From 2a9b5e939f8078e33c06330ae4c57a1cd02b0ae6 Mon Sep 17 00:00:00 2001 From: Carlos Alfredo Vergara Rojas Date: Tue, 7 Jul 2026 23:02:04 -0500 Subject: [PATCH 1/7] Finalize IJDS submission closeout --- book/references.bib | 4 +- docs/research/README.md | 5 + ...rpus_claims_improvement_plan_2026-07-07.md | 637 ++++++++++++++++++ .../literature_reference_audit_2026-06-14.md | 21 + paper/CRPTO_ijds.qmd | 98 ++- paper/submission/CLAIM_AUDIT_MATRIX.md | 49 ++ paper/submission/CRPTO_ijds_submission.tex | 109 +-- .../IJDS_SUBMISSION_ROADMAP_2026-08-10.md | 2 +- paper/submission/README.md | 68 +- paper/submission/REPRODUCIBILITY_PACKAGE.md | 10 + .../submission/SCHOLARONE_FINAL_CHECKLIST.md | 51 +- paper/supplement_ijds.qmd | 20 + 12 files changed, 970 insertions(+), 104 deletions(-) create mode 100644 docs/research/ijds_corpus_claims_improvement_plan_2026-07-07.md diff --git a/book/references.bib b/book/references.bib index 3a9f6a8..b46aa72 100644 --- a/book/references.bib +++ b/book/references.bib @@ -1229,7 +1229,7 @@ @article{delage2010dro @inproceedings{zhao2016p2pportfolio, author = {Zhao, Hongke and Liu, Qi and Wang, Guifeng and Ge, Yong and Chen, Enhong}, - title = {Portfolio Selections in {P2P} Lending}, + title = {Portfolio Selections in {P2P} Lending: A Multi-Objective Perspective}, booktitle = {Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining}, pages = {2075--2084}, year = {2016}, @@ -1237,7 +1237,7 @@ @inproceedings{zhao2016p2pportfolio } @article{serrano2016profitscoring, - author = {Serrano-Cinca, Carlos and Gutierrez-Nieto, Begona}, + author = {Serrano-Cinca, Carlos and Guti{\'e}rrez-Nieto, Bego{\~n}a}, title = {The Use of Profit Scoring as an Alternative to Credit Scoring Systems in Peer-to-Peer ({P2P}) Lending}, journal = {Decision Support Systems}, volume = {89}, diff --git a/docs/research/README.md b/docs/research/README.md index a3bbdc5..9dcf911 100644 --- a/docs/research/README.md +++ b/docs/research/README.md @@ -43,6 +43,11 @@ perenne y lo que el código lee/escribe. paper que pueden entrar con evidencia congelada y extensiones que requieren un nuevo resultado CRPTO v2. +- `ijds_corpus_claims_improvement_plan_2026-07-07.md` - analisis con + `academic-pdf-intake` del paper, supplement, submission PDF y corpus + `Papers_tesis`; sus recomendaciones editoriales IJDS quedaron aplicadas y se + conserva como trazabilidad, no como backlog abierto. + ## Registros de gobernanza (decisiones; no se re-ejecutan sin permiso) - `crpto_champion_reopen_plan_2026-05-21.md` — secuencia gobernada de reopen diff --git a/docs/research/ijds_corpus_claims_improvement_plan_2026-07-07.md b/docs/research/ijds_corpus_claims_improvement_plan_2026-07-07.md new file mode 100644 index 0000000..d6c60fe --- /dev/null +++ b/docs/research/ijds_corpus_claims_improvement_plan_2026-07-07.md @@ -0,0 +1,637 @@ +# IJDS Corpus, Claims, and Improvement Plan - 2026-07-07 + +Scope: analyze the current CRPTO body paper, official IJDS submission PDF/source, +online supplement, frozen metrics/artifacts, and the local `Papers_tesis` corpus +using the global `academic-pdf-intake` skill outputs. + +This memo does **not** reopen the champion, does **not** modify +`EXTRACTION_MANIFEST.json`, and does **not** recommend rerunning protected DVC +stages. The active paper claim remains the finite-grid decision certificate +registered in `docs/research/active_claims_2026-07-04.md`. + +## Implementation Status + +Status on 2026-07-07: the actionable P0/P1/P2 editorial recommendations in this +memo have been applied to the manuscript, official IJDS `.tex` handoff, +supplement, submission checklist, bibliography, and reviewer-defense matrix. +This file is retained as traceability for the corpus/IJDS analysis and parser +evidence, not as an open TODO list. Future edits should use +`paper/submission/SCHOLARONE_FINAL_CHECKLIST.md` and +`docs/research/active_claims_2026-07-04.md` as the active operating gates. + +## Inputs Used + +Skill / benchmark outputs: + +- Full local inventory: + `.tmp_pdf_intake_benchmark/run_20260707_1715/manifest.jsonl` +- Full parser benchmark: + `.tmp_pdf_intake_benchmark/run_20260707_1715/runs.jsonl` +- Current literature matrix generated from extracted text: + `.tmp_pdf_intake_benchmark/run_20260707_ijds_lit_analysis/corpus_current_inventory.csv` +- IJDS venue snippets: + `.tmp_pdf_intake_benchmark/run_20260707_ijds_lit_analysis/ijds_venue_snippets.csv` +- Parser summary for active CRPTO PDFs: + `.tmp_pdf_intake_benchmark/run_20260707_ijds_lit_analysis/active_crpto_parser_benchmark.csv` +- MinerU CUDA follow-up for active CRPTO PDFs: + `.tmp_pdf_intake_benchmark/run_20260707_active_mineru_cuda/runs.jsonl` + +CRPTO sources: + +- `paper/CRPTO_ijds.qmd` +- `paper/submission/CRPTO_ijds_submission.tex` +- `paper/submission/CRPTO_ijds_submission.pdf` +- `paper/supplement_ijds.qmd` +- `paper/CRPTO_ijds.pdf` +- `paper/supplement_ijds.pdf` +- `reports/crpto/tables/crpto_tableA19_regret_auditability_frontier.csv` +- `reports/crpto/tables/crpto_tableA25_external_replication_gate.csv` +- `reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv` +- `reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.csv` +- `reports/crpto/tables/crpto_tableA37_pool93_body_tail_risk.csv` +- `reports/crpto/tables/crpto_tableA38_pool93_body_cluster_bound_audit.csv` +- `reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.csv` +- `models/experiments/champion_reopen/.../pool93_ijds_claim_governance.json` +- `models/experiments/champion_reopen/.../pool93_ijds_consolidated_governance.json` + +IJDS public sources checked on 2026-07-07: + +- Submission guidelines: + +- Reviewer guidelines: + +- Editorial statement: + +- Data and Code Disclosure Policy: + + +## Executive Diagnosis + +CRPTO is well matched to IJDS when written as **data science for decisions**: +real credit data, a methodological bridge from calibrated prediction to robust +optimization, managerial/model-risk relevance, practical implications, and +reproducible computational evidence. The strongest submission story is not +"better credit scoring." It is: + +> CRPTO converts a frozen calibrated PD model into an auditable robust funding +> decision by carrying conformal uncertainty into a finite-grid portfolio +> frontier, exposing the return-bound trade-off and validating the selected +> funded set from frozen evidence. + +The paper is already close to this framing. The main improvement opportunity is +reader focus: reduce the cognitive load around internal run history, make the +closest IJDS precedents visible, and make the A35 frontier the unmistakable +decision object. + +The main risk is overclaiming by shorthand. In particular, the current body and +submission use "0.345 Markov cap" as a rounded lens. The active body/default +point has `Markov_cap = 0.345083740`; the strict `<= 0.345` frontier row is a +neighboring policy with return `$184,800.41`, not the body/default return +`$184,832.48`. This is fixable in prose by saying "approximately 0.3451" or +"the declared 0.345 return-bound lens" and by avoiding exact `<= 0.345` language +for the body/default row. + +## Official IJDS Fit + +The IJDS submission guidelines say IJDS publishes data science methodologies +for decision-making environments and expects four components: data, innovative +model/algorithm/approach, managerial/engineering/industrial relevance, and +implications. CRPTO has all four: + +| IJDS component | CRPTO evidence | Current strength | Improvement | +|---|---|---|---| +| Data | Lending Club OOT panel; Prosper and Freddie/Mendeley external frozen applications. | Strong. | Keep external results as recipe transfer, not as extra certificates. | +| Model / algorithm | Calibrated PD -> Mondrian conformal intervals -> robust LP -> exact funded-set audit. | Strong. | Name the methodological unit earlier as a "decision certificate." | +| Decision relevance | `$1M` budgeted credit funding with return/risk trade-off. | Strong. | Make A35/A19 the reader-facing proof of relevance. | +| Implications | Reproducible model-risk audit surface; explicit limits on live/conditional claims. | Strong. | Put implications in abstract/conclusion as knowledge gained, not only actions performed. | + +Venue constraints that matter: + +- Initial submissions should be at most 25 IJDS-style pages excluding references + and appendices. +- IJDS uses double-anonymous review for submissions on/after 2025-01-01. +- Abstract must be one paragraph and <=300 words; IJDS strongly encourages it + to answer problem/relevance, methods/results, and insights/implications. +- Tables/figures should appear near first mention. +- Data/code disclosure is required at submission, and accepted computational + papers are expected to upload data/code or an approved alternative plan. + +The current official-template build is 26 pages total, with conclusion and +references starting on page 22, so the body appears inside the IJDS page budget +when references are excluded. The reproducibility package plan is aligned with +the IJDS policy, but review-stage path/identity sanitization remains essential. + +## Current CRPTO Claim Stack + +Active body point: + +| Quantity | Current value | +|---|---:| +| OOT Lending Club universe | `276,869` loans | +| Budget | `$1,000,000` | +| Selected-policy realized return | `$184,832.48` | +| Return-floor surplus | `$14,367.94` | +| `V(alpha=0.01)` | `0.035350` | +| `Gamma_CP(alpha=0.01)` | `0.162616` | +| Endpoint budget upper | `0.245083740` | +| Markov cap | `0.345083740` | +| Exact alpha violation | `0.0` | +| Alpha-grid pass | `8/8` | +| Bootstrap return interval | `$167,963.20` to `$198,650.47` | + +Finite-grid denominators: + +| Surface | Denominator | Result | +|---|---:|---| +| Consolidated semantic policies | `50,010` | `27,508` pass every declared alpha and exceed return floor | +| Raw consolidated rows | `51,678` | `1,668` duplicate semantic rows removed | +| Terminal endpoint policies | `37,068` | `37,068/37,068` all-alpha passers | +| Terminal alpha checks | `296,544` | `296,544/296,544` completed checks | + +Main claim boundary: + +- Claim **finite-grid decision certificate**, not continuous global optimality. +- Claim **weighted funded-set validity plus Markov bound**, not universal + conditional coverage. +- Claim **frozen external economic recipe transfer**, not Prosper/Freddie exact + Lending Club-style certificates. +- Claim **regret-auditability trade-off**, not dominance over SPO+ or other + decision-focused learners on every regret metric. +- Claim **fixed-allocation bootstrap diagnostics**, not a conformal/bootstrap + guarantee over model retraining, solver inputs, or search. + +## Paper / Submission / Supplement Assessment + +### Body and official submission + +Strengths: + +- The abstract already starts with the correct IJDS premise: credit allocation is + a data-science-for-decisions problem. +- The introduction explicitly says the research question is not a better + classifier, but whether predictive uncertainty can be carried into a robust + auditable portfolio decision. +- The contribution list separates construction, theorem, literature positioning, + frozen evidence, and evidence ladder. +- The official submission uses the IJDS `informs4` class with `dblanonrev`, and + the build instructions now document both `latexmk` and the robust + `pdflatex -> bibtex -> pdflatex -> pdflatex` fallback. +- The body has the right core tables: certificate metrics, finite-grid frontier, + reviewer-question boundary table, regret-auditability table, and supplement + map. + +Risks / repairs: + +- **Cap wording repair:** replace exact-sounding "under the declared 0.345 + Markov cap" for the body/default point with "near the declared 0.345 + return-bound lens" or "Markov cap 0.345084." Keep the strict `<=0.345` row as + a neighboring frontier endpoint, not the body point. +- **A35 should be the first durable result object.** The paper currently explains + many pieces before the reader sees the frontier. A reviewer should encounter + the frontier logic as soon as possible after the certificate table. +- **IJDS precedent paragraph is thin.** The body cites relevant work, but the + venue-specific story can be sharper: IJDS has already published credit ML, + cost-aware calibration, causal decision framing, and reproducibility-oriented + data-science-for-decisions papers. CRPTO extends that line into a portfolio + decision certificate. +- **External replications are valuable but slightly loud in the abstract.** They + support transfer of the recipe; they should not compete with the Lending Club + certificate as the abstract's main result. +- **SPO+ comparison is strong but needs a crisp takeaway sentence.** The current + text is accurate: SPO+ wins synthetic regret; CRPTO buys auditable risk + controls and a dollar-funded set. Put that exact contrast in one memorable + sentence before the table. + +### Supplement + +Strengths: + +- A35--A39 are exactly the right selected-policy closure block. +- A37--A39 correctly close tail, concentration, and empirical contribution + objections without changing the selector. +- A25--A34 answer the single-dataset concern through external economic recipe + transfer and exhaustiveness checks. +- Appendix E makes routine reproduction distinct from protected champion/search + reruns. + +Risks / repairs: + +- The supplement is rich enough that a reader may miss the hierarchy. Add a + short "how to read this supplement" map at the top that says: theory -> active + certificate -> diagnostics -> external transfer -> reproduction. +- A35 should be introduced as the supplement's active frontier, not as another + appendix table among many. +- A25--A34 should keep the words "economic replication" and "recipe transfer" + visible. Avoid "external validation" unless it is immediately scoped. +- In A38, the fact that every cluster-aware threshold is looser than Markov is a + strength: it explains why the body does not chase a more fragile bound. + +## Parser / Skill Findings + +The `academic-pdf-intake` skill routing is sensible for this repo: + +- **Docling** remains the primary parser for born-digital academic PDFs and + clean Markdown/JSON extraction. +- **OpenDataLoader hybrid** is the best comparison/fallback when bounding boxes, + reading-order traceability, hidden-text safety, and table provenance matter. +- **MinerU CUDA hybrid-engine** is now viable on the local RTX/CUDA setup for + the active CRPTO PDFs and is best kept for formula-heavy, OCR-heavy, scanned, + or visual-QA cases. +- **Codex PDF / MarkItDown** are useful as fast baselines and smoke tests, not + as the final source for complex academic extraction. + +Active PDF benchmark: + +| PDF | Pages | Docling | OpenDataLoader | MinerU CUDA | +|---|---:|---:|---:|---:| +| `paper/CRPTO_ijds.pdf` | 27 | 54.98s | 57.09s | 60.41s | +| `paper/submission/CRPTO_ijds_submission.pdf` | 26 | 46.14s | 35.00s | 54.47s | +| `paper/supplement_ijds.pdf` | 32 | 78.41s | 70.30s | 74.20s | + +Operational recommendation: + +- For day-to-day paper analysis: Docling first, ODL for table/traceability + comparison, Codex PDF for fast diffable text. +- For final figure/table/formula QA: add MinerU on the active PDF(s), then check + `layout.pdf` / visual outputs when extraction disagreement is material. +- For the full 81-PDF literature corpus: do not run all three heavy parsers + routinely. Use fast baseline + Docling on close papers, then ODL/MinerU only + on candidates with tables, equations, or layout ambiguity. + +## `Papers_tesis` Corpus Summary + +Current corpus inventory: + +- `81` local literature PDFs in `Papers_tesis` +- `3,404` literature pages +- `3` active CRPTO PDFs separately, `85` pages +- Full generated matrix: + `.tmp_pdf_intake_benchmark/run_20260707_ijds_lit_analysis/corpus_current_inventory.csv` + +Topic signals from extracted text: + +| Topic signal | PDFs | +|---|---:| +| Tables / metrics / experiments | 78 | +| Tail risk / CVaR / OCE / loss | 76 | +| Portfolio / optimization / decision | 68 | +| Robust optimization / uncertainty sets | 67 | +| Conformal / coverage / calibration | 57 | +| Fairness / governance / explainability | 57 | +| Source shift / weighted / multi-source | 55 | +| Credit / lending / default | 38 | +| Causal decision | 24 | +| Decision-focused / SPO / PTO | 22 | +| Online / adaptive conformal | 20 | + +These counts are keyword/topic signals, not claims that every paper is equally +central. The editorial use is by cluster. + +## IJDS Papers in or Adjacent to the Local Corpus + +Confirmed IJDS papers with local or official evidence: + +| Paper | Evidence | What it contributes | CRPTO use | Boundary | +|---|---|---|---|---| +| Das et al. (2023), "Credit Risk Modeling with Graph Machine Learning" | Local PDF header and DOI `10.1287/ijds.2022.00018`; official IJDS page. | Extends tabular credit scoring with corporate graph features and GNN/AutoML ensembles; includes reproducibility capsule. | Shows IJDS accepts credit-risk ML when data/method/reproducibility are clear. Position CRPTO as a **decision certificate after scoring**, not a richer scorer. | Corporate credit ratings, not consumer loan portfolio funding; no conformal/robust funded-set certificate. | +| Yang and Bi (2025), "Cost-Aware Calibration of Classifiers" | Official IJDS DOI `10.1287/ijds.2024.0038`; cited in CRPTO `.bbl`. | Defines cost-aware calibration, expected calibration cost, and MetaCal; emphasizes downstream costs of miscalibration. | Strongest IJDS calibration precedent. CRPTO extends calibration into **portfolio allocation and funded-set audit**. | Classifier calibration problem, not robust portfolio optimization. | +| Fernandez-Loria and Provost (2022), "Causal Decision Making and Causal Effect Estimation Are Not the Same..." | Official IJDS DOI `10.1287/ijds.2021.0006`; cited in CRPTO `.bbl`. | Separates decision quality from effect-estimation accuracy. | Use to sharpen the intro: CRPTO is a decision object, not a prediction leaderboard. | Causal treatment assignment framing, not credit PD/conformal portfolio. | +| Fernandez-Loria and Provost (2025), "Observational vs. Experimental Data When Making Automated Decisions Using Machine Learning" | Local PDF in supplement; official DOI `10.1287/ijds.2023.0012`. | Shows observational data can sometimes support automated decisions when the decision target is ranking/thresholding rather than unbiased effect estimation. | Supports CRPTO's observational-panel boundary and the claim that decision metrics differ from estimation metrics. | Causal/automated intervention setting; CRPTO should cite it as a limitation/future protocol, not as causal validity. | +| Falconer, Kazempour, and Pinson (2026), "Toward Replication-Robust Analytics Markets" | Local PDF header and DOI `10.1287/ijds.2025.0075`; official IJDS page. | Builds an analytics market robust to strategic data replication; emphasizes reproducibility and strategic robustness. | Useful venue signal: IJDS values robust/reproducible analytics systems. Use only as a light reproducibility/robustness cousin. | Market design/collaborative analytics, not credit, conformal prediction, or portfolio funding. | + +The local corpus also references Morucci et al. (2022), an IJDS causal +uncertainty paper, but the PDF is not in `Papers_tesis`; do not count it as +local corpus evidence unless it is added. + +## Closest Non-IJDS Literature Clusters + +### 1. Conformal foundations and risk control + +Key local papers: + +- Angelopoulos and Bates (2023), gentle introduction. +- Angelopoulos et al. (2024), conformal risk control. +- Angelopoulos et al. (2025), Learn Then Test. +- Bates et al. (2021), risk-controlling prediction sets. +- Barber et al. (2021), conditional coverage limits. +- Angelopoulos et al. (2026), non-monotonic CRC. +- Gibbs/Candes, Lekeufack et al., Kiyani et al., Zhou/Orfanoudaki/Zhu. + +What they give CRPTO: + +- Validity language and finite-sample discipline. +- Justification for risk-control framing. +- Limits on conditional/group/live claims. + +How to improve paper: + +- Keep them as theory lineage, but do not over-expand. +- Use them to justify why CRPTO reports a bound and exact audit rather than only + nominal coverage. + +### 2. Conformal robust optimization / predict-then-calibrate + +Key local papers: + +- Johnstone and Cox (2021), conformal uncertainty sets for robust optimization. +- Patel et al. (2024), conformal contextual robust optimization. +- Sun et al. (2024), predict-then-calibrate. +- Zhao et al. (2026), conformal robust optimization and satisficing. +- Bao et al. (2025), CROMS model selection. +- Yeh et al. (2025/2026), conformal risk training / end-to-end calibration. +- Zhou and Zhu (2025), inverse conformal risk control. + +What they give CRPTO: + +- The nearest methodological neighborhood. +- A natural "what CRPTO adds" contrast: real credit payoff, funded-set weights, + exact portfolio audit, finite frontier, and reproducibility harness. + +How to improve paper: + +- Add a compact contrast table: abstract CRO/LP papers vs. CRPTO's credit + funded-set certificate. +- Say explicitly that CRPTO is post-hoc over a frozen PD system; end-to-end + variants are future work. + +### 3. Decision-focused learning and SPO+ + +Key local papers: + +- Elmachtoub and Grigas (2022), SPO+. +- Donti et al. (2017), task-based end-to-end learning. +- Mandi et al. (2024), DFL survey. +- Liu and Grigas (2021), risk bounds/calibration for SPO. +- Schutte et al. (2024), robust losses for DFL. + +What they give CRPTO: + +- The main alternative methodological family. +- A strong reviewer question: "Why not train through the optimizer?" + +How to improve paper: + +- Keep A19/Figure 15 central. +- State the contrast in one sentence: + "SPO+ is the low-regret corner; CRPTO is the auditable-risk-control corner + with a funded-set dollar certificate." +- Do not apologize for higher synthetic regret; explain that the metric is + different from the funded-set economic certificate. + +### 4. Credit / P2P / fairness / governance + +Key local papers: + +- Jagtiani and Lemieux (2019), fintech Lending Club context. +- Serrano-Cinca and Gutierrez-Nieto (2016), profit scoring in P2P lending. +- Guo et al. (2016), instance-based P2P credit investment. +- Zhao et al. (2016), P2P portfolio selection. +- Chi, Ding, and Peng (2019), data-driven robust P2P credit portfolio. +- Das et al. (2023), IJDS graph ML credit risk. +- Albanesi and Vamossy (2024), score performance and equity. +- Fuster et al. (2022), unequal ML credit-market effects. +- Blattner and Nelson (2021), noisy data and consumer credit disparities. +- FinRegLab (2023), explainability and fairness in credit underwriting. +- CFPB (2014), proxy race/ethnicity methods. + +What they give CRPTO: + +- Domain legitimacy and governance boundaries. +- Support for reporting economic return, risk, calibration, and governance + together. + +How to improve paper: + +- Keep fairness/proxy material bounded. CRPTO does not have protected labels or a + legal fair-lending protocol. +- Use credit/P2P papers to motivate why classification metrics alone are + insufficient for investment decisions. + +### 5. Robust optimization, DRO, tail risk, and concentration + +Key local papers: + +- Bertsimas and Sim (2004), price of robustness. +- Ben-Tal, El Ghaoui, and Nemirovski (2009), robust optimization. +- Bertsimas, Gupta, and Kallus (2018), data-driven robust optimization. +- Bertsimas and Kallus (2020), predictive to prescriptive analytics. +- Delage and Ye (2010), moment DRO. +- Goldfarb and Iyengar (2003), robust portfolios. +- Rockafellar and Uryasev (2000), CVaR. +- Ben-Tal and Teboulle (2007), OCE. +- Hoeffding, Bennett, Freedman, Fuk-Nagaev for concentration context. + +What they give CRPTO: + +- The language for price of robustness and uncertainty budgets. +- Tail-risk diagnostics and the reason to keep Markov as the weakest defensible + body-level statement. + +How to improve paper: + +- A37/A38 should be discussed as "assumption-priced sensitivity." +- Avoid making CVaR/OCE sound like promoted selectors. + +### 6. Source shift, multi-distribution, online conformal + +Key local papers: + +- Tibshirani et al. (2019), conformal prediction under covariate shift. +- Barber, Candes, Ramdas, Tibshirani (2023), beyond exchangeability. +- Bhattacharyya and Barber (2026), group-weighted conformal prediction. +- Guan (2023), localized conformal prediction. +- Liu, Levis, Normand, Han (2024), multi-source conformal inference. +- Yang and Jin (2026), multi-distribution robust conformal prediction. +- Gibbs and Candes (2021), adaptive conformal inference. +- Liu et al. (2026), online conformal prediction via universal portfolios. + +What they give CRPTO: + +- A future-work lane and reviewer caveats for group/source/live deployment. + +How to improve paper: + +- Keep A23/A24 as diagnostics. +- Do not promote multi-distribution or online validity without a new protocol. + +## High-Priority Improvement Plan + +### P0: repair precision in cap wording + +Change any exact-sounding text that says the body/default point is under a +`0.345` cap. The exact body/default cap is `0.345083740`; the strict +`<=0.345` row is a neighboring frontier point. + +Recommended wording: + +- "the selected policy sits at the declared approximately 0.345 return-bound + lens, with Markov cap 0.345084" +- "the strict `<=0.345` endpoint earns `$184,800.41`; the body/default balanced + point earns `$184,832.48` with Markov cap `0.345084`" + +Avoid: + +- "highest-return point under cap `<=0.345`" for the body/default row. +- "declared `0.345` Markov cap" unless the next words clarify rounding. + +### P1: add a venue-specific IJDS precedent table + +Add a compact body table or paragraph after the related-work overview: + +| IJDS precedent | Lesson for CRPTO | CRPTO extension | +|---|---|---| +| Das et al. (2023) credit graph ML | IJDS accepts reproducible credit-risk ML. | CRPTO turns credit risk scores into a funded portfolio certificate. | +| Yang and Bi (2025) cost-aware calibration | Calibration matters because downstream costs are asymmetric. | CRPTO prices uncertainty inside a budgeted allocation. | +| Fernandez-Loria and Provost (2022/2025) decision vs estimation | Decision quality is not the same as estimation/prediction quality. | CRPTO evaluates the funded decision, not only PD quality. | +| Falconer et al. (2026) replication-robust analytics | IJDS values robust/reproducible analytics systems. | CRPTO supplies frozen evidence, exact checks, and a reproducibility harness. | + +This helps the editor see fit immediately and helps reviewers place the paper +inside IJDS rather than only OR/ML. + +### P1: make A35 the central result object + +Move the reader quickly from method to A35: + +1. Exact certificate table: what the selected policy achieved. +2. A35 frontier: why this is not a cherry-picked singleton. +3. A19 regret-auditability: why CRPTO is not trying to beat SPO+ on its own + synthetic regret metric. +4. A25 external recipe transfer: why the method is not a Lending Club-only + curiosity. + +The current paper has these pieces; the improvement is ordering and signposting. + +### P1: sharpen abstract to IJDS's three-question template + +Current abstract is good but can be more IJDS-aligned: + +1. Problem/relevance: + "Credit allocation decisions need calibrated probabilities only insofar as + they change funding choices under risk appetite." +2. Methods/results: + "CRPTO maps frozen PD predictions through Mondrian conformal intervals into a + robust LP and finite-grid funded-set audit; on 276,869 OOT loans it earns + `$184.8K` on `$1M` with `V=0.035350`, `Gamma_CP=0.162616`, and Markov cap + `0.345084`." +3. Insight/implication: + "The insight is that uncertainty should be reported as a return-bound + frontier, not as a post-hoc calibration table; reproducible decision + certificates can be audited without retraining a production-style PD model." + +Reduce abstract space devoted to external replications unless needed for +single-dataset defense. + +### P1: make the baseline story a reviewer checklist + +IJDS reviewers will ask whether the paper uses reasonable baselines and +quantifies improvement. CRPTO can answer with a table: + +| Baseline / family | What it optimizes | CRPTO comparison | +|---|---|---| +| Two-stage baseline | Predict then optimize without conformal robust certificate. | CRPTO adds exact funded-set bound and frontier. | +| SPO+ / DFL | Synthetic regret / task-aligned training loss. | SPO+ has lower mean regret; CRPTO has funded-set dollar value and 3/3 verifiable risk controls. | +| P2P profit scoring | Economic loan selection. | CRPTO adds conformal premium and exact alpha-safe audit. | +| P2P robust portfolio | Robust credit allocation. | CRPTO calibrates uncertainty with conformal intervals and exposes finite-grid denominators. | +| Cost-aware calibration | Probability calibration under asymmetric costs. | CRPTO carries calibrated uncertainty into a portfolio decision. | + +The goal is not to claim universal dominance; it is to make the trade-off +impossible to miss. + +### P2: strengthen supplement navigation + +Add a short supplement reader map: + +- Appendix A: proof and Markov boundary. +- Appendix B: robustness/challenger diagnostics. +- Appendix C: active A35--A39 selected-policy closure. +- Appendix D: external recipe transfer. +- Appendix E: reproducibility, DVC, protected stages, and anonymization. + +Then repeat the claim hierarchy in one table: + +| Evidence | Promoted? | Why | +|---|---:|---| +| A35 | Yes | Active finite-grid frontier. | +| A36--A39 | Support | Selected-policy composition/tail/concentration/bootstrap diagnostics. | +| A19 | Support | Regret-auditability contrast. | +| A25--A34 | Support | External economic recipe transfer. | +| A20--A24 | Diagnostics | Tail/source/online objections, not selector changes. | + +### P2: citation hygiene before freeze + +The previous citation audit flagged several body references as partial or +citation-only. This pass reduces risk for `das2023creditgraph`, +`yang2025costaware`, and the Fernandez-Loria/Provost IJDS papers. Still +spot-check before final freeze: + +- `hoeffding1963`, `boucheron2013concentration`, `ghosh2002` +- `goldfarb2003robustportfolio`, `delage2010dro` +- `serrano2016profitscoring`, `zhao2016p2pportfolio` +- any recent credit/IJDS references added after 2026-06-14 + +Do this as source verification, not as a broad literature expansion. + +### P2: prepare response-ready reviewer objections + +Prewrite one paragraph each: + +- "Why not SPO+?" +- "Why not CVaR/OCE as the selector?" +- "Is the result cherry-picked?" +- "What happens under dependence?" +- "Is this a live-production guarantee?" +- "What exactly can be reproduced under double anonymous review?" + +Most answers already exist in the body/supplement; the improvement is to make +them short and reusable. + +## Concrete Editing Checklist + +Before submission: + +1. Replace exact-sounding `0.345` cap wording for the body/default point. +2. Add IJDS precedent paragraph/table with Das, Yang/Bi, Fernandez-Loria/Provost, + and Falconer/Kazempour/Pinson. +3. Move or signpost A35 so the finite-grid frontier appears as the main results + object, not only as a supporting table. +4. Add one compact "what the reviewer should remember" paragraph before the + A19 regret table. +5. Tighten abstract external-replication language to "recipe transfer." +6. Add supplement reader map and promoted/support/diagnostic hierarchy. +7. Check all anonymous review paths: no repo URLs, local paths, author names, + affiliations, or identifying metadata in body/supplement PDFs. +8. Run `just smoke` and `just validate-champion` after any paper edits. +9. Rebuild official submission PDF and check page count/log. + +Do **not** do before submission unless a reviewer or explicit research decision +requires it: + +- Rerun protected champion/search/conformal stages. +- Promote CVaR/OCE, multi-distribution, online, causal, or end-to-end DFL + variants. +- Turn external Prosper/Freddie replications into new exact funded-set + certificates. +- Expand the body with a long literature review. + +## Best Current IJDS Story + +The paper should read like this: + +1. Credit allocation is a decision problem; PD calibration alone is insufficient. +2. Existing IJDS work shows calibration, credit ML, and causal decision framing + matter for data-science decisions. +3. CRPTO contributes the missing bridge: a frozen predictive model becomes a + robust funding decision with a conformal premium and exact finite-grid audit. +4. The selected policy earns `$184,832.48` on `$1M`, with `V=0.035350`, + `Gamma_CP=0.162616`, and Markov cap `0.345084`. +5. The result is not a singleton: A35 exposes 50,010 semantic policies and a + return-bound frontier. +6. The method does not beat SPO+ at SPO+'s own regret target; instead, it buys + auditability, risk controls, and a funded-set dollar certificate. +7. The supplement shows tail, concentration, bootstrap, source, online, and + external-recipe checks without changing the body claim. +8. The reproducibility package is part of the scientific contribution, not just + administration. + +That is the IJDS version of CRPTO: **a reproducible decision certificate for +credit portfolio allocation under conformal uncertainty**. diff --git a/docs/research/literature_reference_audit_2026-06-14.md b/docs/research/literature_reference_audit_2026-06-14.md index a0822c3..25c4fc3 100644 --- a/docs/research/literature_reference_audit_2026-06-14.md +++ b/docs/research/literature_reference_audit_2026-06-14.md @@ -75,3 +75,24 @@ The weaker area is not the main claim; it is citation hygiene. Several classical or recent references are being used as positioning anchors without a local note proving close reading. They can stay for now, but they deserve a targeted pre-freeze source check rather than another broad literature expansion. + +## 2026-07-07 targeted pre-freeze spot-check + +The pre-freeze spot-check resolved the highest-risk citation-only items that +support the IJDS body and supplement theory/positioning language: + +| Key | Source checked | Result | +|---|---|---| +| `ghosh2002` | Taylor & Francis DOI landing page, `10.1198/000313002119`. | Metadata matches BibTeX; appropriate for Markov-related probability-inequality language. | +| `hoeffding1963` | Taylor & Francis DOI landing page, `10.1080/01621459.1963.10500830`. | Metadata matches BibTeX; appropriate for bounded-sum tightening language when independence is stated. | +| `boucheron2013concentration` | Oxford Academic book page / DOI `10.1093/acprof:oso/9780199535255.001.0001`. | Metadata matches BibTeX; appropriate as a modern concentration reference. | +| `goldfarb2003robustportfolio` | INFORMS PubsOnLine DOI landing page, `10.1287/moor.28.1.1.14260`. | Metadata matches BibTeX; appropriate for robust portfolio selection under parameter uncertainty. | +| `delage2010dro` | INFORMS PubsOnLine DOI landing page, `10.1287/opre.1090.0741`. | Metadata matches BibTeX; appropriate for DRO/moment-uncertainty positioning. | +| `zhao2016p2pportfolio` | ACM/KDD DOI landing page, `10.1145/2939672.2939861`. | Title updated in BibTeX to include the subtitle "A Multi-Objective Perspective"; appropriate for P2P portfolio-selection positioning. | +| `serrano2016profitscoring` | ScienceDirect DOI landing page, `10.1016/j.dss.2016.06.014`. | Author accents corrected in BibTeX; appropriate for P2P profit-scoring positioning. | +| `falconer2026replication` | INFORMS/IJDS DOI landing page, `10.1287/ijds.2025.0075`. | Metadata matches BibTeX; appropriate for the replication-robust IJDS precedent. | + +No broad literature expansion is recommended before submission. Remaining +recent-credit references (`das2023creditgraph`, `yang2025costaware`, +`boosting2025default`, `zheng2026twostage`) are lower-risk positioning anchors; +verify only if their exact body sentences become more specific. diff --git a/paper/CRPTO_ijds.qmd b/paper/CRPTO_ijds.qmd index 364dc10..a9a9429 100644 --- a/paper/CRPTO_ijds.qmd +++ b/paper/CRPTO_ijds.qmd @@ -26,25 +26,26 @@ execute: # Abstract Credit allocation is a data-science-for-decisions problem: calibrated default -probabilities matter only after they shape which loans are funded. We introduce -Conformal Robust Predict-Then-Optimize (CRPTO), a post-hoc bridge that maps a -frozen calibrated probability-of-default model through Mondrian conformal -intervals into robust portfolio constraints and an empirical funded-set audit. -On a 276,869-loan out-of-time Lending Club evaluation, the selected policy earns -`$184.8K` on a `$1M` budget while passing the declared eight-level alpha grid -($V(0.01)=0.035350$, $\Gamma_{\mathrm{CP}}=0.162616$, Markov cap `0.345084`, -zero violation). The same exact frontier exposes conservative and economic -endpoints, making the return-bound trade-off visible rather than implicit. -Frozen external credit replications preserve the predeclared global conformal -gates and produce positive robust LP objectives, so the result is not confined -to the Lending Club panel. Across the four frozen external applications, the -price of robustness forms a positive premium series ordered by panel default -rate. The contribution is a conformal-robust -credit-portfolio decision certificate with a -distribution-free Markov bound under weighted funded-set validity: it connects -real credit data, calibrated predictive models, robust funding decisions, and a -validation harness that rebuilds the prediction-to-decision chain from frozen -inputs while keeping the statistical guarantee boundary explicit. +probabilities matter only after they shape which loans are funded under a budget +and risk appetite. We introduce Conformal Robust Predict-Then-Optimize (CRPTO), +a post-hoc bridge that maps a frozen calibrated probability-of-default model +through Mondrian conformal intervals into robust portfolio constraints and an +empirical funded-set audit. On a 276,869-loan out-of-time Lending Club +evaluation, the selected policy earns `$184.8K` on a `$1M` budget while passing +the declared eight-level alpha grid ($V(0.01)=0.035350$, +$\Gamma_{\mathrm{CP}}=0.162616$, Markov cap `0.345084`, zero violation). The +consolidated finite frontier contains `50,010` deduplicated semantic policies, +of which `27,508` both pass all declared alpha levels and exceed the return +floor, making the return-bound trade-off visible rather than implicit. Frozen +Prosper and Freddie/Mendeley applications test recipe transfer and preserve the +predeclared global conformal gates with positive robust LP objectives. The +insight is that uncertainty should be reported as a decision frontier, not as a +post-hoc calibration table. The contribution is a conformal-robust +credit-portfolio decision certificate with a distribution-free Markov bound +under weighted funded-set validity: it connects real credit data, calibrated +predictive models, robust funding decisions, and a validation harness that +rebuilds the prediction-to-decision chain from frozen inputs while keeping the +statistical guarantee boundary explicit. **Keywords:** conformal prediction; robust optimization; predict-then-optimize; credit risk; portfolio optimization; reproducible data science. @@ -87,10 +88,10 @@ The empirical setting is the Lending Club retail-loan panel, with an out-of-time evaluation set of 276,869 loans. The consolidated frontier contains 50,010 deduplicated semantic policies, of which 27,508 pass every declared alpha level and exceed the return floor. From that declared finite frontier, -the selected policy is the highest-return point that passes the eight -alpha levels under the declared `0.345` Markov cap; it is neither a continuous -global optimum nor the economic endpoint. The selected point earns `$184.8K` -on a `$1M` budget and passes the exact empirical funded-set audit at +the selected policy is the body/default balanced point at the approximately +`0.345` return-bound lens, with Markov cap `0.345084`; it is neither a +continuous global optimum nor the economic endpoint. The selected point earns +`$184.8K` on a `$1M` budget and passes the exact empirical funded-set audit at $\alpha = 0.01$. The headline result is not a single lucky allocation. It is a reproducible bridge from calibrated probabilistic learning to robust, auditable credit portfolio choice, with the return-bound frontier reported rather than @@ -197,10 +198,15 @@ panels [@lessmann2015; @ayari2026; @xia2017]. Recent IJDS credit-risk work shows how richer data structures such as firm graphs can improve rating prediction [@das2023creditgraph], and cost-aware calibration work makes explicit why probability quality matters when predictions feed asymmetric downstream -decisions [@yang2025costaware]. Work on fintech lending and consumer-credit -allocation studies platform structure, credit invisibility, measurement noise, -and scorecard equity [@jagtiani2019altdata; @albanesi2024credit; -@khandani2010consumer; @fuster2022predictably]. +decisions [@yang2025costaware]. IJDS decision papers also sharpen the distinction +between an accurate intermediate estimate and an effective automated decision +[@fernandezloria2022causaldecision; @fernandezloria2025observational], while +replication-robust analytics markets show the journal's appetite for robust, +reproducible decision systems [@falconer2026replication]. Work on fintech +lending and consumer-credit allocation studies platform structure, credit +invisibility, measurement noise, and scorecard equity +[@jagtiani2019altdata; @albanesi2024credit; @khandani2010consumer; +@fuster2022predictably]. In the P2P/Lending Club decision neighborhood, prior work studies instance-based investment support, P2P portfolio selection, profit scoring, robust credit portfolio optimization, and multi-objective AI/OR funding @@ -213,6 +219,15 @@ The contribution is the auditable bridge from a calibrated, frozen PD model to a conformal robust portfolio decision, not another point on the credit-scoring leaderboard. +| IJDS precedent | Lesson for this submission | CRPTO extension | +|---|---|---| +| Credit graph ML [@das2023creditgraph] | IJDS accepts credit-risk ML when data, method, and reproducibility are explicit. | Moves from rating prediction to a funded portfolio certificate. | +| Cost-aware calibration [@yang2025costaware] | Calibration matters because downstream miscalibration costs are asymmetric. | Prices uncertainty inside a budgeted allocation and exact audit. | +| Causal decision papers [@fernandezloria2022causaldecision; @fernandezloria2025observational] | Decision quality is not the same as estimating an intermediate quantity. | Evaluates the funded decision, not only PD quality. | +| Replication-robust analytics [@falconer2026replication] | Robustness and reproducibility are first-class IJDS concerns. | Supplies frozen evidence, exact checks, and a reproducibility harness. | + +: IJDS decision-science precedents and the CRPTO extension. + Finally, recent work on conformal model selection for robust optimization, multi-distribution conformal validity, online conformal portfolio methods, end-to-end conformal risk training, robust conformal decision certificates, and @@ -605,7 +620,10 @@ address whether the method survives two materially different credit products. # Results -The core metric table summarizes the paper-facing metrics. The calibrated PD +The results section is ordered around the reviewer decision object: first the +selected-policy certificate, then the A35 finite-grid frontier that prevents a +singleton reading, and finally the external recipe-transfer checks. The core +metric table summarizes the paper-facing metrics. The calibrated PD layer is not sold as a leaderboard model: AUC `0.7139` is sufficient only because the downstream decision consumes calibrated probabilities, not rankings alone. Its Brier score `0.1544` and ECE near `0.0070` are therefore as important @@ -693,7 +711,10 @@ continuous optima. The table gives the manuscript its decision geometry. The body/default point is not the highest-return point and not the tightest-bound point; it is the -balanced point selected by the declared return-bound lens. The endpoint at +balanced point selected by the declared return-bound lens. The neighboring +strict `<= 0.345` row is reported separately because it is a different finite +policy: it earns `$184,800.41` with Markov cap `0.344996`, whereas the +body/default row earns `$184,832.48` with Markov cap `0.345084`. The endpoint at Markov cap `0.273036` shows how conservative the certified frontier can become while preserving the return floor, and the `$223.5K` endpoint shows the economic return available when the committee accepts a looser cap. The supplement reports @@ -798,10 +819,11 @@ check, strengthening that validation claim without reopening the champion. The second concern is whether conformal uncertainty is doing decision work or only adding conservative decoration. The answer is visible in the portfolio frontier. Policies are evaluated by return, exact alpha pass/fail, weighted -miscoverage, and $\Gamma_{\mathrm{CP}}$; the promoted point is selected because it earns -the highest robust return inside the exact feasible region. This differs from -a workflow where conformal intervals are plotted after the optimizer has -already chosen a point-PD allocation. +miscoverage, and $\Gamma_{\mathrm{CP}}$; the promoted point is the body/default +balanced policy on that finite-grid frontier, while the strict `<= 0.345` cap +policy is reported separately. This differs from a workflow where conformal +intervals are plotted after the optimizer has already chosen a point-PD +allocation. The supplement also carries the reviewer-facing robustness checks: nested holdout, strict temporal holdout, exact evaluation of conformal finalists, @@ -818,6 +840,16 @@ traceable risk controls, and exact funded-set checks. The manuscript therefore does not claim to dominate SPO+ on every regret metric; it claims a different auditability/economic trade-off. +| Baseline family | What it optimizes or reports | CRPTO comparison | +|---|---|---| +| Two-stage predict-then-optimize | Point-PD allocation after prediction. | Adds conformal uncertainty, exact funded-set audit, and frontier denominators. | +| SPO+ / decision-focused learning | Training-time regret on a decision-loss surface. | SPO+ owns low regret; CRPTO owns auditable funded-set risk controls. | +| P2P profit scoring | Economic loan selection. | Adds portfolio-level conformal premium and alpha-safe audit. | +| P2P robust portfolio optimization | Robust allocation under credit uncertainty. | Calibrates uncertainty with conformal intervals and exposes finite-grid selection. | +| Cost-aware calibration | Probability calibration under asymmetric costs. | Carries calibrated uncertainty into a budgeted portfolio decision. | + +: Baseline map for reviewer interpretation. + ## Regret-Auditability Frontier The SPO+ comparator makes the trade-off sharp. In the committed A19/Fig. 15 diff --git a/paper/submission/CLAIM_AUDIT_MATRIX.md b/paper/submission/CLAIM_AUDIT_MATRIX.md index dbdbe7b..9929e23 100644 --- a/paper/submission/CLAIM_AUDIT_MATRIX.md +++ b/paper/submission/CLAIM_AUDIT_MATRIX.md @@ -33,3 +33,52 @@ overclaiming. | "The intervals are too wide to use." | Raw width is expected on a binary PD-scale interval; the paper evaluates whether upper endpoints rank downside risk and produce a funded set that passes Winkler, funded-set miscoverage, and exact alpha-safe checks. | | "SPO+ has lower regret." | Correct; the paper reports a frontier where SPO+ buys regret and CRPTO buys verifiable risk controls; the pool93 frontier updates the funding certificate, not the SPO+ regret experiment. | | "Why not live validation?" | Lending Club retail originations ended in 2020; prospective live validation is future protocol, not hidden current evidence. | + +## Response-Ready Reviewer Paragraphs + +**Why not SPO+ as the main method?** SPO+ is the right comparator for +training-time decision regret, and the manuscript reports that comparison +directly. The point of CRPTO is different: it asks what can be certified after a +calibrated PD model is frozen and the decision layer must remain auditable. On +the A19 regret scale SPO+ owns the low-regret corner; CRPTO owns the funded-set +risk-control corner with a dollar-valued allocation, conformal premium, +finite-grid denominator, and exact post-allocation audit. + +**Why not CVaR/OCE as the selector?** CVaR and OCE are useful tail-risk +diagnostics, but making either one the promoted selector would define a new +objective and require a new predeclared search/audit protocol. The current +submission deliberately promotes the finite-grid return-bound point and then +reprices that selected allocation under LGD, CVaR, OCE, cluster, and bootstrap +stress checks. This keeps tail risk visible without turning a diagnostic table +into a hidden promotion criterion. + +**Is the selected point cherry-picked?** The selected policy is not a singleton +chosen after looking at one lucky allocation. It sits on a declared finite-grid +frontier: 50,010 deduplicated semantic policies are reported, 27,508 both pass +all declared alpha levels and exceed the return floor, and the terminal endpoint +search completes 296,544 exact policy-alpha checks. The body/default point and +the strict `<=0.345` neighboring point are separated explicitly to avoid +rounding-based overclaiming. + +**What happens under dependence?** The body theorem uses the weakest +distribution-free Markov step under the stated weighted funded-set validity +assumption and does not require loan-level independence. The supplement prices +stronger assumptions through cluster-aware sensitivity tables; those rows show +what a reviewer would gain by accepting additional structure, but none becomes +the body guarantee. Dependence therefore appears as an assumption boundary, not +as an unstated theorem condition. + +**Is this a live-production guarantee?** No. Lending Club retail originations +ended in 2020, and the manuscript is explicit that the evidence is a frozen +historical decision certificate, not a prospective control system. The +contribution is reproducible prediction-to-decision governance on the available +out-of-time panel; online conformal control, prospective validation, and live +monitoring are future protocols. + +**What can be reproduced under double-anonymous review?** During anonymous +review, the manuscript and supplement describe the companion package without +author-identifying repository URLs. The reproducible object is the +prediction-to-decision chain from frozen PD artifacts and conformal intervals to +tables, figures, exact checks, and manifest validation. Protected searches and +retraining are intentionally excluded from routine reproduction because they +would change the submitted certificate rather than verify it. diff --git a/paper/submission/CRPTO_ijds_submission.tex b/paper/submission/CRPTO_ijds_submission.tex index 208fe8b..7ff9bec 100644 --- a/paper/submission/CRPTO_ijds_submission.tex +++ b/paper/submission/CRPTO_ijds_submission.tex @@ -2,7 +2,9 @@ %% CRPTO -- INFORMS Journal on Data Science (IJDS) submission manuscript %% ===================================================================== %% Double-anonymous submission body in the official INFORMS class. The prose is -%% ported from the source of truth `paper/CRPTO_ijds.qmd`; keep the two in sync. +%% ported from `paper/CRPTO_ijds.qmd`, then manually compacted for the official +%% IJDS template/page budget. Do not regenerate this file mechanically from QMD +%% after freeze; port substantive claim changes deliberately and rebuild. %% %% REQUIRED (NOT vendored in this repo, per the no-vendoring rule): %% - informs4.cls INFORMS document class @@ -11,12 +13,19 @@ %% portal or Overleaf and drop them next to this file, then build: %% https://pubsonline.informs.org/authorportal/latex-style-files %% https://www.overleaf.com/latex/templates/template-for-informs-journal-on-data-science/sbthszxgycfn +%% if (-not $env:WINDIR) { $env:WINDIR = $env:SystemRoot } % PowerShell %% latexmk -pdf -gg -interaction=nonstopmode CRPTO_ijds_submission.tex -%% If the local TinyTeX wrapper fails with runscript.tlu/nil, use: +%% If a TeX Live update leaves a LaTeX support-file mismatch, run once: +%% fmtutil-sys --byfmt pdflatex +%% If the local TinyTeX wrapper still fails, use the verified Windows fallback: %% pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex %% bibtex CRPTO_ijds_submission %% pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex %% pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex +%% The first pdflatex writes .aux, bibtex writes .bbl, the second pdflatex reads +%% bibliography/cross-reference data, and the third stabilizes references and +%% pagination. On 2026-07-07 the local Codex shell needed WINDIR initialized +%% from SystemRoot before TinyTeX wrappers would run. %% %% `dblanonrev` keeps the manuscript anonymous (IJDS uses double-anonymous %% review for submissions on/after 2025-01-01). Do NOT add author names, @@ -67,26 +76,26 @@ \ABSTRACT{% Credit allocation is a data-science-for-decisions problem: calibrated default -probabilities matter only after they shape which loans are funded. We introduce -Conformal Robust Predict-Then-Optimize (CRPTO), a post-hoc bridge that maps a -frozen calibrated probability-of-default model through Mondrian conformal -intervals into robust portfolio constraints and an empirical funded-set audit. -On a 276{,}869-loan out-of-time Lending Club evaluation, the selected policy -earns \$184.8K on a \$1M budget while passing the declared -eight-level alpha grid ($V(0.01)=0.035350$, +probabilities matter only after they shape which loans are funded under a budget +and risk appetite. We introduce Conformal Robust Predict-Then-Optimize (CRPTO), +a post-hoc bridge that maps a frozen calibrated probability-of-default model +through Mondrian conformal intervals into robust portfolio constraints and an +empirical funded-set audit. On a 276{,}869-loan out-of-time Lending Club +evaluation, the selected policy earns \$184.8K on a \$1M budget while passing +the declared eight-level alpha grid ($V(0.01)=0.035350$, $\Gamma_{\mathrm{CP}}=0.162616$, Markov cap $0.345084$, zero violation). The -same exact frontier exposes conservative and economic endpoints, making the -return-bound trade-off visible rather than implicit. -Frozen external credit replications preserve the predeclared global conformal -gates and produce positive robust LP objectives, so the result is not confined -to the Lending Club panel. Across the four frozen external applications, the -price of robustness forms a positive premium series ordered by panel default -rate. The contribution is a -conformal-robust credit-portfolio decision certificate with a distribution-free -Markov bound under weighted funded-set validity: it connects real credit data, -calibrated predictive models, robust funding decisions, and a validation harness -that rebuilds the prediction-to-decision chain from frozen inputs -while keeping the statistical guarantee boundary explicit.% +consolidated finite frontier contains 50{,}010 deduplicated semantic policies, +of which 27{,}508 both pass all declared alpha levels and exceed the return +floor, making the return-bound trade-off visible rather than implicit. Frozen +Prosper and Freddie/Mendeley applications test recipe transfer and preserve the +predeclared global conformal gates with positive robust LP objectives. The +insight is that uncertainty should be reported as a decision frontier, not as a +post-hoc calibration table. The contribution is a conformal-robust +credit-portfolio decision certificate with a distribution-free Markov bound +under weighted funded-set validity: it connects real credit data, calibrated +predictive models, robust funding decisions, and a validation harness that +rebuilds the prediction-to-decision chain from frozen inputs while keeping the +statistical guarantee boundary explicit.% } \KEYWORDS{conformal prediction; robust optimization; predict-then-optimize; @@ -132,8 +141,8 @@ \section{Introduction}\label{sec:intro} evaluation set of 276{,}869 loans. The consolidated frontier contains 50{,}010 deduplicated semantic policies, of which 27{,}508 pass every declared alpha level and exceed the return floor. From that declared finite frontier, the -selected policy is the highest-return point that passes the eight -alpha levels under the declared $0.345$ Markov cap; it is neither a continuous +selected policy is the body/default balanced point at the approximately +$0.345$ return-bound lens, with Markov cap $0.345084$; it is neither a continuous global optimum nor the economic endpoint. The selected point earns \$184.8K on a \$1M budget and passes the exact empirical funded-set audit at $\alpha=0.01$. The headline result is not a single lucky allocation. It is a @@ -235,16 +244,22 @@ \section{Related Work}\label{sec:related} share this protective intent, but operate at training time rather than as a post-hoc auditable constraint. -The fourth foundation is machine learning for credit scoring, where gradient -boosting and benchmark studies define the performance frontier on retail panels -\citep{lessmann2015,ayari2026,boosting2025default,xia2017}. Recent IJDS +The fourth foundation is machine learning and optimization for credit +decisions. Credit-scoring benchmarks define the performance frontier on retail +panels \citep{lessmann2015,ayari2026,boosting2025default,xia2017}. Recent IJDS credit-risk work shows how richer data structures such as firm graphs can improve rating prediction \citep{das2023creditgraph}, and cost-aware calibration work makes explicit why probability quality matters when predictions feed asymmetric -downstream decisions \citep{yang2025costaware}. Work specific to the Lending Club -platform spans alternative-data fintech lending \citep{jagtiani2019altdata}, -scorecard equity \citep{albanesi2024credit}, two-stage learning under fragmentary -data \citep{zheng2026twostage}, P2P portfolio selection and profit scoring +downstream decisions \citep{yang2025costaware}. IJDS decision papers also sharpen +the distinction between an accurate intermediate estimate and an effective +automated decision +\citep{fernandezloria2022causaldecision,fernandezloria2025observational}, while +replication-robust analytics markets show the journal's appetite for robust, +reproducible decision systems \citep{falconer2026replication}. Work specific to +the Lending Club platform spans alternative-data fintech lending +\citep{jagtiani2019altdata}, scorecard equity \citep{albanesi2024credit}, +two-stage learning under fragmentary data \citep{zheng2026twostage}, P2P +portfolio selection and profit scoring \citep{zhao2016p2pportfolio,serrano2016profitscoring}, and operations-research treatments of the same investment-decision problem CRPTO studies \citep{aior2025lendingclub}. CRPTO does not compete on raw ranking against this @@ -623,6 +638,11 @@ \section{Results}\label{sec:results} $V\le\sqrt{\alpha}$ certificate at the tightest reported level while keeping zero deterministic violation. +The results are ordered around three reviewer questions: what certificate the +selected policy carries, where that policy sits on the finite-grid frontier, and +whether the same recipe transfers to other credit panels without changing the +Lending Club champion. + \begin{table}[t] \centering \caption{Frozen paper-facing core metrics by layer.} @@ -723,10 +743,13 @@ \section{Results}\label{sec:results} Table~\ref{tab:pool93-frontier} gives the manuscript its decision geometry. The body/default point is not the highest-return point and not the tightest-bound point; it is the balanced point selected by the declared return-bound lens. The -endpoint at Markov cap $0.273036$ shows how conservative the certified frontier -can become while preserving the return floor, and the \$223.5K endpoint shows -the economic return available when the committee accepts a looser cap. The -supplement reports the full frontier table and traceability details. +strict $\le 0.345$ row is reported separately because it is a different finite +policy: it earns \$184{,}800.41 with Markov cap $0.344996$, whereas the +body/default row earns \$184{,}832.48 with Markov cap $0.345084$. The endpoint +at Markov cap $0.273036$ shows how conservative the certified frontier can +become while preserving the return floor, and the \$223.5K endpoint shows the +economic return available when the committee accepts a looser cap. The supplement +reports the full frontier table and traceability details. The funded-set under-coverage remains structural rather than a calibration-draw effect. With $n_{\mathrm{cal}}=237{,}584$ calibration loans, the split-conformal @@ -857,10 +880,10 @@ \section{Robustness and Comparators}\label{sec:robustness} The second concern is whether conformal uncertainty is doing decision work or only adding conservative decoration. The answer is visible in the portfolio frontier. Policies are evaluated by return, exact alpha pass/fail, weighted miscoverage, and -$\Gamma_{\mathrm{CP}}$; the promoted point is selected because it earns the highest -robust return inside the exact feasible region. This differs from a workflow where -conformal intervals are plotted after the optimizer has already chosen a point-PD -allocation. +$\Gamma_{\mathrm{CP}}$; the promoted point is the body/default balanced policy on +that finite-grid frontier, while the strict $\le 0.345$ cap policy is reported +separately. This differs from a workflow where conformal intervals are plotted +after the optimizer has already chosen a point-PD allocation. The supplement also carries the reviewer-facing robustness checks: nested holdout, strict temporal holdout, exact evaluation of conformal finalists, uncertainty-set @@ -874,7 +897,11 @@ \section{Robustness and Comparators}\label{sec:robustness} object. Decision-focused training can reduce regret relative to an optimization loss, while CRPTO prioritizes calibrated uncertainty, traceable risk controls, and exact funded-set checks. The manuscript therefore does not claim to dominate SPO+ on every -regret metric; it claims a different auditability/economic trade-off. +regret metric; it claims a different auditability/economic trade-off. Relative to +two-stage PTO, SPO+/decision-focused learning, P2P profit scoring, P2P robust +portfolio optimization, and cost-aware calibration, the distinguishing object is +not a new ranking score but a budgeted conformal premium, exact funded-set audit, +and finite-grid frontier denominator. \subsection{Regret-Auditability Frontier}\label{sec:regret} @@ -889,7 +916,9 @@ \subsection{Regret-Auditability Frontier}\label{sec:regret} predictive uncertainty before funding. The frontier is therefore not a single leaderboard. It asks what the method buys besides regret: temporal coverage above target, an exact funded-set $\alpha=0.01$ pass, and a finite-grid return-bound -frontier. +frontier. This is the cleanest comparator story in the paper. SPO+ answers how +much regret training can remove; CRPTO answers what a reviewer can verify after +a calibrated PD model is frozen. \begin{table}[t] \centering diff --git a/paper/submission/IJDS_SUBMISSION_ROADMAP_2026-08-10.md b/paper/submission/IJDS_SUBMISSION_ROADMAP_2026-08-10.md index 12fd04c..72c144d 100644 --- a/paper/submission/IJDS_SUBMISSION_ROADMAP_2026-08-10.md +++ b/paper/submission/IJDS_SUBMISSION_ROADMAP_2026-08-10.md @@ -53,7 +53,7 @@ CRPTO should be read as data science for decisions: | 9 | Supplement | A3--A39 are organized as a defense layer with scope caveats. | | 10 | Reproducibility | Accepted-paper package has code, DVC pointers, manifest, raw-data instructions, and guardrail commands. | | 11 | Double anonymity | Reviewer-facing body and supplement contain no author URLs, names, local paths, or private remotes. | -| 12 | Official IJDS template | `CRPTO_ijds_submission.tex` is regenerated from the pool93 A35--A39 QMD source, compiles against the official files, and is rechecked after body edits. | +| 12 | Official IJDS template | `CRPTO_ijds_submission.tex` is manually synchronized from the pool93 A35--A39 QMD source, keeps the official-template compaction, compiles against the official files, and is rechecked after body edits. | | 13 | Data/code form | Cover letter and disclosure text acknowledge IJDS accepted-paper reproducibility requirements. | | 14 | Acceptance-risk audit | A short list of likely reviewer objections has body or supplement responses. | | 15 | Freeze discipline | Protected champion/search stages are never rerun as routine paper reproduction. | diff --git a/paper/submission/README.md b/paper/submission/README.md index d9d3017..7e01923 100644 --- a/paper/submission/README.md +++ b/paper/submission/README.md @@ -56,10 +56,12 @@ purpose. ## Official LaTeX Submission Build `CRPTO_ijds_submission.tex` is the official-template handoff draft in the -INFORMS class (`\documentclass[ijds,dblanonrev]{informs4}`). The source of -truth remains `paper/CRPTO_ijds.qmd`; after the 2026-07-02 pool93 A35--A39 -update, the `.tex` must be regenerated before any ScholarOne freeze. The synchronized -submission surface should carry the central IJDS body: title, abstract, +INFORMS class (`\documentclass[ijds,dblanonrev]{informs4}`). The narrative +source remains `paper/CRPTO_ijds.qmd`, but the official `.tex` is now a +manually compacted IJDS-template surface. After freeze, do **not** regenerate it +mechanically from QMD; port substantive claim changes deliberately, then rebuild +and recheck the 26-page official PDF. The synchronized submission surface should +carry the central IJDS body: title, abstract, keywords, core sections, the journal pipeline Figure 1, the bound-claim stack, the A35 finite-grid frontier, the A36--A39 selected-allocation audits in the supplement, the regret-auditability comparison, plus the core, exact-certificate, @@ -75,14 +77,22 @@ PDF crop box cuts the right edge under `informs4`. > `informs4.cls`, `informs2014.bst`, template PDFs, `.sty` files, or generated > LaTeX build artifacts. -Current local build state (verified 2026-07-06): TinyTeX/TeX Live 2026 and the -`listingsutf8` TeX package compile `CRPTO_ijds_submission.tex` to a 26-page -official-template PDF. Section 9 (Conclusion) and References both start on page -22, so the body remains inside the IJDS 25-page initial-submission budget when -references are excluded. The only LaTeX log warnings left are a small -`\maketitle` overfull from the `informs4` anonymous title block and font-size / -underfull paragraph warnings, visually acceptable unless the final ScholarOne -proof shows a layout issue. +Current local build state (verified 2026-07-07): TinyTeX/TeX Live 2026, +`pdflatex`, `bibtex`, and the `listingsutf8` TeX package compile +`CRPTO_ijds_submission.tex` to a 26-page official-template PDF. Section 9 +(Conclusion) and References both start on page 22, so the body remains inside +the IJDS 25-page initial-submission budget when references are excluded. The +only LaTeX log warnings left are a small `\maketitle` overfull from the +`informs4` anonymous title block and font-size / underfull paragraph warnings, +visually acceptable unless the final ScholarOne proof shows a layout issue. + +`latexmk` remains the preferred command because it automates the required +LaTeX/BibTeX convergence loop. On 2026-07-07, the local Codex PowerShell +environment was missing `WINDIR`, which made TinyTeX wrapper scripts fail with +`runscript.tlu:712: attempt to concatenate a nil value`. Set `WINDIR` from +`SystemRoot` before calling TinyTeX wrappers in that environment. After +`tlmgr update --self --all`, the LaTeX format also had to be refreshed with +`fmtutil-sys --byfmt pdflatex` to resolve an `expl3` format mismatch. To produce the official submission PDF: @@ -90,14 +100,23 @@ To produce the official submission PDF: author portal (or Overleaf) and drop them next to `CRPTO_ijds_submission.tex`. These are gitignored on purpose (`paper/submission/.gitignore`); do not commit them. -2. Build with `latexmk` when the local TinyTeX wrapper works: +2. Build with `latexmk`. In Codex/PowerShell sessions where `WINDIR` is absent, + initialize it first: ```powershell + if (-not $env:WINDIR) { $env:WINDIR = $env:SystemRoot } latexmk -pdf -gg -interaction=nonstopmode CRPTO_ijds_submission.tex ``` - If PowerShell/TinyTeX fails with `runscript.tlu`/`nil`, use the proven - fallback: + If LaTeX reports mismatched support files after a TeX Live update, rebuild + the local TinyTeX format once: + + ```powershell + if (-not $env:WINDIR) { $env:WINDIR = $env:SystemRoot } + fmtutil-sys --byfmt pdflatex + ``` + + If PowerShell/TinyTeX still fails, use the proven fallback: ```powershell pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex @@ -106,6 +125,12 @@ To produce the official submission PDF: pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex ``` + The three `pdflatex` passes are intentional. The first pass writes the + `.aux` file that BibTeX needs; `bibtex` then writes the `.bbl`; the second + `pdflatex` pass reads the bibliography and updates citations, cross + references, labels, and page anchors; the final pass stabilizes any values + that shifted after the bibliography and floats were inserted. + 3. The `dblanonrev` option keeps the body anonymous; verify against the anonymity checklist below before uploading. @@ -158,8 +183,10 @@ These protocols are compatible but not interchangeable. - Preserve CRPTO as the coverage/auditability method and SPO+ as the low-regret comparator. - Cross-check every headline claim against `CLAIM_AUDIT_MATRIX.md`. -- Keep `CRPTO_ijds_submission.tex` synchronized with the QMD whenever the body - adds or demotes a figure, table, theorem statement or major result paragraph. +- Keep `CRPTO_ijds_submission.tex` semantically synchronized with the QMD + whenever the body adds or demotes a figure, table, theorem statement or major + result paragraph. Preserve the manual compaction choices that keep the + official-template PDF inside the IJDS page budget. - Regenerate previews with `just paper-submission-pdf` before release. - Run the repository gates: `just lint`, `just smoke`, `just validate-champion`. @@ -190,15 +217,20 @@ updates the template. ``` Use the documented `pdflatex -> bibtex -> pdflatex -> pdflatex` fallback if - the local TinyTeX wrapper fails. + the local TinyTeX wrapper fails after the `WINDIR` and format-refresh steps. ```powershell + if (-not $env:WINDIR) { $env:WINDIR = $env:SystemRoot } pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex bibtex CRPTO_ijds_submission pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex ``` + The repeated `pdflatex` calls are not redundant: pass 1 creates `.aux`, + BibTeX creates `.bbl`, pass 2 imports bibliography/citation data, and pass 3 + converges final references and pagination. + 4. **Recount the official-template page budget** and demote body floats to the supplement only if the body exceeds 25 pages excluding references. The local official-template build is currently 26 pages total; Section 9 and References diff --git a/paper/submission/REPRODUCIBILITY_PACKAGE.md b/paper/submission/REPRODUCIBILITY_PACKAGE.md index a47f34b..3b2d1fd 100644 --- a/paper/submission/REPRODUCIBILITY_PACKAGE.md +++ b/paper/submission/REPRODUCIBILITY_PACKAGE.md @@ -46,6 +46,7 @@ is synchronized: ```powershell cd paper/submission +if (-not $env:WINDIR) { $env:WINDIR = $env:SystemRoot } latexmk -pdf -gg -interaction=nonstopmode CRPTO_ijds_submission.tex ``` @@ -53,12 +54,21 @@ PowerShell/TinyTeX fallback proven in the local Codex environment: ```powershell cd paper/submission +if (-not $env:WINDIR) { $env:WINDIR = $env:SystemRoot } pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex bibtex CRPTO_ijds_submission pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex ``` +The repeated `pdflatex` calls are the standard LaTeX/BibTeX convergence loop: +first `.aux`, then BibTeX `.bbl`, then citation/cross-reference import, then +final pagination/reference stabilization. As of 2026-07-07, the local Codex +PowerShell environment needed `WINDIR` initialized from `SystemRoot` before +TinyTeX wrappers such as `latexmk` and `fmtutil-sys` would run; after +`tlmgr update --self --all`, `fmtutil-sys --byfmt pdflatex` also refreshed the +LaTeX format to match the updated support files. + Artifact-aware DVC verification, when credentials or public artifact access are available: diff --git a/paper/submission/SCHOLARONE_FINAL_CHECKLIST.md b/paper/submission/SCHOLARONE_FINAL_CHECKLIST.md index 86a203a..14551d8 100644 --- a/paper/submission/SCHOLARONE_FINAL_CHECKLIST.md +++ b/paper/submission/SCHOLARONE_FINAL_CHECKLIST.md @@ -7,36 +7,45 @@ template files have been downloaded outside Git. | File | Source | Reviewer-facing? | Status | |---|---|:---:|---| -| Anonymous manuscript PDF | `CRPTO_ijds_submission.tex` compiled with `informs4` and `dblanonrev`. | Yes | Source synchronized with pool93 A35--A39 and the dual-tag provenance passages; local official-template build verified 2026-07-06 (26 pages total; conclusion and references start on p. 22); final ScholarOne proof pending | -| Anonymous online supplement PDF | `paper/supplement_ijds.qmd` rendered and visually checked. | Yes | Local render and page QA pass; final ScholarOne proof pending | -| Separate title page | `TITLE_PAGE_DRAFT.md` converted into the ScholarOne/title-page format. | No | Pending ScholarOne copy | -| Data and Code Disclosure Form | Official IJDS form using `DATA_CODE_DISCLOSURE_FORM_DRAFT.md`. | Editor/system | Pending official form entry | -| Cover letter | `COVER_LETTER_AND_DISCLOSURE.md`, shortened if ScholarOne text boxes are tight. | Editor | Draft ready; final text-box copy pending | +| Anonymous manuscript PDF | `CRPTO_ijds_submission.tex` compiled with `informs4` and `dblanonrev`. | Yes | Local file ready. Official-template build verified 2026-07-07 (26 pages total; `.blg` warnings 0; `.log` has no undefined citations/references; source/metadata anonymity checks clean). Final ScholarOne proof remains a system-side gate. | +| Anonymous online supplement PDF | `paper/supplement_ijds.qmd` rendered and visually checked. | Yes | Local render and representative page QA verified 2026-07-07; source/metadata anonymity checks clean; final ScholarOne proof pending | +| Separate title page | `TITLE_PAGE_DRAFT.md` converted into the ScholarOne/title-page format. | No | Draft ready for separate upload/copy; complete affiliation and ORCID in ScholarOne if applicable. | +| Data and Code Disclosure Form | Official IJDS form using `DATA_CODE_DISCLOSURE_FORM_DRAFT.md`. | Editor/system | Draft language ready; official ScholarOne form entry remains manual. | +| Cover letter | `COVER_LETTER_AND_DISCLOSURE.md`, shortened if ScholarOne text boxes are tight. | Editor | Draft ready; final paste/proof inside ScholarOne remains manual. | | Optional reproducibility note | `REPRODUCIBILITY_PACKAGE.md` or excerpted text if requested. | Editor/system | Optional | ## Official Template Build 1. Download or refresh `informs4.cls` and `informs2014.bst` from INFORMS/Overleaf. -2. Regenerate/synchronize `CRPTO_ijds_submission.tex` from the pool93 A35--A39 QMD source. +2. Synchronize `CRPTO_ijds_submission.tex` manually from the pool93 A35--A39 + QMD source while preserving the official-template compaction. 3. Place the template files next to `CRPTO_ijds_submission.tex`; local gitignored copies are already present. -4. Build with `latexmk` when the local TinyTeX wrapper works: +4. Build with `latexmk`. In Codex/PowerShell sessions where `WINDIR` is absent, + initialize it first: ```powershell + if (-not $env:WINDIR) { $env:WINDIR = $env:SystemRoot } latexmk -pdf -gg -interaction=nonstopmode CRPTO_ijds_submission.tex ``` - If PowerShell/TinyTeX fails with `runscript.tlu`/`nil`, use the proven - fallback: + If LaTeX reports a support-file mismatch after a TeX Live update, run + `fmtutil-sys --byfmt pdflatex` once with the same `WINDIR` initialization. + If PowerShell/TinyTeX still fails, use the proven fallback: ```powershell + if (-not $env:WINDIR) { $env:WINDIR = $env:SystemRoot } pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex bibtex CRPTO_ijds_submission pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex ``` + The three `pdflatex` passes are intentional: the first creates `.aux`, + BibTeX creates `.bbl`, the second imports bibliography and cross-reference + data, and the third stabilizes references and pagination. + 5. Confirm body page count is at most 25 pages excluding references and - appendices. The local official-template build verified on 2026-07-06 is 26 + appendices. The local official-template build verified on 2026-07-07 is 26 pages total; Section 9 (Conclusion) and References both start on page 22, so the manuscript remains comfortably inside the IJDS page budget when references are excluded. Recount after every official rebuild. @@ -52,6 +61,28 @@ uv run dvc status --no-updates just paper-submission-pdf ``` +Last local closeout on 2026-07-07: `just lint`, `just smoke`, +`just validate-champion`, `uv run pytest tests/test_publication_targets.py -q`, +`uv run pytest tests/test_pool93_body_claim_sync.py -q`, +`just paper-submission-pdf`, and `latexmk -pdf -gg -interaction=nonstopmode +CRPTO_ijds_submission.tex` passed. Representative PDF pages were rendered +locally for body/supplement/submission visual QA. + +`uv run dvc status --no-updates` is intentionally treated as a pipeline-state +report, not a submission blocker. On 2026-07-07 it reported modified deps/outs +for protected and paper/book stages from earlier code/book work, while +`just validate-champion` remained green. Do not resolve that status by +rerunning protected stages during ScholarOne closeout; open a separate pipeline +debt task after submission if needed. + +## QMD-vs-TeX Freeze Rule + +`paper/CRPTO_ijds.qmd` remains the long-form narrative source. The official +`CRPTO_ijds_submission.tex` is a manually compacted INFORMS-template handoff +surface. After freeze, port substantive claim edits from QMD to TeX deliberately +and recompile; do not regenerate the `.tex` mechanically unless you are prepared +to redo the page-budget and visual QA. + ## Anonymous PDF QA - Body PDF has no author names, affiliation, acknowledgements, public repo URLs, diff --git a/paper/supplement_ijds.qmd b/paper/supplement_ijds.qmd index 4ea7355..3bdb326 100644 --- a/paper/supplement_ijds.qmd +++ b/paper/supplement_ijds.qmd @@ -42,6 +42,16 @@ path, and Appendix F maps the anonymous submission files. Tables and figures are included only when they either protect a body claim or prevent overreading of the external and robustness evidence. +| Reader question | Where to go | What to remember | +|---|---|---| +| What is the theorem? | Appendix A | Markov under weighted funded-set validity is the body claim; stronger assumptions stay explicit. | +| Is the selected policy a singleton? | Appendix C, A35 | The frontier is finite-grid and exact, with denominators reported. | +| What does the selected policy fund? | Appendix C, A36--A39 | Composition, tail, concentration, and bootstrap are selected-allocation diagnostics. | +| Does the recipe travel? | Appendix C, A25--A34 | Prosper and Freddie/Mendeley are external economic recipe-transfer checks. | +| What can be reproduced? | Appendix E | Routine reproduction rebuilds paper surfaces from frozen evidence and excludes protected searches. | + +: How to read the online supplement. + ::: {.callout-important} ## Journal Strengthening Pack @@ -442,6 +452,16 @@ Value-at-Risk [@rockafellar2000cvar] and the Optimized Certainty Equivalent [@angelopoulos2026nonmonotonic]; all are reported as post-hoc summaries of the frozen funded set or intervals, never as a re-promoted champion. +| Evidence block | Paper status | Why it is here | +|---|---|---| +| A35 finite-grid frontier | Promoted body evidence | It is the active return-bound decision surface. | +| A36--A39 selected-policy audits | Support evidence | They describe the selected allocation without changing the selector. | +| A19 regret-auditability | Support evidence | It answers the SPO+/DFL baseline objection. | +| A25--A34 external recipe transfer | Support evidence | It answers the single-dataset objection without new certificates. | +| A20--A24 tail/source/online diagnostics | Diagnostic evidence | They price assumptions and future-work lanes without promoting new guarantees. | + +: Promoted, support, and diagnostic evidence hierarchy. + | Table | Role | Scope caveat | |---|---|---| | A12 tail-risk OCE/CVaR diagnostics | Reprices the funded set under tail-risk summaries. | Diagnostic only; OCE/CVaR is not the optimized objective. | From 17811d8fb4c5dfc0035f86ac7095088533bfec5b Mon Sep 17 00:00:00 2001 From: Carlos Alfredo Vergara Rojas Date: Thu, 9 Jul 2026 18:34:53 -0500 Subject: [PATCH 2/7] refactor: harden IJDS certificate and workflow --- .codex/skills/crpto/SKILL.md | 21 +- .github/workflows/tests-full.yml | 1 + .gitignore | 5 + .pre-commit-config.yaml | 6 +- CLAUDE.md | 17 +- EXTRACTION_MANIFEST.json | 86 +- EXTRACTION_MANIFEST.md | 18 +- README.md | 30 +- .../crpto_fig12_crpto_conceptual_pipeline.pdf | Bin 219344 -> 219344 bytes .../crpto_fig13_alpha_gamma_funded_set.pdf | Bin 27860 -> 27860 bytes .../crpto_fig14_robust_region_heatmap.pdf | Bin 23806 -> 23806 bytes ...pto_fig15_regret_auditability_frontier.pdf | Bin 22490 -> 22490 bytes .../crpto_fig20_bound_claim_layers.pdf | Bin 23144 -> 23144 bytes ...rpto_fig25_price_of_robustness_scaling.pdf | Bin 66923 -> 66346 bytes ...rpto_fig25_price_of_robustness_scaling.png | Bin 122844 -> 116119 bytes book/chapters/00-crpto-en-una-pagina.qmd | 5 +- book/chapters/04-resultados.qmd | 30 +- book/chapters/05-discusion.qmd | 2 +- book/chapters/06-blueprint-manuscrito.qmd | 14 +- book/chapters/06b-guia-editorial-claims.qmd | 6 +- book/chapters/13-trazabilidad.qmd | 2 +- book/chapters/14-release.qmd | 7 +- .../16-fundamentos-conformal-optimizacion.qmd | 11 +- book/chapters/30-replicacion-multidataset.qmd | 41 +- book/includes/_build-info.qmd | 2 +- book/index.qmd | 11 +- book/references.bib | 164 ++ configs/crpto_publication_targets.yaml | 23 +- docs/SCOPE_AND_GOVERNANCE.md | 17 +- docs/refactor/README.md | 1 + .../ijds_tooling_refactor_lab_2026-07-08.md | 419 +++ docs/research/README.md | 11 +- docs/research/active_claims_2026-07-04.md | 79 +- .../crpto_editorial_claims_references.qmd | 17 +- docs/research/crpto_full_audit_2026-07-05.md | 12 +- .../crpto_publication_strategy_2026-05-12.md | 14 +- .../ijds_claim_concept_audit_2026-06-26.md | 5 + ..._claim_maximization_analysis_2026-06-27.md | 4 + ...rpus_claims_improvement_plan_2026-07-07.md | 6 + ...ds_literature_expansion_scan_2026-07-08.md | 514 ++++ ...simplification_cleanup_audit_2026-07-06.md | 9 +- ...l93_certificate_semantics_v2_2026-07-09.md | 132 + .../pool93_tail_risk_closeout_2026-07-02.md | 20 +- justfile | 30 +- .../pool93_ijds_consolidated_frontier.json | 358 +++ .../pool93_ijds_consolidated_governance.json | 198 ++ .../pool93_point_pd_baseline_audit.json | 86 + paper/CRPTO.qmd | 11 +- paper/CRPTO_ijds.qmd | 524 ++-- paper/README.md | 8 +- paper/submission/CLAIM_AUDIT_MATRIX.md | 9 +- paper/submission/CRPTO_ijds_submission.tex | 502 ++-- .../IJDS_SUBMISSION_ROADMAP_2026-08-10.md | 4 +- paper/submission/README.md | 27 +- .../submission/SCHOLARONE_FINAL_CHECKLIST.md | 2 +- paper/supplement_ijds.qmd | 244 +- pyproject.toml | 13 + .../crpto_fig13_alpha_gamma_funded_set.pdf | Bin 27860 -> 27860 bytes .../crpto_fig14_robust_region_heatmap.pdf | Bin 23806 -> 23806 bytes ...pto_fig15_regret_auditability_frontier.pdf | Bin 22490 -> 22490 bytes .../crpto_fig20_bound_claim_layers.pdf | Bin 23144 -> 23144 bytes ...rpto_fig25_price_of_robustness_scaling.pdf | Bin 66923 -> 66346 bytes ...rpto_fig25_price_of_robustness_scaling.png | Bin 122844 -> 116119 bytes reports/crpto/tables/README.md | 12 +- .../crpto_tableA35_pool93_ijds_frontier.csv | 21 +- .../crpto_tableA35_pool93_ijds_frontier.tex | 21 +- .../crpto_tableA40_pool93_point_baseline.csv | 3 + .../crpto_tableA40_pool93_point_baseline.tex | 8 + scripts/README.md | 78 + scripts/analyze_crpto_evidence.py | 16 +- .../search/monitor_regret_auditability.py | 3 +- scripts/backtest_conformal_coverage.py | 24 +- scripts/benchmark_conformal_variants.py | 1039 ++++--- scripts/benchmark_pd_set_prediction.py | 708 +++-- scripts/build_bound_tightening_audit.py | 44 +- scripts/build_crpto_journal_package.py | 12 +- scripts/build_papers_tesis_deep_audit.py | 375 +-- .../build_tail_constrained_reoptimization.py | 4 +- ...build_tail_satisficing_challenger_audit.py | 16 +- scripts/check_publication_integrity.py | 226 ++ scripts/compile_ijds_submission.py | 172 ++ .../run_champion_claim_max_downstream.py | 146 +- .../experiments/run_champion_reopen_hpo.py | 7 +- .../experiments/run_tabpfn_tabprep_full.py | 27 +- .../run_tabprep_feature_selection_catboost.py | 7 +- scripts/export_crpto_tables.py | 4 +- scripts/generate_conformal_intervals.py | 2516 +++++++++++------ scripts/generate_crpto_figures.py | 129 +- scripts/generate_governance_status.py | 915 +++--- scripts/generate_mrm_report.py | 26 +- scripts/optimize_portfolio.py | 78 +- scripts/optimize_portfolio_tradeoff.py | 123 +- scripts/run_comparison.py | 469 ++- scripts/run_cqr_comparison.py | 2 +- scripts/run_crpto_vs_spo_stability.py | 454 +-- scripts/run_fairness_audit.py | 925 +++--- scripts/run_spo_comparison.py | 5 +- scripts/run_spo_real.py | 34 +- scripts/run_ty_advisory.py | 131 + .../build_pool93_body_allocation_audit.py | 75 +- ...build_pool93_ijds_consolidated_frontier.py | 166 +- ...ild_pool93_ijds_consolidated_governance.py | 31 +- .../build_pool93_ijds_frontier_claim_table.py | 2 + .../build_pool93_point_baseline_audit.py | 372 +++ .../export_pool93_policy_aware_frontier.py | 143 + scripts/search/run_conformal_reopen_search.py | 1261 ++++++--- scripts/search/run_conformal_search.py | 22 +- .../run_pool93_ijds_local_refinement.py | 2055 +++++++------- .../run_portfolio_bound_aware_search.py | 1285 ++++++--- .../search/run_portfolio_bound_exact_eval.py | 602 ++-- scripts/search/run_portfolio_search.py | 22 +- .../search/run_regret_auditability_sandbox.py | 1211 +++++--- scripts/select_economic_portfolio_policy.py | 749 +++-- scripts/simulate_ab_test.py | 335 ++- scripts/train_pd_model.py | 1890 ++++++++----- scripts/validate_alpha_gamma_bound.py | 94 +- scripts/validate_conformal_policy.py | 440 +-- src/evaluation/backtesting.py | 40 +- src/evaluation/explainability.py | 16 +- src/evaluation/fairness.py | 28 +- src/evaluation/model_shift.py | 165 +- src/features/feature_config_io.py | 4 +- src/features/feature_engineering.py | 7 +- src/features/tabprep_challenger.py | 9 +- src/models/calibration.py | 20 +- src/models/conformal_adapters.py | 8 +- src/models/conformal_tuning.py | 484 ++-- src/models/optuna_tuning.py | 1535 ++++++---- src/optimization/certificate_semantics.py | 260 ++ src/optimization/cuopt_adapter.py | 223 +- src/optimization/input_alignment.py | 195 ++ src/optimization/portfolio_model.py | 742 +++-- src/utils/mlflow_tracing.py | 3 +- src/utils/script_helpers.py | 31 + tests/test_evaluation/test_model_shift.py | 39 + .../test_champion_reopen_orchestration.py | 10 + tests/test_manifest_regression.py | 9 + tests/test_models/test_conformal_tuning.py | 59 + .../test_certificate_semantics.py | 164 ++ tests/test_optimization/test_cuopt_adapter.py | 163 ++ .../test_optimization/test_input_alignment.py | 152 + tests/test_optimization/test_policy.py | 37 + .../test_optimization/test_portfolio_model.py | 61 +- tests/test_pool93_body_claim_sync.py | 74 +- tests/test_publication_integrity.py | 7 + tests/test_publication_targets.py | 6 +- .../test_benchmark_conformal_variants.py | 23 +- .../test_benchmark_pd_set_prediction.py | 61 + .../test_build_crpto_journal_package.py | 26 +- .../test_build_papers_tesis_deep_audit.py | 86 + ...build_pool93_ijds_consolidated_frontier.py | 77 + .../test_build_pool93_point_baseline_audit.py | 49 + .../test_compile_ijds_submission.py | 14 + ...est_export_pool93_policy_aware_frontier.py | 39 + .../test_generate_conformal_intervals_cli.py | 208 ++ .../test_generate_crpto_figures.py | 71 + .../test_generate_governance_status.py | 166 ++ .../test_optimize_portfolio_tradeoff.py | 97 + .../test_pool93_local_refinement_grid.py | 267 ++ .../test_retired_search_entrypoints.py | 19 + tests/test_scripts/test_run_comparison.py | 103 + .../test_run_conformal_reopen_search.py | 336 ++- .../test_run_crpto_vs_spo_stability.py | 63 + tests/test_scripts/test_run_fairness_audit.py | 55 + .../test_run_portfolio_bound_aware_search.py | 122 + .../test_run_portfolio_bound_exact_eval.py | 53 + .../test_run_regret_auditability_sandbox.py | 72 + tests/test_scripts/test_run_ty_advisory.py | 55 + tests/test_scripts/test_simulate_ab_test.py | 91 + .../test_train_pd_model_config_overrides.py | 145 + .../test_validate_conformal_policy.py | 46 + tests/test_utils/test_script_helpers.py | 18 + 172 files changed, 21274 insertions(+), 8952 deletions(-) create mode 100644 docs/refactor/ijds_tooling_refactor_lab_2026-07-08.md create mode 100644 docs/research/ijds_literature_expansion_scan_2026-07-08.md create mode 100644 docs/research/pool93_certificate_semantics_v2_2026-07-09.md create mode 100644 models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_frontier.json create mode 100644 models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_governance.json create mode 100644 models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_point_pd_baseline_audit.json create mode 100644 reports/crpto/tables/crpto_tableA40_pool93_point_baseline.csv create mode 100644 reports/crpto/tables/crpto_tableA40_pool93_point_baseline.tex create mode 100644 scripts/README.md create mode 100644 scripts/check_publication_integrity.py create mode 100644 scripts/compile_ijds_submission.py create mode 100644 scripts/run_ty_advisory.py create mode 100644 scripts/search/build_pool93_point_baseline_audit.py create mode 100644 scripts/search/export_pool93_policy_aware_frontier.py create mode 100644 src/optimization/certificate_semantics.py create mode 100644 src/optimization/input_alignment.py create mode 100644 tests/test_optimization/test_certificate_semantics.py create mode 100644 tests/test_optimization/test_cuopt_adapter.py create mode 100644 tests/test_optimization/test_input_alignment.py create mode 100644 tests/test_publication_integrity.py create mode 100644 tests/test_scripts/test_benchmark_pd_set_prediction.py create mode 100644 tests/test_scripts/test_build_papers_tesis_deep_audit.py create mode 100644 tests/test_scripts/test_build_pool93_ijds_consolidated_frontier.py create mode 100644 tests/test_scripts/test_build_pool93_point_baseline_audit.py create mode 100644 tests/test_scripts/test_compile_ijds_submission.py create mode 100644 tests/test_scripts/test_export_pool93_policy_aware_frontier.py create mode 100644 tests/test_scripts/test_generate_crpto_figures.py create mode 100644 tests/test_scripts/test_generate_governance_status.py create mode 100644 tests/test_scripts/test_pool93_local_refinement_grid.py create mode 100644 tests/test_scripts/test_retired_search_entrypoints.py create mode 100644 tests/test_scripts/test_run_comparison.py create mode 100644 tests/test_scripts/test_run_ty_advisory.py create mode 100644 tests/test_scripts/test_simulate_ab_test.py diff --git a/.codex/skills/crpto/SKILL.md b/.codex/skills/crpto/SKILL.md index 2620b44..10a10e6 100644 --- a/.codex/skills/crpto/SKILL.md +++ b/.codex/skills/crpto/SKILL.md @@ -36,6 +36,8 @@ frozen upstream PD/calibration/conformal artifacts, not a retraining run. - Terminal run tag: `champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal` +- Active certificate tag: + `champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2` - Body point source run: `champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext` - Policy family: `claim_micro_ext_body_cap345` @@ -44,14 +46,18 @@ frozen upstream PD/calibration/conformal artifacts, not a retraining run. - Return-floor surplus: `$14,367.94` - `V(alpha=0.01)`: `0.035350` - `Gamma_CP(alpha=0.01)`: `0.162616` -- Endpoint budget upper at `alpha=0.01`: `0.24508374` -- Markov cap at `alpha=0.01`: `0.34508374` -- Exact alpha violation: `0.0` +- `Gamma_internalized(alpha=0.01)`: `0.089032` +- `Gamma_residual(alpha=0.01)`: `0.073584` +- Exact endpoint budget at `alpha=0.01`: `0.245083866` +- Exact Markov loss threshold at `alpha=0.01`: `0.345083866` +- Realized risk-tolerance excess: `0.0` - Declared alpha-grid pass: `8/8` - Consolidated frontier: `50,010` deduplicated semantic policies, `27,508` eligible all-alpha above-floor policies. - Terminal exact search: `37,068/37,068` all-alpha passers and `296,544` completed exact candidate-alpha checks. +- Matched A40 point-PD baseline: `5.875%` realized-return cost, `0.08305` + weighted default/miscoverage reduction, and `0.435495` threshold reduction. The frozen upstream baseline remains retained for provenance and as the declared return floor: @@ -81,7 +87,10 @@ champion rebuild: - `reports/crpto/tables/crpto_tableA37_pool93_body_tail_risk.csv` - `reports/crpto/tables/crpto_tableA38_pool93_body_cluster_bound_audit.csv` - `reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.csv` -- `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_ijds_consolidated_governance.json` +- `reports/crpto/tables/crpto_tableA40_pool93_point_baseline.csv` +- `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_frontier.json` +- `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_governance.json` +- `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_point_pd_baseline_audit.json` - `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal/portfolio/pool93_ijds_claim_governance.json` - `EXTRACTION_MANIFEST.json` @@ -118,6 +127,8 @@ The active paper scope is the IJDS pool93 certificate plus bounded diagnostics: the body theorem because the cluster thresholds are not tighter here. - A39: fixed-allocation bootstrap diagnostic; it does not resample model, solver, conformal intervals or policy search. +- A40: matched point-PD baseline with candidates and operating constraints fixed; + one frozen OOT trade-off, not a causal or universal-dominance claim. - A20--A22: legacy tail-risk/OCE/CVaR diagnostic package retained in the supplement, not as the promoted pool93 selector. - A23--A24: multi-distribution/online coverage diagnostics, not universal @@ -151,7 +162,7 @@ scoring layer. The default rule is: For the current submission, keep these gates visible: -- Consolidate A19/Fig15, A20--A39, paper/supplement, docs, manifest hashes and +- Consolidate A19/Fig15, A20--A40, paper/supplement, docs, manifest hashes and `dvc.lock`. - Sweep the manuscript for stale numbers, captions, body-vs-appendix placement, and IJDS length. diff --git a/.github/workflows/tests-full.yml b/.github/workflows/tests-full.yml index 5ad9622..5b3f528 100644 --- a/.github/workflows/tests-full.yml +++ b/.github/workflows/tests-full.yml @@ -72,6 +72,7 @@ jobs: tests/test_features/ \ tests/test_crpto_final_sync.py \ tests/test_quarto_book_guardrails.py \ + tests/test_publication_integrity.py \ -q - name: Upload coverage if generated diff --git a/.gitignore b/.gitignore index 2e428de..109697f 100644 --- a/.gitignore +++ b/.gitignore @@ -92,6 +92,8 @@ models/feature_artifacts_runtime_status.json # Reports (large generated outputs) reports/run_logs/ +reports/api-docs/ +reports/ci/ reports/notebook_images/ reports/paper_material/ reports/crpto/experiments/ @@ -149,6 +151,9 @@ models/experiments/** !models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal/portfolio/pool93_ijds_claim_governance.json !models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_ijds_consolidated_governance.json !models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_ijds_consolidated_frontier.json +!models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_frontier.json +!models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_governance.json +!models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_point_pd_baseline_audit.json configs/experiments/ !configs/experiments/ configs/experiments/* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8434421..78d3195 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -47,15 +47,15 @@ repos: stages: [pre-push] - id: pytest-smoke - name: pytest smoke (final-sync + quarto guardrails) - entry: uv run pytest tests/test_crpto_final_sync.py tests/test_quarto_book_guardrails.py -q + name: just smoke (publication guardrails) + entry: just smoke language: system pass_filenames: false stages: [pre-push] - id: validate-champion name: validate champion artefacts - entry: uv run python -c "import json; from pathlib import Path; m=json.loads(Path('EXTRACTION_MANIFEST.json').read_text()); print('manifest OK, keys:', len(m))" + entry: just validate-champion language: system pass_filenames: false stages: [pre-push] diff --git a/CLAUDE.md b/CLAUDE.md index b58a842..0097bbb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -41,14 +41,17 @@ intervalos conformal congelados; **no regenera ningún artefacto upstream**. | Campo | Valor | | --- | --- | -| Run tag | `champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal` | +| Certificate tag | `champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2` | +| Source policy run | `champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext` | | Policy mode | `capped_blended_uncertainty` (familia `claim_micro_ext_body_cap345`) | | Retorno robusto | `$184,832.48` | | V(α=0.01) | `0.035350` | | Γ_CP(α=0.01) | `0.162616` | -| Markov cap (α=0.01) | `0.345084` | -| Alpha grid | `8/8`, violación exacta `0.0` | -| Evidencia | A35–A39 + JSONs de gobernanza pool93 | +| Γ_int / Γ_res (α=0.01) | `0.089032 / 0.073584` | +| Endpoint / Markov threshold (α=0.01) | `0.245084 / 0.345084` | +| Alpha grid | `8/8`, exceso realizado sobre τ `0.0` | +| Baseline A40 | costo de retorno `5.875%`; reducción default/V `8.305` pp | +| Evidencia | A35–A40 + JSONs de gobernanza/certificado pool93 | **Cadena upstream congelada (histórica; su retorno es el return floor declarado del pool93):** @@ -70,9 +73,11 @@ Artefactos congelados cuyos hashes están en `EXTRACTION_MANIFEST.json` y **no s - `models/conformal_policy_status.json` - `data/processed/conformal_intervals_mondrian.parquet` - `data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/` -- `reports/crpto/tables/crpto_tableA35..A39_pool93_*.csv/.tex` (evidencia pool93) +- `reports/crpto/tables/crpto_tableA35..A40_pool93_*.csv/.tex` (evidencia pool93) - `models/experiments/champion_reopen/...__pool93__ijds-claim-bound-terminal/portfolio/pool93_ijds_claim_governance.json` -- `models/experiments/champion_reopen/...__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_ijds_consolidated_governance.json` +- `models/experiments/champion_reopen/...__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_frontier.json` +- `models/experiments/champion_reopen/...__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_governance.json` +- `models/experiments/champion_reopen/...__pool93__ijds-certificate-semantics-v2/portfolio/pool93_point_pd_baseline_audit.json` - `EXTRACTION_MANIFEST.json` La sincronía del body claim con el paper la vigila `tests/test_pool93_body_claim_sync.py`. diff --git a/EXTRACTION_MANIFEST.json b/EXTRACTION_MANIFEST.json index a3f66d0..13c1f30 100644 --- a/EXTRACTION_MANIFEST.json +++ b/EXTRACTION_MANIFEST.json @@ -1,5 +1,5 @@ { - "schema_version": 5, + "schema_version": 6, "project_name": "CRPTO", "source_project": "local source repository; read-only during extraction; absolute source path intentionally omitted in final package", "source_project_note": "Formal IJDS rebaseline computed in the standalone Paper_CRPTO repository; the parent WSL project is provenance only.", @@ -893,9 +893,9 @@ "hash_source": "feature_config_pickle_retirement_2026-06-13" }, "reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv": { - "sha256": "fbaf05312426058a6b7691f696198666ee289374305a3867dd84f589bddb0132", - "bytes": 2264, - "hash_source": "pool93_ijds_promotion_2026-07-02" + "sha256": "819b2927856ffcc4b6993aa0f9e35839a562803cd7685ddb3013390e64659a44", + "bytes": 2334, + "hash_source": "pool93_certificate_semantics_v2_2026-07-09" }, "reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.csv": { "sha256": "9d02382c22c7bd58fdbb287de43d0ca09714bae1753b4d30eb1609cb1502740f", @@ -918,9 +918,9 @@ "hash_source": "pool93_ijds_promotion_2026-07-02" }, "reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.tex": { - "sha256": "37b4c4d0d2da6f4dbcae55031c07905a12875776463eecae6cab086dc5e8d6f9", - "bytes": 956, - "hash_source": "pool93_ijds_promotion_2026-07-02" + "sha256": "b951567302b3547573d1fbf569ef1ede3e15f8b2e1484b5d2a13399eff4eedf9", + "bytes": 1233, + "hash_source": "pool93_certificate_semantics_v2_2026-07-09" }, "reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.tex": { "sha256": "20ab4514684f1d2d2cb888be022385d0db771d9fd78b4c87ebff19958e96ad9d", @@ -951,6 +951,31 @@ "sha256": "e686f8b0f76c8b3a63351b97ea3761f55fb8921a1c23b051e59904163e8a79de", "bytes": 7996, "hash_source": "pool93_ijds_promotion_2026-07-02" + }, + "reports/crpto/tables/crpto_tableA40_pool93_point_baseline.csv": { + "sha256": "25a9420250f275ccf927d23de6a2a5cbfc46bf411552f99fb684fc2cc21ddafe", + "bytes": 492, + "hash_source": "pool93_certificate_semantics_v2_2026-07-09" + }, + "reports/crpto/tables/crpto_tableA40_pool93_point_baseline.tex": { + "sha256": "4ab75f6f6c106947d4be50d568f390e7c53bf47f01117cfdf04bfbcc9bd1f4de", + "bytes": 328, + "hash_source": "pool93_certificate_semantics_v2_2026-07-09" + }, + "models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_frontier.json": { + "sha256": "5d3f82559ae60e450af9e30644fe08cb728f0fe39b0fa7b05b50dc29668be6ed", + "bytes": 15290, + "hash_source": "pool93_certificate_semantics_v2_2026-07-09" + }, + "models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_governance.json": { + "sha256": "b698e4269b2dfbd028840a432bc3b926cc7f6dd9ec63b8550c392b38dc445c48", + "bytes": 9823, + "hash_source": "pool93_certificate_semantics_v2_2026-07-09" + }, + "models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_point_pd_baseline_audit.json": { + "sha256": "bd1dfd75fdc5a13334ab71040de1750eabda05898660e93c8ab1723e8dbaa597", + "bytes": 3810, + "hash_source": "pool93_certificate_semantics_v2_2026-07-09" } }, "validation_results": { @@ -4318,9 +4343,10 @@ ] }, "pool93_ijds_promotion": { - "promoted_at_utc": "2026-07-02T06:05:17.323703+00:00", - "note": "Pool93 finite-grid body claim promoted for the IJDS paper. The champion_metrics block above remains the frozen ijds-rebaseline-2026-06-07 upstream chain (PD champion, Mondrian conformal intervals, bound-aware 276k search); pool93 is a deterministic policy-grid re-evaluation over the same frozen conformal intervals and does not regenerate any upstream artifact.", + "promoted_at_utc": "2026-07-09T23:10:36+00:00", + "note": "Pool93 policy-aware certificate semantics v2 promoted for the IJDS paper. The selected allocation, upstream PD/calibration/conformal chain, alpha grid and finite-grid denominators are unchanged. Exact endpoint budgets are rehydrated from existing evaluations; A40 adds a matched point-PD baseline without rerunning protected stages.", "terminal_run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal", + "certificate_semantics_run_tag": "champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2", "consolidated_run_tags": [ "champion-reopen-2026-06-19__pool93__ijds-claim-expanded-refine", "champion-reopen-2026-06-19__pool93__ijds-claim-micro-refine", @@ -4341,10 +4367,16 @@ "realized_total_return": 184832.475845, "return_floor_surplus": 14367.935845, "alpha01_gamma_cp": 0.162616, + "alpha01_gamma_internalized": 0.089032, + "alpha01_gamma_residual": 0.073584, "alpha01_weighted_miscoverage_V": 0.03535, - "endpoint_budget_upper_alpha01": 0.24508374, - "markov_cap_alpha01": 0.34508374, + "alpha01_realized_risk_tolerance_excess": 0.0, + "endpoint_budget_alpha01": 0.245084, + "endpoint_budget_upper_alpha01": 0.245084, + "markov_threshold_alpha01": 0.345084, + "markov_cap_alpha01": 0.345084, "alpha_grid_pass": "8/8", + "alpha01_n_funded": 314, "n_funded_mean": 320.5, "declared_return_floor": 170464.54, "bootstrap_return_interval_p025_p975": [ @@ -4364,19 +4396,49 @@ "n_all_alpha_passers": 37068, "n_all_alpha_passers_above_return_floor": 14814 }, + "certificate_semantics_audit": { + "status": "corrected_from_existing_exact_bound_evaluations", + "body_selection_unchanged": true, + "materially_changed_policies": 10423, + "materially_understated_policies": 2866, + "maximum_legacy_understatement": 0.2413235, + "legacy_under_0_50_excluded_by_exact_threshold": 716, + "affected_policy_modes": [ + "segment_relative_tail_blended_uncertainty", + "tail_blended_uncertainty" + ] + }, + "matched_point_pd_baseline": { + "candidate_universe": 276869, + "budget": 1000000.0, + "risk_tolerance": 0.1715, + "point_pd_realized_return": 196369.14000000004, + "selected_crpto_realized_return": 184832.47584455396, + "realized_return_cost": 11536.66415544608, + "realized_return_cost_pct": 5.874988379256577, + "weighted_default_rate_reduction": 0.08305000000000001, + "weighted_miscoverage_reduction": 0.08305000000000001, + "markov_threshold_reduction": 0.4354954303914803, + "claim_boundary": "Frozen OOT matched-policy audit; no causal, prospective or universal dominance claim." + }, "protected_artifacts": [ "reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv", "reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.csv", "reports/crpto/tables/crpto_tableA37_pool93_body_tail_risk.csv", "reports/crpto/tables/crpto_tableA38_pool93_body_cluster_bound_audit.csv", "reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.csv", + "reports/crpto/tables/crpto_tableA40_pool93_point_baseline.csv", "reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.tex", "reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.tex", "reports/crpto/tables/crpto_tableA37_pool93_body_tail_risk.tex", "reports/crpto/tables/crpto_tableA38_pool93_body_cluster_bound_audit.tex", "reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.tex", + "reports/crpto/tables/crpto_tableA40_pool93_point_baseline.tex", "models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal/portfolio/pool93_ijds_claim_governance.json", - "models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_ijds_consolidated_governance.json" + "models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_ijds_consolidated_governance.json", + "models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_frontier.json", + "models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_governance.json", + "models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_point_pd_baseline_audit.json" ] } } diff --git a/EXTRACTION_MANIFEST.md b/EXTRACTION_MANIFEST.md index 9cf475d..4da809f 100644 --- a/EXTRACTION_MANIFEST.md +++ b/EXTRACTION_MANIFEST.md @@ -8,16 +8,20 @@ exists, and how `tests/test_manifest_regression.py` enforces it. ## TL;DR -- **Schema version**: 5 (top-level key `schema_version`). +- **Schema version**: 6 (top-level key `schema_version`). - **Dual-tag governance**: - frozen upstream baseline: `ijds-rebaseline-2026-06-07`; - - active IJDS body claim: pool93 finite-grid frontier closure - `champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal`. -- **182 critical files** are hashed under `critical_hashes` (SHA256 + byte + - active IJDS certificate semantics: + `champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2`. +- **187 critical files** are hashed under `critical_hashes` (SHA256 + byte count). - **Pool93 body claim**: return `$184,832.48`, `V(alpha=0.01)=0.035350`, - `Gamma_CP(alpha=0.01)=0.162616`, Markov cap `0.345084`, exact alpha - violation `0.0`, declared alpha-grid pass `8/8`. + `Gamma_CP=0.162616`, `Gamma_res=0.073584`, endpoint `0.245084`, exact + Markov loss threshold `0.345084`, realized risk-tolerance excess `0.0`, and + declared alpha-grid pass `8/8`. +- **Matched A40 baseline**: CRPTO pays `5.875%` realized return relative to a + point-PD LP and reduces weighted default/miscoverage by `8.305` percentage + points under matched operating constraints. - **6 files are flagged as non-overwriteable** without a fresh run tag: - `models/pd_canonical.cbm` - `models/pd_canonical_calibrator.pkl` @@ -46,7 +50,7 @@ exists, and how `tests/test_manifest_regression.py` enforces it. | `generated_at_utc` | When the manifest was produced. | | `summary` | Free-text human description of the extraction scope. | | `champion_metrics` | Frozen upstream baseline numbers retained as provenance and as the declared return floor. | -| `pool93_ijds_promotion` | Active IJDS body-claim metadata for the pool93 finite-grid frontier closure. | +| `pool93_ijds_promotion` | Active IJDS metadata for the policy-aware frontier, semantic audit, selected body point, and A40 matched baseline. | | `critical_hashes` | Map `relative_path → {sha256, bytes, hash_source}` for every file the paper depends on. | | `validation_results` | Output of the extraction-time guardrail tests. | | `files` | Inventory of files copied/created during extraction. | diff --git a/README.md b/README.md index 9bc7f02..5b8384c 100644 --- a/README.md +++ b/README.md @@ -4,19 +4,26 @@ Pipeline de investigación y libro Quarto que acompañan el paper **CRPTO**, una > CRPTO opera como repositorio standalone: GitHub, DVC y MLflow apuntan a recursos propios del paper. La historia de extracción y aprendizajes queda documentada en [`docs/PROJECT_HISTORY.md`](docs/PROJECT_HISTORY.md). -## Champion congelado +## Claim IJDS activo | Campo | Valor | | --- | --- | -| Run tag | `ijds-rebaseline-2026-06-07` | -| Policy | `bound_aware_276k_economic_champion` | -| Retorno robusto | **$170,464.54** | -| `V(α=0.01)` | `0.028875` | -| `Γ_CP(α=0.01)` | `0.187987` | -| `α=0.01 exact pass` | `True` | -| Región robusta | `45/45` | +| Certificate tag | `champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2` | +| Policy family | `claim_micro_ext_body_cap345` | +| Policy mode | `capped_blended_uncertainty` | +| Retorno robusto | **$184,832.48** | +| `V(α=0.01)` | `0.035350` | +| `Γ_CP(α=0.01)` | `0.162616` | +| `Γ_int / Γ_res` (`α=0.01`) | `0.089032 / 0.073584` | +| Endpoint / Markov threshold | `0.245084 / 0.345084` | +| Alpha-grid pass | `8/8` | +| Frontera consolidada | `50,010` políticas semánticas; `27,508` elegibles sobre el return floor | +| Baseline A40 | costo de retorno `5.875%`; reducción default/V `8.305` pp | Hashes SHA256 de los artefactos críticos están en [`EXTRACTION_MANIFEST.json`](EXTRACTION_MANIFEST.json). Verifica con `just validate-champion` o el skill `/crpto-validate-champion`. +El rebaseline `ijds-rebaseline-2026-06-07` se conserva como upstream congelado +y return floor histórico (`$170,464.54`, `45/45`), no como el claim activo del +manuscrito IJDS. ## Requisitos del sistema @@ -67,8 +74,13 @@ just figures # solo PNGs/PDFs just lint # ruff check + format check just fmt # ruff fix + format just type-check # mypy src scripts +just type-advisory # ty sobre ruta activa IJDS, no bloqueante +just type-advisory-full # ty sobre src/scripts completos, deuda opcional/historica +just api-docs-core # pdoc local para modulos core, salida ignorada +just hooks-check # valida hooks con pre-commit y prek just smoke # tests críticos rápidos just test # suite completa +just submission-check # cierre IJDS: claims, lint, type, smoke, champion y PDF oficial # DVC just dvc-status # drift detection @@ -146,7 +158,7 @@ just paper-submission │ └── apa.csl # estilo APA 7 ├── crpto/ # paquete público mínimo (`import crpto`) ├── src/ # módulos fuente históricos (data, features, models, optimization, evaluation, utils) -├── scripts/ # 40+ entry points +├── scripts/ # entry points; ver scripts/README.md para rutas IJDS vs históricas ├── tests/ # 26 archivos pytest (markers slow / integration) ├── configs/ # YAML (pd_model, conformal, optimization, fairness, mrm) ├── dbt_project/ # 3 staging + 3 marts sobre crpto.duckdb diff --git a/book/assets/figures/publication/crpto_fig12_crpto_conceptual_pipeline.pdf b/book/assets/figures/publication/crpto_fig12_crpto_conceptual_pipeline.pdf index b54718499f3bd6db9be0ccb55ceda595624c4c12..106f6647d0a7db5d2b99c029a0814985b449c390 100644 GIT binary patch delta 59 zcmcbxk@vzz-i8*&7N#xCN5j?34J-|eOf3y9qBObmee+XX5=&AQG+eBV42;YS;F8l{ IhciC~06$F0C>3wbpQYW delta 24 gcmeyjlkwk9#tlMIoMwh521W)3CKj8eqaHH>0C+_RYybcN diff --git a/book/assets/figures/publication/crpto_fig15_regret_auditability_frontier.pdf b/book/assets/figures/publication/crpto_fig15_regret_auditability_frontier.pdf index 89f5a6d6522691c5eb14e3bf4f21d46a8b73c72a..7e7d9deafa1b0df2aed7ed4216930627a29f4e61 100644 GIT binary patch delta 24 gcmcb$p7GXt#tn9%oaP1=Muw)AMuwZ+Lnku<0CQRh7ytkO delta 24 gcmcb$p7GXt#tn9%oMwhbM#jbl#wMHHLnku<0CMCA4FCWD diff --git a/book/assets/figures/publication/crpto_fig20_bound_claim_layers.pdf b/book/assets/figures/publication/crpto_fig20_bound_claim_layers.pdf index 447bd409373f1a9ecb47cf8c66a34f7a116f681f..91625292a9f1e8a29b369958196e5131124e1fa8 100644 GIT binary patch delta 24 gcmaE{h4IA}#tmn}In50$j0{aJ4J|fb4WG;e0D_hYWB>pF delta 24 gcmaE{h4IA}#tmn}In4}>jEs#9j14zm4WG;e0DS^o9NbBNmh}+4J^D@-Vkdh-8@Kzzb+x$ z_2|hT$Xzi#>bZ>eY`IQj3iX2%W%TsfGJntTrf)g%R5LYSlX>CEv$S?UPb@>N2?hKD zc!+pSCZth=qS_)8ZrKC&a6%j3grl}A`*NxaOmxj-_HOChoMa7w;pgvNO! zEqP_9mPqS`Ti|Dut=cb|&f4q-Rdu8TXtm}En@svtHz|bJVSqzVd;ioOci|^)2OrM*6JS0_!bW-=o@10n;S7Z=zKDv;_5A%Y{BhD zChIoCoYyQ*jgqkFeoEF#lW-k$z$cx8K)C8P`%E)?x=1hNFiO52AeDCK=5kWx6eiXkHSq42Y1#aWu$d~90LRX7CF1g0}80CACr z4iulLu`d}$fvIFCwwYEz-4>tfr{X(-w6AnN-E?8X9M%+i_}YV_A?~P`3>v!C?w5a6 zi@U|ZqF#QSYG989uw5Diti48F3rVe%mKeqLNL)>ucWahBt=)6TP#H%jc0D&krM=Z| z((G}NyWT`2p;}Gi6qusoL;J#>9L|^}F226GR551}Wes$AIaEt)K#RdwW{J1N!YW(R zSeO0!-Z?X#dzdXzUZ@(LiuGq5Lp~3a)+Uxf0n5hJ{7hLVhwfol|?+3Ys~+ z;`ykRZw47eiE~7a#ptYBG^$Ptn)C7DKDH33h8n;6Qm~!qYOZVGJE-98qy8$;{2{6Q z$CXi7!+4g(1x(t-&#TzxHYM~?2Y?TAr|a^q`1EUXoHo|0tXER$l?Tjq9Y^7dzRXI$ z3h0t+x6epcxJsI$(!5egSVcU8v$9BZ^~eqN_Vt`wvi&WwI?W5;r<$7?9oR@o zcPYKeqq9;IRfNE+$7{M7`D?K~D>2xyVc{jPwgMp&0H?DNGOnG{sU~?jKEh(A^2&wA zlT);>Ka8Sg&v3gii#h!@ z+V0Ld)x}qvzxl4$&)WnyO$QI&AlBT)Q`NOgF7B}r!ghm&IrB3jMGVbjf3YIYSWB7` z6r}7Zcxe3|Ps0>EfMBXrqGwVn*3E}%xy1_E0%XxwC24$|jMctSCqqDGe_iNjoTQZN z$S;9AiOr}Bx<`_;t-EQmI)c^i1BdvB@R(Kd#U(`D=It^gEjAs`jFd}!>?vo3li1yf zfXpuqPiSni)t>mo*xg{cr?CIdWU8`fvN(#6NLtcA9y%YgT0vGt9)|R`xBmS{72l5f z0|+~thJ~*tva5%@SB-V6X-00%lR&oevlVAT86D63Rek8DE6+F+L3A+znPE33iQ$2Q zm(7>GDIktq*mxUfM~6PkLNYxNDr=;Gp3p^j2wnAyxmS(_J^3n-Pe9{EzgJWVRV#wN z119UiVsU)FQJC9^ZW^^>Ns%Ns{Djvh;tX3Htl&a_z62 zyW9@zM$#)!Yz}&+Djv};cQhe0gapzRke(CoU$|Znnc&PF%v^uEm>b*w&&bil29AZ5 zgqh_34!Bs@lNP?Qp|WvulQ7AdJ6Qg-B4Op?W@bq`A0`7%yI?&r)bF``9n{^_@4Aay ze|mKAWuMb$7mrG!k(K`nr8FwAgT;a6lm10Pzx!284~McWZ zVM0_rNo6UKK`0Kkm|fnCTc7i@|Hgpsho}1spBxW|ySIOTjv2x4=R6D9|9$z7F6h76 zt7f-5qac?6KnmNa#?M!E_wQL{4xFhs!+$G0{>Qh@n+~WwkjvSDM_EJVP@$L9p?#D~U!AD`n`oUj6l@KC30#6G@~(UBezMhg`UBs_^(BfJnpwZ6%=GGd zpW(b*EAEVgAsSqB`X4srrPq92x3%+2a7Sd>1f13@X~OUv@#ajc0TR(|)$kLg3vu z7nVOZPwRBN6ejkA&mN|$z|;CZJpPB*kyb$V=9f!MLOP8aS+oD91`7w9x(aQLrGRbw z)~_Kd{{b;Q(7YZhI{YW&y63s3Rj)ekqr(3#1aAc;2I2+JJi+1WVqEEr$xQu484IOI zVl>QpFl_Rd0~HsB9QxiZdokn2Jp1c{Wg(ivUy+t10u8sz6t`W~)36@V8lX%Yj=3tx zy3D`hUauLWT9VSnp?f9g?Rb(tb-6mM{>Bs97zdCw%N_7)s3lYa`29f^ap>)saW*mdG?eNHAz`>QRC^a$6^7jx0>XA;krT}tbHZaMQ3xN6oHmA* z6UiE_!^K0@&4YEu()<00V8PY}$tYG+x;a-vPMo<${rvXqA*>x!O z@K}@z{jypyZBg4cT0w2lDP-?oQz!*P=x_{R&muJYTtJoE*C3d>lx^Na;aAF^WHnO0 z#2xwYS9n^Ho(xr@T}T?#3cYuA(9RIAU#~Qspmq==hPNf#(45KjVwVS;0Q4&PE*P#p zUQp(CYD6pZI>Nt54`X<}U{5-|XixkB!vVL5)`7doo_@l*FINhWm^=DT5G@_}4xl+7 zoOftWvJLo5l#AfZx+zCohtN#a);@-aHPp?(7i0LHi!BWh%oZmI26Z!x@X{8P?Zmeu z=0v?S5aHNI?u5Nl=)|=HFt)+mc(A<+tVi97+h0J#I78baAc#7)D`*0R)05yz-IL52!ys6Qta}Q@C-4y(fg-Lq^O725im(|ZMBa_4NNo`2f%A@; zBjSslBjJmj^MfF4_mY1H*7M7ipC{@S1ov-GSV1#SGy~DsZw8XDY1`CJ6wcoSLklBH zw=hA|IQ~erlm;PcOj~Xskw^HQO`Lba4PkynN3wT_I8uLDN78rn4TY}%4Ey*Z4B1|x z-=iKGx>7xfx`THkHrZaWvxHy2Y$)=>2(c0bA)|(GQRb%EASrq;k`1tMoTpBYkWS;d-vwN$EQwsERzo8Eun8n-mpEXzO9`8oVkXO{I&x_gWKsq5u_gZR{arU-C; zeu1$S>BKJ`Xf{z1;qu%E#=rNow5{#nk?&O=ATxc>v*i9R(rr9BTHq@|kz z{YQ>(xjVO}G4t`CriZxSaCZMcAU|~E$f^J&IX`E{%x`|48+!Ma>oc~G`wemE2r$li z=iR+s!Y%EY;(i0%hK_1k6o6RgXWN*0L({Bx+TB~tPi~C+P5qN%RscespXolS&oaP$ zQlEJ~%YZp_B=zabJj?&HNXnn!)3I@ue~9xl>d+Arn*xyT{5<{{^jRR*-P?hfd1N*P zp$YCc%Auq1PpL_k|7V#@)V$IN9zk_NSK-?l-8W{ z6*mNFh~9R#dFTa1&Unp|1rI3o$+Aq9=2?8&DN~LWUn@ty(I(4as8_JrH|tc#oS7>F zvGa~fw(JEl<_#n{;9d(BV_kS8*6{IetLsW- z4cM@0U(RV)*)%WN)E^??RSp&{=7344asI33v|_euZb3~wcG6O%=ZxO4SUUO&++$?U z7B#L^b=(l{g|i8l!tee)CLo_m%CK=QTCSGlYmskVBczJn*p{9>ns!x6OM>XgiyKC9Ia=H}=A+zWRssALgIx0dS6Lm1eOc_HBqX#_0rF77wz43IMlQKNm^-A4$u zZ*iG$^0MDn>a`G>?)%$eQ@3)=Pn5}E3BugR4sp43v9#nPP+#HUA?hIq`1tmma==@O zH0ZtCX*)MO1W)8ORb}7#h?xux{B3LmEFM`^Zf+V?{(*V4nD0cL+Do9nGHPx|bQ=BC zb|5tfR1PJY4oyQHxy8WR_v!+-v(BY6FE9EHXg1U#8ocJE&yCj6)NJ6tGT!mr`3K^QY(;mZjH5J=edU?W zJ(LHb0n)GUJ$~daIjR;8TFniWji!V9<7=tr7CQt9)w!Qt7w*`dlGhPJ-_2 z-?iEps6v9#srF`nIx$*Iyva!FPQ+{93Du(CwAS0a=Ps$$9TeQy(2iz|tz>}qYD31l z^Z@-CVm>|m+u_lw=nNKrSNpm_VtlhJ8v5?g!8;se$hyFusrI5pVsiWU4z0HgN&z?f zX3tK8>JlogMx`Vm75)z%gJd&KG*1xW9}r7Q-8XPq<>9{YqG_o~dEaVB8lBb;ACz4& zH&<*>2BS?)r5nsw2CsG&6*eJmCg?tA7Sj;*K)n03G%fU`Wz*LmM^;8jJzO4n;F31j zgJy}J@Imkg26F_v;FcN7H1fyw)wJD;q$yXKuJ+=Rr?6ZgrQ<1w7px*Ae}B8a6>r~D z_EcK@C5nN;Gad)K!ttbh7-lm>b(wh~etL(qiG6qk+BYvZE;x>crz5nFH)GyCU8n>p z4xaI-y$TVVs+k%Z+YtBghrBvc-gA}?_#%Cg00C67-j~weoU?&(*#0=?pSt7m)yZg{qBkEdN|tA$(Q7~yF^H?hwg0f;48SxXCA;AXrI zuLCQn+63N7AZ6$;hrK`v+yvL`J&XDX#_fYIV%vLPACrF$n+JsyLV7neUa^Xc#!TI- z96weh$i<~BK$@eiV0pNnUOzFllWJ-(h>%JMnj7$&3326XEiU+vS_ma?ypyJ;mtPLl z!DORQ^2R*<4@Op`h;pcTHZhQmvgN3$=tXXH<-lI#_wFCNfi)AC$J1e}2HlesHBiJG zpwh*xy}`_}>@-_t0q85PjjVIZy=c>F8@RXFQAj32ac{GbhOlw4L=<>jRas;N604U9 zY5}cH_Fo-1PE99t<|jJaQ_NyqZ%#G7_oQb1VUuS+Is6myYh(`)(Mwbg&_=pi8kVz1 zPh2j5g?#tmjG|VU*P!QKvNMybj~bbtH}y|>*M9gZ!0)d0%@uLp;_lYUUqd6_Y++uN z_;%@`>24yHx&sXLR$?^$r90j6{HV;9uMv2?0`>pnl0#A_sNO5Vh-A47g?LW)OGj^03to&fa}AfYLsyw5DVIX_f^Tm~ z>0s=9#tV1VDPLS*p}Id|AvRQj9=5RqyS|3tfa$Im)AZ?a_{WEkfHDaK9m% zPh+EDP3(vC_cnlqt?S~`_K5@l3Zd?j7-|LV-6UpX$$eFeNHU?>B+S|9Yye^Xn8*&_a9nl zuH1l>D=BZ5mPb-g_26b$W~koZ82kldRpUd$5KBKN#(yO8qgU*tn#Rnnmb5KUdJ$M2 za-u^z>>kFy94>J5m-kMo+pPf=gc8o--hcDxr;(_qCS^o)yJrI|Ec^-3k4F)ZV8$Yw z5weOOxxfubUoSr9R}DweFN) z^xs%EZA0H@zTWw_{B`}8)-RJLt;*n8XeQwnp#i!-MUP{MzipAtcjW<28_5!@=o>p5 z*q9c7mPCOBagJg@+)}t$bA9Mz!!qtANu_Y|pd)pqiuI;up&@yE<1w?B$foj_i6F{M zW2Ci=Q*McK*aZAc|Bij8NNs$^oU@_G$pO<69f z=O+R~_2yV1So>F+(ms1CgYlF=;XYz!(2D2g|>e@@s<}d*HJuYW=b2q-wU{dGK zavwXVX3-RS8-SG*?S)x;@*oY%fd~aUg~;{V&Sc&x>45~`3xHK;!E}2x8R|IzejM%% z8d#gYz^7SRa%9F9y_0>n*c}BYLyA^_hV;62Hx765f<{ebj$t zJOv}!If~N@_k8^%zfPSNruD>-eJ}&Ke!$cwXcLJOf$OTrVmlb8#N?50LUG{IUvA!M zP5RjY!TJesl)S#^Ya$%zlmjGht2t8F%rC)A7>wMD;+r zF3eY~Y5`SC`vwJ~9;}n9L0;SXDV^svK7UFQc@cwdt zs6Qf$eP8MJ`FH~rT(~CH<_pmNf_kwz!5AQf+rdNIS@_r2?wGHeBJZMFjKLKnl=t)0l319^dCQOtp(sX4pkczbU04R_c-KTL_(1=U}}2YH)eIm12|L)ZVR z+*w<~_lHpw>t2Tvp4YUyk+0^S#-cbDx`DQLe zm@XSHGe)QP{NtA#9Jq2T52aRo0i#y(X5EQ~@>S50lk(l3ahM0&`bb@rahOS3(T7Fi zdwkMuPe#!+=vyk(f|e8CKtMtD;zji$m&QqB<((%N45aY%3QiQpC>VjsEUmH#@^%yE z_55Vbe{XGn*-us@u+Y1u? zlDms=sznF<^pR_TZ|=Ms=wA1F{Oj}LQVS}5hjv*}y*M5hh-~jUAO9E8XYbd3hGTaQ zKZ|6rsF}W7$(c0xt8cdBh7>fLBS+-51KH)u7q*)ed-CjJU8(aNz+HX>AJlxRDXC*p?0IzI0a@X~Va%}nm_{`ThP6lGNq z_+1G@YaOJ#mncMZ5HYcR9;y8VU1NOAjn>F6JkaD^KbsH+wEb@5*1)ok7?p3#^nq0*lpO?pAU41coMx7Dqam&~R%~%o zFE6R_RMMMH?%y?kK(9(NyG-$r3%My=v@K^Ns%E~(68Zh*tD)2w*@=W4!Z+DHiY|iC zqi!vzx&Lno5D88>fY_t0X*@T{P^_u2=g~!+Ye%YrPG-+rcqYx(2>v-YT_JC=pky$A zV&gMEpOlgvJpeVL#lT{MjncrS=`8?4#-b#rvm)nfEd^wz#!-Zq81yc~R?f4P+K;Je-8b(1=S5}Vxv zjlDR81!?>C#})jMx%1TI=}^~#eXt~Nd!AiOUv-vQIZ+KiRc(K?+1_2UZ8^Q_d9TfJ z@-aCHEY4>x%vhe;J#q~qHhEI}X+ZqXzNlbM?V~X;xGOED>ROAit7F3`6ZV#+5xiD_ zcsAhdt~#vMA~gh=)$@}W?w(6>l z?l*nps^zNRQ?OI8lcO1pVp^zrs5%MFCgC$Ub_g669Ooz%LF)MxRTYGyB6TEcztro* zE}{HL4O1M$>oVF?GAssf&Gr*y9VGUj$0n24Xdl!n5?4~F=i`f9UFS55Y85xLx)}gJ z#SUqBM(d4QHHRM`KD<46lnGU$nwsMsop@reGkX;aKS}87J#g)hk=$A{xXRy-%XE~v z%GK!}?_Mo(9g7*SItLkc+?D@Gtdlw0uOb2VttnjAZ>wb*vz*rIf|38l*;9J+N>yrhVkS{6 zJ~w6P5Kk{W|8=R3S1|lx=2leF&P#Yu=r5;~It@YhO!7>2t2$e`H62l#|AF!*Gz;%U zxKleT6wsY}aOvf1knL*CkR6=z&0K}raG98ahCbSq+e4W(iI!hR|-u)2M;`&4^HkHm)y|SzWS?8xGZbTGeXcnDy71u27esjFSPST)QQ?I7=c?}vQL#c0Oqg4-|dVocN}CXBV?Q{ zS(k5Mwup-Pfe}EEf9kkTM9m4g3o2C@+ZU30X3intAp+_`6^78tIE7k)l(RQB5<=?- z1ji)|xAf|$+V|c^NJeZh8Du6p#8yBRXdA>L_l6}%h8t;+7EmAG7vm0qyxTD4FUGe9 zLGib>$wox{;){u@W$Z0|RQXzmGW5YLb;$3cc+8woHEM*a7h=s|4bK4X5#i4+3Hl>B zEiC+%!cu?i@~$Ax*7XzKg8QWy4r2hkF=L7Wq9ZCD?jT*{1gH%pkoJP&J1Ws;=>_|O z8W_n8;}G`Z%()SeR0ix?$-q`PbY6OgzRJ5Hye$4#xlMaGlyd=U>D>MgS3=SmNzIps zz?f5RxM8vmB6ky6N(RE36k<>D6-oVP8*M^j9i^ZQ1qn<>efZjz;$uv$G{(sm`r}Xk zG)G#iu)TZ5-bdop|rZqm4h_X8-2M=9JKM5B*Qr&X87hbahwt&Z72# zROFsSD`~yyzl^9a$*}E82!+c2So!P>1MhBI3sAP(c!xlsu9AU6@LAc&@V1QSm-nw1 z{n%|^g}*L?iyv?Izq{d&FnInGb59aVupEKk^Seg;)AqBgp-rVGs&f0deQ+JR%K;qc z;^>rnP2?}wV8vyi`}Fpz>v{Z5R8&1;337pIL2QSV=0^s}WRl^tv(tCWGV{Yf#yARu zIRd`m%iJ(PNM=G>FY*mpglNxH(XrQHBbc=fqE<>7?O~wt60K|W`I&M%0*-P+Cg?k2 zdz+2A8GnTW`Y47As=@qKhK$8FN9qLmk4qiJj?v~lUKfG(^!2`)PG>v55ppvjgZPAC z&$mzlYT~5nkyHXWmf=G7@HV-c0J`CXMW@|%$#WB+3(J-6@_4JGjiXHz+4*h#%Q@o4 zSokkT@OAd9z$@`%>EqJV1(d=FCR~WDaaf&Jn*ujk7a0M%ztWSMTLooBbw#Oos}{Wi zHWT^<)Tgog+6I((qr^IRnlYFDshsP&=p%D#>JeeT2h)+k1R!T)&RE zP06@`SVyqqwxK!$S*e$hSLW^Ik0pwpcVaQBWN zs+}30N0b?CYvOPlrY~Yf#i3d)Eax`wLTOoV4=m<7FKe&BfZ?!V>LXK`l0D_Hx^BB> zFH_c5l@o!-9pbwF@}ao{O)G3hsL29y9R?V9LtVh+-ZX8;mqGgIsKXT830@bPwd+FH z-epdwFmpO-IXj^Kjrj5NW2L4YTn?2{p|t-|jr2m+UiQrN z#e98{Ax`|eAF%?=)?2wJf z_zT+KR2^luj9|{wuYdco3iq2aYR${`tA+nqAvV7I2R_d*DU+W_>VkFt{&MK8gsEQ9 zojLD8=P2w{0#@TVcOc5D&mm9Twa4WOS} z7i#m*t~k4bJ63$02%0O=%X0mE9qO#E+!BXpEv&Y>CE?wRk8N1gUp2gC-t~fQ3K$oZ zEbO@MwBb~Smd~Ae^{3l^@Xt1-0s33heaA88+lq}14UVJx*QZ|AUYcOS31sla#79wH zg^?xD#IA##jhBtDkNReQTXOhkRzEKYHWEDeRAJs*_@0hbn3thPyoO?}5ri6XxlX>L z@Q=JiN|eu)IFbnE;RGoSDdZkZ6mjg3#iXz!}oBV3HpHrFO2N#McBE)4Xnf<$lO1Af`@!dN-$IYa5c!+>g<- zdbSyx;R-Hs)Zt?ZsGwD)Gzb<8E|EommJbHm(J+<1hrN#Q&d(PXOl8m|BdNPNf5$@# z{q3a9&83FR8VX?y^U}`EbS%1|G+U;^q-Gg&mw5ecXvoZT!luQGRDb^_QH^1U-zEk~ z#=!8FQO7aciChRF!bXje%`Tdcbg-2iHp;|g2|>Slg+N% z%6t2}(8tJQW+{J%&Cbx!Q0#7DU)SqrUs;kJPl@40v+_6)z4d)Np3Q?vf0eKCG%8RY zU8+<&s`T`2_jf`5xaFC_`paWgcnZVYKuqG?v(Q646P`f*MI`jIqKVP!C4~b}eT-En zDy9;4&Mstj;9tCEb-?aXIqAfe{{b&u=)e4Hg95Fd&wRCb>RZjt7D|mBr5d6JVq8>1 z?r`79fu%j;EovK{l(J{HN zWe~q!{42;-(H%`!(Ei~>NzaKcO<7!wmF4(*zZ2V`+wpOvRnsvO&55H*ongdU>*v}f z(>v?y4s$!d+5?aF{e+>30H1a)Bk_jBc^EihKBZC6yv2zjuJo*Hb@l+@J>FS>q!3{q z!&-J!{;8ASZ*{ZMVc*4|;DS*-%{@GVt3UNZWR_X%Hl{@9kkpry;ces`NHgUsqLN^>;{v7@%Z%y5tW5=V!I z;{}@Ir~>6XIxzit@mn{*;hb0{Y%$-Dv*?KYy&y&NudGC*PC0FQA=TPnpTAk&!RHN? zhXn5K5@bz|g$3t=DB7BJ*!gThvD4H_d1_#=rl!y!bB}O7fXz&EMe~tn1Di$yv5~KM zoJsexD}Jh4ZpS`zqPhzz)B0PEqS2p)<^b=Aocj&lm~E1CLIe#kM9xgeQs06jq_Dmv zzM%lyT#BmNg1J! z_BA=?Pc-RG(+{sl*vXYzXAK%O%Y

?AI0@F&3l-<9XDM5vK~St<|U;**hPp8Fm`P z-vo4K*GScyvoivL0}cw?PNJ*>h4MdPQsWpoc4;Y>A~S!^S7_TmRR4JxS~?JLo4hVP z8RPq*S-daBHr7aI^MeY*Rab~I7F*NN$HH?Od*J2b@lbIS30Unf4&D%=wRVv~#C2a|Z!TJA#Xg z*)w^vs$4MjQOW1qD7$JPLwGF=Kw&6FW(W3Y>3Bd(E?NXVAwRD!@bW5$+VzA+P0+Y^ z4q$!99Zv>kM)O@=Gvwg3=$LR1R>w~9Xny12QS*CzAangpE_TjG0Ta4mJ}j-dzFG?s z_%X)$W5)@`uJ=g-)q8(a__A`~qq5;TBw1THzMC6ENuB`kZMWF@e@x#~a57SO2BmJsL8jy-^=?l*SYIQ;kVn?SUHL=!y?Ww;co&mYez z72q&ZQsNFgCZDb#(|^3`G8xt}5V#PwdklBZ%JB6N|2Sx#u2Q6|*d`V=BWQm~uywCF z=f6BX(ZynV|JS`eZ(gG8rFLIRmOIhAz(jF!bWHr8{Em2Tjh=kZugbs4nw2N)cdKE* z&+Ui|Gs{$Sx*>* zM{RdJ4F43zrq*Q^s^yDcOKcU0A0Gf-{alOgxW71`lS>q0B%KXH4C=9fMwf>%PLedH zj%f@2-XE>(o7wU#uRdXhCY974u3+MxR^ZtYg1i(O^sW6xbhJz{K)1rozfiwHK`LXi zh78~p)qf>*B$8n>q0gJll43vhW#Nr~yfqlJEg3_V|C|wi+SR&?SW*NHljQ=*u`h0h zd&bXmA+HTzpJ7oa(?AiHPy|G86rNt>MZe|xao>RGI<5Kj0>mnV~aEe-;IYgMioph;L!IWd-NLXPLD`t>s`WBj;Rj6 zld?C>;To?t&iPEb$_nwb#d}~VjMY)YE#qMB1RuM0QwAUpw{!Y$UnH0>)YA8t`4T$$ z-y=-&S{#|UxMd1{gm1MPk>WGfcO-|mL1pv1HFw{!WE2u8v=5+U{*G?VqrdI$38IMn zXrb}OL|th3K1M52icUn0?kqHBmQbKx8Oy&ibg|7NsL-vRRTO1!NNx>?mGB$y#TJmE zj+#G?5}j3<+3G~%V-yl+hB?Vpj=?A5)mxWU&gnT=;G?Me?&4lt@xinu@JY)K3QQHZnsp|ER!f zqNPnKK1?If-&85k3i~hoqw}@V_@ED=-jsqVxWxNGq2r)fE zlsh0Z@jzi7IGjAZKr9B2*eGmNY`rES2fUD={P$%#tA2}W_rVSLv3A5rYPib?i$=MO zwHMU(*1hSt(*h$MupzGef;sPwZs}4?OyTiM)y`e9WyjderkHJ%b4rFnIHu5`w*@{& z{`Zx`uN!jOESo2V$15W(z4{3HsjK_{QsxwoBL?9RTq#i6C{;+i00Uig3<)JwdG)n88 z@5eoAdoo$5``x&jHQ4(bu-X*;i;R}d-mwSRVYUm7p$egZcmnmwV`WI?DL#Q{aTaC8 zV;@8g@3Ka&uD^7 zr@aO}MQMBB%6pDK%L?gefYc3TWo$18<_p9SEBm$=$ni$;0t~bm0j_ z_s!!Cvobq#^?%M3l2MbJkzZD0pL1(tCVPKaPEk~prbjd27E3@f)HI$1!(ap8J7j@6 zdeiahOaq;+`(k<54^Mm+|5m$0%JIkNf=D*5jW$MrU$vQ{$no``D?T_kLcn{zF!dJw z!k^W+&8zG_tla)0p3d{~T>qnB57O4T*xRXC4<(H3nnbrU*b1v6+E;=noi3_I_F~cf zq(imy-%?n|fPh3iw?~m6#_`gbydV9m=K))WWs~Fa8Jbl@8CYc@-TeFIy8_fxyss2x;)wlXd zXZ+yzUaF0o*$K{@;Ol(dRTs|jCl$3E13Bx}-Y%|^Od2j{Kj9%b` z_1{@uza0Bq8T5OD8DlFhP<=M|zcYg=L(Rlx4{4;hIQ%=-wi@CSO`mo6zr>v6o}OfMxvj)vGaj$|9*(usS>AQe zcGox`F)gxVG11_>7x_!AaXc7sxQadj<|8va|MeS~9g;PK$|xkOG%Vl1Pyd;f%IYkP zms0#a!lp-P65eEBPpT)h4so2MBxH@Z}V}~AHXJU_=^KyAg!tY;wVbjDblQSLW@{0q@mdH!O zS${U!Lyu2L7pLRuMk^AKaQ(-@?YvDJ{yj}6VYbIcfag}esM)P9(Ym#S>GHopBOD%} z46W6DUx`wHMLyUt;OOKVF07PY0-(`qGBF(=VIp1~S5)Fn4#G)=erQJDiT!Wrd6k@} z?Ecg~-f06P<-UZaaga{6T)5uo)L5aS?VvHLhDV9J<0m^&liW0{Qe#O(x^H2$5==?0 zqilL$Fs)KA9)c>}uTv_XO4jH>r4ge#J$}DOL{*KIVY*cDVGYab_w5@ic3{V&$U-g} zj?=WHx{TR2KMMbuJ*~%v+qRoJjYCg>Zho5yr_;gHW5ihH;3j*i$UHP`Q*Y6XZEl`F zpg|OP`YlzH*D#lLWM|n*_E9=~# zI0^%igbIQFKSwd|jMUMHRCDk27|+&&>FjClV7OrnDzY+FH}wW!DTX>-z~))^M7zFM z9iPRGjNFCgCzbg2VbPd6sQR#>ae!7`D}~UwApVD;%C=KTQa>sZOE&jJF?W_6J%P(y zhJYb`nw-)X;e-)zkcjIwgJA*#4FV;(#WN7ur_FeJ&+ z&}{EmWT@1SaKSV{r}dH*BXAmD10j`N0S<9mi@-@cD}Uzuhq|v1tKjQURto)d2_>PX zkSOp1-Co3HjlaKZ@!=*RBl2zorhTBDUag9I<>Q}Ao*HIFviP&>KYZgJb~whqZy7o+ zwx4d=Q1H~qnrh*`=K;3Og-Sdgy=V8_`1xm7Lm*4SV8Rm^zj7baO&@L$oJJ9loynvu z#2E?vx#RjHFmD#z0u{7%2R`s@tO$xNYlEWHRlv%C&Y~a1lIAFuR89UHr>1@k@|6`e zt&}Cn{r*(%>!ABC{66a|D3r>KN@CXxi~c3ift?u@Cjp-7Ygo0{lk;`-`+t}rBcVd5 zAvB&LKP6@WgpwXRgd4BYb$T2-nW(|M^l~z*I~D7Qy2inzgtJR-G3{+TskR6+PZU;t z9tny)B&@y>)F7K7khLalNz-ir6St(a;!S*Dz$Q#c#f23_8mTPa2Ya0n${5u;*3?1{We00aNJ?yjrD-U?a z9=x9)bmCR1Sx`|yKgGTheGh?ul#gNKF2Q=K#NeG(hto&S)Ez?})nVWg04-8Lb`*H- zG!+2NT5U%X?mAHO7iY!eAqlk@`1%!O8m(n|Ci%aIvHsN#Q9VigqA~+ZM!QAfIaa84 z8AMso&kds2(pfj5;pv@ZgGODltHS0nWRqxJp}q^BZ5>G$B~KA_3_yw93Jjej3>5u` zemX*o;=eMPg6ruk;iyD<#} zaqx<09>q|{(X6aqOM(f&aMXrddKY_HJ-6xK2%Vc$%J-a>32#G14Pd8qvuZUbf;a(^ z_`T-PJn7nbOzd6IM+6;-0+V;vI}bw0Av)qd%5`Xmn679L8ZY5ahZu!fL9%GPD+fs+ z1IrAUu61zKgmO|}Nl;b=Vc)-8<`_uvOcGc4Q3SivZecgMFGE<}SV<>(*Yr*0`#(!h zLCuUrY@HURyM>uijd>%>1*0XD#ESN;#Yl0uL^y!Bmb~m7nd*tVd?}U*cNeyzs&HI( zoV%%sgOwV3?(HZ`tAODlc*XWEV9@6FNPt3sTT=ZzkhOhTQ8ua>D}GcA;RyJKX;KW6 zmLa@pL$XDkm?UOs|BG=BHoFMj!(86gFga9-obju^)!G?hD7i!=|E#4+pTF3Xl~9~O z&SR6JR4{s!hVJ>T?Kt6~fZD2R^FE&5*sM1DU!$@C|wsK4B%cq{m2Xd{FT4uo>~uzu$?e4XFQfK!hEl z-)bilQB+IQ(eSiV)HI*Gp~hzj=On%#Iw<+&_j5xj4Dou5{|I$=CO3b1>p8$7A|Ez%Y49n zqxH(iCM-D-w4Z-}RE&Z?jD>ao-72QT}U8VSQQ~EIPxF`A+W+ zZPoNhVV-y!1?_Nl`A)1!U@k{Vr)o0pKbiy@pUqOhYoa1;3-FHtdg5XH8#}Jk8+kIW zXANx?dp8*Qe%5w`J9Hoz5GqpCbbcmCCKH-lqGST4RIitwsQz29y&vvV2KA&)4!NYF zBx0e+%)LQet4gczJmk7p@0t0l@C*4iUWzv(ikvxXf!8^}cA9Ml%RIGo#3|A>(lgST z+%s+i`H~z!1elSWX_Ffx=gocXA(D&FG)cEyJKQ+Dfy;rDGUN9oas>G}{Uag03odp4 zkaQc^W*zF>b)ruHM1sc1r_REMPHl+A(dW? z16dB+dAJZSwq=G}O3fgMf?^Q}w+h0D;fI`8f{DY^4q$q0SE->)wvi?+cPvn*DHR1N zkd*OFu0XP$w}x|QA+vp~nWhY^*3A~qt|sSBKg|zj;aDAm3rkFiw=GLQ&%V(;I|5vh zK5#>B@|&8P7DaXG2=hY0Lr4u6IHh#N>pp8wsFEpj<^yz%3qH7Kt*P)C_q4T6-Sfl> zxe#`j0B%p_RL{>bnyt^x4n1L%SE#La3#DXvfDt1m5|{q?3iz;53x$=- z0?lG(9>j5jc@5kzE&oD#T%&;`4X6`q`B^#((TL2Q>9- zsm9Lg{w+1g6HtO~hvA_3d$LNjpt^8m8P82>86ffjUdVdD%FN;p1Ak;g9|r*~C(hA> z|KT6l%HHU~w4(Zk{h9iPZn^%c{@T*V2KX+V2R>WFvDIicJFlR%T-M5DU(mafykCC% zo31w6RHu1=l(4CTU;F6KmUpjQJYrY4i|GP?ngvqdt`pE@&6U$SR>AU&Az zk?}}8e4NymU!%rCQC_J3*L5#0-a$6XvCk#OLwNvEaQb)fkqy5HSP%{6AvtG-4nVAl zM6$lx)P-N6!QaKTxnSB4OBnj4@mxWNYlwu0ovw0PlHh{d1&dFEMdFxW+{c`9L{o{6$ z+2kWh+;?L2In@0I|ArBgTc2RW3cx*%oyzmQpQ-YXoES#_$T$u}&-X^2gkQS(K(dm# zpe(>2oVdAzwgTvmc+UZd?Cs!X5Xjg7OL5YSd05~AE}is^Exhoi_-TF_>Ai8;xxqSTQyMTO-oxl{iTpd!NFrU3v0K@b-;~;b9;s(79j6N^g4K` z{r5N3IoTNf5D8upTe$BlfkX>b5AqvW4xdzi{v0lLiBWFCzVv|OJVgvz3nVTr1O)dK z6NZl^zd9aKXsps_{^QxQ8q57EwZkT!N~DA4F^9IU?QF_Fh+3yt-ZCSv@vDf^O>yty z@^lrN_tU-uBw5!-;NV_#W<)&TK44Wu$J+u}Y((}3cv={r(8A$1Wi z$!7DmiDDao@GNtJCPXGPBjADv^_dZGh~!!#cuv?Fm?HefcTwtZ(* z>BT&ZjNd#MG7vrf`~X`5h;?Mv>Ee8{YBLdYp?lMD1#;Cit}}OsK#RiJ)NmK#gPhBL z(4ciFY8wmkJWJ*96HGjdOK}bbHL)F(C6*ghe+M0N7-pMQd>kd~eISV!7Tdo%rH9EE za}y6Z6I7`Qz()yu?o3B=DbYnNV@YJ`N)1Uy3aJ}1HBPBJI)EDS8HxjQG%=VXXvz5u zTZB};^%c7Y%c-)=Kiw@`G?u}wmlxJ6Io#+oX((!puihvlqfnOldFTpwMm%(E#ul?3p8b4Mrz;od>(5f$;$^ z4*yof>`X7ozImIy`+aEjD>HZ3d+le-eTm}&Iv;`GO1n1>ZO^Mqs*g2t)=7wZrpD5d z_ju|J?K6C|xz;kthCDUzz!#c}dDDlB)0u_fkSv7)`7;u286)e9825(55+mQ)3*+%m zy3dF?;{hCuly6lwMvNJn5uTTTG!xbuu;tF)3f} z#nIx_n|l&4lMV=r$+3DR0ehj$>WPc#ya_3`lmd#=9bu;|zI?lTcsFQr{(zQ9b^}uj zecNyQ mEB@n%Nj~>?kUY#K>rwSFTsU8(`^Aa*Hh3m@9W}RMIS?R2GwsaPC%qX-- zm_eUb=h44S40UrWJux#eS(-jDlP%kP8Ch!bG_@r4KT^9W8?uke^b_v~A9(!Ww$P07 z4+qTl!;=DJP3qXUP`gW$LU|T!>}01i%5e_EUiXJGKZKS}hu5gZYT?``OLj_hPg=Lz z+KS9CU0pM=eV05!58zICqaFABtZI#G?}PWzu^`Owv;P?V_-w9Kda6fN5|qv}tGy(0 zRJ4vDxb?Bp4k6U1+5j|_&05NfPVQkELVyR|4DgvQ1l?g2T~t`qYEkVb+D7sRlQ8(9 zvE@Hg@pdfKbrl7ruCyB<-XQW2FTE1jXhw-Z%c@!0Wbu*@E0T#QlNm|UEt^FpC<#d9 z^+vjciH}TTlOEV@BAulni-WI<5!0-a%DY6eHFoGHZ}iuy2#(5f{!%paVF?#RP5?!_ z8pm<`SBjik%djJc;8-d?<)o@GyBNF3e~RS1Glg93$pxfGuRhN#Owlp2u^l+X^w+G# zKZmW$t|gYY1ku6~SPt652_j7yH{9}}%mxW#HSuMEZh5QX*DzZ5l;1~5?Nj?-8-9sV*vMsB(L{@D*w1WxtMq8=Px|EyTR0m^yY?(G9n}M zKaC@8-%3@_D)AMHirwKS86F^=;a`KGrSnI$W`x_xrLm>w+|fK@Bct@$tnUPEARrC+swcX9cCQMGG!Su2L*pvVp{*v$G=)5`YROSU1J@ zZd6b$eCmVp#%GCL55EED@(X~<6YZ@dj%kkS7ViQxZ|rrB`AmoS{c@*1Ry2t;t+wx( zA$7Q^Hd+1ADj7X#N-{?uVT`R?A9hNlYmh0s34o*H9n@~EW6%*t+<(f50~eAX;qJ!o zmZSdg`b1ws!+XqedlRJ22cWMyGs z6f}7Vh+6*!6=cc5hbho**ihjqviY?iz8FKW-A%lR?04y}$D)M?P`ev-=#n;GE@yyv zaQ-9MS?nQ=u$9s#Dtgg%5%Bgw%{sERy4BgkXIXO!xF z;t9C~BAAt?1M|_LWdtFC`pYdA%AR!(7@01^VK7J(+3hb`;WBHMFz`-0_*W_Y^o!5H zFdr7vzm`Lh{l68lfY3(okNNqWE#+!`x22@o)Hh3n%n(L2>+(3UEl$6W>Q-2U1)Pt$ zN(b=?eeK*|Y&Cp=_ixR*(lZ}(4=eR<%N7Phmjhdox!CJzks{K4Yd2NtTply}U&odB zxOJnhzVfsigV%eX!cBK`+{@~z-%T3lYv1E-J(bIR_x!ghfFFh>puP+<%B)tO0fT)C zkN-xBO}BpBxvI}5v+HrTTmHJtg;L6>+Ec#>C;ZAxj_+aA+`TH>`|zQr=_b4ES|(}# zbA)Y)2M^BRcYWXJXOCd&mC;XvH>c2cRw6tSL9|l_S_(&}n%QvlV2!C?CzOLiQjgk0 zavFk?33WU|0HTCBE#Bl2#c3;3vVz7a3q3;9L-d;OH=OX`-i}4EG<#*j03iMW5c#H@MT)fA1@l|IideYC zb3c#ZvU{{4yXpoy7x_Ro z0&Pve<*<>C(hhUUq~&PU#001bn5wcd+*cV6)=ff_{&@$Q3@CDR zjSe+H*phN^pH?eh(2@#zK{ePR`9#95^8T?Wa4zCsib?3wV0Cagf&zwk@i`B|1pFEK z6Z8AcyAPin&cu;H!Cj#*qnrtOc0NE<`wFu0A!|hCSiH>40RG)aeCAY&yaNnZ>xW?A zSlTa=KnQ(=doq4}qTPH`&!3xqCVc58Lyv9%2)Y&n6c7=Io=lPEnqYjOOnhL=yi!En zW-+&V8h2_MjT{Y3)_;9~h-GmNFv7Z87T zip;cdGMNOz+w6OimbH>bal&68GtRWfK7~_RwOtaP%A_U4H_NAer;L6OxRCxArO~Sr zikwXjaUfB+V&nuHD+YO*83}s1R+@=`hxA5L1A@2%&Hdk<{p@VXPBKZ?lAI@EVp{%v zg1mXJSg;2{Vo_l;7?@7VL5miUHzZTu>M#k^Q^y{YVr?VC2xN7N*MzhSZQ_JR0Z!P1 zokh$#Jk1ftF1*<1UvKlh34CjdMmRe@IPxRr2lsd8`!|EIRz5t_7AENAfgZ&GrNWVG zyw5cHvk{E|yrzH|eNZ$4Sgm z5m3_5($dgS`^>2C>EOB5F4Q|2`uW2c?f5Mp7t}pHy?u1b>gtU=y0ydZ|NNyX_ZWVs znU~Khut7B2jQu^blynsgDNVUy%4*bIeB|gtw<9sg*No)2ngRO8i35av63E6v3&(4fYA?-7;?rF$k}L^C*w=o` zt$9_XCNEEtvs3%SPyn30S*2EcLI%t$K${uVub1(xciVkn+NE-|u0SkaUAm?{u-(x7 zse7PT3jp9?(xv8GgzH3F5 z*EX-dUpBA0(STiU$Wa9=j9l~Y#wGy)im){Yu>4K*ZAn4MJSQ|D|^5F4OeI>{BfIDPCj1gDQJgGi`f z*Lq3y4==7%ZR$}(TKFjN#o8XAJ+?;NwMp#bN$4iYazJGOO=#nK7P<|LBJ;op8wZs2 zyZh7!O8;zF5HLv>VeufATj`8-i%T3*VvKMc+j}&yujG=L7`bbkx}?tf)VyPtKr({$ z<6?i{0(4;A+`4F+jaSshy*F(OUK`O>sg)hCx!M;LKm9j*raW-#kD)PrkJST#qeL)z zQ(F&TB!hTB1{5{Sr1uFIR=NWwNJg-lbpY2{ap>hdh2`jTD@;)+l%MAfMS$qRD+V&{sOEmU4=@7MySC_A3jJPyaj_ORD zDM4#Zc(e>5_6;5Dt<8ZoQLXNH=YXDyP6t0ez}ttPP7AVc~|4L=H0 ztD16v8q{0CeGe?IcNaW}wM)}h8c@L9Hg>%^X@)-dlpR)9g@?~imKb%{_rh!LT#Pjz zit{=sUF($htS-gcUT~$2i{}`os1EZ1C(lMpMiFQ2j`NnIKS%LL+x`)`a3++99rcq( zpGNi0-#PHnQ8Eua?H{vlOP~@{)I*1kA9rhj0JR;HF}hfV7qr-k79kx?m?Nx-Yh7J` z51v_WfA?=sIw?rrIDp!0>KFWI57k6<3c!`>TS^6!>Y~PqYz~jUp_T}Ra8t!^)ikD{qf-jGLdCJ`dIlPCmux<_~zzOyEAV&qZ86~2LAvcbjdBh3^J`T^>8?m`!BglL6| z~4N`O239DNL`CJZ>Y6a<)XY(w|N~ z-Q4P#6FLF2^Kfj>eM+r7C?a)I$;}*ik>Lfpw_f`nolv=iZjqV`mJ5|+(;_G}0wq`x?qW=UgLKP% zVs9SiT+T#8MX_-POs;HTBO+$qh2)w`{2CWd6nyJLB)({E55PIXMmh8J;t(<5aV#El$+#tH zh3=WFsiGG)hW8`x=LERYl>Zw5tsd!I>tk!;y;C?L?k3`+vW3*L9~z7IHtJ+CjtsKB z*@|!fzDdS^Bm)5vB7Hg0n@$AH_^R582}V%SfvPAVc|ux^m2*ycg({!E~ zLJ=JO2Tb*T;y59jkVXHWBe!2JDsPF{lO{HZ=M zaGC0drd&3**&G2tWA0i^Z6+{jo6Rbe|QQraq4QrAk>7+Y_RmvV9);+HdB z(1bK+e;Y$dPo)qA3WSF8UMY%~S8R(Z(-FrgLpNa~7^PMT+Bsx!7kuLiJKvYpBcd-H ze|**>3DQFRHKn8uFL-+8zpmKsrc{2R@~Ai@LG?JJSenRQz<5n3KvJ$2e z=sF?MUEWV#yNj$oyQ9f}8wt`Anv%6_cmO_gOn=vadp?Hn5_iHM-6Gye^zL;L;Y)Zo zUO0Jak?mpHkm};DOl(TWDt+892$wa-k@Iu!j&z0K-{{w`vY+XqEbSaRH_apQ>yPfWhnFb$J?Mo{Oju!|=p zqowC2G6;9O)}5Qy>=1FlDE4U5II%4ep|zAvL2~Ub*e`%}Q*};_m`Lf5co*MT6}{XR zx$O4yG)8}{Xa4up5AxlBZhnbYJ0p+Ek=u1dSrp5Pp?ZI6y;Bbj# zx<>-c`eJ>1G=)l#7yB3wQkPPMNm3$iDxs)fsK&E>g*V1a2hCz+mxe1e(F^7O zAx_a4I(TPd2nnwp%18hNoo0&6CISJnX=6mVnArs&ljO}KOV$3032^@!_t!9_ zDj2rd7w6Es81&|H44S1|65~UYE?(?2;b08{lP3OYdiS(Fdf5NCU z8ypt;ahwxz;)RQ-_~>M-V-`u}0dy4TDJW+2=x)?m%$V7gT2=p;XeGK&p^g&n(cM52fWoUE2Eb}g6AM(SlE{K z>u~Ktx%Hjam!Ig(*rjCz?NY$Q+^hqF6x_87L&hxdZcM;5*``48MRW_$M+QKDi*Iry zKt%pNbdjEUxBu47&y!M-NpF>tUsYsa$TroZA3nig-1UaVXJZHvA?HUKejfn9hon37 z8}=22^elb^6^PVn-DO^)AeQ1B#!0DC(`vSr;iHF(&!Map4Gon8TgJc5w{#{lA7{~z zaNvMX%}*B7$_}wwZYBF@y{T*x+zM%p%BY)Hj#P@(wfm`Fk%Vp8uXUU1N70k{B6!n& z6>es3X4bJE*q_kE_;0FQe|-SvHYt4~IEV1u6PwCM0%K3)QEiCOF#TDzqXh|{qhsuc zWdxTGoWOVq^~vJA$3K#88{bSW$6ANtn~I$W+Nzg$m*7qLlHSuk`3G|87iDMP*MK?Qvx@~3=IJ>W<_bnO#{+@!Zpyz$Ggzn+d>Y*FOJ(5-xK&$2ATQbkCCbw0hSm+af39b}+Sk-HS4LUw? zg%JEPgfAK}5GQycJ&OQr>Ye@e5CxjVZ_(6Y%RDOTifh78Mk7Zh(e3s!wStpYuhE+u zU|H3|&{ddf#u`6|DGWT55lJ>7#*Y_tWjB)%G`M#_xU&>SxEO%S77os^Jl?szk-8c0 zk#ts0%v;1Kn!sb(vGmE{oXTnT8D_i5d7gnh4wQHO@Vs={hVBJWN}K79+0>g+4oBSa zPx0acxXjat?59@_8_%sX7P`rngUCR&1>9BtDPMHY(g>);*d3RsLUTi0Qmp}da9~@^ z2i9V);C36m^+AW;Pn*d(o||$$)QdoF_5-&#usBx)>fFbWMa0S4)F;4R8xw9%f_EQZNLEN?Y<2EkPe#&G%(8a-8`d=cgml#(A?VRWS$-38ve@4cC6yi7-#Pc zJJL|nKfHMKP*3>|P>CE{YPR;~S<;ITM8P-X%kX)5g56je&5`%LNFLZQE3m7j#e(&H zih1OIqWd=H`+f-R7_O*4{98Xm3pg0_4!PetlQBH#?%@Nxsc76DMCW?!ryRd+-SbZ2 zhJN)^ucQ7WO}_Y!JUi%)0Fm)(GFO#w_B(`8{HBUb&H$>iBWL5VoRtifF!#<`mK;`r zrjvwMx(He@8V|=q7uTskVC%>5yxLc~8#l6c@#*Ty*$+{<_jLXK)$U#kJ)rCpX|?_3 z)+=sPxVQ`8v+hG<^xJapxHUsBX<6{|+uKe7VpCwS%scP&kEw+ywd-Y=&zO%#1?eea z`-v1m(J7u*?iu&DMlUU)laQX@SMl}*3nRQ`suu)@sGj2WfFJlBl#`JA@6XUD>P)~5 zEk1xN2>A;j(Z7Ho=4qJe$Z7U0u@jfu z8$5gAG3z&{a>}<1yg0hNR{f5mo+^+8Izx;!M4OnBZnk8DK~50F8Aygvwn^IC)R`%R zLW=@`b?6}4l@b8r&to|^*#hZVbcG@yKMSYQ9Z?>f=VRS^$)!kQvpiFI>cm(EhIqW;BFtS4}N1=qIUrLhTs^Gl~VF*(LVyS$DXs!+6vdRSrFZZBN^ zFjAzP5vuIPE9GSe*a0)HM#pYS`TW>^|GgV};%e8`TydF1LnYEEmb_D;Lazus^YkM& zZ{~XzlW|%1mlc zkk1B7i8egW61a9H3q*>n=BRe^5(TU}vyu}0^JaFPRqP|Yg(m@yDU7qn!Q(}-k37WhW zI=xN-_jWYdoK49QFW}tz?E2gUc{2HBy@t-?Psj5S)dmd-XejATE^?gaPR+eQUT{Zs zbaf1d-x8os?oI?-Rmom#+Rah`?IYN|sj;~S3E7La>x~?Y8_r4m2Ass0u@Rn;PQY+( zirj?0LU-2h4cG#baU(h+(L)J3Vw27xJi+sHhMRM&n?0>XyihOvv*E2GLU>fiy0|Jz zkWlC%*gYYNMH&>ZjOx8;4FO0A>7tcViGkCc#xoHmMIWFetF;VYR3_EzS(9%y+ao1K4XRnY@klHP%-L$Q5dt|d{LPhU z;vkX4;53IZFkw8Ci<~?GSu>#ngniIexBG(dPbf$*t6u@<1AIzBsukAV%-!rhH!mDW zcLAqmuicX_`BeLN+2$9)iy&-Weif~6rpJ|sq79R)Fmt5mt?Z)PLQb^Y$-FK{=ZmOY zWn9Tu)^$Ld?}UoOK!+mBbx4M?dNg38*F~SFL^!Bey%B|;jb%E36gt^>y&jQ{?YS%} zIl(9q`C@T;;V^c|KTq9>Zsj+p`$U{H8EBpyX#^>q5f*k4bVJ-haEk2CFnLt@mV}9x z$0ez|w7+^DL2$NAc3B~hfHTqJAb!4s{IHC(>j@aN?)?t>vR$Gq^u@QAF%eDD@+J#C z6H3(v+T<}!B5Mw?<0l+k71<;h5Iu~Lm7+36CY&Wx^S-AE({>Ci;PgPS5Qz4dmGl${ zla1bwOA;xQ$%b=kizM(}(|w^39|sJ)9SYh<+_^ndm&~16*NK(+?-&`pXDK$yv85Rc zCYyVM5L;_Q>LxCm)9bwxau58&!d|a^VV^1#7^)UHspbIyItmOa-l21g`s5 z)65MVDN+?+p7A;OD45uk4bOe8YrA}C2!2#5B76k4W_Dm@C({yW#Z+K=Dz5f$f2EAw zdCqVj<(kEP-EIA=WfNp1_sJ;p`ET_mYm8gOfgM|~=arlzyL;t}@^!&z%b_GzgC0CJ zlUpFF$_hO8ZL1+x<4VF32gyF9Xs0+21(Q)YB2X6qD13e^`q_pBGy}7XS~BpO0a(hh z+K3?w`G;3d7vY%VTt_pag&{Yn*0CaX-5(u@rdO{SJ&!j_FP(-sNYDH^8pBbdKT+4} zkkR?!x)~zPHpOObZ2MK<-rKP>;Z+STjmwyJ=h+D;Zn#aU3>~gXKv(GEW@UGCJx8K2 zIm`wS42EW3uDjZJPUL_M^R|~YZlf=7DGY03F{^=s zd#Di;)7|p(a}WHwxpxx9xgr2Vw-(vrkkSXR4{Y0c-h@s*{Cj?S;sn)MA1$!X?bE)- zyOTn@)c(@Ux_TI*8kh)w?na*|ch~FLev&heq6i!P9g+HKZ5F+^Fd8{tRzYdWE*H*^ zke&pkEl-lDdIT0hK<-Ajn{D0z0ebrTMwWKZ=t4J0Sy_2ni(k1qh_hv4dWZ{JW&jm% zT#@Wsqel!gf0@M$Utxwdf?(sLr3_tvNnCfsPYQvDqle_E|vmEuT|}){Zpz`)o)xgUc(@4tNx%{ zG%uZ_4T?BZwAt_9g)#JQ$!Q}gMWkNw!A0H;D_W7cdZ^j6@cMW%}X@+r~BZ@zZ z(+SKl3DvQcdc%^5nmYse$?8J4X-N{P5W+ITYqrT^&nBOt8Ht&R_~#B*lTIFRu10vV zN;MP##;v1MfmUe}=a)#TBwMCiI4*wiX;UeT$+tWs6v|vQ;K|OHOb14q-|D96aanHb zZ&AFFc`4-;C0f|hZN0FoVn17kG8`-OH=yr0o)2}^`}`~1q@A1W)rd0F{h(c`-JRle zoIS*Cvn9v3wE_5HH`uLOu%i*+;t>5EMCeuE9i88?@i!zHqFz%`s{x`G_>i3F1sPg3 zX>g2wFI%Elc%~y?RNYS?<q?d$(V_J1evMF;QRMP@k zI4rDU<n;F^R%G+Ikx~WwM%v~wW@~K;nW1PHZ&L}Nmzl6nN4k=J z4=Lp+?`BbuNR8DQF2*^PSU?$9y9tpObM4#?~mOP|<38br%il?IL@ z3pMmwe<|4a;-LzWJ3dvw^d^JOA|t+W;mcznWGpRzWUN`6=_CeUS627uX4}Rc{I}67 zh$l$bEm%azh$0BA6<<``nFvLa5fYRn$m>jfl(Hb34YBUDKx!ejt{&KM6%eUlk1`0+ z!@!7IJFBO2trI1=a+Mx!t+z$}a>M&*?=CHrrX;y?j)Og+$?YMa+1KKoIb_6s$ClD| zdtqU-g>z$V?`f1y^XxM6eoU$m!_*;8B93CQ1&$XDB57%{^Vgc(q0?hn{y`AC5?h)K zMKI2)AlNVH7O-E7we4UZ9>q#?mbg|@debXQM3-oY56-IRN<9%sTv0MMSAV!yDDxke zHfgPq&S1T-u*!=1|##F-XYaLjG-9dB3N1}h$2-QLFL zcg=xXM>(L07YP&O#c}|f5e;3rr;|Se!lVUM3UBa&)=FjA(Rtd9LBHahh1XR>#phUT zRuQl(jRW)XM=~gM5OpjKaY1-wBn0vB=orfV4!dcf0S#G;?HP+&XnUN zvAvMzKxtV1?YthUhh_P-V zyobTyq_xP?BQF<67~Ux^&t^&Hj}-94&$EOJN;k|%`OI#FNIEaJY;)6^pOob5KI*U>cb1t|}HN^X8iR+Ok*ppK+g=TbWdNL^zVl#^yyagG?El#IE462tMV zgQNI!GRJvB3RN?h1TgbX5^xtb_0z16*;od*se1mFFyz~T##V&9hd;HCkDzi`fj$qI zWphF20vcg-iSupS7d#!(#A<+>J|>WP@+SPU%P0h*477mwQgNXkjK9(23B$`Xhurl5 za^eRAynnKkMt9<)H_IY8Ll)r(zzbihd4<03ZC`mu_v6M0yoTn}ppNtXI@*qY67;|i zGAFdPJJLIsf|2UIKa@_mYJlM3SBmX=zz%d5^gs@}DK)(mes2B;H?XFQAgu&1%!{vK zxx)4a8w+Ux`&Iqs6ZaGvvE2WP8z*G+JipxvXp8pZgw&>0J=28~KIdE4b9xe_csFcy ziy6iV(RsJLDM#4xCCkj0D{{XsS=o|*Z%Ney_#JcjT?cw6Y;VgyTUCl(by7tGfQ%%* zr)qeHu&)Tt;Du&&24jVT?ps zGS1uH2G_n6$W1i|ahTH#TcePpz}N+V+*xWmWA5Ahk5EAC@mt$Yrq}!92H<;XMR-N{ z8{hr?<_7!y@=>{m9R78h9ESHuK<-c&cUBEF3ks~XA0M<%PU5)_AF>=bPAW$r>IwCg z@gdyt#@QIu=8!v%=Ye2UkV5%ebJueXh&WtA=<1{q_YV%{)AUsr=+N1M(%Aw)>4wW; z1Sev_JSAlPl)r4FSzBD?Fke1SluI^_A@m`3xFPIkJALnQ-+|8F{x{}u*O`K6@9;b9 z;v`);ne$Q?_x~gS=VE2b7)}oh(WP}k5H1<*j&kcN@6```JDlXNRSz^0BpjFnW!piP zGjScM`6^v_$7=Ez{_1vo^0mLX#J|zyB=~6);oQhTI5_?zwx(7b!wwgu@Ecs5sFd_1 z>jp5;n7Z7c*(>*7zWK93PO4SX(kdveQorn|hpV}&O%njvm1@{&DJ zpssIfQ;Ngiq!ih1RR%M8;P*}KHwL}Tj0l!}JkB>x%7Us^2lp>e38nKTe?ZFsx4o@ZF3eAd~ zoAh7bD^vWGqpcL#ttwaVWA;t@)q1V!{3~*dH^XqP9DcXGMaBP5d9R_>qZ1^NVFc{| zDmWzpD;t3}0V@Lofi3|X=TDQ3k&%G)$EZud!SJ7+|7~)zLnk(RQ{i#^SbpUB2-tt> zf9CwZs!PDg$O4_IW{2zrsQhhd+;^LECFO7U^gxf&HKNphW1az0W*CKL!E6Nv$c$S(vk%>JT0A`YUya@8zU8*>Os zq+SqmEk}WkWxFQab+pB>JTY5rTW!+kD{m^9ny>HguE%XH`>yBbF3z3wr}U?7^V~ml zPy#@`QN4n5Wg)}y20<=<_ss_7u7JfC$7*;NoYO%cqfL01(rd_HU%&EN z-Q{LBMQIl{;+Nkr{CK!ptUcjd1%eN@q8^6ZVAN+UzmA5|fr24{box8jA^~bsn(qgM zS+_rg)`iaNnl*kOPiw3g{ose<`@aJElX?NKsA)4fEfzym{&DVc%Q3HaFsYO0722w= zxKHlb$Ij@k)#vfN`G3&QZU%XJTCEkI4=*~pAuWb!HWnI8UZLkYr(*BV;?KQPT%T+F!RW#th3!*qL z1~rdRtUaH{ZhaH}XmxvVZi?MY^6dGN0e1y(cRw6$?0f{W!ZZj+9k_cOZ819tObS!? z2`Pfu7fYU$Ef8l6 zO42{*1j1y70pJcwBYQum-yq72F7V30R~W-%dJy=_;EXDc1np}wqo&M5f>qXu<8gfQ z1<72HDH5?!rclDaR!#LS;d*1G5R>u{LgVVHRQ^n6Jqp7W%-*1*r3gwqg8)?zCZWe+juL+k5H0kePbJ)1c@?*i8t$2$18& zKM*^2#U?@U<{oXKs98^fLMpDg-Gwmb_d{)36%_EHefPB&x++hzQdRd za2s4>0I$O?A(|Haz&Y+`8T6bD=g8d$Nu$FHhB1WL7djQv5piUkMBxu4Wq_%JG$W=9 zwi`5M5XbR5VnBBUA4+C`zavVE^j|xXNoI8*YVY8~CDbDcb>e zd0X_(Zq?5N-bSPoy+%CaonjlFoxm;x8?lAp3mkywmx%~~=@IlnuoLw8bqDH!VJGSX zW+&`}VkhqN`$v!t=UQMB+MUQQu;Z@}_YkfFd^y)R;XlwE>3PClNPdV?L9Ot3f}8SplpZQSLU$n}2)X_vL;M2`VK@EGe_W zC_w%TA&$rgb!XTMVkb>Ea4037zd8bc2tYV2x3_X1e^)sI^wxWD=0{Vn5QRUKKdg3u z(F633qci*k{6q1B*9QuKq#szjOFJm&fqiEkvD3>Pw$tAc1}J>N?F@YYgg!I=0pSz) z$k~U#CEVxVt=+G@CH4t=kslPkMaCI^m~jAG=;<8^Bi}6^9waIjc{U&{80@zYf%%P(g8#%Q&` z|6_3eFT^Tk`X;SE6#fh>X8WXQI?Gke{KEV%fG(E6D`fHs*uDY%*pxEAC;|V6KcJBL z1@&LRSI+!mY&!ct5dT?D$o38KXD14ozG>?Z13wFtuzfN#orRXmeuu3;#Foi^pMDp} z_BgMW^nBZAe*Kj1YFYU2jgtSAmbx?dPy4k8_ix*UZ~W#npm}%34zXg86hQKX3C%>s zp%dqbVF>u_R|O8T6KBfBq7{w0LB@d!bg&TF0g4k`Z{8Bf2xXn$!J>bExJ%2o`ZorTqRx}3p@IQSL#-b^Y3<@R-C{IJD{+Wlfb^XGnaiFXJ%$c z&S9BS#5_K%;KoLp7VHzHq~zoymyCu@GrwaGoXKOql}KkwSTK zN|vE{_=OH4i7QK`|IFG#hQkMT{x?!pPo1@!(MQnJFhuIgMtp86!DE9DGxBbwG)8|C z*!dX)l;?pOG>-zdF&8mt*y22>jhU9(XnEc`@w}vlKCi~LPZ#)kN+v_ zD}(B2x^9C9cXxL<$T>g)1b00UfT5i=PSG`Wsc*>J$h6NRV&5 z2xKjK`|G7^O#`*59*8>-7#SG{%wM~oRgT1X(+0=lZ$*?N{d)P-W*}ZHUoNs~Zt5xq z1`PvF{jbgzOR05#@;`Xc7TL%l0kFA3oUxZx+{sLupM@AuTxpUB!DGHEzyxXVShEVS zAa&yM_w#vv>SBx%yux3CFg3@WRyRNOzp=;Xs@{$z74F?}F6~vW{Q%q4mGPmx;}OLL z)B>#iXP5atlmY^LS%{@Fc0oE_0_WK>?7;vz-`wt!bRgE(=mZQOqi)_D#z0bwiU&xw zGWd_0g?J9vSWrRo&)|u%ZyB*)Og)x$@#sh4lJ@Hv+uDvDk0r8~j?PGrb|mk|56@&m zPfVWOaw=TOQ}wai1j4x#q`4W-Hx6aOTH9gpIU;+NwFnDW_8`;?@{6=lCz=Jd7d^lk&?2qvV=vdOmUp8alR&gq3Z(-jBS>gA1Wi}9cq8pQ>S@v&d^PvzvxHlD@;eG zpLuL+74}g&tOt(!9C=UdehSHyqPl#3R1hwe|F|L8LqvK}`S0fGqqsGZa2EkKYh;3okupepjt zySnR!r6pNg>bzg~B0wu%Y|>-i!P0nC(}Z&kV_IHqtBGo+2r7`G2121%AM#buG21HsVNv& z-4+e)RtLflpt|Pv%)1rQM)c(OgJ_0)4mn8!5hJ#rKj6pZ8meK9{xMx*j*@J{xdUr_ z?`4>iPa>LS@fo9+Z}$?CXr`+%F=td3 zw?4LZXJW9tVbZ!ltSovD(Z#48AT}X*1B#)dBK2Q7#E98hLRR-SjyEp8>>e1sflI^V zBuTE)+}zXZUaS|l&Z9b3Cj`cHDcd%``RU&h@K5gwl<wJZp6Xvw5c9`XPis zSTd@Q-p@-lY4$2e8(NOIEj|n6y$tsAXrX70*+W*0OQ@}9F9?v(z*3pNZ*o69yP;p( ziup;LOliKn+-5i+PUYJgqgnR|Y)SA*eZ<@W^y}%zx*f-Ln8Mbmy4>^<>B_yT?wG0NNCQT9`MsT52(WOGtd=Y>iQ=Mua z1%6;F?Nr&1#viuaJvA|*zahDS@3A&I-TVOj!MemA8Tio*6&8bq3@yib@FOH3O8C_DgwIFsiJv)pu%ML|e?Bw;U) zFZ(?`@5o0SOy4)^y3j20z;!^z*L?0grIPP##$Fh3q=dp%VcD{F63#s0&fg@RZ?c;s z3d>oGEklp5vzwJvzgZd?9G)K&BnIr_hq5b?C~_#;T{qjhuhq1kgLE5G7Fai(by{sc zRakC1*L?!Z4+|quGvFzY#zjHWR;|rAejM{TU}*{soQu$D*;OU93(dwFO}{_`2^e2i z72PdJ!$?+T{OUMSRseQA9LQt}O!P`DOEA}5kAp^GnTN=`9x+CeOv`k48kUri!a^3} zWbiy!hJP@ua-KFsIKs&ZA}&sDSbI$#WR%N7S78LrF-Wok&>WQYvOYjc@b7WiZ)WqO zk?rM*+{lbnswGtM7|RK5_D>u)F5FYxy2hk57D?65_(INJy)!apVq~Tdo=jb0mgW!m z@T9Oty@Q-xAKc+XIq1#!Gx5;f1R2*ipHwJ@qS`WeU@AOyAL#jLysDG11TnMHty~Q$ zr#JMxd=DZzUo*5+ydNMYD%mAb$t$om2Qu2zG8;b%5L%MYwDy}Q=Drq0j_Xe=r&#tq z>2sF{;AP-PeFAw(;7;6yp;+C_;&+@E-Ro?IcvgB<6i|7-zb83{xQ}Kx+B|G%rZGD! z`-ph9H*!p#DXp+b)4wQLa2V#95}5Xx_Zd=f{j#-{iwD0MF}6ZNo${p3RgdE}Y1_P+ z`Dk-sjXl8pQ~`PK!r+O-c~Z^A*qdg{gHnUJK7aiwW#IdTx<^4=V`a`E)pm|xN(H&N z;8g75E0q;jOb0^*a|MzlxG}D$Ui7kd8s(fDs++m!fFi<_mE%*XH2rc>o~$wfX|bX{ zAlnhqj{!o=@+L?ZZ6E2v>f{vE{4a}BVE%D!Xf*0yXcH8}$5BJ{%$G$j>;|5Odir`3 z7aA(C$mpq=LYT!~LEVMnJRlJSIyGdA&5pSDHlnG?x6f8pkN`QQ(TN5&qYNn(vCBr6 z!8M2lEw?4a=tNQ7=v&g<$VQ9Z6Sy%X2lLZK|4xy1YJt?d6A`<(^T!39_*%!A5HG}L zU#X_P-E*p}#hMs%x2RH-msqxPo% zR0`ZD92@eTx$;SEcS-*^mH4oveyKmGpD42d)fdS@65BmVaD`8}y!+dIo8Ahz93^MWXoxmgfzAKJ_;*}>7zBbG3v{z9BYyPdEYkgM z8-;2oP6G7LL!Ye#B4*Lv8C@h84LcmC0|`$`vNFk)jpVCQzYsiWXFGv*#H)xGG0>cT zto!%As0*`jFF!NXrSaSqTJiZG>pQvAj_&L}TLms`0=Frx2dCpEDvih z@Vc({v2L*JYb9BEk1Kh-Tl6K3m<(hP>Dp9cf*>hybZ->2&+G#8jFst-`Ri`ma2t8> zW+&?dc{^0nPq2K~NpcZ9t@TT)4hL2z7&tIxhBSRk3vNklSgJ+Okz>{wi zUV3lj;>yy%Rp<+nWaGyMp?YN(F+t%epwr9bngLh&c$l}OvTbp6@LJ)o5oP*wal!kp z*h`Y}GxAHW>%Op(F}ipoDh+Wq>Tgl0gLv11GrcVO2>=0>?_3)1!p{{u{Hvhz$W?@D zStH_LXi-uFC;`g3LezJ8OBaY&K@SPZ9+npm6}e0I3`1F>vSnlr2!j4)#|&jyLzJag z)-M-xLz>~v#R2Mw>SmDWnqo;pkV0=Kd~w`}gNLJm1Dz9MAZVIc-BA5d`Fg5#ia|fj zUH!3rtQt-G0<`3MEq#p|`l(N@FK2iob5!7IcqlTcRy5)FrqapeoG3&wbdrE0xtpf`HQX3A-=WT>*?ozcF4nWI^f`@dw5OCGza(1Jw;`=v zoog~azYFwT@?FXP-Xut}NPd{Pz68fkQphMve(F)z$bR~$z$o=-?5NMac-(bnvTO20 z5+w+Rc4`n*m3|607(M6a?D4}p#yizJ*gNty=@rwx=XY3ZTI7wRy$zcyuB#ZHh~QWP zYm{~WgL{)i`v+Tv*XxDd0UQT$si|Co3OBK-0u1|t(*P~I#0IPn9cN~FY%c%lVbla>qZt5}TO7cJ3o>A}}nCt-py+NOgwOWxMx<5_Jp>2|- zjIbEDZ~A->FZ>N5uer!sC+TY=r)?O~^Ea0xsI!K(f@fae@nguF3ou2}M_TRE4UW0@ zgK=mI zsGIjw2R~`Qn(oT(bE!i?=&2lI+Cq{gbt&1gnX&1dtuRqI$YYavQ>`8~O+-y=P#&U5 z?v5F1;?w;T0~(AbYRDZ}Pjxp3YjVS$0{4>Pr@Iq=`EMrv9Xt+N$2|G%i`l2eJ2qq+ z^;vey`-%dRI^k?@=(B!vLmU>q+RYYI_iF>SELx{ewTEQMzVD;@*lbRq8_Yy@dX#MY z_q0J*^n~7M70~&9Cd7nJi0+2cQ_o zoe&ojb`mk`4ixK_USsOeFBRA2kZTlfi|EKdoc{(X(DJ^Bb91CC6^HRYVZF(Z*g z?CjfD*8&^l*Th7sD}tBLE->R<66^S%RHXl!7;ef<&0MGSKaDQONVVo~tq#n%uj0YHMF zRx(EZQN8w)WhDUXQme^8R#)1DV#o$-6k~LJ7+UgC46xs*=10Rqp_IE3j1sRuCAYD z0+31(*#<2a#YlF@{!@aBR#S_DU`%^~XeI4Qo-US1C>B|u z>#&h0u_Fou=e+%-#$5@%!P8>`p(O5znOQ09xh^p&H0hJF4k!kEP7LayhYzQR5X3>? zImmvbFzx8Sx0Fzn>KXDA)GvJ44~L;CqZ(6h2&xt!a63yebtv%@vXG&tFaZL&~NOEa?NAI^5`7mqBMJyEY6DJgs2l$6i$pLUZ6SOKiSG z5xCfhO{tiL|F#)6uUYc|GVFxgtez;lGXi2X5)KcFBxWH)HnfMo0>mv^&QN>aorvsM z-a@<7YAD`D2tC5R6s;{@Px#)1eQ~Nsq55VCFOM6TzJ+5b-zZOWpWnEGnC`SutklVi z8&7k{?HumFt0q+?qs6-S0#GF>;7TXPU=u)#RDfuuy%Mjp@Em*XAbSW+@ zWQIdkCD9ey6(MX32?NQfN#L*0Ccfc{PI~Z_SNT4|o(=?mQ|(GhN4<`br>?%~Y8;9$ z;}jYjf;uV~cH`8h<*2xHPQ_oQ8Ily2HqiWt#Bc-?l$CPu#hwz-e(@L+$SU_5o9JmUDW??6F^ZU5>`1fX zjylb-{*sJ@BzJf$*A{ClhHUIL5a2R7x;+zA`1E2c9%`KoUo*)L@Go+^XbRWUAF3Xa z0HdFiNY9^ZOmEow+9+W{jMz!O}iMuJE9bhT&@UIC~CIl8G?}v{3b}q`ReL+joGA{+n1OntOZk_ zX>pi_&8I{n`G?k-dFw)>=Cj5)+*a3}8dgrG8y!V;8>Z$h1m$j2yqpQ4Dy zJZ6k)m^mPqCux1^-Hul#I*>4DDfC)WZrI#X;j399Vs+TqHCc_&ZWv21X@6XH93PL% zf(_S#ERWa-2U9qk%uTsH8l9RE^77f?Otf;*XXnBWZ_Qi=3-3JmZ|My_IN8ePW45ds zVozdD{oZ^?VYbrPM`COmQ2LIIg4Cw0NkXTwgiAUCAOY7K;8QJW*T0;_x^Jz%xgY zng?;hF?iPV6ql~cV>y;^|7x={q~a5VmAi5w7Z$!>LE7lhlO<0xE4p7Z35x_G%

!G(k^VT(o@Dd^1xJ@7n*FnY}QD+djj+-C20u z;>_ZN*1e<{R7rdTQnN1Xh-{ODC?0v=MaEb+BMg(VZ)j9;wbxvS6qli*gU%Tt8^jwc zyB(EIx=ViWE>uq3a%qD$3~l9G;}Z2P+cDGeSF*%x79N!%WK?gVn}yxdvdc#6rSre{ zX@)SCMX-NqY6;=RL$#*g<8pPwLz`oIn? zHr6QHGqHDQoHsItqZ;uIyRO4Sa?hs~2;w+B-zUXJ4^Pmh(}D~4 z;vjN=-$)KOn~xse-=8YgXAYA{40|)-=31UgFf6GJuivXJqV54vcqQOMJ~p7Bpk(lJ zESig5xK=tcCZJ~!tY1xkK1JBks?kjVQ-_`1(g?Lx2dj7hf0T2)Ntvppggo~>I@Y@% z<|@i;Ogp()i{ru%POF7=HL7a*{_dtAeGWom!xb*B1mZ3giofe_40C3doy1snPE%7l zYct3dfKN5p>N5C@)rQ9*f~;NzqhTZyr1Pe1#N+ToGv6}47(B1Sn`*BrM z@Jvf~E$VWpww+7Uu`Y4<&I1(&B@PhNA-4;FJ44|z~e zyQ_#*fl#YFG^7`h2osb5K+_f=X-1a#rXavTJdgFzWEXRAClMc#K_15a>*IdH;tdJdk2Iwo@3LXNv1#iHvWQT8?-}?nxpYXpu0Y_y)W|B zlwk{f{KcE@MKVNv<}~TH-2FS@c&3vEU8i;u9YfUa5uAw-)`@+l&pE56C-ABZHO48S^W560bz9i^>=9bq#!&U6#T{w`Z9XOp-8+PkFLO^j`B$ z3CpjN1@Y#D|5(mn;R3DlRrOKFfK}o1R@4-J=Zv_fO7JM<15RL5wnE z_5%zzW&`i5{(Ml4NZDMzu_aEaoXN;O_q46qN4h>p^ZtCHfsXenU7rbDjva5FUN?r{ zZOljLd0!=RnA;#>yMf$`8F7kEyr?eZ^4ajM;JAR?ktCYZgq=eR89E8SxGJIfe6CeO zMLo(Y7UvgW)oW@neEC}Cj{SD?^RacLoOSaZIhVx7CA!I6C>ON6M`agbMxVJnv400*$NjM z^fMK~^cwbuC>6a-AQ|1s64g%=#7-<77kjY^gpwIGqdrf~=)Q$RnHWhpZ-$b=1?Z(* zfKcbxBqLjl>fAF-Xo1IDvI6F{BPY1(IU*~Fu^pDF^xz)%{+VgT2RJ4tVL8epX9Xst zX{pLfj=;^3a1{(;4&AAo#cbZJD8~Dm_`~P`*A}n#T%k16)SdBu>E2aG1{y z4Vz@h%zm5?JzcxywfTbX_9N}UX<&a>EMplexuW|=*{SXYl#H`w)KrHo{2dj+9*Qo= zny7u-x_aP5ihqG|9)Zgc6fIt^XQyWNWp<8Fe>_*tVe!%|P+nNqO1ri!XXpBNOGiX4 zPI&!tHf!;35@%EEd+F5Ri7>YE400V_SA(;|?Lb@H8sdoZRL7oa^kAk^LQzpeYc<7` zjfwaQWBmI3metTr>6@g-YWa$BqAy2~0ZAG?cKl>+=)828zc?0zwb8mL*n>3p&~tFd zA$(d(XF+B6!Rf`8X{l)jNo$>JeH=EC!9qC?zmya(Ycc53P5Du-@Q#17NrR9zj@(Y@ zLotM@B7-0DZ(0){3Pp9-)pM~_cfOWde4&==aF|MT{F#`mtY5Mjc=(fneYejILP;!& z2k$iDPD#y$#1G}e^_S!?-CtDpYlqtKg)_vP`j2;biYUM6NhJQJdf1!FsmXj&my2Ak z8N+==`l?N{M1xbeG?5{QM9YPPk|uaRu1=}xM0wQcejtx6)9TCO**-SHfs{u#wi#CE zve)dm_ebZ&O|*FdJg&e%?htd?2SF8v=zMKoF-%~N_xQfw=3F*nmB{rA4=<_q{TL8E z5;4UW$6X&xe)dMHfW>Oae`KjDb{-!mwq3OE1zj06Mq4#Kf22(^q2a)Q%mkxuKgcg zEtG+Pr0#LR*VD>f?MLXC?rf2j(jr-8& zD&Ejld(FvrAee{sM={w@Z?1VhBX*Lk@x-3f++WIw*XPsr#lR{QY zi0Hi43+&8La(o6?Rc>i)RjD++588bf3q`WSeiDq@`rd&fK-(%WK=Tsxq%oo3aMz6< zFg706qbgvz$>nas%7nWE-{xz_83@hEwaT_VI_R`wj26K#<-GqQHqE5L;PIO`gr^@$x&;C3B3aE_J$v;_`-^m&oT{bPFr>6Fi-@(P zn5titk=((&Aa^-{6{7t3w#Ro8i-H#Gv+iyFU^4-*AmO`LyQmG>!*$>LljB;5zrh*U zVsN;U66-y~H7(bE+Z16r0&ToCO)KX;>3vmJ(C$tqwpK>v`tcj{P6V%EC~hQ^O^K$y zj&)OS^V&7lA0uUXCbJUlEQ3^5Yd~Ljf=GPeRgeQyk~I?;6H>Wte zR$V+?3P$kT!?iDwS2HN$MDQ!egrlYH@ljD%hI7!@h-_k7vnsDX0kYkEyiF?}m2j@G zfM?c0*Z%r>)GZ?gu7%nHkA)-BJI5cSDkFlTll;^yKv`NjxZg#PWvbatc6+i1h4ycv zvch|pLj2859~>?`o6^FD!=_NQGQ{JUfK$h}NG&45k5;u#c-fbiM3F_4s)h#gZtw!; zAD3zW%5d6vH*r|(X!|bmxJ->-x^%UV`_#Omwv3*mxO=!0kVYNX9aW`tHg?{2Izq8s za-YP4I4)zDaKxTezuZ=Kw=ibNExfk)UfwEuVrQ;{f?{nNc6W1pX?~X@<1w3F33xQ+Qx{*uq z`Lh*Odg=(47QyiTfB+TF9cVnv2G)i zQ@a#@{{4m?bHc^sFDM0jO1&h315*Xzn~30iqq zu@bl=q>Qllzv*jScU| z!-OeArLvt z7yl3b6#f_=P+&h4#dbc^L-YNIYC#3j0gu@9Q2d8_<0*$8ZCDV3J5-I6hfn^jw3d-g zVx=xA1*DKedwYTYrBftj!d>$j9==-kW}4xM;-xaB;ZQ$HMm_De$03010$kPlfwD=K zW!&o&Wu&z!oj%ww2Y##=sajDlEiL`KBVO|WJ^Y8I7F@Hu=AQkR>6lr;uqQ|tX1qUX zi6sW_<3^jyei<=lwb?qTdm%5iVf}vL3P>D4b)8n>Si2Z$TU1xn04E2K zE0G=siWb7b#rbbL&_4~hH~?J#jF$@l`Zo^1%lj`|0N|a-^S^U_4}j-?aFG8h02c=* z563@%TmVkae;NY;+yKx&aPPFDcka;tb^-)&a{n`EZa(gJdeZ;Lz555emkG%IPshOb zp#K~A9~?Iy2=GrQAZ{+uKb?TM_;~*Hgt$5X6*P#88_f6b?0ElgY5uD+@1bz~%P|*- h1NdK={3k#Vkc*cmkr)n292p4Y1|iYXiYrPW{U6gyYg+&S delta 44604 zcmY(IRZtz#mWD$J?!n#N{ouhNSa5fDcj*Lo4esvl9ymAzcXxM(Aj7>cGgDpFwg3L= zU()@udhfE}FDKz&Y6u`8I1)%8TmWr*3~r391J6_D8ah(SJWx|ib&cJn6B34-SDyrI**K|N zU!iRvvOKI+ZFo2xg(tA&zz8c4_V@)=ML@G_=+2fp?Gt@G?)u*6aWyb9~xv-mdr>D(-0kV_x|Li zF)_+VbIATJuNP!-pyDhaj01?)QD#DFu+1IMFIU%eyjFCuL{?xP6k0D73kHVp&t_kh z_n|{+%5#E;Xe$j%T*9fdbml^+?|Kjz^FW^2x`(@nX6LG9@rf%TiPYV5-C47P@9eHx z)RqDFIJ!96IV{!OSb+uj0wyWndh%_vgd3`_r$OP$GtlqbyboTfSPBMmCxKPKS5V-kwL$5hqNY zjLbLjU!u{WZ{)jY82dgx1LDR!Y&xyPZV4AaxKDFjo|G7l9ym>uuYi@kT$ATL=Yb}* zlo%R4ZfL6-FSn#DV+UTk2cNJr)^=X71|R;2-MLkJsl56JSh_0fZoXHgZEj;=Z=8#U zc}YS{j_*p*7248y#S$$~-W*UGuJyudrI<~@e#s1>wTxs_Q*7{iDtmpIvFq$grxkGo zg+@V{LpBjw7FI+z##MMOD@kex-d{dqs9su9+P2F!thlw~PXS!<^H*F+T>4oy?#z$k zMXtyVsj7r~D1Xv+_$nS$>bQPt*QtF!(ZrY59#4a64X>cO6&JEnyf}?iqdW1RqO^u= zp|mb+$ZCvh@SIn1XLfnO%4~7N;ATF^wobYZ+@r+Jggw-W($tN#E0} zQ+SRQO^xADH%@09Y>Kc+;G-c%dX@X`=5W%$m0*n+a{#s4%O*TfS{YxArJ){kGn zR`RM+glcv2S0Y|O#&@yvTT9A(zHqdzlempS-@wC(phwZL1MXLcFEMU<`5RHcN#->R z4X~UZ{CgHvNq8K)#>Ex#E`~33}k#VkxDZLOE6xt9jxf3SEEGcyz6ZIiS;X+Q&15!zINn!Odz1$(wIyk;2SMWO?SGd0 zO+Xs<9X$P8VH@4U&rwblnmQ7rBFoG<2V*FO`oXKIg$~?TrOcR?(U_BTK?@8esnr5l zIHJbc=0+Eb?+tjhs`l$ii0MXx-hTuh=0apw%LMKzCEh#rQCxEhRw*$*iW5=UOpXJ9m2IV`{9>>Zki$LobEi;m1N18;%-SHUy2uD^No~9=o%0v z`0I6hp(F3y)HNRCoV9o8ne6+>?|n%j^u#l2xe57l2e0P?SKU2+=6-__ps(i8yRFu7 zxv|R5^|~GIX4Dl%G$fK5#XBB9mrq^?e=BqXCs|Kce&1#QV@#U|K2h|qzmA^Xi#wGu zVGCuOsSGeI{E2#~Nf-_l55!ZAE*Q?)0ryDMEzG zDyI040&&O@+E*$xtzjRanobg)$(v|pCGf|Q4IV)Q_|a{>7dj$MC$g`K6w1yRA?|X+ z0Mo1I>h`B?G=#=XZAFuXJCrVOM`Azl` zO1v`Qy7fSY9Sb<2I^q5coZv{t@!RAL#C@~Fk|2>hq7A%Ebi5>7c>kQ5%Z&(cYG>l? z;$&)Q`=6D)krg~E8wm@^e-|Kj_5?#%c2o{lE)r&0Q#*4P3lcVN5Ld$H06B2l0ppQA z_r&d!ZEkfEXTm5`N2Hm=zl&MD7#&7D=$3?TGgOBzo00jGw26H&3YWGI;_E$)oXAoj zR~`G0&y2x!q!f`NDR8I~pPDW4X&C#wE}VBO-d8T#*X@CYy9-YrKXW1T^={{E^5Jiv z{`?~Ob`=;raFz27>d#I;&Pf_DIn96O=P@3%74|8^vbO%qN>><`xS*Y$4KXf}hDn7aBEQg?Ji6oiIP#h<-Y) zZC6oZ>(1_{L&vVe8CYnu*^6U)kOtnBrs!)L-ddd%}oc9RB2yNv@%y4tLt!V`xXG(yiH($?qd zORT+zE1NgV^`UFlp&hp+gSjc6%k&Az3B`wq>UO~w%e^c7#2eQG zgC7&u$^lgbcy@uB@GlM5+4i&t*|I!_Mhr#%D{!2bR6~EZIxH?_PVdkv)cb1wBqu~2 z&V)ULgG4Ceot_wLQtHv;b|~~P#Gj$3k`0P@TfmeHe8!$-Z;`2yiw#Ah50H&adF?5uuJfq@gG*lV(H zR4EkSuBlxHjlCe-lrIJT9tt5;rk4BTx!4|NQL6qzdZlWJ(%8>VnHYmHAT261B%tmJ z+)BUfTo4IN+A#zC@tGj`B7<35oa7(sks%aH&^)so0)I-rjDui>j^lG6b)9ey`hHp% zSbQe#Hbh1bYp`LC0?*S{)8PuaU1c9 z5_`nIe-SDmhsH7ciNXr&Cx&??Y2*5d72QA7@CRBw>N^C021_bIiq4vDh-^X${L~30 z*%!uOgd$o z#_cbYsMKsg#uwLNA+QNf$l{l~P~{v5M>-CwkWCw$m_mrIM?CP+ zWFQE#h@v|vl?Brg3#?`#VH1GWx8xvyjh_O(JPo6^DX+PUTTmg!`e2aism2b5063u2Xt z2c`?@+vhg41rk5_&dC4|&Wdjx&}w9Q2<7o1o4guPEm%y#FO)jr9s=9KEf7qiIxy-a z_pCYqiaM3hFZ?aKzVuD(P;U0sPo2XthU7X?9tb)C9`xXqO|8C_t+riB2bc=+4s^dT z4{V*7!t-E!mOG;0hhE-3{T{Erf146xH`c-Fn^b*&9oVe`VRu6maPcNuU-6b=-v-!tSg3~> zM-WmQ4Q30j&wQ(S*nN{H7zo@T6-06D!$#c*#?B?aqFjuAWIO%l6SP%c!~TTT0o@Lu z>W5IHJ&1h1LR!r9LBH=I>U%}p2rNe3@GnmIw`KbXxy^@Iiy3;+Yv4rO0W(2HWbi2X zf_4SzA@~ICGjvJr0dZyQfp&%NLEDOr+&dZ?vWeNJu%+5JvDvtLa>)Y*U-E8aUqWvu zBX;}y4lit4FUEL$Sr>Z=-fF&dyH*7rk=KO@{Dny0Qn&Osb^G!!iMI8(K5cJo`u25$ zVTRQ=aeP7_1;E6cfz)qNl@scLh!r4XsA$6@o@5{dGO$tbDtq7G9{2Xp7||g z<~`l{xjAyC+ce`1Y3J4+rx*Z(UhxNxByoy~4Kw@#9iP$uS1A5h>9fnfr#U_^e7MXq z-Y|D=t3O=*pjY~VBYigc_jt$W5r<1_w;$snUE z+3{I#;K=*KHx7E`95|}}C@sqI8L*C=(KpU`!`r!C_(=Ev*niMR*>pR%>L0cd&@1ML z?IUQA<1_sSdb964s(p8O20*XfNW~jBK-kR2KOl6b`Ss(=>TkMy^1@cPoIUwOmS|R54u9V*G?(NdS8WU2#HfM&QJ-QymF>yk5NL7KbqxUQNc; zm{Wt2bgZcsa=_>3yY1Ay3Nv&AOT}GQVovBxy$OD@qy$g$a=qB7`144okuWF!9Ajv{ zie%Tv$!8Z0`C^&GnVW6P0W(%(9)z=wEZqEcgV-Yk3Wgl?zr3CD@#2|^Tp4<@E&z=h zsaB)6voSanaGK~FPd&s ztPL#2&2(xBP{f(TWa+M!I6+ZOAV5}4(>)E&V^w23z?fBOJa-X8YrKM6+XX8#B8mN& z!P0E+)`)lR04j4l#5h+u#c`R?Ey6#h$7+t=t65bsQlfLtw)RAUn^gZf%!$iP=bpoc zjuwz1b0tU4hi)UC7F{u@ZoZubsVAgqkk+w7I$a%waodB-d9*457 z&L=hNTCiawbjmQYS=kYUNGr$N*cBt(ez)j?rMMbnI|uPN@LO(IkB-=tw#VKp}j za&~g|O?XpYwlmYzTv_!t@_HIn*Mdy5v;nmMFzft@D=2U5_bz51Afo8QA6LS@UC~Qp z{|A>5WsgU6G`BmPWh&>-cCY;Rk@zKe7t@V6)wS&L=c)T(iqg9|d^`=Pv zOS88X%b^=Ko3omwsihsMV2^+At^e&64(8Z*W`bnpq;tN%3>j`*9-T8Hq}9!Vsn1i! zbT=}Wa8Yzu$&V8QK*^7$ZWANnLLrAV+}5OnsUp&^`|vG%kaJ>F)Fcu=F2*F!>M%G5 zyk?J!a@?8_IWtsC+I6xl6{wbUfiJEqY2`SLqv);sxkVz8 z>5tC_TdYUtEiYYI>nZIO#YVSQKG-xzu^yA2W1_-1>{ zVEbdIK-4xeM!&P6x;7s|f5Dz(Hsk~w7ydeT!$7PW&DYVrr98W3@az88YSi2-!Qt3q ziHBE9F6%iG_as)|lhjT}&{dMN7CydD?hkEYzl>RC0Cplq)I(<#&sZ}jOBFr&c!#ZA zLRZ2Y@?`GqrKbxY9e(U0G!-itdz`PDb9}-&-*wSQ!(MdN_sMXgl%#7_Z^P^Ls8q14 zda7Qwu&1l*WbW}Jfr0{(qK!4!y{~oXF}mwY<0=XiC^w^05uv>@k4J4?E0TC0a8Ene zF1`gw;LSMx!dnO#3`)M&o&1HKu#O$#%#46kP;A|)nn=mniTbR^gp{Zd2sO-7AdeAZ8i zb#3{niKsX@m>3;KYr`HW@*WZNbvPerz;Kqw#6ME*;B_sqGQ@ZJj3A zUOnL;vFt%n-qm$w=<1^{xzQn3=kBMW+Uwi?q}J6~U0%D>cS{}bJ^yce$KD1k&p3W-e-s1oi**8J14O-09^&V5|UjwoW<2mZyVp(ikP1HUBKQkw6{s^$6 z^Dv_C3EW=`nvR8`;=&nzdVk)Q8AoIiz$H*%v@410{rz*(%gmY`qx%o`g-t}}7#s6C zpZ*zGTt`mbiIblH*D^B3DcsdM)J;ib{vG?ep8c*{u`{IQjfu-z9I>sez}AlmP$_KuC6Wz{R{(i@*P(8E}mwr~|Kxj+n7>p!JAcHkD2& z4C^j4rACD{(l8GgZc^UOf^E&+bBPmTKDXd?k4AxM`qZ4t3n_c>-usI68Flx43`H2J zO%zgzBxSREW&8lj`XuAU@NhHckJ%dnC{*nelrhHbT;7=o@U=39U{`2ueNiKiaxw~& z&7Fk0*@RmTN9rN@CF}jS_vbp&P&61A9l@mQ;Xf^olUVUdVU27bwO- zpqE%oE*kLXh5*pF}cG3BS|FqH*cJB4YwJ#WI z$J^H3^N8qjG05V|9TNxyG0AniQ@jx&woi6?_jMxd_#Uo@86K~*e%1S1ZM3Hr-J<=7 z35hFQ$OJi~+R$&|dAkbaS8Pru)~i42UYGU^Ofd>SjcbROJf}ltIT}IttdiZM-K_58S2xO{XPrpA{pa)oUttd=5JF04q1wP7 z{SThN9gkR&K&(BxUq7S;?Fz6)D ztSZJTI2#H3=v)1l1qbU#&R#r48atAE^T)3iQxHtjeEOd_c^5G=v_&PV*OZ!*vK3r; z#h`4lm!mA8PS12A)t_#r#`_&pBcgbTcOcMMfP$R0m^abGaGW|LLHhlAqY6ypN*shx zjzTC~nB}TCl_QgpFgA{N#Wa5|d}T%)>mEHK5&rMK7SzT=k)-UwV=2}`5A01pgjJ@R za`rS=NgF^wDD`x>~@3`R_gDdD$&V>iB>a7^8m5vA>Z&+z)GKc8JYL+FaOVX`3$}M*-SZ&jMhdm;@{}n5nx`q1D=P&;V?w4eQ(lxsoT32}UUTMw* zN!<0n4uoK>V9hksx~GugeIZ${cc(fYd;uNgS;O8lMA5X`@s9lQRUPy3O7HKNj6ci^(6mgB?fN7=`@ zi6O(hE8bw7fqWDyAk+P`g`!^5!+Iu$3Vv1gaJ$cJ;lZAdD#Q*|ZQ^eg`!kzR4vqcV zH1HD%ft%Q+2Gy}I#sC2XgB=7*n5#+M6pt$`GQKpCrJ%WVYk2=|_zC27^q$Lam&Y%~ zmh78kr&qGFcB91j8A|n>Ai;Q&l!`e6(=<{I!gDw^Ll;wj+2t-y5FNd6x>DrDVjF!* z2&V*i@i$@5H7>iv#ny;0Re6JtXCc3S7Hu|pOq|T#XU#;&IzV53X2~^tJdgbd@f7m2 z@Ziz6e;h9^CTr5|wvT-Z^XEq^6eBeP%nI&g5)u3upyf zKBeuKUlA7>DgcNcS(xA*{!K02lhVeRM%5_$NA5{h|61h%biOr@&pX@hTyoZgG{DQ% zpOz*MxEtz(L;G-r9Bk?KRy#XSW3XLS1g~vn`|zHQ`%WuImd^U58ymiflWq!n*`(13+Ej!5s_**0t7D9x$L ziKfbpou)wOtRa-Xele0AY7XJ&jV~~0VB|+O+O*V}ceu9i567fe|Dn-H7u@2M7$ROJ zwuK~T9UfEsAj5f;HqbIZH7RlEV=$mrX=LYNXJR+L8WOm-wHLR?GfMn54g5E5KVv^6 za2as0E#NA?C(WhN!Jr+X^^$@O4u~_5za%xpWVgMmOuLr-+l6Q6`8O>Mi8I~0X)d(X z;vakJKepD^)KT^Y6T6|+j8;bFJh+3#^JyzC^Eb)_{UG|d3bC2RKWXjox-a=XiRBiO zY6d5Dytlf~dIHsls=l>)GtGzndoOr?kP`sR6IoZ1{rO{>6*rl*|O;fkDC_?*TO ziijg$YV7GJr&+ZcM~Z3dW>Got@cj7~P-jo@{9PWGjmnMgDqT}+KHVAnxA{PTCWC>g zq`b^vwLl@VuBdzG20)VjPh)6vuJ5NaH#{)*B~( z9STQ7+ECnvGAuV_va#MtoB?`P$*~`}3)D2;m8FHQGs3LUfM{u(?2%<~gJNt7Z=*(} z`ZUVCjz}W^i$qI^Z%+7HxgliAPk^82ss7`TEQDE#Xq&Kw@YSvFv)HEmBBG|l?@0dx z161yk5XBJdUn_>6PXlZXevJ-UgR%@&GKs|B5Vqa8h*re{y*GV9C|;Wgpg?NijOsCP z@k$_SGKhMtY!|JMs)e)_-x)WD$PdO_iaHfq{4jfHZ%0GSK-39!H8x>2o&RU9J{X&V zehCGA>Vb0clAspat|$Ki>eA}k{9by+$0JZjIyz^15#l6>EtlDoH1b#GCG~V*SE)m) z$G2|uxh(BXh;6EX;eg@lO_UOVf}xjN7n0i`0>&Vge)puGF-HVr+`rzCs7Mdu0v!go z-+K6Tw`ErV=O64#RD>h*HCq!obbX(Ndiw=Yk#X;71CzZ9P?Hj6(*Yas){!96$ z(_w#-P^4-0&ovVa-2PU@tG$|#Ru;$^u0Nt0?{13b9axXFLG8$cYHQTV1JdmgG7&sNNzpc7(d8y@!o7q z0BNI>$lUVN*T;^C&IU99(_2w`oa!K+(gdqRY@(2P+|Hd$gS?|9VaxiXz_95E5cg()DenRS<*k%qv zK?3+*l(2_3Y~xXGCq55^>Px_|E2Da+_{R1H>E-NY z^aWoerbhMgTL#5Adh^IlFf)-K`_k`@=u+LUP4xKJxrh=w&zh0tMp*Ki)tdcTBv1}_Q6KgFUpdE(Cdp;fgt?wa|WlPbiT4ARB&92G4*HJ3b6|J6BD%|3hmcJ@Oyu(>h8)j4%)M(*z+@CXI7hHm*Gg6e@2xccVVOX**%Ym+w!6Jf#sN+MR)~>z)IWtK!pw zbt1CpOnbSgU$nQ4oPMhh!xY5m59p`({3M=u9b%?tZ#9>q zBE~3x>USWg7*;!1KF4`T+@zRkC8v;V3F04SJOECvO2Q?4v>o6V!u9h%zZ0oAtH%kS zL%nIeE$?3BYORVPho`r23u7WXB7;Jedkn$I^(InC#*b+QL}KG15tXx|eXz>{3DI-;)#n#92u5rc-Z%kq#gA7>iEi zR|mf2sEqkgn`a`__mFvu0?A5av??B!8?38Y`K08&GR&?a=Mum^u{XTsu!3_pUEKZ%+mU%Y=r< zbHeg`mTOhC5Hmae_NE&%A!0H3F$|YCDEr{2h2eku5yHJi_YONgC~)9hAw+0Z<6FPw zM@p<+NyhGO9QH4m+JC^W_6cumlg{Qw!EExx=Q^VwZuEJhvNi~wC5smf@;*a?^6=EWAeJ}R->4`>! z{}M^C8+;#wne8vQC9pfbyc;zv^b38E99_~v@H{VMi|mm-dPU3+^?S&~{&T|46Vqxg zaxc%YWW2D^&%$AOll9^)Qcbb*BOnOzid3ilB&c79wTT;BrvTP?0-gKj;tIfQdsFYa zxgx&#zt=naPVZpUa}rD=4sn*Dv*OMWI00_%BH~9gt>~IDN7icD4*)m7Yh1IWKT@>Q zoZkvv`NxFSfH|HZ`(@-@F!jaF4}B}izZOhchS+&nOJht=iTU>`RzY|Wq=5SpBppIJnE$W*gZmK1!b8Y z3}9atTz$22sP45ciReVO|MV(%yW(;G_(@P~?Gzmpx;#fZJJH2Ir^$cinoxW>HreMJ zL%oQ(jj2=)=;S2a=Do3kiRH4jGUjVKp5Zg<|HtBw5{x{Ks*7IlsQ*9N45(OubX?MpQ z06~Ytq>Wd@dGyhwyNR+67l+q^Dct+IQoc8q#@omK`fQEq_t{fPCY}5C)IH3d_ZDA~ zqeE#(1@0-}`14E%wX=FScK~FXlJ%kd3OUz#a*-?7xI^RRmBx5S$*Ep}4 zqU?*XEq)Zj!PLrM%el+Biv2d5P3SQDz@Tuh=vJheWUCXD1RrG>n90mxHPM}M*@Kw& zX)9dLJY$g~u!2AdO%}B0CO-!D_rXu?r5T~ZDp`rdj z0PZD=dQ&rSy0(6kcBG|=g(99#gWf-y{SpT)HYH71&m$mQk*1rCtjIWGITd}FbeLO$ z+Bjt~szY(fSpyNK`NiG@WjN_h^ z7eYTcCYeCdx)0~hV@*#UfQM^(Lj65`kidcPtV4BcismYKb%c5v?kwnm1WOG^8ne{V zQSSSimloT-G({V3W-cPgHfhPV1IlTbd7`P08K8A&x3MHZmU44x5TR@_atT5*r{z>w z-NjcQKX17h(Ote?I(!-^I}cC94eHlaZJk5fLmzFC(@)q3oKLvASvtExsf9tvSU}O=n#Qj;uPO_vu2Ho zw91afwt=>(wthN5qw`!q2aAWF?WH!YbY#+yPKC3PldH|3eDhy!_V7b3PQuH%+Xgb( zh?l7UyG@R=eIBH0?=GW~m5g)i>iCZ*jj9Y}WWw}(6Q?0X=XI6^#Ur+%xZ5GnR^dOO zt*e+VAVRF=PswO&jNODXyZ_=ppVL>hQ(x%TfV(%mCF2Uf$@|4T(Uao~Gn94kXd?Z> zP{Qt@iL~+oY`v@VYP|sVvD&iQ@&)_bvQx|%+zxt@dY}3#v-i4t_XSVEoAP7v$`J)b z5B+n7Uh)Fr<&anNlriFOPil{aR_>Cz!BNL^m+E2qR8MB=??l{HeoLe&=q9B_3)YrJ z%X$J+)Q?KQ--6I1{0@_P_w?kO^DP-iDf?x4kg|mTYC)>-bHx(tLhWkp04eCJ*gV1Q z=)h5ql~whKdx^G+{GFiqTbOris)g3FjvY^P@X8+n@YxQs6R_oy!QB$F>hm27d7+>n z*Y~*AZB^Y-DLA^>ePNd9;yn3~+cSGLhkD}h;fM#YxUh>LEZXheM{L%Umo{B;OnjhX zE^5}as#5PO&uZRulWp_xQ(j# z@4crVJl{NV<;y8T=#@GsWZ0-fzSGf3+-M)X`Gk&<9~VVn+eJ%s{PVk;9PCdZeml@$ z>vo<89$mCnTe|> zH%@ox%%}`$C9}jKq%>n@j)b}Ly!-qYpt*CeGq>V;a@l->g%Q+o)FiVO*U|4DG!szE zH=&l!tyKp~K`a zxP@qc$E!s)<1;B+Vy*CNz~|a#Q-)Cu@~XJ`FLoGq@I!k#o)*wTqM4X&ligY!z!)H~ zki32X`~+0GDSc>cf>>(lxZ1%-PAAi21QXPu`t_x`9EsSoKiVcHMu~*`7XPlOa_%k$ zI2~;frVZOrWkWn@(tP<`x36&-2gm2V$LNwZ`~aFT%nRb{)xp7@6mq9sO+tm6O1AJ) zRc@lm*#5gJdQKv_SU_wi$*Z~ySkp|F3PqAd`mHW429mccrFQD6CJ&h#2~D-C6AV&te-4+ktQfn`=g-_)(mb=}8-@KGjxJskN7 zCabizGGOL8B8$o&w0EVulx`=b(^AguecSLAkcCUUt8U#^n1f<77yp}sXvi<`GH)LEWR(H{BbmBRM&br)N)ApO;`4&hWT#ko>G1zzJ< zli4#O)KaC|H7hC*CSE3>_}5N0gr6~H=9>4qX_s&W{ko%6vzNkw26;~Lbdms{P@A9* zobLU-@ey~f229=th0af*Oz`_Ymx#c{$!gafhcdgb7DCc;>_PJO{-+RlLb@rG2X87w zaUEmB2Uc@3*)(*V+OM6Lm`x7W;kqV{o9t(opDsqfjd`)W`CTj!=wl#REZLCPcJ=hF zc+O{k48+*s(ERr;gg zLiH2hO!j8>_8rwanZvIzAADx4BMj{%lv0J&;V0N7NVqP`WUuO!odaoqzLpuR2zoK; zOIJSb6z+5$XMg7D@afxI?X1AxGzvELaXFRq154AK8Ge9o)G?`=!X()92SF=tmULVY z0#4Y#vUf`Gqo@M6yfL?3*kGJojISo&?OH8q9wH&WlAa-YF|Tht9A|V1>v6c0pf5y9 z1}ty2#y3PeXNb&-xPYxWb>-pXo~3DqvSW1&p~1qBjj`^O*zUEdql1~TNV=KZQn>QQ zLazQe-zfmCs0>-)aWU2GFe$JmEh^r0BCkR^#ydZVJD{gle;s&*QLiLw``4zQx+4bZW`?^3a4eJdjWP)y9qC0B(pfRy(77&utIB6`zupCc!L~ zpBzA^7|+xq)u0FoNH=87)eDz8n2S$EW1)rTCjUbQdBnk1Q|hojWHR`JP(c1l!`oc} zsoeoI#-Y47aW-;=ml2${qas77=;a`hn0xV!$FEmBu@PUPe2q2RY~*iLxRFY+vm~7S z!G@dL*X?K#5hA?9NQuRBoW1E)qN0U1S6ZNxYasAu?5Jsqhp90Lq19nAiT5HoZ+cYz zj@;be!E;D>W*G84Wy&U6Ov=p|EHnnq+3hiVCD1>UPgGADdd%^z?hNX5oPr{Ae_1Ko z1MmG!AP2K22X?qAP_TXP%(#$!3fah9NfO=T#5RAI&hW()G`&7n{zghyA~u@Q&j@TW zZ9UI=&koM6m4%s54gJ+BH8t!DyNt`=bAo_uN*BCOh`^Fhe*A6$ zWb{rHe6fSM#h!wG(s(53bxB#KwLi07rC-JA%>SQX?Qb126#Gb(GWsWkg_0-Gi|4Yb zkPs{5p)9?Q_(WAID4O+Ij%Y%~I1-BayZ?YVmFh^_++FzfGovRNyP zhjgL55}Qb`+Z3`;Yhjj-5G5$Dgm;095W%Y{f@`b!f9EK5 z>l1>pH?~Q;3%$Px2*VSYeoOlc@?bPU`;Py>`E2Ew^7>&?ER2x==C6NZgr?}07&&y6c)drabX6^p$<1_Extq!VW$(1 z7WwynX*KZ zParPCqKZ%Jd)*7{(t3x|*Q+2Ig;-iU?}Q@#B3j(kTmz5D*wox;;8$$;AfK0bXfe`2 z!&=Gmq)b6dwGiq+!o&FZw{HhyGD862p-T~~qKi|W`~+6;IijM;=ahSW@o1))1b5q| zs<^-LWktKffQ!}ARk1^6!IMK~Z1=gB^-c(L%yB8b5O>XORoH?Yv+Y(Dj@}%8pQ{qC z&ak_Z{>+~}l#6CrfCcCL!nU(~Sa@NqU>CJJdIOr$0joR9k$G^Iqfsp%Qn!5S$>Djo zH+0-$*Gz0NL+p=AniSDoq#GZ1{;YVt!D9lUOWLHftn+j!ovEixxyk+rl+CoN02VDn z5F#mecgtA8(N!T8BWjKXm_nKCDLP4*Jyo_B59O*x17&mzP$a{3lI!{;UScjt9ML!v zCtBZp2ok{ULSK;yXZdsA)zcGWTzin{ejiOYkl^36dU!##XhPxIvuiP=;6#= zx&p6tl4F=7Q2E9Sc9l4BS!iTp(%@@OI+;uq(Udc^6JAp88+^N4-emGkq#e{S&GS{L zW>S}Bq>>jsWVxbqR!je)97P?+pRV_?l zhxvP_h4S$#O^yW%87tO58uoXZTu~hx!DF@~bOCN5pfE*@{Vj0bqSRlCWg3482d!af z3U&dX6fMXsoWFlA0C9YG^IOeGDv3-3qQ2rBjb^<{sZ+!%{Y|)R^hg1ed+T%U9d@z2 z*P4I;tBOLf(NL($xK;F^)$iFGQNH7>!DwPT_24$^5U)!RalPskf31e%cL>5wxi51Q zxwBnrK);E-$P*5r;SyO@QL;=TnMb6?UdVUd z)bAssk#61F)Jd{|R;2?9n6r3p@?w9wHS~xC7~+Yhk=6?E3+uXEbLQm_bIG6uJmwc! z;l0D@uP1-@X6PoBP0`&8#J%9f9MHYm{+!a-0~DiF?Hc^rmS>_=p)*FNcxE|stiP$p zRCOKiCccDPg8iP|y!b_eOBqOk&iWY>y73>A&H3mD?A*ER7l#a7@SJd&emTl4(IxuS z+@@a2p4&L=ZFxp6#~?bMHB0IkE*HRUI)b(+m!j@wix3|n{S=U;0Js!LwUt&~-~V!aZq}Q!M=0|Jt*8u97%-!8jF|$;{lQ zF#d(>3SGys+A(o+*ZSvZ6G~(an{XRr9550=bDtmXOHMsT)8#*?Zos1zob>k_676S| zgCsK#P}`fOv)`&P`0(i_fn&rZT*4+&DV7(y)c!)s*IT)HMevM`@;lLQH)*~-z~0{< zr;@u_y<|%D#cC`0<*pa}qL#@oOf}cmVB?%gPF~rw!}V*^`Io=s5jrC-CI)x7PRJs? z9lGb0b^^km1o3xfm5A2S|KaK#fHdc#ci*$+PqDg-WU|-d<2v_wLHl-Mc$M41~nyeRISdgd~xo-A2 z%>O`He9GO1+E`ncJtfg?Zhq-#{R)gOd8Xan#i?;?q*>}S;!%pdMj4V@2T;*X$*pA7 z)Gn8MVG3uD0|7rHQlp1EHr5eXSErbhRf}uM3`dEjNns)5KfAMvmHF9+VsHr`2 z-(qLfg?7iA=KNBWJRP4N1YAoCe>(8gHjTB3hqTGKk<_7#M(9RLr=>d)G9C&TamO4) zYCJlLms1DR%EMn3ThpQT`I?52z_r|n+%4+5Y{NwFIg-8|AHK6aPED2dJAsW;Rn)-d|EICOgv27 zOvAZ*aD7zq@_zBXPsYaliOCHc3)=YelaWN@Dd{)nE#kiN9pe@4L2B3No9gT9(7lGk zhn;Fm<;mp2B4)ijyx*kLcN#3K0xH#XG z!9=*pn0zd5+fz>+^s3|Mwb)_YSUuLjZo{u>uH!G)`)Pb#v8nY}fH*N8j{pr$saRCu z)Z*XhvHNSlP1#%jczMr4JP%FysHY&sj+fkgTA?VmZdiL_7_FeL6=!1!v{l~_#f+&6 zQc(s=W^kQ$=H!O&XXTJDoeokV+-pm8KT__10F~{WVE6;o& z1qOP-ZxXHPk@9<3Kx0X?*EF;c&vI;u0P7tO?b$U9BJB*)4u~L0&+v&+^$9WsxY4iq z@+ufLYGQF^m6mdbv_r$3pJ=6$BGLX>cLS>KFvRn)QL;auo&B~pcd$-GaztV?PUO!d zK5`if>Ov4!IyH?YMVAtXk`?j?bX#g{n#7jEEn@gc9nYy10IXxEo!5n&W-bLvi$$|x zpke8*-EKk`t>~S>>?~&4o+m>yr0GC7x3h7~T6Jz69h*yKT2+=|(_34Ywjbj(<|1dK ztm_l3co1{^W#zp&71jPaOYB~75q@Q0rJH%g1rV5qQ25xSh?pj+qzLT|+J)P~N zKN1fkBb*_ofCXPCp>`F0jb5_@QV!P>4|#6D#R0Q;ts5}l`v%nfK7!gTrpjxQDk zDoy$2Esk3E1<`Yd zM~L7+K4Rz9{9>m&(xtQz_@X4*lx=@Do{d6e96V(A%T$~RVJ@0xv`d7gWjkwk~0M?dD3 zBk3ZSxa7z#wVd^q+!6rUR+F0(>)!TULqIXo)_V`{9kx+$-oI5d&rjWJ!%MaQMVVE% z5AfhkA%{M+bPDm~VHe!S&7P87o<86~adC%x2FaIS&g!>O0%qNv2fR)Mg72EbDaSIFAO z;{)ZY>lIJmKhy(^*4N?G)FA51(MMAnZ)7`k`oYpARGgr>q3g|XXQE{yHF-k$4ah;F zN(jFP(VrQ?tNpIG7M|IC)>asTY6VT#KQWW&E4oSN*K{E`~S z(CKHWk?+Y6S_0W+JKl5QrW(0XOy16Wzixnny_HkwH65iL6by&Uw*c1bRc zf`We~;+1O=g)r7v}=H1OqRvh4MJw86& z)Xi&UxjaG5BsTq13P0K14badI|2TM3euiqEIcq-lFld#(_U@PA0@~P_`KhVP4Efl-jV3$ztF0)?Cnykf4;-Sk z?ylM*+J0xV^5lLa%A*?_AUEHRC~r^KcWPZlZYlp24qyt?&S~|+hV)b_aJlkQEyvsB zF@HT#ovr1>35MHixjN^;`S(7ox87E8nl|a)c73xkpN;4i1>oM`=BZ2Wo@3!K|Nbd+ z5<7KUQZ=Pi!c#-cA?_o>-d?x_N!LTnvI@ENru<#1eC<$t>e`KD6gSh!!hPk9#1&&% zs&!^1*4g0ZY@&UT0YxAgFUzfM-0e|4Se==1)M5{B`->N4Png6sJxWw4UH#rBTZU}@ z+FNFc!ODpb4bVPDYe`Bg=#pvN9$P(IFcMlfna8K42K{Axs7j+%5hgcHe*aX*CLk%y60PJx!D zj&|2QKjyN=&Md|0rMFp7%s5Scv1P^8ZNiwY2vjB>2;d|QtI>7$S`EdW%brhRPH*BD zA8nY)PIrmc#mzsS&dfoUouYD;yT!GgLdSJ>gr_}Y&-{0!!M4Gf8*R?6w=Z5nXXNVI z{^Y~qcn^{D zoi-Es)iRxZrs`(8>J>s8uSm3*lvqkE70!^fD`57>!jP1ni%4 zEu?8si@zCF=5e6C;grhqCi*!EYqjaa3o) z$!(U?FFn;e7AEG(3vgy#dBJssn*FMM+O@Jc5ufYU(eBn%qr!|z)h0HM#@(I9y=HG- z1`sGl(bW?A5{z=$QyK#2Ru|87EmjkcbRjJ#QNwGv=aQ^S&qm<0_Z8x6YnngLp;=_e zBh5>)FVP2z|7K{+#Qfg zm`JeiEA1hm&cW|@czsy)Zq-@8)tTs*1`ylF=qkN_2*aV=1r-xJ_|ehS=|4VWYf4O?{iLR*x@#^&k69JxI_KdESsJR` z)wPc0(@6q$<%cNl-(Id12BU&`r5w{P8xY9qNq9CtCvX+ z1kuqF8Z8xNlblny^i+&kNz%)Ze^D;lt0D-3vr2n-2AY^8ZL$`Dl5?g-Vad+1CE`(8 z78$#ps+&vFk@Dq}^~Iu-3GeAD7ZQWo)`14Wa}fp-?u5{SYQwXEx&~z9IH zf2$pYtJdp~WipI{UHAOD{!;;`tf-Ug-A+7>R-JvHk3U^A6aDPw4ZXK~f`4V4xPvqUQ<9jLd7FE+D|a7z(LXeTbK{@poaB&d zn$a(Tvl6JAX^yF^aosr@OJ}9WjcY57`S5U$drc@+nk7SY?6(M3j=|e?7?%cdE(Qbu=-sl<+h+c{$}IGH z@WW}44Ud9TsV9*{fCus;M-;MPPg0DRp6noq+XnKYjEUK5q23Fn$06+&Sb7$hdhIQj zt}%4!Cqzxc5JMs1(wjO}@0BD53H=UJD5*d-E*XJ&vg}Z#y$yJqDYqC`{={^me#})P zE1&du86}7KBP-?uz;Bq>(ZOM$CeFgVxajbd&TlYuT+B??PeI-lSPkm_BaUACz4ErX zoT;z(k;U7AEn5~--35o>_tfF^{4{jpS|Y-AiQ?_vWKDyXc`{yQI0EX%sfgozz~sjp zH`kYK1oqNx7m0gC+VV!K8VD(w_m9iJO7ROMfHy2V9+3Z_J(YuOcLf#lXJlU7*ny4|DR>@%4RQ z);jS}k{baE%%w}9pE~V!xJEfza*o&X_+qJMaDiUzxSUQ^xXQk=AO%nHg&>naqjv z29AKW6^J8;)Xf}c2Nkv?TsYeE)pHyUz=I>k$ktQ`kU%P*)it!X>DW!?s7(q9m-zMO zs%|n<;Va$-b5%^0%U51kuszD0O#li?1Y^;reC{4uMRgY(b1P#9ogR zbWgmdg5_=^r{xvvhcm?Samy{mDcY zE_KQz@kXyVr}$2U{79tUJLvX`>A#_y3>sqai<0|fL;ot8)GMPa72+14lF}3}?BSN6 zA5$RN5Wxu;%_du>MyG(Kp$YHG#BPO$N74fUzDEnOcY<2O?&5N}{#ntb9Yz=d9;ycp zOEHJPPMR&sfU85uv^6QRx!tcC$15b;^&uvJriAuJYLE2&J^0f&Rz0*(akNOf>5+i(j09PQR4qWp8E7%sJVkMx_2>9N&a<@uEIN#hjWJzgidVAja0_!qRHMWu>a z?~HjxF>#I!PeP_~I#xo;qjDNUvD-JF##E@07GwtcB3Tn$$AlX?l11ZWNUPgiK@~B% zJesn{VEvKkTp-H=6lT9@EM%JG8^b9BQpHr&hFAp)^j>TDiduWloI*SUy?GVD?CNQz z8{>8R4fryG?nLo9?rnIm5V>k~C<@#LkXhK=ArsYDi82e}5Svo}d`Skfa^{Kc#9n8f zoZl@Qw!M>!RrUmV{Bi4;GC};oJD~`^u9=S-#f4Q2YzEXm5J$dp-c9mC;IW?PRq%`q zVR=4VgHGB7nm`Iu?N|@lih{1Jsqh zU;sty#Fl-K&q8{>Qx8uHwc9+GKUfUjXzbxbW5m@^-2}nIo+hzF@KR$j9USx%{%tBl zu!bn0Q~l>i{dq8{SSx*?kF29D4RPU?`>gnyu7J(82-#XHt8$j+`ICRdv%bBxhn%SS zF&e84x#fBM_Gha3kDH?|TcY+ygBrFTjjaTDcA_`}E3srJuDcgELRJe^S|D$|)^O+T+28Q9lf-D@@IrD_D`2l3x6k zO`M|Y2-dt6i6J_`zl@8V3uOxg^6DL#0>;YuXhm)li2{0EEW9(D+jzOV%CV^5oTVaL ze&zv7R4$9`%~rM+i^QB}i3@8C^^X2aIQfJu0)iF>qw-g53mtalLrtMtZ~}YkrDA9B z5ycq=Al+R8^z;jsHfUqz;|Gmg5fM&_=Vyv@4mp)t#a20hf0wcYx{ zIH_Ag=Z4e+OYIL!%!aDUf}+iS0%k>3f>8!-KFadAwEUJV3oTln`8!JMBZ6Ii8TdM_3{1r3;Jk+~K!={93 z=Xmi~;t~{Fj zs^(_^QhJUqb3$I0WK4W4v(hm0P%IW21m*$%+v&HU%>fke5wi?ye}t{Nsl6b8j4^#0 ziQ*Rd+P3IvYmo-e6%nDx-$Pw=D7Af-Q_J3XDPc6hpvw?o_j z_jAD+GGQkvtiO5LG#M?jQ$H0#D~&*s)`2~B&8ju@VRhzFB+E#Ai(WeT>B%Y#;2)+# z5Z?g-U#gmRzdx%K- zw1wz4;64x$aBR|*kc!cIxJ7V77ASxQo|cb;3aC8$h%--Vjbi91mLq6U)G;(qRMWW} z6_`^dNgxsRq0iPn1Jji?8twQ~3F%W9f}dP-!$T(4k5Lx7YR8iJp1`wFXP3lIpOCcC zu3FsUCrc~ft3u1Q0J!gx-+bmRR!q?fsgzib94W~} zaai#Ak7$2+r1i662PF^H9Wa0qYUDvxgNrvYe_|ftJyGdB^Q*Ea`r#o@QW>2(A{(MY zJKmBqQ7(c(y8T|&(WlAA3r4n#b*&>vTjq21KOBAlN{8G5(+&8JNRC;xkmU!i6*7$mpYo>_IY{0NbKz_S!#$ zN=FrNUxn`*-~DC6-!JQsv;pfL+2nlP5uWtVB?s)$X4PGOFE#>@(NO=WVt2fxOroyu zyoR>z!}Zs8%W-lGL$a9l6&L)~2Hdda{OJo0rU`A3h+o@N+mE|(54CM(<&g7*m*Z&C zqfv$|6OH+)n)uZfb-)l05-(oKQ{r`_F_whp0%R&amIdb`^Ud#EoFoRA3s~;9qr228 zn$zq%{e^LQq{|%u8CNk<*#Dy899cv{MI8fW1H8B4pfv@zM=Qz%XCvH&n0dI(>8#;E zH@!HL`rdTvn*Ymb1=P=ezPB&`*Yl>+an*D0TLaG3=hD;~t$NPK;Mcp|+9==SD~kN4 zv*p__z}@om=Ur{U*EHZGWeDJR*W3MF0X<0b>E!t9UDY2T3d)+ELv6sgEx!LrZN$xq zxX<43{mxR{r*@zJaK^w#GY2?OqH%E>!^zoYsQths)b zfO#Ik~rZ@Mv)nEo46QLC7f{q z%#JFO`ms_(`b)BzGz~0(@l0Ayp@m*hGzV<4%tDl=$x4CbNF2=+_=9&nhGh5?!ur5t zaR>w|m;7y>$3mVUQ7`;bw`a~Yg?Q3*ca0?W9WNRbTj6p>8RiRSF;0R2Ao`bI@Ie+J zbRroj)=Y|g`HIHI=@~2598EutjdTx10jU&HAwgy=mAB*3gm-N{0u9osg5$|xiuoqM zW(P|=yIT9=isekDDOO0-S#KR}iiGes%*V}kxuPf4yzf?Aj1mzXTlR>RBiI4^Hp2NC zXg5!Or>~Q%Ze)q5j1ectN^T`P8AlCBEu9gdhTT;rgal&9g|5UleXUG)o`mTsvo~~3 z-9b>0Ajf-d-UDbe`Byx41Rb7s#U6Ua4>xhI91s zJzm<^QDgI-f4-ihvm;>jt;`Ejsx((on zHGg2pg^~}=s9%nFJ02bh;7;9z3UfsEm+pfi4ZN?*Nz-q#IAH=)O-v+mxcuAJYGS*|`{M->_+HY?I;U#}ErZ7`_f?zKvdvkVF8Og+hx+DzzY^^QaKBbsZRBE=nb- zO=*sBB0qA-v3c6rb?N68-TP)TQ{2TCs+n|cC7#-#I!}e35jg86`3m{b*rIqck{^** zCM}*;>Q+t}ZSnw{JZD|mdmYU=XIz)+$@ePyBzZGGn!Q~X&adm^(r*El9PMuvKPc`g zxbf2A<;6>j5ZrmRoTN33M3#0N#fSE&ZMP+$71>Un#ex`V2`KrlJal}7zGcpsEz4SW zrqEhZTh*Ucucx+keP-_Sl;u^ojkcpZ?u68FyGI8dLDCq5%pG@Oy&71e21@5@hg&d6 zZXJlD59{IOV^OW>AtV8YS<6{+?K#;dGeNYm-`rFw>z#6u7NQqaJ^hLRw18OUNRgBD}tf*5-uT~+?R zoz~ETVdv&~<-x4|A`NwcZ6$PE_64r@>Ry_n(*zO~ghQ^(zAmG}1972)r z$rJgKO&dw;CbZ9!Z(h2;`~i6f`LE7-;{}uAQG9HHMwvT8o{4-lX==LN9E5o{V^rqh z_7A^&&V@t3jtU_lfrxZ($R$ece<3^GInhCQeWwcfkBkUZnEK7 z04mP!QL4GgS%)!4`(AN!ZbbYk5$uP2);To=tAAW=J2%yJiYmTp+Im&AW4~H&`f0mj z3a@@IcbQ8et0ZApUxdE5NrGNy=&^5%z_}VIqjh{99n@0;04tfaZ0L-eu*xuDK|kL+ z%DQdSN0vE4>-`o7=@jl09b^D?nmHH65}0oXhjuyZDsZ`$RK=gt?c$gv7skRD2IjE~ z-DSI#8uBOKw=8=6byp57sP?3>ljA!5Ui0rges>ohNBL<7)r4-kr-~tUKIi+?*e#u& z=G#8~t#=QA4(xtRp4Y+I@Y}hW8tW78XB^R3QKJ1Z%_ntJ<~vs;Lwm|>9^?G4WO^}U zfgCz35C?Fdm7^j_0@H*ZHJXUH8x<;@L~@B56uvTrd>ewQ0h6PJz&xZ2kHW(bqdxtd9DV4a^y~*tY6n^-07tZkU?83bD*tC zdb6VIme_06q&XJDMRQql)0@^7T^>EUZo$<2%Jo;sul@bwgZsM;kMXcyeTbg;?FFEq zwZnW}afWZkany0v`oDZ=IjXu#Zpn>bb{V#=)~>dW*N(s1VLBWHSXJ0C63CcarJ->Pg-Y$9~P5_>5zoqCjl(|Mu_hbZg1swI;EIeECPlzFgqYduas$HSN zqd7!H2F*z@EpU#^UfrkhhutLe?{P}VF6Pn!)Yc^d0#dii;niz3reG;5HDMykyR>z^ zdXvAGsl=?j>YH0X`4{Z3p$^;Y=>5zG`qhV*j+~bX%CETlHqv3Ynkl9E&PV*!y8HM- znI^W-ZZKHNyQM8U+_SxEE${9AnRsv)h3FsbWJ>Yk0Gs?95-vxqjM*YDTya-$9q4Dx3mGWHCi&I7MuFP5HKuk#ba?O}KGCDUVZ3I8%M4vv|pk9*X- ze&0)7#Z2I||0)4%bnM~<^o8JJZ8;oVylPqGbje{gT<*A6ao-_u6QmF*5iH@$@g{px zaO=Y4IPhR{C75o;GWw+Xr1f*+3b9$6FcBXKf+Y(}=`1OawZ+~IaU_b!>quGLa!{xu z@X;Q?5|TV;mS#$xgV1xlxbPiMA3JN3WHEAqhy6qY1^%`OXidBTL>ZH$%D7c0>ngSQ zvgQO-ouIm)u6Myd=;jo0d*59^q+vLd5+G05cY0}p-ik>=O^j56sW4X;F>WR)g1|mR zmH%bTth#U=FVIVdiNveP=kbQYA1ae}a2tsIBf<7l8lbnTv*Mr8xST(2gsI-=)ct3G zrRT+5$$1Zws)!B_020(K)!0fKGIQe^HwlQycHOW}4yoIN+MqLvVO5GgxMk6#7N0uzWClZo+?FODi~RM3oTP7PKt-q^7lzFQBu2*UqiL1d!SQYDB2Z6o3>{gH=D)z4g%5 zepN-pWol&68ao49fJtObo0#@i<%Ro@)^CZU!-ca7QN#N=d0fksClG}J@p9u?(BMFg zlzTv8jvQgeASixEscR;n@gd6tE9Fd!7lv7SfZONLhiTt%4DF#p zP`K_}5{ftj0LwMle-~daO560?Zd`k3`^I=*hwP8PpZBJjqqd_SYt0D89d0cO+ivP! z>g228Jy&Ex;l?Nd2D|1j%e)E}0tXDzp0Q)6bU1L$h}#vN ze3LEX0g^w^GtAt&PlC|fM7zFoND&EKSV#xalnQubS9fRPG2y2I1fzEXgmNO8&6*9X zQ<0A3#|gzOnMJ?rc15LMpH)T(-sxQ-T-KJw%|rIqG5grKqC19L%4~?s%y>ckeEd6_r&4Xj#nsn+E zRQhZNX~)6FJ&apDAAlE5=PHH{cp~(zipOeAaS{#1qsyyHtr8LQPr5J8n(6dPkzZPB zoP_*$Q_(`h;}FW@z~@genNrxO(PO39b$zRwU{b=Kd^rYjY3g0>Hdz{Q+f%Mf8-q(= zDw@Sj!H>TBePUa*s$(y;q!h~C?5JyAYtL|YU5;?o?#laVV2!s!1h%gQ2&!dto@L#I zv>Ai+T3+$;wzbx1?^TFDV70!%j{he886d2PY~*-X?ZSxgxN!jzV?h9N(z9gwXIfIJ zG99lLoFO{}nLW$&X z5UlY$dNwhOKx5pUM4N?y1ufUE17IS|^zX+j0WcgqY6 zL$hDB)PCjF@HL?pn&MS7=QgR2f|?nxpNRD-f(~MVujyF@@etb_&_eVg5d?;fQrcrY^+}O z-tM5dfT_kX$h6H}J%0W)@29Ph_Q4Ni65C?y9R7BefTc|kb9Ny*N?wT1E?-Zb&XZPN zyE5Jv-OpDFUwp0(n5nujm>(i|ay zMg;#|1N`+iz|^0jC7y)A)XzdD_(t*2?@3vw5f=%Om0*i%9mz1#A`vVvij z1={$BA-~#iL<|-3hvqJx~e#(MW#U7ntk^*dcIzx98q<&>-_T_9Ye z>D=-I#J|*2@jrcAQm-^Sj<;>HdAQMYdtmLyqEr?j+QxxoQFjr9=`;yM0{GFTx^GOz z02lxpg(|pm9rQ1cx77Iu-*^Jyg249~Uz_>_dWAa&{`}?{uk&NDbcY~f;#oe8`g$B+ zk&0PaLVNR(LnCRqxi?aUg(aH_S{a? zFh2>3{4=Qo;_$d)?d!3dT%v6us#*QU;`j4Uqk%Ns50vn%OLH;yeAF`5549ZRMQ2|z zceDgtBh9={=tMe!4qT|VThn6KTM<^1ejNFwk1fjw_SrbpQ?uWo;6=8~NqVkipp48+ zNxMI!w&^h|Awfj&nO|hs`!;(vEK^_K7;8KtjYuIjI@zGCE`qF^WBiMDo7ySc%5_#n zgW?fpeDhMBE;%mOTXR)T-_s@zH3MPvFaO-du*?%J%`(gt&Bi}WQe6F-D9dc!R*w}r zj9VPN&dh=SujK}N3S*iT1e7W1%at62gZ-yB#7UBFzW{&2!hwd1l~60B=s6z>=5h;h z!@ET!Vg%tiZ=Q#8SyLjDqs`n#xE)hn03MtxcBrRe=dQ9v7Tngjn{S=-q*E@ZHi?J* zE?y?+J&!F|i7sr*^R~t3fB*P4Zbm*h(`|{dM}(Qr=bbVl2^C*Y#h+h@^7vd^OFa5w z`khF*p#siLT2VsRUfY(BW&3ZJ?LR9c zNl=jj3qgS)GXN7qbB`jkfusFJgSI2ttKDg9l1Kt#+v#tk-#>|-YKsPl#z0vOI?gz% zC|xgH*1vwHJ@-8KT>D;smPNjIuD73aK5sZvLH}?DqX~lwXCL8;`k>VOQ{8ADXP+pHx$?Y=&5ALl^nqCl3d zlUf^_!UrZKZmbaJ>t?!LiRxd;Z3Kkx)<9QV?Y#Hs?9&!`agWa+6ovO4>JR=nDl5Gr z_Svl#lN-OS$%lbpoKrrcZM}2h_9X9X_aND01S!-;AD^ zCh7)#mOh54)!DlywgQ^rEk8u1&0UvMV!I*VJMr&5qdL`}N8PQrMc=p?r|D1ic+CI7 zZEk}`4?NcM@=B{$qfT;zaOB0SY&ZGHO~*cf>iIvk0CCojQl z3!-fR!`Rpc3#A9zU=KQ)rl&QS9-zkuDfzW_f!#^?5AUzoGNdsgaSn`Xw#(ESnO59R z@!3wg@}%B4G<=j6C>96bCjf-YELTkL9OtA2E6NE^m7tk%>$8YqOD2OSEP)In9!51&8G`+VzgmtMKFU!Ap|d%H zy^BRug0K}qv2Rf=1f$A}$|3Ozfp0N35I~KvMNu%3Cd2uQEQ*P={^|u=fc`4PyCvT+ zq%K6c2&ap*a|r(qc5xsw^T(M5Jq_0z6*xcwfDw2gG6#r11R*f~x+OsLml-5{L29B5 zB_#X;;zSggD8UT18RUVA*cRqHAVe3SGa?!yhKeFDNZJ`A0~{2gI)d$O=VSjvw9dy> z^ye_5-D_n)rb0T;hiSkUBV;4Y^;c8~;=l$QWJJX&Q2T%w?w1%OD;LIvd<>tQA%ySu zzzpF479;tiko`lts53;47PTo^fshSPGR%HK`)AF(2aftvfY%7JnNThyVlRy$X3r%( zcm=v9978C}1AuGRXEI2a9=HOBK{zGC1JRU^=g1}_nhiRScuedEd~d+D=W`;WBj89h zff&~>Iup_c)q#YetOLS4G;K&ZA#@S28b1Dm2y{xPhVO$(4{J-@qe_qSV5AY~0;5Cb z4_ZWB?js(QHe}p0trz(~r7LSg_d=}mHw`~W@Wk0W|h+69ge#qB_7M0t?t1bCnfWf^wA*gO2viTEM*!Q84v zX6;%HuJ2&q3a@}H6K}yU6Kw(G)Gy{Z@GfnM-rB8%dmz{dbz#^EWWG~uA+QtMg=8VN z{s0#N`2O06fM30WK8SXrK2Sf1JWMtcK9E)-KA2V#zR(vw9(Z>`yD)Af_dZ>Xo}@jx zHc07Qe{UprWtl$TaPJ+zTi%nPn()thI7fOO|271_s5Sz85uZdO&+(T226nIWhUTL0!Ehzwbo(!WPKICn*3$#z7JMbfS=okDC_*-8O_@7tO^ZE59bu0Ws{oCFV9gN#Cy_J1Q-$U{MzeNFf zpxr8a$bNwOh&clJ#l3KL1$ZR?u>T-$;W!}diSff)5qpFABYVT@cI5ZYZ&3$nnd$zByLT z$0_IdWuHCltCSIxaeZR`oGO>&7ju34nWfg7&-MB1e3^b>^F{gpi$cy9^v}`%67iqM3Aw%@{q*F= zz_@ry@Z&Cwbv#81tN7GKqgKh0szfI?$WYx$CUDgNs9mQ`?Xfb91n0AQvQVTFhp=R{51Pv1@hzMm->q18o_t~j$~vR3ad&T! zJnWncv+TCeJeSP7&x*gj!$jRNN^z7uKq9)s!_Qt=o!6~Zyr}kmc~bHC_%D77L=v{l z?~AWo-a$QCi@*l-1Fz0>NDLv({s*LO;#4uaS3aEyH`g+O9fv)BYH_wH$`v#oFFs`7FgM489MUehEFwnnnr&^bO`QF?e@~njUmo;bXpj8x$3{yf%EQ)?OlKab z{VP-o?2BJf;&*Oib@zk?vW1Abc|Ve&+{kLqoa4A3Jb)fNAYPFcu(j1(PE?xOW+!Z? z@!2UHDyX7N!nid}oawHe2%Y#vjeCYf3pu_bb61AruS}I_<9p|i)=4Q**6OkHb1uJ$ zb`+atzfxcjynH{u^R>Ri@bUJMxU&!Ytp7}wmq$rrwu^EsSBSl3|Cv)Q9;Fc=aF)F_AUpo8mHf! zm)28gRhp2W_Pfw(oc;VN_x?=gW$|+kYSx1pV0jendI!nyNv`-H?W5TMbD?e7MuiFsH?Fx1@|>-Xhy_?Ur?(OAr=eN#&4esv*USrhUgSdnIPO;CCuHUR<05Sf5kV*L>Yu1I?s z2%`U%Il&tX@q;ST;tTMk=GPBNk%?QAJyG<^rzM7`ZC&BA%u|FfD+s*7c&K~?(fab6 zCy!f3s5F330%Ltc^5ZAx+&e;7hNjTfsJ3;a^dyl<&xN7+p;VVq?&MSl#D>Vwkg&E1 zF3xxyjJpYku|AIq8iqXo!X>LvowQi@6v)z=n)YW(R{CrO{xj^TfF;->zIfjuZ`9K> zou%1OeWCpiEP`l2%Uu=R@kp0n`*93{m`Ijk#cNFygTW6qL}`{wN{7rDvoBqoFMj@5 zD5g~XjPAj0Sa(IxpStu6kp%h8VKbjK>B#t53!KewNu7kKYpB9p0?CDX^4A#ZJvua z7Zv+(B8Q32kC1`&cK2y)<}i1AAf<#>17tN&oH9jCu>`NfU}4G*li!e}|6a-(i)E|I z1Tt0>Di)5igPAa^B#1QjGjaM)02LeE3YdewuQ87ODhUk0swJdK?fNNB2?44rM(_Nk zEeZfP(*L^s=J}&52eC!glOhy_;Ud`sHz=G4z9ngCjCZ7UXt2JzWyx%bQ)5XnuQ}DE z&6+~1M^@jg(oxtZgHU9DTwpc)^~+$y`8=<$F6q>w5R51-&`YLJ_$ruGj-u?-7q~Ih zryYybn-Zr9s`R!d2Zq`~;Pg)9?2|BQKf}G`)P`r12F7F*@0&|bQThB}?_}>Bt1Z$H3(k+iMIlx>`^#oK zk2PTH8Ha8|@B-7W%UY}Lhl-K|mpX1mnPEYMFSK|Hqp`6W%U0$b5Z@yWN8~iYfwL}| z`fbB9@2(=wA@xUp5CPp2Z%AvMtms=ZsxD2gU{V6(9#$0HRAt$0^=X7%vx@{e0{$ud z(cjcV<;u0fM>X@3WC7W;;k>A1Se^G2i-Ol3L54*90FsS`1HHhpv!tJzxJ?Mp)1SZs zNTj>l?fP$3S&kTVA!lVWU+}k56feII{xn9TWmMD#u8wXTG;KWLT>6E?RabI0_In|( zAUwA6Cs8KlO*~DMQr6^7crm6>raX}xpPgN!;~2?Ky7M!TUHE9WPCXUs$5TAD*}-pI zv}1}7$6gE>O5Tgax>nBw1c^f+mqF`1{&!xSg^y=rKSk7bAf{&@N7qa2B}$&(t&^RY z) z`!^G5VlZ;coay+E#wD#~cozPuj+2`R4tJ7}IJB&2KuWFZvVK+LKCypGj zagm!Tq$P^xLv&#(ie_4LN^Fwyn(^D16pSy5A>RxVKi`aEny}j#QyMFFTz7q>dS^so zV49v%Fdr9IZLbuMO+z)@BZH(eQhUo6xH>DCh81EaI3xY_Qn-X)V%EHM@a2m{_w0Ka zn=ho|8T#d7!0a+U2@zOdo$U}>=W!(B!uqe|?Px2z6UeVOD}A77A|)5izM@A6LUSW(5Oal{!LngaLp`DTqD2Wee9>0Z7ol zCP{h5V{-&_qK#>1p@oCTqp^WEC)ex_wVMtJw zlKLXS z(bv3H{Ji|E2}ADFD4v#LKCw^Q36~^Czgm3#StDlbKPK+@t0s;_5Medx5Ih|7biR5f z`->*VkY_emtiqsLs!heW_<3=bP_z|l?Qvd3QRTc|WmbI;%k*F0cu#A**vxxi0>XUS>uH`;8tAcK7_JMr`YX}^JX@7cerO3+r6We)i?4lw}n zqJQ@2QD_`!@Z)8S zF_y&sbg8H4jQ%j`7~DvJFs#%3fg@TyeX}cw*-q6Mkeh>fQ2qW)RJIM-t~2BU`&nc) z&tF@Rad-Cof|4K%?hRi3RVYv7ibrZS?n@q;{-Oa zVz5CB+x4MU=}TUVKDRcm;?%>SE5vr0D|GF1^ar=_q420xaa-!UqN}??N4Yk~qg;y9 zSeMs~x67}z9AU#2cSx0c*uC$4QKQr<6IlUG(zm5=8~7d*xk{S8G=)h0T|wZ#5&DF$ zY%!~^Y)*Fy+QR%NLHP39e-I)5F`Gu;W4c*)Pt=@cHhNeZsSz#{3=^SjY?-5!P7)YR zuiMi&o*|uGviMIE@jbhsD-I86t622r)4K+JO(=-0Q&c2grsEn8_d!^B$?5)sh9OxG}U+34M_MShf-Z3wMFPL|t z!Ms;P&(*$+%;_{Onyw;b-QHIw|qes?~6h{JQ%Gae&xiqA^ig+pCHOPkDfg~mz)o&HY%BgL6 z2RMbkERRD2vYKKDqo$eo5-xC{*(CKt8_60&x#kKwAsjX+tzlO)f$R%Cf{3B#f<(`v zt%$dnJXu9t+dA%e^9j_B1QV0Iri9n|C;0QBSdU0gUm?R|3cODWjxF0RDdT|(%c7mC zl{w_lC~Zw)_kuSC-8v_~2QA3K6l(S2(cBr|L!!rA_gziD)Tn*O9c!sjIG0I^60JE$ zJbKY)*KYTxvA3;YwF43leNFY}6`{Uf0wWx{~!IWu&W?$*@wt4C0)w6%0f zq;Z6P=tJOW?~-xXD{B@}x`}%<4y~G4>gkELGuC=C|9}szmFZnTB(Lyrik^BFjAqQb z35@u*FQ(M?*DIgcEbym$iL}$v`?GO2cfId^V^E08NdKSK^8-#oo~9%nMmzitVoipQ zG+1!c)y^uSde3Z5usino&$U=7Spf5@8%uJpL*Yramd~HmR7AqjhFvdp8QcDuxF7}# z;jt>>eF-v*vrIV9P38smBgcdwum$<|sE_;G{1?aBQ&UsCb>Q_7C3`kXXyiJATQ2*v zOg-eye5dE`-x>Gxv(7}ZKNqoYAOdZi@Dtm5L)5%3Mk&Y|Jhj_8p18)<=hcve-x*)h z=XUY;32$kZbRS&8sn1fv6?974wUuBG(q7Or!WW{KBwCGc+pM>4C$FdYMz*qmm0PFr zzbwRBIX1-dtff1hGcIaW^Ow>IB?wkpF@Yc$OJww>l1>9xksLFGGeIt;YTgu%}#tzUpA{gNfD%?zAh@C2wqEq>QB1G0s0BJTGx_n{t~Vc`oSlq zqtcr~z9BP2wd!SE-Ff#09RtcrDaI)8Ge=Bj)un_5OB5DLywhg=P-9|nV-TN=?V%7A z0d6ZrUdCT9ezfXQ^Pd^KB9io{9^2<%iA79NcB|CXb=6$X?zXskpaZ8q;XkoIc5;5gGi^NdA0+Hqj|LEVqF#3b%i_p@Ewo4fN}saJpvv4206UUT7PuA# z;kC%m2q~ZX)i}j}4;Q0(?x3*X;K#|<_Hsj|RCNw2n+U{SImbf7&INFU5oM~ZFV(re zl1>s*X9n)!s79=*V$m@$)iB`v8P#XhR4v$4q*KzckmxmK5f+p0QfEqJ#6yErcX+M$ z7bj_Ef?7Cgw&suQ7fstdhXNgXD4o*;*_s=(U<|sZO_5$`G{hwpm`uZv;OgVLb084DICoxJr_&4yn&0tq#=& z$U0%?r`Z>W+50`qH#5@O7&k*`===OPoxjcO2Hw|@pjUGK>OfrzKN&+Oy=)34`xMAg zw6-;O$i$tw4&8%Yn2dned>N#2r{IM=GZh|DX$io)cVnyRB(ESl&Y4Rb@KXim) zrtl;C>~{G`Q9&Y!9;vGAt}Tq?^=a=CBNqw9CYJ1)W4I&Aoi!jMpdUicO_z@FLC(cB zIOd1x`KG>1c*igf23DhY~v0M;O}QSkhl z*fnIV&YKNnyfmW+n|4R&aK`943OoK9L36XF`sS0%w?Q*!&}XTP}; z2e*7uMw_E#t{5*zU+c8KvKd|P;!UiZSovXc;~(KLl;NmW&zaZAAow1#P&TIkWB}UHW+J`))n+p0WQHK78!G*$KQ{xtHzZ#DNG){0N4|}HvTgjJZ|D*e5o_k)x^xKg^wp|1L zd;64XTJm5FaR?}3J5Y#Q6pLk8{qEU~x!vLzeS4+WT>-#1kZwe)oDMVuVl2pMaCAbQD*Y%$Bg<063TmD6Q@2~=4ydg1i?4|stUC555W@~HY9j4x@nR7y->o|p;esA5^ z5{5l{BCv%AARde4%I_RQQ1FBPME$i!`_48aPit^6yjf{LB|MOQ_K0S+_q3F4WlZI0 zF!$hxq;lKH2icd8=(!>Ua#@Antv*cfWyKezKq&VZGN-qM!+#1-z~aef6FKzem*@^R zFC43%BRAG1LJig3{tVF1YiqjR%1aFpnq$Y-q46bW6-=kHX|WAhwN77lr4q?nRX(1u-YteR1j|M;ro;?f7yK}D^O&X1%pKu=y877~UOo~d4O^{r zgrKC>+B@nZrIJ+!X{&k@;e+dh2>54zOV7N&%E)tr=Y1NvIBA&-C&ZFO6P4*SaDH!` z%}!6KRO0wE90}(5Zm(E!f}mVssfZ{ytDv2*P>nYW1u^Et^#~b>R>&XJt}aZq4qSZ)CIZ zMe6slFcZl0Ezfc>ji(vlxpn6TZS{G=?{ctjUyyeU6i^V}J$|tXfPRwUo~KLBKFf2f zL&ims{YFKX!tN5~2;R?A&o4{9@+K&>HQrS(e7Nzb|9iCP6mD&-Vj;PgwOgLT4cT8V zDGNAX_tc7CaJmErXhvmpC z$kyO_;z-=;cIEF0X$Ei%Z2rDygpkEdlwIxye<#arTL>O7sSup&u}m;_CSNfT$NHPV z6(LR5MS3m1pw`MJxwA?e1&^C$6SKLe#8tC|$8@!NV!%7D9kZ5D(!Se#UHT*OEu}dx zmKm=niD$CGr*kJJ3LRTJi)bTf;pQm41#|w_B4ij zEA14NxN25Zi4vUHHtWYrg+;C?O$Vwu$SB=|GwN9_a2EyO@(d}NuJ8LylYdr?@ zz4e%ldISgTA{wa(NdG!?hrp|qqAgF$x&O6*!aQW+L?>&_jng7O9M8&oDu@DqCj!+g zhAdfAOFu*nVOOU`njzuCFye(5O_H-s-cbYMdI}n(BazAw1y2=HPpD%a;~ik$(3?3n z`T^=yv(_k?L<@80w7}S){l|Cp%CIH7K1K3m$%T3eK(o?s^RNO<$lOKI#RgzKo*nuz!E{Z)`6V88id8Z=VJEP)8wk9~k6m5pD zcFkfwH(SNsv{t#_pNg0D-k-vYg1!PqiWfk|WEOtctPV&^o$ImHi`&0&e7hT{$4jl} zPgI;_@v+h^TSa-Nk- z>$PQUYyUI;!Bw^|Kfh!fI;Y3G--Y}w^gVsG7{09TGWDGaq#hkaCDA;Im!>?sjM@wq zBeHTi0{a!1J7{X!unyFF%cCa^O~$girRE3zN~ zz8+#P2eD*e#fvUz2rB4PV7?nOH7J$qV=YoMuFSz1 zV+!wOz}{SYDe`6Jm-R@M+A{Gem(9^&B|74dT=?#>ckcpdH@_liA9+#+|wQvUYF|Q)evq zM$w|KMjzx#2;qC^)H|B2!Razt%}4&2z=HnmxVYOXjCF1YA;EkLiR4$3NFIjIUk{pL zf)Imo2s=xUWo__k_xc<*(b;00E$|UeMNTXef8DjSE2yd}@>a(C+ zvt#c+j>0RPG3<)eT?yljKVWoPxjs3ouIctJx5hw6&EQqbg0U|sXU6T;z(sbqWS~Zz z5jL&q&sKa}$XKSJZIcN@oSL z5}qN7@~mNY3x8M+KY7-lqIa}XD(^cSgk1X#Igyzrv(&77XZwb?OdZd%7{itec}H`C zWtkKiu2QrVGAakIZ+epUt~VnIw}?hRsM{wd(HgzuBI9t#-*0rPZJP4%a}0-#jl$}d zYZ=i7hWBn6Q$-(1iMx8s>@1@kkB=v})`wX$W~g3>qGDxqWIk4+2R0LA^Stw?p?(R> zL43dX!O?v^v-)qajoVOuLdx8C2#Jhj1Obr~m0yPc2p*cGgu6q%S*zr|gJ z=NAz#nPg90Eozf%k$Os$)itp3S9nyIXKhNrb-F3JvC`%dMo5&~BN_CbgXC9?So_g= z)+7E*g{u^Ss*p?_ckNf>FHNp6R6i20N?a9XW0>GQq*FxTd_ZbMNjQlTDsbi%%0}o| zKX%m8W_ANem366pz;{RrTxB*|7rd3+l+Q*n%%EK0g3+dn`X|LW_;YgjeSur({%ya~ z*prdLGpPFW8`k>q?Nb8t5TqfzS9=m7S@^6O(VynHZCuW0n|HBBOE;*C zl3;S8B{LZ=V0C#K>7Q{yU2FB$+}xe;oR*F-c6`wC$&~4wd~|aS2jn<$ZNy9js>~9~ zoW=>OuX3MS={xDqJw8}(Jn6|n)_R}$-rJ^*60kk!YBf^&2ChwBtsW|THT4>O+V*owUT)=}8m~sxE#j}!C zKm`KMN91KO*|Gb(Awb>Wdt?_8D5X}Aw3I;3C+*NraSJ0)jJ)SNZAY;&oOrPJDiPr- zq{i(_kw z=xwl0mNUOSWN;hUV-8VbI4d2NbU=tWRPicr{<(n}5kZ+52r1=2SOlu&cZq?CypLWMLA(M#HkTrYvoE$imxo{Y#CJrLa9=W~}zSuRMTi z`i&hBL!x9G;Cel-Y~)l85P^@IQY(sk)(#F$zQu2R`gW1A8U4|4!kO4Aktnq*r;2EM z=_U4JI;=*Ydzu)BRk$-+Z?fbPes6k*!#&5=l3ClXgc z^hg9oyeAtmiNVg@v;jB_zh`KCT*;m7LlK512$0eZ83*8651pI62ukQHWS`b2&Ioai z*)dm1v{RiKybaKmmgr^P8%bdieZPC1UP25EuOMnTPGiM?lzzDF4D~g9D0Lw1zxfkS z483B0X|1*YV0q11-cs8>sHWmM!_@tZz*YyJ_h+1eS}laFWL<;cxLtPK6kl?BHTa23 z%9Y9;61x3n@UmJjI2p@R31dhOz<<0nbf$anSE)hhkWyfV_Wi>|sc?=VZ#L)0)Qp`^ zU)FZ0%WqI)gwGN&sg76*m8gw4dpLq=M~Pbd+c;>*)C@fqd8J1<8U6lS&21Tv=MjyICZ#Osmi{7y_blf#8BlL z&VI4LTgGFMn#SlpeTeQ3M~uJ-Y}gv2d{^A8KE!9QcFMNnZHY%RUZhdk-@1>qPq?2x zrZwgW^u+S7r`*It78ms0FmxzukVFs;a0g`ET3rfcy%am8oey4LGcmzK zbu?Yu1y#oqld4#F9_D%c8dI8vpf02?!Y;Fb>P@9mJ?`F4sh{#6_mo~HN07ahTA`KK zcs_iuwXq%k>rWT2CM%Xj=Z>ojdY>xA z|G{N?qG10`F~WCPi0B%Gp_t(?UxW`3Y&pZbeVw*uA^D;?0K)qrJEO5Zfg4$#e%8>6 zXCLX*p9ycR^m}pNz(g{6)F4J0#Iu0{`Vk_x_{x=KVnGHQb*Uh57l z)yY+o$b(P6`$B8z^3iA6BfH{L)qx1iUDHU;fmGnz!>ax*M61fbDvS6r-Yk7S_??W> z6Wz`j^Gy6eJP5=Te)@dvg0`^G$s|_nU@D65<`YxQj_VFsk-TQ}r@?YnN`lXbc(G2< zwYGz)&a^ILww1;W`M4SL8U~rDp%3-Ec>bC*=eV9x@sIo-hP^}Dl1f)Ror=Wt2yZ=q z8|X~Dia03!j4Woa&|^B(Dc;rWtxZGVnH`l4e~99!UmpZqP`gq?chv*rmKmA3!d0)% z_+zte!JMTNqTJzlZ-=Y4$t=EV*|Tc69367mbDSD@6LDzEq4+op?nq9%hfF2&SAZ}m zP7K~=u1y2I;;%bq2XLCr7{*IWP#5GQuk7n%lrhX!8Uz@2|61bOOzXLZ)71Z!z1fO> ze!`IYI}Vxgcl`-|#?M~fA~L-{7aY3y3oI1nX(w6Kuu5iu8)@BhVul;}d6p#gOD}Re z;FNCfn^Q8&+dX??e;4{uOrpFE-DE?iygv{g><@4sM(KLnetDh3Q+>#6baQV7Oa6$A zKKQxQv_FyBM567iJaMSkYMwsZ`{gwC5}zrv?pG`JB^MFHp*rjmH#>)G?(dtMETq2) zH+R%sws0r&P zeZQ`!-;n^Z*NzS}B~YeXQr!daJYj)d9-63PzwBef`;ZWaPW!vlW6Dh9P&kjR9ZFUB ztEvNk_28OWse*O-X1CtG^Zhl3$gYTH|JV$o1VzmAgCRew?G04m;k@pepsF49UxaYF z4>TK_Ymb-u{*e4A5fz7vIfUL*+RrnI%@n`Y@Ng*fuXZamEb8s>l!TS@pM*7Ff5%>L z%jgpvSB!T{`^byMdrB6LBND-+Xw8P<*bZ7Wr$&s;-r%E%S4!v29h8Mx$?9A?->tsz z-4Q=9JdwPRT<=D(+9n!=83g-yDsEMo@mc)1nqcYAPXW{{Jru)X{0AfSA0W{GcdyBO zR(OZ>iZlWMfnX4O!rfnKJT3tHC%}KMPqv?s|NFoJ0((5C_Nq%D|le zBjW&olZxNc;Qng?z{bYGm9+ns3c|(?{$DNjf7=6D**LlWX$J&w0YLxCSXtTrM+RW~ zAA7IK_0A zvVvLv9We;N#rm%kb}lgIzfL&Vxj_H6<6sB9V$1$JiU0p>vU76%&kVBz6v6+7AM^?( pWBXSb#P%Aee+{zzPp3IRTpaAsWw^KBP&ioGfXGx--{eG*{||N{*Bt-= diff --git a/book/assets/figures/publication/crpto_fig25_price_of_robustness_scaling.png b/book/assets/figures/publication/crpto_fig25_price_of_robustness_scaling.png index f36d241d62d08f3a81998bdc5f832e60cebd9c33..966ab67a4cd4e716365b7ca2cf905b64ae09e1be 100644 GIT binary patch literal 116119 zcmd432UJtt+BF(PL`A@kB1NSt0!o*T1w?6Lp$8D@B?yEPs-mLOM0y92CJ=fHkYJ%m z=z)X|(jh=-AwVd1#dFSk-t&Fq-f#Tl-usU~V`z3r*=w)zJoA}zuKiv|OO5#``%wr4 z!mNJpjxGdp7#z~SKYR#0z32Wu0G?#s?&`Zef?2tFn7dd)G|k1o(=$<(9*!Wn3f-{Wn%$ehq#ue8~EXMDc z7`=JK9$-hj9}VcD*8E3^N;lLarEDX!)G?_uN1hLPySg-0XH$GTs#W+SfANA7|Laj- z$UsH?>jmvOZS&sWUtWZK{QFB~$VJe;za9Y)<{QJwp zkiRzsavbvi{fTR@N=xhcZO;jum-j4s{rYutih}n6yHL_l^&>8?$rHK8)g>Bn*S`@O z=*C`|)VQk?kS2Wm{HjYeBt`uBmi)3Z2`_T5-qb55k&hqF2;IMP=XIGwzb=>e!i}4V z^_;?BM$fxeR#`q)Q&Q$MY`C|xrftf0YB^5WsG|8c zs{(JiJs~E7N3G=(-vj*`2;}0$H`{+k`tUV5_l1u}u(~HK+=>qCMXkSntu(t33XB96 z2P-MfB|2i%Z*9_}4#s8*FUIfG7p$UvsqEB^EX~B9&&9S-GLWIu91errw*pg?4L+;v zV?}B-MC#Ucx3-Ce!XzS*j{eZO3`&wL{5}#jf0gG_npU!`qOhoraWI=`%uI@R`?)Q? zn>XXnfYo)t@Za%IN1?*n?m77Zc};EYSYPVKd4yUViY6j! z+GUh%OS&(>YJU?i)A6aQp>2Ar1)(TT-57*T&F2+bF%2pX&8K`4ejp<A`_~{=*E_yD60}Q!T5# zdZt?rHq(&;+U!S;38ox9bM@f*)qMTp&ry6FY>=_hGRMr8SWy^naX-#IHthR)6Sx1p zTen{Iq_a@Z{Db1 zWMXBFEqOh00J1{>R~KSrb{enB#LOHWEP~28_~1}fk}@n2O!e!3rhDyGnW|0AcKgCQ z{A}jaPg+SbmkR4P4Bz`%r>*`})JfQ~dwtY&tO^3z*M?ko?V)lDG99RM1hb(2PX`N< zoiMu_C-=2Dw@m|+Au=M6i#i-ienYEC7X?8>kH`Iq4gsdJE|d4tf0XDUjr%=Q5)!Uj zccchAv)fNJg~!H=TO*3f0_;`dhlYlHNU=hCT34Uu+pLX%HztRwT}4Q^l=>GhOMdp> zR3%+-^e4?e%GOeU`ZVT2lgOC8fOblmJnPY;tG2Yj4Dn=~-~^#A?Qu(U^T)T$7i4#? z2?(g!^?rRaHqjzfSf;VRP}(7zG!2$;u>tasDYxIIdAx+JE?W5`nMYh4XInkfo-8Ci z>QgFVk8eFG2)-E|a$a0j3N~Uq2E&t5CGBI5amaLL>5YK(^}MghywUMEyQCd2hy<1K zu#Oak)$dHA_Y$QXR|`s0{r3zGF|+eZ+VxyN@Nl@sBNm)a#W-F46bZABHuhO*(5-QI zd>;}L6&H7IqSE9=!J@x2(HI)4Y!=C{Iho?U6p<|F zA)5rPOwHI`-=mT`qzlk`eSV~S&z^k_;ZiVUm#{HAwwu3p?7Y~|zIJtWb(e`y=@vFq zpWhJ^&p_X;js>!^52|2r8mHJsyh#dk1Lb+eN^`Q@AUfuo)I2rLgx8nZ;zCLYg{Gr@ z(vmCbU{yD+IB|}xPNIsm`#i_r3-F4Fv{!|@XWozgqE`rQL>Sf~$~BT@pE%W#pI#AB z*h}?URnL7|kl?e?ZyiJNSm-k^^_O%Ucub4xH=W0n)3HM(mccg)xCEvx=iO7}y-Iot zjP#yaSWIOlI(=GScMDo0Qb}kk(ikjL-E1vRi`^#$hP|7+2;ynABYJqS`uRKL*Fqjn zEBbEhk4VRBtnN}qFTlf`I%a2WCOms~8y=dhfnGgN4O_B6F_$5G$%QlF-)CW+zBypJ zZoNvA#gw{YL5%wZGe8VgBNk$^;A3HdjI8Way{_+^>RecV9E4gsQ6-rT8XsoMd%>!BCk!l!+ZBLtF(5!rcQKgh$cna1eQ~m9B9jBbT zMEjOup{>am^+%u=B`YFQ2+3WJNVq1(Z#Tpd+2o3i5uO6Q5@YJW`vhNTDoP6jpS#J= zO!Nc<1mt!#xD(8*G>Ptx9&SW@n@wPx<4{#s4t-fIYQ)V~)^(xpMn>{yYB)aH(0Ufu z?YmDVm`W>(hTy9=EaBvIM|$h^(pV_sR}RJwpKsgGMy}i?`R@xAB}c}_S}#yh9kUgQ z*Ueu|m4X?M4HiWUg7hOj05jAJW0!2nsZnUSFvFcnOBG&$gyvu zlHddr>m;>xs+)h;1SeRfJ=bZl1X^bH?LefITw6GZ_+6P>QjP=9?K;z5{^$T%R|8yS zxovCQjjVU?4ku0#Etbc=Win^>b9m6vxtS5k<(|-TGeu& zJoyZ@zem-ao10^ZQ72Sy)ueb$pP#5>RotvE;h+W~G2u#kedh6Q==NkeHbiT_DP}~% zciRmYtp~LgFIa-L)7F=2{MX01&k!JMg!-=MW~YdeiG&WT^QfWETx^J;Fm9!ho-gXB zf-t{91ba~xx_;u!8B4>G=jxZd1AeVjM*R~F%WTZQ8kA0nTc*bS`+Sn^<%Uk=E18b-xzm2(MR70wWXSMB=c zVRqjOq+ADqPtx?l?Lqx%-|st}1Oct2qAe#!QtfJsP$R0dg1{_2WVV9_@x9B+G(0p^ za)Vy4qkKuxjZ^M3RSQID4adO>Nd?M~YZho_huf$xCH|@&(ji`N@WQ~^(ySRVM>Ras zWYBy!ZN_yN!x?>d(9NjA;l`Pu><-ox*^zRVHLJn0szpb86gnxHenAr;KbgkzVHu|( znF%=%=zsaZtA!lTK_CI2Zv8+Fu$F2j11g5p^spf8YmStyfRKnpE0O}L)1xD7}AQn0v zs772GF)*_L9jyv^l8sY#a&akXn@P|~CwA3d(?klsKL}yonJ1NC#*pD4T*sk2NKVGh zl~6*Mh@kzli6{$Eb`vDoHQ#WgD6?6@48aTsCbfwcq|+=&k>Hqx?P6LA6LFKfe?aHT zl`B$fr`=j-8y24y7@kRh6uC^@x2T)lkau6uM|bRR6GgFp&pFrKDOH=Z8Q~#jIfs5O zhEr4hwtsDg4Z<+PanQ(f5;k_6&cVUK;;#zZZFKI2Q_^n-u{Q-tGDGZiKf)pP57RPo zh~E*^S;d@hjJpLJ?n^_qJ7J&sA5>Re)6jE^$dz-j&PfiUMb}Q}Sooow zRuDq0c{OSC(bVKKw&!B*9v9V@yowdAyppJuH|k9=@rW-eDREH5S^Sh${xWF0I8Ywb zAz79Et!>b$rfpizEwy(-FEEE->>65}I*Yzx(HP?M&4U?bQr2z+IDZNE5jX zd8M7@+vtw+JqgC@WIo7{YllMPOoD@zwtp%Afz#4-%8`Bf#Q76Es``GZ6$%T))z_~N z^1RE#m~`agiI0ex^o%U$$QAHs0bAz2twdBWqySwAkFNN^Q67Hl_HE9J@P#=K`{Atj zrU-}G6$evyUr?{eNm1WbSaUeuX5J*uAT$O@j0tAA3^mq7av8`}*qMapQoG)(8MPba zK&}i!RvVXK{sgo_;9UQd&_^ryy$T5|97j=kOJabNFb@T zbT9K+-du?h0;aKMrDD>RR;gv2TG=Jp4AF-tgd6c&%~zjE3aPFm+zYtgM*n z7iq?bd_wfb+uq<(qe`dp_2a#ew}*sU&k7qpKO!}obEw@$)SMXHtY_+5jR4$!(w=zYG$_)o?mu|2HjC<1)>2Wp z-=+{k))$&T44Fx0iFJ0Z!Wp;&?Cv;-!+IAnO%%se(`Bx3z|~l5NSol5%XI?a2+Fz5xwIE&mlnt zQGLdXWNF(mHSamu`TkrwA!2}uN}W;=EBDgsUOqJf9#DAEkg>{TA?7Vy=Ug%Y<>%_| zkZ=mh!7x-{8*ZGedMZOJMc%+nd1$9ILQ=k*m3YH~EB6-tTDwj6%R|})mEpeMBW}Lz zs7;m2P#BlgSrVjE4s4xXi}3C6vPUQiX&neq7!E3+FAa+$l=EMj6+D8+*$-!D_zRj~ zrcaozOV4`^XwVa-3W>y zAbx|Z6q>bhxRj*%m2Ipbs2(5Ogy=HLCe(lQHS0@ZVRYQ(Y!}%xG`&u!kAbt^vu9uK zo@6Fk+PwpEbG#j%Z8w{PeNLc2u(x!b01}MXk`yJ$Tokj z6XU$WN;1)5waC2v-YE8ww*|t*B+k2<&8>z34nWXZ;#tUjiY;;Uv&v~}avuOo4v6mg z+upYaaP=R_66~H`ui2=+y?c1B1pQ1Lk#hQue<^vDlOdvvF$bP90edQ+2+9 zQ%)t9F_3wC&EoT@r5S@hG86N~M(lVM^D@R7a*4#tmJ`I+*O<|=pA|@L3c-?N#1LO+ z(^IrbGwO!ZjPc5c(M`VdefEi;UxrEIEdFWN>+5PqAneEBQ?)sRHD69a@`rsX!!dK^ z9ENO0xHL?KW`3lv#E};HV|8IUI>(boWo-)8-B9x03GaR{$dB7xAX+T+6|;DE5vvX9 zDflC=!e-rtuGzqn(=ryXY9=_}p&!O$yWJn#%F+TEMRl+za?w$5P(pS*d{_7^N7uni zCi^s2F`%1QpZ0W8!mJ*$13(YcB<_)#v?HFERkW zg>re0(Vf5X?Ddd0sg7$`I6;RBkP~~yU?ogcEsW#E_Ti&POX5A2hN`z>*iN0YSVPSB zi*QG|ahQbTK+kIJlf0)a*4H?&w4PK{Z|iCqA9=DWV&UZQds^*c37|ZTS6+T-HDzh-{Q2|aj?<*g(fvIET4^Ao3F?p`=O4^mR4w9e z2>_0|xw?EoxHo{&0Y&`LAyJRT0Y0VO^~jZ4I1$glWse^;Zi^Rh${>E1-yd%Xww!8- z*7DzS%9~`41oRdr2a~1WpRnG zL@6%->>U(Yj~ux%&p;fNvF+yl)zTuC!pbg@Q$I!A0X0zC6d6FRsd56;wF)LD7oHQY zxZRx9egRZdKh`!tg5d-4Hbym+{rsqjj_+AcP7@nnD_B(Uh{Js6?ysL4|n%kTQ#x=p8>k(?j9GIp|GMl7aITNX^<=npBxVs6>(-~?jth5 zJYKaR(jNxqnxNcni1b-nTVpk)hj~Gsy&>|<57b-Zwh3lwDh!|y_BhX~2fo}QmFE9*Zv)&aV^2UAd zWAtWvmg=LU;}R-ox%{@fh~;PowFMC0&wBI_BX~#_4+1SbYn(=BqVP8Q&8!&_57<3M9KNC}NH;=+~7*Y1moQVyIqko}LGaJ1#RTs|ApH zLj#HkXphum1Ol~!*j}GgeU=TsPI$OH3+KPO^_2L?L$VM0{)<#?exm)DQxUuh|F02Or`jS zLNupIj{6(AH3QR+l8m<(Yy&zTlH{1TJCbhVsy1%|x&v?#3UUuV)DY(vBl)wKJk4vAoPR(s4D|9SkK9x{9Z@&JP)+Zs26|CWeFqrK8@$$Lu(BSk>V3 zGpYVg>zDCk#ws(fG9SbmzIrQZk9V7swjnuaJIT2ocvVBjNEYlDoPtQK%+QF5FmcVY z&JKx7_lPG8*cQ}?#n~@kXexF~edxZ^07%r4S?0p`2N}rjqUuXX*y0Pv;YS0}JJQJa9;3u$`82DRZGT3;O0yDvnvSln6b@aPk& z`htJMAD|I0c9+Y4=LJ?VQy~UE>%#e4CGY^k;q=*^wX@8EhKvs+oGm!VuC#9xc+->% zX2#e#6>gvUStebJw`9-Px_|%b^^bNYFjQpi#1+)2T}gO0(O=W5Gp%l8Pg+0*P$&&D zN_NizSgMhgZ;H8&4l8!4(0?7s>+|uZQ{bMhMLd8c>%s0m(hsM0t9nluxmx=g3Cz!J zi(I2qCYsfm1j9RaDn}4l+Y7oBhBXx4w>ZBSEE?_)F13t?-c%aVhL86e?XfU{NvHoR zO7Q&b_!iV8G5TJhi;brSqddTK{gJwh20f4jsDw0*2|nS;y1@xoBCMNkN*9DdQf-nVUUqzyR&{qC1zr_CGLH%hGGMT8JL(XxoZjmMQ3 zSP_fnaG|Nv#4D7*NsiiYkxdB#_=b#d#jE~1D{5$Hi>wA{`5o`WBF0rZKHtzmUj-2H z2?Pv2V`(UX;pJ`0;-%Yf%4W|;)B64K(9NE=NN3PfH3qDA!=n~yj|j*A|}ydbZ@ z`-a@t{qk%Kl96f@X;ZHYv;Oj9#~$WpJw7oJn%+=YyLvy7QXazY*}IkMy;OY?(-wwF zf|+{$X0Yf(h-)?$Inl?IyC-J1vo_ousrwc?s0n-UdB)X|YB-9zOGxHtvD0m!sfj<| zpW-tV0eI>AAGJ3?S3S^;-9h9&49+=jBx;x-Iz)6?LQ|bq_}7l3U*4kMla*gegrRMZ zRiZ$>S-sxqVbHA?Kol1%NJIW;msu%N zPj2K6fMmJ}UgP>}KT(}s2PE!&et&Rd&>3*9-57uW3G@h`u#^;jx;*bka?O2M!K;!f z8im=O3ow;Hk1EPUc>fCN;D&RYLuH;rxLRibz7J*6-z;-{xbXTeN^{iA56 zyX{Rib#0Sud?Q=OSisWYm_;c_QS?QrVLA}!Th=S!^!9Y~SyGAliRWugoKjpUYOKkT*W)vfj2gO}erp zDBV}m{MhB1>!ha8!OS63q)!cBN@dq~ixyBl)=8>lxlg>l%P?Fs)PFNAZQC?ll5^Xi zh%abdbXf>mB-}xLAGpZAL)gAGD9Wx;mQ^WS4Fuhv+)+QbHbC9pIIA_iovE97U-jhO z=g*(N95wf+p7S7zYlq~FKX?P65Gdqs+uFkHUvhJV@93py5bi+TLyr|*0D3a5O3Rs zwrmxD&haOraZd}O!$cl|sMB>yd2q-*Z}%FpFD1L34GhBsoT<=LE}hI`02?-13uION*!^=`p9)ywHRc;8IwfydEEayjZvgfx zk$NI8Cr5Ci0y41XKiiouHF(>bIfj?`8(*%LpGO&Whws{p&V+;+q-;fBume!Qb?}hrLk)83tKs%d^&TGo8P~mhG_1EqE2k6-RT30~* zGIF*V3Y3E>u%vHn93c)~ogEcq7+wX!a~f|1nA;Et=3`Ql!#c7Q#yk{lcJfT`o>lr z>9aKAFFOvDMRb3y`iI~(hTYfza!;~fEI)(oNgWLhja)}1I!HiC@E_dZ8J&$#9`P9U zRXv^#7WI~*5yr2s3;Wo6FqBI{)>M?UPOUP2HbeLaS>A#S#|*lo#F_RMGbNacWw~goG@=1jf^_39VP#=! zM8)Q?ke7}iluTTeZNI-h!jfG)(sAn4DW+p_`lb%@akrbYJ|!DglCEx8 zM!j$*JtdFEu00?|>-8C%5e6U6ejH%vO7WBp-7TJe4}aLXV>kZuJs7N{J2~IO0zvs` zxFM>iy6imFcOIbgHj*bW--L2iGSF1eE}T&$^5{V0$O4~fXUU?Hkd2{UxS9f+0jmd8 zC*Z3@e&BcgRMw;HjljXF z*kiuUwCzXTHpLQW{pL{NY^ObYQT3>$vO?Av5fi)8ltmMG?-MD7VugpP`h}t%K|&su zCd7AwsKwSY3fvr?Mc&3wYFfi%@w-4`$eJQIu$f*1!m!3mZ12I>fZo-I4yK+eLlcay zoxW39%74-rF_#HdAZ)GfjO9)YUm!MIpm7&jsC_?BYG`59B99eNJp3`1Ae|iMD7z1q z-Pn^2wmy@&^ye_ZG&I(^k`L4>aI)(MUF{7HM2Nzy+4e+%3WX&Spx&A^y-XLT?RxS1 z_JMSGeJ@%dpC$e4k2mwnm<_i+XW<~|mpb5xS@m`Dzx2pkxLWQ@9(36)I_#ZU$M#;e zp_bZ@SbJ7$D%00#szjk-`PI>n%^M0MK+fNAlGz7Fg7_i53u-w|i$KmMFv z{Cq?-457s~)z>wW#-kcibR@~9B-)8yt&+Y|_7>5yU}k}I!_idWG(H%&$D!Mae!f4r z0Q>zr|JWN%dtZ4wNSoiL=kUJL_AC$2Ia`yUv!`iVK-zUY#&Tn!UsX3qa(F|wMv2~AVZ>_FRGY;S^ZHCWWxL_|&h`75zQYpl&fLtP4#DOQP*SIf!scOAFBg7ez zR^%QJrqCSIgCrfAeFd05`qooK1(gdjO5)U-x=`3I=e+@B2}{{LqU*%TqfAwH$_t`{>sP zb?|mZP)43TMVnT4xqa6opy$iv5#;%))mj4sgDmS*U;Jqf4hNv;e!9ZL^J97qNUaI7 zZnI|BC;)qjI2!=HBT(}ujS(^|w~PK^8w(bR_#*I;S#D`yX29exi{JYl#9a{&rCrH?SS~1y{bE=*?n`paE(M*x=e_ZKg zf&;0-F=h)GOa?S)yj{Wvdl0Flu(beQnIw?vU(#o7lZi&t_5)?Pb3k@y$?hG^z&*~@ zZ7B1lx@Th@fKi}rwZfr)+!=K{h$S8j$8!yu^nO;q9n_Y6JBq_FC-jyTZhw!WgnJ-b z_gr>=XD!rs0i)>ZXU-Kjafxq8nn-W^ z0U=zIk(H~pGHD#iq`8KjM%UO<^pA z^zN;KSO6s-JVGE4=~MT#w4@}#svF&=skJ8lyW}i^6wmMaFq%#~5)Pz4KxxPhjQ|%= zajPlX#|t-y%TD$qg5u>k&s|e#k9E&G5 z0nbz7hX>&kfU+FS$8h`a6xVwJjox0oZpi}KxtL9CluzYyMKLeTDFQtv_8e|j(P9+; z$kx_23-tRSM^^}5LAK_v<2E#OOwqm^likY6;XV(9{n z`Qu;ys-nRBlS9tE)k?OmFCUAm&(Em;ZQ?O(2^gc8HhIK~fvAh2Y1P6g2J@opai9_K zD3!-6N=p+23wKu+xe+&XMZXvV55$Mjb(_3RbFe;TK<7hWqf@z zU=cYFxwK+$z<5kVw`&$fuZ{vKRaNI2A78`Nt~&j5lnAgm#lzyaI~~6aB0!1AAYu(_ zk}P1#8^{+OW^83chI1=99gKw65EUx`&^ZvB{c-~s;SzW6pE@nkU+;5ir*hO!9A6$| zbn41_-S;^9wlFSs4q&ZIxRLC8Ai&QZfTDihs<^%bXjxXMlz)F#(Vhj`4p{SbAXO!R3#|ngd0a=S)Re+rgc*CqbW{QV z^=W~fAuy-?7=1VdH?K2sGr}x?XKGZmKr%_hWsMf5DlOO3XN{NanRIn^HQpv3TPw~O z&237)DO0GzM%}*U3lN;Dm&A*2e!jMlCqz2yZ#+aq_TAvKZ;<}NSfojf!?fnTd)F&+ zGk94vC%Y{O9Ipu$`jj(NNT{NOO*WkXGe7Z09vypuePhUZ3*fsjM%RDjZ?`eYfDgVu zB@i;RK?nDpMVyy&&;J4JV0jt4J+c8X7KwfVh)q0jW4hy}+Y%o%$T|nb^vjyr?so>H z;$0EY(tQnjbA(@MHrdpi8tyj(JdP95zWT%~#I-E0A3$lZ&o2|{{CT$kM;0&OPAXz!CO@vQV#lPND2Hvfa2R@=bs&nYLhUt6R`(;8V45v*}V~s^j1@s zpLQ*&z_`hpr~&7OYDxU=UOuZr_?*`auvoaqO+GzgvzN;t)q^qK-7MT-Aj{%->^^{2aPJpa?by2nzqhsj6m6A_4XP9NT^w;fXohoUv~orMr}6 z&qzrEV%xSio}3W`9dqw-ZIjKhN6R>-nwc#E9|qj>#qiJt_r0C1t?6P`h>UliX)XTZ z6Z18j18=&aL2oqk>MplQEKknfB7=+;3SF?mztm^G*H1}bSd`D`%V&(wT%YGU+0-J_Jm8Ml@jM}5CNDhyW0aVYde73^3%ui<^vAoDu(q>%n zggJomjZ!WW;Ye#uoflbE#P1KEy=~9_bz@pqTkU%spg(?{%EzeDqh+_>@(a!q=X>*v z4j8j<_Kv+N4M@rub=-HhJw@r~d5Nbs< z^IkT9sz1APt_-W@hgKp)?dwR{BlJ*`y74(jC~?<{%+&?_%o0nKJjz86H}2JL4HC~j%?Pfm z`{{9x&bQ6Q4T@whc@?0xd;0ou#-K*qT5?=qT~Db9J}dkQP+u1YACSl#+}xsU#?CK) zc<-W0c0s{s2|$R;`s~9atgO;Y)O-pD63{&nEjg4He9%auJ~oR|TaKPt$u}qs@r47v z|Kyo7Q*JjAqZdO=!hBY1TR`#`;GR1QcR7Q1s5IY4GTcJ6xpGIr?ROnQN&432=U8^yjrIeZ3$`LH$#6t|7Z7xHjt`fZbeeX?=2 z`=a5*CWu%GFNT0!)cF1fJi&i|x6)yx)*y`I<}-&%uwCFBCCFpOCySy&K$!tEc%#cY zbR3~X2H@cW{1aEko8+J}uC6UOXCg@x2{+i^+YN=EI=nuVJ~K%32Ei-#kf6udS&4{t zHaG5`Z4KhnVNqbPz6jop0)p$6w%Qkd%lMTgTfY90_O>=jd>&VZ_r}5v{6pYrFZR2S z8Ud5V`OF{C0D){i3;+c{J1dwp&eIqtu;b{jM?LHxEYCmx@Yg1v09`>*(GJ302qfxO z5;u5P=lDN&@Cd#EOqKnbtgvvKwIv-SitE3=4YU!&6P0(-?1f1Q z>qr41z;li_AtlAhi<65>3-sVT6QoQOQXjtJ!9_a`z+`ALMKQangPixT$3@y+kTC`o z6_t~~U%bZtuixJP@4x->(7w=TYlRlio&hTGXSU@59odJbkTh_y`^q)U5HbU#ezYr#q2&;m z)uPwqH5%XdY)e9Pz(+tk_JT2Mv}4E)r=ww-!x)X^|LgI;e=FNRPyOFcoQ@WO9DnyJ zFp%%+)t2f1yViqUdx9?`1^>FX(Eqr$G$XOUuIzu@#P??+9=^jLsSd2p!iBeQ-J*Go z|A)@)9P*D1JzL;mZxE>(1343w#0U(p%=8QL*qQ*NM4xxZfcZ_NOEE_wg{@_%s- z{QtPBUxR&g$`_A<`~jkHA_4Nx8N4a_pELL%q@L;3t5>`dQ9qbyb8UbPa$Imzj`eTz z>)w=&!K*kE;&&Z+j~fqv)X@V`Bw?<0pm?5fO2cLj%qHuVrcapUV<-9aQ#xR(7@# zG$<%262w7(92S`%%N9GyjI`r1k`3;FCB4a~zcu0iU621+b$^XmK>6!OL6fDNc~ca6 zqvzOoi>;?x&N?3-9}fLEA93c3W@dtm*RD)8`UT_r!#HE$Ty+#8#1K%% zB0{2ddf#8&c2G=ISU#)HvHebC);XO+G4`_3(u1hP7(X6I>$`V&hRS>{3@-f_J{QIudD8uw` z?Y9$Rd+mjLFJq0MBn^ct;_d~a>7VE_S^t?y|NQ)pu8z(}iT6mmwUL2HMxUo#^@Ll# zCScQ=w$!Cdm!t~m{?*~8e|30^ZlV;gpuS&U<8b=AQ`NsZw z{)zDE>@TX()0UU(&wn+y{i65^g)K%f`FRWM*Gku2##QX|i1LX<)!AleS~kqDdM2V> z;2G{L{;_@pHaS9zw^0l>OI(fb-m5SE1zBRbxQn(Ai&2)S+{D;lRptKoT<|xs-l-c^ zPgSMmw9j()UkTq^%X3h4)ks-%j(Bl_;owG<-_hf~EPG4%p5uU9& znk}yMVt#pepX8MhyqD#MzVrR|t2fP@wF0y2shTiUvS4I; zc5QU=EZ(a*LHFZNko?Yw?X{0R*YeyQ4|W3cWn z&%MXgmyJIbf|KoqJ0xYCIP(k7aoKty>Gn-2uunvGkB@c{XB=V*Z_kU~EP0Scu7%F) zq=XGl0TNTQNSNmV_$Yl;|7q-OK1!2=r1w&U&hgXYZ4hH;~(VcW~&x&FV z{#Y)+I&g5y0>c6GN=}ZoKW{tsc=~2GcFCX~re+4L49l;EhKvF8Qd}GM{(Z#9$r!EVaAjY&(%tvA{lOadgTaEBuw0~^P9D&K{Xu? z8tAOd38MQK7K>=!CtV@k3+cVSzu~gyzv)XfIv6*nv+r+;NHu;Yy^d~orL@FO0g=&%Dwk$*h|aRYitN3WEmX!Esr-}*Yu zOv9KNZDfk@lgn5vj2;#(UBgm*tf`PCGrM_OyvbPPNB_dLL@w&ot$pSJ>G#NIUP^>0 z0e{Y+X=gv#FSq_)PNiwoRDcz^7PWBf*Dq{&Z3WjWQ~Di{U>v=6`@{Tsc?1SM{Y;4| zB{rQVx|h~5Cl0)7sL=CQVM(TvHoEEUr7PtzMeY6_z2d~ZD2FMJ}RYS@H#-cI(`)iy{WY0B}bb!4)41oGbFvKhv$ z_BX%{2SQ{&I&T*li_P#Dc#j8gbli(9bVo+t5VC4n6h@s{9;`Nj8>sxP;0n+c=!g;+ zT`&|C5bGGI-6tc-pAsD9G%!3&;R|J6bvu1rze15k-K#Zjy-E(U2x{Ng zz^BR!_Ip*Yt`8^)laz3NX=R1FO{<2nE1Ibhx_p;gR!4Sz$bG?y!tRj^1)Afcm0AQ8 zkkPY6&HH{-^TAQ{X0wQe@8xBN_^R`F;$xzscvn_dVhHcwzXxvd8JrlytE40jbuF!m z+OW{j=+4glrL6;y9yW-LDI~!EH%Xp$4Dp|VIS#-)3S&*grNSwQzT!dB#ECV!L8La zOq;t{o?y#!`#UdMSM&T}MXK@BQK@PC3?;W@wpj#AB;V!|d%I@p<&zrGd7Rtm5!VNz zFhOdM$tXrihRco(u5f-G*Vy&hn za{UV3UiiTPv9N;>(sCuJ7e>({Lv!7mUgb%2`}EPQV?8UdNNCGB*ft&@_S0$;szfBB)f3Fi$ztDr{P{y(#o>bk?1v*)A^t zb1=NNC`5=zXC6=1@{p2ygoIu<@l&e0D^ftXzX4Uy`h&mrw$ekO+o3+y_Q^g3kEqAT zMD_Ald6N|{dmuMQrsmi^+f6lD$T$m={SUS82^ZdDkXA+(G>``E?7&+Y}`L z{`m9|!Ctt{FYfXLVeb-#yYKRv2pVU5vVQ&))$;o`W^zDv@yK4R%Fuzd*A9afnps-O ze{OTK0~EYZ$;l?Vrp$kh+rih9`L3%A+c(ARO`pBQ4O1xEs>`Os!>b35upWtCl4SqY z*48dQlIUua7y@0@EN=M}lYHgJSs#(Pn{S$SUDj47)1N3k*jG~g7-;?I`PP{CWUol9 z+_Alx$*Gn_EJpg({`~6p3th>Q+9bX5lodpOv4R*J>k>2?333U~SzkU$>|HE%#`H=4 zDC7Q}0|6Jn4IZx1Zu?MX)AfDItGl~0iEmYl^}Ii*DcNZuQ{(-D7?eA=0~wdgl3M#XtI;Al7@073LvB3iRKYoh_zp_*Pt^%WsNP#h@Y9Z3f;Wy&=cLf3>{+Y7y*Yq^0lcpx62-gN!3e#rID;H_0W&v`kNCh-gwl zES{=SIMTQlG>%ZOX(?w)Ql#vA?_e{clrWfw0mH_}Y+t_#~th=<8dPk=h|sq2R^= z=lXjID<3 zeO&(@M9^mHAR`vsI^ZI#cIE935IIktK0Uem0XVl|G=E9s8Y7qI#nBns4Sc|^{_`0} zhTI1o@AkX@)q{|uF92+shBp-X2leP(=|BG|A^G>JtbBC52(4ftnK)-kDPF;iQzm(T zs!M;^g8Pm<-@!Qaiy4$We~-ofo9!FZH_l9uDy-z?F7@KHWxGhIuNIQf@&7SiIJJs+ zA(n4mb1PdZC4ygxH2U0YINR#fR#u1LmSuh0NzSvEh+3Rcw90HSW(s4u`Hr8{a)r+W zYe_c5{$HHEbzD?y+cvyF5DQTZ1Qb-d73mhFQ$&$g1nDk^9Ak?}w@3&gT}n47iol2n z2uOpJbeF`u=R$FNKlk(de&72Je{DC+tXb>2t}~9~JdWWB zt{4;6r|0L|YQKjcn$GZ(zz6M}2gpCQx$fR^qiYLCF_z0Y_VT?E_AVJSwQiVX5sb^C z{hboG78Ut|CkO1W*YI&?>v{PUQf(n<^3VV1(WZ-d(1T~*+nRuBI1B0R%##6*FsX8N z_v?VxRMP#GERz7?EnV*!Rl%WqKenM78HVh;uI__@d;p8;T{vB7Ew#=iM#nNmhXg9v zw|gz}`llHlDlvHWkGD%5wY;LyvN4(5&DRzeYy49J1-2}0dvEaNh<_{PHXxL^oonx4 z^!{azUo(-|xLwxBRGNIsR|aJV$*@NY_YYYFb{D!N2kaJ3rqv z`-|oB&*PgNeAZn4FWGPh_b(c+zY$${|Gaadul;J^hh9jC1P z!3OcOKs`q?6IV5OrB*At1CxxeEOIR|O*^TjAAra6Sg6n(aq-vLD9aRglJlJL3jNAV zVO735)aMV;LQgj(NyDF=tA?sD9B;Lu1L2E)sAKb-jWhMcipyKCAn0j>X4Y#FF7|kX zH-^%Jo}0zSmEY+RlvI(M`7-SK$nfx~z;{H%#Db@fo&)AYG2SNw(ieYk-dKakp~D=$ z5%t>o`f>BN;Y?nkqOtwq%i7PquAQ}J@;ug^#xw0Unp>=`X)CgRSNQ7eo7r{y{??yw zU1`}lj4E*(#c5UaC+3B~SPS2R zv0GE8U_<|BRlyQ`gifT+&9ZXEf}atCKO3ve(8!u2)y>`-VEOg!xc#Rp`G;dnmzo+L zW_U)B;I})?)_K<|7ryDQzuMq!NirARU!;@MUPSXH_bV&K9eG3`)+xJsQgo%;Y&&6k z!Th^nNd-oeQlso20Mx>i5ch>tGwV1fzcQkY9I8*D7v#}SC)%PQ(O|mNYUO$D(Iy3R z_URkh)lSx1(JCg%IDf^va*NN>xwGZTqSyJfdY!AoVy)aGLbv-l$W)Azar1sWcZdit zeOCQW;U!)I5E$3%?7As_+slq5lDEDnNRlaCa^Zf6gjM{Mlzqrzdi*GIp@XcwJ_(X# zwcNj}W?C}qkZCg}|K+1+sLoyMI+U*^D~Q1?sJ)wAm>jgSzZ~wm;M9LrJ?+Wmn{9dK zd!jqa)bU%^c}Fs6`Zy9ZO3QnMBfPj|FSQuee{hl4kbBl>!bsAzXxVpS|KaaKj%Q)p zE51Fwc!B!%?Lv}R(MA>SyjkAE_#`R&n*QE!Z(7Q5{d>XM#U}$t*7Fxnw49x&`W!PcxwU^`dE38sQ9N)aDDoIZ!|>u> z+@r%3lw=OFwi|A$9lo`=5)?o;@@4m<6^S2dXfjvk0TgQjcLa^_uf!+~jA0vav_o zdtGK9UcEVs(G(E64KhKc;Tmsk>;qu)ew@E@=IW7&eY-zhuk&&1IyHV(`^~6NJgVv~ zp}Bkp{dvo)m|b(y&w@RluDJGl7jMMy7m*<^e*!^YrcPQ_YSl9lY@=R`DS4M(-6%0# zFYVYju3vGxa=p;1fHc`gPHd{(*sc?6!e#!+Ajcim#YdePb2Uu>bZYXKh zj+wMoSZEd0oPwV+)Q#75!4?=IJ9&z{l`iu7D9q zewq&n6>7e&;`>K@K3&}CTa->BdB*g(c7$KB*4eOs=-$f!=6y~?zkjYYe|hW_;4C<% z8^AR2ScinsH%`EVc!ksh;@$!Ou1=tJ651*VYFy_{|KXPYtvTuwS7|%2`EP zES)@klcQdXvL#j0c)haUpZvs8tpS_*MqFsHZ&k!V{Mo3!Nmty3?UjC)>qE}m6V;4S zl@9*UxW3k6pEr`SCwhC!BYd(h?+mBoB!9_6=MtAdCe2_78qhn)weA%fS_iHnR8_!1 zSFC+L3iJ8Y0zP=HZ^3cnzPcFu zJG!d~v zQ8r%=lU2E<&F5LnP|DW32U;Ks2en1`JwZZc2!4&zXL#P$9E-Y!5Rw5L1`zTbS$;R# zRs0RH-h#uecq$j?cl)%xy}c1oO^EmQH^McH`oFC9W@-j$wrICX0H*&NTKSD-Rm1}| zncIEMc4!?qMhw6p9N&kqf}$r3YLEE8{F62zNW(sIq$emk+M+&)^%H2dalo*8NO(8{ z%uY6}mK-L-1C5cY3FUG6`1?DhF#q+Z-_8B9=hmvMd zIt1|I&cmEYII88jhfL%wsyBCtiboFRhwdjO<;Vodqd%|XPl&LRhiJumel&`M+E4WA zvJ*&JjetZbo9*`e&_~?6e-1Lx(_i5PHp@5+-(~6)slbr)xjd7m2oR4rdE@U^^8x9O zMafOj@>JIq`cx(M=R-gO6NP&Ief@tod;SZ(Jo~kS7$Lk0EfdgT{Z_tr7q|5Kjay1+ zS+pkKHK}{!U=|V{$u{`*oO*>WvCm;&_WL1(%!g5g{>w2_kd(Sv^qWo*cQe*Bkm}7y?tvc zK!$q%@uQAD6C&M9$W)>V}p&=0x#YM=_u8XdN6Y;bX3uhFp4VPF0lF8uWe%yfKA zwBa;){o%Y!WcM6PGE`3o&E)l8j3BFaDJF(fr=+ALBqYQTbLQAFx^w3ax$W8AFPguh z$Rqo#y8OLY>lqZ2>j(XKa@;>AKo&hWMv}V}ad$9*MW700kfntv8C1Yb5g^ z-EDH>zwb6VBg3QV5;eC53I?j2$gA59CM&93wBbPFP5*966ZWvl`x@0m(J?IOxw_qt z0)d2RMGV1PKL{;d9ue*YS}dgx&> zMsxro4jqyArF(bVk1W40l{pK|fpwr$6dfH=H=gOB{ydC~G!ix|=;UI#|3NtvJZmpt z_z-i3nwl*k;k5NGr|1t~DXH;|LmK1&M^X02m zPm8$8=&R^5(^~7XjM?WU(A0t5>dZpn|FR)tpQp`EL_&QXJ1(3Wpfv{ z%Shh{6Rc02|D>sLtjmCzgY^d2o7p{cTz`=VudfjkLyPkKXMAYlUoLotj zlWIwY83m=t*qu(>(I@wG#?qZe-tc*pZ;K)ef0NOejmcHcgve8mqLA%-dd8x$9IC7O zG*4$4T#CviSRY4*0{F20YEg}{eCnWJNHAlcx-hFu<)p*IM|-{RO$#Fef49}u6f%V> zoAzzu%f-{=7CWGZTdJ(e!f3M4p1i%NwUv5vJ)zslzO8{$U!L&G0KUg}Cg^cy5Yxw~z6E3TKdYp)jLtypvuZYPC@hy&r~8A8xNwU-{6G zd)WnN5gpm^XxU5NSwH)DxaZiT6$+}v!xyH+2i*SNHw?@GDpgH&(;quF+Awj9H8b7QX@pW!W*Ldxmj++1)y@1SAs!AMLua!!Im^cr}WmGiX(Q^sv9J%FJypPV2 zC=#!#DpB=~|#ohkn>C7OM;7}lwkQT9(@17 zZ#-BxI3)bsthE+n4g+Vzu_%*X8yAHj-QL$(epVd@UDvaxo5NM2PclSVjOBLIjg>d-3*x=@lR&5xn4 z4E)Ms5bQ)&eFvfzN{v(nJx}%=IyyQA0fA>uq`RT>+V9XA@0=VH!~&F@{4*I0!dxBG z;b=GWj<$0o$&jnUvVAi{sk#|TtlE!YBUN64x@ttn;NT#9fnCbmsb5DqHrEPm3ZA=N z>0V$F3#E&7lL1S4jL4CWYcp1DQmoQfe7L^+zTMCyG)4Y+E!Tb#$@%+0d7TS@QdIu!xJ~C<4#Dk;v8hjY7(j}Poa85{K9>@! z8U7WlC8A8Yql^ZFW4Y~WbfdiP$p*K}0__B)bqqgFj%7Ii=)!4ih-~AY-Rr?}pIO5X z4V;WBz8%LdUYH_>Gu3)i>Sxt$kZseWFZobPL|^7#0y6qn%nl1m7;X!j5G93YY!<3C z>@D1|FgS*~mm}FGih_d%ZUsgjMS7(FT_8wBvJu`4P7dk^cKn2=#&6qXevyWP0|XGigk4IEe8^pSq9tEO<`7Mxw_07e0^0 zI&m`Th;>#f==T&$th`X>Y~kk2Zk4Pppgt>V&a-fD9jOle`zBSR*z;;pW{_s+$@*Z2 zG->VH6U#&X)m|y-PNMo^$Q;Th-F+Ro5OS%hUKq$!lhG$}X`dX6#>fsI2(h3phy+54 zynB+iJ_{Xk?;xxe3fTAZa&k}pJL+RwW`Ukl9-RMv-*$?TNwKJxsoLwFFZp>Q&B-|c z3n&wkq6}5wc+L@$ZfmA%_&6Q>y>9m%`Mquxa)1N1pzvK1<289ezNy~XZTm@(T1k=? z1twFO)=VuSmxvXmVE#>;-2B{^e{G?q_Me6MmFsW-4XtcmO}K5Z>3eW^aIH_BjkT7| zm-34_F`d)u~7{OlErc zKtz%$F~#sy7WFiLcTYQVa&UfuK*9|2um70{pUbq^O=4gCPGZ#-BF^-3pfO4Our)`^ z=5oWwh9Rq;4ULVZawT6buuSgMmFUBHw!=x;`Y%Rv{BW1v;$a(p*G(>w+&tI-H>^2tjR5%10BqeMV{bPeiqRy3_5J%ZWKRo6vgR;x1iKWmhXOqh_8DY$&C0?! zc%BNO&(bYVAhyT;g&h#)h=(RkbvU1cBPerC*SfHKQTFkp2EfSbfgke<_*8{^k>f}I z{)-TlcJ={Chpp3ixf}Y~-HVZst6t>b3iAH$rs!|qY)OnTGs5oY+IRSH1fY`t3pcp1 zke)V_f_$_-vaDS`o26;G?HIMb%8W&?)Qm-&RQi<&ljR8H_eNABWV-q%Y*7CW8(vn> z9AdP2#?(J8eUNhUtub@#@ft_FgOb>EF2UzQk;Xfcn zb{(Kdy88P1+6@o~i-gKw0VH7<#KiPr%KI!Rm4(8{bC|~bzbo~9`VQ-$WC74HX>Hkp zh7WbnAVhIQh4t6_+Akq%cL*V^3(?BETfa2kIs)B`Flf(vx581TrH_Xx;9C4oeHh&u zd2W#%3=S6ZN=knGzTo-r`z7oP<+=}S$r}@@Y8bF-OA7@L9j$;Z=xSeT$%;la9 zl^GCahqx>Tv-=qE%`ob(^r&@{UZ-*}#O}qe&U&8!rUZ zUK!`7X(*zn0SZO^iiXm%=+hK&J&0wwmRyIns$eLZoRX5oso* z(bKUK5pd}APj{XTJvNG~Z7n>*qt{kqt_K;aL`l9r+y2An=M1RO$%)5ms>X-Ra=b_z zz_7sp^^G8!z(sNy3>)7_Rl6+YJY_24sD&yIEx?Fu9S|U18B}?qbJ%k5Z=&;mo(|Fw zHU&}MeoPz9h%6Na^20f!J|JQ$2&&ePil*lRj&9cc@}IC* zU_uZDgONSZO1PN=x!*`sz8pU z?ni`{s`d^l3a!~p10oTQ+VEt9-%6TV!h@!UKfPtO=P{oI*wACt#!8>nB$vAMs zGJNCAu+x}0?qs+n`5{>K5CkP!fMLsX9_yZ)?kwEg>XqxG)Gs3byiKuFwqARh9$ zZoO!%w|CoyRzyi!CkD+ZcbY~Guz`&CBX8fn6(ie@z77s89WXlf`!$!gWmdnS{+)2$ zw7Wkf7)>UCm_HnFvUeCrOzJ7_(rBDldul^4uzzD^Iur&qucD6=xG3STUtdVmD=UnF zt!dwOT~mI2VZ=l4%I-U3)h$dK_5(P79}Qr+i}t;7;zI0_wfR$=kVS+|jJ&K+mq8@3 zObGN_8&H)tbk^Tan7|E{X%>MQ?4894lfxeHd%s`U7IR3?qsk#YuWcI3_3rln!B?)Gonv{xzS7qr2}I&!g?K5RV=|cW0E3hZfVW<wmhrDywe3by)br zApONJLwJhY=!pj0Y>E%WyPq{za^mAk-Fs38XBGPOUZr=7g2s$+=FG7xkd>ulDujJ0GxT@(Bzq zDpyfb3M(!aXxF?remgvRqu(P!Qd06jZwEn`GQf^7Lsl*R#TuoWRYy(-{Uhs4bsxX( z?E-0nKdVLr6CAnJ&z&Ql9C^`uhDuBZJ>8yO$5p}p%5@l18Nx$gD&(P5Bn z@%48-vPOYM*CVJaz0)3Bf+Gv&(qC7bI!yZ>)OI?|xMH+$(ovzNbWkg@{5doSJhVy#xuK_e%2f>aq4uDKCn%e-l~pHf1Es2s`#_RT4s)x_$4}aNJC9f-32BM=it#**|8yK zNaG>0sHF75LAwcwNAB&Ky@JkD&#p*I%L5)c0=6w7AVV)_YWikI9V>Z(5bH8LJS_Kn zD2;uCdl!$`9dC?7d4jVxTIX`;IYJW0)P3{)*d{*Wik#wZH z@%=-nQjoA86~vez-eUcPK^mAJxPXeI8Gqk)5R3t1Bn>?N$$&(ba! z1`X5*Rk8Iepk-|V34lMle)-LY=OA7zK?PI7s8_-!u=sL8-DVxeq?2F-TZUW$oWOE? z_wBBz@?oef{d5+~c-E{#L<(leiXP$z&|Q6IRW8~XtIEpq2`nb9o)#wN-p%mrT_6nRp` z{mK1A#?*dNVtDO>%d4FJfiaLWQLigT# zVfNF5S`_!Txc%^xRx^;}HNw!O`W)H)ETUZhl*1@nZeco(G{l4(Y9ye1qEl{uW}SM5 zbcFbodIXD5!9Ial!3pE;wbKNyOFzzmuTQa&v9Wyo1+w)j$8q(PiVc37KFdu5P&+V# zw9F?0oe|%89q$E3?#NTj3;FIB`r(vi9dP4vv^cf(1!vVe9hLYdZ)AJ*bz$0x&Y6-k$CSaUyQhFK0_YKOHr7ykJdw z;r2VC{RbEOkM)D>3kQf0a=WhQ06q%O8O4IRLukH&)Dt}jB}MkYfWJM+ZLgPtiQTVP z^?pD&p!|=78m8eT9lnq7o1SuFMg!W{26{Gw$gB6WUk)M6bPtqV+gTqQc&|k{c(0%LH5cI5IIL>qqx7~LBI4Y z$U8>>S63U2i*(~}uYp1UAnPW&;1)h-|q{T zX@DyH{Ojb!N&B7z<7RFx91eGhCVOGnre#(c9cR#dWB9>+WU;9K_>*p*{`bzTq#LMr zQjC~Km-@*pE=j3__&q^GX+}k%zX??-%OAj3kA0Rx!$3`T&04tR;q-eTbu8u#)wj8R zd*KPgBKPb=k^kQt)(xgdlns}d-){eX3dXzFQE zqUfu#$329JzgLd=_YY5<9%-torv!=|3m38~s(FI#1Be1iTNe-Rtp`&+RJ@^LO();< zTsWI%1ZYnUZPCdFpjL;UVg#?P%=UAEOH43``*x0Vqilsc`PgXdQu$J+Nk&zX`9~tl zU$1czODRxSOuq$-%wcfb3lnu)IrVlO{P+}LQ2C;{B*5sRKRsO8h$~_x>>zvWBFaA4 zH4a;G*!G332}!`yr`$B{j-Hwgx0E~5w!~g;7=Q#g4mUS!^As3vY4rIInw=(_s#uN` z9jo%0)}UGu6}>-MQ_3DdMY$pnr{FBiG9kJmYB1~ev&y4leuG4l%OiUr2$DvEn!OAxmgf*P zat+G2BycnM9w?8107eL!G{PP8BXwUGmM;NONn=nOlF20^4D?_~&~icsGAq9v6GVzz zDxU-d+?6K42C9+mfo6EaW3;bp-sFm5F$=a-6^7(OSFIw!4^WFznKs*uRh9uOR0-g% z^CkiR83EhseeDz{H#I>4!(c<_VLCDXG}kZr8yv?-m`bx&ch zF!DQ;-kn}2#Sdd|r!OoT1Br93eeEB9z2H$N@pC-TP7@pkX%ruvSABGjkhC6>iY78*)a{ou|9d@{jIvd@=0WyIR>9&frZL_R$jZ z6BsU{II{Y7-+MQ=xgBcfY1OYsi4!@;Hs1?IIukj)uQIJ^;T*Fe-S&KGrEy`~dCZll z3vg__)hVVDR;2a;M66waU}AeLMeQEN?B4u#EHpLB?5nxF~} zax{A&Dg`k*926sGp}`SOO3gRC;Xd0}(E`>?ogmjTuWe&@!SdT{axSJnT{~>1D=t&QhUNTL~-ZZXtKc;7uV090p z<>IloUdxczox;0-4dURKkotpwr(YmG!+`=P`Um*5xx;5_>40st&?UmUcNZ7i_W6a2 zZBKtI(ZImqno@7Bu6o`j_|`D{CBr|>9#}Narfit*n&gKq+UcekG9js`WiJRWS`+0% zym5haF)&8ZK?Y?P{>=2}7tt z<6rFnPl3|I1>TEb1L99|1)})~lw%+W6-N+UqzzCpbk329!lEx#&%_F6)$ zdp^);WUE69cYQ>v##jk=a5w?Lf<#@OZX#rWzd?pxkBR$RCUl}8*pZ8ZFR0Xi2jlH1c!>fO_k?ycdogC!sX%5SA(uX^)W zO{8a7&N$4lfmonT8nq-I6^MU~7q=b2q887CJZR(uRY3i{hfbY^RG4F)dv{?T*@qIo zJ2;gNiY!!;;M=gnEP(+PhY=hnI$z3ilyZ_+f+n@zRsPg6* z0`{x^gHt4jPcx%KGAVJc3aHkm&(o*ms%WO-x~cl?!exl~#)sa(r1kCxV5`GS8bS?L ztU`SDI~W5+2hfpdg#shp?roSq(b<1nLm!bq*qjNmU_lxM9?6oZi!NF+1{2wWd?jcN zNuegP;EL2aAh~j$6Mb@pG_M*Tl zJtHkk3-{*?ZB?{Y8QE?Dv+6wop!BgO5`x4-p=W&PBObCks@3w2jqXW0H0kTe;X=1E z4YXV?S|3wOkRL)u|#%qAQ`8DZ3 z<$2N-;7GWaIAoOvT`(w2I?dI>?b>c?M6^xPZf*UZ1<+PD5wM&(Pe(TbOEMCsx9Z_I zGp8pq{o)DdX04R>luM-GT0gjGqgG|QAX^{D&b|9JnTfeUv8DA4z7AEO=*w>f80nva zXsCdh>qLI?}EUlLh+0!Jvd*4;gWi-B4{G zr~dyk%)7e>3HwnN@ysa9Ewva524mWPAyj5xRRE!4g4v!GK&X&8eufa_78f0z6m+3j zfdCr(^{WO5p+!QkLk?ni61X-o=*4jV3sp+U3q{%8=$Hoac+P-=Me zuHG=vhhANEJb?yG7&i|67adjOn^F|!;9}woLlHX45<%bJtgb2T zEnW94qIcwNaU3u-d7=UG`i)l&5L0c!P{;4lXtl%qyKrmOFSr#u3GjVzJ@qKAqpU+d zu6}qWJ>$5(3eA&A78n*i7k>Wb&1J0jPV6}!vU?BWzwL4!XjiP+TQ=M1*y?KLD@L*- z#uOcI((4c**zh;8`Tl#8x8o=UqU}2r>CXd_sTzy^B7=L{DseB zyI08#?ir;VU3;-ZchLV&x`Qn=RO6~CPwc*V@#!`ER`p5fz5O7$C_sZ0FU(PLCJjFxD;GZmE!=3yg$*z?uNTE$9s# z?w@T*@@<@i$gQsrHY4?)KINB#m>=&t^xTZWQMr$karbSFZ)iIwmsc78Qy!8p7Z9fi z#EtA!TCSlk(jb-VlPFA##D~!%O5jBZlCl)INg!$0U)R(#4n^iZU23X>66AvscjqTJ zkcle+?8F#!DO*5*+lFHIfnd2A1Q%n<^Dt56m1b{Z8kAhzwt=mWDxjd*-EPYaNn$1# z05{tps0Fnk5bU@Y9NjD`yxT~H2I+YbfW(0Wk5&zknK!^)@+W#AJX0(MG}>~#Kke6s zM?;im3ot~b>*OF<-yR#X?-VtF@)b0HF^G2XF*iw#Jbbrwg z6SEG7#9z+6H*Yu&J>w$ZgZHig2jqQiZP$NMq=}wN zjhhU&y_a6tZa)D^7Sk)6zzDNf%;Q|R84c;Xq%m9o)@5h^G3z%l_5C_bOsq4l55UmJsVJ zRLEUGp%qphr5STzIiK#=fj-tfKtalVcUqyMk|k!Xj3v+wKJB>zYVg`|lR^0B%mYtW zUWvKP#B=~W^y?*Yw##|6SgBLMAbIvConW$;?2wZzsu$CpUIzRpMHqKjMb9WKEdyf8 zcNH2JHNgaq#a?*;fC|K*<6dat)I+no(2(QSLuobKf{-benLI?PR5`hG`19*2AZ=hk%yhA2W4u?p>Wv`Jn-zYiMZYCG=8+u*7po12%1jy&{n zvhIF_r`5907PncANK^sW8z57%Q*Z>nUyTsfj^R4-?z{-levpANfaEsuV5b93Nyb_{ z^zK5?JO<#`JMJ_`kDjxsSdOT{jD3ot1Mu=f9Q53Jx}}Ll82{p&oe6kEuzP5F#rN(7 z7Ee+yBO%>pRLbN%KHlvAQb;d6*|5%9$|8kGGr=SsK(D8Rfn-{Uo}`41Ev zv(ZY7iK}?%xh)c}lARHv&#C;Gp0rw}(g6`4cG27qc^*;GUKleLiGctDPaajVpdH#F zcr*Ji32l_ubo*E1U(%HUgDM$lo~Xc&QD^`;&G+RX0)%#_6VG7J=LPK*bkbHvLj2sh zw?CG1iuRrYQDIbZ^KKX1$a?6x(1RKu;0#y&nSjOt_L^^FJ)PP0# zM&S%)zcuzy8Z8r~VTH#LtVGSdKQjzrhJqjr0{SE9Qn3_t$dtns$Gb{V_Gee=!UU*Y zw;rmeD!2R0k@({cpBjVBgm2)&fTt-utpT4kA?Lag zTTn$c>nVQYcUn-r)OmVh(s1`e)eKu4a)%SGyl{%t6b)~eF2V7dw5(lUT)5|z&hp-! zM2fj)Kf+b96jHNF>o#=}w+N#yXz$$k?E_-IRwWpbM18uVL@>EUzPq(8RyE1UxDM6t z%8>URzE`USl$VgRCF!@RK_R5dF;)E|TG=ATA_QmI*c5ln-7#0LJVLWDZq&-gEW2cs zDoWg|vS!`DeO}t^+v|iL&yDH77Cpnvf7Q&L?cYo!pNAedHMQLeY*n z6zPiapH^ZoF5Y3Y(k+`cLO*DFhG+s45pbIfF{8T&9zI`Bu79YcKZ8nDW-Nl zAXYIPI`p`*=txCvmE`V4Z({GdAIY)kxKon6()FD~c8vlKxJ~dfQ_s7rJcR%8!!)`H zMuvvrkbg~t2Vrkk*an44DfN3UnS2g|_7DzgXs9mF^>KpB)vuSn(|~1ec}TIGSWl_? z=aIVrKo|i4(Ow>nR$ri6vAlBzc`djSV)xF>77qouLmvRD$_BLCHTIln4q*GS=xmTO zZz;CBgZ90P{!m-69a9QGA7^9_LRq;m77AI|IUdOwYfPocaX2``a9)jPz@7~594NL1 zgIel{n}Hi%6}$|{H);7SpFyErGswMkk`OE2>>&i|mH#E}7%%WYsYL5{YH;-)S#EV6 zS*~*7bqkh<5NSyiUWJ1%Ui2sQcw%R_ph87Qy6_xVOjDJ>PJM6_e32r-hff}6YarNF z-{Dzn5<~feSf=Jm1nGZC1FZe~FTdGN1fk${Wq;mPrSG1yMd?RJMt}xr?}9r>&nOb& zkHp_ZQ}c%%$B=wYJSrVg)ax8gL_$J6YXx{pZ6K}9_=&6+V5RVBa8oF@rF-{5AZlG( ze^%Lqlgw28? zKxpLvsM`A)t3<@SI9N3_qf06kLF>y%@*BTJ_Pj`81LrkcaG zVgqKZA?4dCcUPW&)Iez)^>cs*unRmw1uhkA2(eUP<9G2uHGoUZ0L+j>bU=Y%ur-$h z5|L#xiUlw*G09T#S(=LM-X~@&;La{ARw~O3cs17l&r)|Cuk!6 zo^6Uz#U0-VdY$M<+=8}j$-e#P94K~KKRLfxKkQ?V{zqN=R4~U}6g^ONwR6IWNR9$^ zmV2lC^6`VikN3G|B9r>$|J1*{|C`7Ilup&clo;lg1LM~d-}nItzqj3*^YZ1(nHmVg zQ#4~(qbC@EB+q`Jn!pT9IZ$2dg>6`$et768O!@vfKyT5}0YkTUJ`kO*no<_ldsX0Y zdwLXVLa0MC;m{tq&cIiw2A!mG0CrkRoh;I>CO&qY_$m)7w3f-8fCr)R-F0P(1t1;P zw3oXl^Q|FJLqn&$sJ!qGdMXU>20y?{O?<3RSDXzM=PHdI5C#pu`&>2Pq<7;_%=E7Xkg)GIy>ez*m*uy?Zwd7A6tkqEZLOtn}c8 z_wO1DyUKLw^Ul1C4vb1fWgdcqfB%7qU}Yf`can#Z29!yHATO8@w0XNTfQkX)9ZF_m z1;gEbFTBV7MSoyY=wn7s03|OBtOI2$f%5ckX029xT3$66WjMR(+(5@B;4{b$2ctmS z$F8Md5GpK3KLe`{xjLj%)OfIh&;^+ub9#bTNlEpnK+KPFP!W#=0}x8wGYDK?a}4^s z|1ea-2sT<&a47rI3(5|dekvN{q9A8<$^XF!c~!8`2pcp-;X5R8LZ^BSy39LK1-Rbx zaqIYZH_0}{0ctE6PmV6Xq4IlwNDOob+e;?Xe~N5!$qt4^STq)R*f+>c+bNPndv%uv z1eYAWL)ig?#thtLuZ>}XJTOYZ6dV118*HxO{}191T$va_JaqI?&6J|CZmGy=akmqCODMVNXtJOOTL z?N4P3>>mYZ=fZjIPz22*KB(`;M#B34ENxfu-^$2-FsD}r4pmvYl2y3uV>q{TSp9^w z##F_^z6g?oe8;B$8`mNo-mjW+iohpxi~)$J_j-D^as#Xwisg9Sgi+0Rr@|~ zT;o{p*wSfyjBoRCn{G(FW-FyU&8MzeFwRNl>y`u)2kF!u`B)2EHtuY{)1U770@9-o zt^K6V*Zp9f1GBTtV79%9hqxIY5ph?q$Y=-;)LLE`ww0x5HiiJ99r*AZj2Qzs)%Hhl zn>p~^OG%F+ewo(9^9A#31ZB?JtEI|t5sKPQ;UD~iMWDiQ$OxJD_1n7zTn1g>Xnwg9 zD0-J~4b_lx?^^b+^}5rAMfVbHfb6H|cNap;70=uEE4z**Q8ov6g}A@Z7Wv6*nkHN3 z-|}oUR?+_k43-XjN;d*({(%DrmVj(&I^L4>^yyPHCct#d@l&T#p~TIw>8~sWqmr)> zKhxPGaE1{7e!V`QFFI@RVp=;Vs@XT6ZTguW2AJ0>*Wmr1FixATLI1HpCEO2D3n?or z>#VjB!FDw~$+-AHPr-A>=lsry-{e8WR=)?8c23pTTF3mf-Jo?9yX&)JOy3>;5z{CX zCACFn{x!J;M}mCDqK{hxgJ84#1jkW>t_nan+Vqac_xRP~hljnauK>Ahc<|~_l{J!1 z@N00~_ONad`sOs_7(rK)N%O|1dg*jpX3isab~xL-k6t-H8`&z{kpZRRWMW+@f16<4 zyu1GT9?REvZy{1GO+RKdzFd65)fi`grl~$(u;N3bGOUPfZ#o&~a+PD=XlE^+G5rkz z0waufXYr^TQ>pLn8>ABYbdI|+IG6N2(MkJSZW-O`R;=YHwbA#y#i>|`h~HKY3RuKE zm-Qg>Ow_GVY<>7x-c|hC{LtM!UYIeu+cS>8Z+2?M;k(JEH$JIXlvx#UFD_cD%udfF z;2nQtkdz+2mI2O?Fd%|pFWCaQK}O~=XYl|bhhAyEMxrO?%iJzeqd?QW%TAOw??HpRU?=3%Z+0=Ml)alr5Cx3PC zzDCY_Nl5+0Y*+l|OUdX~63kCi^|8yPOjGP_tMe|u-Xj>^0~l%++spIw&EQa<>C0Iv zhvIQn_$q_2>Hezz)1si7(bfE6cRdMw8y)iIp%YSf7)sPIl_h!Y+UHryKgHhLX%-|| z?r~-(SecB@Az0uS=B?r($~uDEeRo(Pq}D$n!^;4He}PPB7yWvD)Q)@>Ki(Yt^W3@i z>?MDXJx*S0YRXdr5(II5;b9a&3d>LMja8^QFwT zrUZI|%ddCXt_Gc23GA)>xq-vVf%>UQ-Y(tYvK!SuT?GsuWm|My)1Ntvrv#WRzWuep zPg2N&bHF_P2e{JavTcD&g;Tqh$#vI~-R~COJ%_&|nfVjH^!&5dT%4Xi{`NWlrI9Zd zCtT|HlrrRV`9s->@Y&z{@hUPh5}rc_I0>6VV~z@p>6OQazJJdLzQ1?)?!vfL_28YP zM;^cSrI!>e6Kj{t7JgjMDM@ywPV?1%T>pJ^&eH7fc~Bou_JHY0fLO>#ZAEUv`a64N7&d`arM$m zx&jV{=4`V1>c?5jrbSn&dNUJydV1F439x(Yln)l~^(H~?-MjZpO8mqm=!+?j`{(Pf zz3n{pLTf0G?DxTq?Yh5#Drw^DSn4|P)0@|tWVl-5mWG@?1b}53Z`LbgZaU<=xqT41 z{%9vyl>cK74IOPT;wms?q5L!LYGC( zvlG;$2@ncB$6(0OQ!}zl6##e4L3wvJHa1%95!~L+3W|!YaLzh`T!tx_RRsL$jK)72 z5{z=S!r5088vmm*e7CdQrl*&^G}($w^z=o^SShMEHt5+Qg9*%I`EbVZw zn<>@R60xvkTnFxxID*-?K)#=ZF&jI(1|$qlogY7bJmgA%w(xbKkK^!Nj10@xMj@Nm zvZOx6T|rr-FupFxm$I5)Jej;wv8>iHXilNge-4@i#iN1;BuMum7htX9ufpUxicmUe z)9t&pFkF9_g<_yC)b#G%WT#FX&KE>Me}BNSA@l>jQfyobS$;Rl;OezO+rV0vCs+|! z=50+nokQ}@J55^Ri4^qxY#SYpV;3COiiDuqHCw(IfacHFN`Z2!2lOljRn?AzT;&6= z0dCxRVcEK4ot2JhdS(?GG<~(OW3#m0B%7OSe>hlk+-qH5Me>4xZ>wT zMk(lqSUx9=cbaT>1^OJX?SO;_FzQNwy!1V-y_+K1GT7D2@?3HauI#C}N-X^kUkcg$ zLYto6-pTCG53e_U+_@Qu1ZfOmd}?Z4wqrs<0{EI?Q6zF2q<)rxtGAT<=Nf&4Jk?qg zPfkp(HL`|GSM|29u5@?|N;%1<+&{3eVE1d0Jlum!g+G6u4lRnLSFe5mb*sgnzwc)6 zpK-;MCE4%#%7IV625uC73$IKw2%_0Xo2(t;JLYW&2?jJY%JQ# z9Bm8-07K^qDfajpsrSBiv>5?sYG2J8Ny+r zY3F6`RfC7q(#+NI)s54gcQk3vJf!|Hm`&%XTo$*GU##hhhCH=(_V6ezaA{kH7O*+w zgr(a*Cl&qo-V!Mq8=RcX04FvbG@beM@^>he$hvk@BXxGQHnk}*XQI4c{B_2Bl|?M$ zPxEh1PtUAkt*5?t*LuMPFntf2e-638k{5o}3wB!2O!=b9iIXQMz;yBkG}0Tw`%Vc~ zfBYy5WJ9$VVjjnj92t0G_~%p9)NZDh%wK(e^hDaT;Tq?W<`mC^%9DHwcXS*hK4htH z1i;0|KNWC%mkiFx%PhGK|js445S-ZvErSsEr%DHTTc~k|w z)Z4PW(`x40nsAu{wBCD>Q;aGxsl*3Q3jDPCb?=aBW23RL@uAqsrJm6bUD2NJvPC|d zQKhH#%1mU|PP0O`f*`3$1ju9j)~r|V%~RY5vTL4%IlH)U6&(7L&wlWr!9G_vFd82D z(Q9SF!*)0AuyImX_(y4u8XjK5S3h}7wbG3DAYUzYTM4rhfO%tSwy7#7_XRj5U!g4K zHT_bZFhY3xLvek^wUoocr+#PW@eu=JWD-st6f7lLu9vLC^#k;hU&S1XZY`@<*32JI zmOU@S{L;D=jf(N3L(fmzl3piA)M&sdss%<*vVe4OGciaekR}UsPg@V|{r!y*&Csye z1kED$j!8SVu#jjg2UiziNW5Ys=hMJIE?z2$9Q}%XXuC{W zfFE`12~FLE@^X_ZovON^9PtoJ$l$AdE&#jBv-ddbK~K$ zu9!aD_iP=Zr8X#I(v)wB?&NX^=P6_kARe=U>ieUw&!f$$8EDd8J*A}FlGp6Jv!AgqD`p1CpkcO&frg4v>)*9p3P25?%d)qJ0G_w{IuF*_8r(&F6Ic;_abedHyl} z*~JOTn>SCDl{|nlA@e@R(UC zC8}nd8ts{{ug6j)m=o=TVGdOS0)li7zaQzSTQX{ZyRf+E)M*@P2C$`)-S0*F)jlKO zy@l(NBv>kB=#58j5Kq3%u33KZ#dBeKE^`TuP}L=ZBH$3Tv7!yMo9yE6Pt}h~qr_qA zRd}wz;_#;x-_9TRPut_S*Wu11Y}YNoEuUYw{ebcm^!E<+LXUl6&UWqg?+;W{_Uo6T zpzTUCd3n-&eozk%Ccf8kGy$g)sJoU!B897%;871Oo%T$+2cu8p;CIz5I2Ln9491L{ zfPiBLHu3F!_?+^Er-EPc^}u?~DVr(9xmy$M7ub-lyx4I6V&i3Ki01=`?jLV&O_yEN zS746+a$*WT@1k#kkX?=*#__Oqs%)wM*XvfLxSDq&jp%VSieT$QnI>V(r~^nGDS+ZU zCqO=k;{6ZaN+j6?W|*q{D){?O1PIT38LaVcbOKXTCKEo8!V(2*TywCf0`5uU+27w3 zDNwzAHP}-uUzuz6W>#aGER_Fly~)qpt-?61PjLEnr^TkguVT%CPY1OH^u0bwU{Q7| zF){JzzXf9CWi_pm^r#l`d%Uu>ZQU3UunVFx^HZ$G)UDKE+ zY`!D(C~-@?dqvt@$2CL`vt;u{p%uO*6v|n2spy6qxy}$OO-w}e16*cXA<@c&!6uBG zTLwJeh2>>Y<6e}8=>GL2N(w^ld>2cO&Rw`wjh*299Tu?)VLCxMsxDa_pMFSp#X5h= z7J1BXZL+dP+xJS86iwGtI|&d5w|R(#WuKg&wWpzT>e^|g=REZ>3)SD8?*OL%>s26i z!!u5-XWKh2Q0t`!Wj&O$`gtnPyR2UA^ojUYVV75ooCHVK-X{c%I7xs0QYID_cwgzYNlr0p$y1tXcR!s1Z-Gkq%<|Ra+ zQxoSp!Qz3+E6V_zt8{rC&WubT>zl*6?JBlQBtLfS#Oc$S(DE5i@TTETxssZiC>!{E zy~_}tJdIdfTx|SUb#au|!No;-!F>6y>iyjJ}>suhYEHy-o2*rhJ&CCWbZw!wn zXLLDs)EuQduAkzwPUZC8AaykrCn#?Bnd6|-m65MUO-hqTTmzeEI^67^ZZM^gA{WGN zfj%vSpblunp}liKE4%j z{FaCxUh_S)$C8*{xaUe@paaSxC|IIPTPox<5wqY5n1M3wEpc)2#j7E~l_JK>EFi{4 zLz5)_xwEsg;S)RF|E95ZXk?^jT8G~fD2CXNTBZAc^s)Lb&=%GdXE*bI82bvSD)zPS zL63z37O3P=js*fLjnokZ1?f`Spu43pQ7M&@5Rh(=Zcr4IZlqC~O?U6_`R@(hbKduU z-}>fVcdh$gH#4(m&-~|ke)TZl5eu0-)&IaTEMb#Jc50?Qak8#wH05k9Qr;`{ny2ro zc-j9&w=S<(dNA4Ty?$_|JN)fTyN}d>F0OxP1A3*)&82HIQ^4z2k3PjMpIrG`BvtgL z7~RaVv7#mS4S+aY$O4O|CorFo2zgnE2U(8-W5AEqqHs^)3}briK){++u z`|2qeKBtz=9YG;Hq1#N+)6)a3BnFxOv^X=Hpq{QQrLf8FsOrZhlbuFCjm_J#SWeCN zE^f4ZW?S}UctV23iLYm4&_jA-T2-(2#wNb-nD2F*^MpZn%`n)NcYkEzSI~FLB+Qh& zyEG1#tkR98-c)E2Btc{Xq(aiKw-HQG713=nxEi(`D=sds{Qfo#KW@gdFYvql{b7dHdzzUR`;BQ*TKa}3L(^v6Fy42qOQg@8No|WRAsNL zj8t9PA*`hpo^`V1RH*Z+z9dk#wZ9;NuL+G8rTf%(OI!Np%}Y;q0;pXEFk%b9iUmDL zTnn{(=J^^B(W9~4xLH6fsvGthl<^<0R{jh(eZ|sCz6hh{K@{(60l(4Av6Q=ctiu{l z*3PDQR+blCDG|$UG`-L%e`GASX5A}GYQ8?^hk`-s2?q>x0_;X;5bqeqG|bad zUe^nR*T-zp!%1E`b80V z;}F^J`wP-z{)T7+B{01w%kKtjT{|77^~d25lPD6^@UM0kvqKMPG3@LZ(Ja87ZdFUhv{~+7IcW-V+B~%a?e9M{z1_yWZasZQ4;mWh8!xg9 zZ8;Xnns@#zP-V5V%~*IsZ~pc}Gq=!} zDskFkHd~NYROepx&g|I2E@`|fis%fF`9-@c_c?X7j4*N(XGG#Se!Be^98jmBqh7&=>)dlNM*ow$FUx$a#ct(-IQ zcfSyP(4df-dxtQvgf^31@i1ck1~d`OEs;I#Pu<)=GI}QnSSk(s(bJ$$kb-Q5^I5f! zyczE6do*LF#09_EqNGGP9v05^4SK-wZW>O&G&|)EPgLHQDQiV#j3_rwPpT6MGYc4Ql@HZG$K|f>|_jAn~oB)>!~8 zowBVg@#|r+e)3RW*d#S&1Rpyuxbalo&cmSNlhLwJqO2O8xU@|5YC2-}wn z4Ueji7R(;+jL3))^^n6qPbT)I!pXrq!gJ5M*yR=!F!S)-hP&-D&9}3XK~YFr(GClL z<2E`Qm)qQBRFq}yrvZ@smvLeb8IV1h06k=aSdk7bs+yWfp?|E8mp==+-@4GO0FQ?{R{W3Mq<8ak+YKZTMhxnI`ed~ZR2NI^Qk|dB392;rRD<(23Pr&ySwGU_Jq>7 zXEw2PO_mMr3p2IWmCP5?c%59`awbPOgni1~65g1Gqjv?|-UCOm)6jX1!5g6oN@=U< zzQ3UxWP(}%tSkz!rds#d4POE}o3Qmo{CexH2Pgwt(@9^E8cQ+!8cZxGIMAxD{yZ)v zCycn)W(AY$uqn=nf{q}F$2@}S!tLo)9624I0vjaEf{js;^17wIsIch6hq>0g+ZdI_ro>iqVkbYc~qR^zTZ*TlBqEulMH(=%-YX&>?< zn}m1St;FS{++G8wT3J`axVML}&Gk8HL@C3HqceDqx#3#9HU@@& zMvsZY1fsZj9x;-XPd{0F!K6gZuHyk{*WC#c<8?ZaQlOq4EsCJ7hxiXZi098XYQN`o zI1T1JcLTbs!)DOWM!c@B9iK*MGsnM~z*6+{6CJc$eBoX(R&*bN`R3o`T%o&!KK7(6 z)dK|H@jyrzrXt!8hu@sSLy0lW9)rxPo6G$al3+-HwoeY!C@smV++JIWynSChdiQ|E zvBqdw(~hGjg`aOXcRk}FMr-R%`rjZKraPph(0Y5_FC2=@SKfHsy&k=FO$ZzT(zMc# z3nb=|qMmZX!miF>0TQEXLKGttP>}hwM~+0MrKQ2IMf^4f0l+cVNc_0RwVo=J-J8OH zYK)_y*@V$rKXGrK>IFZ@Ujv|k%WQcs=0?H5#pnQIHhYkdm3xzBePd$*(jDRfd=bBK zLk3LMbSg^W=|`!kn66yu%pdRXXWY*ngVdmAaQ&s+*Z4~XcRhO+M8gn0D!ZRDm_R%> zn|?sV;}0tJBzce42SB{K2EDR8+DA=GkJUqfg>ifOby!f9pn|9w?aVPX6@;lX3Ba%R z0Psv}ez*n%#Hc8B2Ug1Zkf3SWRxF!E8p9bUK9Q@sg#&^CDmmOBL3MF!cJy?Tzn z_pc)%(tHe>6FGQQblV-5CjnnxoLtqGDkt1R8=$vHI>QBk-o(G%$Y%C#sNpj(_;Q0x zzsn!^ELf2;F;`G+jsh>!2$+2hc-94s#2Jnqi-onaox_@VwK3NOpEm)KwmHcSN$;Zh zeEkfR>5pX{V5iw!)A{1JZ`a6e%5cCAp;N&kT0F01RoCiZ{bdwOb&OM3~OTgwS7&8R)yB6Y3H zvThWt%-pem}60*2@-Sl901`t^sbd)QV&oag*$t(Qe=u ziq)khUmpM9^@)u4BJ5UJaxZoqo}={ibR+skJhZ+SmR3cl@Nfw_4s4wvCgb$~1+AHB zQ<7pAsfUE1HRT1Na{%u)Ip#X{=D(n2)~c~2`D{jUK`VQ%L?N9A#&jIJ2)0f`Oc>

%1Wpq3qs6px!O@xpy5`^P(Ak`m`Xw=9Vq={}!4u-W^=E~b+$cER z)+pxZ;|cf>W(Lq7>J`aXwrIFg-ekVRsQ~bdwe3`ezv*JTjowEy{}5PUh)xO(4b2DD z=Y)9h`}fHZ#Q}gMD49fcG=z<{9AhMQmmjYpg#B*O>mRt7n3x)WiCh6q!B;oic0aMs zQhW50G|_95(3qs$jQ^+vVBS_aDKn@%2=55{xf<>T81ci*%38ac-*B6niYf;FUrjW! z3IuJ+E!TH4dCu6S06y$+^q=@}R`O4Lz=nAcJ!L#mMPX?sBG70Ym15AOPERGehXuVo zWaQ1gc0fz?#{8@1F4Fz`7XYpq%Nq`r*U$*Hi!>L%yE%E<_~UJ>iLACUfEg5^UrdO2 z$jS`EJs>5sb@1K0HkB5*+J;E6-^e*3gb%+GrFCp(bz0aiB@WVD~^f($?vi4pGz`^AKleKc=}^Ez

t=vCT0klO~6YSe+O`KePj977Q4k2NL|rsE2inmec#^i^3Z3I zV;y}0tk1pAn?bf$zM>30|;{~_G z(i_Js``;?5FWmTYVXmf?Jh5+{$E&c-1oPZ>GSqoq($l4d0xX00L1_GFCa&xazy-J#{nFFCqkRY z4c6I|a!k}gL7Dh_&F9JZGHz8wTX4(tOR|EBhF8s~K-Y!*k4~m&30&MR1hK3OSZO`# z#ha+h;=mK9YGPvIx2@nQJwaD}3$66_ww(srL&NPMJ)PYm_|bZ03M;x8B@%05HHl?$ zV1IIOC|tU9DIcUO6XHU~9UOy0LkV`4Lv+M@*%kLn`bx?$7h9{?l~$&l>9F`cRaL!Z z5;?Zi;ot9yw0vRvhIm-$Hmi1GkfKIl$&Q?rQh=hp>OCtcK>;RjKpuPw3JRzet3M2G zsVfX3CeFVMIzbm-XKO4&pbDPwJ6S0Qd;*OA;BVCi27^LN-`n=YLr~Rsqf&0_IY{PF z99o4r4XbcJ%vUm}o&)7R?WzmdH^I;TCURTJ+C!9K;XeS?;l-U>3o@ph#(D|b8c5e8syY^gmoG|={LtK1bpTh1mSa$Of(9!FFzH^S z5pQpAq5gc)vXTZ|Z+*n|{TuM!)j7D;ILQRF2o-k19w$n<3L10fDH=pT)#gv`G}ynT zYQtpa6mMd#1aQxlWTzF_7{IZq*xKe!vqtu{Xp^i>)1^qKCP>?h_}IZxDq22ucR$5s zJ=CgMP(&8Y>M`8b^tPh;VA#3n8{pu#(S^QG`0j+pE000FsRv$JL=y~E2S$SL$FRbS zuClrsWpP{;5kc*kWW0ITR*B2Hxi((Ocp>Ty$TgywRcw6< zzRHaf?vg&ogwucJf_`ggYpFCl4TagNj%j9$*mb7WFDJKT9@aF``&Dh;eD!>e_)|>r zIF@GE#)=A8FqpRbo|oq^|8-5QaWW$TNz@yOb#r6hQFu@PWFnmJfFUl+>m`_aV_i&1h3&W9Iz|a_AYGP9j+&+L$B^ zOtO1A^2OP{vMS!Qc@D+WKund2Z3`6bEkzpaRFMF-Y^}mpy+f#;p zCk9ncGN|E4;NfdVkUMf|_4bx@je0ih#B}8q6cm{BzIJ!N=Ux5Hi5*|E|@TZr)vn~_GuQg*7W4e*{;IN|dD+n7hfH^ae)!`;$F0&ZE**1)Mga#ue;X*KqCssI2Mh^U&GpN$`hkw> z#|Ofr3VRh-v9wB!5o2qXUi&rPaY=IH@x0cuS$D_Buyy{z9|Y-_ff%epz-iS3EvzTs zU-AiLRtQ9=Vo@1L+lDH?HYu@QNmoUhwa-tkjCY*>m!3oUyR<3#3R;S%ZBIfJKs_YD zDcrt&&i(5>`3@)|M*#`9z1>dymVF%_&+)zFM$ow^(|hJHdqYz&5ASpJ?rb-HmPumU zh@JCh+AjhrkWl8f!8;Oo&EiIQKM}SzB@k`c?NhCOj$f!)1pM&Y`!k5CVl}2bX-jhj zs!r%IwC~eiCVo8E7j$Sfp5Be~anD{!3f@zucN)v7kL(uJH*#kMUqa$MVze<=f#1$B z2?VnJPoYIirb#46g8m)??u9k;Y`p}raHXo+x+ZwHj^)+3`@Mc+1S?6fb#wEJH`I-} z#?i>Y3J?1Jtrr6p=XZJ5Vtz84lb{RJjcz9>DB!CIlBp=!ub>Zym+!)k9z8lz!`<5q z{XeJO>;rgKlPtt{^IWpEwSD&VsjmuQh7FQh&HWOr%ZnP&8THKdv*)!5^cqCv3uwmD zCApDiOdZDsav)g;Ks>e~6KD~Tx&bV0J&whIQh+2g>T=~S9!_pkhdP#*yNH194XLwOG>f}Z zFjojF0!x629EgOjzamvQ-(PUoanXHcdHLJDk7h@IucA=Wa*B+jK zt&290x&&T-)G^eg@#?K(-pU_idaFfu9+!b5Qw8NJ?^Xd; zJqvu?HDN-ULL2iw;G~O&sFZW(&)&}9 zmel5C<=|(2{P=O^y|1KD*raulAv)!ihetD%xHbHQGsXJWE%whYjHWCEohZ6d(BuUx z$#nTdH9fv8I5YMq1NsmxCG50pnMNOaqDYj|3);9Q(D@I&-dfnh4IT^*k3N}j8Cj`u z&k|z)3d%6U)|;TWZw?nVCL-{d3=MQv;St~}f47B#l-`UU)W*F}PF%P*ewpsOcm}gK z7z+qhOQQ4KRu9;G2Ht?v3kxm3zPh(ScVE!xP0c9|vrOaWEr=JT)Ld%d!59+4TUU|( zQwVRuweHrUKTfRh30i2({c8lgDwWQ;BG~eGo5Uk}&n6#9#<+heF|n{!49(M}rlxr-;!)#2D| zi!bx0+{Q4ob}iZIQia*5)fsCfd4qb(z7^z3a-{uptbzgd6+@Xg7YY^!zuLA<9?`_;0^z8va>BEfS)DHR+A(}1J=2aJzEGyAE?)mlpz!1 z+K^v$Xda>$(=?n{wAnYlyM);+qhYHzbuiX1u)#ny5DJbChOpppwf=D!=5-YlbbxsM+|ZWlE?!hhfW zP~1+FJSzLU>-?aUJSGezL0#|DHGe(&!4Y?149$EEPg6m zP6y=xK+@T4jq9sx+%u6fpMN)}kulf0w@Irk0y_u{0cf59zqOxY;D1;@Igetod3BWl zdtlB|Bi;xpOtD}I8owQ@%grX|xVjHW!j{Y@0JbSbW)q<-*Tg?Cm_ew<;5hw=m(118 z&DeAaz_Sq$&Rh;#8f~ih`!w;3nUNt;zG)|Q(ZP&sd)VcHJ~eW$4WhmzzCn|nuys=z zJS5^35Y?oKj5nxtfmGQk%_dMC=tDQX+Kb=RWnCI;kpRE0xz!d~Y7qV3*vBj)`{+=l zv$YhJ?%RR9bq2dhAoq|4E$IlZ^&-`}MJ7W8D*{EpRAVhHRSO;dl;X8D>BO=ubK(P7h~g}7Fm659eb+$M3&6`wh;eF$9V8-m zv17=~h7rwby>&Dtp$6Cw!X_gK@Y=i_6lYo1S&eg+Mm@%%P@HuYd(*!6~uL*-;0A7s+K!{l%D+G zP)1AUTA^eGIg)u4zwAvqwZWMm!2(?!t{CXdA_ASs6XoHWn4k~jC-xQbj2~2>&r#-m z`Fj<~Yw=wx<`7&puaxb4~mK1+w_S+T13Pt7J)*Ex@*^& zX6wlTE~1kWlh606m!w<$*7-i{3A%pa*}uGqOhNc_u64YPRxpxf_=)v{Rq?*rPSinw zh~cC((W(f4Ad^7^P~g8kIZ8!2=Jxb9auBTEUz@08jF38l%#8uruU=o7^G5BOYt^6N z;FMwWgas0p4yHZuNFip;plJ4i^78D0N0~6QE3(!d>_g)2^B;xgQDRw37 zD}mmLNcN&#Vf|ts1_mDBEP$)w=U>hTD;`lY%q-S3JV-Twn^UJjm`(>Ucgpm-oCb!l z-WYT|dhQY)G*fVJOINa6!unXLAIjt%l3W2*jY2n2AZSy(a79Sk^lP$$n`vmvvY+LI z!8!xDi2UE%y7_zMiRg0gkDeivR%*2BZxg>G2?|JhlkEIws7?V>R*`+J~XBhsXs5S5s2XB4b{(OB?~m6&9C`rKZks ztGCAl)4x0(pJE*hcM0Kh!!S;Ok{P%cWX=JK{rsg%9}{}Fj{Y%f{6NuR0G^H1<9)lp z7>LvhP6eSshc8_P!^9Rh1vIprpD7h3Wi-kef^tgnJ)d`M+G2JQ&($0S08f&wfV z3dZxOCVmr2P$Cg5{cS_y`nAYUMe`JED2K+~bSn|mFM zck*DDPblCa9+2d4qco~e7e0AqRh~xziS3@xjY3hrM9Yh?h=uFy!q7|$q(iC7G65QU z9J9~mfdTGR>kHzxDaChD)Tb~w)HKgd0# zHeY&5Cc85X^CD1-;X}JHpvu^Eb~>ATyJ%`^!puw@yqmRSf`Ok$a4TEMx*YNVX6iQ$ zRC~lzK-v>*WFd>KBhx~@(dzToAyV8}^@)bYV2j|K{qMa~}nsk5^n-yJ=C zfIECzjoQfnYxc-97aMq&0U+-OFIDKN&)6wXz@8Ce4A~ez~=Ta-XVth`FR%K`! z&T_Ss_kRIxtwE!1YVL)>Waec-#X9qdp8gEw_hAB&&gWK6Tu3Yq_lhg-f3V-{NX5&# zP)nyU@zye&6m30?i*sTxxp9C%5QyeCVQGzaVpy?%7@DmS>yTN|0T3<=hAGw$z?arD5d!F-yWFYJZU) zzH^oQ^S4AF@oK-(A6^f1bo{S)S9Zb9zyJl8q=7BOF4&;o{9Z7R6z5=uuG&-IBt|LQ zH|Cd%r4?O>y5uF@DXX?4y8la+e73)$p?Ar&Np&AC>&#&IctZ!vu(oH?7)mv-1YS;?O+qSEBXl@fa&R5Qv<=KoiqZhs}a<1Eg+P|P+h!ne)vuTFnNx0-ytfhGrBN$C>}+a0$9P)lD4x=>%e!} z%S;umKk+2F8cwpI8hA{tGw}JzofwZOVsE<(?noQo4zD@?PqDvV>&^J{@u>3TJkn;o zEg0!~Mcc)>g2Yiz_Z@+lfN1FEf>N~~JUDmpq71WAQa#Xp(l^1@h7m_ysJY)3((MVN@vacnU8=|4{jXq%9DrH zG=tR?LPyj-QPp0g5H$|zd_vGn_pe=}ljRz-00FdRo>#cn^8U6owE)Z1yZ8-xaNQ4$ zw@00Ta$JI!$>E<~f)Zmh>2FS~ZD}E42i8rjbk?bq7FR|es7=i{G41a?TMTK==>HP@ zj14%?V5JxDec3w!(@lVI3au4f8)x5*t%I;*h;|XoO*5qO~quMe+My>^THu zZY-7HTRb4(Iv=at`L zIs~YRnaMiqA6w7Vhsqt;s}p41DB{bS7c%TmoH(K5?b1)IcVyZRXEZHXfJgUGj-0+7 z^5~jnj8szn%un3FvKCS!A#@lGK!?j)9zTn@`kW~0eV-L8g#H&Wmykb*g8YEPQ1N)C zlg<+J1HeBz+EXiX^nrr~;ZthNSa=k;^yA?_v-9&wAbBTjXa6S}GP)s9Xyp(~5=6_T zAi4gy`uqVOA0Ik7qV$!?{kq}<$T>!4Tcn3bRxeo0#9h!VR)gG+{KXE#ENIOWn23g! zxYH0x+ag&t9Jj$;H}>8*(HKm;i?a=Y;}J}}^}{q=;L2URe0zFpABJww*G@lym@|5hCAm5Iaw z7N#kVszisb zGX2k!o(nf}-sDGEbLXkj_An5Z4A+hm5Q{jhoI;O}1^9*@H@CdJEPjC)-x)UwO~-<@ zVcmf4O06?q{6%SYJ74u(Di`OQv;4HMXv`z;cFrx*&1ax8nzsVTkl>ths zkt%f>4o3So*|BFjvO0eQ9v|f1Ni3ms5&+7$h--X;UUlbnKmFqfr2WI+)5Qpxc&d; z!ndCv-Cxt-V}UHucqq{z#`-3#^LZ;gK5sqo&d$!OK#I1&@13v}g%!7>!t(M0L>NQn z0qN;s9njHq*F=C|qTct#p)uOVE$s_n!$^&B_t%$~=w0rDU(qH=loS^lF9#2(OnOQ&xaOZzvB@0`a1{QM1Zx?nPY(e2*R4|0~e-&U8 zz=w|EjD`@E2O^4nIQqD4ero3f2(ffJAtu?I=6rIani!Tb8og z?4g<$Ud;go+zItwt|JH8G`qXC(Pv7QZrys5DASyUY(MTj*j&)0)^N2 zr^}g%rnZ>|$WT#ImG|oKwbAR(R-2hoQMd_D}$7f4n{a?E~( z?~ndew?D%q_r}fUho2bEl@HQ@o|=r+R~)cdzPI#Tob4<+FDxbMDhY;0+Wq>q`D(H^ zQ{M*#K2`D|YY439%=1qk1Cn?B72(%5I~7W`WRa|Eo}s4{_A%e!`*6ZO{tHxY(M~S) z+CjNvG=oP643)z6CxA`SJWLQ@ssvb2ro$i4fzz*+rAsmCEVD0UPRA3DEDRv*UuVv! zm~Q&KeJfQ^ivSF0rdo`?q@FK2O87{v;outq9n%%_^uW(G$r_7DW8(gn4Eqndy= z+7pkk4xqpw%8Lz%boTB*m_-%HNP5)!BH{3^cUu}BTJJT`>ORpgvKHu4^3uCCinA9d z(<;=ohregTNTA_g+Dztq;}c!qw+aemy*zTSWc}f(-h?@4e0Z~A@{X}{ zo3rt;Nd;^I?bT}#8fMMUEuXxO)4Nwe64b2ynpfp>$sOm1IVKu2em2o`8O1jD*Lqg9 z>Iad@lD?Zq6Y95gvEEe{kdvPCG;X*K5}=^`pUN9-f+}tqaxHw}DJD1`S1BSi(ERi~ z<{@?mX~Cc)Kr9DZjccavdi*(!Gc|XBk$R!C0PgmL+D~Lzc3nt9QQYhP*B!!`IF8OCy1o*lvE`K!eGh_g%Q>_tj@h(96A!^`2bqcc|&f z_@1&vxD7CT+}nMNCIu5*2nI>Xgib=Ri106^Uk8s->GglyNnJ3_IKkQ=vhhH4YWXdk zk@dn`wq~xhlm$+1pcsGY4B5>@NLQDS*gRU8hwy}f!AbBcgc+yk77E%;6v~P8X_gpe zx4W9=3OT;xz3O_2>{HjzwnUAW!jF6u|1utaEvj>WC2(9)D8>9u*gF#Pa;v>g)R_ve z1H30G@P-c_wHI)Y;cb%S?LrFi9t~9E`s+)}4qr%MVS3+vq1kAiyDM*1)v|>8jLmrJ z(B?^A*tp@sfUJS( zl*?^dBdO1a?oAo1Mgq5>*MHq-LBYF0$8-)DM5XXpZ=I>0DC?VR)eF114eryODJk|Y z;eG4+)4iL?IZGR>=#eLy z4t0)%If_W!@|j-Db}31fs;g?K`dFoKs9)u|;lz)Eqp~?K^NeUylfR24GhT8GIB417 z)MxTB4s2bN_5|?}CiCe$R%r;4*Bvm2Q6<|b9?J7hUoP;BZLDr^hB5YC_@D6n+O1K( zuJKEz^P*wdcZ;(ciuuu#3yCsAmXW$H=2M%$LTA1Pt?UXAAD;Rd4Z%rBl=%Gq8fVf? zhmCh>(an4S#8;|&y^3Oy0KRPV7=v}k#Prg9x*%Gh@>pQ;l%Kr;Ht}FpOc;RbPrh=FpzY@*D$g+{smWyLPoM2`UxhrPev;U9_kE%NZ|22T)T2MV|Q0o3eNB zo-6{}8f-Q*GtcBhYl2^R2Lr3z#DvHRf;;5_nu?Wm9f!3 zdVuS=#<=ApkoBZhb8L%-ij4hGNS(7Zn|ndI+hR-VS`L^Jb5~R#axY6~TwOaxqFRbA zOYOnj6~YqdYCwU-X_gctcywqe2GU&{VbUUOaj%`pRK{*7jg}9v&zx`AxoJh#UY@r6 zylC|7CMTP>Pt4+F`<^`xEB=YJls8qW*p?baR;MJT1^e?In!IrHB?KSCLL~KO$%LE0UpJggL%T&+GDN#p7Bnc|ElKI6P^=i6K{0PnmqXl z9uDa{x?**mXgo+Lof8_~XZIh?g@v+P^0SfLvVU zfcNvQd#$t_XSin~%`)hjN*E0z34i|29Qjr|Fad15$!&9VwrO%{<-Wk5S+>fb^C#X{ zbOYRBxHiNqfXAPi> zwugzg)(}u~o-uO1%S|`kpoOUjJj~>53^MItpg^IWPQUFe;f^54lVH&u3eaGcQ2Kk6 zccRMRYG|{aS^W7OJ)Om(w#h3R4vk*=E~4r5z01Lyw>o0s?g_+~fS|u+ZZRVT51gr` zhC?I?KD(t?&qYT(emy6KHcK$|hKk5{jkP-~(K$1pK$PHbs8@=X&s-LQUW_9NL`7MH$e{JN*3!C^?uLAJ;cI%1lrg<0DG(3Rl`-7I6MGN}u%YD`-4>C29+b6m~Axt!0lye3MSX(UkQ&sC3ozOh9RpH9u{F%ZE6{ zg;rMyP+Fx>iWjUbpR^8JdP|BgcCz&e=2Evx1y@>Dip%hrY0b|K=7>DawDRncB-`xd zkHu6}jW-f2q~yY5>-3v00OvW(vfSrex*rBd9%Q*PMV`faY2(xLs0}_`TaNFHt&H0t z*FyuNk@bDMhgQEv9y|WH$Z=jRb&m666V~zvA>I>g)K^mUN!t1V0f6WnJqVj{DFTjr zB_~(5zHf*L9Fk1$h9rEN?Hvx@66;IQ&htJR1hxh1h96@8Qskenh64sQJVnV&39qIy9Fm^sA>BLbJQ34&*8v;ahBr z5V=A?a(D|R7WIzZmIiIhx2hU-?i3#DF+s?{qVft2zbn}h#Z;a&3kedk`r#caNe;{RfFuS1qLJ&=uI9=HE|W;;mhu94 zP?~GrKVME`8kd$PL*;H#@Aq(0iYhcC&9u%~7P-d!uJR>NWTHsG5AOO`Ty8BI9NEa!pt1d2<_Z^fUmE1sew z5GN^M@=W8Uddd;xILOJ(8y({!I1ZlumhfSOy+I}-vV8$$I+BS;;+EzT2v!4;7B&Av zGwmW4!XBk?dP+f{=u6?wN!c<_u}tQFP-pwMc?#=;h;{@zg*r^~&gDS@;Y!YfKZR!u zCOz_y$OSsG}85ANgJfChQj6%n!eQOrU#C4ZZoaH}r8XEkLXD zzAAAG0E860{Ld3!YQA1rKw7G-s^VsO0a0ATr0$jR4fPZMdR!EEus)DDU!;KmXD5(M zu1%F_!jBj>*2X3<-v{%6IAG>5;%hC3s!xJbC1}R!;*Psp_GjU*D3{oerK-8egx%AS z(z$#AU{dSubT&@ix)`}sHlg$i#|wO!@>Asb)iFyNP8=Hdk2yy{UEJCRLBc^ri{L$% zG%W9l7A<}h-I#Rq7Pu)uNgc{6Vr3g}GGVx1bIfJm=vk|ZAXCn?yM|D)G@hJ-SQ5;a z;XfjeGzE?)rzQM-Nq1kgUkvM7Hq8ZHViv3_8OoWLs7oPEkZH9Q-72msti0S3(k83w zy=TAp=Jh}XRxHP6018nE%5b=!*W0G2z)g*3pR;WFEOgDkBsQCwD$Mz+x9NIvv*EMr z&r||3rM=8hwnM^j9{eTEl-)*pJJSWG#$jD`7F&sDrgMbK)%8f?G}}%P$LV}|-B9WR zqx*dGOu$UV2?-E$yu_BuwNh)o-o+YXRlAn1(iM-H(-jPzce*%Yk~nOT5ti}R{&YR+26hs&hoXnH6EAZBsQ;pTBDsp$`%9Qj5%TP% zGZrj};@w-$q-jJKauc2qv!9qNAW4BPsYQz&7RfF^lmZEH1GKG!Dnh)xcR+xhAVUl& z$qHyVi2vEgY$g5U&d}?1K<79FGsAkQ&2Gjoc=;W#EV%&gN|cgIKV|N{5^$h{-oudnc7!Y*Qsx5KHujJiI}Zz0P^?9Yoy(jBQ4+^qGK z$pxd&7mJwZLkVXXZWA~C*C41BhI%0k!q;#Ep2T-9D4r%|K{@I62Nc^3x&X>^r)9Fh zsO$HBL%I$^1R3ree|2Qb_Um@MjcGkIRkJ0gpgi>*K> z|ATR%5Jx*CROAliB=|*yA=U=BKm3*5?JmW-PEEITxtc)4Srqm^#X2ZD2CKE9$~Ivhe5?!^zd+;wJ=?VZ1EZeo8jEcl9;a(K zERrPt`it?9XmEmEX9SHHQHJ`-xReEyd*#+1KR3N0eFmB#P%Smk-X(lo#-Zq=_OqTJ z`Ln)lbENxgN_rNDa-IL6j={cW4h^$#q?Uo()=5gC~X%&Z_k5PC~B8oU8^UzT%fsamVM*5r`!N zq5H)Sku$5eAlA$;=&iAQC_p3kH=PnH^Z8P`AMBs}z;a9|oXy6#Ig(}=7wcrM;n38?dTH}8k$lMNJt80)?CtY4mQFc1Rd=Ku5|zKdPON<2o4`1;DtJx;XP$UHSaf&@2Sav*S5bZWvav)Oj#~`|ndTe83gYxj< zQxx8>v<+{Oyng#OA~Q3yl8K;4E4}_;Xn0JBc{qsdC&PjKuA%w(ymos_96{ptvi$^C zEO!iTZW1xux*NyYv=isdp~+o&A7O*`THVrosdvt9DWS-nK2S2uvw0=c7Jw-rh_UkkyeX z9ZY5%?+ERB?5JKaF9B&?qx~n9{a1Ti2O?XCdt9XVY2D(MNlJ^maYfFjIp=RkTO$^e zxmsB1z;_?!qM2T;0Xt|=M?dXhNSZGsoPAc^*(V{k z^>&F5-E3nEPKKm~1V0zLiJMnaJRYkB_ck^CN(t^nRUhZXrvDRT?cG)Z-_DR{-X#jini_uo=w5f@@ejE8hdTy}Dstf zH7|W`O0je*(9qd@UUo-+Nb~-eNB559kSYVNS~EqA4Np2Q%X~$20J9o$VrQ_ zH0MV-K}NG$?_{^pW~!nvT~VuPNX-<{zpCa44xH%!*jPfy7k0VMMijdy0`A@d%R)e2 zEnQ+sL)gtVJZEasV@Ict4w34Ed#9)6dUvz5#QrS-FTHe&g@mx8;nb!42+xTkyioT_ z>`=LwzJ1+XGq`syw>d#QL`r57e(b(~LX|DsIWWlPJYQ^T5~x!I?O<>(O{|!}f$+kA z`Bd5)CP=bG{~-<{-G9Ye<0g9M%(xQ0%MI-nqMUmebd^_ioIlzB>G;lM;>VPSz%=33 z#|#dmtRUE$sYN~38p{giZz)8bqL@fy_HHGy2KVv}HV*|M!D6j5(0XTaazT6nPQy806KkaStulfV&xA)Wt8HC)B zVBMZzvxYK+${<5%{l8=giI4;n=G(w5-*J=)xs&)^U8$J`fVeSHO}=wU!kUMp&aCbhH#WyKh;vAc*HG@r#C+-z4JHV}qb=^7qlVw092PvERnxZ2kw5D>bEU z4t~O7`-_gbDtlV339pg; za)_bKyRE+v2ZdHclGPv;)w`lmn$_Ls>YFO=I_Lmm8rofWJ8W`Yeua>|KR@3ZRr|zR z`o}?NxrhT(;J_hd#EaMW7{u5dx=UwR07e12x~(=dO1@Pe;ueZ<+-+&)6IPI@MLL`C zP*}M7B_u}{-O!w9kNO;OD!8d_VX;X-{rL#kN8`P6G@osmmSuvsWb>>KR4ZkX#v6?~ z@mTM=+{XEMhet(-K(JMBK7+M-K`7$itq+*!&8?+Cc@1$_3+h)r)CWxlXyVq&o{wcq z?L68yYOG9@d}CR3Tc^BZcK?1v=-XQ~O#x4l##?mg!Zi4dQIQ12jgdgJbuRyfNqmqA zaI7epAFasBd5|(4JjDrJfujKbL?w|LELGe(*{rV6%=`I{GrB4+MlR3hEd3wMlDph-)>_^2t>;0C4wzHTbC>(CNE|ZvC&dCwG_q<_G%#43OvD?6?WUPY`!Fi^ z>mbnhJ-#E391TPm%dxe-$etxPx52JovU_9pIp9-qwI@B3$nZi_-ry0(-?){=+zSjL zXBL#D?J=r(qQ|iW(tmZmaJCAcZXPAdV;+r-txUZr$KMkz7Tv6>EfAscW`M|yH?R1d z>r)Etn~h_@#-y*7c=1lx-tg-+W&dJahy|7~%Yy?grU%xh;?Qn3fA-eNLFRz^D8O&p zT71{J;^wGQthx2EJ#h#B1tN!ECkX!x-tZQ1pJx=xQK{?DQgjobI$T7 zC^~>i*uG!By)7$9SlGDoM@t*@E-jbCrLz3D?kUPU2?d19;95DS+qh}a`<`Qy_)iKH z#D_QB|3*zsD?DljEM>sng2hzXaLYNaMoGNOWhDO6iA_9nqq;V3@)TijSngljfBlW+ zf{f&A`PZ7=*NOjRc60lJ9|Yqi-rZYHvTtHi_CNo;&O|r^sc$3NCF|czHu7B(S6)c) zCIm%1@^-O4Mm%Qe>8YnN9ObN|;&UvuYK-Pak%;-S{O~``Z(wSgbz%}B5n`QEL zt*Dqk@OHjLy+L8r;y=KcFlygs`kzDJA)Hw$>p!(H#M^(?|_8cq`Q_7;Aj~Wwb^CP!~>@2NKw0+5+c4W}y%qG1?e!j8y80+r7gIOQxclRCtPfQ=Yh-gb|Y9#6D9gE;WDCPVE zQY3+>2GmMLb1prXxwNS>U#+cEdg93#4)Rj0n*I-6-vQO+wsjjs@Sp<6ihzI(kuF_2 zHc${ydJQ6-P^1e1g4mU&Qlti@*H9AyLsSH$_Zo`Q0)!ALp@k%G$K$>Czwfq5 zBBL$uhG>NWCz=j+#B}+uZmyhJWpinUTur^Lbfp2rJ!{m8n>9c5Kc7Zoz?Qj8E@>L- zZHHX88k0yFa4DAXHP-)OUw%n~M13S>3{rIkz)#rss5JjJnK9Zv-a&V`yEow$IXDBf zs0WlkzK4?FuiekYzD4)EQ_N}{b7>bnQ?K{Dp#OI-(6+a*C}Y8=R89QXHX0-NQN-g$ zLcPd8Yg6~EzX14-3j_KLm6+X+W;o4}r)2AVjZ<{x-T{fv&l+rG7 zxz%!4&HV!_ujmftvssvckS&nDuZ?W&uQoW6K2)65R5f1=`Vx#IantAZ@fYn(XJ0H6 z@*VCsh{6A13PaLfg7-8ac4ddnaW861*OrLtc*DvXOf!61F zJ`Yn_3}X5%fJt;w`H%I>2l%Z@EAT0SbU@W1Z*uEm2gNdDrlgS}t+mwNw;i9N?SI4) zoS9h|)As9a?Lk=`y|U{71%!Z3sHAdLTYFKmU0J8+;LX@Tp&WJ%mOS-X4M18?6c};H z5a-)wpWmHJ?+_dVY@bA{&1gyS_4%erT?HFv0`ugFd@neutil~JzWdh(Koz!QDIlZM zeawJqj-nV+4q*C%mh(%ief>Q^At(S?zBzg6gg{G!OQ{;CwNnjGGTAKJ3ZzUCc$6Z1 zmnU_V&^5P_=+EQ{Ies^tTVwJ@CaTu}d$FY7g;*PiU7yKi+W{g~fKHe?wAXtbw-xUx zg%;i}tBHQaO)_zZqE*<0ydAA8o736>t&K2zQe`)BFhRs^AX--Z8pM(HGq2}TYaTW8YqPDFG3 zX6Jp~Lu3F83j2C5`8R}P#}|Ph?;sFpRkb#U&GtMnRm#en;v~Tf06oPs177>dtr?UD zY007pbvtEqkjuGjE$}_x*;9Jzf1@oK^HuI+tmcmVaDN~gm@G&NL<1wZ4)$%o* zTXfdlaj*QXn^~Ck2c|zCphVk>H;omjOoR`9VvdFOdy^M|@RRbb-8r94Q81ePzjBs7 z$SFajK{Jv^YS&_fE!a6ZI-@U(=aV*bv({8mS;^9axC=C9txH4#yO z*|FiF*Hw+o=j*Vk)yt7}1H;naQ#JtEr%w`8Qf8@B#W1GnQHZ!#&a%x~y2eC}f!|Z8 zi>+71JRHnetjXwsZ09}&Oj-JE zoa{r#U-QH98lQ=8r#ga!x?qzXTB_h6;%3{ckzV(Mm8k&CdrpRtW<3B38Gcw7I1+_^ zNRjs$7|QD)ETYv-Zkyy2lRD7#$Aew-=9XVDeIRFl&XrbuItXS}4L}9tsN8aRaGHEV zqj{au!qeHxj#G~Z97K|-Lt8d{!A%*LJ|ttC=kGedoFh->0s)WYVBUv+$NVkWJ3RVB zGv6K-T*IK2q?N1NapNvO3J-4BJ*cV+jy0P1FlXM9W+wV)E$4r#x9nEuioS!k_c|!7 zxPmFI5FpN41tdBuKU^m{p2UN3e&QnslEr1H2558IEh+#e2_VoMn_qBHl;BlokIo2b zYZ|jxH0j@~?Tzg|jI|e6^=vtx_d%8~^ENhL`gZF7;D1#<+`1Jtu_DZitw2FnpNz88 z`_AI^y?ikrlY^n3=8u4Kh3#E+rZi~f>2=VpNhR{4456xBDW$*2>_4WOx~L7$lF#lv z(f4<|Ksa_W;jg2YEU)(la=AGw`XgH=)~ES+wqNl6xEs~o3-b20&_KX4fsw&MTdy|6 zqs>i$C573upWn^9YWIX)>2}vn$H18M@GkDq!3{7)3e(_!vq97pXSgWR`7d*7Pvd@W z7_c%MA)+-SOhgYv0ilaD{_VYqR*HO#q>P}tWiI}E?Dvk!?O9#6SSxYn#+t2umngl+Kv}Xg;TcJ7SOo1H88L1&Y;{< zpf0+}buXu4Q^l)SG(p(`kiUrI50gZHuDXqeuFyj2yDT`pd+TkY8I2%c8~_lrLpwz0 z;;O=`(!{=e(nl{~2ly|3{X-CHRKbufQ_|2NU1ed=+Is+u)hGa^SI!d`wSZkvUuyzm z_r!tE;e$0^j3+(sE*5oJSVrCSfk9&TM;%=8cAFIPPacVLkk{mMI)!(FYtNzem&U-_7>xZ#- z3ThCPc%`Q#Y31E*OJTmK=v@_dr1HDMGj5#1{8+<{{@f>nKi9k5*A451AJt7vpsi+C zej#xwmybN1npkP1=FN1kH|7>Iu5VvQuyJ4O_jC^%2gt^jD6fdjBM|0SqHzf!La(eB zrXBa%ZNWoQyQ0_hZ)!JGAKH5-G)9V7qFhd)pq;cr`6W?p<#YHb&l);@{O@qz zWbcJ;9Z)DVw4Qt0=CqVIMFY$V3n8*ak@#^$p@^bnS!rca)aT=He-1ZC_J`L%uI4bL zRzTLC$mS5B7G3|m@N7$3dfMVd)~)rzA)5sCB}S_ZqgwEv5od^rW-bt@1v4+*pqm0^ zQci(@!X4l&YKDTAq@|r~{PxWi$agS(bo{@t<#D|@khxl{R?&P?3Sm=pzmkyKgRelV zqJmw7ZDdNo)XnGXz`vJC;5*2n2`Zhitgzt654Nq~IsEF>SMW`q%_wQ-b|1SpGHP`) zv#)ww>wXT0bHaL|n&&-4t79awnc{EI5=`|%yU>0&Ov7Yf+fHahIqMuc{@9GO2f0g_ z?^|@3xd$LPxBrmNI$_K-G^pWoGLX`-X)@5V0@zXxoK5rH^<;t^8f{0=6p}k*$ZKx|Y757m&{l z7D>l{%Ccg+RLT>U`Z;8-YCkafy2maPe{g?!f{z+3S67)Lw<*SZ<=uL}y>4W&N$G(MyuXHO1**U8xXEui zu-+7Mzy%^Q*hh#Qj?S2uu>wk)vIKi*>PJIBBI$b-CS6tf7APHp+7Yu1BJ6lBl~XDS z6VrV}y;F4Hfl>QA)D9ptcT8l9_*NX{v$5NJLQ`m8!<8$Lb^&jBf-ICK`zdgD*4WT8 z2&g|KV`NaPD`U5+S5a9n1opkVD!{W@D9N40(mE`pdG1?ELntWPTEUZ!E8RbF_V^eO z1Y;&R^vZjs%FX^QHRYQ&8_$(Pe^CMU2deZ=$7dE04W&bIht<8J?Msm_UiLGEi0vy? z|3H;K)ii@(v@{S3E4AIo0e*qT?g-DinbXXX7Ua^d@ybH;!1>J5j74Rur0Qj}W!j#P zDh^_aeklR+@cTi?O{1E=gr%DB3eIM33nV`!M#e5w#=&)K3c7hynRjk-c%~tzms*>& zYzs{U{I*vzlb>Wl!^?eMX`mJ3_y_K_yk-SxGUXx6W!0ru!!Lt+Pk7C9PNlP(9t?2>g8F=_@<)AV*n~H0O7VqDnYBb+n@lQJZSz|Bc z((|57960^_o-@D{Q^F3ir<>_r4x~?SAn4n9y}iBbz`TknX%ZvG^`n<5OxFp7>B_qb z&U)I6s(d{u8}e;8NEa*UHx6G(q4-Wlr#58{L9>(Cs`~*#b0hLSWG!&`SG$)^tK>ii^=kzj&g8p-zNZ{ zdhWa-=rFWVeK0@kI$ zQOXv@dLcFTS)QeNqPe9yDLNwSp0@Na<%UvmIcbOWq&F{3&S88FSdCv-If>SAvQO{& zTjxCtoK5gI;yog35MM)ofy)!PFgFXWC4|4%MJmNO=a`2;w62>r$y_{|<{y)GtkNXr z2jOP{3#XN6#kc2sn7W1Ea|KLnqx`1utkMI1*e)fV0gDhv4DYRbd-KwYL6ZY8Ew5bV zuO1ligkSjHmS@%adjtj&v09E;b&FpBE!l3M*b=NejgVo^pL5b*@kl$z?n#iqm1*(E z3Um)Y5lY7Pc72;C*}jp9T^-P?zm5M~o5ROHCh6sz0opL%Ja)Z#EV|Md!zcJ^9z1ZL zrnJBamVJP8mBQJt&07EE1Q-6q>Cf{W%0XL{oVjZn0MzjLtz@Vb``rZeS(wd?h;zgv z8G$Q65u-zBJv#Uqr?x{x&w&cH1`E5|c}9B^qCvqE{w|m|It<8)VoEo|u0)v$MN9kP zfJ&S(*~2z#PWb89Kn6U2F5m@6ZTPmw?`rR6bW{HM@XNp>!jAJLIH$9Ph55`p#x=4C zB{GNyQpH4?4Oc8TFE_Y1>?=W-cJjgvKfD`GP7yA$gb z{B@j7^Jj$kn*Nd!W}~<5n4ZqN zh-(*R@p%0@smKt_LKh{40`k64+Z`}g)qcqRc$(bhW>m11&_OZv!&mj~)A&UK`R=ke zdgAVqP0Y9!`_Pkoae~7k zqZD1Q=HxV+AwJ`bG%w^)A{1j;Fgughdv=fG97Ex-u20$Zd0ZN-@QQDz9ijhTlk(SP zD7{>-}H$u7YF zjtpvV$|=WN*1i_~+(3=OYO9awZaz>SaVj|rv?)J_N@$P+g`sQNpiI!|@!iVgLlB~1 zHSrx!i2l@sF24Tlqa(T-dlW-r_iP0*Cj32-|M0gbf9fdh2DYj&)#x1vr#y_?{?E$NEBTX(?faYFLMCGSwQ zA6EHTw2XUfRuy+KD&{h0J1qLhkVc81aHO~*@{*0%-_tYiDQzhq{wbV8^8X?%#2D!b zXwrwcQ+_)LeSlOOMC4&Q-3EJ>SSG?I^kxlQ)|WKj{ywt1URr(t^ua1A!qFLPe%&{| z^pv;Ce}*4&p1DiuftAzf`uKu!6y@V}q13BC_@g$Szll~C@s;fm-PbuYlkvVr_^F#> zuF-`89c%?_`qMOf96u*Tn3A^nQd!`t538<>)OxL!hR23o$zBrC3$ zD)N=+i50o|8xg*Qob}47jtTar@~y$cyodi@tE1C5G>-!P7a&h~k%0hZnGp9r@?5_l zbF(bg2C5%hTFAniA+l%)Lwh3+SnHht4mX>hRgDBTqGCpW90Q2=+Lq7A5m&d4`E|V3 zlkyxGPm=a z21Vdl!416eVLdj1qtg6X269okwx61IQr>j+WBQn@oiE;Sf{r*Kg-@^w$xdL~hp|TD z`U#On)R~zDmO0^Xi+$kJ%t9b6kbw)pc|lnjpIX|lHoFXK5Mq_L267!Ka8;rCBW{vBI;s{hkj`D z^_E2^3XMBRdFyMq*T=pNw|=25?6B;erG#kdnrPWpSl`fe!tvvceUbcgTy+L6N(=W# zJLG$jUex7zwfy#JlTWe52K3IvExXr`>J1yIS)T&f0=_NO7-7xdq8IR6uCO1*<;cy( zg_v%Zxp+PXT3^I)pknoxpwpaV8&MwHmAbQc@exZBu_?i>;0IZ7Q=1jDa*;U^`NUx$1yt_<~`PmS;!I(q%nfSfZL_pwU?`E%;HeXV2_ZepaqY4n|!{9zuYqm$jGNxVz9S1^rh4|+r;NbF^b^};sPVg2(C5{xP!oyova z-dblKkW4#d`J)*~&U#$|R{r??To%xK)*aJ<_w$-CL`kKZg=g4QJ(_n|Z@56)E%UOZ z*g+x9+wOKJTXctF;;&V&Fkt73cX5x{fO-Am(J?K+0G%gb%b8Th?U8qt@VDCxCP_P5 zVCP-e+X}gL{6kML&tzG!slKwgE}hVL!tH$4wTo@J)B8Gfe1fGsw!2*S4xvc}$iwLQ zBAkx(gCIPgpS+NR$rBfNOpiKo&T>d$gZcwG*^N2tr6&A$iQ?cPk0Z?m=+KQ$6K>Tt zMWC$1^a*vrZx$5qM3R9oU_G$01ip!$Ch0?P7Zn|!@TuKz+cyn-7$(rY5R)vX`tmNk zQ<;!&jqw7DzXUVU3xJvBdo=*d!GwxUDYxy)I-4+1l^k7X*;reXW6CG}krHYbJ*L~* z*$Jkq7oIj0j-HoBA)fLoomlAW(^HyL!7C1|Py4T4fBo9KWi?=>H0xW`K_V;mUj~|z|D_v$Rjl8sT z-@JNbNYhC7|2qCQ(59u{g}#*TMWvH2rSI~J)!+>1$g}l^+H_&!^FH>Pw8tt_>ox@w zMI89`m6&ShN0gSc;U|DMN+Ai;CHA>N58jy?+T_VJPcPu77fQdpE;vyTHxlbcdCuWJ z8{`i8GOtpy-j+#RZ^{U;@{{;-lG5-FHRc+DmGh5SjoLX!L~7$7vP4rFKEu{`(X&>F zskl$TkfZSW$J_wE+!Z3OPg(yelaRb-zgGhIi@j_9{$SU7^e5+NzG1raqB-t~b~T4w zAH7!7z#*JbW|AB&xPT20F|DNIuyv5P74D2n;gxj~uMdtP369{Is2kY{6ZqgpkX z6e4+W=`)wa?L$XyA3YSGxV+rCZHIg05*irj9Ha=?Nj0SpM_#yOsI9g8O%gOnAtTDs zDz(kaZaobui6}QzJFOwNc=E0MK%o(5U17)Rz>v2wKaYgs(m$jdZf#7(%@wrW_C`T!bJEoMD1k+zv9IabjrmYY3#mK0NEcFg^w zo15JT&Rv~ol9N$gYSB=bDDl=y@TC#|&+=Oyc^@jsPW@awgHA7b+P($en`|3hiN6E? zvht1jwRvQ+CXIiVunjFHU#LfJg-=mi_BWFA9=wLc83NJW&fw=uD)M;C;OO10F`K4(Rn;ErWzUODzRTHWTCu>-5ZDa*nMfH_`TJYf z^F3^RN{(bKub~o&XBgAwf)^+W$%dyVp3Mlu=(_Xr_fihR-RDl!1ub$9datLJ9>hG| zw`E6gSXxcA&DxlXHt*X%(6?J6h|KHvIOG?eC21#b*)rVrYU7MEg*_9(tcnF4U{9=E z9NbVJ*9Vr8M>bp5>rEF&>nH|>L9!(?`PGTxP_BHdNsYxf^;5k>D)QYzi@!FR;(t72UzHP1UzV`e7t5=u-}N88Ob_=0VQ|uNAu1yiv)+y2k0ghOPsXvFKX_rBS6h zxKf77shz+bf=(CINxaUH+mPe0+LDR-#(FyqeDySAvsnZ!mIU8~>x-LIW+?FC-w*KP zFAr?$D$R7VMQ{3HGA-~ou1eWVtV4oc$)gFvHEIONDl5SQ;tIH$PXq{6T0J^@v^01x z-OmK-cf(8{Kbr+wY@{%F0!6co*M2YdlQ!~uE~P=r&Q{@*T2 zZ1E^vTOt%&&S|&d?M6g`#`54X*wbb`Xq8h~%4XrW3hA}UEaxNCkdbx?tdd}Yv7Yra z-D=OTwK>b=7Ge^<`BiQ*eglZ2c%#>G0M+(AsZ z9?=+;qku9wtJfj|d5QWk!NvMry$ZPAMZP3o$b4{$<&;o6Qgd>-mMEqA>oY3}$Dnza z>Td!Eq)KqIDC?1vfmWaZ5R(3h3y?XRp=DAG{Q}&i+gC50s5x}-m04gpFsWV8U2%Er zBsPY3Yg?|mdA!b8HUsxV)@^Jy(lyX{!ooD+!n# zU8Z06tR2ll#J;3r{V<+o=-Jk1jEUnHTCkE;svWZ9{q&2fv^-1$!j48rLaPh67z`Jc z6sg=NY@tfvr1L+2Ahhvr-IESWE4-R`C&^k1d@bj+M#dH4zHm0v{M$WZIwr=y9A<5X z=Kii**?j*K(xZ$+co;4rq6DCMH67zsP3n2R?cuy&*@|C7Pn19N5|{9vW-S&9>#PI#hLt_a{|UrQXkBdhGdh!KrY9GaCu&?;*0G z9wRdPJgQs!UtddKl#7MeKdC{5udwB?`^o?gGZ(QZ>sd#tQdc;QVqRn}9n|{9We9MI zOMs}>PaLpAbp+BnEknV3Di+@czg^lQ_r^$%6&mSNcfP+WzP#Pe-%rS;Ej5*GaG~&H zPX)s-w^vZeor2yjwDlPNhD%o76rmj(cT-)OHQ$5zNYTJ04C;xU-FrvTB#Hai#<89Y ziIk1$UW8u%kd*P443cl8z+{mfVtiq~y(kA>jdnZ`@I zSXG@cTIqNl=`p!CR+BW|L%&ZY*JJzC5))?o*2S>_b)KU&F75YC*(?x)*u*#S9{3=V zf7f<)tgmPRdbu4gnf!(ZiE}FaEE_&>k9pdJ)!a{^)%u2S)rn!?+d-&-Z#(_oZec`B z#oVzXxpRNqEMKx6hO7>on%g>|>URohKpA>7VA@64?l-`zHq=s9|ObTuhRr#@@F`mQa9VsUH&Mt~0L#6KH z9jyrx3ConMq>l2sCT@(Kc{`@JMKsWB&0ynmco=DhJn4VzuiAcBS6i531*`kgF~O@U zb!3|FPZJ`wqwNlXx-BI#+9+A~m=W%>@qp(X14cQ#8ANQYUTQVlH?p9w(j;)fZHtao zMD%6&q&$5Nyz5FGe%ak}u@vN&`L1(>i**@-J0|V;=O{-d(bli{pYjyCn%SXz|?XGZMjNBq=#1XS=-oMZUQ~wC()$jM>Q`F-#`^ga9J}u(G7)M*iU*AvW=+R z^5*%)aPRTAu64S4MI-M$aTX`?1L2zPY?c9~LsoBJ_Ik#0RN}wFF84tJmHy64(+A-| z?1u!dGJ3oA_JwPxoHJMOMs`3V<>3d{z8KFz?{95j6OC+GVT?(SwMoUHx}paxzXvW& zc%}w=VMJj`BWzg{7~Bx5EdBlXwt5>I7l|$6>o9b_l3{}#ekz$Z7qY{s2+bhrYVo5^ z2Xhq+I2v~2!u4R51?LHEkX8N5?%=a|{MTpu0yKvCRad?pAptdXa1_)@%n^YJwVO!V z{YWfOAUCU#b&2Rn4H^K8uZgt?Gp*KDv23s85 zxL6uA2ZDj;##7UcFJri+1YX<-mb!ty>aNs5wq%EVyGbZSlvqx+?r>XQ8VEPIE6sQW zQwVgTQ59X?O2h^}B>mwZ#p$1eocj)xOC5puT`g(2yn(5^7Sz@byw?sZs6yiYR?tnt zylA#{k!H|R!?3Qkl-EHvL0+o9x zSKk^lhF%_Z^AsDZVBo0{@}>r7Bu73?EDLImJ(WsxFT!}D3v*hna62WkyAs|ry(5Fu zMTDg>XDYsJ+@L(NyAAPeidSkL)k}Y+B$Atis~0-@%iNHW5s+&6K?5E%WvYN&cl8H%_FbcS2hQU+ zvC`#c39y{tu3Q295;LA%CTHeY^<-Ma3b@QxxSwN3JVYpEpXxf%VQ&8wI@Hn34+DZ{V^`lqmt6~VPE|bc17|(7JZ1m*Ro{oJ3?j8b#gVifV zJeH`R(xdA~yeecxg&WA*T_|KN^Jv#b#}tk<_L@8rmqEzg zF6M6v{~5{f^?x+TJf^UV)n&-Uj=&cOhv*wMVIf<=cL-~%c29>t=o39maBU@_Ek=pv zgWi+1c}a{9Zc@K+u4}cALk+MM3B_bh7Vu1bQ#6n z+NRl}CuH$=sJ)PFNBTMb+Pg0)ux^#zPls7Q=`!NWQD%i6Tjk`$_9zu3-<(zh=fU|0 z*#ft--V4_zF=nduY|9YCU)>blN75csRyFg@&JwM03F72Q^!|UxfKPuuYjou5#QT7h zxl1fR9%%_b{0^n$yk)3U!h$*<-`^9lhkE+J6QrKL0oNqi1baEf?USs7N6Ov^+lN{^ z%ZS|9Y4JzHB?_*+@L#)xnC7cnN{TIkb!Z3THJ(|Y@fa>2sJB&(}^s>FK#&7!vJg;R&{5gZMq%57th1`x_WUNNn!a^k7`*!VEfezQkmZj zB-{8>#4}$bRV!K15pXcf(#;ljGq1WUl%tO82W^^rjmOU+lq)7C7V3pM1-E1LPCi(R z>+3srV5TaG5ntW*m4(5?QUqls;i4o&hjwW(&WmbNF#Xix&rP^}F%C?@-LR_je5Eck zD?+?o4Snih2r-mM6mq#lNu z)|l09c35D(a?w{UdE$B>bW_c}e}p_=%#G-St9SZ3=sIe>`uK`PXeX9uh9QS#Snyr{>CnyUgZoBBd=EGOjp!%=I)hg6HeXID1xt=ofn z;YI#kl9RPPR+c65IW%)G{E>b;oI?@O*6t_Lml3;+sG`;0t1lU;Z`4SmCJA87t(l+? zUJ4!2-I%BA?| z9S?kvjxF&O{r4_oI|g*{US?P0F~boYlf%j#Do*c!S#-x?>f{V^iKf*upH}O)J0!Iw zJyfOqjS{7nZ{`5*X(`cX6t?wBk@h$hWJoRYSbfUJH{7*DUWWy_cPbKXZG|7k67;LA zn>Z_VHZ)Z_T*fbM8~|}Ai@z$6V-Fu{C(o&JysDH&@}sW*TKmvRn8h6|zn(f+4itjcWu4 zTkfs1xNwJ z?q&GHuUI9RsPC4kOL?6?vxpTISJ2wmiZG=?$S=OQLTyTs!PSl33N(M8$PpCvChz(c5ZZ^UZuoy0Bl0Z5Df#DBt0;=kvaYx_e$@7aytL zm`s;Dt-t3pOb)?4DUE!yLx!Iz+v}uER2D(2EeV#D#s`wydx@J>DRUtUxX^AhPoLTdVjoW5vaWs5ixEUmH7!oe^^6$`aM1K4E~G+vEE5oitD!qo^E%h{XU8kz zXuV|J)EfI73S&Wl8cIJ}SJucPCHf;Tr}9W|o82V0t-HMcWkx;4+7c8YB?|U(yn@8p z<6FpTkd+Kr4-^8~f1y4rV(K9nE!fg=XLq>XOr+*iPNigsu@sNiwiG5vxezhl)tY9K zFNtB35*e@0v;2s&(qAgN-ZxH5uvW_1wu>po1l6t%yTpbz3v+!6{=VS$h7EE`b$NNX zZTw8fc-bf);sK8TgaebB)i?LSY~B|VqZwRf8%N!+e_T6c(QGh%GUK~A})albzf-fHs zOkK~KMO!(ES&%~{nYaUkG37Lg`UmYoZp(uw45_(s`78?X<95g2$I;Nf^^}6i)^J-H zO?^Xp)C7;^`yV~D*g&#fW!S5*OPkWFUMRGp$4`?@8*tPwfgkCv?K~5q-_?{#_e2G# z4uB*wvBYU_L}cMD-J41WTMIo?$0y%+<3NxE0&FTn!);I3)s#9Mg6Jy=oZJU7zFw8I zN_oRJj7Qm;4%VAgG+0gDBp%5?Jv7$A58A@Tj<=709E}lNc`5uMiScl8)((|JT$n7k zvi*11yM7Xcy*R!8y6}-i_34|3 z@}-x@mmLn&ipI*OC63~`=jg}ib>6f@v>j!~GDV1up>XhP_amvYL$6#uo-2{&(Z`JN zn?136>yvV2u>XRs)X86(70?6!{?LsFIUrhl@?9oNZ5c5`sX*jQSeAqRESw!2^{jc( z2Af+kEbz3(-7@xTj?Ln-%rW@HQm6YA2eI4(CNQ0^JcKpORlc2;WhI>JDB%u z>?)kaalCW9Zs`M3vnqDZpbDg|!`9Ik3!qXsILZiJ0@eXhneiTqE6&yqVc z?)D8z25pKbN_YaDI1RnE;7uVreD@RM5~rM$LC;}1GzlrlsA&LAPb~6 z=l&vi{-;{H0@CR?$bLG?woY+3Xtp&$Qus-OW#mv=~BWoz@`lwgMt(p!Rxb;m`V9z}F_*Nfoy{^~U%fgL@2x)#Q_0D~A zv!*|xNP7xwXt%d%k^LIi*A}D7m(L@m-`(S;QLy7w!(f_Rihq?gW2=`9Kjb|*_lC`g z&THjF&}^~q#f^mXmhJ$gGm?M{pIT1RH7JkeQpNMR$ojv0Qxa~W(g(6!XJlup#{osM zGS9D@duwv6+<8G0;fQLkk1DY*(=_+RK}f{dmtcMfGpyP-D@)#1`}_6Y+xg?;}8JP=Aw`Sb(U1IoL0wzJeO+31P|){f!txgzVLG5 zs6PheO?AzQ9#C)d7~ZK!n7)`vB`Zv^idhgD*(hbAi%#C{(rC>&J~4C1cVl9Zhu+%e z)8{xWM-flnHhg#S`cG^wJSRJzqFLtBgDjVn)SmKBaNdDxE5$HT_ zwNAPX_M5ge5(bj#N<(;t%Q554gS>xN0gML^T{Ya8&la*(h!nRDsAL2#IVG^Zuv(mD zO!r!XJoY`ibFrP#j8z7n31t?W?{xUxCvtZ(ykL$kBf#X zHa~4gI*!AX%2P)LQs%;R*cGi})a`ns^f`4$ED}Oc<)^H$SgxOVIHX zscU((>X32;F}ndla%dDNnWi78ZCm0j@)(XRC=FwTw@BbCU^-VeqGk@?9IP{cOln&;T-A#wtRH7O*Y}n?EzeriNyVSvXHtsJ| zy@cusiq&KI_Rv<|nSYcCz~bQnhsNK+TatXLDklvF$j-q49)kGzMe4OgzTlo_LIc5GD+);)F&cQk*p#1cdxA)h@MmM@3F67QuIH}?o zX;ksqFe@UAaQLr&SXIERI+o1iF)HV^2Avhd3rp5!CqJbNUJ9rZmlKfaB2S)OY_vkz zjQW&ijnu>P(pRRfOUEch;|A4-?}V%(d-Jj5>(txkzVL-a&gJrGRrE*-$$Bh|E$2)T zJ4EZwhifIs#LhrdYY_F^`-X&blMo60vZX<|NIG_rL@KQRv+A`z{Lb16)2y177w(@xu6zy>E?&Nb}g4@MJeSUv$yK1i#+*!@(j2NuvYKqgwNMsAh^nq z^E8Q}jhKIP5Wt!)Ius*&ve;+}R9F=Jl!^mc5R`RBU`rIJ2TY=@!psLjNicD+W(Nuo z@jn+7=c4=g${_{Ro#I;=Y9pg)PkuVQH~Z!wu-yqw&--wf(3y@7B;nLC;^w6uqM|eS zqt{qLnTZIe1lspm}AYytVtJ#PVD=h_H5Na%S!F}J5D zJy)SriTF@-Sp zmw>9qXugx5$Gb@StK1P#2{^o}oM5Dfe;u%iW^MxjR_}3@|3eLjK(=4}9`HRXc0jQ- z;7d4X(kf{x8SO(rgL>1m!T?)t1RIUJJqy&k|2(fF=O7S{H~^^<#LW%Xlpvs9K0=<= z4B56ZAGN#8XyI6zMLZ_XPWFZj!Q2A&TSy~1r6H#fJzT7rSgG$rU|qb4wgM(kWg?h9qu&CzwbSK*> zwN@P67_XC15FK$(6!ks^I+>B9ZHy#P|7?5G=Q;efV8c8zWZV3LWh)n|Z)``=)d@?i zF58(E6PfY*=N>{bGf%TXZbSglF8p8#W8fsXtYn>W5Pg9iF79% z%Enbrsrc4riu18T>%2uu31|Q_xgTaS3XH>hj(bC0>?G5??4(kh8+d5xWl*}O_)wtw zE)^6;H5F>+)8~A{Ss>f50#86lY>>>d>9pe)1Yb@h+{ugxec?6nNqINO-CYda5?ATC zvwcI7m}U`%<0BWEJ!nI8{i@d8-%T1LAdvl>)G?@5Xc+}bi5UI^Ll@?&8D zm{JLV54g70y!Z-cYv3|NgJNa%X!SZSNt3_>iNm=*1D6F>rfR3#!{13CBx27k7Rs~p z;ohw0L#K7;T@v1~vFi&tpxRHsf&wmm^>Po&&p9VdSX7e&b@2KfkspkXDPD9S^|;<+ z#RS+BRyt@yNWJ-2Q4w-V@|pGTB`iUpz-!>lGIdXKo` zgUA*A*^Wh)xt~lgBqJR3S+(xS1dwD^cZrwL57u^U(wE8NJfoKz*him3d8~_CtEa*z znKg3|Fl@RJTT{3rgKaod0N^p<;yk~=;x_ts2#oj$&Y@F!|KqXGM&96aP%*09KTu*! z*%{0VG*-pl)G2zWdj*`W)da+uNJd6((L(GZyykbe5f(_AAc2k*$Tk@T8_k8^)>oxd zje0`{-Mm0^vkDy0bNRA=UR64@<X06e|69Y()?(4>=35I6ohdu$`c+n>!_41NG!$134z^;lOTl;U8aUu#I_v zE`_5-q!sq6-`T(a`++M8XRw^ou0AGgGb=3YCj52c80@1UegBq_QPGP81JUHTu7BN; zU8dRV9evw6iPHca2QB=Mr)WWi{GIvUtisqc=hTI_HoaUWef>Lw-m{T({eR;KNN^1E z@8NWjGh1Z0+7x;MHKQ%O>yeuOiPK-9OUV?S-#_ zV@Nv$1dT?f*`oDOE1N;oA;#K+Vk+ZPc$fdxhtwX8vr}|U#xpx1B3%su{xItxt)u}ygnnED$`@h5SVMxT@+ZR!~31UyY#trN+W9+xK&Kqj3ErV+LH_+s~ zo|6*1Q=?_?DSdQ)BE2Lc;NGL5m3pZ=NS=z@S1`stAE0z~cct+o>!_jL6!XH@la-IJ zgoE&PosD$yf24Q-0G(O_OSk4UWc7^(`9;fFQcbByg>YQw^8HMuGjUkB`O;nAhnyD8 zLw#zM)Rgma7lhX5y~FscDk?+^iXHz1;R}yJ-t?mwU*E{ywR6^Puqm~&Y2<1;@QLW} zZ489vn{SrhifkF-P$8UqknZ+bMf?l#Qfmo08CCT#JS2cM7&f^`;rOFl0?|6nY-|w5 zM~-;(b&|F#6xVB{ch#FZJu19A_Y97l7*d6^5N&G&<%Re6kG6Mlm zG)*Y6vvo+pr(sxnj1v-v&=vM`|Lf0hJmEDDezXXrGGvj28`K?(&8$yBU4A#+$aP&^Kf;aPj4Lkd zS=R?R)RWjc#LY0Egh9X)vgUuk;Z^1vez=b>u|E;C?9X7;^2s6Y2Xt0WMtVw$E@989 zB;kb=Z{3$vO&oZ9pUq|*K67X-udvJ>WF55pE7C>yfq>kv^cMe$lM=Dpye*7&YK9gL zhcr3EFsP_l#5c+x{@U%7${iNs82!`Xx~piXwYgu@ z3F2l|Mxw%fY!%~SL*nY9?qwPU*EqxhG7tCWo6(9L)n8L!$%L1j-`*HuK9@Lr;L@=D z+FXZTU*9pvU7M?WS1)kf;dovv%3E&V@YJlr@n)MixbPA>zw`(u+#Nw z;(+Rys;c%cl|WEvLB1y(Fh(u6=kNXimWK@p^HQ6Ugnd_5{H)$`1&$AQd4lB>Odpvw zG9s;?7g*c%V+O5LcW~7%9VxT)eUR(~q*KeoDW>@K@~3RjNe>PrA^&29fRifpp~Zp4 zI-6d$mz6@#V`#NiEc5x77Y~M5yGTC}IP|?!+tLrL<3^nvkm#HBg?VS87q{#RO+=IK+o3%yn`+r_7eBf3UcTY|1>T6l4_eFImG9p_>tFE( zZaT{{=}Ggu+1k;3i9BF)-DQF!fLrkGoZ@Sqp7JP*ak|EeGnzf(9w`OnR^oC^qcLhH zQ?9=-H^(Sn>H^}c;NM|^v0k=7!^;ftGCkABAau90$I{k;A<<3nvrL;BU$YV`g;d~9 ztjGR$I?@CMRX2=LUJQlYS20B)QLqoj6m|~|NF(%>t^%o)OJ`;wK7HzSRfLmFaA6k#;o)Y`ygsvUwA67fM4v@r_JO8Ds31d%U$)2@90w;>OJ zu8rk1j%~DJ?U4IdAb2fTS1|rwk=bGZP$T0UVtlxCACW1hqV@Wa1dZIrAS4BVPElQ@ z$S}8_%Lu>*+9mY`g=ssUxiG&xrG@XmfE4)A(>Z#10)eI+5K++=++AOP#WIpk7qdB4iO!J4aY&6Z=*Fcef}` z6(s0R`@&bM#_7%G=(Eor!oiCz$gaDk*4Znhf{5qH!ORZM2@6{Sle7jUn(c}Fs$pA! znD2dEpTL8Rm%mzWW`RF26Ww&IaD<6QfVI}!>AjNRWwvuB6V`X7vjJ=LPsX7oVLT*N z>D7=?&cqdqx;tUBSCrJ!a=9Tj!BC<%awexBsc0w6NrXf@*M_gUX|xWfZVj=jB)Ny68Wkk&$V4)a@b zp`S0SK57D?@d`;$*U<6JNdN>8mn3rG;}&L*d_N5Iwel zin%_R!^Gv(1~CFCox-q05trwN!$cq{Uimc? zRNg&66B-*fhj#p05>I!4-y{h`eDiY?kc7L>hklHjgGp9lx_D=}!Zte=w0S|MIsvBW zgOLKFs0k=q#MQ7?!Q!*OgCwe8j?gCd6i?G{fz*}T<38hHvd3mh^1$MOp$7pV#wKVW zfWT})c}*yEuLh`ujtV^=u>ywF6-nNE^E#3F#`S~~CtH`}6A&1)7)%~P`1y`~7AS3d>TxU{lD4|iu%wt$5l0eAoxnEv?goS<@w3)IqB#5A(7Y|y)ZVb*=<80S^= zqW27B>Lcb5^p0NjRR9_^64?j8*KlPhbj0WQu;Tb=zt$Hmr3C_E?R1<$ zK2UzXORq=o2|Ui60q(38G9`6}$^6X_^s1o=kmxVRB0^AsU&5J02;dtyh*Sf{s(-@N0|)ahU{_$iTYutc4ZLVusD!5?U#u%VeSETw&}NPhifW`A_@xuZEi{wE zO>FSm*ZX-rZUMh9n1tFhQta~iUgowjbD>>^v%M;ZlM6uAzNgnid=F&*mLXCRYDYgQ z$s*%7-PE2i>!%8R2T$m7Ly_YTEltg@mC3-Xws>5CK^g)ht;tm=X)|o z*HVVWwQ0d>x*Af(Wd&qD*unSI<3eC#${ohV4OS3>@4)pFG~}6LpDv}bnx2KQlF?=u z);inothB{)*S;J3%E87t6njOaSVLr+VzNVj@}#75Y-|LvnB-8CVl+Bq(wrg(=SolK zHwBdG3LsHm6{=tnvfpFKXel4D=oWj?W2v}>M_AbMCkIqT!-fxX`wCcn|J$!wJz401 zC3X}aw+w!Ud7-b1l3G`toO=z>e+rYF3D)OQSZ=G@*x2|P*E8Y`q&4eyXFlj8%rnmA zW+yLT?UsN!^|{?p6ZX8ix&f>VgjFJ@v#G%aX!{QvIMiEQMR_`YvO&GQ2idg(W}bzc zZyWj$r{yPuzn={4yRY)4{7>J#b`OWt##*0*n!_)y07bg*IAnKH=zl2owy41iflLiY zIGBvu0w0tQPCWM{6kh%Tv(d#O-sw+2P{ZY>+SfByZ?+G)!n*|Yb(|QS0Qi_vLN`~F zt+F8yK>VUlX+MAResW0o6O-`EjKj4Smgin`uQvT(0k7%K>CAstUO$>Zel*YZfAYL% zc7ZSgAgy{&PfnQG=3zaf{rgj@CZc1bQ>2ShL}PH&dDvdavh(^ixKj?@kFizBr^DO8 z&54+3Tj=-@bt&~8SgyCqFAN<3XWOcMt*j@-VC3*EZ*wIGd+j;7^gg;h-5y_Amuk}B zZL-6{aMH=4Gk7>uVI)$Z{W7?p!>CXpTmqN{p@t83H;BUyjlA0DAi_f}>lpIA8vCJ= zA71Cq;0wCM`_I*xTjp1R_%ryROLVfW!^rUm-E-u_G1qiuE^rMehwmRnwxWl#tiBSa zYZuzp#_zs=YAM(O8tP#Av3sSpgQz*<%ObS-@l|Iri}%i_?xILw1x48Ck~s1G<<`ch z`lT($XsJ4$gJ4gq>(M|ID#_M0z<(7ezPq!03>?v0E==4-mT`MGluPk-%WP#bQ)3;T8khOlq{X@ zWcLNm?er7n5SRDvgn+i5_7LnpLW`*;nyZZm6T>B5`!B{&yYOR{Ux(bo8_Dm!2QJkB zVnLdY-2n($`9KWcdGKjqbqi4Tp8}8l{>$c2z!t#%a{G0^MGq9`qvh&G*Xlu-vIAdX zKC1u{7Ja?1I^*#h*fJxwG52z8a&Lidk+Jny_89OvKB`)jOvmfw>*1&X%Vr-WqcrDE z_+ozPSEeTdEDN4#wl9DcUnQ_&7FT&sa_tLsBYBH*WXg?;7Hw<7w}6d##(tVeKcxYeto53zfdInu;CHVLK4}VOCJ?4^qK&+q^iAs zO99uOe|9G=RPXAh>Pq*@6;=Y?Q#UaJEAU(70`rS6WY5RSJHYsTT1hPfijBBYkbZQ) z7fn#)I)8=d1u=tb8p~A}?$gndaNjJg!I(H{5pQ%Q{QUf2&1AU9h){on5Ac3}=2o|N zlI$#cy)&$>4#z-x8vv%gIYoH!V<5Kg^SpL3u~vK^e1G&X7rVWIPUAJXQ2M9xOz0R7 zWRfsdRS)WX(I6ad5KzvRv&0;IDWq*U&WUT_T0OJd&G4vg8c~LXVgAPz9(FZ zB-gnN5pMD0gIBucQcd0{6-mTtC(Yc;|8}Bo$6?fu2oc%7Soko9BvvvQ^9;Ca*VG1N zPT)Wa8jXxg#=NR&NH-qi3M>4Q7ZBDL8C{c{ftS~`yhs_8YpV=X-5);3{&6B%cfxMs zi=TCX0bG<8_w=Eac0_n_$Nhx(O!n?i%= zcn-k`Spq~N$Kl7roLbp(WLi@;U5p z*!dH(>Ty_c1SaoSHWG>9q# qz@a2ugDF!(J#sg0h?1PgmIr^qWiv$we zcv}9*(MUI(Fvcg*(f8MMJ7N<8;%*Eav4UCV?evS2D$PE164~O~4GiCnKI<)N4a*G6 zHM!d79Ubluzv>c;f0np#Dx1m#DTG_gCmsXi&l+@w*M!9JZiKB_SwoqGwKwTwcnY=n z&KFfNQs1gT`E!sUeekC_pEfzLcjOVcJ8CfpHjM&uo-aI%op7?tgX-5Uch{#i@c!ZF zparB|*p4j|wjNiU<7KSKmP`<5nzh+38-Y`m8I=?Jk~uF)`YaCitTFQum0=YQfYF2a zH~`lv>GW8szxvz^z%4lLOZvH>$Fw}s+|)@hV#=O~$lvK9phBcT>Ud>s#m2R7*jT8_STnXLKlL@o}xjMNm2(gKbT78-V4xCc5(o1h)>oa7w~ zst+8q*QUnRQzt|Nqgg6aq}PPs8AD$Kzo#TxqegG9#4dH9bfxqTwp0)5n&k#eYLm@- z7&h1Pv$!&u1~i+tReM4aZ%wDJxvEnCf$pFdM6YjRYyDEnQf|C~uL*LZ8z`YOg%& zh}dG#Sh0Xo(Y@a8TQm+LCj^4GZHTL5eQiXYnEKAB#u{Ga%-&CS56wHX2e=4Ry$g6v(jr5jujUs!}7GA(fE6vw^%Qi*JB7_x@ z-z|=vI>U6CaE3NthtK8Q>DP#2>GnC1uk(ZI5LB;j$?It5vivleb4=Sx7c^E}M*9kP zb~g~#<(^7+R%89jkFAFneL8Asw6{iGx*~b7g{hw`WjA)ep69E|@Y5?R;&7B9^CFHT z&)7_s@y)yT-A|vo49%8hnuvtvd`1HHaD;?d3z(Wzk-*0oD$9)n~{2`+uQHSaYJEP`-B_v zhXOZ8-_wvck2S(O$^4WcBp1)o(}GnNG*VTl1nJBk6{DZQ^R+cVPv1dJbnUe*eL7UU ze>Kmsjl=)0L_FvE>spSbhw$CtYc%eE}s98Q|qzeJN_a{KgjuoN=sG*WFE%A`EEf8$2Fa4 zEHhNXybWA&E8t3y;x(vnI$nh9e&(qrCyCQ%N~Y6X^X%5b<2+v=@okt`D$Hb??NFI_ zIhE|WD=2X`vqHkEIj?<6L{M)7#;4E z2Xg;Zcq%9|ln%Prax+d#IQeF~ZTY!F6M=r*aMg_Nez4m5IZZSrw4ANV$bYj6F0q$! ziT54#o?NpizlacKACr}R*j;ps?G;B*E6QbHLrL!v+Z&Ez=_Wg9ovfb|DonU)v@Bw; zON-ir6uP}pXq{)jy0Q^;)7adAHp5G)1zlDlAA7Fvn{inmqq2+Kw_D4nt+oh%mSZ-WpkXlUcR$(Ci;r~o5ymnwt>s`AvJ7iUVc zoiJ%@VYF4c(wQwW`ndtJ>A@Di6-^UnaiR&kyqVQ8XgHmv@L7?@xzRY z>lH1jSj-12&2X4$wPzJud72fx^RXb?3zhZxdYcfAbl4NESDUuNu=5hGa6KqOXTN+9 zrC->l)Z38FXXW*2dh6gvG)_E=l=Vy4tD(`FhuiBtcdsh@7Nd(YDjNP9M=z)U$v)$W zu#e&VZDn=}uJox=Xa5+*+;^9Vwo%ghRz_AVxC8m*1iAF-9%@*;Tn>>pS(+xTc@*_od%^j5b<;g3GOreLB#2D;a8IdsHV zl$P|8DvgU;w4ZlG^u{n8w)P42i)T;w-Y;aqaam5Q9r>4udY?UIe;$eibfQnMSftyE_c7b>u*G`x zES4X5Uts^chEo@qgB!o_L>u|w$`kdbqYnk;jVSui;R>*}#hMQ+c=71MGuWlF!r$|+ zJTGd|kzt&O(qaU3z09}AT&(m{9(Xxp{6(X{HNSe3F7@Mt!flz;6&Spxwx%K8oIm~b zDU}EG+P#*GgxIUXS>-FZ!k)dT`eAgK65bD8&;DrHbrp?+cXIgVlje$K=LuSMX-4Vm zV!pEY?3?q6NzF{u&rwF{x{`t~&#D_D4Q}0UP|^JO z7%zG&P}0X$G4sA5yfQCS+u=isTbc% zmbZb8V+CKT2#PG(3{GcowMT<$KcNR2R=!|frPlaQAZ$Q+ia+ltgv}|VP@;O-uWBt@4)hKhw zE8FOMc5X+#_lK2nE6lcI=E}L`Gny5p+`Mp5&Roe%)fnE~3pXv^ape>SjQ9-Y+`w&c zS>A-joS|6Sd)XfpH-n%|KYrpBg}pJL7|(~BeDF!ePsc)F_jO=vfD*esY<1zgN^iKK zYOH?`yyqp?hL*)M0dGxF1|PV7AABG_ZbqLnFPfZoLTe^HS?i*hUV}XQ9M}EKrMq5! zyL?<;pRB#+L+&JG=A`Zd4&n&C7_t(ET|BSYq<<`%%Lija&I!Uq_aqBd`RO7D&eeGf zSXF!8DtO0IueK;3bViDCTzO+f?ZAK0Ak)K1GXssC4$tTL)PfjOM8$Xyz4SOT{b|M0 zE1q{a`L~7hLLdVEe&Adfe`dMEVrf~!p`v!3zzXBG(y=W|kIH_5_YP*A9J{`@A`ZO;@lmjowLu}LuiDY{$yF()(~7azsW0-HB9M8dTZN? zES{2Q_g8w}y^byD_MV=XFOCk^PE_1TJ;fqc#PS3e4cgM+ki?}e-e=4)ujJ-;;TJe|e{-UlS|eCi9BP`ewvTI|9$RvEV5gw| zL$FiL(kr-f2BR*G?w!Wt!=lAeMjjSsHp27!@cI;3lH`k}o9Qy0EN@Wj#RZ z4vlW}i^Mq*5lsWF7Qg1)GvSpM0x89qXuj$xn)r*UX5s@4 zkvMqU61=oA{Q8P>i0D|EF-DAF#&mUIZaUBCJK=7FS9@DSeA1SD=7CoWuaTP0WB)%W zWE^WpQK6%4!_8r((akyszQa1pZ2R`5+8d=pvMshlJkEBZ<;PHHHiLe7Dl0%K{0MND z4I#(DpQ=NUPG7TGGpXzZYmzY*T%Xce%S$RfzZJ38VvJxpvZu}a&2+-v@!`_9b6m<9 z(^@YpvnX6R9V-Ss+ds&4nUgRm6SdMm%kehPx0c-5=H9KAE1EAfr;)V2R6?mRa_-jr zv38j2e%A)-_4}k&S8=z;?bzhmh1_NQd%kaJVg5tvA8)Yf?L^Dq+|6vO%XkGS z3q*YQvl>aVbER`P+xe{}HpGSx;VRz`EN&CTv-tRl?ARwal(=wvz2d2SZec7_?SU54 z8v>2gVw;iY*SW9t`aE4IrK`H)L&K2qPKX$R3#T}qJD^+s7wO1s{CB2;sQWXR-?-_;l-KD2wD){S4g;ok(R1 z%bP8aF0oW;K~$wGT2DV7QWl>p7#oZttDKwZf*X8PZ}u<19IarBK0K zKbeLkq3vlL>P7F^q26Ar`kR>rN@O-mw_4PLe(*(*`(l|d(muJ_4c}Ts)#GCCgWJV)c$?a`kraNsAGb6A~lqaH2ufpYu@!5j>Y$ya$5>XZ@ z`}e&?8&L~YSD@I~-+8}el787R-@+^A6R&@yF7PQZi*4$)++#_4uW==he3`IK8k=y; zu+Olzu7$>{NxXInm1o{RI;FvGk1H{ZZB65G85ifUVyVt&%zDg@hg10vYwmxJPKmir zXD`O_&6s^A?NGlc2(kA!-ul+53$gVJC8JRl<37u++e_cveDuA6WUBaUTdQ$=AGw2?UU^>N3Wu3|Dwh|R{rwH#AZxq3L7cq_eUwy9*SS_SWZAgGGQzSY@+ z=N$;Lj$gWqTM3cHk2sAg;c=!$wgW<%XT-*zJ{eYtHSK@O<2nkjgcd0_m!`0lXer-& z_R05!D4j-W$wy>Mb$O__Fs6K(3BP>AZwra(sU>Y;`%fgHX>@~#2@RC`Ue6k;#M?_gPVQ{vG1Gwm};3u+l8=8lwxx=G3&8w0B`|5$!e6xS1} z)m9b-2GT12VDY=Wl*RZ1F=qI4uOEJ`H_ixXzm8L3=Z=y%>)(Pl-!i^;gT0E{hFQ>4 zPA-VqM$74!O=x=l$1+>ygH>wAty?8bJ;iBlkfycZ7@pM0hsd7juC`E}H%*lW@PUJx z-k%mP40+jX>c9r*T^w7&i*%M-T=PX)SR{CGE zDO>D2yN+%m&TMFQ!8=ODR)o)epDqWh+pfc+aE69-<@`nAg$defy9Gz=qFPPo8t7%V z)CoJ5u=x92Pt z9=T_Rf6&uL-(ruw9aEN?e5`c4d$k;TV<%}$O>TtZh7Hxi+c@W&#(_R{11jKYVYUwXJKj1Jz!_dmccrEz^ba_v+cj?@2;2ID*0^DI+siV)w+ zna-Nnmew)7vdeZHafOw7XTs@oWq!6~6?OQ$xf*2vKMh?~h@g1vN-ooTu?#r_cDX&8 zoz-vD-x!`p`KbxIx~r6>u9sM}8-7-yR47}zJ$_N%qiE;!qmqLl_r1LD!4jV|Rt+iR z$T6@5PDsIpe2?Yo7K>jeumcV5tKUJMM(0?BT`}1VtB<~nd2cTc$#2#p>uKeCfq-{c zpS9~ZoE9S8j2}i%$^TKbjQc3rg}Oj#u;p6^G%!FLu^iG z8IQnh#0k9&m9-tnDNe!yM7j6SfU zr$Ds7HKcCP_L7@EHuH)bD43A|>S3{-pGesj=abdvTodj+eVn+KaC6r&J;v#xFdt2| zCs#-YG2Q<9kii(crj@wUfh7oyk*B$2H3-(ky)pa z9ml9Ni)b5(zT(2e!;{^8{r6D;)GW(eT@+N0z7RuT2YEd~B~s-}K!8iWM&>f&((3*I zXadq2A45v`kO1G_D)N^eFp)Hn;UC+coxHg>()&SX#r^{ehHP|%i(!QmvgEU!Gbt@l zqr(A)-Sfnjeukz9hN9y9bR%7@6TapkD!uPsZj_m%ZFOPwOyY$+im8~wdIh=04O3{B z88pOeWw5jp&b_>CJ%kohQ@jkTn>meMqA{usz7nbRvIT4<(=}X4NkU($E4;px`GLU* zr8Rqv0q$f~^IOP^PRr#&*z z)$RXc3*R#g)5>VR!@n<(Y&=~|p7Ndzvg5emKcoN;>hW}$?_oPg;IHz$l7x9B9dN;M zQA!G(bZ>wD`?oW;2xn2G^s@h#rY`EcA(&B|lG3hQ-?=vd<~tMq)P@tt>F>an9f=PS zk50)V-)i#B0GD%dNln#rLVH+=t>dtYlZkZTu2*FP%@%LP$=NRUg6bbX|72OT1WW+C7a@FmlK- zvVZ*;d6$^3r2RHFDf=ZNYVk=6u;>}kcwxMm5Pn}pYmsZ6etS^^L4FWq$m3q5%6OpS zlV=nh`{+^_mz9F~ybn&OyF$H|mRL2grEw=Kn$C$lFMqU^W#ve(Pdt6=F+#PBNV_93 z9ix>-1@zV?tQiEt@Wvjz3lSWUwU7Y-@?WR=3)OXv)odTsSGhd=KsXK{%u0iEp_W$l z6Ap)Z^|NMcWPB{zK^eAkjDg70RM+X3>wf3@W58yX0!~Zq&_&^tiW>eQTI@0Y&fWUW z9;e;`v>2*fTg=w0$P!xENA|c$*@#*RxAtCq&s?=@gci^|h zNq`Gdc{iB47y!%Kg@q@>-DS^T$8H3}zkY{dtXusRuorXfv9{l%lFU6P9t^lnAFZ`o z3{(-1+(%R%j!qRy&m-gD>0;n9-p%puN9exsncPEv%7BdPD>AL$t(w5RFfm+wjGybj zr0hhLs4^sMEqF1HRGM~-pDf=kZFaTE#iF;Mjnt>1+uNaA{;u0(1!sWFwp5>9N$Cpb z&nDCdT{SzF($$A(<{pwNqT{X5(hGr_yE%)&HO>V#nS{8SfSi|W@EL8Wc5g@p^^e*H z44(hcb1~~nDV#reOIIw**J(;1jm`fQ1ray%O*b&Ce6(y_u0d@-9UID_c)NtrY1K98 zV-}4mtxNY2(VrN#pPcM-u-0<9W@Xnc$jsRYxt=%mmikYc6u&J(FvB& z>7V#+KuKxP&HhJ0ZHFN%hm5uduaZ9QNSc-aQAjZ-wAdUV^p zhMzk=(VZU9e$f%iR*mjgKi$F@ z98Ya5e|})$73~~Cx~O?vORj*?Sum!_*TvkLSL22L^t50EEUTH(M=7kdIT=@jxT04^ z!oHlvVK15sGJK0?(uy4Sd7s--q1x%WD!@<13X1NyUt9NCJ`P-cde2QtkrFw}DD;Jz zEs8;6JnAPs@(gTe;kDw=^HF4`DT5b@Rh!crbR)r)Fj@2RpjNX7rIW~wbURJ2XR9qA z1BBFR-qOGh5|rUhnj#Z{rElZrepi0j!n-Po%jTK0u9-XAc>N$y5wes%1_`DlhkIBk zbK#^Uk`Fdz?86<@1il9n&O;Al?IMTnT_T)8_vE$RCLA{c48fE5ztkn)uM+?=tM@ab zuAnWmtF}gE50yBmMg|()`qDgdJtP{^9JS`>A(W5SJ?;9`{#!=gg+2ThANUZY@6U?H*i*Jzm($FD%`)opBjO$JZL?FAk;I zrXJN)MT?ga(Y#%{XZr;?eZ15anF#^O1Le%Uq~h#k_cmxnf37@UaNJ`>ES@%%M4J!! zWY4&Czn~M{M9lvVH9Fpkb@NZT8Zr=pojUE|E#J1>kiA%xh%&P67tdPsI3ms2N`ADyD zOR6Q5oP?Y`Vl2tt`l;1|omee}<>%D|nJOuTYeWZC6o)??)jgxt;jqfsI97BhIKY1@ zLPhMV@Q6JRQjKuz`8dk-A+lvbU~ByN=-~mJD9wmOZsa>l56hm@Ca4C7>^@R;9J;^!wSmS zRB>?;Bk}%S^HW;v*WpdQNBN}74h*a*kc;mhO>A#lwUlQSymLx9OZ$r_Y@`$E18-Uz z*6-~ zDBJeY!jJqGI)xBfKTGJkr=XnCq(wxRg*?6 zkkV{fYo4bszBtOFVoFrKM}IppU`F510Y$0phXtUx5%lUhnOwbG`z%F4OveH)?F&u{IDHR@ELr3}@6^nXF(!3#<0N6bnW9=cy`eQG4;d<4S z?9Tg2r_>+6rbaHnz3#adB{wI^eYUHop(*?5o-*JF;AEo>>8zgvn(CHH+IxPbSIB=x z`mWFw*f!OS73ldlhG+TOP0jxJb6E=@yJnrbrk%#MG+$#rDRGL%FX>tvv@_M9emu$#|yQLMX;Pd%4+xO_YdHVjNqDjGZ#)i0W+uB#zZd)@STWy+KC`!Om!0heX2`+z-3`#>x$I z?u?S%YD-=O<=!NM7(CNUox5n=*zH|VaVPmHKMa%aqI?R*097Z+UK;X8lF28T%qgo* zD!Rd&wET1UI#>mTY+lo9YnkOwWJs*`({q$6Q2NiR6`F>z7s@HKS<$*~QUVL#!n z`Zw6c+S6XRa%Ma)^tQzgv{6B zl=Zl3=Kdq`8cao7Xn%aDsuTK8>MkxQo!^fHO`YA8&7O54pT}o*MfSZ)3?C?u)7lS1 zvFxExkN0l26L;Gc&wgjSYj6$nYPV+eO_*f%HD7bV}~T`O_DpYK&vwdCB;hkEY=Ur@TGaQ(JAI z3`QY3nlhqJVmxtHP5Z8#8d5*c)+`rvoppZJ7UrcUrA6e%1;AVHLcf|Jc1UH$50I4r z7H;ut)=8yTSL|$0&6=Mn-@yH;9}7doB8Qe?=G@AL=kJ3MKW{|SFksF-D4t`qm1{|3 zxp^y3dsn(LU%h*T-C$a0IIe%?G^n>cFAMj{WDpthANVARJ%Al?SET+{GNrt4*sTs0 zkclkWMt;_)uhO(jTF*YUv2rT&>NO*$93y zltD+jxwICxoF>X-b}V3E0wy^29AVnh$A#oGue1L(Rxnd`B!MdAE++4@K`K`inP zwBwe4UGi8SAGbvm=L6*?V>VU}Gc^Q7%#SN-yJ(g=oz#JC-+^^5y@J#n8rm{de_G#x z?h9>^Aw2rb8Ef<9RJ3Qk0N{M&Y7KdsO^hJWb8KSX5Ffh5Z?Ztq28%mw?RNMRGo#CX zS-fgFrMD!#kE_L+q@puesH0-351e)MIW4|%amtZ77Z{-E9%mND zJes&{asa0a4~x0f5g5+m(>o4-Mw@_NqrZ+<>B{^5?r3{hz2zVO9a4g^ydj2I zBty}-Tqa6Lak?N&tmH4RFYjbAU9Uxq6%ZL9@W!FhI}^pbzQUF>|$SQqRHwodu; zNEHpu<}S>Y8&Z_tz)H%{SwTX#1&@o{{B4>0uL-A5)R4bVlz^`G+3Ia%d`S_1s7J#& zhR&bvNkAbRk;YI8x_kY{IR5dN0(yP;9m30;z6kPA*duIN6{Eg=W$O(nt-8*HM*H;( z(cD$JUA;BIJWc7Es$lQnr_H|2u9#Oijnrn{*i{DYfgOIOuief%5mu;6kHtnv%!enK z-GEAm=Upf{enJaBOx0Gu%qWH@UDE!ur7O?E)51fXL#;Zy)tC<~Ts^vahL9+c4+MlG#Ru&VwXLr4Bj-CJ6L`vW0TGZUgvvV)!M(XZ~ zGNngN566%d@f&%w^wuqql-h6}ea|=?e>%8~KhK?dUlLv3$Q7kX#8gUV zGe~vp9?3g)u%g#n&F~NFDOeoMp>6qy@@eYJ4P1agN3KaA#uaW%_G9Ry@GHY?{T~-@ z<=I~~XM3EnO=}AS?e#|PM+vdT&FA5y0L1(iXw-xALbG?E8VIj1T960n#dYVoN%RBs zTF3^#wUTJGJoDotjZ5D(<4HZbJ3lh(A(5~COFE;aYFIb_?Edugo@Lsve}oV^44J4T zEspy)QYIwQaQ9*vowdx2TMUj+Op*s{so_txZF#DWR*sLl?DyVWUM9c zZue-(3nDDk18cX(JR%U}28 zMLU*UJLpn7={sn0+Js3J*fF~I-U;A4E?VD|^k#hc{3CZr5zW68>)jiG(evDXy!Q^X zOOjDfT1c$&uFFoF>m12FWvpo4%3U(!3NP|6(7)V4fXYaB6XSr+ zH;t=uee`lsU?bC~`go&c)5kF_6bEQ*0cvo+y8PueKr*R5ECg)%3LxtLF#LpF)}=T2 zH$StEm*;5rCNH|DiS^~C)Iad3JFK#+IBa<7*P)lv5bwMZe4-3-dZ1qC@;}NDi901< zI1Ui{rC7(w=E4htu$1ScHPzSFTULL6(#wiR{pGj;G`flWb;sA&UgbKBHw=_nYYF|`RDp2Zqy5!b zWpUNz9i>PGA-Ld!-Ed)&4wvu`o0z+nbV@XT2yyJ8-1#xT(HO8_rkDjRa;jfdGd*eNLN`6_$WC}zAg&{Xo9JI7$7=k7}v zPZrk+zvcUXZ;LPHNs~AE5d^}DQvgicSM!Ro&R`OcL3!2OUkxwTcIS zL~pN)vT10pPigs@W&skXHI?Zh<%#yrqedXV6Ef}w^iyDg0B`ML^c=ADmRG{AJwyVq z_l8FWB=rlcG1(MTQlKkb0_Ub=3_8;a5T@^dmHBj(9BATv0a8A+mkv59Bt0}-$qneZ zv_F^I;%gd7jfK5{t-;l3NeO`&YI6I2AF8l7*zuHc@{ZO$_vrS2E*KIV8cfF>z*m*G zxeTIAKH$(5(nuPU>VVGmk6!RKjJkZ1LQVxja1(yQ8 zO>OZ{G3cFl(HvK(rtbWp0Cd82(K!|wa!Jhm_jQ~+OL^^bO*^BIozp3wSC~Q1(|M1Wu8IS?Af<99I=gxrv zbYIda;*>i8lN+u!0tVOC$g8f*pK0#katbuk<+e6^)rPr?>Y`i!J6-`*o8m;I{)CFw zDYD|U%~!vvsVm2(kePM2ZuPMXDi#vQhf2=T|0l}+{aydO;~TXEphVff+6HafYa~Ih zy-mmCBK^{4%6~P_{+=r74BU($Cp1&Qz!)Co(=Gg1Za2igut=zMaL81OeYp(~rqeRN z{((^aeM@QrD;XbrzIx!j%8_kh~umJ^nzQw&SDl1J6bfwHO*Iz!26L zk4IHJAGHCY>SvFKQXG#pAm-oCz6yI&BFLARm6bHyiH9r+^X@zV~qk4fMpS#f?-R)GkW3#c*CGITlb9 zge>(O*fhNIs3APZarpwQTtl1@p{uM>w&(sQvs;zf-1_!cF;=(u9d0=p?o`Mx!gW3{ zQrdrSw7RvG8pe@0dudO#XWMFZ|5oxAM(7W!?2<>f9POykO%bbGB?EHi*-SH)&CCY_ zgZ@HkJBAT&E9+Ckz-v=#!rMQ+a^FM#z8bN=b+a+OETS2nJzd${xELygsgm>lA#Uc? zV!kIl#f@#y=&=*D9r)EF z0C!%p0v(d`DSfA0IzLGVKHI;Y4Y+RuSVUeA_o8=HizBsmVEfH zb@+ul9@hc6Yx!nkgWW^B!HeF{*Qb6~nE#9h=K@8BfpmFl7gR5$choV+S3(|JOW^0G zKZ5^~wz^c3;Qc_kEv{gG;dTb9(OVj8UkY9`^5te=8wOhVeyP4 z&iBY=INZVn-GKWYOvj__fWGPP#?t@Z`lF4vty9IIeLei=N66^mM8vu);XL834^D1L z>Y7H`@Z0>fuvYhLQT@&d7fjm!GYLSHxO5Lt@N9dtR7Q%_P#+kDul(8! zK{Pj=fA;n}*ZBVlv*5&M3P3sxM#uGf#ZrjbWp&po^XvbFfwwJ<6ke=&0Wjk{eZZ$N zG&qwM`hV{O!lQ1u_=yA5`3UGGzA&~mP*K)S`JYgp-S|K<3ZJ2P_Q0~U3~=-A)^UmN zPQ6>zy8*c<<3Yx1{NEz;Um35jxqxt}?=A*rcsD3`Ru4fEx7ZGAWN zf@J=mL;V6sq$XS|C!{napm-Digz!~>ygGEkAg&yB{%}HL9snE#nXdo2*i?FjpDPR; zRDHe2V9s>@Pjv*c%~S3FeE7+~sb@ir^54Vvrtd$M-@iW!_WegQ0}FVpqiDR5_0j1Q zb*2-2zTZE9$Tt`*|4I^aT!OuMyeoG3zn=wRwWR-_i-P=LpM9d!^WTGm9Prcrd;EW0 z*#DnzKDm1km_H&P^2H?KYzZdtaIU&S>?ED-|hBDP8 zxIV}@yqrNFK`H6`bq08SR$RIzHwc1c{z){hb#dUu=BRNr3(8 z<*)lE3B{VR9&lH#7rmd8hIFC*5`RV64j@IRAT2<;P4 zuap3{`#}VQMs50K5(qCq{fdl3vdIU!H{0=Hw(}~o@;T8x=y1KA4@5JeZAKF9{#<&* z68Hlq3e@+e%2g>)SkP~{iEX#A_)N`Wa1}wZ1JWxIuxBf%nBcE{K3lXX^VORJS(lLayoJnnP96KDS%OgE3|36Y_xu(k zKLvJOA!!;SH%|0lK>810!AGp0Xe}H<#*57YliX(B-%UqJERaEh98bV+%vMEAWMwLYQ_P_J!H0OQct}c*_E)-{8yso4}nIJISB5{wbDrxC&D7j zZj^$rKw_!gSGxP*k)vr1J}0dMAW6&>6%{L7*G%)kovHbM+I#bGs@L{?cqu9rC2gq; z4Kh@UlzC__%1|4b?I>hO=9wr>+f+hiY#>8R$}CcfqR2cCkukF@T88($?frb8@9`e* zalC*2{`swco;vmuYkk&zU)On^*Lj^+gOHZE$;Z{Q-GA@@39jS9W6BMsvtE@H^|b>k zmEm=3rn6WFYZ8O#M%)gYc==@wz2_Yzr(4iT3h~QjQg0k!d~>-jc~*Wig(X5walUrN z>Wym;%m3x%l%>P!u{nm3S3 zj(FwT{UA|Gqd^17D-Gr3Na088^5ke)=e&5v&;#0q9e2F!T&%?xavWj2Hdb6>P_JVQq#E6MtHnzFrIx7xH@Yc zg>upiZ)qC-C%Q{Vh(g&BChCz)3l&XTKs{ey;5s>GuiPIaVZjkaQM#5>&qxu9u_>Ir zWZ9JQG}`X7FiKoyS|jY1E+WUjL@ZM1b-q6~&&at4_iY`_?%vb3tH>SC9~+q!Yqm=u zJOPBHO)pD%H+%goq>tF^xZ@EWTE3Fa;UbOGA%Uhy=}E)KZ6wZKme~25!|@g;r1BC) zz4OHZ2Fjfv5)25OBv^HGvdG4tib(1D$9&8?&OG!~WGaffJ}Ma3xt)P>OFjPZjj?%S z=11nfmD;K2#kA|67Mha~1DmO6y8E|;V+TPQX5s5!6{YA6Ubw-)3Guwydy#Hbznaps zJmCIsWUHI*Dr#YZ;3IY(!Uq(OcEg0n*Wb$n1aWmltU&7psUYudB$cGUkgDfaF)0TK&Wm&jto- zLXPkE$7AUU!bNQa%HL2yn=b9`ashq>-10k$~72=V2J)KN=3R zNDK^fG|2T}IU-&&_ z{k`P&G>)VmN`&)bl;BEL+2Pk?o?iJ3WuG#48kkMIR|*p~kddocMWHk~cHQ98CAx+A zGSj+L4@Fn-glyVxm#N#5djhm|LSKILm!H*0Ywieoz)!6f=V_H~e*1+#O}D#`+EM|I zeTcXz#(Tkav$*Lk$DfTM8v#eZrkE?yNuNvvF`ff?6j zq;U2YW{S^+|5>g{T31MtwRya z`1vTTsOU;r=`>g?{G}TkClAjkAY^_cA@c)Pt!^j`o~%(*JZ5kLV9I`Gf#kiRAq|s} z(+|Zz*c6N(CztL+YgPP_VT`q>Lo`*b3Z^ZBIw@enKyft#t0bE4Je=8((admE^YQUD z&Z2^-AM=S{>U+yG1dGGiZS^K|q^T;@)6GMo$#jcxt7Yxshb*K%> zH6^O?WwEHQf>G(wcs?DkxhOP2_72MYUaAB|V(m$+b5*@Y2Yl59e`#Htq|qcgj?AGg zd7RxR;R<1Z7eP<DVS81 zyfgv4KMbdpQ?xn2=3$hQwlpG;^XFus^xT0nP5vm?uW3bq#A*oyk|qptfe^ZxK{cX5HzWaRoTUEt*hy4D8yH;NxNoyF{ z<2~r2UNE@#htGGxT}9mzCEZKj1&hQ}jA1bjNV-Jpg}w!z<&xvmA^X)7q04ri2^1Jts)h^Dp9(poRPveiViS!dYzU*Ps7;mc*6b zs4IB~X}+wYWurPw>bJZ~+rUXUmGU?@LX2d=E;r_5X+&ayHgg6d(}>$yu(uQD7TB4- zv|X3XedHTGRo>7rK89K*3ot&H?G71Yyn9S2OXG!p?&@95`rvv=T9Z8OQ`q6K=kS9U zoKz{;Xtmi7?qiO#1r<h@Vsv356%*CjwV0tGD@+S{G%0%nerY$$w zy(yvhwqQ&;iimQ9AazF{G#Wn!N_}5-kVU-=SN^{*rFCyRzyJ5I6ix=@=KsFb8UJ7Z zZ>9fDY*3DY{{COD|ND6H3^GVT9VOj{%$e zJ~g*yT)ayhD(mBqtR$Pye?0wgm#c;Bj>b{TLWt7?{YwnwrXgL|IV54%a}o)-X7_;u zb-iDsqdHzK-``T;`l`*7*5-tU&V#Y`2<=wGLAZ<_HXBQ(A}t zIB=1f)-CnOPk#b*^;scBe7NKZ!@e>S(_1X5Z~rf5kR+L87`8jF z37arcjWWU-NjHe~8T-b4ZwV)fV*XXv`=q0+v+x635J?xCz@tp`yjoX;o7RF+qD_QdGJvBMo=M&G7O?nPaK&o^Gyirc zxd9Wa{yF$-4>4qewQ7Wgp8{8B&_w{|EVo~mqvTvO-hkO%B8%_|h98>62>&rOg>TRC z+J{XCF84H%uMHBP~w>Pzmdv)59&mz(`WVbkBXYyoQQc#SVd{q?Nxw6QrVIYo<^__Nx00J6i?&~Oj|SN-MrSA-7+0M4ZY zhF07cpB&Ulo%mi&Dl32w4nq+=fknbb`DP%~tWivy_{q`;bXz|^U$S0pX&pV&HQ%@B z2;0$#rSzD)FnZw4_7utwvUx+<`jHG+<|3^(#}jd}eznL-kN<8XEd zuYLd0UZ-5Yf2i_c2Bt1_4Y(Ha!PPkd3Fr|rz@#)#G;SOORGyNaot@o`BPpFWbHqti zt);l`p}J>TZgFH<)O6+FI2R7aTwClryyYCRS+z>wanFe{#iC{= z&Q|LucZ8Lf<>E3JR1*(tpAzh!ATMzZ)WWA5zJV{ijLYR2Sz24;4430$Uh~O2ybX7u ze-`5v^YK9Vc@rqGjXF?d$!f3Jy7%*C!OQfL&Bmf-EY0FSn={M5!gvwR_R;c~tQ9k& zq0I32%fRhFIjIlswsjHAu+w`6Q&Ze$r(!0qocPlduNpkPUb6}{wwydd5fUCCW`=br zJ5e>R_~)(!GyfV3ys2Eh(kP|6jg(p;Mn%P+S29^g^@I^RuxiDD?luL~b%gLNr{ zQ0khl1dfzaQ{0SqAf&qVM#us=pyn1}$eITWqJ!JqclF(P@}z(+o$|5)cvjEcFLK-_ z7I{^s;L#1p7*_}|w2`?}FZ8!e3RUYE#FcHBcg9#XzZULwDEvLR%YF7o_%I%JhK%Fj z`Ky*(>SNg<$>+)lu`sPoK6@@G9P%nVhGOsJlHfw8=te7!*7_KibR+`Qh9-G>JqwSE zx?K&lo@dW&zEbel4V^>!{@`F+t@)UAfp7XFs0g^ak=O5^B{ac|X&9WL8>43T4(hdlIhW~tWPVR!cb*#>S@h?u;A|$1O2oO!m#u>^l z72kZ&7;$JRtSBkl{(U9jxd)-S!6~;&l_wt`!`=NTuR4pUeu

?AI0@F&3l-<9XDM5vK~St<|U;**hPp8Fm`P z-vo4K*GScyvoivL0}cw?PNJ*>h4MdPQsWpoc4;Y>A~S!^S7_TmRR4JxS~?JLo4hVP z8RPq*S-daBHr7aI^MeY*Rab~I7F*NN$HH?Od*J2b@lbIS30Unf4&D%=wRVv~#C2a|Z!TJA#Xg z*)w^vs$4MjQOW1qD7$JPLwGF=Kw&6FW(W3Y>3Bd(E?NXVAwRD!@bW5$+VzA+P0+Y^ z4q$!99Zv>kM)O@=Gvwg3=$LR1R>w~9Xny12QS*CzAangpE_TjG0Ta4mJ}j-dzFG?s z_%X)$W5)@`uJ=g-)q8(a__A`~qq5;TBw1THzMC6ENuB`kZMWF@e@x#~a57SO2BmJsL8jy-^=?l*SYIQ;kVn?SUHL=!y?Ww;co&mYez z72q&ZQsNFgCZDb#(|^3`G8xt}5V#PwdklBZ%JB6N|2Sx#u2Q6|*d`V=BWQm~uywCF z=f6BX(ZynV|JS`eZ(gG8rFLIRmOIhAz(jF!bWHr8{Em2Tjh=kZugbs4nw2N)cdKE* z&+Ui|Gs{$Sx*>* zM{RdJ4F43zrq*Q^s^yDcOKcU0A0Gf-{alOgxW71`lS>q0B%KXH4C=9fMwf>%PLedH zj%f@2-XE>(o7wU#uRdXhCY974u3+MxR^ZtYg1i(O^sW6xbhJz{K)1rozfiwHK`LXi zh78~p)qf>*B$8n>q0gJll43vhW#Nr~yfqlJEg3_V|C|wi+SR&?SW*NHljQ=*u`h0h zd&bXmA+HTzpJ7oa(?AiHPy|G86rNt>MZe|xao>RGI<5Kj0>mnV~aEe-;IYgMioph;L!IWd-NLXPLD`t>s`WBj;Rj6 zld?C>;To?t&iPEb$_nwb#d}~VjMY)YE#qMB1RuM0QwAUpw{!Y$UnH0>)YA8t`4T$$ z-y=-&S{#|UxMd1{gm1MPk>WGfcO-|mL1pv1HFw{!WE2u8v=5+U{*G?VqrdI$38IMn zXrb}OL|th3K1M52icUn0?kqHBmQbKx8Oy&ibg|7NsL-vRRTO1!NNx>?mGB$y#TJmE zj+#G?5}j3<+3G~%V-yl+hB?Vpj=?A5)mxWU&gnT=;G?Me?&4lt@xinu@JY)K3QQHZnsp|ER!f zqNPnKK1?If-&85k3i~hoqw}@V_@ED=-jsqVxWxNGq2r)fE zlsh0Z@jzi7IGjAZKr9B2*eGmNY`rES2fUD={P$%#tA2}W_rVSLv3A5rYPib?i$=MO zwHMU(*1hSt(*h$MupzGef;sPwZs}4?OyTiM)y`e9WyjderkHJ%b4rFnIHu5`w*@{& z{`Zx`uN!jOESo2V$15W(z4{3HsjK_{QsxwoBL?9RTq#i6C{;+i00Uig3<)JwdG)n88 z@5eoAdoo$5``x&jHQ4(bu-X*;i;R}d-mwSRVYUm7p$egZcmnmwV`WI?DL#Q{aTaC8 zV;@8g@3Ka&uD^7 zr@aO}MQMBB%6pDK%L?gefYc3TWo$18<_p9SEBm$=$ni$;0t~bm0j_ z_s!!Cvobq#^?%M3l2MbJkzZD0pL1(tCVPKaPEk~prbjd27E3@f)HI$1!(ap8J7j@6 zdeiahOaq;+`(k<54^Mm+|5m$0%JIkNf=D*5jW$MrU$vQ{$no``D?T_kLcn{zF!dJw z!k^W+&8zG_tla)0p3d{~T>qnB57O4T*xRXC4<(H3nnbrU*b1v6+E;=noi3_I_F~cf zq(imy-%?n|fPh3iw?~m6#_`gbydV9m=K))WWs~Fa8Jbl@8CYc@-TeFIy8_fxyss2x;)wlXd zXZ+yzUaF0o*$K{@;Ol(dRTs|jCl$3E13Bx}-Y%|^Od2j{Kj9%b` z_1{@uza0Bq8T5OD8DlFhP<=M|zcYg=L(Rlx4{4;hIQ%=-wi@CSO`mo6zr>v6o}OfMxvj)vGaj$|9*(usS>AQe zcGox`F)gxVG11_>7x_!AaXc7sxQadj<|8va|MeS~9g;PK$|xkOG%Vl1Pyd;f%IYkP zms0#a!lp-P65eEBPpT)h4so2MBxH@Z}V}~AHXJU_=^KyAg!tY;wVbjDblQSLW@{0q@mdH!O zS${U!Lyu2L7pLRuMk^AKaQ(-@?YvDJ{yj}6VYbIcfag}esM)P9(Ym#S>GHopBOD%} z46W6DUx`wHMLyUt;OOKVF07PY0-(`qGBF(=VIp1~S5)Fn4#G)=erQJDiT!Wrd6k@} z?Ecg~-f06P<-UZaaga{6T)5uo)L5aS?VvHLhDV9J<0m^&liW0{Qe#O(x^H2$5==?0 zqilL$Fs)KA9)c>}uTv_XO4jH>r4ge#J$}DOL{*KIVY*cDVGYab_w5@ic3{V&$U-g} zj?=WHx{TR2KMMbuJ*~%v+qRoJjYCg>Zho5yr_;gHW5ihH;3j*i$UHP`Q*Y6XZEl`F zpg|OP`YlzH*D#lLWM|n*_E9=~# zI0^%igbIQFKSwd|jMUMHRCDk27|+&&>FjClV7OrnDzY+FH}wW!DTX>-z~))^M7zFM z9iPRGjNFCgCzbg2VbPd6sQR#>ae!7`D}~UwApVD;%C=KTQa>sZOE&jJF?W_6J%P(y zhJYb`nw-)X;e-)zkcjIwgJA*#4FV;(#WN7ur_FeJ&+ z&}{EmWT@1SaKSV{r}dH*BXAmD10j`N0S<9mi@-@cD}Uzuhq|v1tKjQURto)d2_>PX zkSOp1-Co3HjlaKZ@!=*RBl2zorhTBDUag9I<>Q}Ao*HIFviP&>KYZgJb~whqZy7o+ zwx4d=Q1H~qnrh*`=K;3Og-Sdgy=V8_`1xm7Lm*4SV8Rm^zj7baO&@L$oJJ9loynvu z#2E?vx#RjHFmD#z0u{7%2R`s@tO$xNYlEWHRlv%C&Y~a1lIAFuR89UHr>1@k@|6`e zt&}Cn{r*(%>!ABC{66a|D3r>KN@CXxi~c3ift?u@Cjp-7Ygo0{lk;`-`+t}rBcVd5 zAvB&LKP6@WgpwXRgd4BYb$T2-nW(|M^l~z*I~D7Qy2inzgtJR-G3{+TskR6+PZU;t z9tny)B&@y>)F7K7khLalNz-ir6St(a;!S*Dz$Q#c#f23_8mTPa2Ya0n${5u;*3?1{We00aNJ?yjrD-U?a z9=x9)bmCR1Sx`|yKgGTheGh?ul#gNKF2Q=K#NeG(hto&S)Ez?})nVWg04-8Lb`*H- zG!+2NT5U%X?mAHO7iY!eAqlk@`1%!O8m(n|Ci%aIvHsN#Q9VigqA~+ZM!QAfIaa84 z8AMso&kds2(pfj5;pv@ZgGODltHS0nWRqxJp}q^BZ5>G$B~KA_3_yw93Jjej3>5u` zemX*o;=eMPg6ruk;iyD<#} zaqx<09>q|{(X6aqOM(f&aMXrddKY_HJ-6xK2%Vc$%J-a>32#G14Pd8qvuZUbf;a(^ z_`T-PJn7nbOzd6IM+6;-0+V;vI}bw0Av)qd%5`Xmn679L8ZY5ahZu!fL9%GPD+fs+ z1IrAUu61zKgmO|}Nl;b=Vc)-8<`_uvOcGc4Q3SivZecgMFGE<}SV<>(*Yr*0`#(!h zLCuUrY@HURyM>uijd>%>1*0XD#ESN;#Yl0uL^y!Bmb~m7nd*tVd?}U*cNeyzs&HI( zoV%%sgOwV3?(HZ`tAODlc*XWEV9@6FNPt3sTT=ZzkhOhTQ8ua>D}GcA;RyJKX;KW6 zmLa@pL$XDkm?UOs|BG=BHoFMj!(86gFga9-obju^)!G?hD7i!=|E#4+pTF3Xl~9~O z&SR6JR4{s!hVJ>T?Kt6~fZD2R^FE&5*sM1DU!$@C|wsK4B%cq{m2Xd{FT4uo>~uzu$?e4XFQfK!hEl z-)bilQB+IQ(eSiV)HI*Gp~hzj=On%#Iw<+&_j5xj4Dou5{|I$=CO3b1>p8$7A|Ez%Y49n zqxH(iCM-D-w4Z-}RE&Z?jD>ao-72QT}U8VSQQ~EIPxF`A+W+ zZPoNhVV-y!1?_Nl`A)1!U@k{Vr)o0pKbiy@pUqOhYoa1;3-FHtdg5XH8#}Jk8+kIW zXANx?dp8*Qe%5w`J9Hoz5GqpCbbcmCCKH-lqGST4RIitwsQz29y&vvV2KA&)4!NYF zBx0e+%)LQet4gczJmk7p@0t0l@C*4iUWzv(ikvxXf!8^}cA9Ml%RIGo#3|A>(lgST z+%s+i`H~z!1elSWX_Ffx=gocXA(D&FG)cEyJKQ+Dfy;rDGUN9oas>G}{Uag03odp4 zkaQc^W*zF>b)ruHM1sc1r_REMPHl+A(dW? z16dB+dAJZSwq=G}O3fgMf?^Q}w+h0D;fI`8f{DY^4q$q0SE->)wvi?+cPvn*DHR1N zkd*OFu0XP$w}x|QA+vp~nWhY^*3A~qt|sSBKg|zj;aDAm3rkFiw=GLQ&%V(;I|5vh zK5#>B@|&8P7DaXG2=hY0Lr4u6IHh#N>pp8wsFEpj<^yz%3qH7Kt*P)C_q4T6-Sfl> zxe#`j0B%p_RL{>bnyt^x4n1L%SE#La3#DXvfDt1m5|{q?3iz;53x$=- z0?lG(9>j5jc@5kzE&oD#T%&;`4X6`q`B^#((TL2Q>9- zsm9Lg{w+1g6HtO~hvA_3d$LNjpt^8m8P82>86ffjUdVdD%FN;p1Ak;g9|r*~C(hA> z|KT6l%HHU~w4(Zk{h9iPZn^%c{@T*V2KX+V2R>WFvDIicJFlR%T-M5DU(mafykCC% zo31w6RHu1=l(4CTU;F6KmUpjQJYrY4i|GP?ngvqdt`pE@&6U$SR>AU&Az zk?}}8e4NymU!%rCQC_J3*L5#0-a$6XvCk#OLwNvEaQb)fkqy5HSP%{6AvtG-4nVAl zM6$lx)P-N6!QaKTxnSB4OBnj4@mxWNYlwu0ovw0PlHh{d1&dFEMdFxW+{c`9L{o{6$ z+2kWh+;?L2In@0I|ArBgTc2RW3cx*%oyzmQpQ-YXoES#_$T$u}&-X^2gkQS(K(dm# zpe(>2oVdAzwgTvmc+UZd?Cs!X5Xjg7OL5YSd05~AE}is^Exhoi_-TF_>Ai8;xxqSTQyMTO-oxl{iTpd!NFrU3v0K@b-;~;b9;s(79j6N^g4K` z{r5N3IoTNf5D8upTe$BlfkX>b5AqvW4xdzi{v0lLiBWFCzVv|OJVgvz3nVTr1O)dK z6NZl^zd9aKXsps_{^QxQ8q57EwZkT!N~DA4F^9IU?QF_Fh+3yt-ZCSv@vDf^O>yty z@^lrN_tU-uBw5!-;NV_#W<)&TK44Wu$J+u}Y((}3cv={r(8A$1Wi z$!7DmiDDao@GNtJCPXGPBjADv^_dZGh~!!#cuv?Fm?HefcTwtZ(* z>BT&ZjNd#MG7vrf`~X`5h;?Mv>Ee8{YBLdYp?lMD1#;Cit}}OsK#RiJ)NmK#gPhBL z(4ciFY8wmkJWJ*96HGjdOK}bbHL)F(C6*ghe+M0N7-pMQd>kd~eISV!7Tdo%rH9EE za}y6Z6I7`Qz()yu?o3B=DbYnNV@YJ`N)1Uy3aJ}1HBPBJI)EDS8HxjQG%=VXXvz5u zTZB};^%c7Y%c-)=Kiw@`G?u}wmlxJ6Io#+oX((!puihvlqfnOldFTpwMm%(E#ul?3p8b4Mrz;od>(5f$;$^ z4*yof>`X7ozImIy`+aEjD>HZ3d+le-eTm}&Iv;`GO1n1>ZO^Mqs*g2t)=7wZrpD5d z_ju|J?K6C|xz;kthCDUzz!#c}dDDlB)0u_fkSv7)`7;u286)e9825(55+mQ)3*+%m zy3dF?;{hCuly6lwMvNJn5uTTTG!xbuu;tF)3f} z#nIx_n|l&4lMV=r$+3DR0ehj$>WPc#ya_3`lmd#=9bu;|zI?lTcsFQr{(zQ9b^}uj zecNyQ mEB@n%Nj~>?kUY#K>rwSFTsU8(`^Aa*Hh3m@9W}RMIS?R2GwsaPC%qX-- zm_eUb=h44S40UrWJux#eS(-jDlP%kP8Ch!bG_@r4KT^9W8?uke^b_v~A9(!Ww$P07 z4+qTl!;=DJP3qXUP`gW$LU|T!>}01i%5e_EUiXJGKZKS}hu5gZYT?``OLj_hPg=Lz z+KS9CU0pM=eV05!58zICqaFABtZI#G?}PWzu^`Owv;P?V_-w9Kda6fN5|qv}tGy(0 zRJ4vDxb?Bp4k6U1+5j|_&05NfPVQkELVyR|4DgvQ1l?g2T~t`qYEkVb+D7sRlQ8(9 zvE@Hg@pdfKbrl7ruCyB<-XQW2FTE1jXhw-Z%c@!0Wbu*@E0T#QlNm|UEt^FpC<#d9 z^+vjciH}TTlOEV@BAulni-WI<5!0-a%DY6eHFoGHZ}iuy2#(5f{!%paVF?#RP5?!_ z8pm<`SBjik%djJc;8-d?<)o@GyBNF3e~RS1Glg93$pxfGuRhN#Owlp2u^l+X^w+G# zKZmW$t|gYY1ku6~SPt652_j7yH{9}}%mxW#HSuMEZh5QX*DzZ5l;1~5?Nj?-8-9sV*vMsB(L{@D*w1WxtMq8=Px|EyTR0m^yY?(G9n}M zKaC@8-%3@_D)AMHirwKS86F^=;a`KGrSnI$W`x_xrLm>w+|fK@Bct@$tnUPEARrC+swcX9cCQMGG!Su2L*pvVp{*v$G=)5`YROSU1J@ zZd6b$eCmVp#%GCL55EED@(X~<6YZ@dj%kkS7ViQxZ|rrB`AmoS{c@*1Ry2t;t+wx( zA$7Q^Hd+1ADj7X#N-{?uVT`R?A9hNlYmh0s34o*H9n@~EW6%*t+<(f50~eAX;qJ!o zmZSdg`b1ws!+XqedlRJ22cWMyGs z6f}7Vh+6*!6=cc5hbho**ihjqviY?iz8FKW-A%lR?04y}$D)M?P`ev-=#n;GE@yyv zaQ-9MS?nQ=u$9s#Dtgg%5%Bgw%{sERy4BgkXIXO!xF z;t9C~BAAt?1M|_LWdtFC`pYdA%AR!(7@01^VK7J(+3hb`;WBHMFz`-0_*W_Y^o!5H zFdr7vzm`Lh{l68lfY3(okNNqWE#+!`x22@o)Hh3n%n(L2>+(3UEl$6W>Q-2U1)Pt$ zN(b=?eeK*|Y&Cp=_ixR*(lZ}(4=eR<%N7Phmjhdox!CJzks{K4Yd2NtTply}U&odB zxOJnhzVfsigV%eX!cBK`+{@~z-%T3lYv1E-J(bIR_x!ghfFFh>puP+<%B)tO0fT)C zkN-xBO}BpBxvI}5v+HrTTmHJtg;L6>+Ec#>C;ZAxj_+aA+`TH>`|zQr=_b4ES|(}# zbA)Y)2M^BRcYWXJXOCd&mC;XvH>c2cRw6tSL9|l_S_(&}n%QvlV2!C?CzOLiQjgk0 zavFk?33WU|0HTCBE#Bl2#c3;3vVz7a3q3;9L-d;OH=OX`-i}4EG<#*j03iMW5c#H@MT)fA1@l|IideYC zb3c#ZvU{{4yXpoy7x_Ro z0&Pve<*<>C(hhUUq~&PU#001bn5wcd+*cV6)=ff_{&@$Q3@CDR zjSe+H*phN^pH?eh(2@#zK{ePR`9#95^8T?Wa4zCsib?3wV0Cagf&zwk@i`B|1pFEK z6Z8AcyAPin&cu;H!Cj#*qnrtOc0NE<`wFu0A!|hCSiH>40RG)aeCAY&yaNnZ>xW?A zSlTa=KnQ(=doq4}qTPH`&!3xqCVc58Lyv9%2)Y&n6c7=Io=lPEnqYjOOnhL=yi!En zW-+&V8h2_MjT{Y3)_;9~h-GmNFv7Z87T zip;cdGMNOz+w6OimbH>bal&68GtRWfK7~_RwOtaP%A_U4H_NAer;L6OxRCxArO~Sr zikwXjaUfB+V&nuHD+YO*83}s1R+@=`hxA5L1A@2%&Hdk<{p@VXPBKZ?lAI@EVp{%v zg1mXJSg;2{Vo_l;7?@7VL5miUHzZTu>M#k^Q^y{YVr?VC2xN7N*MzhSZQ_JR0Z!P1 zokh$#Jk1ftF1*<1UvKlh34CjdMmRe@IPxRr2lsd8`!|EIRz5t_7AENAfgZ&GrNWVG zyw5cHvk{E|yrzH|eNZ$4Sgm z5m3_5($dgS`^>2C>EOB5F4Q|2`uW2c?f5Mp7t}pHy?u1b>gtU=y0ydZ|NNyX_ZWVs znU~Khut7B2jQu^blynsgDNVUy%4*bIeB|gtw<9sg*No)2ngRO8i35av63E6v3&(4fYA?-7;?rF$k}L^C*w=o` zt$9_XCNEEtvs3%SPyn30S*2EcLI%t$K${uVub1(xciVkn+NE-|u0SkaUAm?{u-(x7 zse7PT3jp9?(xv8GgzH3F5 z*EX-dUpBA0(STiU$Wa9=j9l~Y#wGy)im){Yu>4K*ZAn4MJSQ|D|^5F4OeI>{BfIDPCj1gDQJgGi`f z*Lq3y4==7%ZR$}(TKFjN#o8XAJ+?;NwMp#bN$4iYazJGOO=#nK7P<|LBJ;op8wZs2 zyZh7!O8;zF5HLv>VeufATj`8-i%T3*VvKMc+j}&yujG=L7`bbkx}?tf)VyPtKr({$ z<6?i{0(4;A+`4F+jaSshy*F(OUK`O>sg)hCx!M;LKm9j*raW-#kD)PrkJST#qeL)z zQ(F&TB!hTB1{5{Sr1uFIR=NWwNJg-lbpY2{ap>hdh2`jTD@;)+l%MAfMS$qRD+V&{sOEmU4=@7MySC_A3jJPyaj_ORD zDM4#Zc(e>5_6;5Dt<8ZoQLXNH=YXDyP6t0ez}ttPP7AVc~|4L=H0 ztD16v8q{0CeGe?IcNaW}wM)}h8c@L9Hg>%^X@)-dlpR)9g@?~imKb%{_rh!LT#Pjz zit{=sUF($htS-gcUT~$2i{}`os1EZ1C(lMpMiFQ2j`NnIKS%LL+x`)`a3++99rcq( zpGNi0-#PHnQ8Eua?H{vlOP~@{)I*1kA9rhj0JR;HF}hfV7qr-k79kx?m?Nx-Yh7J` z51v_WfA?=sIw?rrIDp!0>KFWI57k6<3c!`>TS^6!>Y~PqYz~jUp_T}Ra8t!^)ikD{qf-jGLdCJ`dIlPCmux<_~zzOyEAV&qZ86~2LAvcbjdBh3^J`T^>8?m`!BglL6| z~4N`O239DNL`CJZ>Y6a<)XY(w|N~ z-Q4P#6FLF2^Kfj>eM+r7C?a)I$;}*ik>Lfpw_f`nolv=iZjqV`mJ5|+(;_G}0wq`x?qW=UgLKP% zVs9SiT+T#8MX_-POs;HTBO+$qh2)w`{2CWd6nyJLB)({E55PIXMmh8J;t(<5aV#El$+#tH zh3=WFsiGG)hW8`x=LERYl>Zw5tsd!I>tk!;y;C?L?k3`+vW3*L9~z7IHtJ+CjtsKB z*@|!fzDdS^Bm)5vB7Hg0n@$AH_^R582}V%SfvPAVc|ux^m2*ycg({!E~ zLJ=JO2Tb*T;y59jkVXHWBe!2JDsPF{lO{HZ=M zaGC0drd&3**&G2tWA0i^Z6+{jo6Rbe|QQraq4QrAk>7+Y_RmvV9);+HdB z(1bK+e;Y$dPo)qA3WSF8UMY%~S8R(Z(-FrgLpNa~7^PMT+Bsx!7kuLiJKvYpBcd-H ze|**>3DQFRHKn8uFL-+8zpmKsrc{2R@~Ai@LG?JJSenRQz<5n3KvJ$2e z=sF?MUEWV#yNj$oyQ9f}8wt`Anv%6_cmO_gOn=vadp?Hn5_iHM-6Gye^zL;L;Y)Zo zUO0Jak?mpHkm};DOl(TWDt+892$wa-k@Iu!j&z0K-{{w`vY+XqEbSaRH_apQ>yPfWhnFb$J?Mo{Oju!|=p zqowC2G6;9O)}5Qy>=1FlDE4U5II%4ep|zAvL2~Ub*e`%}Q*};_m`Lf5co*MT6}{XR zx$O4yG)8}{Xa4up5AxlBZhnbYJ0p+Ek=u1dSrp5Pp?ZI6y;Bbj# zx<>-c`eJ>1G=)l#7yB3wQkPPMNm3$iDxs)fsK&E>g*V1a2hCz+mxe1e(F^7O zAx_a4I(TPd2nnwp%18hNoo0&6CISJnX=6mVnArs&ljO}KOV$3032^@!_t!9_ zDj2rd7w6Es81&|H44S1|65~UYE?(?2;b08{lP3OYdiS(Fdf5NCU z8ypt;ahwxz;)RQ-_~>M-V-`u}0dy4TDJW+2=x)?m%$V7gT2=p;XeGK&p^g&n(cM52fWoUE2Eb}g6AM(SlE{K z>u~Ktx%Hjam!Ig(*rjCz?NY$Q+^hqF6x_87L&hxdZcM;5*``48MRW_$M+QKDi*Iry zKt%pNbdjEUxBu47&y!M-NpF>tUsYsa$TroZA3nig-1UaVXJZHvA?HUKejfn9hon37 z8}=22^elb^6^PVn-DO^)AeQ1B#!0DC(`vSr;iHF(&!Map4Gon8TgJc5w{#{lA7{~z zaNvMX%}*B7$_}wwZYBF@y{T*x+zM%p%BY)Hj#P@(wfm`Fk%Vp8uXUU1N70k{B6!n& z6>es3X4bJE*q_kE_;0FQe|-SvHYt4~IEV1u6PwCM0%K3)QEiCOF#TDzqXh|{qhsuc zWdxTGoWOVq^~vJA$3K#88{bSW$6ANtn~I$W+Nzg$m*7qLlHSuk`3G|87iDMP*MK?Qvx@~3=IJ>W<_bnO#{+@!Zpyz$Ggzn+d>Y*FOJ(5-xK&$2ATQbkCCbw0hSm+af39b}+Sk-HS4LUw? zg%JEPgfAK}5GQycJ&OQr>Ye@e5CxjVZ_(6Y%RDOTifh78Mk7Zh(e3s!wStpYuhE+u zU|H3|&{ddf#u`6|DGWT55lJ>7#*Y_tWjB)%G`M#_xU&>SxEO%S77os^Jl?szk-8c0 zk#ts0%v;1Kn!sb(vGmE{oXTnT8D_i5d7gnh4wQHO@Vs={hVBJWN}K79+0>g+4oBSa zPx0acxXjat?59@_8_%sX7P`rngUCR&1>9BtDPMHY(g>);*d3RsLUTi0Qmp}da9~@^ z2i9V);C36m^+AW;Pn*d(o||$$)QdoF_5-&#usBx)>fFbWMa0S4)F;4R8xw9%f_EQZNLEN?Y<2EkPe#&G%(8a-8`d=cgml#(A?VRWS$-38ve@4cC6yi7-#Pc zJJL|nKfHMKP*3>|P>CE{YPR;~S<;ITM8P-X%kX)5g56je&5`%LNFLZQE3m7j#e(&H zih1OIqWd=H`+f-R7_O*4{98Xm3pg0_4!PetlQBH#?%@Nxsc76DMCW?!ryRd+-SbZ2 zhJN)^ucQ7WO}_Y!JUi%)0Fm)(GFO#w_B(`8{HBUb&H$>iBWL5VoRtifF!#<`mK;`r zrjvwMx(He@8V|=q7uTskVC%>5yxLc~8#l6c@#*Ty*$+{<_jLXK)$U#kJ)rCpX|?_3 z)+=sPxVQ`8v+hG<^xJapxHUsBX<6{|+uKe7VpCwS%scP&kEw+ywd-Y=&zO%#1?eea z`-v1m(J7u*?iu&DMlUU)laQX@SMl}*3nRQ`suu)@sGj2WfFJlBl#`JA@6XUD>P)~5 zEk1xN2>A;j(Z7Ho=4qJe$Z7U0u@jfu z8$5gAG3z&{a>}<1yg0hNR{f5mo+^+8Izx;!M4OnBZnk8DK~50F8Aygvwn^IC)R`%R zLW=@`b?6}4l@b8r&to|^*#hZVbcG@yKMSYQ9Z?>f=VRS^$)!kQvpiFI>cm(EhIqW;BFtS4}N1=qIUrLhTs^Gl~VF*(LVyS$DXs!+6vdRSrFZZBN^ zFjAzP5vuIPE9GSe*a0)HM#pYS`TW>^|GgV};%e8`TydF1LnYEEmb_D;Lazus^YkM& zZ{~XzlW|%1mlc zkk1B7i8egW61a9H3q*>n=BRe^5(TU}vyu}0^JaFPRqP|Yg(m@yDU7qn!Q(}-k37WhW zI=xN-_jWYdoK49QFW}tz?E2gUc{2HBy@t-?Psj5S)dmd-XejATE^?gaPR+eQUT{Zs zbaf1d-x8os?oI?-Rmom#+Rah`?IYN|sj;~S3E7La>x~?Y8_r4m2Ass0u@Rn;PQY+( zirj?0LU-2h4cG#baU(h+(L)J3Vw27xJi+sHhMRM&n?0>XyihOvv*E2GLU>fiy0|Jz zkWlC%*gYYNMH&>ZjOx8;4FO0A>7tcViGkCc#xoHmMIWFetF;VYR3_EzS(9%y+ao1K4XRnY@klHP%-L$Q5dt|d{LPhU z;vkX4;53IZFkw8Ci<~?GSu>#ngniIexBG(dPbf$*t6u@<1AIzBsukAV%-!rhH!mDW zcLAqmuicX_`BeLN+2$9)iy&-Weif~6rpJ|sq79R)Fmt5mt?Z)PLQb^Y$-FK{=ZmOY zWn9Tu)^$Ld?}UoOK!+mBbx4M?dNg38*F~SFL^!Bey%B|;jb%E36gt^>y&jQ{?YS%} zIl(9q`C@T;;V^c|KTq9>Zsj+p`$U{H8EBpyX#^>q5f*k4bVJ-haEk2CFnLt@mV}9x z$0ez|w7+^DL2$NAc3B~hfHTqJAb!4s{IHC(>j@aN?)?t>vR$Gq^u@QAF%eDD@+J#C z6H3(v+T<}!B5Mw?<0l+k71<;h5Iu~Lm7+36CY&Wx^S-AE({>Ci;PgPS5Qz4dmGl${ zla1bwOA;xQ$%b=kizM(}(|w^39|sJ)9SYh<+_^ndm&~16*NK(+?-&`pXDK$yv85Rc zCYyVM5L;_Q>LxCm)9bwxau58&!d|a^VV^1#7^)UHspbIyItmOa-l21g`s5 z)65MVDN+?+p7A;OD45uk4bOe8YrA}C2!2#5B76k4W_Dm@C({yW#Z+K=Dz5f$f2EAw zdCqVj<(kEP-EIA=WfNp1_sJ;p`ET_mYm8gOfgM|~=arlzyL;t}@^!&z%b_GzgC0CJ zlUpFF$_hO8ZL1+x<4VF32gyF9Xs0+21(Q)YB2X6qD13e^`q_pBGy}7XS~BpO0a(hh z+K3?w`G;3d7vY%VTt_pag&{Yn*0CaX-5(u@rdO{SJ&!j_FP(-sNYDH^8pBbdKT+4} zkkR?!x)~zPHpOObZ2MK<-rKP>;Z+STjmwyJ=h+D;Zn#aU3>~gXKv(GEW@UGCJx8K2 zIm`wS42EW3uDjZJPUL_M^R|~YZlf=7DGY03F{^=s zd#Di;)7|p(a}WHwxpxx9xgr2Vw-(vrkkSXR4{Y0c-h@s*{Cj?S;sn)MA1$!X?bE)- zyOTn@)c(@Ux_TI*8kh)w?na*|ch~FLev&heq6i!P9g+HKZ5F+^Fd8{tRzYdWE*H*^ zke&pkEl-lDdIT0hK<-Ajn{D0z0ebrTMwWKZ=t4J0Sy_2ni(k1qh_hv4dWZ{JW&jm% zT#@Wsqel!gf0@M$Utxwdf?(sLr3_tvNnCfsPYQvDqle_E|vmEuT|}){Zpz`)o)xgUc(@4tNx%{ zG%uZ_4T?BZwAt_9g)#JQ$!Q}gMWkNw!A0H;D_W7cdZ^j6@cMW%}X@+r~BZ@zZ z(+SKl3DvQcdc%^5nmYse$?8J4X-N{P5W+ITYqrT^&nBOt8Ht&R_~#B*lTIFRu10vV zN;MP##;v1MfmUe}=a)#TBwMCiI4*wiX;UeT$+tWs6v|vQ;K|OHOb14q-|D96aanHb zZ&AFFc`4-;C0f|hZN0FoVn17kG8`-OH=yr0o)2}^`}`~1q@A1W)rd0F{h(c`-JRle zoIS*Cvn9v3wE_5HH`uLOu%i*+;t>5EMCeuE9i88?@i!zHqFz%`s{x`G_>i3F1sPg3 zX>g2wFI%Elc%~y?RNYS?<q?d$(V_J1evMF;QRMP@k zI4rDU<n;F^R%G+Ikx~WwM%v~wW@~K;nW1PHZ&L}Nmzl6nN4k=J z4=Lp+?`BbuNR8DQF2*^PSU?$9y9tpObM4#?~mOP|<38br%il?IL@ z3pMmwe<|4a;-LzWJ3dvw^d^JOA|t+W;mcznWGpRzWUN`6=_CeUS627uX4}Rc{I}67 zh$l$bEm%azh$0BA6<<``nFvLa5fYRn$m>jfl(Hb34YBUDKx!ejt{&KM6%eUlk1`0+ z!@!7IJFBO2trI1=a+Mx!t+z$}a>M&*?=CHrrX;y?j)Og+$?YMa+1KKoIb_6s$ClD| zdtqU-g>z$V?`f1y^XxM6eoU$m!_*;8B93CQ1&$XDB57%{^Vgc(q0?hn{y`AC5?h)K zMKI2)AlNVH7O-E7we4UZ9>q#?mbg|@debXQM3-oY56-IRN<9%sTv0MMSAV!yDDxke zHfgPq&S1T-u*!=1|##F-XYaLjG-9dB3N1}h$2-QLFL zcg=xXM>(L07YP&O#c}|f5e;3rr;|Se!lVUM3UBa&)=FjA(Rtd9LBHahh1XR>#phUT zRuQl(jRW)XM=~gM5OpjKaY1-wBn0vB=orfV4!dcf0S#G;?HP+&XnUN zvAvMzKxtV1?YthUhh_P-V zyobTyq_xP?BQF<67~Ux^&t^&Hj}-94&$EOJN;k|%`OI#FNIEaJY;)6^pOob5KI*U>cb1t|}HN^X8iR+Ok*ppK+g=TbWdNL^zVl#^yyagG?El#IE462tMV zgQNI!GRJvB3RN?h1TgbX5^xtb_0z16*;od*se1mFFyz~T##V&9hd;HCkDzi`fj$qI zWphF20vcg-iSupS7d#!(#A<+>J|>WP@+SPU%P0h*477mwQgNXkjK9(23B$`Xhurl5 za^eRAynnKkMt9<)H_IY8Ll)r(zzbihd4<03ZC`mu_v6M0yoTn}ppNtXI@*qY67;|i zGAFdPJJLIsf|2UIKa@_mYJlM3SBmX=zz%d5^gs@}DK)(mes2B;H?XFQAgu&1%!{vK zxx)4a8w+Ux`&Iqs6ZaGvvE2WP8z*G+JipxvXp8pZgw&>0J=28~KIdE4b9xe_csFcy ziy6iV(RsJLDM#4xCCkj0D{{XsS=o|*Z%Ney_#JcjT?cw6Y;VgyTUCl(by7tGfQ%%* zr)qeHu&)Tt;Du&&24jVT?ps zGS1uH2G_n6$W1i|ahTH#TcePpz}N+V+*xWmWA5Ahk5EAC@mt$Yrq}!92H<;XMR-N{ z8{hr?<_7!y@=>{m9R78h9ESHuK<-c&cUBEF3ks~XA0M<%PU5)_AF>=bPAW$r>IwCg z@gdyt#@QIu=8!v%=Ye2UkV5%ebJueXh&WtA=<1{q_YV%{)AUsr=+N1M(%Aw)>4wW; z1Sev_JSAlPl)r4FSzBD?Fke1SluI^_A@m`3xFPIkJALnQ-+|8F{x{}u*O`K6@9;b9 z;v`);ne$Q?_x~gS=VE2b7)}oh(WP}k5H1<*j&kcN@6```JDlXNRSz^0BpjFnW!piP zGjScM`6^v_$7=Ez{_1vo^0mLX#J|zyB=~6);oQhTI5_?zwx(7b!wwgu@Ecs5sFd_1 z>jp5;n7Z7c*(>*7zWK93PO4SX(kdveQorn|hpV}&O%njvm1@{&DJ zpssIfQ;Ngiq!ih1RR%M8;P*}KHwL}Tj0l!}JkB>x%7Us^2lp>e38nKTe?ZFsx4o@ZF3eAd~ zoAh7bD^vWGqpcL#ttwaVWA;t@)q1V!{3~*dH^XqP9DcXGMaBP5d9R_>qZ1^NVFc{| zDmWzpD;t3}0V@Lofi3|X=TDQ3k&%G)$EZud!SJ7+|7~)zLnk(RQ{i#^SbpUB2-tt> zf9CwZs!PDg$O4_IW{2zrsQhhd+;^LECFO7U^gxf&HKNphW1az0W*CKL!E6Nv$c$S(vk%>JT0A`YUya@8zU8*>Os zq+SqmEk}WkWxFQab+pB>JTY5rTW!+kD{m^9ny>HguE%XH`>yBbF3z3wr}U?7^V~ml zPy#@`QN4n5Wg)}y20<=<_ss_7u7JfC$7*;NoYO%cqfL01(rd_HU%&EN z-Q{LBMQIl{;+Nkr{CK!ptUcjd1%eN@q8^6ZVAN+UzmA5|fr24{box8jA^~bsn(qgM zS+_rg)`iaNnl*kOPiw3g{ose<`@aJElX?NKsA)4fEfzym{&DVc%Q3HaFsYO0722w= zxKHlb$Ij@k)#vfN`G3&QZU%XJTCEkI4=*~pAuWb!HWnI8UZLkYr(*BV;?KQPT%T+F!RW#th3!*qL z1~rdRtUaH{ZhaH}XmxvVZi?MY^6dGN0e1y(cRw6$?0f{W!ZZj+9k_cOZ819tObS!? z2`Pfu7fYU$Ef8l6 zO42{*1j1y70pJcwBYQum-yq72F7V30R~W-%dJy=_;EXDc1np}wqo&M5f>qXu<8gfQ z1<72HDH5?!rclDaR!#LS;d*1G5R>u{LgVVHRQ^n6Jqp7W%-*1*r3gwqg8)?zCZWe+juL+k5H0kePbJ)1c@?*i8t$2$18& zKM*^2#U?@U<{oXKs98^fLMpDg-Gwmb_d{)36%_EHefPB&x++hzQdRd za2s4>0I$O?A(|Haz&Y+`8T6bD=g8d$Nu$FHhB1WL7djQv5piUkMBxu4Wq_%JG$W=9 zwi`5M5XbR5VnBBUA4+C`zavVE^j|xXNoI8*YVY8~CDbDcb>e zd0X_(Zq?5N-bSPoy+%CaonjlFoxm;x8?lAp3mkywmx%~~=@IlnuoLw8bqDH!VJGSX zW+&`}VkhqN`$v!t=UQMB+MUQQu;Z@}_YkfFd^y)R;XlwE>3PClNPdV?L9Ot3f}8SplpZQSLU$n}2)X_vL;M2`VK@EGe_W zC_w%TA&$rgb!XTMVkb>Ea4037zd8bc2tYV2x3_X1e^)sI^wxWD=0{Vn5QRUKKdg3u z(F633qci*k{6q1B*9QuKq#szjOFJm&fqiEkvD3>Pw$tAc1}J>N?F@YYgg!I=0pSz) z$k~U#CEVxVt=+G@CH4t=kslPkMaCI^m~jAG=;<8^Bi}6^9waIjc{U&{80@zYf%%P(g8#%Q&` z|6_3eFT^Tk`X;SE6#fh>X8WXQI?Gke{KEV%fG(E6D`fHs*uDY%*pxEAC;|V6KcJBL z1@&LRSI+!mY&!ct5dT?D$o38KXD14ozG>?Z13wFtuzfN#orRXmeuu3;#Foi^pMDp} z_BgMW^nBZAe*Kj1YFYU2jgtSAmbx?dPy4k8_ix*UZ~W#npm}%34zXg86hQKX3C%>s zp%dqbVF>u_R|O8T6KBfBq7{w0LB@d!bg&TF0g4k`Z{8Bf2xXn$!J>bExJ%2o`ZorTqRx}3p@IQSL#-b^Y3<@R-C{IJD{+Wlfb^XGnaiFXJ%$c z&S9BS#5_K%;KoLp7VHzHq~zoymyCu@GrwaGoXKOql}KkwSTK zN|vE{_=OH4i7QK`|IFG#hQkMT{x?!pPo1@!(MQnJFhuIgMtp86!DE9DGxBbwG)8|C z*!dX)l;?pOG>-zdF&8mt*y22>jhU9(XnEc`@w}vlKCi~LPZ#)kN+v_ zD}(B2x^9C9cXxL<$T>g)1b00UfT5i=PSG`Wsc*>J$h6NRV&5 z2xKjK`|G7^O#`*59*8>-7#SG{%wM~oRgT1X(+0=lZ$*?N{d)P-W*}ZHUoNs~Zt5xq z1`PvF{jbgzOR05#@;`Xc7TL%l0kFA3oUxZx+{sLupM@AuTxpUB!DGHEzyxXVShEVS zAa&yM_w#vv>SBx%yux3CFg3@WRyRNOzp=;Xs@{$z74F?}F6~vW{Q%q4mGPmx;}OLL z)B>#iXP5atlmY^LS%{@Fc0oE_0_WK>?7;vz-`wt!bRgE(=mZQOqi)_D#z0bwiU&xw zGWd_0g?J9vSWrRo&)|u%ZyB*)Og)x$@#sh4lJ@Hv+uDvDk0r8~j?PGrb|mk|56@&m zPfVWOaw=TOQ}wai1j4x#q`4W-Hx6aOTH9gpIU;+NwFnDW_8`;?@{6=lCz=Jd7d^lk&?2qvV=vdOmUp8alR&gq3Z(-jBS>gA1Wi}9cq8pQ>S@v&d^PvzvxHlD@;eG zpLuL+74}g&tOt(!9C=UdehSHyqPl#3R1hwe|F|L8LqvK}`S0fGqqsGZa2EkKYh;3okupepjt zySnR!r6pNg>bzg~B0wu%Y|>-i!P0nC(}Z&kV_IHqtBGo+2r7`G2121%AM#buG21HsVNv& z-4+e)RtLflpt|Pv%)1rQM)c(OgJ_0)4mn8!5hJ#rKj6pZ8meK9{xMx*j*@J{xdUr_ z?`4>iPa>LS@fo9+Z}$?CXr`+%F=td3 zw?4LZXJW9tVbZ!ltSovD(Z#48AT}X*1B#)dBK2Q7#E98hLRR-SjyEp8>>e1sflI^V zBuTE)+}zXZUaS|l&Z9b3Cj`cHDcd%``RU&h@K5gwl<wJZp6Xvw5c9`XPis zSTd@Q-p@-lY4$2e8(NOIEj|n6y$tsAXrX70*+W*0OQ@}9F9?v(z*3pNZ*o69yP;p( ziup;LOliKn+-5i+PUYJgqgnR|Y)SA*eZ<@W^y}%zx*f-Ln8Mbmy4>^<>B_yT?wG0NNCQT9`MsT52(WOGtd=Y>iQ=Mua z1%6;F?Nr&1#viuaJvA|*zahDS@3A&I-TVOj!MemA8Tio*6&8bq3@yib@FOH3O8C_DgwIFsiJv)pu%ML|e?Bw;U) zFZ(?`@5o0SOy4)^y3j20z;!^z*L?0grIPP##$Fh3q=dp%VcD{F63#s0&fg@RZ?c;s z3d>oGEklp5vzwJvzgZd?9G)K&BnIr_hq5b?C~_#;T{qjhuhq1kgLE5G7Fai(by{sc zRakC1*L?!Z4+|quGvFzY#zjHWR;|rAejM{TU}*{soQu$D*;OU93(dwFO}{_`2^e2i z72PdJ!$?+T{OUMSRseQA9LQt}O!P`DOEA}5kAp^GnTN=`9x+CeOv`k48kUri!a^3} zWbiy!hJP@ua-KFsIKs&ZA}&sDSbI$#WR%N7S78LrF-Wok&>WQYvOYjc@b7WiZ)WqO zk?rM*+{lbnswGtM7|RK5_D>u)F5FYxy2hk57D?65_(INJy)!apVq~Tdo=jb0mgW!m z@T9Oty@Q-xAKc+XIq1#!Gx5;f1R2*ipHwJ@qS`WeU@AOyAL#jLysDG11TnMHty~Q$ zr#JMxd=DZzUo*5+ydNMYD%mAb$t$om2Qu2zG8;b%5L%MYwDy}Q=Drq0j_Xe=r&#tq z>2sF{;AP-PeFAw(;7;6yp;+C_;&+@E-Ro?IcvgB<6i|7-zb83{xQ}Kx+B|G%rZGD! z`-ph9H*!p#DXp+b)4wQLa2V#95}5Xx_Zd=f{j#-{iwD0MF}6ZNo${p3RgdE}Y1_P+ z`Dk-sjXl8pQ~`PK!r+O-c~Z^A*qdg{gHnUJK7aiwW#IdTx<^4=V`a`E)pm|xN(H&N z;8g75E0q;jOb0^*a|MzlxG}D$Ui7kd8s(fDs++m!fFi<_mE%*XH2rc>o~$wfX|bX{ zAlnhqj{!o=@+L?ZZ6E2v>f{vE{4a}BVE%D!Xf*0yXcH8}$5BJ{%$G$j>;|5Odir`3 z7aA(C$mpq=LYT!~LEVMnJRlJSIyGdA&5pSDHlnG?x6f8pkN`QQ(TN5&qYNn(vCBr6 z!8M2lEw?4a=tNQ7=v&g<$VQ9Z6Sy%X2lLZK|4xy1YJt?d6A`<(^T!39_*%!A5HG}L zU#X_P-E*p}#hMs%x2RH-msqxPo% zR0`ZD92@eTx$;SEcS-*^mH4oveyKmGpD42d)fdS@65BmVaD`8}y!+dIo8Ahz93^MWXoxmgfzAKJ_;*}>7zBbG3v{z9BYyPdEYkgM z8-;2oP6G7LL!Ye#B4*Lv8C@h84LcmC0|`$`vNFk)jpVCQzYsiWXFGv*#H)xGG0>cT zto!%As0*`jFF!NXrSaSqTJiZG>pQvAj_&L}TLms`0=Frx2dCpEDvih z@Vc({v2L*JYb9BEk1Kh-Tl6K3m<(hP>Dp9cf*>hybZ->2&+G#8jFst-`Ri`ma2t8> zW+&?dc{^0nPq2K~NpcZ9t@TT)4hL2z7&tIxhBSRk3vNklSgJ+Okz>{wi zUV3lj;>yy%Rp<+nWaGyMp?YN(F+t%epwr9bngLh&c$l}OvTbp6@LJ)o5oP*wal!kp z*h`Y}GxAHW>%Op(F}ipoDh+Wq>Tgl0gLv11GrcVO2>=0>?_3)1!p{{u{Hvhz$W?@D zStH_LXi-uFC;`g3LezJ8OBaY&K@SPZ9+npm6}e0I3`1F>vSnlr2!j4)#|&jyLzJag z)-M-xLz>~v#R2Mw>SmDWnqo;pkV0=Kd~w`}gNLJm1Dz9MAZVIc-BA5d`Fg5#ia|fj zUH!3rtQt-G0<`3MEq#p|`l(N@FK2iob5!7IcqlTcRy5)FrqapeoG3&wbdrE0xtpf`HQX3A-=WT>*?ozcF4nWI^f`@dw5OCGza(1Jw;`=v zoog~azYFwT@?FXP-Xut}NPd{Pz68fkQphMve(F)z$bR~$z$o=-?5NMac-(bnvTO20 z5+w+Rc4`n*m3|607(M6a?D4}p#yizJ*gNty=@rwx=XY3ZTI7wRy$zcyuB#ZHh~QWP zYm{~WgL{)i`v+Tv*XxDd0UQT$si|Co3OBK-0u1|t(*P~I#0IPn9cN~FY%c%lVbla>qZt5}TO7cJ3o>A}}nCt-py+NOgwOWxMx<5_Jp>2|- zjIbEDZ~A->FZ>N5uer!sC+TY=r)?O~^Ea0xsI!K(f@fae@nguF3ou2}M_TRE4UW0@ zgK=mI zsGIjw2R~`Qn(oT(bE!i?=&2lI+Cq{gbt&1gnX&1dtuRqI$YYavQ>`8~O+-y=P#&U5 z?v5F1;?w;T0~(AbYRDZ}Pjxp3YjVS$0{4>Pr@Iq=`EMrv9Xt+N$2|G%i`l2eJ2qq+ z^;vey`-%dRI^k?@=(B!vLmU>q+RYYI_iF>SELx{ewTEQMzVD;@*lbRq8_Yy@dX#MY z_q0J*^n~7M70~&9Cd7nJi0+2cQ_o zoe&ojb`mk`4ixK_USsOeFBRA2kZTlfi|EKdoc{(X(DJ^Bb91CC6^HRYVZF(Z*g z?CjfD*8&^l*Th7sD}tBLE->R<66^S%RHXl!7;ef<&0MGSKaDQONVVo~tq#n%uj0YHMF zRx(EZQN8w)WhDUXQme^8R#)1DV#o$-6k~LJ7+UgC46xs*=10Rqp_IE3j1sRuCAYD z0+31(*#<2a#YlF@{!@aBR#S_DU`%^~XeI4Qo-US1C>B|u z>#&h0u_Fou=e+%-#$5@%!P8>`p(O5znOQ09xh^p&H0hJF4k!kEP7LayhYzQR5X3>? zImmvbFzx8Sx0Fzn>KXDA)GvJ44~L;CqZ(6h2&xt!a63yebtv%@vXG&tFaZL&~NOEa?NAI^5`7mqBMJyEY6DJgs2l$6i$pLUZ6SOKiSG z5xCfhO{tiL|F#)6uUYc|GVFxgtez;lGXi2X5)KcFBxWH)HnfMo0>mv^&QN>aorvsM z-a@<7YAD`D2tC5R6s;{@Px#)1eQ~Nsq55VCFOM6TzJ+5b-zZOWpWnEGnC`SutklVi z8&7k{?HumFt0q+?qs6-S0#GF>;7TXPU=u)#RDfuuy%Mjp@Em*XAbSW+@ zWQIdkCD9ey6(MX32?NQfN#L*0Ccfc{PI~Z_SNT4|o(=?mQ|(GhN4<`br>?%~Y8;9$ z;}jYjf;uV~cH`8h<*2xHPQ_oQ8Ily2HqiWt#Bc-?l$CPu#hwz-e(@L+$SU_5o9JmUDW??6F^ZU5>`1fX zjylb-{*sJ@BzJf$*A{ClhHUIL5a2R7x;+zA`1E2c9%`KoUo*)L@Go+^XbRWUAF3Xa z0HdFiNY9^ZOmEow+9+W{jMz!O}iMuJE9bhT&@UIC~CIl8G?}v{3b}q`ReL+joGA{+n1OntOZk_ zX>pi_&8I{n`G?k-dFw)>=Cj5)+*a3}8dgrG8y!V;8>Z$h1m$j2yqpQ4Dy zJZ6k)m^mPqCux1^-Hul#I*>4DDfC)WZrI#X;j399Vs+TqHCc_&ZWv21X@6XH93PL% zf(_S#ERWa-2U9qk%uTsH8l9RE^77f?Otf;*XXnBWZ_Qi=3-3JmZ|My_IN8ePW45ds zVozdD{oZ^?VYbrPM`COmQ2LIIg4Cw0NkXTwgiAUCAOY7K;8QJW*T0;_x^Jz%xgY zng?;hF?iPV6ql~cV>y;^|7x={q~a5VmAi5w7Z$!>LE7lhlO<0xE4p7Z35x_G%

llmkDAVNmF4L(2U zHf3JUj=w=i>{hntrg9vqGi97dtf&POci^JG`-~+>1*dn+g6DBu+A`p+TOKIgyOE@( z+J=?Ohl_cC{BAAl%&#}tafSv-x?Oi}3eAaRJkUW-p9I1iVd+@?W&y;2Jzc4XbQ&!T z6NkAghcYgJgR&{Qo%J*C80$&1<3mfd!H^F9f!Y(P`Kg*VCo7cm^S_6vxla0~&fI^X zY#_~}U0;mg;wwp7V3~S0xdLatpKuZ~SG4U^Lfd!sirMXGbmetxC3udI@`47*1MjQf z#VCb-A{J-AGsowc2G-QYp=ryin%3T&XzGs?E2C1gR1&n{4VyU6J)bJb8pqYKLEJd= z>Pk|=;nyZo?OO#2N0J?sZ1CJ5Kg-gLc(~G0o(%geXDf5q1Z&5;LmQ}`g(wPbn(`e6 zzTZ3Hdz0iK#n*TIOUd~QEu0W&;jUl@>|54f0sckXuCp zq8kv(5g=r>BwV*H+M12~_`kRV(QyApXXOZbcc+EmKixJ5lhwvEduOok8#n%3-a&@kMMbn9qLE$XIIn9M$vmS z>qAX1Cg@B;nYPj6<+fMdT>G5OqXx z{RX#k#4OrTE7HPJ!53{U-c2}~{lFh(V~#JUjjU5T6&#TR%;n5n;arQtXY6$CW$>F8?W%@4uRx z!#RKJ;%udNtEEcVZtiAuc_aJg77VF|Z<0#8fKa+(yfIOHnCV8Cbj*BBWl^0KG>gx8JGiPIbl=u9Q<{~{E<>CEej*~ zYg!~h#bD!IC#MK37|$#3c73V-jTr67bF}Qo(Xpp4RW35s)2uH@p?>|3-RIZ-nwL?i zeP*r$WU1e9Tmf{tMf36D7Bq`)NwZ(h|88;O2l@!?2W(m!c}l@&KGAC@G4#)8ryttp zKePTL2%kkvZBBpSXVkb@_+a0?KLC$w$7gSWXV+K6&z_ zs^s?i_m$S5EvIniZW3XRuX6FMM%8oRy?9dcOSnz-SHWbMgYEDTM*ktKdi?d(vPu?z z=bw#YuP=Rxl&Jg?A-4TbkUwJ%yskYVa;GR&;-R7j%}iOLjcd6(^Ly^8RHycxzm|f& zh(GC&-4pMvjExlye$g2j`w^Enrgs+IfE&jHw+z@QZyKZ4xJ?JqqZCT>CqD0bPvw!c z(T}tMLq@g9AC$H>^E@sA&C105pki1N9o$8hp)*^R_yN@5XzB7QdD<(gnkwN zHn3wqBn+#8kv_jMq(73p_?c6Z>ZZ>u6T->aAeP#5!*@qJBgOj4Fsm{p0)2-(#-O{R zc|vm0KtiQfB-?9wUHH1012XVKYbf2?R08OH|r`l5s*YrL0?&>G>&gZ@ggIDK(ZH~>S8mqZ535M_1xmU$V zVcKt5m;B?IBYJ`Xp2GeD^9cvM!BuWj07xH^aH z)1p|-I~;1x<#YQtg{fe_dCTAAjc)MrjDgmiukLR~Xa6bMKaD3>qU}F-?F^S}o zuEfVXu9YdXjz^;79_2qhbRGwsYd=FL1TJ?U?&x)=t!&vT5Y9vpWLF=|7h?WmryIU; z`ys@j^vO(@OL0D`?X$XS%#Tye&vZhTqFzR3JPc64B3yK3BNo#w42ljwDH?*FE!155&EH|Bl3!)x$QGcixaU?lhqH z9<#Kxn{?5LqI!DA^O&%=8R;nsSnhh-Suo`1RxRHf z<(u>}?95O$s-D~|EAY74@K)({q5r`IY^Sn*mFC;MLODi1>aVV27ud9#TfC$Vb*pCu zqS5M@WI-t6Mf4ah?9GY2*jur`@--TYxjoyMk{sN0p)$Ks9^z@`O zOQ+e*4)09s5oFZ}=Hn0nDARIcovrU9Js$O>a^#0rRe|*$>L@CT_L%r&5ko7z9rhGI znc3RRLoW{<`j2vk+h|`Zn$HS2GByNotR_#rC+79@Zt@48)xv9^@`el+r&7XvZPCgW zjnF_kI_<}+I5+0!2^kY9FB9Dj-;_Qu4%VTzoqrRypXHrRN1=S%Btq_e`XWSmw%IPG zlE#0q|7xU7KGg9k!eRk#1?uM&6G-~GdWO4kE9>;*Tq+{OoIY}!J;6`=>KCh_?tZb) z-*~j$xbt90=`lGnAZi<&Zmbzc|rXrWxe?PP_HimD6RR@L&7PCw5N zp-XQWi^VN$;XmU(uS|?~iflQ7*#L|3E5r>y=fd$z^jmK4`Rpsi4gxjPvZpr;uPhwA zo^;PAG0q`1>)dTUFM87OS2oQ{RdN>di=bFCh0i@}M4k`3Bt4~5$a5#PU|Vxx$OazY zCAII}apZ3rzkpDModdn&DavIDtdcMFglH7!T@my`dgt5*DS zso1T=70|wZws3Z`uZ76NfC7_usSFF}gx+WLF=iepv)H4bvrpg2K$DeI|L%#@>gn7r z>=$9}sNDT<+jiZ0!jiUWk2aAsSw6HYAF9%Rn~)!QniG0Y%Mbj0*jC|ga_WonRw>bp z=KYJ16jtt|Y_3iJ>-}y|Ra>v_Q|CV=6Ap+-VYI&}-4;BgoS?CRfv1BoQmk`#eLv`y zvLvLGdcNNApn&tI0I&4*Qxe79%Eewxx%l2Tc%9bV;h|>7V~+`MtJ;B?C(CcwyDRVx z##4Xw-M<@J5!ba5ejLk|mf%2&Ew;Sd%Y4#*cABh7AT7P z(%-mUJ+Y$FUL{>JuO!NR?WnlWObUO|bjnE) zn*^WPkhGhY+lQ%>*CMN)y-X>Z?cGcu@Ar|m*mErUuEi6Bv%(v#K7Y!on|)(7nCpc{ zl+V$od^7@--q6-mvEq=T(qo3hn?s)du$N(Tufo2EUgeHbk#6cBrI5w%-j;*_9_mFt zMX!P%iJdYU&u1%+PDcc#ipMjqGdiV24`i=cDXg@&Ikxt(ykO{)CoHS46pZ`VK`l0y z!@)o{m5}avR8X98m67IoyTjJyXn6N0%gn?qUbYlPz}q6A{-bW3V@u)I>|T=1-}3%L z*emg#^*PR<%4AjRN5gRl$WOJiVF}yO5b=xYP&$`wdZLwT81lSVdCgh+;4dow5GgR+ z_;>k4w~2RuIAzGg<(kH=@ZDf)_v@zx7&;6Tv(Yq)+%J|?DZRhC-<5}TG zS%sKRJJVpPjSk^R{EcYWm;nH>X)CI~HIna1JQLNcJlO%nBs!M**V;h2i86g1wl(_` z?$MKV*X{PCRu03o1HvxhLGitMSru!TgWFivC%eR5yj=a{HvR7!RoOk-{$ig}x&_V) zY|l7L?z_*U{~X;iFtz$hC*?~t^K?z3fd0gd>b-%v-_uG1HgoaH%I;@L7f?pgXD=Z2 zJmy_t!D|j?0*wUA4_emm9zvFE`O5ND9`ou=EM-5O)>7lj%e%MbbkN6J_CM~N!p5an zQ_>>gF=zyfxO#)%X_4-G)SbM|bSke*to}`>35!x+qVXy3fCI5Q399L*3ziMDc$|=# z8+&iy{w(Wgi&7oOVD0nn>0J*#jW)eXRH^>9mL>PpkY+JVRTo+W(i#0)C0lKeR6Joa zO53_0M7%*{ks4TZ!jHCIP3U4n6>o7{!4N+2mq7J?nUc?U9(CHFCjJWD<|J#yrH26p z19?r{`e<(W+FT>Y`LazQYm6q#MwIva=4fc^Jsfc74CRw^v3=M3DO@l-Wc!Jant!ye ze6)?pJqTnw|DdIBtsaexk34dL_~hN8Jr6P}1F9Rx0|V0RI&Wwi@^V4(q+0WMmf+_p zc~aJ?sFvM9y_`5JE4X2g9MdbJL4-a1QGe>>r@h~%XFI*_8GSlo2y*rvjTi+kqm8qz zt4>slO?rP==yoTl6X)9;pdhc7X5N#8-|^O+p{r|AcWtT67x75CqqIYFtNP~W%FEkv zOf7+jY-yUATO(%T;+M_YzZ4pKvE!jRJ+1yPGMS-)DUQ);>5e~yp9}9-J~;#a%5OH+Q)9Dst4m=DkOF% z7JDfkTr zGu9x#TUtAi>obVx79Z_L&0|VP=v8s z-OuJiPxo4W)!k>X`@7ymXR&BQKh8_xR#quWsDRs zibWGSFYfg8`J7YsKe2 z0>a2pNLpHYE|LhPryEJl(lwzsbh>~L2D+II&BEpc!HCCU@^5JB6_ zT2a)-pHr0<0Ba;bn*m1J?6?WZz$@~HF4`AD`Gy|jA0#Hvg6pb6TRXh>0HN`dpmMV? zl)PMY@?UA*$I{SbD@pWVSX@y7}uy(2bJE|w}O_!gGGlZ%SL$>f2O~$E-(XI91oJf zzb_l_Djp%(-!1{|rLF<8UQNU36?i7~n`}Jqs03cM796b>OZ0r~F*^~DIh143VOdeJ zs(A9PW`k0>Ais`axW;xkKJYQLQX*3j*4#vF4j;ZqN4frfIJu+J_x0+?m2Y|>ELEk= z;hPV)S*-b;*BfcsIu`>jazsiEM<<&H62MmyqeSI(1eu17sJ*|!cPL$_}*i>Sy(-aYD*pj-2mD$uCuHuNUqsg=DR>63AOY<9&`@?I#0ZW_A& zcrZ4}(kkMTTzO7?9*vw;Wbu}08OI|&rVLLdY_}bbCy<*dpRAk{H2csmfpFgZAb@-87Xbh{(is5$k}gzMBNX7>rgz=Y57`_Ft9d; zex%63+Rf85j?K@|RbbV`&6`5XErHqHSIEx-s-WYDu!(svH`Z)P&E+?>;p1x3Ss`F5 zloa#LOXKB*4^?P9Z2BxJz$PR#Nc-}$>US-Dd$#r^&ZG5o=;^M6GjhoN;V_48g20y{ z^!^{(%#z05NKV<`%gU5jvFsF{<$tX}RHP@lOzR855!Z}aBHQ;=VP@xD>? zY*20wMn-$)S_o=a!+kAPKew7`c$;5Ra%1Q600jo6H%*yF!egUvkng#_X1A{&xWQ$3 za{m?GMLy3djn;rGYzP$Va=Q@v|`iIF3LJWsOZ~cg^$FZBtj7xq| zNgdqwP3W7_=(%l2(%T4j2$oguk8S1EBHtKgZPZ8Q1)^@3`hj5QXyUZ8bYm=vDNkVQ z`h;Vw1d?x7li5qATLLTzevtHc2jSb+ShLAbMiHvzrT}`{ctUxoun7?%1nIpH8zp&% zC}6DMKKj`~SC=X}@|w)D@?KHZY&t_i}#3i8KA zgik6_ONk{u7Rxxm^Ti%L$u^fM1aD@=#ZQ8($Z^^M*WkH#IEa z{$`TN)}hJF)6F;0W*Vv21Myb#^KOXAS4-YsGS+?&Y}P=j#wCmLWDgaBG;ALQtC8=S zj~;hDR1_&VnEInci2jr8{uPXIdAEm-NNNq;#S80mo8wHBYw9MRN7ErvXubLy=eAkL zEc!v(tR{U56+b2k%k^83zxO!nOJWpLWw;f;WVe98hfG$$qgUvQt?oc%Ki;sy++bjT zofa8to|^*+Z-u6-7GhJwaZ_i$J5PQ|d(7Bo7xiz%x0b2bjy#+bu3h?Mfe&;)k6vs8mXy8IPuX1Oxqa7pRbHm6KL3t9iJne?p~XvY~$JhxDQ_a zULtXA?C-z%6x;E974sqS7{=#Noss+A;h6#!P*g{-y<`a+jh9E+{3}=c5bRq{iaz<8 zx}k!7>H;e?TZDEOFM^%>`XX;oBp*y+H`{^=wj~-oz2)LQw7ri^?qKo#^*2CxjEner zK8*j?m=yj)oW@$WU76%jn}1wZ6mtU2AO8jyPyuQC)K4vfW9oQ7t5hB{TH2N9WOPKC ze_X=BXPQbRd29+Z##++&@WK55k<#s`!V1Pgu%M_2%@^W?h8IP@FWB&xA1 zK5PGhDf*WvpLB7>QUOo!$Afw>OHkQ3gLG{T>G3q~a5J~rqY&z((UzZ(Orx+rStC$} zMpxc2`8xQo{-ZU3+F|>k>x-DG!Hd*5^F!%q8bryvXYpX*T5F@l0u^5ek-r%POYbLx zWkl)?B0F4gQu`4Y=4QyiM*YoPFlv+rW`0&)1wUFT1`tWAY+9Mlf1b{ zG%2BsV<4lNLWR$f^{XS}7$VZag)peQ0r=YvDME!v<#ICjOE!nT0QXP!_UZJIAI$qd z-#OoIxHSycreM!&v1-{=WZw_ND*+ qe{zNsTGs!*{(nYK)8CCQGTgbn@-R_x}Ne{AZQ` literal 122844 zcmdSBcT`hb+dUdYv7+aQic)QWsDL89iYSN>P*GYaDxDBQ3xQCTqv%0`g3_xrkrIIr zAP^7>5s?lF5CSNj5F#b?guCMVp6@$%jC;rLpZm|v7_zg?CfQkQJ!epN4`0;(xA?NBX)dNE>upgLm2Oecjp*0+Bz(`^OiU zryBs_gFuX~>fH%Uo2BpYx9z2Tm={MUVs0xazpc7?<_le7ZrOV!N;t4uXXKN_-2MYM zqVd=F_RvX2+D6@hIhTe9WYVXTn1iXcE9x>|Jt}Q>fKUDB zfp>2#vHRzQckJ*#{qNI1AlLtWdK~iN-=~)$$HBe+&m#hIdE5V-fG2zr+y8yK4YK>+ zr`sV1|9!d(^8fh_fvtHM6DaX5TSPT0=>r8Vx)KmUwNOFK^% zN2i6X(9sBAj!lV0MP^#y&u0e@9y}?k>gCBi$%pv(_u+G62{5mhuU;KWzHt8vTv+Db zojZ5FTDzu8`S|$w_S`aTF14#oK@2e{CGj+`+SVSLX9c0sEM%$EP|<5pZTFr%3u`P( zOUsr-hcMYuaxC)oa8+S(@hjD#K>hu?o8IQEzoMdyV_N9{uh>>dAI=&twynACMPge_57t!JdN=M+3|;rn zEcGG`@{Y?#hZb(-$VYvqc96;&dI?+rT! zp1A#mY~WA^|6&_hStVUHG&HucNvV0M<=J;Z$iJ83DJQEIhzu(s*uYy8;D}@bw|k^s zpjr65)jco#Ip{96wXt~Uwg@nYxLi23bYg;_+b!BB>=07s{)J*;Be;z_Qr;qeiEmsE zM!*N$+LkaHWbEF48k$Yo<+hM25Ral|BRVcmf#LLQGvBdeht3vfIBWx2S~DTCTTt+I zYpSX-+=k$pEavn1&#qcDa*eYx%p`vfnoM`K$%S9Wtjy7Ur}Gk17(%3`C7Fk}kt%5V z*i1#c&LwIxrnTQ9&Ip8_NTdWWhB~CD1SwU+VK52I)QRqEuA%SVy~}<7{wAf=?p(v> zLbMk6F8NZg2*)!oIvPa3zSUA-7v=G@5r!m5-MV#4ZW;E_r1m~%c#A7zC>e^^UPT(u zb{s*AYxuu!D4j4BNf z9y(+r!|0GhWP3H)ryMVoD{kPf!-b}nl}QHOxql)ahK{{7BzUm&F($KpWuNf7(EBu2 zSi=tXVQbDCaOM}+nV!O*$ZdH;A>cZv?@1T)CwI_~M2QDjRkUr*Rb)C3S31o;sCU zS^O*Jg#MG&kh_}pMKO}P_F#o0J=*m%O7-N4Qz!l8-r);mNp1V^7;{_>EJt{kWpZ+| zw0qUtw{O1$V|-0iK!)CR=yy8IYVDMNP-z@)xNZa6LQ9hKd%Zsy$zyLqBhCCKBQ(^(W>^mmvNn7>5jMqx|=VyLzIr~hH&*^ z?zDrIGdc9;9Hu+dQBGF2ur~}Af8SHfF48S7!d`#Ir{iYq9Rvb4qg-yUyP>}=LgCBj z&-j2p!AeUJ|DD6#;=8)LNhw}n78hDoht*J{@9gUAeZCx?2wzP!ikI%sgfociHiSx- zbB&?SjG8P`htE^-S`&ekuG`|=AG`=A2j~yLLWR%RY*@idI0nH*YYbe2NS#2Zy*$2U zRYDtcJqyvI(6D)%fS^e;p>WnB_;|U8zbgvfD5geF%cJK^H=-y9x+|5!98j zzl2OL*q!C~9v-lWjbCq?Bg71sOrDMY^3%sg@D6c*bPhYR_A<$|O5q`txv&LtQha}f ztLl2~WmqlOZ5sZE_@i&iKeFl<%S{$Q=ZvpNU4-+|rIxH#4QaQ8OxKr|L(nZI#CbV6-(=)NqrOt``*$6)Vv4YI7cZ+s{ zXT`sgu2%IqfIfs!=Wq9UtgYTUGEv59U|3=1<#f(G>IawgY-lirb5a} zRcW8*1*#aXrJ1S*K0al23*XMUkTkqf-MjOfAMTWJBNWMh8m#eyd0LC^DGg=!TDsW& z00mnJ(h>UWi?D;g=;sYK)gjNkz|>1Je2arlpVo$M^t)#E!tz0E+~k$v3f*aq0Z=l2 zSK((uGT%Nv-kS%j@L`Q4YG`nmE@O~ z+O@hjOS$!?IPyd=i{y&6D{Qh;f-@|fI-}rWl9j~GXq?#lve%Wjq zX^bSLB$@_;@0se{8)}~jtBMqOG})Ru*{kIbv-fS5a*+Vo#HGJBfa!s)1zEkvGd1Cj z-miiT%#EfPQ5kW~0HtRaL59D;SZ~+o|R3s#}tNuHB#=` zuhc>k?cC#J)>Sw@6PtKl*tS!r@2>o(Pxy>=QcFN+5j}V zJK5)_71plF((?1dcM|(5!vbb?NIIBJxZOK;nM5}xqbu|V13NMM@G4A+5#T3Qewh&u z;(gH5ke*qo!-G7ep(ikc=gXavAvc?_;^ztr3tck1woulk)j#*FF{_3}(xNhqCBip- z{KRI|KX-zNli^{xelzxJ?{KG|zT=cnDZ@PWE-a!te2eQ#M^TfB`IYYCdF(<7UdElU zeb-XN9!*pCTXxS#N%94GG}Td$nmT7m!mt&|GvAXhX@|A@Y@whKZ+e5a6rE`-Uhqj! zX`-%}DlXV7v@rwL&+DR6;ycri4@b)x>KKP<_P6PRO!@5WY?on6qC8uz2az}V8H5V4 zZDmQ7y&3rRL{#AG=bc@XAksG|umi}x_xCd=1wjcvSqviYvglHqCd-QLOIZ#Cg@=(I z27<%1n=uN|5NdtXVQW26Sy{Q48|31WzvelBcc*|zD`3qq>gu%MC{!hTgVj)#;rmUN z($R5qV=_tt;}+TS-ixlmI1wfGAYXSP$+5UyXQP-sp6EcZ2jGeF;>90}VS$0w?8n0P zZ_zq;@7|@rylTE|a?8jP@QKBr(F!)a0>2mvOE%S6bAsQ$pIxaA;I9>{ZqXDDfR(bm zd>aO=LzqL*_+iGWSgi-*DxOIkRgmC3IqM9ZcH1O)lhO*inPAVp_u03`Z2{F%d#ByT=#ZkvBwyH8-k?E&SoM@b_5|S~+XM}UtD1Ci>8ZGzAtcuNaIrwg! z+0~<$T%K1J7Z*2%j)1p(Z*c9JH5n@o&)6E?;$*Tjw>As9FNTgh+-U>W{y;I~Vd*h7nLAwP(VCWXLZ4{tU`@_eQDaF1n@81d~9+uRdJ>ctF!LLFgMe7CTvtv?cq>U zQ^FQ@ZYR5wW6V%WU(Sl$U#`Pl9nJqvwe^cpR#fb&!?om)AA8B5{b^|^inWW4c7T66 zoRi;z+rXEcCpK(?IwdHdN+AzXR&J~?b8}BKK*xg&UHuUS4GkZSQPn7k;w;nJi4VT) zsT!t^6KCEP&62Yu;6&D7*amYV7)wKR=R+w?U8v2mcr%T3za`Kh-KWH7#CwLV{@BUj zKcIDwk~4}Yo+l!w)xW;ZsN$i`U~y`hqag#-Zm*e3uhY{JI%r$m+)V7TOD;Wys#@Ad z&RcbisB{yrS4G#d;NrwE4hC-Swzf4N8?M&NBw*$M>|L?SbQ4!{@5;qYasZO*X9#M` zFRgZI+z4;?vBzM&D;>%E&5>?)qiQ8g4abdT=ENq`ovyPo=qD&&%Y9;RJarM%ER~q# zXwmX^Qi~SyC^E9Oer&oLg%0gaJ0frECax9uKC~ZX=`cTAyMTV@{6U-g`B8E(XG2O# zin3R}`h)FOTF^xD#DF37b%r_0o)BaUz|3;!@JrQ=$^r&4zsJYMi)7!yF^p|}($-?0 zZ*mcpG^!k>in<@lb}=gqBm0bD>tg2C@zdf?i&2mxvF z0;(vQaIbtQx94&O{N@$pHm{+&{*<`5+cMX4V%HWYQ$NHB3?Lqcn5q74*P48ZoRW~> z^5{>Iuc$n|5c1k|s|kAP$+8bw-1U(#Xt^BI+bf@o^blHI2djN3-@e@dQ*!wZ5;L=@ zR84ShIa1KB{66cU92vzd+>91hP*Dv<93VT_vnjB2o$e@AJ_k`jQx_+vp55{cLg(ez z0;2w+)n1l@ZwQ~uiN+8sfIT?^l?esLiSnqpgG<@8n=9AQLtv2!2thnu+S2;(W$D=g z*gCP8>P0MUbFD?QR2q?JJyy`-M1R$5xvx2~AdhspYkFL8q?X!Iwl*hqCtMm3;xvi% z8fk&0+5YZz&^%ggtuquD#nu%jiczEfB1xs~Agg(8(`VIhc%`>1bxa-2?Y#WBc6)ND zoiJ99ANKK4Ed;=skb4JaRh0QYB)lDww|d(Wu-xyengq#uW<-d1tJ;K#66E&G?}+I5Z{22f$?D#|L;JuC!Pg{16Lcmk?%Fdo zH>v6HE0HA`-Jv)6KVs5mf*LQ%eqZibEK6wa_9d#6!Bf&RHvQz{7^ zTLtAQZCk3%kTRT-^kICrOxisTShlTeMlG67U;ilh{0BMr56ksrN}^5d^rkH@dzelh zfd($2wnb3b1ZQ2)q0jV0Va`|6!ZN1RojyPPQ#xYKtCq9SqF_21PO|VDuilzbjv8eV z=Y}!+J)UXIQ60kBGj+vEfnKx&q{6+7B2{+UJ(voaaWr&Fz1ydsL0hq%#&mJ4g)p{N zdD53W6f^9Rwf2muAmi9SwPU9*PE@C{nLAs3uxx3*%kGa#nV3d%JJ9@&DE#K8E`N}L zxOMbZR=ymKxrcb>S$kje-UmoQY6RxgLjA_vrA&AP{m46 z0)4i@W!{5ak?LQsAUADoJ#qY%>{?9?Bsr^0z6uTr1?{!z!v|=kz@OD~qBGi`Xxk%r z6p2X=>2Qzm)o3i1&~C<$Dn&&}hfTYR;Fqp4qVuI8)fCBR&z?!dX zu>$MLQ%qT-Pj zN0BQ{g*%lK`+Yqijo{D_hN3i3E@E4L78XQ2oSt*Unu|t#wDF))!`6F2LW1jn~k)k z2BoY@Y=z&MVz5w?2YU7mn)^&4<6NcNdUJ@>LC%6SIN_WcN$^%8gd4NH$X zSdmwjuac2epEn3zgXveSOeJ=(&fDPMHnQN_Su~PtzFDRhZc!`dODWA^Mb2!gV!S0W z)u&Zk&fMGk{XS8Z@9yVUU!SwofR3tR(IzI(X>0q$HF`F&F%sde7YWrAoJ|wJR8>Qx zrsPRNvXXmTci+_dpo&CbC=E(}gw{GRdrrJ|P^Af_r}t1cR+DI}kzdow<6QId@+2lV znq@MR)~{2ILdg*dQ%O_0oTYemR!!G@H{$$}7M)mrWYKV&&W2~fN@n2C-4!MN#fx~n z(qaasWHKF{0hr|e!i-+OO)!@B1v;IJVfq_OQ=K`aflP6RCf$`+RsHsRtJ14|MrU!L z&tn9H;p*4l@kmSgx?flPLPwubim?8ntKS4w2k$k#uHuvy_;u>-6d%KX&*1UT6JkL; zDj77}b@mgk89@dq`e6CSs7D0{OCX^(G6ORejk)Ag@uK*Z}}`a>+;;ja?IIy#|L_$ zO9In(E$y7cNeG%s5Tm3jS6R)=lD;hR9-Ygm0a#hb!9wt_>b<^2qp`a7bD(|~tcVF6 zvbhlyRKr<|H`AH+A(lCkDXE7#kCrknM`~ou0=Cl>hYAm1gN2_o-T`?oaEGC(BNn+$ z-Kv93deVZe*Vfe9QM_2>0u-SZ-RTlvl5u!tHL~J_*fxct?9#Dd@(e!*D=s zbhO|49N?&a2GnYUvMeE#uXyPaRPmPi^t#4UhdvWG95g8&+wHL}g1Kpa=Z@2(%?vm3 zT=MNuz5B>ESgT$nF=Jbz$Sd@|M_&Eqc}5N6B5Ad=l#BD=z8v`Hg#x>ZAPJCE-31(20W!+bm9 zZ7BlrLnPZ~u0LA)-PCbv9E<$X_~Mk)Sr63(xth}AQ_w9ys-#RxhWvUMq}238P*jAz zU=F$mZ+v~$|J?37>CQU+$*WXQpH_>uYLs@95#vKmmeY!w!7)Xk?R>~_?7j%2z zw(S*`f5iiqr+uo8kGyBAl;Ol$Frd3lx))EFLYJ`c1@Qn%Q`hRHrTZVr3id*IY`FX3 z5|V@?>ZUYp@qZjG^Kj zkQs2s*g7WI`*mg}awb2)bX%F!nZCF@TcJN~-Q%1?l_HRg`7zMXUpj!CF3TaQD^-$T zQ{IgZh*13^RjIdQz~nE0wDErVd-fvd*QW!}*b`vJ5xf7JFGNDpFWi6R{%4^o8qFMV z_L_h*&M|+|Dk}?Q=4_UOTT)rvWg60yepB^o3$t`~09sZO?ZG@~7r#Tq(SaCB$$E8) zY142*M64$(S)jyI`}pd~k&Yhbj6%!1;aR8<yiL)1hsk)zYF&n2sG7FASQ$n)z z<=k3O)Cjq}fPfDA?3)V=p1!T^mdWwaSo6%p1q0p;)=&a^Y&=ZM($B8HPbUqHGu z?%25!O_grG<&!}_yvA&^hjq-ur^#<^;uaf`7rwb97NMIOhPZ zOycrLxq_13CxW~eaXMdv{#`XUdpkc8Zy`>l;Fpw9#_5ig!WWs3jfAxDG8Z0g0s30* zBoC}W_^$Dv60BT?L$TV2l=4Jxk;|MacieGAMzErH>^-7JT2`7XIVbpL;L%S^w|&c8 zJMHUm!d-q1(B$Mb%dt`ILt+mF1zp^YHJG6vDYutERH@QO0Mqk2-}O<#u16|6nL2}% zILlfN;#*m#7~=};9{hz@)AM5!mD1jB-q&HW9xtqGyU~I}b9)i}xQEvTR|jlv!WAP3 zUVu}Gg1)~YnGts+rO3SzP<@j{6Ylb^UtmCsBxvV$!N@NoOkvop`NOwC| zv@Gyo?;ghLkAWFWvJG`UEA`St{i;lXWPAZ%UbATFZb-^%Uh}SjxVu7Ho}?pgtV1bj z4|1G)hg_8IXG_IDPKkFc@}|u*j8mpYkD5?F)HB{|VwIY&*Je}M#VEY=lk+e_apyA` z=y;%92hKq^({``Y6A68+-miQ@9oo)CaAM<=;>H_nB(kk+J(OPHNrT3h;>l4>%V&X9 z&@r%is+T7~y2&U7P96P9x5N%t?nr~B$W>GN<~>3GII=WA@$JoA>n_*8o+}ZV(Wz12 zC!+M#Ke$MhWJiq@AmO9^1=d;~h3~ zbxf?t$XtQ66i3>*jh;cYL+Fozfx#dCkHoJt7U*{ywpV2*Y-LQYRYjtFboK3!H*?!C zmA+NQc1F#+l+>+`9w{QjY0yeexShL^&bOPgTCMq2suRpTEpd4=S6? z$%)nRcoa2ovZw4ZDPV62U+#}jQ7wS$OSQE?ye_4$!Hwtn_p5!FzJz#Iq%Z6S=~w>H zKZr8-@+#%Y8PdPr7#$qQD<^#Re0Jirt$S91x($8i*Ew7RmpJ|B7@)!e)=h+Tl&&Z(JmR*GOxd~-^=0=fXRlU`{c^!r?djZj6NPgs z6|U_nMmk{7{+os)4oJz!&@$oCc}?D)*$g#FCJ~0z3Wht>S_UBK^&x8u2C<{>RGn#8 zZhp&5=q(d|`2D*{W^8{6A5t*H42TAUm(fb7nL9Pyb;g%WH3`wfhYv3R)=q_1)(J*Y z?$>ZIOS?ih9Z-uXVkPCcpI=OPX~`>PZw??;DAVuk$%)i^&kET2EqzJz@5C9>umd6@ zN4LO>Je}5M*IwuwHzVx0WxR};SI>%e<*J3qSnph&a^KredTKVHb4?3f?(eb{gDt~5 zZmisA1JWTi`#RomTOOnPqO6|K;a798fx{{#TcU=woPgq5yV?y(*#yeOr{t7>gU-y` zoX?VKb1OEbO=-iS7keHN+5&pSj7+gD#o8CDrH`#XNQUKL^*^O7iUk8G-2$W~$8Zkd zI#Pzhz!(nf`H$5^Qbf%=i?fy)&Xr|QaVs;yA{{k=Et<3#&^pyQa4I`WBt0a{ zAVy?-$p)r}`Bb*C38>GG=O`v?)7@GXG2j$F=T+=Tm47WNRE<()etsrI9~#pE{DM~mQ}}{1JqH+Pwncdn<7|$Nkp47MdN!}!9L_cPf7`lI;Mzs z5~ud~1qOEksRLkjnruJv<6pZsLa@L<^-KA`)8RX-V~ZS|X}yYct12gp`8_v(VGfJb z6i}~}dZ${_to$N()KZl)PnKl$l)NJZE>28TI9d*3OA0rWb>)g2g~^~za!GF&=kF-h z7EsFXEjoqxF^3g+rTpthAHAB8=7r5N_3LC`#4NdOzy+T|QW7yFj`p#iB*Qm5-QT{LxB=<-ybU_=zH?;a z9Io!c6R{49DJ*`&hn%}p@HXQD-@pz6oYavqO@x2_`6Zg!9;-R0R?t2aI2vQopGo?G z85|0rAtV^0)La$l(f3lR_(M!>{iv?@)|DQwzo)9=jA&*;OJXZX=uqbgMUnFRp9K8m zr+|8@bj`TO2iwvuHu-BgSE{`X6EfwkK^PKWO7jMIT@@n<=DKY3Oxwxkq$6aIR1_aX?S8Ho%I> z0%uj1Rf>OBiKKXxL`T9)b|?Fuu}OzlFlyEZNk#ji9uIY}@J;0yeO^Z&xLGgY4O%eY*kR4Lcg;0uu4d9iVXwQpI|_S|dV(MHyCS`u1_rTM zp|kKJ*!$ELk=2j+(s{Xf89(xMeie!31UWD?(Tr=6{a7Q2$QWg;V!bWX z+1c3_$Y)e_d#%?vS(-lM4t40v9KfLlt)D6rw=sWw@#zz!*8}0!gM`YI4s?$_mQpvG zG+3oTJuY{PoR^=ELG7n%$jZo&xx{Rz1IQUDT6?$hc-o*XwReaD1-xkq(rp)0?{a$y z;k2N37=CW0AR`uJ#laK5AtEA<@WFR1P#-TVhFtYH{zZjbt||sYs0Zv|VNT7*8O!?+ zvkQ&*kif~6o|=k@zXjtC15obEBCVuY)M0rNztN2wzC_s)dK&na%%tsfME9xw9Lq*#|}NES~L>m-vSE# zN8g)UwZ$mGzoxi>Epk;o$5!a(_h2j6=&YI1)zpD+74=7AO36jiDsHzQ7hD`eg|ia~ zPUt*7cd|t?&>UasAJFc>Q_$LN{;4w<$`Mv6qo9WJSi&!3W;>O2;F#jo<<|m{GmN`e zBehZn-}nzxyy6gVa&n-LB62k6s`Vv+^FU*_1+h)(ObNp}SkHT(ea-&hYz76@Us>Aq z7r(g92ZP$$Dnu|^Zbh77iKY!w1{PoF@AuiW&pm5`C!$vDBidD}9umxAlrc-c)M5GT z9PYiXA*7z8TL!PpAu<^AI6hNEKNGwE?nu&|0G_{Si_0N#<*1YB=9sHs@D`dWFDv?eZ$=veS|Rl zPChbw)@oRpjOqNLE#_XHvivQ(glRW@v7p(BXlCcvs%*~`g3={;c8qQvji3dmpoL}` zZBv$&LLl)-2mBP-YXx;Z0njbqX8S7`CPf=F$6V9y+x*SANbiBaNO(Ic#&`gI=PM`= zPu~Fcz|^b9$)oS<$CQu^=(x>RO-6C&*fuLNSP6ZsP zF3@CerUwa%sdfC8s#!IY5|?m1YK6;FH4*QCfvbI56$zGdRrl+J1e>V+;vV_xpazs2 ziOw5Z(*e&+TP{5KysM%l^~Pjpfth4M&=zQVOUjGaYq8_in+q-4@}qz*a0=5rdGg~r zkm0GYHZWVA?)5|k9l!&hmy3z_lK}t&d zh5{r(uq}m0?YR$Vvg*2-{(h5f%8WL4Uk5wOx!Z9Rn1d=LtAN^cD?JGG7gMeT7N}M8 z5{$R;rW(yAIg7IA&*SH!f%l{NyZ*{CGd3kb4p#i~_16!H*s>T=)d7YkIjU_7_yts2vQUmd)BnD~W9F>A42=`c z`)C;0=~k5#0c%~7`|@yjcvZL37eI-+D0~8V&zyzdS|#hwcGm+K6|*j`%_XU4^nl-< zQepxP-gStI@Bt1FnC`~xr44*bf}CH9MSDpG&xcs=Ff+;%+*MwWB6sfG8CHv_6lcM1j5SPbT~R^1D=)kPfkA%e{DDFh8vzGaR+ zwHJ7G?qwRpf(WK!Z1+=x zt<4JZuolqQ(k61K$Dq6ydt@^cjB!z7xE%Z()mPIL-Fw$EdGSlEE_aI1<`+7(PT|~& zW%E2lX_HH#+LD+zGS*q;H&{RIozl|MHd}05c(Niu_#_H2)+J#O^FO~( zKz~qxC`Nq&w%(hrn#%b@J&P5icntmCBJfLD;ebz-QJ(~epYD*#{mTiAFf}01Yji7- z)d_$_Z6maQ`{t`DNnOmLa5D0NVbKloUqI0z>M7FGb-YZjV`jZWG*@IK47e>y04!x= zhD6?Wta;q*LS`$oF)pOesT=&)F!8HEM=f2Xb2%F@up)gJ4&j{ffFRq*t~m)C zX|m|poZAOO}--y2=7KvIv zdf{FKy9ifb9*)b(`Y=l-vYefqtXx}DdhZ=mevjkhH>26d8!KISXA9iGKY?@Xz4rYt z7pF>Mfvft9MmRTQ1L^*L=Ig0gJZlul|9&uOK?Aim8Q|I3o&_wECBFq5C8C}3u3SE4 zvO6ji4w(dZQQ<`=_7XyjZ-!UZOm5yLkZlk8&$MH;69Nz3k8GIJze$# zaWYNdh;knemCcy<|8m$eX|%agzI!|%%{~^U9RPBtE(5g62ueE`#CzKk3AsnfQ4;5{ zG!url;66s#s78svtkU&&eCwe<)D*M{OWArBw6!}c4LmEwm|nU;o5hnNRCjh-7P~Tm{!%_Rqg!NP z61Cx>Zs#7FMYE07rLx@K!{U$Ob%v<$U7R5xpxkyHve9K|bdHqFH4qiF!2(6)caK`H zoTL7wPQIGFE|GDtTB(`?DzBQ9vl``YqOI>UYcFj}_cu5Es6HydSmHv%q!eVRBG?h| z(6^Ie(5q!qy-YZ18dH+n2rhK1(duQ-%??zyoIWz`J1}#B93@>jqv1fppwd|$p~V~7 z_Cmxkvy}tT!EOXaY|;vf=@~ObQ&||h{N3W9Qq+WENERrQDQ_(uB|bx4H1sK8q1>SV z4Z~HiDC&sK@ZhN!O=OVAwTX3*%=FV)Gc2VgZL}N$p(@ROLW2M54#+V_dIrYpU%{LjIP#Q2X~e3U(S6W4!`=G)DKS4vujST8;Szc~ zhr_oJ+8eVK=>(DhN%?E`79*rm%E{3tlA%?LVpz!hkzFz+7MVt9Uvg*l}bnR%_uY#}C%>eO-d*S~oKNL zy_X_qHS(^I@ljdrez4nxTQG6jH-Dg-)d)UOdQyNfX>o^_ylYeuH0e#u{n>YrN?(5V z_uq<*`Rm)nZIJFr>C)j|X9~(=5m-yrm_;^{8kC@vklqdz>bQ?9@D)i1PA1hk2D^QH zC7ls?No{tY?+N6xGqvR#8wY_OwA#5i(<*Qi=H-W5C&`Zu7UU5VYbKEif^tOy!X#4ZFTBH4(AR{Xk@b#+P0)&YH55=)hkD8~W^N!u zLA08Zr1~X*?_(ZEy(Z#W)w|n9HEeA>k-aRy<9Vw1LPFjC7`%9I_M6>{!R}8N0|#wU zX{VrlzKQ#Co@HUFnFK}n!9vPLiq?pm0|Uyu_(9pn5Y~jVu4o!KnlWQWjjGM?P|7aP zW=)3;@>ZXi=Tw!y9qsh#(@JV2X-9zFR-=z|3M!*`VgY2}lDP0P&{|&>7oXp~d-qE) z0Lk;g;^fSg1gxeeaK2sADGYNUKa$`PmR2>bP_o|^pr5|?m1+zXVKaGV?qeYLZcmx8 zw-DcJMJ}L}rJ(i=0E|}A>VMvmd@-Js=Xi^;`VH*Gc+JzSrHP31KIHkoQJEb+O`<7@ zb7o-s&85*lX-%WOW&W{hKX(}!8!x^y0TAdlAQ1KA;^M~M=3wJMrZs~15m80QYGGCB zr7$}MAl2uB?IEhGbkJ3b8`_f}QCP^q>Q1pm;+FqZhaT|DPAYNz%%@kstZS}!IQRNw zR@(FS%M@CIeL^CY;U4td#V+Tm-l9Znd*o!9Q}e-!HWUZI^Dy7<7s)-x;OjtbjN^&j zp}qiDbPf7HpWq3%Y+RKt5DZ(x-jWUV{Rf$XqAE$_R2Ji+jh&sGU2T9e*}|CBLd0#~ zxzpIdz#y_)-j3Ka8$~p$RI(EWQAe^me?%Q5CYV#L@?GFMptADdhLpmINJ6SDvwOfxCbB8py+~^*%F&1#{-wLwAUo?a7x6 zfo1PFK-)@_um3z{kQAlcLA=_&8vv$DsOP}bW0IerpVBTYZ+nJ^Sa`y!i)fHBBNdp= zjZIClpTB%*=HZ{VJj}kzR}-b9{`9G(a~8!HCi5frYkWN)#WTz|_I%+nCf#8}VoM>} z8!W82rV{|Ksy1H?Q1&v|1TTPwr`LA z(lv06w=yFvuKjb-wkz)Lh20FW6fDmD_T=LQ{@H`}#~&{38O$XN&@M_Wj_`iZO;FFG zsCVFO!#>^_1fp~ce7z80peP3gF9>wMPQaUHnwbgQwu)h!f$9L{JA>IRM)EG^i`mXQ zfCt_hSW64Ue7~L$G+hBG=i6Vqg+gvi^Ime3`<<(KPkE0&Adg_R|9SeKpIOuYzubIx z9Bg=q_;AbKULHhbMZ+A3i>n3(1drtZ{cc$#uyL@}AaW~GK(M^Qp6Fk=PfYEF@_$E~ zcbyK{N;pD3e!BybJUkbLFd|6m9 zyE|;FwmppgodW-P^Z(OM#=i&w|5?tzf8j&=mo?Zpz0%KlVd8ap`GtbE$wocBgQ6lQ zreBsRsukv>rCq$)##Fy^^2)BrH=%pVZAvH<`R&_2l{>dM@f=W80IgpG{}dSD0V3G< zCwum6c>^{!Hu1%a7Z!3tkgY4)d=VPkAe(!suKyV|Jy@NWT=Afsd#1^Yv&JzTl98dE zJl!#VoGe!~#$D6B;f~4+YCwjhm}o61PRGBX{4FJ5vr``om&u~v(rM4&7dZY1%r*%1 zAY^2CRdYWCwH*kKZqd||-9K*Szi!BR`{KfMW}lwdF*Q>iou!bCjiPAZ@oI;mSLNEf z`7Q@v`6nKE1JEEWGOLacx?RN=cjTC3IH@$SAcPo}9eYDy^}s*D$-55kuBWHhbxyu_ znzW{=T4fUyK{+_!@lSB_u8=!@`ixcmK(&=u(prfp&CV%&tMPt z-nF*tFv-$X1zhPSEJ1kg#qr&7r!?3JhEb9-;4)@6sr;ux4VoK!`(_P|JEXZYQ!a7o zm~1xz7q2;C^6Kzo^@WZO$L|;R?Ao`)`(ponWVON^>bj@def#O-OLH47hEW-ot^;|s zgyzP0u2~qJU~%K!xv?7O?%b?5DmOFi%8j3JSwTS|^SCYr9lL`MOs$nwDAmx+^pn0(Fs0TXH1w&QIlFW0qy4A!eDw>YT2J+juy~V^bvoQ7 zM2`0D?_50b_5^eAyeo(MP^)Z}sxe+#T$oT1zI9nXh5O7u+UQl9pY&JJo#`hWoaJ8L z8g4jK;a9=zwx@*`YpKrUYAhxZQ88j`-J*(`$oNB+cfWqx8$OQ_|Fre;pRWmySglcEZ9Ni)mZ8Z zOM7J3vps_G5VMNS2muFD4VA72#+!>a|wgTa~9nr&O%p=zq#{^slSaowHS! za?I*RqOqngxcf_%dRI~tS|%!3iFJz-aekP6Wewr1&F1{XZHDMpj$VCVwbi4@M`O?a zBpU}_e>*LE@8$zT*>`6X7rConWz|nyY6|N2YcU>MXwXU2nGcRK-nDzrf`@nx;@^9u zFQ>k8xFs!FoGvrDHDAUdMQSd+aKP5wz7nbbopH}+zjs5KNB$sb>qq&>E7*tiY)XC35cLJbiywA4lxnB3Zir>C-N9x=v;=DzXRznE#P2-cF=?h(dH+=U) zaBxPuRYHZjzHc%9Mt@U)S<;57SNRMziW{Rw+v24PhNLZ5G&`0IyxX~kO79ao6VzCU z|DCW7fi$IY@x^dYtLoiPGy4z`X~Xa&vEG{4-TA?BMRV(?d4= z1hJX|455#wnC;3Vx<1{$>IIWGjiR5UQW^pZbwf8WL97>kQm0gm=Dd!77}70^G(WB@ zbg9ft$>pdR zPQ@??M}F=~afts^kgqM38*|oynshaI+Xn*u+HIS=CobOqQ!#w_4wlv) zv)Y1Vmb1%hEd3elfbIVSEqI@e7fsI@)ea-HoCuVdJNSz#za~BomUm`@^7480O)}L) zcyl@E?p#8ALER|_iGY`Baq3?g3pLUbhb&&FZik4^eJ}_X?+a^->`H6Fei*4?{e~~R zcLrNUFT4!4`Bp2$>S%2!n$f1SH242`FUOA4t}$<*K9t7@<+N2^YDoB&;(==s7CSVy zq#>(CG`(`gKB(~3UCSGe*PhIZcR?OL+Yb21gj62&NzjKtqdB)9XxhQSB!18);?uEOP?9qXK1u}zaZgY3K=Yj57 zyo%fKIr%T#u$bPHx=3s0@`goX#-5J)KcL$;kILNpZ)J}lJpNb3*4C^g|FN<7=ACaL zp{LJ5GoTU2$BicnzHbOaGsR?arHbuHG4H3FK7+~!MTu2W5%+w&dL1%LYNKb{V`m1b z+|4QZ;LM_(!kecL%WLo5tZ|Igs+v833*Y-N>fh@kti8;+E{N$YyjlzQ;9pc>SH?C} zuk4!6eBl*N%0JW24Y}4hWU-0fCxW!uqXX%+6i@|3-tqU`L9s>A)IpE}XrQKD6b;xguNYJC{G4K3s+K!*I}T28I6a z>a4!m)KiAfHqzXSCZSX6rT775<$vJ+e}!5$x&bbbe2w)XUd??MzCz*yBF!> zdA!UUwAFu0#^}2CC#}qNMs2zGg*_YUi0+qbnk^_M#<0z!C$ zk!PIv4eWWfZurkS1wZ>_GLcqic&YDB@Gcwpxixb|fp4mYwm+8_$w#0lk7T6K5wVj+ zT%Urar^Y@T+K5|QpMN@xZ$Mk13~oo(y?))Ac-v0&bC6%G=)QG+wVWDL(PIatBjYr_ zX>Ul%%Q&l%Tbj(mu5tV7hq#y@vQ;T_o30Zssee7udnP_kc^LJrUwdW0;fNdDI$NvU z`)`whx+?*%5|f-BJJv462QTYhQG5_T+ikZ-6WV0 ztTp#AQ)uf@nKOWmr_X%dxcC6FU1w%Dq~}k_h(omw|7CGLh}+(3FN7WNBT+Zo%>QJ1%SUebBTuhD5M#t!(=jnhX>Fp8@0j-I0GUi-_1e zYWuc|Jw zY0hX^kogl1D~O_-a?Ne=I}n8bd5knaoTh)lOkXNKjz5F>=YM&}|I;7;_yah&{`dNb zga?&2<2mfb@si#OXXpFaSJXQ#A3pErRc2p3+>e6?2elCSx>lNIJm6qE82ndV@K<&F zP*;OCgEeV*;dfe%b=OHpXA_$zgB@G<3KUzLTNA<+QqFdxixc-4S2}$@-MoYJq?sAn zWPJ5H=f@COK(B)*%@S4)s9VaIcCyG zYxfL?+F`we5(m#Z0=Uq!{#=R0DMwY>Dsn2MU+3k%yVADG43#Y^_3$1LiN5BQ&Pf-W zA7zy1;jeRbeqe$EdYZfBRLa`$y$K>U3oM1a0gsC~v$msW3SVASRIE%5;#jsB1UZin zdxaMi6K}U|jXF*rJuHJ02RmgF4j}gOfdBc^r>`Cq5=#Gn>XSO`OJUT*)T<`mBFhqdAf19&9lF`gMIV{Ki#0hvZOI@)9P1En1|*Tta2w(Th$q7cGq*TE_dKlI;BV=WQiC>B+QEYEN-9S|b0&3VRB8M4e~sT4 zR#szvOoxpFcZE0N$JF;JCG5%*RK~WMle>l0F z@@}5ak5srF&t|X_9_X$fF|uu&VX8{qy(zWE3a9#-d#kZn@0`ent$5?s%gcQA)t4fHLpH9*A$A(85=`yuthe4Rdv^a5ZF7Uy% z5$s1TruT~^$(f?N6XWb?_A2;mWhy^LQ0vZb;yT~T_il&^@As3`B_wtRa{il zrsuBY1}>g}83C>WEF(TU%uRrJd6ig&W9H_(e00br04XcfHY~~ZV8pGfcxM5!#qI8KE7D_rRVXev1@KW@~8k^&+x#mGi%4Yl%#Y*JDJU!1PO*u70!sM5;R!=FGp@nr<-IlSu=KsaK> z$aa`A0Pq2LM>+PM`!1xEZ#;@goKV}3$z-oxS}Anw)w?xTHbKNxX}DNSYT8y{{NRWO z$VRHyOmLV~vAAEWnfGFVT9e>fI(SndY<>T#d$*R$-A?;(*XFvzN9hN?@p5+N^$wb_ z*-y~b%(l`gR1K5nGX_Z|w9^Lcl1p-)g#9?mq9cgenb8NgRG!$Z;$lRr9a z6S6+Cd4@!1XZ{vVMfBUA1d>_Y(z+Hsy~gyK^me1fA#ZburBqtrBGdkunY1$@+Qw)T z#+Z&LEY2iRQoa`-t1UGm=7Q4dOO8545%I zBB?$woQ%OgE5H)IudU919J|SU=FGbr*xe-QZ85~t@soaD7431_ZapTvg9KenS6;=; zHw`Wovy>Dk8S=`zlr_r~YsFugFS?iI>d(wlu?ri{H10?_bY7RIGUGXKq4QdDv*1b9 z7zT&H784FCyUB*rm(6|HpHlBHz=lD=3HfHyZgPsFB>Gz5h@0>)B@$I2 zbE=XatasWw&5;^8>MM2NUH^&8UmiYX{QU{=Ukuf8s3`a6mhr=Px;&ivi6;>1bETG& zTw`CCcQb-qVQjP{RuZE}l)}FcUb=jsS*)L09;GEPo}6K{_1(-{=00~K+nbZ8d@YD1U_62OZGo=`;{n4SXR4--o^Hv>%}`ge;$@nd7DMraa!A?=<%v{v@TB-zIh1mVYZmE zJPIp|!wVgs`gACbr(UpOH>;!niwFA&!q4vB_KC3_?VqzwE__GWv%WA@{GyH&DMu9* z_>U#!o;{q~b2ihPhd;d?LZGM=I`3mq`B}CF3L3LP^edm_bMn$;tjEw-uFizOnxP`)?Tvx9%JZrR=Zv1(iiJhVT%+Ou?vMSkb)p6jFR|l zJ7S4B`|QvS#^S*OD17tW^6v(4*5la`JaF{}8_q1;n}we@sc%N7%OqhV&%oO)sPq=~ zRL9NfV{RI`!K?2#se=PjdWSlV&{eS_0y34#OraU)DQ!t)(lvEEDrIwMb>=0CoS%=2 z^}oG8DtpaRrL?8YwW^a>G%>X#-~5R^aq@1v9xkB`&Ek+#xsJ z1++uZCGTZpO)j0HTf3;~tBS0=e*7>$d#0nCL0(@!{?U`bfn zYfw^ASxo7_*=af}aW;iGr;Wnc{7+22*SnX3`w7aF1?Ucc9Cxw8+S0`6l9QaA+z_H@ zxkLh6iCdxEI_&MbuIV6nk^(j1=us}7zdN%W+qA>eZxclR^B~@@24(RV^E^55`cTJn zoH^qUmIb)@t1pJq3;KAgcf3zaQ$SoKgrHq(7<%LoFPYrOVSoP`s^4#ntXvu}^xF>K zROdl=Bd4nyLrE0;3LjwOMOwphtz?4qZ5Ws;frVmFN+>pL4@%2Ej3_teix)L?(VGqn zC0SXGFDIt04wr*|og3`lp!zOb9TAQtOm(Xx_Br9aM$Zw~1n4XH`yC|LFCRdqpE!B) z68LSE#t;Lf8PAiFlap~z51sB&vg^5D8BRjl_`}8BAMl0zlb)WGbj=Z(jL>5isJIwZ zc^(C7g@@?-r>fdXPD_bO2X~~J)bWw}Xbwc}+~9M7Y5IOq(_888SnTRZWZm`4D?6rG zE-Qb`mp1V6MIBrLJ^*L{PxSY-eLu14{vy~P%qUDC*q>x3^wf6*N>MP5yClCkn()UC76+_xImfxYzzz(ZEy_qu>jx z<>H)L7+T$HlfPuN_w&?e-t1^CJG@;Yo{%){@OiFvzWdn&KX*T2ufb=+UK3*Vg;S$Z zsqOk}kq$YB{wjA{;}M?g<_T*ybv9#mKi;s`z5Vc^Bh_|{%pZ}gY{tjOUtk(7?AeuXAt6i)ow*3l5esxF?#o;TYd%fPcI7-TNQ- z!50@A{bBP114c$g@S}@##|2j=zQX?AP}n>0LZJ zFCc*B+4PJwr>^~p#kGTx=^R53hp{!`I=gGWiAmQGUp+MmI>LQ_B4r z-J3!v{GJ>E<2bSlk<-1!@@i_KhQ`JCg@uN)Li5kaDoAEdfR$rOC1Mpe$wgvQ zj^V5X<*IlX4;z~_Y`No0(I7snNlA_|N0);09o@xycf9ciC5IdAMGsZ>(PKD{DDE}G z4Tew*{L9Hn)1AUk5ZlzLv<_dq-T%Py!lS{I;R2c^#&t#qib@&>igwp1yY2=d^`I3h zDyk{mPzQ^#woZ~{kJ57^eUTf|_e5_-od~+iwwfDyU5OgveD;8^R*7nJ8dB1#R5+lDMl8D!L$zAKaf013we-< zq2YybC+Lu`dwbSCxu>1sW?^diF?o(xNx^T*o;T9MR@#KERLu}t&6iKJt}At$C*fku z#hB%t4u!4{k#FP-J7{6r^`Wz9p?wp{9dS*L6!!h`kd zi(?l|fp7-imJxT-u z9vw*<92^`5pMym+x~6$SUrU6U`+b2P|8;?`HdW)R)Mgqto<#9+R3p#geutYsAxEFT{JlbyzKxx(S&qwwSz zpDE3NG<3Lm%f&L*mA2j_t5qAzU!S;35+|=qsWGay^o)sp-1|VSE03m{kQN z)xe+{rEcUCr`aZGF*PNAd+$#Rz8j|6C;$v;3*w*x~Y6_9@%-yS^p;s&7%~rS)Zp^tE*~ zNWb4>4`667&=TH2yS0D|V^Op!ztc1zu&tGNYC8v!r{bp z9f}kE=eHGC2T`1AFVZ+`12hXeN6zyJYKKmDar3LaSn+-y@uQ8K{NU_%okLe#&wOB) zBw8u7wgwm4Ss0m6mAWxA&hPU(I(s@9DX{7HfRIm)#LJ zSyIZizLcFC&l#6r{j}8@ep8&zb|P=~kkuC~Gdi-iCUyU!MI&1Eu621PNlIi9JcChF zraJw>M2p=Z8MAayIM+sp>I`#xR1%6(G(xW3)ijLuIRJPw%7AaV!|8JVZdtgOPN|FaDb ze#j3y(pfy$P8uDu3LJR9H#YR-#WONMSOzww(DAaC3iM<3E*Aj}>AJM(K{-ATRKrl& zs>Jh5s7o-bI;gTsi0T7L(k}#m`sbG0U}cGP_Dobo=YlQVPMa#{Pq=^I0zpClkVqEHnc!hVvh32YGn0 zyUU%nJy^-&0KFT=wdfMyjTIN??zLah7Z%WBaUzdhO`uy~a znc^mmNL6g59-9eq`BEt;1lH>&J&{>0{FRs?$a8sG5PoHMFc=3ZOj-nh3 z!5SeLNFSHz5dDg2Z^lNhoa49r$p#!0CabI334mWF5AQiZ9q{I?j-kEzlb5oY`&R(X z4TVf~u3(~#`s9uIiSJ=Jc?*+jsTUpr?DBQu4SbvzudwbD(we3hjj<*@RJ(#K-y+3r zwfutaH-KJkyC(kqbxq*ULjEb_5;>`}3daWEUMNL$0T9qxMX3H$moIm>I!x$zf5p*> zEr-#DDux|ss|#xWA(uINgytAG?eZ~h!2_aK)>sk>u0JDl#$hd({zL?fEC10KoE#n2 zG|KJGxBIKKDH!UBwvXuw9;jg|hvg4UX7j+0u+XpWYlPC38J% zGPc!B8RxNo#Y7JO=oTgQ-pMx)_+O*HdF+nxVb5R>>zP;K3-(&J^aoVp{#}X` z)lUP6x*F|BIQND{@q9z*m*RZcSKkI&xZ(!e{A8xs4VrnUdRIZ@Fl$B!zP8%}x98CS zbzO6>&3p)rqgH`lHNE0G-IoqC?>O9|NK4Fvvr*;wnfmZC`hgZ(lqb*jce1veA3#n` zI5io`;fsiSnQ_Z$gJ`W~KZPmduX(#Ymp!*_?F-MpnTPCWS!Jz~;A4|F9yY?kAx>yk z8EoOFJ9L=+Q($q`>qQ7=TE}NW<`D=xE;e)rYGdcNDEdzQ?yz-uba6_)5n9)ZiUf}{ zHZxCrE+vm^{U)D6Bbt<;beQq-2P%~1yF-5p&^zi@DFL965rGh#b51=?e`#DxZm(aC z(_p6GjTAbdtv`j5dp_9q!y0|9xVf@+gS(Z4&8RVuD)VM^m%Y7)J%{Rfzf!EjzJg=f z5+Z@RKuwAAWNM*`gc9<9Gc8XnLN67H2#W07Cen}b}kep=rrbhDF}4fHmuA?uiZ zATQ%RmpvF0x|vr&BvLXkGhggT9!7aCKmWUAo&rfFMREYpXYbxv%6`L)hrRgk5tE|X zf}I;a$&Mi(vSebzcA&0r4`33;d0Iv~bM<0e>$$Zgs)Yi%ki@SLQuk4Y97^e(>|L37 ztE~`Er_6}zXWAA#XHOTi*BCnXv4)34vhdmE7&+?ZPo|w&a1nw-9)5=eJ7a;=Kcs{J z)eE*LR0-VH{gvkC<_YiKRS|{7#dR-V{_uhPcdD!YGu4$eFykNsp`?(7w;zrT<4WCY z=U#T|=BfCp^tC*B5+GL0XHRvF4tv^c!8Y*3Bxc%Vl|TCakMP0s$&ZJMi8~=cH2@`9 zWAWa-XaIABc%Aink21tDb*k;I&iE(qqEy&42NwQmpu^&()fD7;9eP zynG|9c=OujGw#H09 zke#tyJcBGV>OW+4KeCI9d4;$=iF0HD+ESeLt(i!piw^6nO*Bw*9o6 zrInroq6msrD>Bm6Rc&Az=EjAeQ|}KtbhdYr4DD|JO4d+>4E1AMbTch*3=ON96xCKI zcJhh{YCEb>^LK(KGtMUw35rS23!-URw!006O1BA z0L^k9I{C-|g(|C{`%~Lv&H}Bfsh0H?Ifi7UBRhrkkEAN&0zmfkR;9Re2N6;#W`5qU z$wuHEcL<3YRMga=|7Uq-N~ew6Yq4WfNRyX?M|)(%DirR$*o&l(420`V`9FF?Bq` z|1IYn`bDV`cGqS*?abn8#Pq=Lk zjfU0J%2NPU>(X{buB5B@Zu5Kiz7Vu-K4(YaRdz1d8Jo4shBEK=$&YIxS81cfY1U`! z?>!?8LRWSie2pZuuVdV(_U@g`q@kf1JCB-sbNYdnXvrx}BzAd{pvHpy{W}s98+EsR+Hp{k7+Kwjsjcbm4k5*FK(UlEXC@SX z`JPEEm0Tlwyv zw}o+dxOR64Ry==(dwI2J21p+Gb&7(-W3xYGv7hP?RDJBgJpr}<7H2eEaf2GfcsC3-FN>^jNMP#1JVBN+$n2%#^>xWa6wYL zVk?exWm*|srG{`?9aU1NqN4Nt=YkVuqZQ(iN24$v=FeJjsY0A#eCNrn%vGNM!-Qbyi@$VhzDF#u8N;^j@3N%f!S{%L+9FyIJj z9PUcKxu&5VH3}Qx>etthl0b@s6l0HO&%mjYT(9`uDX;;hkD4)>fss)O9>FgW0@L7D z`AS4e&aZ)PKJYAzE)kEWyh~X&6Z+^aWvC(7E=Go)s+W)|s$`XiMjmAxS)BXeuh9N$ zLy{{Y32;zBHREi%^v|nFggfu&rL!YYMBaH%!3Y^j64#G-QEj~%lT@|M4K*u#etqMeg< zV9pPAsOB%6(O6cV#v~h|lUA`%D#6iT$G{mt`TpI^E9rm+ANGL+?99{}pe^}=`2M|a`QhzrkWfBJ(4$Si3%gXdV za_ig~4ej!TLZT53ymoBnD)o)~ykFZw&`|e6@driU*2a{S{|osvrUCU#Io+N=*?tEd z%2|tN4+%XMn=xMHf1NJ0*}Qx9fI4# zEq(cwvfYmwH0f_vi^3fJ`|ovAe_b93&e8Gn?l>oOs>}Yn4b_UD9PZopiuejdv=mwlZ**2|xVeHgT{L<5*tC(VtE8kZr#Ptw>iqqCLrz^JwFsJ0)i}g`_-A%28D~y4%8Xin@@L6ReGnw@OjRe%7@T9YS%qV^M!GncG%!#v zZO3Z#m>A->(p!|LBp^Y*UYH#sji6An<^sF4i701?Ei%p{d2?yikQ4frhqO*{ZFU1;_3E#%JdT~a~f9- zUj;E-NOJ^I-0p-LC{%5)JbnZ~V1{e)ymDPI5Zs6V`Ddub7QW)obw~r^67cfLOhC$6 zn3)s)2X)bFJp>Xh-fk!sI$a`bUCWYm2>%MdKEhD+}_YMX(dJ}@?S`A zosRUawp+LX9i*%|I;(dxcy!@N^x;EXNQn>lP@=Jn{*~Yc99%Apzry;jXE+;lN5OGC z3Vn#p4`TQZs`m(@1m)(maFs{irlDx)xre@bFv#kdlHf`aKz`t8G(9|1eT*K@G(8_r2aF^D5y}jUl8-h&s^0 zlRUv8;{^m!5yZP4G$Jyl_x2#S6H4iCNIaARO){V^QQW#QIW^TwT1F>W#dzCNI7o*8 z**Pit+zJkf=dM4QPh4*{6k6HLddAa~%M%8F?h63Oj5KU%UCZ^DK%v3%JK9Lz`Lk}4 zxlmOlGfjYc zb|;btLr0#e6x*FYEQ27h3|f_;&6i;VxYza;%GSJ3b<=v%7C@#!w0 zUrC>kyVNZZmE}sX4i~aZ0^b5DwR9~O7{?b3>gfd!Fop&Rhm5)TdoWLwc|y&=Zt(SG z05yZ~O1vrjAmr8M|MLb~h{@Qn*g@2;`E?1=3^4_Tnh{}FsK#IZnMm^9vr`WZ-W4NgcN?$?u?5$YYB2aYGdDNKrK<4rlTaQtpxmnY zNbib3e-`aTpf#Y*gzml5H;Zc^eD*PLch2 zg9z!XSHnqQL6Y#{g9eCIY7M~OAqR|_M=$T%3C>Q>OThHjwq9uTg8KcYTA<)O9?*Yl zg@%lYxq%LgU4M8$XfO06u$#X(s;K93)mkVF6rqFY2blBWK|tq z2G_iM`S1D5Fa?hb+;xNQ@Z=iu zAG!m=9M*t4Ym4La3k?!D;KYZ)MlcMF`{er2B=;am6ov49BJDr_bm+0MvPKvl`?CSr zU;~QorfWMggK-*`;1HAtWGjRn*Z1T=*+Du0Aa524@7yT61zI6Wl`E}8Tjt&hv7;@t>R?-R{NyDJ52}QU%LN7h30Hn+dWvt;SHgyDf;3(75Y7RS zg|3<_`aTV9N`aL~HmRvfs5IC#g(@7NeYG?own)(5N%;z05=^0d7zZ`+yU8X%b6Oz9VF0{y0-x! z3JuKnC{f1+Ed3Gla@fo~cZsX zXm@DX99sC+BPJmGHNmKB2k|GXg&ShX%81U{KVFcL5n zhW4m&{Qi?|-!wRL;_vxvL0CpRyT9n?tzqczL#I{^BzIa*UdrM8A=Lw*6#i&q#$go5 zZA5{PZO%1OHd6!!e;!6U<(boM{dTC9O+kxYdK|rphU))p#(MzJL1SkKeSgVqFgqqe zUglmF-l&*zr``!p(=*9*p~zE3|2xj#EI)KQMT zEg@*Da;$|y{8rkJ!U(1ZA_F5hoqF#3Al|UX7s=FZO6H#QxGmHnj+Qvk1hgN!x@Zw` z?uOnh8rs|`NI%0-*Rzf1ypYWS?*XK#;)s}DOU8QTS;+n8m8ZhBo0hVVFiJ3~rNERCHKhg+n*E!29!) z;)Yd!A%ULn-FQCBVa4-fgK(~gMz_UJ3&xZw2x6#qh5mKM#pRT z)PB7AD(I6Eii)}@h#(%|lGvE_c7r{c#O8@iL&(e$a`q=efP{1CA zE@#$MLv<@0vBWYwJc#8#&_t=5MO9<1kSRc&o+xdZ1DD3?Ry~9|cGNIElg>lz!u8nK z9V_RHVRR(R>1!_$QXf+xaTs`g!v|E(U|KdD|AMVnj!_fe#FLySh>Le>Hat zi-}s8n@?s!s9qU7_Pbd11CS~r{$0-Pks+%br(u9@8H!)Hm#MgZO}q%Je^M&2SmugM z>PzNh_mGx(qL%8)kGM+Ma0P#jpXVfkIaZQSQ0DPOnK1h9Dbwn5Sj-9hpF??h9U8yRbC$ z^p-{IBp4Q=fFnKu&!(F>xw(p&WCvYfcC3QMY;J?NHZogQ2qYTAUR2!vV0Q1N_8mEL zP!w5OXnu$Eq_8t75e>cU%D97v|KVI`bI2)}gaqKMJB`Y3FdQa(_(-AN!T7YokqC<+ za3l$p=PtYwgC@E>^YhCjh=BS9E4nuJ*7#pf7{Q@VeX$=k-jV(G3f!5D~_F z*+ixaJ)i}H9q%hwuE@BChJ>)VuPp|EQ5Klx7rDU>H-rqLIpTNYh-3mZ+KkK~VIfBW zfp_q?wE7oRTlj@9eN8Zdyn${A{;|M|iD(8Dl8DLn{mw zg7KXJVA@j+vkH$}=4A{Xv(B3(^8P zNFe(L7T)fyQs{!~{vgS@Cg@{qT2%f#V9R0fiH;yc_%Ni#LE9w=;9@G(g2GQ85vf@c z{#5l>%++Zo1uu~>`^M>~db6UDMI)EaWk#LZub>B7Rn|MC1~lJ zdTeoA^WS}#o+rn)Ta&;a%mImbk#C*J5+SA#iR(67(_WE*T5FSqqcMCF`9k?7*4EZ< z%W`1{BW8Y6nf%pKK2J@bvU-`4dvh>M$DJ=urau4Xfb^Psv@+-~ZwKAPIPG;u^p~=# zYSUBH=V0(!Z7~-QB907wHq{!j9s_d50&KF_$c6lv@6P6{Q&&nW%cL19=B_X_@wU5O z4H<48e<7XFoa5BzwEq|n>6ra%$^dn3E)@-(lW+$N<%RGKWk$VXCA;&ZNj=Y8r>qbo z`NW-2G87utC6QV*OPM*6otSq08gVwh#R;ZV&26*}rdAr+X})n`JVehfWFm1r`HPLV zxWtiL)1mhz%Gf#R10~v`9FrGcDlfe-Y93#laqQ+D$KTJlz0c5;PY{bls+H&hS>sWo zNYdzJkL9ww-rm3ybpHO_5IC#fiTOQ*RFRW8*g``BPk48_>#L*6hGLbYK7P=7@1pQW zN2Q#miMFh)EP+x)gLyk!Vah_}@slSH#L`;`3E>H#7Mn=sVyCb?&snw3j9`F)XW1!-ap1Dlkt@PD%h%+bDJB1ALe!JKc+}KCHckkXr-V2;@^~yU8TA~i4?7QY?HLx12 zn~Ko_Ff6g8ZGyjB@m*UVfqsq<#T)zJ|VT}fe}u?^w~O>wtM zz9m?hAQ&O`(W7D~ZPg)Z`nwNyG)gYQKuIpRitMA0WITXC|6ZqUx0e*P&=o|7hFIAs zQn~e-fu!{ODtLTq7jIQojRPOOqH@>8K->^&axLb}=mzo4iud&}HOjix-qB?VHEX?% zpj%q+jXrvQV(DU8VcK)~*uN?VJ7~KOn@Gu)Z4|jYuI($79f3w@`R9%omv{%L4xN0% z%USqh{3F`*vNYX4(WiRjL2e*NgG7W|FD_)yfc+)iUPzTs5oD%%+XJdG z7%RR3Q>b~JXYT~LBXi4NOK7crItcHSbS1R=?oZ0wND21_m|LlShF^iaH-O3Q8mZ}quHy5Y_Unz&1sQ5B`Azl6Q~zkxfu2Op?UiOZig3X}%9D2@?aZZZ-HuyNsgHxl-ty}+Cl4(j)F3PC=N?^jVkkb>oKbWv67}^z<=*kb z8H5Mu%}0hI(_*xJSkj^Sax&@8iTTk`&W%e+Ujm*(j=c8bO_~?a`ON&Xc9`~j?K_J)0wyvEE>)xo9q+7%kS|^%cX~MdAR8d zw#H9ettO+!c0AIbypSD?v{Za4g6d+pU{9d82L}i`>$IQnnn=vVPEn`aL>tbl5zl4*e zKe~QJ1u4w1PWNXH%}<{kt$Gq4Z96eJWB6k^&(?C(V}9<#!eq-WZS$dTs(iznOAED5 zr@f_yA7yL~Hyx^kb~K7)e4|kf4^K#P*yfh)2z28lx>^PzEsW4cd{z9m1YUe9qGdWb zW-i6!etmjIM(D?nm)9L084v<)OJn--0;XN%sT`{yfd%HZn;s7>RCVX*%7Xu4WI=-D zSgSQJJ3!@0jv>KM|6QPKAD$X4iV}{uvP_ojAs~u-BwE@!A)!)@!K>@Y5~G9AD4SWH zUB<+93SG3amIjPxGHgzeodkc`{gT?OP8zHY9ZHj~`|p>I^*awZchR;EPw)H) z_s@6e1*j{YSzUCtEj*MF+wGBxd4E|~oBt+sP^v^538%J$$A)jvC$wALcrnKdJEINd z%<#~PACHfqYrLiW0zg{UsiJj8nzZo*+O1BK=-i(p&1{7R^;cZ(e~INaTGv`#Sealx z3ms@q9D5zB0whYgwQlbr6YgIgzx6g`U z-Jn%H-RLo|^*l#TVcPPw0b0!$!+r6m6<>^etmz$bN}kfN2|QsOYadkd!L6uA+$4UZ z#41lgf+##r7p5Ppu5>kpesy8O*gaYLYt8$XnLq#`3-7lT%y)jhT0gsXd zJot6SFWJ366Nf3p#DT)dI={BIE)XbHc}<6q6&2I#S)GJD4zxQku)&}?dL`Gfe{rf+ zH|&{k{QV9>gbkkR^I$biXtfw3MR%{0V3cA3)=w1es%5# zKONLTAJrxB+>0}K{6;KHm*UMsLSCF$b7&crqjdmHatYT2gVwv^8nC^2prd<}Ym`U~ zSbQm;kELb4boem6;fPCD=1jyL`^*pj6qb=RTCNf{rOaKAHxlx`6~6)wjVjq)pM2%V z@})sQ7i*vmdG#)>+;gdT&(tKh0#LKla1xkPne~*S*D_@9HGV&MEy+j*G^0J@W7ii} zLAG4wYBBOu@HY4a>NnaZU;P@fWK5I8*5OjU*Z{qi-YW;F#pHlD9ZDkekOyjd@8MO;-!zNDM0nkRCe-LtpQ`LLQRV?H2@8 z|2r7JVVt%m*yk*@l`-&@!ysUex6*#gM(4>x)ALr$lB3(Yt|DREQ*G{ZAbu#|>dj@C z@hP1p<*!u=HajJvBNv1++t0Zee&?C5ZI&3y5kJks&8Z@;A<|$^mq9(<_^D6P1Gq6} z=_~RcSHs;f9uGH67E@Sd#LO1tS_V3dZ&$`Fj|yWwD9w`OU7aH4b&h14Ox|;1)V+9h zFS}EFeZi-`RiJuuCD72{TKJ(s#_Z|Dbuzq-mUZBfX`%i-E*49iEg~PkM?b*DB`EQN z@y)7$Yf8AashmkJJ^O`=pP6qXnn0Kqp`>>m~TfHUxu)lxsue+Aksv)d=*Y z0JQQX(%aRZPg3{vJ|M1vlvFN50n*H(q9y##Y0d#`Xi%itytGvdPc>n8F&TR`qDL)* z*GEXE2fAFPwBs#i^~`V{+Q6pcoFlop4Zc-AdJ;+D3BF?R)iu95{oulz@Hj;z*_|f$ zR$R4q2ZCVr<-E;?f~+{(oUiK$=K|XsV4pFD$+ov@-J&O8mEceq%+FTQG0m@y6a`oKn8-hTK^rd&Y#Llz4D}p zCzbU`hg;d*fQ>Ad$D0mDimx^coVwcZ8C@BqNLPk(YzU%y-r!pH#EOH@6=)6^-VPWl zXeEjDXRha`w<7=cA036!0mYSOz7j-dm#Mt=1b~kn#EX}^EQUV}`UW9w ziPXYlC-^RugAXAy=SzucD*>x6X2cC%_WcI{922GeiUbqEP@Hjr2wS3~RdMU@lU?#| zLM)&r#wRR{GPpLZlyLN%vA)>2K9}Lp_FXwvY5M%>PfMGEXWA<7(*^t9^jPxPEo(PE z*_?xKu{oeT#m=W(l8yZ1u#2VE&XT(^{D8W{z2DQ4BZ1q%o)Vr8*kg37> z=|dT9`bQUg65j44|ApYZvb*zuV616WI-p`Wmti%s3c8iL4?`I`Ll*iaQxT;mw{svJ z=zityDEn}q(lYl`eDI6_W}hz@lbrSQyI59l?~7}?{oqStO^z52De%cP%B{5z8qXPLtt>j~r`Do0ut9#7$<-#0bi=GAt=4U-5)wwVy&ywGa;imb z^+kfYFc+k9#7tQp=H^d$Oaao+g}ul|_AvFk2nC}hOa!NxIKG#dnw53VWwA;6i+;9@ zqobpD%LJ1NLMw1;WJv-9Is{L#qg{V1UXp8=sV25eD*wK1&Z{8POo|-=W_CD zCTDH7giVc;>SL*X2dBIR-aetRbII*tq}9TbqIsK^w-xhVQ??^EOjV$0)!j+b8P&lz zdeSa2ax>6BP*}N-V;34F956Dvp5f~OmG9xFN*yHhT9&YWM3hGk=GilMMQz1FG z5j{teZV4%A5k7NIcekm~7uDhU?)^A=2S6&n<|+;f_6d&0=7-*tY?_kdTF|6tSg9(VQFj zbSly2azwbUD^C{4Z-d!Tc&9Po|2yu8f7`9XN&Mn>5|t7}B|SYEgP`}gcEy`=Z5saudNj1bqk z_VVA8=*_$WDhY~TN|}|9A{NGxkPdfA?<^)4`0~ z294dd^>v|@v?O-)u%$mVCGn|ku8R3oEAF5UeVz4GxP=6!2BIJOO=vEDLoI__N)@@Y88{zSWWHRdd+M{Z%3lrpjBQy#|kowlkv`l6CgWOr8*2 zWU_d&W0rM=gc!RKF@6&zOn^Gy1an77f~J;TL4iJmf00xL?lCOb90gwe!^5pFFF|+G zCy)mx)i2^4t-@SK6hkE((`ZUv?>`_^%!7yHa7LlTYe@_@R@`1E;lrx zz_i0)MqX~;j(>iU9GLyWk3NAcWQuDY+|pzn%^rgCSx8tHsJIA8Z2l}$O!GG@UPRP@ zDi@hv=U;3F_P|nW9S;*tlJc-@o9l!yKmmjLB^qV;`T12~)-Bp<;_i+=lB8-Tl5a~s z6HrnL{)ln~A*6cunKT6=LQa5#5j}&B!7;{+9ebFG8qz@Fv(?K=_L~D+S=8@Eg^b}7 z=-p6`5!Dp1?l}WJpy1(rYIALgFE-kF;Mw70T|=?tT@0nNZ`tlvZfkob&N}tfBlxF@ zMQ5zllZ+14!fxnf0B64~g%~7|l>1p5pDouh0VR#H96bRaddEEUU5W8~UE69F5Y%nG z@bBOPNMBX_MFDYHybvQacc`eG)<=ZOFO|}*dbb3j*RDXL2l9@_z9vpk)4(h46{fj{ zy)zFKbD(1RZ|>;ge<;!6pG)$wpiIL~m_$2-B2ZV-6568Z`;+5CeXcVh;F6thjtJ-A z6>$@qG6B>X`e(M`EYIAk-1L#K0T9oTRY$X&dL)484M5JdJf4{c?I}VqRAdOjS7Dgj zWi8xIWigiO>grz%o@AsN^vD?P6frNJ7xd%Q)b&5DHaE4>4?R=Rav;zFjWWNP8|06o zmV3pME%feH>_ou$8U)H{5M6lLQ0(Fg9k|0Vx+V-}=&mpROmk2X<`$ZFfjW!gMAWE= zwg=&3)Ghz7hAB`(Zhb;svg~G_JNz|XFooAFv`s{^7;tJh!Ih4|+_Em2yA}>s<_IVB z@zW>m4Z)v_w5*DBP~o~K;VHmf6r0&-eg4^B@u&RwsirqQ>d3$Klx$9*@B|aSRrm#4 zp73Sv>oxGrdBSuY!@&t&-Y!>zGwBNn$&e)^^uVUWWE|Wja!H`Qg%x#Qxz{V|;xN2V zGZ^~~^GN&zcoG)iHH3;s{oNz*%Q>gOy()8lGs!3?F-->A&4v=JcMJvGtt^_lyjkWF z1lZcm(g?yD-*hM#;VieqbvFFwu>=kh0T8!c7QXhSdc1cxZgV$13+ND+>L&&r7KS&> zm~fu}qgnT%&pROhd<~IH>O`?E>;=qNr`z!E9JOYSMKN3*;yU-wzxj^Xx|IT6Vef0b z`UIItit-eQLd7A45T9|aKhnDllVn+gDDJ^X_v*9BQKtsKz6wf6NN|sFl9he70S{vy z=)M|SJ-3s{m`m7tH87Xuq?8Y}m3hMyE{ zk58~W)dd>Xe*Ci$8y(3dy|GdR;jQY?NogE|cO}lBB*xajxhYi!{#-<1;se=dFYk+F zP6q1OPSs`($1P6VBHpy-bl-Ks*tE8$j<@?J2%whFrm+*oe{3-xudRz_GeHtBwTs!& zA38e)ww?@zPf-bVJzCg-w|5TjPUxL)mglL4h-tzIhSPxo0Vk0#E{wV%xMBJhnQ<%{1bkP+w1Vq&F2&$mM6o)bGmieZhh zRbNV_TO(1kD+78H6eNby><3BoZ7E?-ASyLnpLHyHR}T~I#J zBlbVj@#G#n|D}`jbWeV`Qa~+^!$cJnmdu#a7=CM;6A)o2n$J|euUbgok{Elt)8^%7 zDR{H34mDCGbdQn*N)Mj_|6l&Nw9=iTqKp61wPz>W`#uk=YAi()tj}@0qr+CA;Ad<3faU{0Vws`oV$>(!l=9Zr_R=7|;U&*i zV#D4lyZwOPzVH#!W{|L~vrI0&jI2@aL;DWYENxVfXZbgM^d&|Bc}BABMwJjF2E|k5 zUOTA_>mvB|voGSavUU<-MA@0Q*Ior8#4=D9>|(CVmMH^dR`Y+Fj3U0aG@?cq|2>7I z_N5L^Hayx@m5;9q?<83PUZQZKy{(RPwe*RIdxp*u3G&$HmQOR_EcJ$RI(EEy6U$0F z8RpLoDEs7K1Ho6>Vec)b-o(rnCwo(`g=BYThEoQ^dEQC{^Fnf+yu@pvlU>}o%M|N? zz;+8SB__scFEyK>0o{z)&-0g5Yz)q$l2 zlZp>j#^NW6r<0@n6U@xyTLi{QPC6&e0`2Cbf`VFtT}AJo=Pf2&f5xf9OY@RYSt)#L zIrC{J&oi^LkA-)kne@yKscH&G+(8>VU|F9vB6416b@1O?>T~ zB}-CLRdxd3NPhkZmU!wgs=2vkxPnEZCcM=!h3?o{Wxm841Dm_0?rk zXiUhDnt2YU5W%)hBT&oPbW0f5^5m7=jxq0&x=b$D9kV{s_lnneLd7M=>_F~q*?X)k zos-x^>4u#f2Fo=Sk7_^re!O-1f{Ji(IyVEZ{akx`e5#Cp?}eWhtsiy-Rl&UycRl@S zC)-V@%%E6;b*)|T@bnQ2TuIphxf?EaQ+o3s--idD48N(N$vaK2c?VZ7oTWOgE&7e( zG~NcC*Q1i`K=!K7RkUc4Z53Nb#W3w!I6OQrm)cjTTPZRzQq?^%DmUDJFTBwz3-dDL z)EC)LO!)I$15>@a1?g_zmXb~#zxn^Ddh56*|Mz=*fZz)iP!Lfh3`$Z-X%tZD?o#O( z9nu(x7=UzxNW%c>(OATQAvJm^F>2JPF~)w+>HGJ1e7=A32M@UK`x)1D&biL<5_K8A zf2Gb|FPUI4pd}>*MbelBt+S^f+~NDqDF;Z!#K>mH*)Mx3t?|p}zJ4@G5{3NyQ3BjUuvb4d{x*)8r zYiV;$fTFF9{$M3=V=0g+e?lTD|6bY-i)Gh$t1_({UvuAOzMiXyL?c!b?Gi#BbBnjs z0`R`xzy8z_9#QC;ok&ZnNM5m(1%Z#7=7H5>Z&QO+jIZ zEAJ27_#`dmfj$2lH2h#j<^V+51dYO3lsdP@HXR`lK zNdnr4du96U+#fICA#suASevqiCHy%h-*&NzP>Z@D#PibJT1r{gpS5{`n^RV|r1}Z< zrt2!OEk9JXtt>WP@2dYlm{?#=QI>qoGoA4Q}5VP)}&c$+&cl7mc;Fuke$@ z*Gccck%J#gtymO%lACUPhduEKsHoOb)>pVaV!0-MIT^k05ueaQ%V;KJdM1NM&N(0^ z{ON$nL8Y{puKs${_lq>x_qpMmrqE-F3!hycT`BeXTi$ZK1bY0tQR;>VCV7-f)p~Y%o1u3WiNEie3FGNnv-qcEBV@pQju=yR}URk&H7wRCpo zwMf{#G;f}9^p_AipBceY{fdNP^63xZ`#YBjq-+|s|!XLw`+D!jk!b;yV(}&twT8Ry)eu?KOD%zra z2d8{FR15j&gT_pG@hq0N!1qPK2Ba|y--&y5`c$ud(pt|O<5sO7z`c#RJp)_kh!zg} zGcYO|G9luS?M1L%8#>7<2)WPf`&1N;mL|gnZgHcBUN(`3(bX~w2(=EMc$({FlTxsmnF&t z1alI@wf!VGDhs@)zTbDsj?fBU3|^iL;HFr_VSj7_{=_-|&Y9>9uXIDGO1*k<$x3Ft zA}O5C*`(h5qp$4WmD7>+2o&^kxs1$^Gddya)gkqMB`v6pQo&d&lNxxMD`6-t$DY<9 zAnhcYwz2d;N;rxSW4&-xbWY{Ps zV!h7VPiF5`AD2EX$g>LGs%#KRX}+W-IiCRvfD-o;U)+sArxkI$cb9kC2+_EP{Ss6@ zOh&>i+d_K&SjN9RZunM@XukHqw>R=`jPk;yd7lKft&P`#JdKlWZW3v=YmI4V ztqwBk1sZG`YIN!att(b&E&521cjOt@-YRC=W?+2s)Xg?vGv3Pf74oUD~>J( z)lOPaePvy)JN9#m(wwa`J~D!GohXEP%WzwVO*Kpys(V~MR7t+Jbs&rChKB2itNo`@ z!RD}E$&u1D@PRkB+X70EmLq-s5T21wMK|D5{p~9BsJBH~cA*f@nH zdB+v6oP=muRO;HXG0F97Pw-U?VvvS?adlOzT5gsv??sOWYxM369yF2yng{ z=Vd?AzMTYD+?;WGald3`B|iP5d$1)HpG?5M)!rn6fUQgpzN9Tl!Anq@sJMqz>#yRw z(hL@3oqG&denEF4hjPB_k4bz*Z(sjDR^Df*Z@YP`TR5tB28L1);&iusOiA60%#1Vt zq_^X;HY-}}aDv&RZn1I+N~bN9uU}{J5$=u&`?>Dyxk9c!{7ndVeI1l&QmcxqG4Ovtxk`e?5lqD+>i{Ty5Uhc+yA3DJc!_Jh+NntMT;y?cxML0I9c z6ftCEgJ9}oMkWsEi0W!e&$%NbI^mlHChastasRu8J;~61V%gJug#5e0RG$hS38yU^ z`N1v5z+j@KL0t%OdYfnB9`alAnddRg-K@ts=mQESJ}7|VgNS#2PKQg%YjIc}eO6N$ zB<3=I8{T)nJ#S#!KuoZu?6~=%sXMMI4dX?3=nOHWs^>-Wq#xMMbaX+5kaa1f>+v`3o$ov7 z1?-+ren8UE&}V%sAJXdl>4h(gX`SKFA17`&jeg?MVQ~yqzk+I~Y~p!dR3HB6#l4lK zk4bI~sLZ=_Td8dUtM$|gc3%gK^o4J$@c1^X3Rmi*e+LRY-OuPa0>$@T-;17wn39ra zRF*3VeZxj*&%@%j-fCou1R6wyoO;C|bpnl%T9f;nV}+2ANKbeEvh&ARJ$~Y6L6@in zevP}?CslSOK(MUZrqTkG;!4dcf3X?}-!1aFXPL)1+L}~sp~Id8OwdyOk>+>cxtlMC z1gqoEWh-``iWpf;T0_)Y2aG4JC=Z&31NH&FXXND96yb4uM zB`hn;mb&GK30v7+MRe0htsSak*VXkT@#EDc1lSd7rb@Z*(O-rPBg=%DT_)lKdTCy# zmRf2lrC0gXg^WDOr44hQ3WJ%}`PQ7hymsU}u)I!b$x>5Z{bHn|rOBZlmep4OreWM< zJ!ZCv%Y+t!tVn%wHf$1k03$dKRrK-O$#u0E9!<$k!1@tAl+XAlq@wRH)Lm1>2 zZyRFFjwUX9$rK-ob^u+OD82;6P9J@vqEDtu>uLOafe)yFx$`x%(X#{=(zS>21Ksh~ z-?+xx6A_r&w-q-;YM1Q?YJ#*!^7A+k${V$L7)+9I-pC{OXG!{T#nZ zZ*cY^4Uq%YW+}0vD_N7(Zrv-Z74Y|LQ&J<_Pgp{diZ?1%MoF4&?gOZL(i=|D9zd;h zq&A&3AcKKfDQq)BS?fzkzY0ip@Uy6pdixUC3nP3M;5zE~#VV(YI{E2%%#(b(5||dg zJxEC%pp@Bswi_Ul_BewkNMW`%4veWqI;QW5Gvg+`R54%bU+lx_j?Ii#%qJ>Y-i28U zrN`OoFg+KiP(sp`#cWDpB5V%ay`O2@H%lI}c;7)-FI!66ADE|1U10Q77;jiTq^J=J z((a9M-`pff$*#*Hy8TVk=04GM<~WUgn+_1)u00O)%)m-d7Ib8f{#GK^27J!c(n!*m z<`OmaX8nwtchpC!TO0bF$#wA@1&d#R@Cp)MG$6U{KR-ki#|+Y=Ha}V^84DC7Z4@ce zy9sRB7H|!(C|Em>%64Jo8W&C*dQg%)QiKN;SFJ@17Uug7J2VNi(cVi#|PazsYIS%MYmTlItlmd zBHKahxSKH`J)nRw|0C3EiLn z)vM@gPC-|wnnM4DTYA0w6HHon$eDVV*@!h2X@#(vMeccX8lSGI0e$N)^-#`xsX!b+&=T`rFh{hwR{Y z#GtyJXO<~fGqX1@3)V3>ltG%6GI_MHA7bmc&6qC|wwGSxE+6n_diYq!`8Kf<7S0C4 z!tsNe_dbc^5kaw_Xck2^<*dIfq4q1k<|s27B{B2R+9?lFnKF~Lc=ERM_f8;YO(dux zc2jcPa%aB23EI!yuuF)>jXa#Z^5oY?D|*=R-en=T^t%ZYy^3PvmC+ZCQDm_;z^J~{ zLI8R06BrIMnAyX^fNY8EaUQ?0fFt3JDlDb1NV#W{m6BP?ry|(&qRsCn6-P0$$)0oK zPUr4hQ4Ar5dRJUxH>`mt%9^CfG;C6VVnFZ>)<;`=>p-6K+B=;d3vK~j#+B**1F7Z2 zH4Mg39>%rs9VRlzJzfS?&KP-zf795htGHZrXXArn z+B{AtTjO%osu!9LAY<#cfbj5gsLO@t-;V#>a&{D}h{|Q?)$ILP6 zi!lJfNiejdKH4K7o0y{9hC?o1%5u0@)KtY!*wY_%PXBk~=)Px!%!c5!@IbEuRYa8< zX;d_C$V8or0CCD@WbgGJ&veY*@~_X{;KlTU$ro;`COczyN+rEq#!+HY@#O2V*@@jg z>5?P~#kvBcU3_4X)k?Y+wWoEN=vi-qrCR@8#Y)MDb?4j5kMeBe#ATmL`wHG#TytE5 zRAV#;I^8j^*b6XJ1j&O9b4lKTdk6B4J2q)o7dB0uD}?$vou?(HFi|$SM`LYgW{q}? z^&Zs|tp2#DLjt;wI2J?7rFD28lY&q4J?cvMBlePb$5@usS}*)CvP3U!X9)D8;CK`p zM~heI9ojtR|KA4(Pn;dNbOqqFHu+C~yxjrW^BHsPtw{nVy;nV2p12bZ4^xv#Kzos2 z>po*XR%zqG_#b)F8w`Q_Vs2AhGbvW_v*C6rTjnnuJ$P|bwjVk5=c!vPRNk-8--7hH z(c*`UbVXV9@boMm)+^YJ^@-%yxfZZqbli9u)N=MSJ<3b_z4HVulk=GG>{^dTH}gwD z-RD-Fk#4t{X6k$?srlN;55eG1&?>3E$LDU;XApZ{D%C} z6Z8DrtC5RQc4sl)EAq}~WTp+;uXsI2b5eiOHk}aM{5AMGt2#8+MYgqwW%WB`;parH zp;_>sIe70$-(x_DW7;j4lf9o`$C%>*B8u%b7KpRs`?o1x(L!n*`r|=VGomFBmk-2Y zW*p&UZ6?vOP8Xorg4QjjtWxf7VPj(?n0Z}1nZyJ*em>yqHrV)e1@U;i8koKPhU7Ch z%xfxQG}2o0i#nH%tLu|Lt_-QMjhxnRuA|M9Z})y3*rKLFjo%)c!dX#dqDuQ-%qj%@ zmZR{PuUJSGC>?14oHTC)iwW_%tqg4c}r{c3hAkDV>*OUvI(c!Z5X(BHGl7MCVrY-4^?FVsZqaeIZp zdCo`W^P4j~w2u)bpO-M-YcB={{n#{HQq5@!hMg;&xL+Fi;8Bscgk2BZSzkNTPcy!n zgfDgZz}}R9F}VgKBWLQLBmJbA-AKl?qIB##*4Bx`OpX6kWY5R%eI?ManTDRelG%w0 zSuK5+x6^e_)j&#gDVqM=M`JxByS?Z>VTm;z4*y5r2i?W@Yi;S~GBxwMj_}4O!TsBb z>m&DiX2iFP{Ctfpu99LFOEvlji_h}h3>0X~p_pR>ZmXw4Jg8exk))#S8=5%^R%G2| zN+$_KV;dw3nms*HRuTdCMJ+W+s5ST?xi?7wu{v5dBdwV`K7@ST-rkPY*C$cI8jbSk z%YW4>mEDD@8#?YAe0I7vo$a&5$IDXVW-Xv+<(2)1Tb4C5ZTNs9uvtq{`eP%c2}DW^&M_eVIQk2B$E>nY?o=e+&j>jx*y{4}G-7ouDE`I`dTz ztk@V|DWK~OIn0g{)@4F1_q)1!5oM$}<~szS>n20tDk=2!@+2Z1u0I3|!m7f17^)S2 zCw@DEDINU~E?cJxi1ts~TP-P+XAf%&{%v=q)&2`!zWUaFm8D`p3xKEl$Nnf5`Fww4 zY;w1Jt&Ck@VV|1Ge&v^Z_4>H=mFhX!3IEj)hg&U*mKli+XVFNACxAwuQMNTu&LUv6kPJy^d_8okDa7sj zn+x?*lRGv{)Rw>A7JUEyH7DadWZEj)Di_~AhrR&X@q^=P|G)x%yA2V|9q@{3P4K|GO_qzyQZv` z^bB{O>28@M-cRfjEWN=gW3{q*)-Je>2D_#is=*&hDs>JBSHt(IP-s-9+1dd-g{5_! zj1f8wU751&ivMFj6aBJ2r?PVNK1XdvjLORuR<4TM&acGNB=zqTHPY;&rW-#QYv6U} zE9ehGahvKZyM(Tx-wmV$h>c=k(**w2%MIo=GRi30kn&?ap{IE)b&3;e?)0;~GaTmP z!KU}?w;aA<818@aJ8LOSEBDARXWq~z{bCV?cHQx~byTPq*Y^8$&(b$0O*t743=?zq z-Q&^-VhMFi9Fx}*20m;SB&Q&-dy~?FI8-NgqOGDj+{$lUSa0+3{mgH+51H6=2&M@e z-gQD>^q9fBlTv zEZ8`x7*@PP%e@s=vf{i%=)zc}^je*5PvIFp5%|bt9giR#T9To-kQ6Tk{2JZ~NwCCr)^)g{vY*H@oDa;D?o5^H@j`z%U*WV|NY+v0{$XDtaSa_#&+Z_gvv zUrurFPGRiV>U7dp4IHIL-R+kw!(C_C%zS-QGPTtt6wV0Xk>CI&;|k__!@( zY(-|uawo^P$+*d|m^;VaOmI!xPTO_xZr+P?t|JqMvr7f5`!DIK`YwxKdDDESls^dHp`{!Uzg`CQH45~Y54lpm%L{k?Z5J)b%+)aJd=oTtD_aC*D;x^7^UffS zM6hzz+_9)s_gjo!Y?qcR6x#gU7Tn_-y0SIZMIT#WpRYmWoc*|HBF_&U;C7PWGZ^5q zsQq;Znkqsjb=Wdxt-op{B`bi^|5Ox5(yB~3hKx9K$p(|M72K@@zIE{SDX_P#rwB%0 z0G*P0P50Y+^v&eTfb!cuS?!ZevDkzDTI&YHvXI6C0*mP1G$OL2MN6yYe$_Vb!Y26; zVOfKrRTDHsVmt3i{zOrZjp(EF2gc$?Gg-Zk90Eao;_u4stf^)(O2;KWpeafZMD{qgg}0y`~$= zk0xNusn5JM4!0J=2t*NBlWvKGTYBZB5f1OyQrCo*%_8_wVn+HCMin}faC3L7+rLhU zTN;%i*9Z|M_ZyDeJQB_wt_e)xB<>dWP_nitWL|dU=NDpr`E_Af8`TStAghsT^XSQ) zlmqB4k2ElhSfoh14Sd~Jcv)C*3Q|WyC&d+GNB6+2wz~h!+=3&)muSz^a%~&K`0&LsCAcn7 zKhP+oA@`cCafjIV_QcGlMZXt!A&^9+?OQ#CI2DHuMckjvM6=7Kkt(T=!ynQerj%WW zeJ*=c6<%_m^Yq_Za4YUq1PF8J6>6BiK~~$rXW>3YWLuuuiY0j3g@fm7 z>WO3dFRz7%mt4Lmj71oWSzQoUQyL}xh8hU1mYWm=i8dp3Qz}10i*Wi$X;awYxL;^4OOCHQV!C`YC*$R$Xn7_;y&2(iO1EVtAQn~>leYO-BNlrHMJ~>E-G>l}90l5fVsmlq6lAZ1ilG~fM6gjR9mOBv z8olNhfR}{lvy;x3rx$_#B0=x-gS0siLd;}eXq%m?b>hZpJppVBa8>Si<;N=vS=YEF+h0XngK;6yT%r+PLJl!^=02w|UR zK+hd&53G&iLA=lJ-xz@l|D8}p99;diddA?RqTfLiTTao;nYPKc`?Jsh`%fQ0(|E0U z<;_^;-D?`L8%g0$MX63?bvQ+Kvy86uY5r4Jm`I_Z2xG($-QU?ANaDFjeRVl7C+!1| zyCPOykEDSlz7O;yAUSw(44a9SP1Ag+F*lx1Ck5QfqFmhaR(}kTy@0^+KKR;GEwquE zf5{tR34z3ot|W3vr$2izP*|jP%SQ1Q=SBV}6th=-$F%)fr;%=pz<^r5dg=SU8e*hc zk+%!G)t;pQsc-dU_4-G6w*Nn^@iVEQXe!9tY0lI2;A_wj$(QR`w(%4yc^{WF6m~LH z07!{)Z}|ks?W&-JJPuI^gnOW*CR)@P!%Akq%r4Z}_-DrS>|{1Qu`hs5ln#A<$_|J7 z3&d93kJZu^#}cqp_%5z4mZQ7iluC+ObQw3tOU0IR6&xQL{0k;*{7GVkK;BxS-qz)t zKY9gd@FWYC5Z;^Sdy=FHSdyFb%1Ys+d+&zd&X0ZjouD;^A=uNbfB(tPF@uMTFF%}J zm;$Pe3OVUt`H)u4$+`y|HYa7%;F3M(F{H%`_^BUefL)zc!*0DO7DZ zNN)Qg!U^7^+@-#rpJK_A2H+~McnTgC1kQ<3cK~!LALO5T0|B9W6g~UfAX7-heO=+( zhGPsNmOr5uJmGBP{H+dQJYKhY80Rv~h?PAQ@&aQ?_9hDap!?d+258Eod)onH$fOGZ zElv!ng`@1A-vNaQ%Tx*iAtaJgl(Ba)9DF+f#~gI!#D_oK?*>frVoqvnwWvcoS7E`$ z)w{=`AqDon_A9!^0tDAo{}VqXx6Nb`rt*-@pIDGIVcLG+@Ir z3KIe}yE|Ya9|8Q$RtBz9Lf%G;7Q9#h^>;tO(JsQQieeFaK3xdVd5Z*DrORyrInU1U zES+dU7gS$NQ2oqsTB05JzDVPV_T6!7>BP&tk+26&3!oXZFVg_cKH~zX?(+QKmj;>b z2W2+M&`AQ9$;7m@30m*csvR-?fH#P)Nz0<1F6zzCFgzG*tH5=vcN0|X18>$)tRs#|q$2ee%UxLrN6uFLtNUZ;>pMA&Po zE)IE?WuzOX{rY}YzN6ivCuh#`2?21x#|dL|ov&QC>)$I-=V{=9_^yxtI*xKW*qE_D zKHS!?b1{nQ;sv2W!$mp`b$7u~5d5#Z&i`-}5pw#F9GIGOkcpUkyd)1q4qN_TdZKrJ z7gCcUjdSI{c3&VBfkufeHOYZmxwzZBuif>92c9&D+8@XS=IgPT4gbDl7UJ9Bn{`rh z+~V=M59<@2oNEs_;*$7mugC8!&0rtb;U|G+95p&*#uRS*`#|&w#wBgY=go@uQxD9_ z{1E;?&-?Cn?X+2CR!CUSwG|1U_aSZ2l0cX$kt1H_x4l&iZe&8`%FcmG2{S7`5FrylhsyU}O)f>tXr>!@lx$MV2iO z)54JYo`0HF2nW#J0&|#0Ki)F;`3=Kz;tZ#EyGs~ zZ6Iq>%q{ep9|5#9`3L1+s`n;~*v0w# zfW@%psQz6)x}uB_aZg3x-eJ`awYGu!ES@Dx|D1HcgZptV%cs9n=d6byBA5kWJj#OJSH|xXrv}a(rj(6s5>d zATr$fRoLDS1@gow>E5-JTaExp|IyXUo59k<)E|^eAboiEQKUhnLB7&>8Smmiw*)4B zhXRq|=mxbT6IZujOC~BodrHcFKi?G`d*u?_S3UTcQ{UZB*M&%#P3 zxl)+DcPy^-y1iTU`oB%Z%}?Xu=PV$@Jqa(cJ(pBVrN5mKRX;h|4jsAqEsdVWkW$-nl~zFQ`* zuD-MG>`;w<);n!T>Qj_j2fS2aaP!z?Nv(No9DIzboa>nOvgB$)7L2)bziqs2`Z~om zp_?=Qvczeb^{Gf#4KK%FWTrROvrzR*Z5ivx=!Am;M8WTZy9Ggtu8%?zekc;#~nlG`{Jz&O`Boes%m zCi3byKzk0!K2zE9X(Y9^%}o(7fZBvu1R!WKs?-J2#pfku<#6j;-*1CN!xVEIGgw;ZN0gI5$20jkM$v8bm+F zRkqx5e!}nQ)i>K9j(Vq}OLVN}qrK^`>?lj0{{d%# zV1U~s?_99JT&wqiViZ2#h)4n^b6oPokjP;W(YgpcJ#TjkwoU`hzka2)dm#7V{ z<{zzed!I6_MM=>iUxY_qo0~z4u^*=PW7GJRM$anRT(LlJ?003bqy>J)TDe_a1=@fI zS6v_+I?d&#-lh9m$nH4%J88qF(z5BkKYu%}WtIZ@;{u?dv6zq8iui8*drSKxGi!a* zOMoAhR5YO#z6r#0D^}|KJ-a*=_CK8`o`4zv?ptr zrqBr&(Q-W&vY2(fapQ`#Kkh#~cBgsq2Y@>Od365xU+ZAq31k0@wI`e!1SW@YFifN= zK?2%$ZUu#!rV{le{`wL~X6x||LNx67Spt`Cdz&rwMaV_oAD`&zj$FFVOks6N`j`~2 zRmR&;WKdwESIT`4Wje@@Bx4k(nq0zsAToPzn&%@k`v#gZ*bpp(`JGJIKz?RYm5&-l z)A}mKx5YY%8vFfESpBde0sZT@HBW{c%hH%yEgu&{;YoK0T8^n}u97KJYjw@tFzV{` zS|6?Fes}nV#r1x=%vYi&ReMfk$tGxkm2dENrBnhO(IOaOy=`;?{1L?u?CtDmQf}P1 zu>dUV5<%qD6b=w`LgXR%mV?>gYL`*`nM5#0*yNK6ncIdI&kykQl+K^_uA9y|-5|>a zxo8Z&S(VgZ)$fll3DN4(C~23;Y`W^*aarMj$z<0Y;lZpxQR!dTudLn&LaP#0y3&bh z&CgG%wZmr`)Lu)j+XZKbDvdr2abqSWvjUAtyRg*4p*hf~d9S_L%nx~7{2p!>hdnVn zUEgZ~M97m&r@043l!A;Z1S)G=?ytT|>EH9GN)Gf#%ggCKQ_B4dpbppw5D}*%nfI#x z9ZD@C6iPGXS-Z1;(|18$W&+>^G=Q(x!lx*X_+_WA{Q}_omXlvts19`epxjc=YpMUV z0ThbIK5muDTl__fnX(t1uDV#AEKH~jzYYSFJ={f^Xq0l*S()w{8ZhJbYbEPbVXlbd zMgxq+QnP_~kV|%UPTBx}p?bI_21pzfByL(2ZlZ74)(d`=U9C&d`tp!#e;vP#qKh3c z+1s8+gx?{oAZHIICKE_N9qx_JD~Z2;r8=K%%O8NrMLI4U;p>Lb9EUZi8Qzx*F6u9Ki{Jx@33{)<`)#G zSz2ad9kHpuF#+4mOIvM{Bd3|PrKfX-y;*&SQD06zXG)G_f54rR8x?>BWxT7IDbjfa zEAi+Bw|B3Jl?Q*~7nS7RbLAN0iJ|NDQ&MB@x9Yy;X-c&5c*@s!H@%;2QsF=KIvvtl z0B+zw4%8+=vWJYQ0ED@~WHD%|O_|n9)NMt)#vST43;3H)0E{)`A8}Pi-D%GaYS2wW=4cOR&pfNVuLgQ6 z5-D-i=Ig%V=ea=Tt1fh_QwV2iESsmCbmx;WXK;oJ4={E7IJ?6Tw>;FjH0=~x;jIr0 zJ)VJH!rr`6mTe>0c~mpa4>0r?+0}Z)aB2)sb~0I|GkWY%zv9{)-n&s3pO`%HMX88d zchc-ysc+nsc5T|KjfLwLERcZI+CtON2>=c_Q- zvBY?e*^CWPPUtb5?G_to{XgBe(K0Or7~@ullfp_Y8Y*7B&Sp@^y6g4T5;zMp1MB@n z&)LD(G84jjTqcOcn`y5kM<6X`gz8cIpU;KB2m%fn6MLANYo}d{=uMnib5lGho7A05 zgryg41C3KSsc*%~eGxY7L zUj9I$8V09ysUn)kPq!!W@W6K#_ZOG)06(TSYbIr4tr%c^6-ZR%8eCp;6&0vYPuL|0 zKBm5DfYhS>{CWhMFmDRhhw!%(e9yEn79*l)8I%Dk@PWO0w`h5O9YK=Tk)iW72C3D| z#q9?d7NHGH!}@Hc9Z=*ggX^fusJxWqY~j<4+V(UDS>}6dd(U4ry2Uzs=xixeS7`c)^yn7f!dE7)J^&Nx{aDE(ke10#Ysi#o1Hk`M{mM! z;j0lJgcBS*iYM-8TW>e3b;x8so16{+Q!c*NdZ0N&3`sy#HAplv@z0Jd_fFKJ=Gu&L zu72U_L9>NB+md@<5tB_`{~k5>GB3mlK^iSH$(Z{6`}fcn+lRLd3g2sH$w+i9lHH81 zNPFkIjaS{{2s?a29?=4fA($GJ99ew%HQ&sR8;D~DsZ)kg2vA&6@wr_9O*ZgV(LfaQ z!-Gx=$G?%AYWjl@q<{iIpIY*tPVs6)!*538TwBIUW8LHeRl+73^b{sqOcgfn>W*dK z<$4#hAvxNkc+h0#)aBHb{9_Fcu0R!a*r6$O7 zZ|aCY<$P1RXwWRmk@cDuWjaDlEkDp*Chn!9{3cWP!4!$IEa<+lJco+@Q;Ljx zGa7}gq}e0(g(PE>N0C0m6{ZfH(>&-+bAdH&w#0OBzuzO2EvB>f=esGGry*r)l^?ig zdRV(EeJGSxIM_I1&sX=n5(ve*!F`t;8JAhENsYPO0;3K${k|fW@O55KChf1gdXn<7 z{+8x~aYmd00rN3-_{h%lMfBWF;DG1$M19Zw5IH7yEK3OeIDiPR+l1RW&8xG)% z8A)x()@#X;BOKs=>_9VJy*Q#bMH~}#0VI+>_xCS1#+b>=VuQnd`iJGtjSiF!5bV{2 zWPS4q2@UCo>x-~>n=KwK>7NfWEERALIF>>oH-X+7p8FUKfutHAr%@2+%_xrnf5_2V z_L4}IZ|sx^c6z+|r(gPF;ycq=zJ!*%@I2=sO>Op9)2az>?iE?h3o8Hq`-@(bO2ESg zL^I37>5pW28BAk1PO@S9W!>gnkPTDv2&6uF(R zV2XZii-U0@eIc$JHQwloc=VYq6MA~&=;mQ28T|uYo2CAGA-%U0ws$*+$LcJ0@86qI zr_6bOu86km?BL)h<^DZ@RJ$Gg=<3n{?%slyjk?0{1;!LBDzf7}$bRl-%Z-gCTUez_ zd2IY2mPFAJ`Ldd$lPA{$ zosMprWeV4ix8nK}7_WZsH$4n5_L7i>Vm(GZhBvo<@vm>Gaw(j$>Dmv15qf=uf|w`Q zC?bDYfct&=<^Smpk64xggf0LA13rR=*bG>PdUawrrM*?$;ft4;jvGozQ}s}dEl%!a zz*(L;eOmVT93+z$@@zgJ8cZ%=N)L4I)~zZ!yXUBBUD6?s8`uYj!4^>aM42QVz2O*@ z;eEH;2DEUCscNIe5(k!(s~5IZvVAwGbq~oP|J$=+ycBNlZF&*JBlq zieJ5YmD;tA$(^osPS8xfeNTa~E|$Pu8S9*+1BM>q416V-;S^L53@s%4$m+%&3041$ zh&J23HlttRH>B^$rD-1|NHm7G#iN;);0FhL2_^C#2?N6qdpcj;ZD@_rHoFb5$)!Dr zhhZjoC2u(gE{F2dPgU|N^6A=q7wdOsBZpz1Q@ihYV6#M> zfOn)_wP@IrDfO*Vj<4aJVhdIgCfL5Wh=+(DD*nR>KXst~X~!b&KY~<+Z2w^SJGBm& z4v|C4EXl_pGy?z_H)n4$+^6jLfo#A5^DM+C<*2?zAR`kEIrJ<6spk}}%3x*U$rp1J z<9s#EA#!Fm$5VaiEJo*S!xnBkY4!RA;g^IUD==#l5~{9!b@P7!F3YknWOCm|faLmG zm3ds|UxlTs0&+A|alp#go*b19ESICBqdy)VJ_EDU32&JN)X9BYX<0=)aO+|ji_?vG z1Zli`q&LC^nTZ7iiDVLKvz^p-i2udP%ttT&91G06l7P=@FSCiI-pDr!NJ9vVbh7Wy zOM|F2P|#0?fn_~ph!5uH+`co48NlNA%&AU{lzRu`XVn1o-qIYF-BRr^rrLTy(H6F& zju2QVagZPXA8_ivBog_EDvfmxbr$k)QdIKVwT`~#&$U#qXUFm(1Y}%8Z8Ei7|DPZ4 z7D3KJ5->MmDFbKnxbuswNZhH2%z+C+_c)z3DofPUT6#+CTm&!u*G6$F-Z><4ByRML6R@YA0rNv7L zKd8@<;}2$hYH#!BmUL3QV78##wZZFEIak(WYpLtYWQ4T%77nA=0Q#C zI&b>VZB4x(U?%r+6t16U{KKUz`rTl~a%)ui#(L=iy9Gc-&)GNKWXS6bqdfU{BXHlQ zcntd59Z=GzUHbzFUd;%_23#!AxU0wSt)ZqSNi3(qCMw(s2R@99jEpxVfFa82^(4O2 z^lWipw^);1sK}lr&(6qqAR)TB)n0wW zp?&>sR-IqK%x&n5l~Iskx$>I3)UVBkhm1P1Mw>ZRM!TExKKlA%Lq81gZ(VR2D(P>T53=a01;DrSaN!5Csq@5!!OVj!11qYZ zU9p@}2%BJR-B7t%jogRPJhcZPxu?ub2i)W2!~|cx!HP4$Y{@x$)*}?TG?3-`V(X~s z=)}tF2Q!)XP3in?k4WwT^n$Im4pWvd$3BQ(X*_pnO(9*>{%xSmw{PDHz=9mAdZL_m z$M*HPE27`!FaU%VB{%{xMg2Az)p9^vtN~T)s2{+35G^bP>Phhy?jSoHnGKC0TbG!) zjFu+N?;K4GVk#`*L#KN7nllbo*UgF0w(v;3zN<_zCvy zFZ+|z0~~^P6$ofrX?Cfg93IVOFK6}~(0Ha47xVdOy4;WSpS&#Y6z97()?dB}1WA95 zxA#^Fw|$p&r(9vyt&{5+i-?qrTeo+`97x$qdt#=8^HkopYbBp|I)75{VB5>M6eKWQ znp$TD5&2fG3)_qBy)Zv>vYf5&7~Rc|Bb>PXfzO?6<2s2xouEq4Q5WD~58ElbuR9K3 z4ec@}23ZMl-B-smBY158Z6u}WxiM1K)jLmL^<_vQPq}+vJ#1Lm!$&k573+4^KIRm2 z03?9fa|OB%mt65%w|Y(~2(jKK2TuS`H-i&g4k;DVWPH9Izm&mq?Fd9&q$`#EICJg@ zk59`jrb+i`#!A*`)7>}$t~flGUHYjiDNN{6UF_RvcqDYOnpCfQ6>=-Y56gR~SxrYF zwKU}tjpgMLL3J*pK&{L1k!;84FwNQrMYpvEuFrM8+yPrR{f_f_gO~BFuSSV^1_8@Z zx@i%sCPVke-!x(mFbla~Y`fCwsnV~VXJo&ATBoZwwc$Z;>X#hE+Gd;3h3D7d9A!hH zM?JhbsgNV?6w9YCc373h?kE5N10I&+H{F!E(G1K@wKv)b?%R8NN&Oj82ta2n01HL0 zM_&gayTI_oVZeu#PsF4mQ!N$fNJKQhzG^$5V->HGSG<0`bGnk2ok8)q%1oQ-G&u?* zit7XLzfr>sWGw=vb_#$VGger3$N84AqdqX26p3!wJnGb0_Icpb27R&L1cR>}=SB+! znSFU0Y32x=bu|CHrupJ0+Z3?r9uqnfRoS|q4uVqB{MN=Lw(q?}129IiE5`rpAqcF{ zeDSKS%XkZoZ!7^(Re|kfS4W@r4UrVq)-Av9-$LX@=ihldxk07LXWU6GqxUI;0xkwE>^ z&5GZ~%|=+H0SxK8j~N_?B6FhDtHo~hm`&lPjCOOh)j*|c#=mU2*JkV4?qHe^7a1oV zAN+HW0ipyx3Z=3eMxe-~^>>jdH)g=_cz(L43iBN;waL9Z7*Q~!RK`4LQT6z_VTGj` zN9e8|)A-2a_MwO+U`1+I_M+a2@;S|~x@t>-&k}qtzaYglVtcr}=ZbXST{>)>B@RHK zhn7S&%_#}bQln1r4&%x0t{B5g>zOAltbC}57US0z%&4(LiQ6_nb;YD>!WywY)ldne zdQS9_ST3P3C3F?A2$ zt*W6SZ6+pl8MI|?7U+v{?KG$4yFZh5zG|g<lR zzPPKbremjQU>#$nZmv12K}FhGF0%|Xo5qw8G$G*h4fSzzf}mv{i0h)813Z|{p;k9R zFx1a|)+R-X66G#u>Uvq<<;OAHm;+s=`@0f#H(!NrwddBGIOkf~JnVA9+S2K|=9-Hf z&8{}#hK_7Mii zSAX}%oBfw1ZhbYc6!ySAY#pdYNB5Ytu10dvwKHtTDNHvhr8m7I)pejYjdWoVPeQLtv0&@zZRae7z6VvAT+%t z@1BtUM9;eYTLPe5b&h**ouLBb;T!wg3#Yd2uAdZuIt7aGIJ=cowk%@s7WdDr12R`a zrrV6Zv6c_!_ERy)#TD5Dv78HKwDCJXuy!CtG`r$)C7p_`rqCvd^X1}=CD;{-$5Nhi z9fE6BjE`%9J|8T;>?EEGPFv_~8d(X?Vk??mpMrb3T3mzbPoPQeoT;@8=Pf9JDeTG^ zl&%&?jI4n0+%aq{x0S{NyW-<}wT{?acI<XNkD z=Pnm#*w&}~%=bw>Lif4CY?dU;%)1P@cHLf=msqNwPMdfwvynQTmdqc#zmfE5_-C(e z3jJ+pu@$%xVK|uNQF=)3J$nq-tIQX_k4{`h+e2@gFo}snU<6Zv`lL3NNxJ(7vHtd+ z^u6YrH5EEk@UgI}rVOWoHsf}S=Wa7R7)UiS@%#il6+60OR-4pDGXk4ySB(`a7&c!> z=3r=ou@0h6{YRi@=~d?%H>fET7RKx zbPNZPpN4Er%2xK|M-Luv&bBUv$#2vt-vfnHAI8jL+f~;_F?fKMAxLh`b(vs`RZS5yZ%4I zzB(?dHEMSgqJUVK7#Nfwph!rAO`}MsiUJ;B!?P0MWj0h7!ag}8oKkY z?J+pt{qFD1KSz%YGkfnh*89|gqje>IhQVPJkJa_`rLDk%b}!_)3Ei9#qRJ6YGKyBy zjKwr45OOpFYZnrWajjCtg6$aQ{g<$+6m?Z%4T+vg$9<{fNfdKVQ7Qp>#+z8rn@JjN z(H}CC@oh5<8TfchV%2mwc|+$DEsChd#>P)dPwT|7W2p)TDY;Z?rk~x{NUBE8yi)wC zJ6|s~5xqkK8 zZX9ll#?1=hPU{#6JHN=p;YW7z?n$Pm4&&`V)*H!(36ctFG}@a%@#vcHpnPg%j}4;4`Zs|J!%F($~teubvl-bvvhhfg>`y<`i=3ay*{L6l9?*aE_z)? zUBYsO+w=GT6oYHv*8@IejUg{adk4lZ;Re&K5hj679Oj{yiG=05*%%Jf9@l0$fd|xg zxO5A<)>!kXF2O-##Ok=pV=Pr2C&SIQ%-PQ6FJi?;spPR59j4nlNZwwY8W|Q2h4L4o z$(-oeb7?e2GbkBET1iXLr=@oP@#IfK6=#toUn4~U=v_o z8rgg88e+v-H&|k%(yyGo{X;fvO`Ah;gY&IpQNe>&8JWRs-e=OX!PWc`Z}4jd*(O(# zINSK{-c0z!m!b{Tq)88j>L-OLvt+)C@#n5nx(S7RWLJ&)yg52WC5=*aDGCm{?w$k) zJH3ZbQabja#ep^a{mb5S3|(2`Dm2Edb+LOp)5V; zYjoN3g#SutszOMEM;6sr+fC!g1ryxFfncbXo4YbUQ8d`rMv4-cb{X#TvHtRes&zi3 zrQXl3FGWmAsLY$|LwRm;saug|l&NYZmmo7TK>S&InnVT9@md$!+Rx`E>*_un2`v5E zgST|@`|`fX9pycboMieH$DE(Gn?|Fg zU_3r4sW`sw?L>TSj_X^yhy*R~at+Z$?MqsFzt}e2kp$|p@h9$*)wvKQRU01v zRSDd*cIpLKJRH86E5tFke}=NcGu&?k#fbomLZnH&q-~@TUHiael_$n2+u3$yXy|x% zGwjHpYm(6xyWu@jrCi)4ZQK}tEolX3o@d64WXEUVi5L%6ju`H|aUQ}tdJdGlp0w6B zuin}O<1?M}W{47zPEIcyN^gBgTl+L$KvhDb$WnQbNtF=ce`L;$+?{7{pX0-koc-mF z*H|H*ePetdqIH};Xw3|3=iYEKJ)yUVMQfzrQ4UEV|B?JHq5Pw~a`dXS|G{xzQ;aYG zbz@cqLdsrEsiSj9&fO#>`g$$qnqi_6#cW-9E{pZ0qRqC#rtb3zg~5`YOMWOXuyJc)R-LH1oNWi^o-d z&8lcMCPXc;sgni{VCF%9gBXNjIuFog8cvOW@D;OKobu30_nHf<=!@4bOpAX*;t1!2 zVw0!!JHu2uR;!KuZW0^*s~2dih0c9A0RE#kj~O)jotB4P`Oq?JVoKKB9>vD;_-#awW~Ubow#{Bl)gu`(tpQBqtE zGv4<4O|<4vRAX>Bs^%I4yo$z z5YJlU5O4ys;{7e^VU%1JqeJIaT zegL4-k#t5If<~4Z$?9cYv&aCi9kD#4*|RlG92+Is9R;P&n8>?kpZ3j{evrULTeBZ| zETubH`pUg}SSGo8VY)!0#9S3;-jlp}v(IQ+-_vb_C3~_fvWIf*$_FMuudBJ3AHM<; z0n`qP`6#@ps5K=_J*5NKQG;F>)zxzyxT-oISBaf$T+~P%nYlAVFC25?@h(Eq`v{ScH}mOH}69;ze+wq7o} z6n8k-taFWV_q6~0ccBCo{!`lT`B0lqg${sAiH4kO|A@rCz1 z#F=#PD79is+~hh_hBxphb6v?j>g=(Vr^D9nx+ga;N>O~zo#dmI(44`X_Av7;(bzQD$x{o^qo3xr-~OoZCouUY4n>97T$!_Ejow&=&AXUn%t>TFTb#m z3QcS<^{NFwF0Z62-INB%4W1>BwcuMvF-c2p7dq-WlcSWAmhePFJynXtPUG+Qn2`+g zp6_>Mg|2xO+NDMLUS;|uqh|HQgd@r^tb_3f_tP(HF-;@9BU#M1f>b$}FPw7Q=fZ!* zI8rvS{iN=4?De6{;8zO4Gt(nY15@f9YNsUZ{U|@h$kfuurExUr?t1z#Io(G3$)kwc z*9R_c`0>$c$W&MH$(`4T)o3uSOHRta7?JgT^ilL=pq*h#d_UiCg6TA$2WFc(?zuKG zVUrl;Uc;lZEt{$l_9FA|v`G_L=J;ggo?LDm?ehaE_^V{DjOIYk1uIDnuSR!M&Sf`4 zV_;S9uTEMhJv+2iRllG?y)#U22L2jKV%IIq-+7c@)}SWj?ullJbQXr;F|E(gbh(e3 znwrHSN$G*1gw{wQn#BOfsES!0Df1jP7i{yDWF3vJ3H`10QFf1yoY(VkESkH7um8u6 zJ;iHmp>+nl=~imKwnc7Sr1E!BWJ#m=`>uc_u}GE27vt-b`MSMXiVb6Txa!WX{w%?R zq=fM|U5ppBdAiP?;=Hy}dxH4J{E6?YE-VTjA8Rkj$m60p%L%-s6c3XQ_h4K zDyn*a-Yq)!nlofn7OGs2cu76Fwb2$zY0Y<Q#Pw-$6esCrQVHN}^Qc~u5*m{5LD5ZqA#YB9Y~o#-yrjFFxFXqPY3_GN zJd`EEo>@jK8aw`ME?8MvTFPZ)R&vN{I<$2&!k$&8n&RQ0tP$h1!l4cwE(sM?aB1rZ zZ~M$1d;7uM(Mjp~D&?BO=2THsvcCTMC`57dS!ZF$%b_O@W}hhURPuqBWCsbSCro5nS4xgX1&;nn!b zH?Czlh1?9;czk{R!YIEs=x%tdhh+U7nsTfk(r33#4^MwuG}3dO>Ma?%l(Q${vbOhp z81n9ZlkLcK^s_T{2T0O{KFnol$R_Au6m;zM+^#>`68Gu8he`(~mO@kv3pD7{>5+$W zPSh&>?1Yz*oiL5F^seL7XCuHs#jXfaM4 zMdc?zsB{bxhq0=r?vY$aG6^fR0-|TuyIlL0vcsKyN=^69npl(irmgWryz9I2=Hro= z&10%#RZd0g{11}r7qs$7HBOpZ;u9GW9X_>p-*TPj&oixEx4u|6Ni@l9b`u^i53m)s zniU(gG&$rD#bX{Q`#^e|0o4*;wz+C07SDJ23^Q-<+MS;{<#4%*`?7K4QV9oer1Uy( z)7t*Kvs==X>s?t<4ix#+pJ5lKi;0S6lz%qp#&;ggB=2>|ZE2q`Yk3$kI2dA8fZuO*%v) za_FkW!dm~f^Zev#5}vd(v?<%>>QyIi5IW~?1nn}wI(OJpBU4l$C*R_LC%G!vZVbbj zPCCzyG?SmmLS`sz8qKsx-j>r!()2q-+|1w}XYOS-Xw?7$b=cD<)s7;RKlM!{0@G}S!=`ooE zbUY}7rKO*{EOb0H@EOZ*aD2x%g`Kt7TrAqOB`hKP6TxWn!Y23=cyRA&IKA)ab~ z0j${O?W%ebUF@$xj1aTe-Sutj2G>6878Tlzwn$Cn&&qccy5>PY zfOr`zq?~W7`_8~wdOq^@oh>HtNdIthSxwE>3F3D{l#u}nG;ZQdKYVyAR{xhr! zk>@_&CAaXuU06f5dp{*G^Ru7EVrZNq90{y%38xE(bW=rjiAsy}(>TqiSVte<`keVt zLMVPnFmu()t?RC1#TT7@to*FsZSPtzWYwzN0Guc)n^K#n#00kU_0*8%2R$<#P#b#i*e-Kdt0hX zfmhTJvY$AIZhvSeXV^MjI>TT>yK`rN)nJtr%wQ(o7UzWLWaiD8CatICkl{Op zpFj}Cm4@H$@m}%Mo!6-I@nuJDMuV>N%?SQ|I_xP8y1#jn-Y|j_WegBlXl4L0 z|L;~9D_A4;mDbnS+nt@5ZPYaV{DR!{^R3&CosQ%avo`6GO)R;_@8p(e?xdf1Jfo3) z309ZrDa)-=ZSE{NGQH{fT34SE@JbB=$$***&jO96S-#DIoNnma_3O`B-_h|YW{XUV zkS$LA4@D7asH)@#YIhu(S6oBGlu=R9AutU8mN=1)BUbn8!Ucg`z8I&R5CrbNTcGG* zLL2PBss7M>12VAJ^fOuXaLdjYC;}aJvOsv)$##83=-1_X>Fo}05l>17*Ir!oYaO@dp~f&x^75{=+OlsvfL{JA)mq9Jqh z#fqp&+rJ6z5GL$Njp({eKL!iA6s)yf&@3bEZYfo$xO*XK8r3GF-C&^h#&zStYT^i8 zTRuXQ2&>*=vVZNDR}nLlE2LzaJG5{G#W(<0)8?w~&RicW+-LD7X~ou2yWVf3^z$$0 z3Iv+h^+a&!;*0}xB1Ih2w5)6SAOQ)sI%vUR?W4iS$hZt=VY$m5gRo%-K{fo{W_yDf zIU<6kn}>K=%WAC_5b}A@vff9t&9zk1LtV0`&5vwOO2f2>vT-~G{ER+~*BMah)z1B% z8XkT1tXG^T@}6Zb?1mWtT~97l-v?Co%_q6HVL_Xt;+xC9Cl5vTZ-#~rLGjGWUg)Y5UCR}S_qfPRlG#aIwQnz-2wjg4`%#w+_nQ^aiR5Q)@} z)I2LP?HdmjOs}j@tT@h8g{k}bxw$#dG(TAh3{tI70$W~{3i90&zEp$W1|(S3T#JyG zLL(Oa!h_f$v-qQy-an7F?U`EShbiJ=&F#_m80`4Jg>kSGySU`X_1b^pTnDwDbMt~O z>MsIqefXeb*vS<09x+gP*g}q1_7QX_FE#3klezyIzEfO)4nS|$nG)R4w`BbEerKVlbAVF# z+jlO+(*4qf!bkKutHRm^>AE=?mx-cp*5t*qt zGnC0fee$B?y@851pYC6N#lRN>WjC;f`_Tbiv!BLSbRopFh`bEFj$apmAiE*o(PIu+ zUMAPog+P!^_#=AC8<_K}p4(5nhGpZS`UDW=f!i5f6n8O-nM7*F${#;FV&mRzi`V|C z1sc0>|Jy%B?n8uFsRIfk(~qF>+>MRN%?<45r!)gwKg#1pFmA38*+xZG1!5Z67n@-E zl>vbvwf^?)+p9re#SOzZwRAShWk$-VJ>}ib!{;S-9lNBLC(gulRXTTbW3h;3NjJ~V z^oXca2Ims?yW6U1oO=4FjwqPs*WUR2Qv0A+*8BK4a>vl{v&+UZ@n_3S(3gFLD1+Ns zX4=#=G*vTQ!V}r0TD^v_rJ`EIP(ml)yE}2WclmCoq^tk}O6~ZeM8GI80fc~B1D~4pkxu&-1k`iO|LZAq-5>4VS4^PB1Agq z-dm&>CI+@}B|g9gnxyh6ug4qIh8YxP_YsM#unAtwS{&~_)EHU3SDwI zPep%*r89K#A~!iX`PZHval$;f*9I`>a^BUd-!=Xj>Gp4e|Bl`QVq zXiJ&O`UdYP5&Uw(lCEyAqv#F&#m$y$LC79PMs8U44i!YA&HD;r88{nfH{O0XPdmqI z@L1H*y?cLH<*Z!-;QE-b-8qHoOGQIajIe{VrKVX3ElK3UcrRql(|%Z&1bNti%XTJM zgU^Ce#BMpnr+GB(wJOX;Kd5GN)~;9rYGgIs@XjSkHBVH_Y2@ROorOip6$>-iNLi#< z#CfF#KzN$wN}ax$hOK0ID_~xScRF+C%wQNM*>brZi17#Ty%xA=uZnBkkHj zZk=<>FeB|l7b^vCCLQPHU1lo+Io3V-3a9bT&BhPqUtS)$I8&h0U6dOYF_J25+L_^k zg|@AQL)jMd<-O264E|BnZT?c*`dIj$3vO#mF4$n}NBx4A>#3bm#TC#dTSJ%XcGvAC zTKZIc75tK3bi=}@*ZI-o$-SXVCw{D0g_s5)NRNorRmnTY?wWtkhZ}thamRF7mLF&F z#IIb7dF+ce0t64Xx%Xm3q1@Bj49lQTP&@8 zDz0jsp|^bX-WSsa^;j8}Lc;Ch{v_@*xoaTy(ev_iy3GCzU6JyRGcbhpTay%8Kx%qu zJ~zs3B@S97_q}g>qePV;>Kdi}K)<}KOpsD>UR=RSBT5Xfyb#azA+)1|L@(Mljvr%x zp;^?D8*k8NnWtfH7g(RGpr!m&xS7;2Z>(4^mh!Mj&hdxw+qp;3)U7v#Q!q&&Z?UrT z)A$0mFmv6>+9^tlR6gD|HoE55y*ltbSv6{m{c|K?@ocPV)9QG6ved0ZPCZsa3-VE7 zBr91sYddkzi3v7a=5&^K$O-rn?ko6mYGYYg1nuW5s#jw!-OyS(cGo@9LM(QWiug|= zdXtX*INoyx{!q5~kF1L?+^UT>i~1ghm8?WDfph21Ro{mp-NO*MAo}HqJI2B%WlGfR z=L`{-*{E+??PKV$t^46jY56LA58lFD>r9c>4F?)U9WTS=jovo;9yCILIndRktu;7) ze3%R}7Jk3j1iK=UhD=k<<_*n+S4lzrg4z%hcpAi>0@Z4ZqL9mQ`*z2}huUluU)|iM z$A|vy()#ba9ay7Yz8GnaqA)#@K$>8Y4`+kk=l7x0(`W*A0K~O{Xs8lMthksvfc@YG zg(Y7Wy6`)Kq%Yld!h?*WMobBIr400-vFz3jqU$QqnayO>0DYj}TkN^Zx{7EpO10|P z&R1Dtd}u;s_Y;;k12);YVa`=yPeo%#c7)@{jO(m8j+BMMBTswGg-sIG!ac69y-}xu zO)za*-beBmwJPmo(GKQQel*KJB19tiWOuTtlDrZhkW)~kn{M9g8DffEtZc;?Wa+4q zl+v7|v|#tJ%8?WD`TF&%h}_+KMX1V%hJMwwiSX}Zo^!Qs4bo+7zJFR^(jPvhzovE8 zR+2Sj=}#f;~H)2w&%+_Jb#AH!EZtCfuk!7r|x4@ZoRbKsoQ;1NLOje5nUfufnscY#0_ zeUK^hKsT(B_S6pnchK$M1@xjr3(zT>4ZYc!^`ItA06#}VR~vC;me#*ecVTk=mt_OG zG@19WF*mn9+b>c1&5<0Hd$Z{*xly5bcrpD{!2#e#-jYisC_jV~Dz>+%DVYqgS00C{ zUMWBldfcf-zxQ(S#mBS^m5N z+cA~1amM&$PS72k&k`{IZZgDT{{ZN$Ovhpr8yynZ{`ZXm$I95rBq-X2eAfVZRp`ISm65t-TYkRX-9G7(n3xW9cAPE9_x1HP z!eGZ5o-g>BRJ=Yl1JPpDa4!d-+zv(yDaX2ivr08Mv9|Pw^`{U28%Or@%M@dT5kqA2 z9#`kL4b(_5JZyNja&3LR){RX$J^=NJWSaL(Of9V~&(;P|10os#GvQ#zr7d4YM`W9a z$_z0mDk|#gwQE1a%$BBpKx~Si&bql#<2Tz?mDY3FZ;<)&*E&y&J=rLVRg8&WJ z5~p+n>Pzl8r~z*6KJfSU^vWqAY7G1)Y*4iZqvr`4yMd_8MX11%@`3y00ST?3)xbY3 z391UH?D7b|NytdZ-?t-dp|E^R0Y-2W9N^vSJj_p2R(y{NB!7Oo<%_eORYH~k+wi8M ztAW^^KQ9Nv?TG1(A}|uzC;dK9qyM;nXg{)mc2$Kz?hne{YH*RDO%B0f5aY585x)0t zj1aQ8o45Y1*Sp1zz^l&y4wr%8E2#o4G}lrWh`+1dH&=C0)pXv|Z30XVyr!Kk3$wp1 zOT-t^jP)5Ayo7paPz8`tsn|B?QHAs{Bhev!lb5!@c{8?)w25k|mo;$4(0@TC*;D7C zeW-vrSc1CL3zzz6?RV3$E%3HaPEMNVfkXE9Z*l1V&u>9JIxtfGQOu#zpVib;reLBq zAsAKtGVmIng!%++Bg-lu{#x!Wf9O?DG=CUuP8jy6d4@mt4erog@?uT%0%q_yvkYTf z*P%=eM3EB)KH~rqk9q#|6X=r&es|Z!N)T(rK-G^5z0u5+zb{{6+gSwdB&s>+4v31Z z^Jci$X3(zVLKR>$scj)c>wMv-|N0|;zRN?^0<#@;{JyPnmm@zcY?lbODxy;d^m9c! zC8K)zg^6ws@Q~UQ6+Q2r6&2Nni!l-3PDS+`i`AU9LmUo+>jPa8roSv++;VTNFifd1 zGKX>@dASLqg+;h^p@_FekMT#ZZ+G~{-TUYfCE=Po6$0;H;vz%J4!M_aNxKc!q({s^ zF=-tU78W*X`?b6rwHDD>-1&R8hyx=;6QOJpu*;mNHyqqiBHZZU>&P7yQIFXg%4~}NK8(<850V@EU`g%<7Z`nkPk1i4IOSaJ-99YSj%%Al^ zjoSv&9-w;{P^~uXD@Kol(7Lk;;ODii&%k5fsUB(sQf>UvttM-36Aln=$fsPJ2E z-$+qu4CtP5TjJT0k`naTP*}KJ z$5GMrAR6S-+tPkR3g}z&)eGqnF!OOm2;2Y60si`op7z}EZ)gSmk(r72?-x)ct|eai zMencJBJ>}S$WAf|l#cl{xb^=|^-_9xcpy2j)Xd^cZr}EUN?O*%#Ub~q4s{WvA$Xjg7LGq zqb+A^?vj-Sx$&PqbLNH9VdVLR|9xD|aK1bwrJxXu#+HMM5e+^LfO#baEf-0CU{yp@ zE3&>j$)F1$+E5e~;G-t{KQ9>a&g8`%=?@2U$CCZw+|b=s=M%W|HRR%<3nsY1glx7P z`Vi#WrESG$d__*9@*C5M88{n_!1T^AZmqJR+_8IiFw7MxS=oS)ki+zIQIJ15vZDc{ z$m$@i08~BCdF#^Zz#FxbGtb^1`1LAOQg5c3sCEnOIqtoEH$T}XZ79EKSSU>|hz9~t zC|Uc;RkBV1d;Yt={jwvV7c@1u3(<9*56mf?sp1&Sn=JHiXlT&lcCO{ahm0U_jyR2~ zBClS(nr#^)F&z`-nVo%!P~Uk0;7RPwa>{6jfAbKLpBT^ItO#yh`(zh53H-E&2-zA( z!XhH+O%+Qe?2*<3u=8q zFj?IJ(pRE<%Z}lZWeGaSKvkWyXc!`_YDA-)fdTy^VlLS)8Th2v*4AQf_?|F33VhC= z*Y{%hcyvC%3iB4(02G9KX$_v`GL#ZAfCuZraYYcSf{7hFYY7(5pm{{Uv&&aT))XuE;`Y<1}B?~x0> z>p`$K&U~d;2$9%?eJt)_Gh|;_5@pDA zSUEZ6P(BORt;`o#^LI6!WCD$G^84uNXY;iHy>rNoiD9yq{m?t&=s8OX`5=g*%YO}u zE?`&oITs#mzwJ z_8%kaMNujdJgN+|VQpLO1f&OyvkE@d>(_LHET_xJa=4&1K<4hmGcAjkp?(ML!>>obBZf-}G}1i&6&G2LG-cbb-3@6RWl z2uB)#c2!!(nqCFNak>b%`^ub=Iv~}W->CKrh>3}@NEqXm@i9l}kkWLc(>H*_0eS`P zK?^fu-m7}pQZd)Y$Xb?7VLawtd9qGM`r}x$Kbwq4EIL`hsf!NleMgQM#p}(E!g5%P zxAI^m2pt)1>3diT97aLl4Hr8ep zIv|bXO+ldmlkLDq8aIeV^9u-ILa{MzN|85qmatU&25{@@K(4)NOX;*m*$DX0mwx;& z=yZ{=YN1{Ux%E0I{Hg$|H}ld(>!v#iDxBr?K6QsgL1+MFjZ1RE12S`q@%9U-q|t~T z8*!00B-XlE^QYalCy9|@;=j)g#CwjTaj`V(xB^d=lCd0FVPxRJ5e4&zBFp~k*7C6{ zJp~z>S1m6oAZ(GSeH@75GBlzcDce@$k3d-LjuNn3nzBo9GVI7DWfHoU{XA}H7P%GV zaB-0}Lt}WQWh}MesZ*zJ!#H|>!mnJ=vhR`9c!LicoTD5g`HP$B5fRtlq&#HMK7)z~ ziU~0^wj;+>d4`zDcOh_-%`Ag?6{>sykIz^wP2GLwAvmC)RQ32Jf}jZ7$fU24_o=RPw~X<)LF3oce0wJ55ISh=?> zKobd23kC%Q>|ch}`~l<|8n`OQ-+2SUhg~M6uLQoc!Gs^OwN>xdsikbm85UparHBLW zQOmYf84$Envnwki-Hf}%mZIZ?+6Y3Qpwg9sDEk4>8`juGjq-Cr&k0yIFhcIbGk6{6 zBwh$sP`HnqyfX!Qy|DR5j~`3te(no~Vi)tuwRpr)J)2Lr++1ILOT<_xd*`l%O3TX!!RnN~GlgT-BNsJ} zg5nfzxa=O4gciun5uFV=pootXfL z8gpBoD&1ZCAOE5A^z<~3-MCuwT?u+hc9lfK$jvXb+##!cjm0!HM1w@>#S!$Ri*98y zTsdDU!-@cEzxa(c(&03IGM&ZP&_f%f)Gcj@8>GX!tk2`oFaZBYz47#3g^WTA=wYl@ zpvQ!~5rwGw>$%8!LDS6RG-sGz#OrxEOM#xE|Nz=n0mOdk#dLJ7sW7(=yHAe zX9uOrKTuPz6pp06Cyaz6(fy764nMv;-bIZpcd&w@aje-i#mAa%6vJ}^wH@Amp<^RV zqDRoW_FJ^}d97`md{UBG>i*)qw=TN}4~wOcK`eoVMb$19x<{c>9jPQIL8e;mAdU3e?_5j$^o#g&O?W z=Cj+LPeQ#w041d(ofjreHp{GA_O-J22k8VnA6uT&yfHm*t49KrDsIIYs)Zpz^h9?U zOI5{p606+Da2PiX7C&s@AcKVp6fI3~deYoZzDhKfXmi|f>NY4aEX!j-3{zCh!C8?t z0}u19-V~fdWgFD9$|J=TguRC`;=pfZp2V`;RC^}bW~=ipzP_2LNr=ZL9ryu-#>L8= zIrH#9Lax62>CX!bGmMtBz=$~$*{#71_GHXCX)@!fwT!(Uy1@PII$Syhkbol?uUD24Eh)r!Qvd@ zYicQPN%4?)``9O{cI38 zCpu#XVCXebKA+jQmPIbu*(8#sM1?$-Oqk>QoLpi~S3GvjJ6}URiByG?D^l^oI6Kj4 z>^^qse6>)bu|v|=%q+-Dw@#9xR3SL5BC!Y9T8&-kGL27rQhDP(=26wm<48uY@icXv z_FXAvn8%qcLu$N+lgF-R#%HI$Ez+ULm8NKg%8jS>hwWG!U*PCUXRa7=VM2o$-Tr5_ zR%y)X%rI6SnWQYQO2;D!g8&kwb+-*r$hBC=U#_V8i8fK9N1rQtprlk0s`V~_wlHDE zHh|R}^V*aaKE~#1K29;v=GH(eAVxZ;-N_xZT5%yRw!EiiQaSxxOu_pHgKRyW9C<^( zs!vd{xk|7^Qe3xl{fjc5=J-<s7$S1K zAGI0}DP^(Ohe1y0TJtM@c-eX_p)X+{| zMQvLKMS~9$A!j5CD|s&|H_Pz7q?zHp$C)|iU~VrBpO@CvB@~` zE|_g$QSo{-yU~1b0X(mf{o(!}AmG9HlWW6b%SHi&Rp?U8trVis0d||+_B|0F$;DY! zyq*;XiOO{~L>J;-!7?K4f1UT%b?=$vee}Fi-<112?8+Jjw#z9tJm3`C)vY5Zk&?6(pXy4N#VIzxJGPTcIi@H9j4aAkV*eKE$k!Ooh~_naf`7#`iN zN(hiOj|!KoZ4Jw^n~HO;Q4knh;KUSvMX#$+JHJWVj-$eiG1?rKgCH79x(3XAfwhH- zQGydmG1^muE_&2j+fupAbTfp?6Id*{lWm0>-O@P$6N>c7=?%9zE80bw{_n9AfdQb-b=xZLYn+aWZMKy^tWOqb9YTK`qO0Un#I% z1E8)g`s{jd+tPfy1&(e-Z1kZ-aH;}(!~Ke|nX+Yv=Jk(KoZzWAzgp`wfp+=Il|)+@ zsOK8|9*#)B)yIC2&`ry6GP$*5!P9|QoJBRD&0JtuZJ+c~Wx>;tBehsL<$K{hjo70p zy4VD1;|N`C@1{<9ZNjv$>KhKaLb0i03InHIaw1ERnwBndgMuWa5|Jk1o!DWTK=lUj z>dBOFmE|$3h1u_8YV)Z8pZK%AJJ8)Zq0O-f{Kz{qbKK` zGEs;nNhpgk-SuRba#^mOry@g$9o!Do1mtNXwx$m+-`8&?(Z#^PyQCmg-_M65rs05m7zHD z;zOT-C-Rcf2ZoZ#yM+S1#3qi7w~a1BX%a%v*fx2r0!3Y_Pi9Z3ek72_VM`TgCEdMy z*C5P$e7LZgX+McauRvaBcqOn^PO?0v5&;yw0EW;C{k4$Lo6;jP-IhajDgOSiC8c#OI`j)y(x2?^nGslw1J2wf}uS7`=Cp01$`8!WRRCs8+ioXesE-W)zoZ#!NKu@fkGFt+DMS8Itz zT_J}@6TWd?qq?2-nrtXf8Pfr^3@aV)UdN8>s?t(-%|xV- zYL#S!%%M)3**MI$bgCO*IX1Zv*v7DwVgAfaS$q>_bW^S^2DRIi-dFLsqglfW`h|JS z0bDYiczBoMC5PjbV#A`d#d^Zrpe-n8v2l@Es^1P9EsYT~T{&~o3&`aUkv4z9a!T#M z1jE-hTebd-+|$GLDguZz!{Ig>hwekLPZmD0zTq*Rc~-d14Ii_eE@RBjkk*BV6Z8zdnVIDp@*L}&xdkfsJd9gDzd2G4 zK#{30rs=ULrfwC8pGuCTG*yQ1O-QvK$RwCmTa6Dsj8dyt)mX<8K9^&;5_pZS>ZZf( zW|6w?yAEglYRKp?rREIV5uKIP&j!dCkWsayvKNOQb{stZR-W~4(dM7=CLKTfCeNi| zp&7`(XK@m)aE>q`J6lYE8ki%B&tfE=ITzKy85GXsSQEp*d@qROas2X#8qENyf+HF? z+;SVW>k030RT{#`@R*uuEuUecC%9al!$#d7Fvj56zu<(lxEWqTI?VuuDKS~O1vRjk zF7OSTY-{CvjYSPC(xyp6U|`8g#d{>Ftc_Xo-b_&1qsF-erEh1J`D6-br;&uNCPmHD z-HiMtI;4>3j6O~8AGwU;E5m0{+yhOOc&VDDoKT4#b|T*rQb)xa@D1xW^`eU*>q(Ud zg<8GLtf5Vb_56m$v!h`a)2L%4hKiJV9-N`5j?ht-q6z9o?1xnqib?px;>Z~6L+UEY zT4!NnH1Se0{1#w z2;zmVeDgLN8c_T|UJgh?GE;|B-#;UF-&l?f9lA`|)GYsWA~tt-B9o`z%|)#14QitP zE`<3g5V5^ov;exU9{C-J&ILrKrXogB+Nc);fa^9su4uK?__;AWHmZM?cgG^I6V)iu zkHQ!bxEh8et29Co&{$^M*aGiKk$Ycvgn}16AAmy^G70Z?hH=;7cTdyg5*y_y8UDfd z#!=-aN#s%4qsz0|B4AN%Uf*eI+~AP+Nm{%76KAV1m}i+4lS^6;a%zMiSe5o3P;9fb zC-UlHO~3OUHc{@>oP)-;hCJRZVruu(L}26lbjDMj{=88WlhQjZ)*6IXVbf|>!YT=Q z9zIImEuGU6ixEDg(1($!q^Gnf7&=mO9$=S?c96Nx^BA6ndx3)YyhfbaDm!*t-FwZ512dQL39D5!g@;bq~p=5rp ziB~5?JGC%Vx`aEJ1LyusZNTtTC=t_=M9!z>FOS^sjK~XfSJ&ishooHt9g8hLhQ>|W zM3Tx+ILl4|duS5%VOFIFGNYhzLZ(Z>H&Eu)xIl#4$EfuG53lC#b|?|l{{R9{c-1$b zC7&6CgE7cwJzcnWJ5+P)$;93M_!TeE<0y>e?}0=^RG<8F$imG)6p9yBik%0A{T$G} z5sn&V5fFbxY@7cBD?;BO<^qbJgbBd2{6{VW8Iu4; z1_R$L=(d&(MZfx&FJEYk|AV^Y$Yl_VYYCgP>9YFzF=&s6NT<=WSubg~JYVzvHcF2d zt$^2k+M)wQqCe9kUS3}M6J0rI6qpOAd&h&|O2{gsb%9<3Aa)f3eLu>Qj5egh0N{oRVcvA{=rIEqw(4=n=!pkmX5pWpdlg6#XRTPHn zRKu0sm8cR2ZAn}8^ggTpT4RK`e-@j)%bEY(7sRxG%re52Gs4=+k>*v^|)~0VPgA%R=8s*Mq8BT2ib6p%OX=WW(r8kPADoeR@}#uvSJ{(4@i^y&a(ap%2)z^ ziQ=RkDR;fWM}bTE50_nTy$ZS-pv(uM@Z=8$jZg)g0?<*1QcG~Rf}*3NzZ}~tgwXk* zNRNy^e)1$t9)Z@}piTy~eiummVXe<8v!_PbT>fLTOQ-&3LMw_F_Jap>fhyNMv&y}B(F$m@caFLz4G$fU22gPsAHtZ^PR$ajDS73usB zGt54)XvlhRqIslIXV5&-C|}M$nvnzl(Tp5Y+WOJG+u$tw_x1n#Vv2^;2vBxE74r~V z$O02IO}}y<^tl4Z8KBz1pa;LT>x77!TIrSrx|jo01uj+#7Z%1zzx;Ua~!2=-BwEw>c zb3@Q(Jj`^|iU14eOIVmWWmerE!XU7MttFILFyE$WP_^ajA!z6>h(>*Z+Pns9C@X3a z0$+@EQj3{J&0?ziSQI(T>9N%{QtFMS%HEH*4Aq1vNN0C1u7Gx z-0f;88(e&bLd07>^am>68nC0GLzuV!@AV8UiM?%_=7G?=u@ZD?)&(`R-1+EQkhx6= z_P}%SrS1FKwv6K`7g!athr%#1&_w_>E~7EC)-K6n-#^M6a)e1NJ0%W_jF#^)!P_pb z-9lT%3?h+5dAfSAJb-ja6?kZ0Uc&~7&P-DbISQWSxWKynQ!S8b5{in}tJCVmRuj7% z`d>|)cz>hu>py5*F{F@ai6vL=gKnsS@aKao?OX1sPB^mXzyZiYJ@*C*Q?vx$IUy(|?1mwt?s#efyJa=$FG` z<&MD8d;~dI$3zt1>uC+;l#3hW@;7g$tZT*JISZ={EHK6B0`zT3c-NMcVj5<$VpNLa z2)D*mJW9w`8rDa-vLc<==@95&1t0YQ6bV41(%EgP_o5B=A>l>`gabDm6E=!~fQ5#a zfA3P-RN%dPpcR6B^L9%qTP9kkzpVE-g{XdHjV>CE7i1JA@_1`#71TkkLVc^xI|Gl>kQI~E^IrbfU^UPYXC~7G}lc(s0T8}2@N`W%1 zt*GPM{ApLKZ~TFYi*kj0PMIERQZ_vjyiU})`H(EJ#8b(T@}ha7r95TRs%xs&`IceAD_HI2rV~a+DPbcVNDt3aHZd09JEwm40$StDu@KRyIlc+dNah3$xXd+AnVkAN zcB8W`L63{>*MR=tIsp+iBAl_L-5Kizb*{u7BBY*rpz{64e^kx}I`khq_v?nQxU9Px zrE1g;ZHr=EEK&$xytMG|m?~Kmh5H!Zle6N~`L4c`WnH)~mlXH^A9%4x=Lbw@zAaWB zA>2v(&gvc;x8wOW@?tbn13dAsP;)&T2d!|jpu{vXMblXfb~TjWKvhgo99rYg7kokS zB5)K%gnnSgTC495*s78`1qkoom&ZMjMx&+zRGbV!UY8ffJFH8H^$>4tLl*)I1NGY2 zH9{}_T2Ts|enZD#r9D(EyhVvW#k)Lq^tB9}MP{J>tO7_=&SM4e++a}Ih7dZXuO_Sm z>coxmQ3u95d328MGwjvHQczGlfNnw)$KK;_%E|HJ@mkh$w{~w4l=YfYBco9C^KyozBZsey?$2kXnEfEqNpGFo0E~yK##l@@D-W) zCmgm;7U5mfcsAdyQ21VE5lgVUdA>f%hl2oCA>W1TsBn*TDlAB^nQnp7_B77d$A4W;+zN zMS#ZZD8I>TaG?&^Z!K;E*p}syY_JwTfDLwN0ZMUwxN7yti_lQw*{fHzi@ks>fLN2Y zr+UkNu6TREnZ^TR+B>}FKnczjWlb>63N23}4$g1c%#p4xGCj!ent_vLC?EvO1N6c5 zXok!H#3AsMtbsLC3Z@Y=7+2vL)mU|au$TAvw!~dqJ_#v)rGyMKvx1!}7)7Rj3LeXS zYG*Nq+dZl2SqAL?*>ZmZ4lhTNpUj#l+H?bDRvkVK#8VI1xBLLo)j@VLi)z;~HgHAM zK-`~g3PGR$J;?np!=M_@ z@}*Dc{tt2Q9o5wObqfa+6%bJbu~952ML>}v9YsY!rAa5CAkupiA_Pz=7Q_Nr=tz;? zOX$)B6i`a&y#^9MAfe~3eeUlZJ@38W_l|GeZ;b0d$1`He-urphv({X5&h-YEBkg^h z;I9PiM|`C4&Ig;qcmBZ&O8*O1kRP!Eu(=ri(4j+404V|S+I_O=_Yy&=nLzFw101A@ zp89$vL{8EZ{>Qx<3Zh3r^^NvV{31F}C6-LrO@?lDAp z3)5Q+9f}d85AxwD(lAY}KC&6>>Uz9`0|WPBv4vP0ng@IT_k43i#{lzb_q@M+xD|25 zySewg^jaz4IbY$B{bmbwkoz9MR{r^uIRrQ!fEEbC_}x|@B(viiN|aa&!d8f!9d51XrbRlD_*+v?dXJLK3NsU$peu2 z6;i&_&~5qOAQo9T)P_99omO~!sortJJ)G<{NYUW4WzjgUk+&=MaazOS?n{jy8ea=u zB5N=_IkPV=?YjE)Al6W_m+=I3Eo!Uf-Fs>0xjrw_5}2cF=E|%)N1bnp9Gr`AhG*2n zB+(a=u?i*3=mat=D%ZC6pUX#~0@=Hzw)?ou4fGQ_)2)G8dxKJ1&*2`CZ`UtnQ1A!j z>GR+kAfq~6m;TGKp6HFJS6M^vdV#ZBl?iP z<~aS9+ym@f!h%9dkrGJZZDFDF$w?FQ+H3%s>x{YhC{6!Qm}Xaqh2BuGG${)hs&$~L zwwH?*t~CWP$qi($-GH;^n7ICNN!+RX|C(-k1PDWnOQ%nsEK=g0D+9c_M{+X46sm}( zS^hsZg@X1Tnk*RWfMoW4&UCAx+e0EEZ0e6XKL?WKchJ*VBhx?9b+f;CWpL7RVrI`_ zplc(Nt3Qh0Tmu={s=GNIb{qlzGychwF$qLNRbP#!dtbcKz^+sr6X2RdK$qND8J zzn=&M1n8(~cLPptyZ_XUEW~XLIW|55TMx<0cy>*Bp?>_xPxy3rZeLIJ4wM?eibk|3 z6Gn>pfB9HA>UJsbxBz08FSqberldlbfCKWU3<)0yH@ZMJ6v?&vvk2`B>+2^6%yP3H-*`f*?kwA~?Q$Z>0MylR+n8sA+GxPf*P=VO@1e(h8AhpAQFg9Kp)tW^EGgS0yEZh63 z2nQOLHOy6!!dL(CCe&Bo?LLr0|B#Ja29PmP@+9`47TYLB)>{^d zKdAreRD?XDM=`Xs^J&y5hn=x(Xn1e%$sg4Pl|OWd1O>$ZK%9K4FD33r0YYdB- z;0q(WBuf8vMRnVKlA=5tA>i9}CxF<%?$2$=?b;AVk3hL*Nx(V2xBe&ao>eVNgHj+X z13UnetYYve1z8#&3_U6kjigIbv;ajq3^LRR13Y*zaR>`dD<{l(RLxmV z0WqC{h6YQ(Zq)L2pJWGN8kFQq96Pe;xP^zx*#eE>o$0lTWVM+`M z7=Z^UEU1vhWDL#k{d5JMKtQ<DET_^>kN-u%E74UdW-uY1>evm7K^m0D&oObS} zjREke47NT0R`4HBo?Y?JJN3Uj+5d9!|F`ct6m_9z^hEw&@A}?q7}9h`@_iF4tUeTp z2gRK^^sgv&?q7@JzgsIi*ms(A0qAK9x{ED5=pFNFF8^}-=a@hM_LyL30MPXn(k?uM z6<$~21Eh;-Y@|Ha8y4E%=gGl;|HH(ZKkobg*^2!C^Pw~pm(ie(tNa;Mxb=}p-yBb( z`yDUX=^^%O&Q}29U0ikuWpWViG%UYB!M^A}@5Zq|t`YU$1(E-<)c)r_>jtR71`>!Z zJfx?EfV=)Ti@s&j+4zfZ=0Cfxu13GEvKB}*Y75Z%CX z$g#z|!o2NLFLJm`eRP@mpERlNUuja@PM@PAi>tHM%<_5rPF(E*Qt|izba&A0ih5mF`(CH`((39ng+xFb+mE)%T4sRh6^Uy^7GXj030UBF2 z=vNJ(eTw9p(@|L9E&&!G5x(64@R!v`RE+`~y(7Tccb#gjLdFsSz?=g<1!fC>E8`Vh zm}0>yX~M;M=1I)B-3!$9lv=P$Aot-}2rCL(``iIqcR$Yk-e&6EF_*(o;_F9-kHJ(k!!TRv|J>wHS7@lvxG$~;1+L^BU;t#@qUF_2 zd&eeh)VCSk)_@9<#EI(yd+MhvtJ~awEi#Y^;|+J+0P;P^13zn?rci-*YXeotQOo~) zsN^LYDv&68pf*Q9b)V`1WEn%bp5aab5>)+i$lEv9P+#xDM=0Z$0I|u3V!31zN|xsr z`m^gfXSU=aaVrCI>H9wA8V0RhjZtJyN%TcjpciDlknD07IAC`SIi#Lg!hG`XOKbw) zE}wbD+DR7{lp<>EQaJPU0`!&EqTzu`kGQ}@sd;%o6mf{;=TP6J8WaDJq!KvU-)Ul87%ed^FCj+~X4HVDWv~&z(1UDL#M$(;6uZv_3`%6jGpV>K53y;+n zS_-777L}0OEIy=l(W45tzV^C@2lzrOcz{|8jsg3U5U~(-6(q4}-^Lm2-}KNx+cn=_ z&8#~YQ-S=%c$@}hB2S|bWa-PKdmZ|%SAlK~eoe44x^rSqqF7Kv{DY8EwqZ|#joQ(L zWM1o^mRZ%2M+s?X=IF zHspTp)rcGYsMRzqV>Y+&J!RY>Du=Ri|DI%Q^o-af-%PN*n^3^w+p>Cz)Uf!05Y~!i zf6u`JTLT=nq{YJ2{zG8>Sy!fgU0z{1SB}#YHB=VwpQ0H8s#wF)Sz7+DzUU7LWF`Nc zJC|(xabjqBR5IqAtj3JC;NfAMQJY5>wAtF5zNUCJM$2QLQhu$;)sA|YL|;%X)+1k< zq5sacGO8k46qQ4jQJ+&MC!Z*Sizyl$m8Q(EHg`yg*%pOyc(_K7EXdHW*LcJQf8AIf z6GQ#Nz1UM$9?Bgua5C26;)~}e=@U9ST(7=)+;zHnzuC(fz9>RqYfH7*g+}Xj)|Zof zd)MC)&Bs@$Ki$lGy3mkXW;~ z?&`f#rHs>chuO>+QlfUvieDQQ^q-mF6?-1tNZD^nshfLH)4tDRRHoLn+Y}x`fOEH| zP8R{)+M86Fk-tbl9u5mEzsnr2R%a{$^MM5~LY;wTrGAwkU9O9vX`OC!k7;$$kGEyd zV`gh7msxufJc`!yp4&0IdXPCkmKu@c^sbi9BDVm&(XI)sE*G(pJ;aOLgjEqycBjj) zj$i3p@nj%j=xJWr^c9_(az!L2`6Q51l_=d$R=b~f=e}0>U0jh!E|aC#f*D0_Iw1Z* zZ<0e-($@>BZ_Dx?Z*f;I*DSx?*FHX6TMc_+pKQ5vZozEr=rZe5ucyx!S&N^mlZ&E$ zKAYjoDSaqgtI4sCRG8XIY)DJfy_zO?I;?;vHEh31?`y55pak27l<}IM1@}6fOe_s@ ztR#!`1z#n$G_Ajl{MU;tCKd*lDLQ(NnPC&2IbKX(-vaw-Jv!5{SY*V-H+|WeN8iL)cDp|`ec_M#0KS*V5rv%3 z3pZX|m;;e%s&2MPilU-c27uN&4@x?1cd zz4C{lG##Je+zFJL9s^L(!BrFbWa(h6*9i={aNW0uz$++tWpf!a8>P@Q%Wzorz=+}D zMgT#|G#@HWNnZ&Ph6Bh^G1b`R$CU!0CyR!c!CYsA1l;8me>nx3v;zpcwdvV4RG=fY zhjU>T^$%G*U5XzkU#iLyPu#P=yTBn7rlS{H-AH|wAS7n+q=HMAYt=6xCg>SCGzk`V zK|kmn;X)&LO6HpeOV-^yGP7h*M!Ug426n_9NDhb$u z_+Y6wR7%Wrz-w0X?M;e z@5Qn;LMnR3Nw_P?=P|CdSpZ2bfZ^zHG4xt{kA|QcP}|J2x1=j*ul5+xML|pH4j8J% z!`a@y8J5Xi>bFG*rUGZh(f6}MyNP@%E5J7?+UW#+(gcs5$8He<5P6zAjrQLr8{}Es z?f$IbOPkThm;7M>qcis^obT0T!S9QgE;T?90kHE{E-WNErfpGLZ*4wlQ`wCk+-+av zB_|W7A=dYUpv+!SlXP!ZuoG2To+(v#0$k`A4z`h)(w^ zM|{SME9mc&THQZ7TyK?tgB&`16udT_`_q%c0%JQg1&Z|au`eMvzY?KZrf2bBFOLS_zRGnHWBm*|9(Mi~C0L5UOR4bG z4|3_JOt;l9dxq$1eBh-ShVItun7{hE*VpsNw zj#Y$8foJ;)A$X!LV+A^HL}=Z*nWB+VP#H&FUX%E{hu~AHn%=aH+Vbp|r^+Z0Jv;Mf z**6D_OCKzVPD;+C<$fVsg>cZ|OSQ$Fq^HswQyl$5}ES+eDgK@L&T+>aZBL68H|X|*!iM8 zIee*gEIf`nC#clBw;6!`p@LNCs;S$)@#j+0sDeOghat~qm->9*4zT#P`$b{wnOBv^ zP(m=R3Qzn2+b?<|yc*!p{@OT_9{}p3VZ~qu9qt%6gtaqo{JT8e+XCJ+Bl_R?AB0Dd z>ck#zGYQa_PBqK%bV51#5tx`Hj#+7O30yyI#OTQ+_y#f|)@V$%a`KTa*v41Owjknm z!GU)luu2!ft&Ud75BH8Nhs1EUcO+E!Yh71?x}%Q@vT902;U=|3EYB?&fyEJf&_b{2 z*t>MB(OQ`_9yE_?C->%)TNn;w6^vK+QQ3h=2E8N^Lkx_dDw22`Sq4eSobb@G>WJv5146} zj?atY%kJv$?5%J#dP-?pUTI=kWV`M<&@V8MFKBH1YGm{H3;X;yC62=d$His_tnco0 z-{4p>y5w|M0Bg6BmZl&bUTV=hvHQk?cXoV$-tY+_Rp+hzH&*ZLsm(DImS?wfo%+3% zod<}EN?pNpd=*EPBhJ<_Xv?4P@L(qO-kljpc|Wo__Vat%jr#Wb(pTy719m0eSiaiz zgdWxG4K|JYlXOy9UQg1o)G5}v$;^i-p;i)d-(Bsu)|`5CN6J{|!X>>DdBSZ%2&L=d z-|}BXM}&=R`sKr}8QoHO{`B6cxp_}{{vh0_KD}pWvhNT^KM?hnKih8o@?)K2w{u8m zv)(e!JoCB3Y-holTMOP#jAnjDxZ|ToZD%MF@GgtoW7?nYI=Cwjr+Fp$u9CM8vhuL< znvS<@34Zr*KjYa`&{85%yI%X!>nB48DRg4Px-MzfF9;{&+CEM*{uU>r)jnR8+;k#d zh`G+o<6d9*&e3ui5mM75M`O-I71`TICBoV{y2wd9V*++Tt14Z-#3oznz{%X2dn!U7 zeozD|o$jje8%n-7 zao_C@zG9-6VjT0Of^@v2kDNqmozE)YR$@c#q|%qj+RD=TeS9HM0bOgQk$3z1}r4{&*rfkKgjSpYgXDXKg(xOH03Ux zb5RyE@~X`n?ym?w4;;<@Z?k$(>}+oG}7p*lyQz z=?m}JbeU&wy0-)~i4VWfu^H18)-P;M>KT%@bLf>H^G%u!;KDk+m}UJI)c$kn<&#{S zslcS!H!bBkFFqFNetum`TnZ)<%Xf0+L#~wI`E3uW``%6}b^Y5QHI4kzz-z-XU4ytb zI2xHGL_9JhL~Wg^*$6v2>~5UDn-hQ!Ebq|%-aKGL9-NG7F)r%TFPNS<%_&wY0L0HG=~0>QmA<57QC~sh2nwJo%}(UrN5*tG93Qkg=EfBd&v%jNWLQ zD^FQ-;qdLIqXS|amw2GK*)Id~(wh_gWV`-?egZz#R>!EPEdxhTRuMAfBf8ERj#G&M zla!IaF^g013Akq&?)nPhMLqjw^=g#hWS(vT(ybwgiscTw7XuUeh2Ys z)4FbjHFG{rn{faXi$&ZqhzT{-UUn{bSzuyi5Wi-`CEsmycs)uG@vS8GjFZs~+hXat zcT#K%AC+w78z0!1$jBBp-@Mc7KAX2=d8T_C^BxuF3W`0oX?7@frdk!f$euCn>DOu6 z@R}zBzdiq|wokit)kBB5`#LStrJGcRqMQOpj9vV8n$T z>sMC8>w4iqIN^f`{^wf=cfl|q2f+buW*!)oq8v8gEL`>g9l*E4%p z^9a9S4$)Vzyqao8k3Ti@Bg{WfwW|$iaA(*3uxlMqLwM zjf{PHo+n#F4h{QX>#DGQPhH2ieO5ez8BSQpm~=tci`Wmuz6&StQ@!W9@1<&Q+{YWW zwVYCY(EN*NV7X3t&vW36eb8zfNpN`OKz<*!n$xo~qiCpI(d<+xP9|{9$DqP~@PL7Q ze&Kj;AwQwZ{OT)|kIiTsdovaDwagl^?2Kp_>YuI(k?SI_uGrs1Fggx*KWxUS5*XWN>^t}e+xJIa&sk0k@ksz$p~RS4 zVp|g!Kmg)yd2TpV3bqRjts)I|~)^Fq#{@ zSFWN*fr29s8qvLwtWV;;Wojfmy?L0DhV~#53%u5wS?h-ZqjvDc((P2_vyk|etu!0H z%Gf_+XP#H+xc^iMQAnpCtt63kMO|WjtpcozE-t2afQgk$_EWaTo`AVhi)BK4R>K_D zDjY4_YUFJjSF(qVRr{=Q^>X6u>#YjY&DH1B!^Qa)-pN`vAGYM}eK#7YWw*vRX8sa} zy1_@`xUGkZpkMcX5s z1WsAcPucwe%`Iv6-g!c@LWMt6`|Qh_i6-lB1iGw8PvKBS`UL+O4ea!s%Is&Qod#u6 zV#JSg*c>^X0V_PyK@<6JdUrB|BM;V|lg=XnPM{a=oS8RjAXfM?snjANRW#qxzI3`X z)vf_?2E;cx#q{nNu!O8Nmp%HB)#>O55sdeuJ-p>hw(5MzKTaEWBNj|JDV^i>03Vp; zTX8TDHi6lbk??Da*u4!?2dlSIlh%#Fmquk9OU-t*-Ze zsrh!vSW53~1S!&Hpm0ua!Q=`tl)f#_y6aG)4M-}gJzeC#Go9ERc_!Qg3a4qmj|?{a z+!^OOwTw2#Yf>2>%B@dEhq_teRnRPtn=4qv5&4M3#43@vANdGRM=;_e=y2wPSvyd=>b_*t_QP0=C(`kB>V{S9OjYs#770!dt1lT7;P3* zqjeiz|8ULEBvJ~KnbWH`&A5B!UbWA>YM*2>qC_`o7g1AhOS#-oM0nlFFxS<|@WYI@ z2M2Ugq$j{sD<2SzuKdQqBF8uA2dg*9b06|MsUPHgkBo11P=cq{v=3>b6QmwOTP=g# zcP5iyDceGSYw55{Z@&0ynd;_sp2NYLKx1&3)Wcg1NoAC}#h|@v={j>(U(u|GxIMq9 zFDlu->sNFrwI}?1MLwArtU4)qVahw#0gInSbGgo*k3UF@3cR~!qM4XZ+4XG}dV}<+ z>9X*^*0bADAy58iOU@+J@9d^sQT3_Pwa3<48Dnn_ccgZG%#@q<8)Og{P@AN4WE=XW zUz=i9PAiTTj}J+Wz5Ag1rQ_Vag|37voMc6%PcQcmHTM+Y|`5VE4W!p_vvk3%pF?`{5PxR z)Kv-F0h5^?{>N*^q0FghgL7rH-`nJ$Bkl|6q)^!dq(0osZ0>$Pi3W#9Eht;Z(#G;wcx z{ZlihOt>$QBMg>P2 zn7=qI?(xgO?cO7t_)ISs@pY;l)qbtqOQRD@-I@uGCttjNGh?f_sQc8dt+NE@X`Y+O zshhg7_cA!7_wbmp35MlNa;SfKu2lV6Nij$aD7v3#KAS?DQD`_0MoetgO!e+ya&Nfe z3xc+(<~JLVU}YF-A5vE^#z(Ly3E>|4FyZF1V^;ZfQid9nSsnl(diJ&K+S0TsqGVE; z|0T2D_pHF9NxQ^L!MG_90Aamah5*lVAH=;9Yo&g(R05rWBo!@Y+1EL?TD<9rf3QDq_(_txXASxCjKxsDqAqqTIHt=y~?xUJjz{~ zjE}vOEXO>{ASd=w+@7hIUtId}k!QVcxrvAKnV zkrVpy0U9kVah+ofLQfkb{DuLpPvcb9 zA&^<lm|YQE z=&-JCIL&bwr#som7$FeIVOP53bnLnyQ}9Zr1k*AnxAK9X=WIZ!NG+8YAE60%IHt^5 zG}(M)17G}Ful+^2#DNmevloD6S8bH3+`6z-GIAVW?Q1~=W-USgNfXe4i7YQ@J5m+~ zvrxY6m@1H23>WAqGH?HR?ngmm3D=FnmWqu|S!NxoCD&5aQ7}Cu9jg%B$_)2u=S5}s z6ofzA8_W{gnK?KV?Ar74g`F<5Y=QmD45f63w>9H@hP>nR5-jiX*t(vndrozI4i7)q zB2R1#43k{(t$&oLZt~=H!R3a-h`vEUUWyA?>#lt%+B8;dwxv<&u?E=Y=I%QB1S& zftlaDFYaKzYe_=yCcj16o27ES8Chr&UPL?G?ia`SmRE*FFB1}uPU7@K6-|rndkbVO z`pjL+&JdnT-j2OQm+n;iwYqjkLFaC*A!^lxXTJ62qw!LCJT7-;=36^A^R2hSZDM>Z zL>X9Dgj;b~;~qt)iBn83_Y&fa`qP`KFQe>1vM@oOGSeyAlI_hQCNVf=bONollN7?XhnGuZOZDnXTc%A*RGHfhx|%t?S<@!se}JZu_zatwOFCDYq0xE8uz0N&&@AyZBRQbmA4p_t_yN5 z4}=`@62euIidI`&kL8v}njR6~!pAGz(Fl+bRpMUe9%5>nUz(a?2`bIO3_oG-@lqt^ zN#$>)QiRJqW0)|rl9<(6b2Q64&z+Ls1K+=w`PJJqjuu4$@>}vYm(DOoTkX8(`*?## zFuOALcvy%$PWhr~*69P+JWOAQymh;3>c{gqS_12E->_zNk&RIvd#NY<`lU-6k6o1{ z$SRl0NA6A*KF)J?VPRQ(V(jj;Lel>doivrhshJ@SMU#?5o#7?lpJ4|@229%*Pq95$ z(>_jpwfEZM+H1GOuBhbBNW%*oxc!StWhB&-N&Bap^DN64Nrb&Hv9NI9h%-`Qfp)lUMkYuEZI-e|~O!;yT^&w{7#?Z}}1h)J|PV zj2d`m*Z<^r+Iof1$DhWxJ~y4$2%qUuVRZHQx)3AOQh<{=H?q0M)uYDr7MjyZW>3qy z6XDnG;6s%0s`^Lkziu}&{O50hW>B%UY}T2Oj5M&6lSkJCRWdb;D|Q=(*I9Hvs3*xC zvxs^uij{V``;_{!>|=;-r9Af29I+L{O2xSfb7YaORfugLh~J*kdz*q;R9c`!V{yEs zDox|7m*z!&>K6K`S`~W?NcP+Gsk}^Fk@;+KQ{~zXj|#T-r;_&~=@>cqPwb@C4@gIT+XVkt1Tiu4X(A|v9oI8%K=XMiokA+k; zt8cQ`E+6f3-HH|Y@C#>z8&aX5-`wWSUz-ZqCawKpPGL;JHHEe_{qcdhcl^UV5taA$ z;U8T+Jh*6?)<-H@*2j<%=!I6Olg6oOHx79fp4!e6{X|8SSL0Kl>va3TsnX?gFY0K^ z-KLutK#E&&#Au?iX}zmTHo)`l3OS%y+JHlnOLddsyY16}eEsNxiOtveOaCQ>0D0J{Y1a-`s2g}%p|*kxoKA)nj~D7@Ei zP%-|@TqLU31&!5Pw(om{y~OW*4RP^Gq#Wl!Y)ML5V5T6~(iU{#2m^RBqRe4`0ow3& zN~TVxQTex@=jIEJ$*wQevKy}yo{+8ALHgSi55zcnpc#51YhcmFy0nfB7eX7#P9TD& zzI6caekCY<+LT#t*X&=Hn)p!!=8%0$NGG;^#t78qSnQHa->^^|t{_N}|BfHR&usI~4dO2wDEOl%8DD9p>ErZ+AQxSEIvL7+eh@J#F zVaoDDNEyq)5_}y9Gy9jez!YRG>^%5{Q87q~jLE$6?7+V8B-6K1%aapjEWcU6d+SR{ zxXEwt-^BG26=DUa+61yf=fec4F9ah$4tUN8;-65UeFg4DM$WRwz5%lU;nd+--1~|g z@7Aj^EA=a4%IF^cYB>E~&ZxM`dNuuAW_$u^1~EC4u<@x=3MJr{ z#(eu(B9j?f>VN#&3yD1#?aV=ebv->&e1B42ouKM7oXi|>5Ra4(LwB_hhCwTDj_(rwp?0!S z87|cwDmEfkc-~`ZUU1kyn-?ub9Efm-fq3j_mshFlaO1e#uKA0YZOz!?F`O3+6?wg| zpE~0&pE>q-XAb3aLDZ_97r&fGTSS=9^Od7hJg%-LlfUq?O(VW8c47R=MY3*8%xiKg zLgxSGNw)_)nM?qaaiQO5G%s+NLAT$bnrTbn7f|uomrJEG0ET(txi&VYs3?+bO=`b< zxKNrUb7sk6n+N1>W@@xGpj;sW?$^8^Jy$v!#pPJDk3VlIn z{lSfj1$FYiOt#F2k?_F7(6nzDk4HB3Id$rp7gK@fIGMHlLZVa&!2dXgQYBNsLWHJ2 zJy@XnH(7x7KiioH78dV;aUj$Y%Ni$1g~HRJX%@hjr!A#9doPFvNQ@~C%^!T#_|%^@ z(DmfsR36k`b5@5aFm*ekqIdBCoxK^j>TEe?Z{xE_IS}*Say2z32misyx2=8b_KXKz zh*)ASZ5zYA&rb^U^N=ARTio`Zec*xa)~>EpJbC!M)Q3A!em6mM_)^M(4@`NSq1iH5 z!6#<`ooVXoK-q9TagS1!ES+cmEQlFJQ#{_D+*c$=_bqP_F}=tvyHnU0Uih8e(7~6n zAixvwKfl+PHR4Y){csY32}!IMB{?*0>-mx*MvP#7-*m>49Nq?Q(N|Z-iBAPGvd%YO zsHPeX9h@_2GI|*{$@4t|`mlH3>_x__N>98q?OOzha&@pJC1;LK2)LZq;1ve9M@!sA z7HQjym_jK@!p^Ncd82(?S)0)v{XJ3|qGR{yYj^Ni=jw2R*GJ{G;hQbHEWx-=LdX7d z-+gu-=Lgqip`gmsjkCU4_KDZy3cXxxyTIdQ^T4#2Uzs9cH%3XAavcQptII#IT>w-wZ@qPVz zRr$P3`QV~J!4}2tD$U5QUcMAEV7uTaZIzull<=ApS{4@1*o6x~64nJJ^)zkjwC%-A zp|2er<+R<(CodqO-Rby`trW9mpix>v&<3}p-FTh{5d1g#)l)+Y-3L1S5T7NADh0x> zKU1?mV+{YHOK9uVR7$H*_K6=kpwC=mox@zrfgu7oD%*z6`^AD{@KF<57=gI!bbY6O z-TZL1<{5#X=hT|enO6$P)GpJP`JhNH5lI+|YD}H5g+W{uPnLljK#H=C1X-M$?+ScC zHw+&_&zOZq!nR$*5A&P}#6R_hkw!valToqkGTPSnh%G?QnF>o9GJ}>{KNX;?rJdV> z1vzW%k8krkeb)C_{@n|-%Tqt9K^`;~B@Z+Yw`R%7B3FLBNzj|S^?n+ATuB-)8-1^B za&0JI17Yb%mEL3KcmyNHd*#1_BMcffa3r|$qr3)s3_UqqP_5!~ExquSL)qbEna_JP zC7%|M5UW9K*Hef-Z53!v;v2$aATuZK+PMFzIp}hgZKhmsdxy6+ql|=dMBwEhA%0aB z-NHbEGLw6p>A?@&u|6Ka>NYIgh^@ZOyZTM#41I#sjfU;~6|qaQ3ApQu&k;YDi_xD? z??E>uMY?@}M`fW*(cewh=xViv7=h`N%R3mMHCvbCr-0K1gK zVU246t@{YVi=yVj^oJGT%+JWBs>S-v0n3BJA@iWM5YVwfpso^{qU9^rXJJ(=dx85$ zIncUFR`dFSh2aFC&d-`cv!(-75pa%|pgx3?VhN%5T6lp)gL^b+jJlA8xhCHxRc3b8 z^DRgg-N6sE$TXZ2u^h?-xYzCwD5EJ%oHAYMFbJ-H%N1cb+I;{@pG@tBR1~Ov=k4o3 zEbw+@5u)q5Ja8#5kOF0br^d9(5 z9ebmiXe|%pCkSEgFhgeriplALj}U(iG8>;18vQso`y*bdYc$Ojp}bve>MRhm>+A9| z0)F=(Nak3ViW@rs;6LQ%0g(9OCdAf@Z2+D18bfc7 z7h$KYfz3^yIT-KXZ*{MReA5dyDOq?CW@ynHOoBOMwFI>O3$eI5m~~^M<1PzGA7AjR zMl(1tHt|6^w`4&qLGA#GpVt7erUt=8x^EpAj$|-3!{d`wxeAT-)Ts$qdow(4Nll+L zc55s4g1fc3?$GawU!6(3_Lt+vbA)gKjqDka7YP3Wq@68o-#h zeMcYdqZA(Hw+=7Co~Sq`n)05uaQNH1bJcY8VlVx;80yMERD7}RfiJOA2?QE*&^=fJ z(&bqTSx;UIQBZkOHyA|wvDD3SNW_bLOkgdb?`T!Y(d7ecZx|u6dBMOT_ckM#fWp_A z#N7nefDb|VX#2D!ZwvG(D^VU3hp;i{=3~b_SY3o*dL*L>U-IY5+eIe>A;~eD+Tc}7 zUItp>*L063=IBM)rMJE)Jx~X+`IqsQ|Q&rN^FBVj3K;w&nwru8b(bπ9(lZ`4V+QpV6O2dJA+R_3%8*{+0G5cBDk0uV9F zq}_XjBb9i19t)}Z8r+OG{rq?zM-yz{0CW^>9|H17lcnl=NKjhgPNL9?rz0Hv?16(T z6m-024n@ZqI;IXTNQJ6O9IkH}Cw87P=2biFi6+Dn2f+TR>+(Aoie6;fDn~eIy_%u~ zgbSHL_)=n6Dupds$0R5?)-Si`r33we5xO45%yES0IQGV&_W?{p`?L#(-CV{Ct;RiA z7|GZj>D0(4bd38gA+mKwGEeT;Yeknnw${$-Jp8f4xC-xPuL;9wAS!CoO z6)2?(r*Axo3ILlnfesIF%dj0hSATRgxH#ij$LKKOta`DAf>8rIKf5o?%$<^JD^Iy$ z`JP|pRmMj(J7(+S!OQn z=e!*~{sfb-ee+Fk*PoJ>1w@h9?tq_}+8emu{SEDmA~3plo}T6BWSqhuYQQ z0;pSz9)KEY3r`p*1wnK%?xw<&Sd!hh4q_>ez^vmhR#%E+)uFT;o`c+5ug-Q^k)bNMVpLg?Rk*)2%wUiW@{mw#7`{wDg-7??4BU+?iBzL4K<++xl+saD}wtyfN$J&swQrXbdr5Xoddh zE!J6lSt{l0_aW-A322HDQ(p{Nvp8ckL-v4KNG|yC=t6ST#3r6 zQDjbgz%n}O+3lJRzXmOk|&U+JJO3W7!KiI+nmpsn) zJGX`p0(=u+whGmWIXG=NRu#0`)xy4YFM%_K2-tT1M>`xJzJSfA8-mM5z+K2n9WmFC z0ij3}C^eimfbNpb8uTL2FPRXj7z`EDH@f&+of{|s=aEB|7vdeC1~sVmr)&H0nVGO>bduC#lB{!_1kr z5p-R*@^ZaS+|djDYa^w_`@xBx4BYP+rg@B(^19^v@L>L%|>fp1x%Km zRd(`(dn&hXG=|xN*y5{YkYxGQzi*Yuo$dlAx4!lyn^sjWbKVuMYm3&B_1O8hoB6gU zCP8RYL+1&KYTbWnzW=!SU*(>OXJ%_A$(pCS*?s>xzE!lvB|Ti7Wb0X<$zb^08s3$ zpzg7@vv0m@4blzqJC!IMB{(6B)gudq%U6yo6%ZXR{j;sP@4t9Qee%6~UNtyPW#-U1 z`2xQ%I1;iE`w+U@Wd}-r$b?W3u;_0Y%1s#lLiefK1F|4nz!1q`W~G~kIN5-8H>LnQ z{4g*_Nib8l$RXC1*wcX+Hh6S{SM6-ziQp`ty8R@!4PU-U1@xd95>iVpuO)Tlra;!w#{10NacA z$a(8lMaMoyo;d0T+2eh<#c^N^XgwW8fm`F!pb8N?ZW9WN?i{axk7IoUDAZ^7aF?M< zEIOEnhv!upBlrv`v?GQn5la1xK2pc6Ul^{09--wvZRzz=IKr}}K(5{dAgP-hEgfIv zB_$=V3~340i$&_+IX;&HvWAps;QB0+UVRk8O&!O;gFbPJcZ>~kShp%FA`U_(ka?g% znzm_nt?b;VtCaNEUruw!f`WpCP%rv4V-#qt0oC=NAVRe`LkEzn5E%{~eHaUt`*3~F z=_S}iXX&VrzRqH|pMov*A89`MFWGQlTCUlg(2@0e)nbigBdTezEsW|i5zJzF0RW}} zh~sj-Fd<&QCR@1L0(3?_VWQ5DUpT0bdZBQi^X!H;z?0hj!uz;Cfarr}>|Qq`2(n0y z*;%O>xTkMkUH^T)nKVC^pX?6a#5kMv>(8yoPwE>jS8ih-BcI~lZ&J)g2G$7ir;4y^ z^{p{in41+2)z~*=$nrA@D<Vi3}0lv5!W``g4Km2zX2@!0sT?+QX zh%CJ;Cr}J!1)|fpWUG5Kl!{3C&dM?_X2D>eBt6q?^5b^e7zq`=_YZVT%r9}3C| z@&sz70(2)WgGW#z!Q21-T!n(JVr(WG2S&%A(_WCuqtnv(slvNqzF;QStHlzMM(3xL z_IwI*WW%_%9N^fNP!OY^@RJUZ5%kPe>^!ai5PZH7>2<1lygwUAXy}yYibnVJ=2>b! zH4{*X?<6219#F0cfLjDrJ*w1babb zr}gYpIy!7*E&W%(fkCg}yVMEOj>25rjmynjI5AL5rh#EI>4Js9OHhcEz!=nib6@BM zm?3-OV<+P^2tLg>jg!@mS@#uXBbsNv&TJ^0@%t5#DxvZrq&wmd=I;QC_(3}3bfn8` zFF+l`j0l|Tpw`F(oc+q~5I!|3a4!tN(zuq}0KGab-9(g$SnBigZ+P3ZkGK?G_*;)I={D+dJt z-V3uXQY}`#>AzrnB%k{?h;B3b_X2);-*|Vn3SDocX9!sa5+6eyLwyk&8Tn19pJ&0U zH=kcCrHH5#;|^d2UvMuBa~N-o_xun7O@(emU|xWpdF=!#9R7})-wt(A6R+WZk0efGkT=V<$ z^<_yz@qQUS#)2yK08|LOH?b^lI6$Z^LB-QPVYomjnt5O$&$hd*cnw}9LY=zj(B zS_}iPz^=5>{e0Kb?o5MGgBhaVo4ho+0nGetti28clGP0{4W%n+zxzAz4xY zm4x)@^YeY=2e&iwGNr=!A2C#>9_95(c1~znR@qfFzR4!=AA+c*V}_JmFplxn;`7BedL*`w%R`Pyxn$s z)2>xSYe!zDeo~_{Kq3My`d5n{@3zNkQ9ci}uWAcZmHi#R7e2d zu2k(324y;pR@I9<`BUt4bqE0F=V(QfexTpTy-B!q-cR<|*Aqg!{sMa)BRVdTV*@yy zXzb|xJ2YG%ow_)q)?|}^nE6xX0Mbij))!ElQdhev_$Xh9?lN112Dhf~X}VXeY8vx5 z(?7CplmJ(D6L?W;`ZQX^09x|_@6T*jdElKs=Z9s8@{rx1td))T)1y&MSIHB!`jK|v zbKEMcnx@Z+7xA0wL{uIn$Z|v z4Z7R$v8W>U3tCPWlfQ+@<{#8ybcx(X7jb3$vaf1>TIV^fvts)E)>c}X)>?waWiwan zFHx;AOWqf(^3yxF^!+*Evzoj^C6{gXd2Zdl8oB`s#Y~~fD&Hx*(kIJvwpEV*SXGHQ zYbE|y=$Tj}`n=4z%k*f9@^1ARciG#SprEfTu%N!Z5tGBukCmdV8TV#v4+V8@QA>M? zpTyPDf<;)W?$@`yN>1Z-yiDU2YR|E)!;xPF$Lac=clGk*wRV9gukYMe^?2(NV|kmL z$DG1dIvVHWEqeQu3zGo%ob4kPmH*M)n}<`qw{7E#tO}KqQZh9t5*n0Dp%PIt6lE?c zGnt2EXwW1Y(10j~jAdRdR4S#Ad8Q=uJkP)L)4lh-pZ9r=2mN}S^xgfh< zM_!#FSvf~Olym!uQgso@Rry-CjHpQ!Q$`Fmb|SH>)*LU@_K_9y-pHblOHtSi2~%Ub z?XTLR?=6zCSR2p(fKz=*l|YWk)=}} zPYkTX`@E>17i0_QL^QD;5-N=;SctV?{ERbH+ojG&mWSz%aFS>FVC4=Dfs=w1L6)CF zf@?&2Gp?;#tW^K9>IaLqurg;1q~kWzm7bZOSan}5p{gb2N4Z?&=s3d01FOF;S7wn3 z#_J&BCzjdb%TXUK|AHpTNxexQj&0?8DF4hpZm5l_dSYQtD_5f5O*=+CT@Pk%AKc*& zCL+7F4livgW#F|=;f&FVn8f7dmW9U&?VL?d<+}_HC|EPgB>9Zw>S}l#-dMLqtYe*f zP8n?vW07xgw0#H9Mga%G8_EatU7re{R(!zmtY8^6W!1oHYMGPGfs6yD6?#u~YvN3E zGJ18>yB6B+6VoVIsmn>#d{pMdlh~(G!ent;OzI;o_{2V^@C8NT#bNGS2F4WV>#n{QaHVQhu;p1JBs z6J3w#n8eWodKq!uc4FIC@9lGKaeTb_0b`+Igk+g(+g;b;tD3CF8r4ma4>%%BqHJbl zyEF|MP9zC989%$`vZRcwS-P%Qt9Ia;%i`+ZT-VQA3?+U$QuQ zR9PGuQr@XK=E(ChJ%B~rqxJJ5L)E*2lPzdhoa>iAR_B-UqB$>E>F|!~wE$(;=UKkG zCw`2@v^_ZomxvuMu#DtO+}1Inni<8BwCHAOyJ6Tbfty#B`L!5G!-Zg2toiv_Zm~{< z&#PIgqMZ!P_%zoItxxU|lIL3!dMu&GG-__pJyhLT|Gg5^SRMQ0f)|$ZbpI<=LGC%I zjBV;Uj#MWvUYk%~Hf5vk>+|Jsrj%m)aoNX$vH9I^Z3nVps8M&u*@DdK_nO7g%0+b%DOh?smmHy@bbU`~1OrRo8Tx8iMN+ zlfd?lNaH}?)9y^^J^l|W_9Om|pG z^;4a+QRgGEwBb(bRE5)}y(}_*yvGDg%;tj*KBLfSbIWArI%QYiO+Pl>k@m~~cyWb6 zF6Yb%_Wk0f{K?!0^%mx=E|2AjyB$nVO%$N&ud-U&_(?f<+o_+|)Re8I>AXVE+Nf!} z&mQEb4mw#2b|ju=M`_uv>(>1^;P`4y=|hnQb$&y-zDgcV`;#PiLh@Jl-c4j(D%>*a zyrMJ5byUy&i{YUzKBu;%ULF29bR_o7OI318bD7Gha>HmNPIDqcK8 zu-`LWNpX(0o>__tyZqgnUmjgo{fbQL(lIFA^}cABW^~LpS?2k$;!lBdQ5p4@{Dl$| z^D7^Z1K@DDL5Zb??%FUAsWR^+NvU0nmI;B>k2jo4jB#hw=9K7 z$l^I#;@2*k$a5zZaZPELwTBbslMh@L{kx?M{J^rMJ)@U%-@`~Zu}70H>1i_|`qEq4 ztE|q5Wk||tO^55a8r<_})rwvnsL_4Yf0|pFVVD1or1{F5)3Dk-l6ImUv$3^%6!^Ed ztgOTM_xr=LCcA~;4H(qeB$XFgcE(4t+n>o*=1A3P831g2bcbPvQoFKI&&K*nSP2a_ zX=+GII2uuwZms8K5PN0G(i_jh8OvS?r#!bA{_?czK?PKkZ=2J+v=?)8txpR#Uw+EU zkp9=>@|HRFHfF;l%h`?#7Hutq!47poL0zV<^`nJH_(h}LS|#Fw27zztqXhYy1#B|LwJ$IMP4sXW)edZ;oQhf<*JY!( zf!erB%E*1URZZz^((0MJy0^$Qe+=`|?Dk`_qYu`tGdN7CT|2{>W;e$8+TsN74IR-O z`E$;TmkhWYygxsh7G>-nH5?x9iP3o?tD{Ht(M?@W8+XYb7Bp9QuON801r{rJZnSvm zR{PU($L}1qU@UTudea`Z64&V4LQ(XZnQ>lGnC|s)K@iPn@aXuNVx%%Oo`0bdG_*zc zMO8`>s6#*V!gpz9yf_gZ;}?Pk%`fZ_eT|v)8_kyS%Xe0miH#>2Q&tE$sGjm@+L-;u zkl~lXe;6qOljvTCVf)q9-!wtqR*;$%1ypIy`#Ye#&@)TSdaJ={dtlK|c{BoC@Mtq4ozeOL5{t%tG-ucLFc)7#jF>BolopP0NAfcKBKeRvR&IHDvKX+f6`W{A4{u6w z(%S@Zf#&7b^;%w5;I%$GS#2y`<<75XQc8M*GgOgariDi~riBL-u)4Y_1gR5) zmR9v5Vk87k9NUzBV-)EPol)7>O}5WVY(sT(g+lFHb&<68THEv(w4K;QEKYT&xofv{ zf#~4$R@KQmF(TPlnPtj+rmNHwY$@AU(`&dM-aIH`X$t6Ju5}P_<|2VTW;vGBarV|5{ zTP+~N(faZcgz($93lDaDzPN{nRe&0EH7OoehN7@U@iNx`v|r&fy!%S&qJX(dFLgG= zjk+1v-c0ob-khWtJ$B3>!cX(mz!&3){2>Truct5R*egisr)q5ctX!*g?>+WJf4F7O zXsPiqhG^DWd>`XusKO0>(hT#nMu1!rXdEa zM^3ejveS+T6vUio53&`j;nf`*nFRAHC9TJ;o?Z8oyQ#uw3U8Q}z~W9Wt*kP)a166l zxyzCnUIpOn`N&c2;NF#>}Kp}V@*e%@u6+%;|)F{v{l!&jkj>I=#+bj?^D0<@{3Jw?OEEdEo9f#C?95MN57aAF$`YQ!!wV# z0t`i_76{c`1Wq}1Insifu)4F0rv-;mGQcANv%H`+{1#y|z+x7OsQS1B_-cM1x!oeL zUGVzL;J0;R)M}2XHt0QOZJSB7c))nnZc%`e*Y>O?KnXL)#sYxfswP6eZm zi1(^YV(13SiJ+M-_3HzxAO4-Qc&UpJdJ?mG?pX;u1iR*sDCqMn5>tkppM`xZzbE{Z zGUW}8335!uR@`Jnip4XGmYpw3$svIWcOvhU5@q0uq-l7<73YSQ<0!Ch#(5 ziO_m^$HcA2V_J-Ye~*88c1Owh)bTgI#pY4k{VqoI}ZDoR`}2N-T9>Q1ok)PYZccfb&cmb<0r4q?WtVEj%OX`K}K(ZKN3nD(vIG zwe4iGe6!sj3fP}}`#M2+hrVIx1ec7Sf&83fo?~U-B*$FsnjDR`Wx)jA6QH+39^E|v z!nc5+?z>;90lxvwpugbVI>pm}D zmt;5Hn0djZnQ)Fktv@pOD1DF-bX08VtCPB6)Z3);L) zFS2Zk_z+mtk1*}w6S0~f{xEzlk}z4XIrS!IfB3VxfP$MnN}%WJ_*1)FY(RA56Q2d{ z{u76~5o{(gAKCihywo9p#H_l-_;H?u$Xc(&6HN@mBoHx?Tp!aL1D{S|Rg4yfMBtuC zpGWHS}oZwj9^swF1#JUJk6f#+tj;9Fe-gA2eU5K!Bg4~=os^4+c$&W zk`aJgX@L0fjf2@Tv9k-E_;4DMxxzSu^1#k}x%Zq@Vr2cxZadsAxg<;D!==s?h6M{V z-H4&nA53#7jsD(`&86KLNO~>-{xR0-AjLvMD(%|->iZ8LcC3?fjFFy`VvBrgnMHCv zliORO?us|7;o$9?BMn+;KxE3e(}V{$=oShd#ouR```0PW(ko(%>fEUc@)PT35P^T`;0 zI_ml5x7Ea_H+p0|Ute6UXTNM*%Dd)e-)v zKiUyt$h>@Ag(+P(H!k{0KCgnEt2*1NuB?OagJk9=S}|KLj`Vn;4#VYw?N0vBM33hv z8S4oB+oWFhk4gQ8ZlN{vRi8Qa+TQumcuGWd=c4Y5Zk}M{-Yl+7qw%x#NX^+j8-q3Q zC~qp~p=~5nIUcD57zed9G(C{%#h=zZl+MOtTEUoD*bvgrg1dS4cK-0-|&b0SId)ce7{Mx>Ppj& zRBr1ZoA2J0=uoui9m?K-X)kmpo#9*P!|TrzI)Bw`6Q%6mk1h9(xJ|epXv)AA%_b}r zKu#&(cyG***i!Njz81KpVypJY+-`fiY|T%{qL7CVVol92iL~R{mk;r+Am+vC;I@v> z5OZdn{LyZy6+u&A#LXmCe#r6FyMB})z?Jc*%57+!n|0r{zX5@)9tG`;$=f)^y+51p z@JRO%dO7Cg_s)iH`wwIK5@od`Z=|1upBc_>wwmqYvsJt|9Bp~&Xwsy3Uhwlah9AVH z5U$0)`*{s-Q1ERtp4jF}{$$X~e}h4~Gnhz={LKwO*&(S7lBGLT-c#U9Ejd=WG2qwO zOYU?v>0a@W>w_3UWL9HErh{*)7r-lP?@=E3O?Dp<6d=f_!<`~ur92?CYH1-O1TCq5 z7+l@wFx@}IU4Nto>;!9=Db3U1P#hc@`t^T6nE8}p(%sa!XRah`tnIN}@263(`hGpF z^wLC~ocOeDdf)0!#8}fUBr<+ZI$V-w;13p*aYY#2JW1w+)p=p1HRI23efl(=FWD|3 zfA@Uv!+j@pcJFPHJzJmKm~Jz^tlq&mZKYadKFAeI+Prc>f zQmrrW8)*pte0hKU=hfri{x?|#@>5>=YD?zaCK05-t%$!mUyp}DRe{}qXkh3@ONPMp z%*@ORK|cSL<*clx=6slf+sZKs=JV$8ZTj5%jgfLtD9MYB;k6C)HgoN?MHO@{HMqrJ zKmBS6?4^S0pD^%W)A*-3AarT<0V9Tan5Sk{6HiYdr%@fq|5`qM*qrjF3XH1;`rT+) zpGpQh9{4x-#D9QX2gfL-Y>nU$ganBi)UmAIR=5I08$Z(wVl?OSb{-~paO=KnQvXj> z1OI`F=WoHPw<4_N(-d$7DEVG*y#$W_=9oJ3_jfrhLCmcW-SIa^I!<7uVA zZ?F62uV(N3>oLn;!&)sNm#jNusR_vP+(%&~Ttl85TG~myx%RUIMx-yQ`Hz0`E(F2Z!2u_>Hp6- zTK`+#|MK=Wwv?Zr@D_ay#SCcT)#(C;JtnA%`XCFahB;DbJ_a;gE>lNT1feNgvw~-_~MOoQ@1ZDUS|0Pyp)5M=g z%k-0wt$R^6j^rg`zQ0OPonI$IC`k{=oTch4gh2$Chu>X$7_N)_FG#|F{ZyXz={8H3 z2}1~yx~oKHSBAvKM4^eN;XC$GxGzDRh+GbNs~>Yq{z>5A`+p@1;J<%`{E`_G6#27Y zAf%w>V4^Q1m)-ZD|26Z29UL&R42i$D4S6ePm;Lu|;XfrV{P!Qk`` z5B`&O!mYN)s>T?#Wg0g7DO8T}P&#M>LuqK*h6UxfTFlUjctIh;TNllE0*cHk453ty zY$A+Aer;4<1uu_1wUKu2Nd`6aBr?Xf8RcHgqMsu3t-(c}A#ipnyU8JRhEQ8bj8o^7 ztTEh$#~{n$%eWHCzhx!bB+^$LJWOaI}&X z-{41qfy)PXpf=fdip{&`4M<+uJ#UKQQ6gs1g(zq8ffghH!RlMoN(gLlSryGj6Vk{x zJ@tGRz;SW!Q6}$9kaRMR&x=l@2ypON$xzVJY=-kQ1K3lhMxisx?8_u1T?w3s!V$!97(`= zy61dxD0eLz8BHU264p~zwhc%`1>Osx(WItF*>mv>q~ zeQ)HuiuTa9ri7J4&{$YIJ{hjh4llj;l5zjK9cQj|pE~H|Z__uA%8IAv61kbe?JSfx4=?fK=65S78LfqMq~OA$pT`f)AF9uB zO9^unNAysa1b6HEV3e{sFLMsi`;TPB%qst zq>;92>+!-y@B2lI@}JWb_D!@bAPvY&e|Nw*bc7YNk-4_b`v?`RXr`KPp5h%p3?ACa zM>|db09|rA!rLf=TT;B9cQze)>ovu1@AGVx#gj?{Vo5mUE<6-t?_T~wGdk=?9$WVq!rTOEmq1D73IlRQ&FWdxNO zf@GRFZGs7YU*WTaMS+}xgG!@|WD#~FpNB-+#9Q-_d0e3zsehaUmeXO#2LJ8ZA8eFD zE^tcYc$_LU5~<1VCJc72D*er7E4QdQ8VDEl)TWSKEc8+%U(rOe+fdC3q*otuH{}XH z7%_Tw@CL^sr2|DHXt88b;8(Iti-*BSJpp7??*BZg1C3;hz)Ang&6K^eVs0rX}hCW%keMTn@BDq0@Zl zK?joKzizA3q{!!}m(D>Avt->&2nyv-qnfRwqAP7&PwzH<0RKBFd}lnWU&jPX2aq|( ziF#s>hHy3YSGx$EsMW5qt5+F7z_u%^k}|ZRe<5%F zkE4$!LiFtqzbZGgs>Km8_31H{a@HR_Od6j7s=uQ3)a9eiW?hfWLE)V9R^0@Hd#lpr zxB-|nXB6ng2k$`@=VkBh&a!y7Q`d2cEB*&n4zbScD=U@YdbUu$2TGyB&1AP(R*Df< z;Zv?ZNEi#D?K_x$i)9TPQNQ<=C@?>^XmC1UdpbAvL(417I>sn(7~#%D5nuK+yh2>kZN|JUs4zb|;-dDu=o$uoAzmK6;^?VMvv9W6FY7A`MKeko|9Ko?=FZUZ-$;wOM zQe}@;>CKu z2D}m@8OZ&xPSXBtqRkfX_rn<*(G(r_nFUanwdNgiTTF<;gqQ~ky)1W8F<$VlA~#^a zAJ4Dp#n%nU%RAd%95&YuR^J`>ebk{{8+PpYj$`Z$(t?vA4mpKed*&(Oii6LZwL}Ez z^#y)V?$x2hGlBhl-z5f0;ONNn7+vkoPgU5-UvS5^&mZ|*heEU^c|l>?p9kiLqF)z~ zOb<@)6g1S?`!*FAKG+0rTZh0g5pwks7#dcX=h2R%Rar*i-XngkE7iPwMkJQp zIqWG61z&BIu;im^(QOmB0?Mk8AN?A7f-!Zh&_UNv%8j>O|ac zuUr@t9FGP#`Fx&8H&d+EJ@Anr{t?|I;jt(fafw6Mz>z6yQ5+>(YWcrE z2eC`@N~{`>J(Xc&P&4^vKbAv0P=P*4xrqf7y9v;Y^{gQ69V1scx!^q+_9!J!OY2;*R$1yq<@Zo+Cj(l6vegUvNJNX8wk5=wbNAzM8o=ym_$p3iI z(8NvI8{cWWlUU%BQ>`G72$nsl7>=nGbP{V{L{=yTaM$j^iX3#O0$(;QohFGV^ODU< z@)uEq7asaEJ*ye6)G@jmhVPy`D3^ze@y*3`;^#^rgSW_Nbk++N!{RG=NlXAlGg}x5 z!Mg{=YQWyG>Y!~DevfZJ2dR?_m#oQ;T-XASMoE{F&2n=S&xxqHviZ(UPB9ypZLCmQ zP6=F(LB7*Gnn>x6StOU#0#WY>FL}6Q<4;>^)A(I3IpRAQO69*s&a;iP8o&LY658dp zuyb%g>GN{RbHD+($n#~sCpLFHqJ{P#Vr7ZTY(34%QsN_<_XPA24CC_%?}L7hQs7Mv zSjgkK*MYBS;P7qtifAq0pTrjcP)G6i-UZ(8zh7Um(F)YqDu7W9O&pXRV-j@;ZEsxj zg$O|)zKfc;h|jwRSLFB%svqMuD=6~Uh^pN4X-wKY^On%SpTm&{{AfR}=p^WxqCq(_ zIEh}lJ3vDQN5>qm6{TP^5;?7+#NLwA;B_s)uk-i|9;NLhR>TvWb#3^H5=dI}D^Bq7 zdSq!5{<)CuKtG_~(d~9*oED#BH*z(xjfe&Miq^?rtb1dsS~^czW;E5*k7)Gi%Z34v zV2X4^p1y%cxFy6pX2%>bnlAx`lt5;}h9E#SMhliGU0J2SLD6ZB7R&y|Datao&dGI0 z5A4}DR%6mPnSA=ae!g5OfJiG+C1YdPiexn0JF2&YQrJBnb(1fKGvBURRz(^A6`28s zjl?a-d94b|z?tRbNV;iAA$?ADpuBALNKfsqCu2Z9>f&((V2aokM5wl$8d2D%dc)8;?C}POi=n%CDyNhE5N7^6}2|b zhViTOL_maXZosYp>GF`B07Oz^N6=^jk5a%wMZ2kGhil830#VG=GYH6)M?YUv(As z+f_(c?tP%8_PMH;(c2YXk!R;0##+4fc)wJl2Ev{0x#A~sUb%D}2Z`atuLui?pV3YmT^#C&X>~J1wRCAa3wrV_MYF|MKe22Ik3z97FVF&_$wOuk+ zJc#zx>U=vy1&Ibn)R34a&EB`a5x0|uVO#Oil@a~BW>X|&}(6wY)<)A97 zB`5vd!{fg*a~pzWB5v=a494P=vANx^EdF~O>2>ywPbk0sIC2QEx;K71^c>r94A(sE z;C_ny$4pj@?_v_dh`mmwh1^`upp@U`hXvq8R_coaB@L|D}xnLp#G?{@+Z>{7>-7|L_Oz zutE`?MQ4^zkGx18?f&zh#m4>8`_1POMy2I_S?#<;0rZ}iEo@5= zoCz+~ph~%XgsN!-6$dFJ@9>SIXQ@NSWQBjG6BfXy>VS?fKj42AXg%5iwUY^S$*%$X z*h}HldT;yd*F9TG$SQ!x&G*4s%b0{0;%FrTF+ZY*Yg#g$=RT{NM|b zRE!OCN)_;}?47X0HPCzJE3`p6%*hIn7$DbHTJd}5PD+Gf?A8ZEile8hmL z7KeK|J&}yoMCl`%DCi!xT>;iy2Wx=@U|-H?#zCf$J%@H4uVJG*tU6v30UF^Fq4xTg z?ta|q7-mtuz@XAmbYHkhp2-n3BjjtRG6Vc=+&=MM98lnF2{?=1gLFQ65LF(9gf21b zHy6nOLsEQK^4>9`$70p@hV~M%uVakBIZD` z?Ln*#vftAc>4`+Z2a?Ng_h|JuN7Mu)Vm>hoD4M$LT)wjpuPp=6+9@D=DYM(hU~q$% z+5Y9Vdn)<80xvM{n=2zor}vR3yUyQ9v#Kqd*DZmhKrIepD$Cv}LYr&(NG!I*pX$aU z!(k?Q?*b_HA_)snHi--q9CsxJf9NvQ$B61YlHCi(65mv|%>&9ZP*2AdGgnd}o6VR` z1dvHE(74duoR{~3k?N!RK=8+xSI9N|3PstY`X^2&HP@afngFMvo?r+ZPl=5}71*HT zGtk61CcHF7RhXC~^h5dm!YgB3%WvFc78mzpe^5~-9Zfcu*HOV!9WYw@JZLT7MO-ej zq55=);yuAe0gM+;_$oh3(f3d&no8(9@D)hqxFQT}VBQJ#c0Eu`5o01Vaf3PAR*#;N38s+|LB0dNB&Zmy}7RIs&v4hY>`@AVs;*WK&BjJK?I&~$f}VGi8R1Wce-W($-vbgk4_e8Qn~8a?24cyx}U#>4;4cq{?hZC9P4=Ss^WN@ma*@#uk9x*E2%_ zxz&)<+5^>b0@^C)%u3elvkp;BLcefEI<>9B50)SKmt@~tUmeCU5!KV<{p~WpE})Dn z87sJqqoo!yQZ1SNMFECNyA0f8(H|{O!@^3+L zb<;g`h#=Nd&OIIJS?JfaU#P5-2Eg4*sMrn3IPTW%yO>$e=Hz3Du{F{og9x4H_wZy{ zT0C*YK#gP#=+J4Jlw~-{o)BgH>1ehksf*GDtKAP=?w4|oOl?3!L*`&ZrF%Bo&nl2n zNWJ^*(XK5YMhL(mydW}wFqDo)53l{^*g1&yHp_mcP4nnt&Oo(WcWQ|j(VdhpK_SWn z>~Y~GO4w;MttmaU!j66Mx@lkjRLQ%&;MLLa$3Uqsv-M6yLBMUGhq!B;OVYmP^Lxbr z^UUtnewP4#>ino&5^k`9CN`p_MQ4|uV}_4UcEw5jvoWi5Qimt{>@`d=URa)zn;p3y zw+_F3TcVtkY?8x8wC8mITrYpQ=6-y!%TgujUj-OC8>~G+*F^iFQ<8&P_s=8*6ox0i zcM`?vlI{1;#(@s!5ip%svSU3 zh7aPnP*bMkVOI5rMYZ|J!xmCMc;b2=EOEiLyzpBJ?BSYm-uIK2CkA2rhcS{(DI#Xq z*kl5WqF9@h^w)R_xj;3hh{@=ay%(OwX`doCljDfA zs1?D{AD&B)*m1PC(+FwoZ7Lsg??x<%K|x!*QC^b^_SZCtkXS_7W)3iXy+n+7ZoDNvJ!L zh}BSz#S`euJEeiV%oHUU2&)d%rNFYQHXy z-fn>q{hwWhxMF2uY>wA9W!M@OSgSX#Ix4%BfDK8gPy3`U0h?%;BkI^(;9Zm5yTek< za44~^ePx8z=l!1N@57gw!qT#EVG#J(foC2QQSbaDR2(Zu9{HxQD0ij6o2(gIZ}usv zVI|y0%!oFo-B-n@{&|EJEWkVghD7w^z!~!dE;GlU%FA*I5bN%qJ;DR=#;&-UJyF21 zO!eRw7$=gVyikkW8}&f{;EAWFI{1ijFjjyfIFVGa>FDz}e6Miw z&GlMg%iUxWs1z{V@Q_`JQOIr6_6r2g`RvH(-L=zz6^upudS04{&*$?#rV@`wwSo10 z>m)@9_x^v4$$uF$<`dAMZHc#^+akjZ_FLjfm7uy0+_V;qrNLmYJ{% zAJ^t$OGTa3!N_B#{OZCodP}VDK8p)8ATL}o&@onZrEbBe3QNFRI){$@tw?uQT+%XW ze@1nm_Dz~CDk9UPlUQ00&Mj97fFY+)kK^~gX5bk!3X)+Vc4~WnFwMW&kXE(qNRzUN z9apd8U3SWbuy!-uIKSsRx1f-4pdWTE!m1p;D59G>0P5NYsw;@0t&DZhOLznua(?f- zoTw`BkQ<}y+y#r4T?sp6`b96;y|Hkyz%ADzb>R`S7}K=sGB21IikkJNb=aVOsi|t{ z+Py!aF{t8RH9LjFb=>`yy06yWt$aR*choi8ImYKOtsC9H;*R^zf#Bzxd`iPtKTCrE zI%7#tx3ZK=ANLN17R;h=JlymF#yWuO7>0ux_?}+299+*w3sO>`^ql~ z`fSmk<(mqNnvzaveKpLR>7@ej9;HBffteo*{y?D|w8dHief()H4#CE~?SRFy{nzaH z%T=$b zCiD@&4vUUHZ*%D^`!Megd`KvG7^Ls)3$MsHTVk`vUi5Dxa?Kb%5nEv8;2Dw&R}s8obo@Yj1wr%kdeD`DZJ7 z9VHdpj{s=1Af$r%S=0uFP0&P193ocKRtSN%iJu~Vb)8Nt640Gg`=RGzF;-B-3dVdo%1YD3Zzzr%79gN+jX81n!g)Sdjgw48BhT$!HY`i#&K4Wnp$ZX0HAx_dF&&a6c;jOP#nU{hcl1`L>gzykiLhn8}t(kXe)+*tA4lNbU*@~o)(l?K^ z4PHgv$v1h<^anOGXH{cfM4tayX6mzw4VoekSLJIi7ll!r=tH;(z7W66i~cv6ID(#4^`ibC8h>~ffjgEHHjT7eQiT}C#m8{F($%iJvkwRg^2GKX{~7FZ>A1`JEAHn? z_jFE>t9uMrbsZWuh7@*-uWm}tZSej~TQBEio`WT{uWCr{&EJY`QO z&E(UsWl2VI{TQRrjYIUs9aFgkifu9C*jBYed%1x~-+O`BUs3tmKO+x3yt2~O0IqlZ zE&wWK`(rtolLWF7vj~Z<$37=&0duwcJ}!&Mq}Fx;PN7E%?|10K!O9uRn3I4nHZ1%| zk6bUl>ChqbrnsbBsSa75`yjtm3M?|)doMoI@OXgx`>iqxMm8uE})Hs1V;I!@y$( zKJ8Vxv3}IBW@07-*t?eiABUG2%CDC6GGGe!Nc*{LY54h|{HhLi>t_n7(-AF6zB5~O zA~_yZ4KPHSG+C}$$a5@Bo=@CNJ#h=gwSJ9tpziwwCe25Sk5JV6w;8hE^&P|=y2PWW z!aA>d!9jO-pT%H0l+=P+J=5AKJoppWR2rwz#qVhkbV?|Z2UfUtkGV5_W>FDT?_W^I z8`cU2S+-OXQrJ(q(_>sN$H%KK@Svy%5X_QNw%Wkqk58Zc9j%N8&^B&Zet*UP@MGc} zDl~ z>mm(`&O39D$ye(;_;*k3)0CL&;>qv6s&IJOkB@h%b4y?gZm-p{LqliSoY{H@UtQ4^ z_JcX$Ovjyb@-OmJI~3avi`4Bpa9E|w=Ld}a(WwegH|uQ3$NMlRoqPQXq(~(hp4@rB zsfdh(l9m}3c~$NvQ3)^EW9+-!nYnY<$E6dP>blEct^E>B??Vh~RI3W#aoi~ZG`+TX zjv9fikMRLip9Jp(dSA|+YbF*=+`Se?33IE?qFQ@p=@US>A!V*W0#jRf(jIw>|X$dTU?_D7YXVOK&XUJ9)Z%{yWC6b*KLS#;8 zO+tUm5T3bgc$6tlsngw;$dbN`0Tub$amYj*{?cX5QoDseQICmHZ=<*W;Ux=JqaERI?kvT$*335jAH0?G;>rHjSS07KVsn6!)6gHJRRS`C0+OW zbtR9hVGy1QC_BEo;$9=)mCTEW0k~)Wx-t@jvb(hV>xkJ8jqTY$fW}bXL`#K*%&;eZ zT@-x3L1o`iC}tLO?d$ef+4)SH`7M@&dD;7AtdX01$;|HH4dU=*P_5Dg$AxPT)px1` zN!Ug2?fP+mzW9o!7}KkbmF3>;gusD9BdfhDZT&>0-jVU)+kxGU=gp3T4I0)-ohCFryao$=GC%)o2wU0kgOvQklgzBxXPZ_hi=jG(u#$40R>`FNXI*T#a*c11OU+3GD%9hYo$?5gaL zcz$38?yt(l5Ckv}a7c_z`kQq|<2lGouffFHpw9{9JL2?4<{FA8(#sC`^eK~Do6I~s z5-EB2*ks_Z{c?Viu*9w{yM7jxr!z6XG6|}zRu$)%VcRy{rMNyV>Jb^K;5G=yWU^~K z|H`tZ^Q2MWxBly3S*@{rbMgy|LojTYuv)$)u$o;;Jej9PMcl6ESH9`XRd&oT@->Yu z9h`1?k{koGyRSU8i+X^VHG;3;%TdItFRAVTQCrFYbyWjb$O{oLQ`ZNI=AQa z`p5Rs2Sf4`dyU*XUmW)nn80#xoR>=M`XW~Ww5-*A5FtRyca}Wg;W_0<*g`cW71BPs zCAYdF^P^T)?t%z`?NxilN_oMs1xw8%Hr>caA<(Tz-+ZN8f&X#0KppJ|aONc{-cs-% zFtFD)3mX7zk)5BH_};F4OCbA`oPw|%1Lu<2ggE9uPES2S{mE70bt{ldcscyi94%JCD6t5i>jdwI z+xam@D$-{2BQYmmKhyh!_0adH`6VXg{l^||U*EG6M$&6ijQ~TBnpU8hS)M8a&g}ph z94M4u1mZ@pC+blNV9TV>zKHfGz>9`Ed&h^|plzqhhypRgcFL%zdK#L*fXt&f-|R|v z{)tMVVYk=I9mQ<^A|XA_r;z1S4M@!8Vvad+$Q-W)J*IP=giWYd9Tv2tnI~A0BR~kg z@~r!YXv9m+*rgXSsT@3}1u;W2B6B=3*c(G{)tDELzdM19fthkT2zaM6X7STlx3&__ zZoe3!&0Sy;I91k-j&D$`pO6#oc5kTjEFINDsn22BN^%57`HWrv@Dt}_L#=eJTTEQ$(w9&UrG5PV5l`0#tJ8iT-$TScz`2^=iK@1L(^Dg?Cn={e zR7bM8q^GU)qumLI;JKLuAL;>Ix;EYmdhB?0oS`I)YfAT+J^rIRZ3k**d~+ze)d`y{ ztGD<2bHD83`}gl_h5A90FJ_jjXOs281r6)`V8y3tAwsm+KsDdfNs#^t~&P+Mt zZ$D-LZ%CewbKm0VLE!DNKW$`t0`37Dj}x5Hilc!Up;#Q-|5xDWY$>KO`pY%nDpKFS zr&@%&cL{M`hyAQcMUerfqjKBZsnW*)$#C8-pFSVv6imR`A+N;z1m9@gr@mc)xkq7d zUrIg2n4#ojB^HqePINQYXREIP%TKCWH2)pn2_K&V@TvTk_FdVR+N2H`YgiULDi<^* z>d|@-!#HOu+SjgFQ^>lccxy{S>x-5t1DD32Gmp`jNGT@wfL)~Q&c8(W0igCpBP zO$iZP?piz|2$#)%cu_jF&lAY8WEg`yrlZl6@ZyB@Q|-EM-=Iqq>~IJ(q8dU9wl;MR zaDie2#OKpAbHVVk*tK^0YVPCzaPj^Lt;A5~T>9vV|%Ud!?-T3ZzCu0Ar@@Lc<% zfPtz=fAeUCRQO}Lk&7=B70!NhA>8s!f%%pS)r3yQB|ZyyMyqF@aDC9w++r+LSauGl+^|UuHr&-Q>h!8ypmLohv=7|ZVIoe?g_?*dLF@1?*mhXFs*md)0=jskYl6Bt zn!0(09DvGyL&yr9%$r_h1jYLb8>*Lj2%5E0-8o8+dlye15s9U1_gX|gs!JD~h~u}? zt(U=DiLeGDm~iI6Wd*#d)j~1Py`3$(nX89(>*_{V56ib4NVu9d+eOW1y(iQXTIZ&2 ze-JNAYg-&@R8e)V#1Dh$in%H$bt0rn(K}t%o$QX(MgbFu!+nt_7+=wT)(SaLm~*DG zE`9du(?YS*4gpHc?(8o?jPv2R>Z56mkGE7{)B!Ps)EuB1=Zf}ik4;`u?6i$I*ETMq zK0*OF6T!>QxZq{ijrA!x@|pLXLB$VFbX&EidIyl<5y%xXcP2(H9z7e;tG;}%{(e7p z^_-^djC_C4+p1WT$J%h)>hpN;Aj*(EXJeG^Apgd{GZdN(Y65<*$wPpL~j~-@v8%9Oh@31cwf6Bg=v0H?^s#w)h zx$n#VOwuWZl3c=W7-`sq4v{>H9oY*$tueyB_e5c-op;HzyCExO-7%>xJ@Ci&wohioO@pmJrvY)TC1Z~Udw(}MbSd4nDbb+8$$m36_YyD8zLW5g zo(T24bRSmI*P&iedSBPu_{TPq2_jxx>BI=RUI23y>86^U1|cFKliUApLYvtWbb-Ae zpC`e~c77PV!_2{aIQ`uXe z7EYM=1*TJR`V-UYhMG<|O{7D=qa>S@sOWDgXPi$F#*e*LmE@_pRu=%Pv<;dtmaVrU zO^Pko?G7te0Ab{h=RYVP#r9u7Gib;Hh3=|wJ;pI$;@6$R&wu{d3skl-h(qep&h(9W zqTaF7nsj2yDnB16^3{u*<(Oq3QJ16Df1@H$Hg_`t7*Y(@Mk?K<6#Q0xW zjJA-F?KASQ3j*A|oQ`(Q6Gk0qj!y*Yp+%)|FZ&nf0RMMWBGDlXURVjpc)(CtM6A?# zYt}jZLK=&j9qIukS%XyihHNP6JFQ*N5J^vUJh$#Bjobqkf>raO8J%>ZH@)Zd>?k)` zU&g_^z8xTc0L}QWZxO~k7evYdeu=3cThe8{F^Op&JIaC3NDP1#YAl+6fe5I?=<8;B z-5ZB-`cEQ+`OOG*mup-$;Aj2==>N;v#JB9ZeER!nEyd}t$i z5SZzY&vi1-6_F7rt+6_gz3>v3lzdw^invs~2m3-4&)$xIF@h-k3qujrmUzqEzjhAu z3ElX|a9beLdSU}lz(taaL4`A{{rE*hQH=ptQQY0^$2XTmpFV$BTW4HK~C(Cz>XCCCA*c$puv1@7S9!Dd2%WVGZ&zH^eurjrV{zweL9AfI? z#|7^K+O66r_XT+U92nRxDk^%$&292l^sm9e9bKdDJ$-$L;wsB&zJ*o?p-MjaCNJz+DSy}%xuh$I!939=wa&^CBc4J=2z4A6` zY3aKq?Iuo6yMu#+fA;o@@bdDO8MjPLN%i&h{Tv$FX^~cUynUYe&~ptdYiq@!*rDIQ zH+%A-JDXpLpGKj4efaB_n?1Xfr0G+uo5jA<)6*T}(;A{7CcFZ}PeVfsDl1RWKF^2+ zsR$Go7oRaPY1R8Z{W&~5{EU~E+^om7;o;$vCr>^c!%Ewp@vnEd%WNC-2T z1nmS#3gzOxKOfHc+R}xWPwMGC8Cx@8DzWRAebz~R{nU5(Sq%T0XyI=adN-Hz{`uSZ zp9vY#JN@}O>;12P;(vF_|ND Mondrian conformal prediction intervals into robust portfolio constraints. > On a 276,869-loan out-of-time Lending Club evaluation, the selected pool93 body > point earns `$184.8K` on a `$1M` budget while passing the declared eight-level -> alpha grid (`V=0.035350`, `Gamma_CP=0.162616`, Markov cap `0.345084`, zero -> violation). The consolidated finite policy-grid frontier contains 50,010 +> alpha grid (`V=0.035350`, `Gamma_CP=0.162616`, `Gamma_res=0.073584`, exact +> Markov threshold `0.345084`, zero realized risk-tolerance excess). The consolidated finite policy-grid frontier contains 50,010 > deduplicated semantic policies, with 27,508 all-alpha above-floor policies, > showing that the result is not a single-point artifact. The contribution is > not higher AUC, but a reproducible bridge from calibrated probabilistic @@ -64,7 +64,7 @@ robusta verificable, no en otro leaderboard predictivo. | C2 | La capa conformal Mondrian produce cobertura útil y trazable para decisión. | Empírico-conformal | Coverage 90 `92.97%`, min group coverage `91.90%`, Winkler `1.111` | | C3 | El intervalo conformal se puede mapear a un conjunto de incertidumbre usado por un LP robusto. | Metodológico | Definiciones `u_i(alpha)`, `Gamma_CP`, policy modes | | C4 | El bound controla no-cobertura ponderada del funded set bajo supuestos distribution-free. | Teórico | `thm-conformal-feasibility`, Markov, `V <= sqrt(alpha)` | -| C5 | La policy oficial es el punto pool93 body/default, no el endpoint de máximo retorno. | Empírico-editorial | A35/A36, retorno `$184.8K`, Markov cap `0.345084`, `alpha_grid_pass=8/8` | +| C5 | La policy oficial es el punto pool93 body/default, no el endpoint de máximo retorno. | Empírico-editorial | A35/A40, retorno `$184.8K`, umbral Markov `0.345084`, `alpha_grid_pass=8/8` | | C6 | La frontera finita A35 muestra que el resultado no es un punto aislado. | Empírico | 50,010 políticas semánticas; 27,508 all-alpha above-floor policies | | C7 | La evidencia CRPTO y journal-package muestra robustez adicional sin cambiar la dirección del paper. | Robustez | A3--A36, Figuras 12--25 | @@ -130,7 +130,7 @@ selección sugerida es: | Figura 1: pipeline CRPTO | Cuerpo | Explica el aporte en una sola vista IJDS | `crpto_fig1_journal_pipeline.png` | | Figura bound: pila de claim | Teoría | Separa endpoint conformal, identidad determinística, supuesto ponderado y certificado exacto | `crpto_fig20_bound_claim_layers.png` | | Figura 2: alpha -> `Gamma_CP` -> funded set | teoría/método | Une parámetro conformal y decisión | `crpto_fig13_alpha_gamma_funded_set.png` | -| Tabla A35/A36 | Resultados | Muestra frontera finita y composición del funded set promovido | `crpto_tableA35_pool93_ijds_frontier.csv`, `crpto_tableA36_pool93_body_funded_grade_audit.csv` | +| Tablas A35/A40 | Resultados | Muestran la frontera finita y el baseline point-PD emparejado | `crpto_tableA35_pool93_ijds_frontier.csv`, `crpto_tableA40_pool93_point_baseline.csv` | | Tabla 1: métricas core | Resultados | Fija PD, CP y portfolio sin mezclar familias | `crpto_table0_key_metrics.csv` | | Tabla 2: champion y comparadores | Resultados | Economic vs theorem-tight vs balanced | `crpto_table1_champion_policy.csv` y A13 | | Tabla 3: robustez P1 | Apéndice corto | Post-selección, temporal, selector, shift | A3--A11 | @@ -141,7 +141,7 @@ Selección de figuras y tablas para el manuscrito. El paper no necesita mostrar todas las tablas en el cuerpo. El cuerpo debe proteger tres mensajes: la metodología, el punto pool93 promovido y la frontera -finita A35/A36. +finita A35 y la baseline A40. El appendix puede cargar la evidencia de stress y trazabilidad. ### Paquetes del online supplement (A--F) @@ -254,8 +254,8 @@ Una versión fuerte y honesta de las contribuciones sería: set y separamos explícitamente el tightening condicional de la garantía principal. 3. Mostramos que el enfoque produce una policy pool93 promovida con retorno - `$184.8K`, `V=0.035350`, `Gamma_CP=0.162616`, Markov cap `0.345084`, cero - violación y pass `8/8`. + `$184.8K`, `V=0.035350`, `Gamma_CP=0.162616`, `Gamma_res=0.073584`, umbral + Markov `0.345084`, exceso realizado cero y pass `8/8`. 4. Documentamos una frontera finita consolidada con 50,010 políticas semánticas y 27,508 all-alpha above-floor policies, por lo que el resultado no depende de un único punto elegido después de ver los datos. diff --git a/book/chapters/06b-guia-editorial-claims.qmd b/book/chapters/06b-guia-editorial-claims.qmd index 53b7ed7..5bbca67 100644 --- a/book/chapters/06b-guia-editorial-claims.qmd +++ b/book/chapters/06b-guia-editorial-claims.qmd @@ -46,7 +46,7 @@ empíricos, otros son de ingeniería reproducible. | 3 | El intervalo puede convertirse en conjunto de incertidumbre | Definición de `u_i(alpha)` y `Gamma_CP` | No tratar `Gamma_CP` como presupuesto ad hoc | | 4 | El bound controla no-cobertura ponderada del funded set | @sec-thm-feasibility y `V` | No decir que controla directamente una PD latente sin supuesto adicional | | 5 | Existe una frontera finita robusta en OOT | A35: 50,010 políticas semánticas; 27,508 all-alpha above-floor policies | No decir que eso elimina toda incertidumbre post-selección ni que certifica una región continua | -| 6 | El punto oficial es pool93 body/default, no el endpoint económico extremo | Retorno `$184.8K`, `V=0.035350`, `gamma_cp=0.162616`, Markov cap `0.345084`, pass `8/8` | No mezclarlo con el max-return endpoint de `$223K` ni con el punto de cap mínimo | +| 6 | El punto oficial es pool93 body/default, no el endpoint económico extremo | Retorno `$184.8K`, `V=0.035350`, `gamma_cp=0.162616`, `gamma_res=0.073584`, umbral Markov `0.345084`, pass `8/8` | No mezclarlo con el max-return endpoint de `$223K` ni con el punto de threshold mínimo | | 7 | La evidencia CRPTO fortalece el paper sin cambiar dirección | A7--A11, apéndice condicional, A25--A34 | No venderlo como validación live o universal | : Escalera editorial de claims del CRPTO {#tbl-crpto-claim-ladder} @@ -128,7 +128,7 @@ checkpoint MRM/fair-lending sin convertirlo en un claim legal. | `V` | No-cobertura ponderada del funded set | Cuánto default realizado quedó fuera de la cota | | `violation` | Exceso de riesgo ponderado sobre `tau` | Incumplimiento del límite de portafolio | | `alpha01_exact_pass` | Check exacto de bound en OOT | Semáforo para promoción paper-facing | -| `price_of_robustness` | Retorno esperado baseline no-robusta menos robusta (con signo; `-$14,465.69` en el champion) | Costo/beneficio económico de la robustez, auditable | +| `price_of_robustness` | A40 empareja candidatos y restricciones: costo realizado `5.875%`, reducción default/V `8.305` pp y reducción de threshold `43.55` pp | Costo económico de la robustez, solo con comparadores semánticamente equivalentes | : Diccionario operativo del CRPTO {#tbl-crpto-operational-dictionary} @@ -202,7 +202,7 @@ improvisar: si llega una objeción nueva, primero se busca aquí. |---|---|---| | Diagrama CRPTO | Figura 1 | Explica el método en una sola vista | | Escalera alpha -> `Gamma_CP` -> funded set | Teoría / método | Une conformal y robust optimization | -| Tabla A35/A36 | Resultados principales | Demuestra frontera finita y audit de composición del funded set | +| Tablas A35/A40 | Resultados principales | Demuestran frontera finita y trade-off contra point-PD emparejado | | Holdout temporal estricto CRPTO | Appendix o robustness checks | Responde post-selección | | Funded-set composition | Appendix | Útil para reviewers de crédito y riesgo | | A10 exact finalists | Appendix | Defiende el selector conformal | diff --git a/book/chapters/13-trazabilidad.qmd b/book/chapters/13-trazabilidad.qmd index 6f3762d..31b4fcb 100644 --- a/book/chapters/13-trazabilidad.qmd +++ b/book/chapters/13-trazabilidad.qmd @@ -7,7 +7,7 @@ page-layout: article Esta página es el **índice maestro** del CRPTO. Cumple un rol específico que ningún chapter narrativo cumple: dar a un revisor (académico, regulatorio o interno) la capacidad de tomar cualquier claim del paper, encontrar el artefacto exacto que lo soporta, identificar el script generador y verificar el guardrail que mantiene la coherencia. Es el contrato de trazabilidad operativa del proyecto. -Nota dual-tag (post-promoción pool93, 2026-07-02): los claims C1--C7 de esta página trazan la **cadena upstream congelada** (rebaseline `ijds-rebaseline-2026-06-07`, retorno `$170,464.54`, hoy floor declarado). El body claim del paper IJDS es el punto pool93 (`$184,832.48`, `8/8` alpha grid), anclado en `pool93_ijds_consolidated_governance.json` y las tablas A35--A39, con guardrail `tests/test_pool93_body_claim_sync.py`. +Nota de gobernanza (actualizada 2026-07-09): los claims C1--C7 trazan la **cadena upstream congelada** (rebaseline `ijds-rebaseline-2026-06-07`, retorno `$170,464.54`, floor declarado). El body claim IJDS es el punto pool93 (`$184,832.48`, `8/8`), anclado en la gobernanza `certificate-semantics-v2` y las tablas A35--A40, con guardrail `tests/test_pool93_body_claim_sync.py`. | Claim del paper | Artefacto que lo ancla | Guardrail que lo protege | Lectura rápida | |---|---|---|---| diff --git a/book/chapters/14-release.qmd b/book/chapters/14-release.qmd index 132c3c8..0cbfda8 100644 --- a/book/chapters/14-release.qmd +++ b/book/chapters/14-release.qmd @@ -128,7 +128,10 @@ Manifest de uso por sección del capítulo 14. | A31 | External OOT subperiod metrics | No | Sí | Sí | Prosper por año y Freddie por quarter; registra el borde 2015Q4. | | A32 | Prosper default-definition sensitivity | No | Sí | Sí | Verifica que la definición de default no decide sola el claim Prosper. | | A33 | Freddie segment sensitivity | No | Sí | Sí | Combined/green pasan alpha01; red queda como sensibilidad honesta. | -| A34 | Cross-dataset price of robustness | Sí o appendix corto | Sí | Sí | Muestra que el premium externo crece con default rate y que LC seleccionado queda como referencia favorable. | +| A34 | Cross-dataset price of robustness | Sí o appendix corto | Sí | Sí | Muestra que el premium externo crece con default rate; no reutiliza el comparador histórico retirado de Lending Club. | +| A35 | Policy-aware finite-grid frontier | Sí | Sí | Sí | Usa endpoint exacto para políticas lineales, capped y tail; no es óptimo continuo. | +| A36--A39 | Selected-allocation audits | No | Sí | Sí | Composición, cola, concentración y bootstrap de la asignación promovida. | +| A40 | Matched point-PD baseline | Sí | Sí | Sí | Mismos candidatos y restricciones operativas; cuantifica costo de retorno y reducción de riesgo. | Manifest de tablas para decidir cuerpo, appendix y tesis. ::: @@ -158,7 +161,7 @@ Manifest de tablas para decidir cuerpo, appendix y tesis. | Fig 22 | `crpto_fig22_external_replication` | Sí | Sí | Sí | Resume gates conformales y valor LP robusto en Prosper/Freddie. | | Fig 23 | `crpto_fig23_external_candidate_sensitivity` | No | Sí | Sí | Verifica que la réplica externa no depende de un pool pequeño. | | Fig 24 | `crpto_fig24_freddie_all_candidate_certificate` | Sí o appendix corto | Sí | Sí | Certificado all-candidate Freddie sobre `1,396,053` candidatos OOT. | -| Fig 25 | `crpto_fig25_price_of_robustness_scaling` | Sí o appendix corto | Sí | Sí | Conecta default rate, premium de robustez y contraste con el champion LC seleccionado. | +| Fig 25 | `crpto_fig25_price_of_robustness_scaling` | Sí o appendix corto | Sí | Sí | Conecta default rate y premium de robustez exclusivamente en las aplicaciones externas congeladas. | | Fig alpha-gamma | `crpto_fig_alpha_gamma_bound` | Quizá | Sí | Sí | Figura técnica del bound. | Manifest de figuras para extraer paper y appendix. diff --git a/book/chapters/16-fundamentos-conformal-optimizacion.qmd b/book/chapters/16-fundamentos-conformal-optimizacion.qmd index e520edd..beb3d9c 100644 --- a/book/chapters/16-fundamentos-conformal-optimizacion.qmd +++ b/book/chapters/16-fundamentos-conformal-optimizacion.qmd @@ -647,8 +647,15 @@ En la configuración del proyecto (`configs/optimization.yaml`), el parámetro ` ::: ::: {.callout-note} -## El signo importa: en este proyecto el precio salió favorable -La guía anterior describe el caso *típico*, donde la robustez cuesta retorno. El campo congelado `price_of_robustness` se define con signo: retorno esperado de la baseline no-robusta menos el de la policy robusta. Para el champion vale `-$14,465.69` (`-10.56%`) — **negativo** — porque el funded set conformal-robusto promovido no solo no sacrifica retorno esperado frente a la baseline, sino que lo supera (ver la instanciación empírica del Corolario 1 en @sec-crpto-results). Un PoR negativo no contradice la teoría: el corollary acota el precio *conceptual* máximo; el precio *medido* sobre este funded set resultó a favor de la prudencia. +## El signo exige una baseline point-PD emparejada +El campo congelado `price_of_robustness=-10.56%` queda retirado porque la +baseline histórica etiquetada como no robusta heredaba la restricción +`pd_high`. A40 reconstruye una baseline point-PD con los mismos 276,869 +candidatos, presupuesto, concentración, `tau=0.1715`, LGD y solver. CRPTO cede +`5.875%` de retorno realizado, reduce default/V ponderado en `8.305` puntos +porcentuales y baja el umbral Markov exacto en `43.55` puntos. Esta comparación +no cambia el certificado seleccionado; aclara su costo económico contra un +control semánticamente equivalente. ::: La frontera de Pareto entre retorno y robustez se explora en el script `scripts/optimize_portfolio_tradeoff.py`, que genera el *tradeoff curve* variando el parámetro $\gamma$ y registrando el PoR resultante. diff --git a/book/chapters/30-replicacion-multidataset.qmd b/book/chapters/30-replicacion-multidataset.qmd index ef40c25..e9689ea 100644 --- a/book/chapters/30-replicacion-multidataset.qmd +++ b/book/chapters/30-replicacion-multidataset.qmd @@ -140,13 +140,14 @@ condicional sería excesiva. | A32 Prosper default sensitivity | Main, `Defaulted` only y `Chargedoff` only pasan 90% y alpha01, todos con LP robusto positivo. | La definición de default no decide por sí sola el claim Prosper. | | A33 Freddie red/green | `both` y `green` pasan alpha01; `red` falla alpha01 (`0.9850`) aunque mantiene coverage90 y LP positivo. | FM48 combinado es el claim promovido; red queda como sensibilidad honesta. | -## El precio de robustez crece con el riesgo del panel +## El precio de robustez en las aplicaciones externas -La réplica externa no solo confirma que los gates se sostienen: revela un patrón -económico que un champion de un solo dataset no podía mostrar. El campo -`price_of_robustness_pct` --definido con el mismo signo que en Lending Club, como +La réplica externa no solo confirma que los gates se sostienen: muestra un patrón +económico que un solo dataset no podía revelar. El campo +`price_of_robustness_pct` --definido como `(no-robusto − robusto) / no-robusto`-- es **positivo** en las aplicaciones -congeladas externas y **crece monótonamente con la tasa de default del panel**. +congeladas externas y queda ordenado por la tasa de default del panel en los +cuatro casos observados. | Aplicación congelada | Default del panel | AUC | Precio de robustez | |---|---:|---:|---:| @@ -159,7 +160,7 @@ congeladas externas y **crece monótonamente con la tasa de default del panel**. Todas usan la misma receta sin búsqueda de champion. Fuente: `crpto_tableA34_price_of_robustness_cross_dataset.csv`. -![El precio de robustez crece con la tasa de default del panel: las aplicaciones congeladas (Freddie green/combinado/red, Prosper) forman una serie positiva y monótona, mientras el champion seleccionado de Lending Club queda como referencia favorable por debajo de cero.](../assets/figures/publication/crpto_fig25_price_of_robustness_scaling.png){#fig-book-price-scaling width="85%" fig-alt="Gráfico de líneas en eje x logarítmico: el precio de robustez sube de +1.00% a +9.46% al crecer la tasa de default del panel; Lending Club aparece como línea de referencia en -10.56%."} +![En las cuatro aplicaciones externas congeladas, el precio de robustez es positivo y queda ordenado por la tasa de default del panel.](../assets/figures/publication/crpto_fig25_price_of_robustness_scaling.png){#fig-book-price-scaling width="85%" fig-alt="Gráfico de líneas en eje x logarítmico: el precio de robustez aumenta de +1.00% a +9.46% entre cuatro aplicaciones externas congeladas ordenadas por tasa de default."} La lectura tiene base teórica limpia: más riesgo de default ensancha los intervalos conformales, así que el peor caso robusto descuenta más retorno y la @@ -169,18 +170,18 @@ prima por comprar la garantía de cobertura sube. Dentro de Freddie, el segmento sí sola --`green` y `red` tienen AUC casi idéntico pero distinta prima--: lo que manda es el riesgo irreducible del panel. -Lending Club es la excepción informativa: su precio de robustez es **favorable** -(`−10.56%`), pero ese número proviene del champion *seleccionado* por la búsqueda -bound-aware, no de una aplicación ciega de la receta. La selección encontró -precisamente un funded set robusto que además gana en retorno esperado. Reportar -ambas cosas --la prima positiva acotada bajo aplicación congelada y el caso -favorable bajo selección-- es más honesto y más fuerte que afirmar que la robustez -es siempre gratis o siempre cara. +Lending Club no entra en esa serie. El campo histórico `−10.56%` comparaba la +policy robusta contra una baseline rotulada `nonrobust` que todavía heredaba una +restricción de endpoint conformal. La auditoría point-PD corregida en +`tau=0.1715` es A40: CRPTO cede `5.875%` de retorno realizado y reduce `8.305` +puntos de default/V frente al control. No se mezcla con las cuatro aplicaciones +externas porque usa otro contrato. El claim activo de Lending Club se apoya en +la frontera policy-aware A35 y la baseline emparejada A40. -El mensaje editorial es directo: **la robustez nunca es económicamente -catastrófica**. Bajo aplicación congelada la prima se queda en un dígito a -dígito-bajo-doble, y bajo selección puede volverse favorable. CRPTO mide en qué -régimen está cada panel en vez de asumir un costo fijo de robustez. +El mensaje editorial defendible es más acotado: **en las cuatro aplicaciones +externas congeladas, la prima observada va de un dígito a bajo-doble-dígito**. +Cuatro puntos ordenados son compatibles con el mecanismo, pero no demuestran una +ley general ni que la robustez sea siempre barata. ## Qué mejora en el paper @@ -192,9 +193,9 @@ la receta conserva gates conformales y valor LP positivo en Prosper y Freddie". Ese matiz mejora tres claims: 1. El método no parece depender de una idiosincrasia de Lending Club. -2. El precio de robustez deja de ser un número aislado de Lending Club y se - vuelve un patrón interpretable: una prima positiva acotada que crece con el - riesgo del panel, comparable entre un P2P de consumo y un panel hipotecario. +2. El precio de robustez se vuelve un patrón externo interpretable: una prima + positiva acotada y ordenada por riesgo en los cuatro paneles observados, + comparable entre un P2P de consumo y un panel hipotecario. 3. Home Credit se descarta por criterio económico explícito, no por conveniencia de resultados. diff --git a/book/includes/_build-info.qmd b/book/includes/_build-info.qmd index 692731c..c4dbb67 100644 --- a/book/includes/_build-info.qmd +++ b/book/includes/_build-info.qmd @@ -1,3 +1,3 @@ ::: {.build-info} -Build: `ede5dd1` | Rama: `main` | Actualizado: `2026-06-09` +Build: `2a9b5e9` | Rama: `codex/ijds-refactor-lab-2026-07-08` | Actualizado: `2026-07-09` ::: diff --git a/book/index.qmd b/book/index.qmd index c775cf6..1fe0819 100644 --- a/book/index.qmd +++ b/book/index.qmd @@ -20,14 +20,15 @@ ![Las seis etapas del pipeline CRPTO --- datos y split temporal, PD calibrada, conformal Mondrian, conjunto de incertidumbre, LP robusto y política promovida con bound exacto --- anotadas con métricas del cierre IJDS pool93: retorno $184,832.48, $V=0.035350$, $\Gamma_{\text{CP}}=0.162616$ y frontera finita A35/A36.](assets/figures/editorial/diagrama-crpto.png){#fig-crpto-overview fig-alt="Diagrama maestro del pipeline CRPTO en seis etapas desde datos hasta política promovida auditable."} La superficie IJDS activa promueve el punto pool93 body/default: retorno -`$184,832.48`, `V = 0.035350`, `Gamma_CP = 0.162616`, Markov cap `0.345084`, -`alpha_grid_pass = 8/8` y frontera finita A35/A36. Los cierres anteriores quedan +`$184,832.48`, `V = 0.035350`, `Gamma_CP = 0.162616`, `Gamma_res = 0.073584`, +umbral Markov `0.345084`, `alpha_grid_pass = 8/8` y frontera finita A35. A40 +cuantifica el trade-off contra point-PD. Los cierres anteriores quedan como procedencia histórica, no como baseline operativo vigente. ::: {.callout-warning} Algunos capítulos largos de tesis conservan análisis históricos del champion anterior para trazabilidad. La fuente paper-facing actual es -`paper/CRPTO_ijds.qmd`, su suplemento y las tablas A35/A36. +`paper/CRPTO_ijds.qmd`, su suplemento y las tablas A35--A40. ::: ## Mapa editorial @@ -40,8 +41,8 @@ superconjunto académico. | Superficie | Qué defiende | Evidencia visible | |---|---|---| -| Paper IJDS | El claim estrecho: una PD calibrada congelada puede convertirse en una decisión robusta y auditable mediante conformal prediction. | Teoría mínima, Figura 1, exact funded-set certificate, frontera finita A35/A36, frontera regret-auditability y réplica externa compacta. | -| Online supplement | Que el resultado no es frágil ni cherry-picked. | A3--A36, ablations, stress tests, MRM/fairness, funded-set audit, SPO+, tail risk, diagnósticos multi-distribución/online y Prosper/Freddie. | +| Paper IJDS | El claim estrecho: una PD calibrada congelada puede convertirse en una decisión robusta y auditable mediante conformal prediction. | Teoría mínima, Figura 1, frontera policy-aware A35, baseline A40, regret-auditability y réplica externa compacta. | +| Online supplement | Que el resultado no es frágil ni cherry-picked. | A3--A40, ablations, stress tests, MRM/fairness, funded-set audit, SPO+, tail risk, diagnósticos multi-distribución/online y Prosper/Freddie. | | Tesis | El superconjunto académico y operativo del paper. | Fundamentos, arquitectura reproducible, gobernanza, trazabilidad, réplica multidataset, bibliografía extendida y future work controlado. | | Reproducibilidad | Que las cifras vienen de artefactos versionados, no de edición manual. | DVC, manifests, tablas/figuras regenerables, tests de sincronía y guardrails de champion congelado. | diff --git a/book/references.bib b/book/references.bib index b46aa72..ac527f5 100644 --- a/book/references.bib +++ b/book/references.bib @@ -1254,3 +1254,167 @@ @inproceedings{liu2021riskbounds pages = {22083--22094}, year = {2021} } + +% --- T. Literature expansion scan 2026-07-08 --- + +@article{sadana2025contextual, + author = {Sadana, Utsav and Chenreddy, Abhilash and Delage, Erick and Forel, Alexandre and Frejinger, Emma and Vidal, Thibaut}, + title = {A Survey of Contextual Optimization Methods for Decision-Making Under Uncertainty}, + journal = {European Journal of Operational Research}, + volume = {320}, + number = {2}, + pages = {271--289}, + year = {2025}, + doi = {10.1016/j.ejor.2024.03.020} +} + +@article{xu2025profit_uncertainty_credit, + author = {Xu, Yong and Kou, Gang and Ergu, Daji}, + title = {Profit-Based Uncertainty Estimation with Application to Credit Scoring}, + journal = {European Journal of Operational Research}, + volume = {325}, + number = {2}, + pages = {303--316}, + year = {2025}, + doi = {10.1016/j.ejor.2025.03.007} +} + +@article{xu2024profit_risk_credit, + author = {Xu, Yong and Kou, Gang and Peng, Yi and Ding, Kexing and Ergu, Daji and Alotaibi, Fahd S.}, + title = {Profit- and Risk-Driven Credit Scoring Under Parameter Uncertainty: A Multiobjective Approach}, + journal = {Omega}, + volume = {125}, + pages = {103004}, + year = {2024}, + doi = {10.1016/j.omega.2023.103004} +} + +@inproceedings{farinhas2024nonexchangeable_crc, + author = {Farinhas, Antonio and Zerva, Chrysoula and Ulmer, Dennis and Martins, Andre F. T.}, + title = {Non-Exchangeable Conformal Risk Control}, + booktitle = {International Conference on Learning Representations}, + year = {2024}, + url = {https://arxiv.org/abs/2310.01262} +} + +@article{hegazy2025valid_selection_conformal_sets, + author = {Hegazy, Mahmoud and Aolaritei, Liviu and Jordan, Michael I. and Dieuleveut, Aymeric}, + title = {Valid Selection Among Conformal Sets}, + journal = {arXiv preprint arXiv:2506.20173}, + year = {2025}, + url = {https://arxiv.org/abs/2506.20173} +} + +@article{hullman2025conformal_human_decision, + author = {Hullman, Jessica and Wu, Yifan and Xie, Dawei and Guo, Ziyang and Gelman, Andrew}, + title = {Conformal Prediction and Human Decision Making}, + journal = {arXiv preprint arXiv:2503.11709}, + year = {2025}, + url = {https://arxiv.org/abs/2503.11709} +} + +@article{djeundje2025dynamic_loan_portfolio_profitability, + author = {Djeundje, Viani B. and Crook, Jonathan and Andreeva, Galina}, + title = {The Devil in the Details: Dynamic Prediction of Loan Portfolio Profitability with Macroeconomic Drivers Through Multi-State Modelling}, + journal = {European Journal of Operational Research}, + volume = {327}, + number = {2}, + pages = {703--715}, + year = {2025}, + doi = {10.1016/j.ejor.2025.07.008} +} + +@article{stratigakos2026decision_calibrated_sets, + author = {Stratigakos, Akylas and Wen, Honglin and Spyrou, Elina and Pinson, Pierre}, + title = {Decision-Calibrated Prediction Sets for Robust Power System Operations}, + journal = {arXiv preprint arXiv:2606.02081}, + year = {2026}, + url = {https://arxiv.org/abs/2606.02081} +} + +@article{chen2026polyhedral_conformal_ro, + author = {Chen, Shuyi and Zhou, Wenbin and Zhu, Shixiang}, + title = {Learning Polyhedral Conformal Sets for Robust Optimization}, + journal = {arXiv preprint arXiv:2605.08506}, + year = {2026}, + url = {https://arxiv.org/abs/2605.08506} +} + +@article{wang2026optimal_decision_prediction_sets, + author = {Wang, Tao and Dobriban, Edgar}, + title = {Optimal Decision-Making Based on Prediction Sets}, + journal = {arXiv preprint arXiv:2602.00989}, + year = {2026}, + url = {https://arxiv.org/abs/2602.00989} +} + +@article{huang2026oce_rcps, + author = {Huang, Jiayi and Farzaneh, Amirmohammad and Simeone, Osvaldo}, + title = {Optimized Certainty Equivalent Risk-Controlling Prediction Sets}, + journal = {arXiv preprint arXiv:2602.13660}, + year = {2026}, + url = {https://arxiv.org/abs/2602.13660} +} + +@article{baesens2026foundation_credit_risk, + author = {Baesens, Bart and Goethals, Andreas and Lessmann, Stefan and De Vos, Simon and Bravo, Cristian and Martens, David and Medina-Olivares, Victor and Mues, Christophe and Oskarsdottir, Maria and vanden Broucke, Seppe and Verdonck, Tim and Verbeke, Wouter}, + title = {Foundation Models for Credit Risk Prediction: A Game Changer?}, + journal = {arXiv preprint arXiv:2605.18147}, + year = {2026}, + url = {https://arxiv.org/abs/2605.18147} +} + +@article{distaso2025business_cycle_losses, + author = {Distaso, Walter and Roccazzella, Francesco and Vrins, Frederic}, + title = {Business Cycle and Realized Losses in the Consumer Credit Industry}, + journal = {European Journal of Operational Research}, + volume = {323}, + number = {3}, + pages = {1024--1039}, + year = {2025}, + doi = {10.1016/j.ejor.2024.12.026} +} + +@article{ballegeer2025explanation_stability, + author = {Ballegeer, Matteo and Bogaert, Matthias and Benoit, Dries F.}, + title = {Evaluating the Stability of Model Explanations in Instance-Dependent Cost-Sensitive Credit Scoring}, + journal = {European Journal of Operational Research}, + volume = {326}, + number = {3}, + pages = {630--640}, + year = {2025}, + doi = {10.1016/j.ejor.2025.05.039} +} + +@article{wiberg2025ai_or, + author = {Wiberg, Holly and Dai, Tinglong and Lam, Henry and Kulkarni, Radhika}, + title = {Synergizing Artificial Intelligence and Operations Research: Perspectives from {INFORMS} Fellows on the Next Frontier}, + journal = {INFORMS Journal on Data Science}, + volume = {5}, + number = {1}, + pages = {14--23}, + year = {2025}, + doi = {10.1287/ijds.2025.0077} +} + +@article{morucci2022robust_matching_uncertainty, + author = {Morucci, Marco and Noor-E-Alam, Md. and Rudin, Cynthia}, + title = {A Robust Approach to Quantifying Uncertainty in Matching Problems of Causal Inference}, + journal = {INFORMS Journal on Data Science}, + volume = {1}, + number = {2}, + pages = {156--171}, + year = {2022}, + doi = {10.1287/ijds.2022.0020} +} + +@article{chen2025cost_sensitive_adversarial, + author = {Chen, Qiyuan and Al Kontar, Raed and Nouiehed, Maher and Yang, X. Jessie and Lester, Corey}, + title = {Rethinking Cost-Sensitive Classification in Deep Learning via Adversarial Data Augmentation}, + journal = {INFORMS Journal on Data Science}, + volume = {4}, + number = {1}, + pages = {1--19}, + year = {2025}, + doi = {10.1287/ijds.2022.0033} +} diff --git a/configs/crpto_publication_targets.yaml b/configs/crpto_publication_targets.yaml index 3c0cd4a..af7752d 100644 --- a/configs/crpto_publication_targets.yaml +++ b/configs/crpto_publication_targets.yaml @@ -1,4 +1,4 @@ -version: "2026-07-04" +version: "2026-07-09" project: paper-crpto decision_status: active @@ -38,7 +38,7 @@ primary_target: include: - "Pool93 finite-grid return-bound frontier as the active IJDS body claim." - "Frozen upstream PD/calibration/conformal chain retained as provenance and return floor." - - "A3--A39 as supplement/appendix evidence generated from frozen artifacts." + - "A3--A40 as supplement/appendix evidence generated from frozen artifacts." - "Regret-auditability frontier as the body-level SPO+/CRPTO trade-off." - "OCE/CVaR and robust satisficing as diagnostics, not new selectors." - "Cluster-aware dependence caveat/proposition in the theory supplement." @@ -70,19 +70,30 @@ journal_strengthening_pack: requires_new_run: false pool93_frontier_and_selected_allocation: status: include_body_and_supplement - role: "Active IJDS return-bound certificate: A35 frontier in the body, A36--A39 selected-allocation audits in the supplement." + role: "Active IJDS policy-aware certificate: A35 exact frontier in the body, A36--A39 selected-allocation audits, and A40 matched point-PD audit." artifacts: - reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv - reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.csv - reports/crpto/tables/crpto_tableA37_pool93_body_tail_risk.csv - reports/crpto/tables/crpto_tableA38_pool93_body_cluster_bound_audit.csv - reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.csv + - reports/crpto/tables/crpto_tableA40_pool93_point_baseline.csv - models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal/portfolio/pool93_ijds_claim_governance.json - - models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_ijds_consolidated_governance.json + - models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_frontier.json + - models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_governance.json + - models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_point_pd_baseline_audit.json + requires_new_run: false + matched_point_pd_baseline: + status: include_body_and_supplement + role: "A40 matched Lending Club baseline: same candidates and operating constraints, point PD as the only decision-semantic change." + artifacts: + - reports/crpto/tables/crpto_tableA40_pool93_point_baseline.csv + - reports/crpto/tables/crpto_tableA40_pool93_point_baseline.tex + - models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_point_pd_baseline_audit.json requires_new_run: false robust_satisficing_margins: status: include_supplement_or_short_body - role: "OR/committee margins for return, V, Gamma_CP, zero violation and robust-region pass." + role: "OR/committee margins for return, V, Gamma_CP, realized risk-tolerance excess and robust-region pass." artifacts: - reports/crpto/tables/crpto_tableA13_satisficing_margins.csv requires_new_run: false @@ -228,4 +239,4 @@ current_decision: - operations_research - informs_ijoo reason: "CRPTO currently has the strongest fit as reproducible decision-focused data science with OR substance." - p2_p3_boundary: "P2/P3 are no longer a blanket exclusion: the journal strengthening pack enters as diagnostics, framing and theory caveat; method-changing extensions remain future work and are not acceptance criteria." + p2_p3_boundary: "P2/P3 are no longer a blanket exclusion: the journal strengthening pack enters as diagnostics, framing and theory caveat; method-changing extensions remain outside the submitted claim and are not acceptance criteria." diff --git a/docs/SCOPE_AND_GOVERNANCE.md b/docs/SCOPE_AND_GOVERNANCE.md index 199e900..f69258e 100644 --- a/docs/SCOPE_AND_GOVERNANCE.md +++ b/docs/SCOPE_AND_GOVERNANCE.md @@ -30,18 +30,22 @@ CRPTO does not cover: The current IJDS paper-facing CRPTO body point is the promoted pool93 finite-grid frontier closure: -- run tag: `champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal` +- certificate tag: `champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2` +- source policy run: `champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext` - body/default policy mode: `capped_blended_uncertainty` (family `claim_micro_ext_body_cap345`, selected from the consolidated frontier) - robust return: `$184,832.48` - `V(alpha=0.01)=0.035350` - `Gamma_CP(alpha=0.01)=0.162616` -- Markov cap at `alpha=0.01`: `0.345084` -- exact alpha violation: `0.0` +- `Gamma_internalized(alpha=0.01)=0.089032` +- `Gamma_residual(alpha=0.01)=0.073584` +- exact endpoint budget at `alpha=0.01`: `0.245084` +- exact Markov loss threshold at `alpha=0.01`: `0.345084` +- realized risk-tolerance excess: `0.0` - declared alpha-grid pass: `8/8` - main paper-facing artifacts: A35 finite-grid frontier, A36 funded-set grade audit, A37 selected-allocation tail-risk repricing, A38 cluster-bound audit, - and A39 fixed-allocation bootstrap diagnostic. + A39 fixed-allocation bootstrap diagnostic, and A40 matched point-PD audit. The previous IJDS rebaseline is retained as historical provenance, not as the active body claim: @@ -70,7 +74,10 @@ Do not overwrite these protected files without an explicit revalidation plan: - `reports/crpto/tables/crpto_tableA37_pool93_body_tail_risk.csv` - `reports/crpto/tables/crpto_tableA38_pool93_body_cluster_bound_audit.csv` - `reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.csv` -- `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_ijds_consolidated_governance.json` +- `reports/crpto/tables/crpto_tableA40_pool93_point_baseline.csv` +- `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_frontier.json` +- `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_governance.json` +- `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_point_pd_baseline_audit.json` - `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal/portfolio/pool93_ijds_claim_governance.json` - `EXTRACTION_MANIFEST.json` diff --git a/docs/refactor/README.md b/docs/refactor/README.md index da21078..4fed59d 100644 --- a/docs/refactor/README.md +++ b/docs/refactor/README.md @@ -21,6 +21,7 @@ Each plan documents: | [`CONFORMAL_REFACTOR_PLAN.md`](CONFORMAL_REFACTOR_PLAN.md) | Yes (calibrator pickle) | Full public split executed 2026-06-13; `src.models.conformal` is now a package facade with strict-typed submodules. | | [`MAPIE_MIGRATION_PLAN.md`](MAPIE_MIGRATION_PLAN.md) | Yes (intervals parquet) | Runtime is already MAPIE 1.x and the drift report is green; protected reruns still require explicit approval. | | [`archive/FEATURE_CONFIG_PARQUET_PLAN.md`](archive/FEATURE_CONFIG_PARQUET_PLAN.md) | Yes (downstream stages) | Executed 2026-06-13 and archived; `feature_config.pkl` retired from the live DVC DAG and manifest. | +| [`ijds_tooling_refactor_lab_2026-07-08.md`](ijds_tooling_refactor_lab_2026-07-08.md) | No (tooling/refactor only) | Active and full `ty` advisory scopes are clean; `pyrefly` is experimental; `pdoc`/`prek` are optional local helpers before IJDS submission. | Executed lanes now in `main`: diff --git a/docs/refactor/ijds_tooling_refactor_lab_2026-07-08.md b/docs/refactor/ijds_tooling_refactor_lab_2026-07-08.md new file mode 100644 index 0000000..16f6782 --- /dev/null +++ b/docs/refactor/ijds_tooling_refactor_lab_2026-07-08.md @@ -0,0 +1,419 @@ +# IJDS tooling and refactor lab - 2026-07-08 + +## Decision + +This branch keeps the CRPTO workflow intentionally narrow for IJDS. The daily +path should optimize for claim integrity, reproducible paper outputs and low +maintenance, not for adopting every new Python tool. + +## Tooling decisions + +| Tool | Decision | Rationale | +| --- | --- | --- | +| `uv` | Keep as canonical environment/package runner. | Already matches the repo, lockfile and Windows-first workflow. Official docs position it as a fast Python package/project manager: . | +| `ruff` | Keep as formatter/linter gate. | It already replaces separate formatter/import/lint tools with one fast gate: . | +| `ty` | Use a pinned daily advisory and a blocking final full-scope gate. | `just type-advisory` keeps the active IJDS-safe surface visible without competing with `mypy`; `just type-advisory-full` is clean and now blocks `submission-check` on future diagnostics. Official docs: . | +| `pyrefly` | Do not gate before submission. Use only for targeted experiments. | `uvx pyrefly` works (`1.1.1`), but active-scope trial produced 56 diagnostics, mostly pandas/matplotlib inference noise. Pyrefly is stable and fast, but not yet lower-maintenance than `ty` for this repo: . | +| `pdoc` | Add optional local API-doc recipe only. | `just api-docs-core` builds ignored docs for core optimization/calibration/evaluation modules. Useful for technical inspection, not IJDS-critical. Docs: . | +| `prek` | Add compatibility validation, not a full migration. | `just hooks-check` validates the existing `.pre-commit-config.yaml` with both `pre-commit` and `prek`. `prek` is a fast drop-in alternative, but changing hook execution semantics before submission is unnecessary. Docs: . | +| `commitizen` | Do not adopt before IJDS submission. | Commitizen helps teams enforce conventional commits/changelogs, but CRPTO is single-author and the paper/release checklist matters more than semantic-version automation. Docs: . | + +## Implemented process simplifications + +- Added `scripts/run_ty_advisory.py` so pinned `ty==0.0.57` has two deterministic + scopes: active IJDS path and full repository debt. +- Updated `just type-advisory` to use the active scope. Current result: + `ty advisory clean` over 102 active files. +- Added `just type-advisory-full`; after the final cleanup pass it is also + clean over 131 files. Optional TabPFN/SPO/cuOpt dependencies now load through + explicit optional-import helpers, and retired generic search entrypoints fail + with actionable messages instead of unresolved imports. The previously noisy + pandas/PD/conformal typing issues in protected scripts were removed without + changing drift. +- Updated `just submission-check` to enforce the full `ty` scope now that it is + clean and cheap. `mypy` remains the stable contractual type gate; `ty` adds a + second fast regression check without introducing Pyrefly's duplicate noise. +- Added `just api-docs-core`; generated output lives in ignored + `reports/api-docs/`. +- Added `just hooks-check`; both `pre-commit validate-config` and + `prek validate-config` pass. +- Added `just complexity-report` as an explicit `radon` report over `src/` and + `scripts/`. It is intentionally a refactor radar, not a submission gate, + because remaining long scripts include historical/protected search + entrypoints that should not be rewritten before IJDS without a concrete + claim-risk reduction. +- Split the policy arithmetic inside + `src.optimization.portfolio_model.compute_effective_pd` into small helpers + for clipped deltas, quantiles, blending, and tail selection. The public API + and policy semantics are unchanged, but the main policy resolver no longer + appears in the complexity report. +- Split `src.evaluation.model_shift.interpret_model_shift` into named + structural-shift, predictive-degradation, shift-type, governance-posture and + p-value-note helpers. This keeps the MRM/governance semantics auditable and + removes the module from the C-or-higher complexity report. +- Split `src.models.conformal_tuning.shrink_group_multipliers` into a small + immutable shrink context plus private helpers for factor normalization, + interval application, metric calculation, constraint checks, candidate + generation, tie-breaking and reporting. The public API and greedy policy are + unchanged, but the conformal shrink step is now easier to audit against the + paper's coverage/fairness claims. +- Extracted the shared continuous-portfolio LP algebra in + `src.optimization.portfolio_model` so SciPy HiGHS and native highspy consume + the same constraint matrix, bounds and objective coefficients. This removes + duplicated budget/PD/purpose/slack construction. +- Rewired the native cuOpt adapter to consume that same shared LP component + builder instead of reconstructing budget, concentration, PD-cap and slack + rows locally. This keeps the optional GPU backend aligned with the canonical + CPU formulation and removes another D-level complexity hotspot. +- Split `src.models.optuna_tuning.train_catboost_tuned_optuna` from a single + F-level HPO orchestrator into explicit helpers for local/global search-space + materialization, feature-prior normalization, constraints, incumbent metrics, + objective evaluation, storage/study creation, trial enqueueing, trial + selection and final fit/refit. The CatBoost/Optuna defaults and public return + shape are preserved, while the HPO path is now much easier to inspect when + defending the frozen model recipe. +- Split `scripts.build_papers_tesis_deep_audit.write_audit` into small Markdown + section builders for inventory, paper-facing literature, extended thesis + lanes, visual curation, bibliography control and closeout. This keeps the + local literature audit regenerable without one large opaque memo writer. +- Split `scripts.generate_governance_status._build_explanation_drift_report` + into helpers for recent-period selection, segment construction, SHAP ranking, + feature PSI details and per-segment pass/fail rows. This clarifies the MRM + explanation-drift logic without changing the governance artifact contract. +- Split the `scripts.generate_governance_status` orchestrator into explicit + threshold/path dataclasses plus helpers for train/test loading, JSON sidecar + reads, drift metrics, model-shift interpretation and status serialization. + This keeps `governance_status.json`, `model_shift_status.json` and parquet + paths unchanged while removing the remaining E-level main function. +- Split `scripts.run_comparison._gate_ab_no_regression` into helpers for A/B + return extraction, current self-gate evaluation and baseline comparison + warnings. The gate still passes only on the self no-regression rule, with the + documented selective-ambiguity cross-scenario exception preserved. +- Split the remaining D-level comparison-report helpers in + `scripts.run_comparison`: artifact/status metadata now has explicit source, + observation, timestamp-skew and run-tag-coherence helpers, and comparison + report writing now separates gate execution, gate field extraction, quality + contract construction and JSON/Markdown emission. This makes the promotion + evidence easier to inspect without changing the gate semantics. +- Split `scripts.generate_crpto_figures._crpto_fig8_alpha_pareto` into helpers + for semantic column detection, variant styling, alpha sorting, tick labels and + annotation offsets. This keeps the IJDS alpha-sweep figure logic auditable + without changing the plotted data or the figure contract. +- Split `scripts.run_fairness_audit` so SHAP per-group interpretation, + CatBoost SHAP preparation, Fairlearn sidecar bootstrap summaries, primary + status construction and SHAP status writing are isolated helpers. This turns + the fairness audit script from a mixed I/O/analysis block into a clearer + pipeline while preserving thresholds, sidecar paths and JSON/parquet + contracts. +- Split `scripts.validate_conformal_policy` into helpers for config/sensitivity + loading, namespace application, alert fallback, valid interval extraction, + Winkler/MAPIE cross-checks, compensated Winkler policy handling, material + check construction and latest-month selection. The validation contract and + output schema stay the same, but the conformal promotion gate is now much + easier to audit against the IJDS coverage/width/group-coverage claims. +- Split `scripts.run_crpto_vs_spo_stability` into import-safe optional SPO + loading plus helpers for period masks, deterministic per-period sampling, + coverage aggregation, detail rows and summary JSON. The output contract stays + the same, but tests can now import the module without PyEPO/Torch, and period + sampling no longer depends on Python's randomized `hash()` seed. +- Split `scripts.select_economic_portfolio_policy` into explicit selector + settings, decision inputs, candidate evaluation, hard-filter eligibility, + A/B-like ranking, fallback construction and payload serialization helpers. + This preserves the champion-policy/status JSON contracts while making the + economic selector auditable as a sequence of declared gates instead of one + long mixed orchestration block. +- Split `scripts.simulate_ab_test._resolve_robust_policy` into champion-policy + selection, selected-policy normalization, robustness-summary validation, + summary row choice and default fallback helpers. This makes the A/B audit + policy precedence explicit: champion artifact first, summary second, fallback + last, with `explicit_champion_only` still failing loudly when the artifact is + absent. +- Split `scripts.benchmark_conformal_variants` into benchmark-data loading, + normalized search-space construction, variant accumulation, global/Mondrian/ + cross-conformal appenders, calibration-size sensitivity rows, final frame + assembly and artifact writing. The benchmark still writes the same parquet + and JSON surfaces, but the conformal experiment is now inspectable as stages + instead of one F-level orchestrator. +- Split `scripts.benchmark_pd_set_prediction` into set-benchmark data loading, + settings normalization, per-variant prediction, calibration-size sensitivity, + benchmark matrix assembly, slice summaries, promotion-gate calculation, + status payload construction and artifact writing. The binary set-prediction + sidecar remains a triage/abstention diagnostic, but its evidence path is now + explicit and no longer an E-level `main`. +- Split the high-risk paths in `scripts.train_pd_model`: config defaults, + CLI overrides, replay expectation checks, calibration backtests, Optuna seed + replay, tuned-CatBoost/HPO orchestration, decision-threshold resolution, + MAPIE statistical calibration tests, walk-forward diagnostics and SHAP export + now live behind named helpers. This preserves the training contracts while + making the PD replay and evidence path easier to audit. The script still has + a long main orchestrator, but it is now C-level instead of D/F-level and no + D-level helper remains. +- Split `scripts.generate_conformal_intervals` so feature resolution, + contract-matrix alignment, tuning-grid normalization, 90% Mondrian tuning + search, 90% coverage-floor/shrinkback evidence, optional global rebalance, + 95% alpha selection and final artifact-table/payload persistence are + explicit helpers. The conformal interval generator's `main` is now B-level: + it reads like the paper's certificate sequence instead of mixing tuning, + coverage policy and artifact-writing branches. +- Split `scripts.search.run_pool93_ijds_local_refinement` so the IJDS finite + policy-grid construction is organized by declared profile/family rather than + one F-level function. The candidate generator is now covered by per-profile + semantic-key fingerprints, including the terminal `37,068`-policy surface, + so future edits cannot silently change the paper-facing grid denominators. +- Split the `scripts.search.run_pool93_ijds_local_refinement` entrypoint into + parser, path, conformal-source, candidate, manifest, resume, pending-task, + progress persistence, serial/parallel execution and final-output helpers. + The candidate-grid fingerprints remain unchanged for every declared profile, + including the terminal `37,068`-policy surface, and no exact refinement run + was executed. +- Split `scripts.search.run_portfolio_bound_exact_eval` into explicit context + paths, exact-evaluation plan, resume/cache handling, pending-task iteration, + selection payload writing and final status helpers. This removes the D-level + `main()` from the exact finite-grid evaluator without running the protected + `crpto.portfolio.bound_exact_eval` search stage or changing artifact paths. +- Split `scripts.search.run_portfolio_bound_aware_search` so parser + construction, typed grid/execution state, run paths, budget profiles, + search-space payloads, selection context, frontier artifact writes, + frontier-only completion, external exact delegation, in-process exact + evaluation, success/failure cleanup and selection output writes are explicit + helpers. The protected search stage was not executed; the refactor clarifies + the finite-grid certificate plumbing and leaves no C-or-higher block in the + file. The targeted policy-family grid is table-driven so segment-tail + families are easier to audit. +- Added `src.optimization.certificate_semantics` as the code-level source of + truth for the eight IJDS alpha levels. Pool93 refinement, the bound-aware CLI + default and the regret-auditability portfolio command now consume the same + tuple/CSV contract. A sync test checks the code constant against the + paper-facing search profile and active claim registry. +- Split `scripts.search.run_regret_auditability_sandbox` so sandbox-local PD + config snapshots are assembled by feature/profile, model params, + Venn-Abers calibration, HPO/warm-start, validation, output paths, threshold + disablement and sandbox metadata helpers. The resumable command scheduler now + separates phase grouping, resume skips, launch, completion logging and + PD-phase winner selection. Command planning is now split into PD incumbent, + PD lane, conformal, portfolio and metrics builders; the former C(20) + `build_phase_commands` no longer appears in the C-level report. The portfolio + phase now defaults to the declared eight-level IJDS alpha grid instead of an + older seven-level exploratory grid. No sandbox commands or protected stages + were run. +- Split `scripts.search.run_conformal_reopen_search` so parser construction, + resume-vs-fresh phase-1 materialization, OOT confirmation and optional phase-2 + promotion are explicit helpers with small dataclass handoffs. This removes the + last live D-level search orchestrator without executing the reopen search or + touching frozen conformal artifacts. +- Split the phase-2 calibrator tournament inside + `scripts.search.run_conformal_reopen_search` into explicit helpers for method + normalization, progress-state writes, baseline metric fitting, degradation + gating, holdout candidate execution, candidate ranking and final OOT + confirmation. The phase-2 search no longer appears in the C-or-higher report, + and no reopen search was executed. +- Split `scripts.experiments.run_champion_claim_max_downstream._portfolio_command` + into base-command, frontier-option, execution-option and cuOpt-option + helpers. The downstream watcher remains an isolated experiment lane, but its + portfolio search command is now inspectable and covered for proxy-vs-exact + sampling, exact-python and cuOpt flags. +- Split the legacy Pyomo `solve_portfolio` wrapper into backend solving, + result extraction and termination-status helpers. After this pass, + `src.optimization.portfolio_model` no longer appears in the C-or-higher + complexity report. + +## Code refactor stance + +The useful pre-submission refactor lane is not a broad rewrite. It is: + +1. Keep `mypy` as the contractual gate. +2. Keep `ty` active scope clean so new IJDS-path issues stand out. +3. Convert pandas/Pyomo dynamic edges only where the change is local and tests + can cover it. +4. Keep full-scope historical/protected diagnostics visible through + `just type-advisory-full`, but do not install optional TabPFN/SPO/cuOpt + stacks unless an isolated experiment needs them. +5. Stop live-code complexity cleanup at this point: `src` and active `scripts/` + no longer have D-or-higher radon findings. The only remaining D-level report + is in `scripts/archive/`, so further pre-submission refactors should happen + only when they reduce a concrete claim-risk or maintenance burden. + +## Current validation evidence + +- Focused `ruff` and `mypy` checks passed for edited modules. +- Focused tests passed for conformal adapters, calibration pickle compatibility, + TabPrep challengers, MLflow tracing, MRM report generation and the new + `ty` wrapper. +- Focused policy/portfolio tests pass, including exact regression checks for + segment-tail and segment-relative-tail effective-PD semantics. +- Focused model-shift tests pass, including structural-only, predictive-only, + mixed and stable governance postures. +- Focused conformal-tuning tests pass, including new regression coverage for + temporal-factor shrinkage and the initial-infeasible report path. +- Focused portfolio tests pass for sparse HiGHS vs Pyomo equivalence, native + highspy vs sparse HiGHS equivalence, native fallback behavior and the PD + slack/min-budget case. +- Focused cuOpt adapter tests pass with a fake cuOpt API, covering shared LP + matrix handoff, solver settings, generated log files, allocation payloads, + PD slack and non-feasible termination handling without requiring RAPIDS on + Windows. +- Focused PD-model tests pass, including small real CatBoost/Optuna runs for + tuned-vs-default predictions and local-refine materialization. +- Focused literature-audit memo tests pass for editorial sections, experiment + rows, bibliography-status counts and the no-champion-change boundary. +- Focused governance tests pass for overall/grade explanation-drift rows, + insufficient-support empty reports and the public governance-status summary, + checks and artifact-path contract. +- Focused run-comparison tests pass for ordinary A/B no-regression and the + selective-ambiguity cross-gate exception, plus artifact metadata coherence + and the causal/CATE insights-only run-tag mismatch exception. +- Focused CRPTO figure tests pass for alpha-sweep column detection, variant + labels/colors, alpha sorting, tick labels and annotation offsets. +- Focused fairness-audit tests pass for threshold resolution, auto-selected + decision policy writing, SHAP categorical detection/fill behavior and + per-group SHAP driver summaries. +- Focused conformal-policy validation tests pass for MAPIE current/legacy MWI + signatures, valid interval extraction, compensated Winkler gates, material + status JSON fields, official-baseline run-tag fallback, artifact namespaces + and sensitivity overrides. +- Focused CRPTO-vs-SPO stability tests pass for artifact presence, + deterministic per-period sampling seeds and summary/detail aggregation. +- Focused economic-selector tests pass for robust promotion, fallback, + breadth-aware v2 selection, A/B-like v3 ranking and breadth hard filters. +- Focused A/B policy-resolution tests pass for guardrail champion priority, + summary fallback when no champion artifact exists and explicit champion-only + missing-artifact failure. +- Focused conformal-variant benchmark tests pass for namespaced shadow output + paths and search-space normalization/deduplication. +- Focused PD set-prediction tests pass for namespaced shadow output paths, + settings normalization/fallback coercion and the guardrail promotion gate. +- Focused PD training config tests pass for CLI/replay overrides, feature + resolution, split loading/sampling and the new Optuna replay gate-tier + ranking contract, plus walk-forward stage normalization and SHAP summary + export. +- Focused conformal interval CLI tests pass for tuple parsers, tuning-grid + normalization, tuning candidate counts, split materialization, global + rebalance no-op behavior, 95% alpha tie-breaking, tuning-selection + materialization, learned floor-policy application, temporal-segment + eligibility and final artifact-table metadata preservation. +- Focused pool93 local-refinement tests pass for all declared candidate-grid + profiles (`stage1`, `expanded`, `claim_expanded`, `claim_micro`, + `claim_micro_ext`, `claim_bound_closure`, `claim_bound_floor_closure`, + `claim_bound_terminal`) using stable semantic-key fingerprints and for the + finite-grid claim-summary protocol. Additional helper tests cover manifest + path coherence and pending candidate-alpha task construction after the + entrypoint split. `mypy` is clean for + `scripts/search/run_pool93_ijds_local_refinement.py`, and the former D-level + `main()` no longer appears in the C-or-higher report. +- Focused exact-eval tests pass for completed-cache reuse, partial-cache + resume filtering, full-universe seed deduplication, alpha-grid payload + normalization and priority-context ordering. `mypy` is clean for + `scripts/search/run_portfolio_bound_exact_eval.py`, and `radon` reports the + exact-eval `main()` as A-level with no C-or-higher blocks. +- Focused bound-aware search tests pass for shortlist preservation, exact + aggregation ranking, table-driven policy-grid order, budget-profile parsing, + shared alpha-grid defaults, separated proxy/exact sampling, exact-work counts + and selection-context path/search-space coherence. `mypy` is clean for + `scripts/search/run_portfolio_bound_aware_search.py`, and `radon` reports no + C-or-higher blocks in that file. +- Focused champion-reopen orchestration tests pass for paper-facing downstream + candidate selection and the portfolio command builder, including separated + proxy/exact sampling plus cuOpt option propagation. `mypy` is clean for + `scripts/experiments/run_champion_claim_max_downstream.py`, and its former + D-level `_portfolio_command` no longer appears in the C-or-higher report. +- Focused regret-auditability sandbox tests pass for protected-output rejection, + sandbox lane materialization, PD snapshot writing, external output-dir + command planning, declared alpha-grid propagation, phase grouping, resume + skip behavior and validation-policy scaling. `mypy` is clean for + `scripts/search/run_regret_auditability_sandbox.py`; its former D-level + `write_pd_config_snapshot` and `_run_commands` plus the former C-level + `build_phase_commands` no longer appear at those thresholds. +- Focused conformal-reopen tests pass for phase-2 design fallback, resume + source-path preservation, OOT confirmation ranking, phase1-only phase-2 skip + behavior, explicit calibrator metric-gate skips and final phase-2 candidate + ranking/confirmation. `mypy` is clean for + `scripts/search/run_conformal_reopen_search.py`, and `radon` reports no + D-or-higher blocks in that file. +- `uvx radon cc src -s -n D` returns no findings after the conformal tuning, + portfolio, cuOpt and Optuna refactors. +- `uvx radon cc scripts/run_comparison.py -s -n D` returns no findings after + the comparison metadata/report split. +- `uvx radon cc scripts/generate_crpto_figures.py -s -n D` returns no findings + after the alpha/Pareto figure refactor. +- `uvx radon cc scripts/run_fairness_audit.py -s -n D` returns no findings + after the fairness SHAP/Fairlearn sidecar split. +- `uvx radon cc scripts/validate_conformal_policy.py -s -n D` returns no + findings after the conformal validation split. +- `uvx radon cc scripts/generate_governance_status.py -s -n D` returns no + findings after the governance status orchestration split. +- `uvx radon cc scripts/run_crpto_vs_spo_stability.py -s -n D` returns no + findings after the optional-dependency and aggregation split. +- `uvx radon cc scripts/select_economic_portfolio_policy.py -s -n D` returns + no findings after the selector orchestration split; `main` is now A-level. +- `uvx radon cc scripts/simulate_ab_test.py -s -n D` returns no findings after + the robust-policy resolver split. +- `uvx radon cc scripts/benchmark_conformal_variants.py -s -n D` returns no + findings after the benchmark orchestration split. +- `uvx radon cc scripts/benchmark_pd_set_prediction.py -s -n D` returns no + findings after the set-prediction sidecar split. +- `uvx radon cc scripts/train_pd_model.py -s -n D` now returns no findings; + `main` is C(19) and all helper functions are below D-level after the + PD-training orchestration split. +- `uvx radon cc scripts/generate_conformal_intervals.py -s -a` now reports + average complexity A after the conformal generator split; `main` dropped + from 83 to B(9). The remaining C-level logic is localized in + `_build_90_interval_evidence` and `_load_conformal_inputs`. +- `uvx radon cc scripts/search/run_pool93_ijds_local_refinement.py -s -n C` + no longer reports `_generate_candidate_grid` or `main` as D-level; candidate + generation, claim summarization and entrypoint flow are helperized and + fingerprint-tested. +- `just complexity-report` now reports only + `scripts/archive/search/monitor_regret_auditability.py::render` at D-level; + the active `src/` and `scripts/` surfaces are clear of D-or-higher findings. +- `just type-advisory` passes clean. +- `just type-advisory-full` passes clean; the latest report is written to + `reports/ci/ty-advisory-full.txt`. +- `just drift-gate` stayed bit-exact after touching PD/conformal scripts: + max absolute diffs for predictions, intervals and score-band edges were + `0.000e+00`. +- `just submission-check` passes with the full `ty` advisory, body/supplement + Quarto renders, and the official IJDS LaTeX fallback build; the current + official PDF has 28 pages, References begin on page 24, and the build is + citation/reference clean. +- `just test` passes. + +## Remaining caution + +This branch touched `src/models/optuna_tuning.py`, conformal adapter code and +the conformal tuning shrink path in small interface/refactor-only ways. The +latest `just drift-gate` stayed bit-exact, but keep it in the promotion +checklist alongside the standard submission gates because these modules sit +close to the paper's certificate. + +## Portfolio input and baseline semantics audit (2026-07-09) + +- Added `src/optimization/input_alignment.py` as the single alignment contract + for the canonical and trade-off portfolio entrypoints. It enforces one-to-one + ID or `_row_number` matching, preserves interval-origin columns, rejects + duplicate/missing keys, and makes positional sampling reproducible over the + full universe. +- Replayed canonical alignments at 17, 5,000 and 276,869 rows. ID and + `pd_high_90` fingerprints were bit-identical before and after the refactor in + both entrypoints. +- Added `portfolio_model.solution_allocation_vector` as the validated contract + for dense and sparse solver payloads. Migrated the primary optimizer, + trade-off wrapper, economic selector, evidence audit, alpha--gamma validator, + body-allocation audit and pool93 refinement. Their previous all-row indexing + or local fallback copies were incompatible or redundant under the modern + sparse solver payload. +- Corrected `robust=False` semantics: it now forces `point_estimate`, `gamma=0` + and point PD in the optimization constraint. Previously an endpoint override + had precedence, so the stored `nonrobust` baseline was still constrained by + `pd_high`. +- The preliminary read-only comparison at `tau=0.175` was superseded by the + matched A40 audit: same 276,869-candidate universe, `$1M` budget, + `tau=0.1715`, concentration and LGD contracts, and solver settings. Point-PD + earns `$196,369.14`; the selected CRPTO allocation earns `$184,832.48`, a + cost of `$11,536.66` (`5.875%`) alongside an 8.305 pp reduction in realized + weighted default and a 43.55 pp reduction in the exact loss threshold. The + historical signed-price interpretation and preliminary unmatched comparison + are retired from active paper surfaces; protected historical tables remain + provenance. +- Focused tests cover source-column preservation, deterministic sampling, + key-integrity failures, wrapper parity, effective nonrobust policy metadata, + and dense/sparse allocation payloads. diff --git a/docs/research/README.md b/docs/research/README.md index 9dcf911..9b572aa 100644 --- a/docs/research/README.md +++ b/docs/research/README.md @@ -40,13 +40,20 @@ perenne y lo que el código lee/escribe. limpieza local de peso y frontera entre refactor estricto y nueva corrida tolerante. - `ijds_scientific_upgrade_audit_2026-07-07.md` — separación entre mejoras de - paper que pueden entrar con evidencia congelada y extensiones que requieren - un nuevo resultado CRPTO v2. + paper que pueden entrar con evidencia congelada y extensiones que quedan + fuera del claim enviado salvo un nuevo protocolo etiquetado. - `ijds_corpus_claims_improvement_plan_2026-07-07.md` - analisis con `academic-pdf-intake` del paper, supplement, submission PDF y corpus `Papers_tesis`; sus recomendaciones editoriales IJDS quedaron aplicadas y se conserva como trazabilidad, no como backlog abierto. +- `ijds_literature_expansion_scan_2026-07-08.md` - scan de literatura externa y + local para nuevas referencias IJDS: contextual optimization, incertidumbre de + credit scoring, conformal no-exchangeable, post-selection y comparadores + decision-calibrated 2026. +- `pool93_certificate_semantics_v2_2026-07-09.md` - auditoría consolidada de la + descomposición policy-aware, corrección exacta de la frontera A35 y baseline + point-PD emparejada A40; reemplaza el memo preliminar de baseline. ## Registros de gobernanza (decisiones; no se re-ejecutan sin permiso) diff --git a/docs/research/active_claims_2026-07-04.md b/docs/research/active_claims_2026-07-04.md index c22ef66..9e77c51 100644 --- a/docs/research/active_claims_2026-07-04.md +++ b/docs/research/active_claims_2026-07-04.md @@ -1,4 +1,4 @@ -# CRPTO Active Claim Registry - 2026-07-04 +# CRPTO Active Claim Registry - 2026-07-09 This registry is the current source of truth for paper-facing CRPTO claims. It supersedes older research notes that centered the `45/45` local region or the @@ -16,6 +16,8 @@ Current body/default point: - terminal run tag: `champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal` +- active certificate-semantics tag: + `champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2` - body point source run: `champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext` - policy family: `claim_micro_ext_body_cap345` @@ -28,9 +30,11 @@ Current body/default point: - return-floor surplus: `$14,367.94` - `V(alpha=0.01)`: `0.035350` - `Gamma_CP(alpha=0.01)`: `0.162616` -- endpoint budget upper at `alpha=0.01`: `0.24508374` -- Markov cap at `alpha=0.01`: `0.34508374` -- exact alpha violation: `0.0` +- `Gamma_internalized(alpha=0.01)`: `0.089032` +- `Gamma_residual(alpha=0.01)`: `0.073584` +- exact endpoint budget at `alpha=0.01`: `0.245083866` (`0.245084` paper rounding) +- exact Markov loss threshold at `alpha=0.01`: `0.345083866` (`0.345084` paper rounding) +- realized risk-tolerance excess: `0.0` - declared alpha-grid pass: `8/8` - fixed-allocation bootstrap return interval: `$167,963.20`--`$198,650.47` @@ -44,8 +48,9 @@ calibrator and conformal interval outputs. | Claim | Decision | Destination | Evidence | Stop Rule | |---|---|---|---|---| | CRPTO is a decision certificate, not a classifier leaderboard. | Promote | body | `paper/CRPTO_ijds.qmd`, Figure 1, exact certificate table, A35 | Do not reopen unless the decision certificate changes. | -| The pool93 body point is selected from a finite exact return-bound frontier. | Promote | body + A35 | `crpto_tableA35_pool93_ijds_frontier.csv`, consolidated governance JSON | Do not run more portfolio search unless a new result can lower cap at same return or materially lift return under the declared cap. | +| The pool93 body point is selected from a finite exact return-bound frontier. | Promote | body + A35 | `crpto_tableA35_pool93_ijds_frontier.csv`, certificate-semantics-v2 frontier/governance JSON | Do not run more portfolio search unless a new result can lower the exact threshold at the same return or materially lift return under the declared threshold. | | The selected allocation has inspectable business composition and tail profile. | Append | supplement A36--A39 | A36 grade audit, A37 LGD/CVaR/OCE repricing, A38 cluster-bound audit, A39 bootstrap | Diagnostics only; do not use as hidden selector. | +| The conformal decision has a matched point-PD baseline. | Promote | body + supplement A40 | A40 table and `pool93_point_pd_baseline_audit.json` | Treat as one frozen OOT trade-off; do not claim causal or universal dominance. | | The former `45/45` rebaseline remains provenance and return floor. | Archive/Append | provenance/supplement | `EXTRACTION_MANIFEST.json`, `ijds_rebaseline_2026-06-07.md` | Do not use as active headline except to explain the declared floor. | | External Prosper/Freddie runs support recipe transfer. | Append | body short paragraph + supplement A25--A34 | external replication tables and figures | Do not promote as new Lending Club certificates. | @@ -78,10 +83,18 @@ For the consolidated frontier: - eligible all-alpha above-floor policies: `27,508/50,010` - nonpass or below-floor policies: `22,502/50,010` +The v2 policy-aware rehydration uses the stored exact endpoint budget instead of +the linear-only residual shortcut. It changes neither denominator nor the body +selection, but changes 10,423 policy thresholds materially. Of these, 2,866 +tail/segment-tail policies were understated; the maximum understatement was +`0.241324`, and 716 policies formerly labeled at or below `0.50` exceed `0.50` +on the exact endpoint scale. The max-return endpoint is therefore `0.697056`, +not the retired linear-shortcut value. + These denominators are finite-grid denominators, not continuous optimality -claims. If future work adds new policy families, gamma values, alpha levels or -solvers, the denominators can grow under a new run tag; they must not be mixed -with the current frozen denominators. +claims. If a later, separately tagged run adds new policy families, gamma +values, alpha levels or solvers, the denominators can grow under that run tag; +they must not be mixed with the current frozen denominators. ## How To Present The Denominators @@ -130,6 +143,27 @@ Do not present these as: - a live-production coverage guarantee after adaptive policy selection; - evidence that more policies are always better. +## Baseline Semantics Boundary + +The frozen Lending Club field `price_of_robustness=-10.56%` is historical +provenance, not an active IJDS claim. Its stored `nonrobust` solve inherited an +endpoint constraint and therefore was not a point-PD comparator. A40 replaces +that field with a matched two-stage LP at the selected policy's `tau=0.1715`, +holding 276,869 candidates, budget, concentration, LGD, solver, and operating +constraints fixed. The point-PD allocation earns `$196,369.14`; selected CRPTO +earns `$184,832.48`, a cost of `$11,536.66` (`5.875%`). CRPTO reduces weighted +default/miscoverage by `0.08305` and the exact Markov threshold by `0.435495`. +See `pool93_certificate_semantics_v2_2026-07-09.md`. + +This correction does not alter the selected pool93 allocation, its realized +return, `V`, `Gamma_CP`, exact Markov threshold, zero realized risk-tolerance +excess, alpha-grid pass, or finite-grid denominators. The active Lending Club +comparison is A40, interpreted jointly with the A35 exact return--bound +frontier. Frozen Table 0/Table 1/A2 fields remain untouched for +manifest provenance and must not be cited as evidence of robust dominance over +a point estimate. Historical A/B proxy flags that inherited that comparator are +also non-promoted. + The IJDS framing should emphasize data + methodology + decision + implication: the finite-grid frontier is the decision object, the exact checks are the auditable computation, and the endpoints expose the price of robustness. @@ -142,6 +176,14 @@ it is the weakest defensible assumption for the current selected allocation. A38 reports cluster-aware thresholds as sensitivity; none is tighter than Markov for the observed exposure concentration. +Every paper-facing policy now uses the policy-aware decomposition +`Gamma_CP = Gamma_internalized + Gamma_residual`, with exact endpoint budget +`B_u = sum(w*q) + Gamma_residual`. The shortcut +`Gamma_residual = (1-gamma) * Gamma_CP` is valid only for a pure linear blend. +It remains numerically valid for the selected capped policy because its row-level +cap is inactive on all 314 funded rows, but it must not be applied to tail or +segment-tail policies. + Do not claim: - universal conditional coverage; @@ -150,15 +192,28 @@ Do not claim: - a CVaR/OCE/bootstrap-selected champion; - that `8/8` is an external standard rather than the declared grid. +Literature-informed boundary added after the 2026-07-08 corpus scan: + +- Contextual optimization and credit-scoring uncertainty papers support the + framing of CRPTO as prediction-to-decision data science, but they do not + change the certificate object. +- Non-exchangeable conformal risk control, valid selection among conformal sets, + inverse/decision-calibrated robustness, and learned decision-aware conformal + sets are outside the submitted claim unless rerun under a new tag with an + explicit selection/calibration design. +- The current finite-grid frontier is strong audit evidence for the declared + frozen surface; it is not a stability-based or independent-recalibration + theorem for selecting among many conformal sets. + ## Reopen Gate A new search is justified only if it can plausibly change one of these claims: -1. same or higher return with materially lower Markov cap or `Gamma_CP`; -2. much higher return under the same declared cap; +1. same or higher return with materially lower exact Markov threshold or `Gamma_CP`; +2. much higher return under the same declared threshold; 3. a denser predeclared alpha grid that materially strengthens the certificate; 4. a nested/prospective evaluation design that reduces post-selection risk; 5. a reviewer-requested diagnostic that closes a specific objection. -Otherwise, append the idea to future work and keep the current pool93 frontier -closed. +Otherwise, append the idea to research notes after submission and keep the +current pool93 frontier closed. diff --git a/docs/research/crpto_editorial_claims_references.qmd b/docs/research/crpto_editorial_claims_references.qmd index ee2a516..a69c736 100644 --- a/docs/research/crpto_editorial_claims_references.qmd +++ b/docs/research/crpto_editorial_claims_references.qmd @@ -1,10 +1,11 @@ ## Guía Editorial, Claims y Referencias {#sec-crpto-editorial-guide} -> Nota histórica, 2026-07-04: esta guía fue escrita antes del cierre pool93. +> Nota histórica, actualizada 2026-07-09: esta guía fue escrita antes del cierre pool93. > La fuente activa de métricas y límites de claim es > `docs/research/active_claims_2026-07-04.md`. Las referencias a > `paper-thesis-final-economic-2026-04-06`, `45/45` o A7--A11 se mantienen como -> provenance editorial; el cuerpo IJDS actual usa el punto pool93 y A35--A39. +> provenance editorial; el cuerpo IJDS actual usa el punto pool93, la frontera +> policy-aware A35, las auditorías A36--A39 y el baseline emparejado A40. Esta página existe porque el libro cumple una función distinta al paper. El paper final debe ser breve, selectivo y persuasivo; el libro puede ser más @@ -32,7 +33,7 @@ La frase anterior tiene cuatro piezas: riesgo [1]--[5]; 2. **restricción operativa**: conecta esos intervalos con robust optimization y price of robustness [6]--[9]; -3. **policy económica promovida**: usa el cierre pool93 documentado en A35--A39 +3. **policy económica promovida**: usa el cierre pool93 documentado en A35--A40 y no reabre la cadena upstream congelada; 4. **cautela post-selección**: separa el teorema Markov del tightening condicional y de las validaciones empíricas A7--A11. @@ -50,8 +51,8 @@ empíricos, otros son de ingeniería reproducible. | 3 | El intervalo puede convertirse en conjunto de incertidumbre | definición de `u_i(alpha)` y `Gamma_CP` | no tratar `Gamma_CP` como presupuesto ad hoc | | 4 | El bound controla no-cobertura ponderada del funded set | `thm-conformal-feasibility` y `V` | no decir que controla directamente una PD latente sin supuesto adicional | | 5 | Existe una frontera robusta finita en OOT | A35: `50,010` políticas semánticas deduplicadas y `27,508` elegibles all-alpha sobre piso; terminal `37,068/37,068` all-alpha passers | no decir que eso certifica una región continua ni un óptimo global | -| 6 | El body point pool93 es económico, no theorem-tight | retorno `$184.8K`, `V=0.035350`, `Gamma_CP=0.162616`, Markov cap `0.345084` | no mezclarlo con el endpoint mínimo-cap ni con el max-return endpoint | -| 7 | La evidencia CRPTO fortalece el paper sin cambiar dirección | A35--A39, apéndice condicional, A25--A34 | no venderlo como validación live o universal | +| 6 | El body point pool93 es económico, no theorem-tight | retorno `$184.8K`, `V=0.035350`, `Gamma_CP=0.162616`, `Gamma_res=0.073584`, umbral exacto de pérdida `0.345084` | no mezclarlo con el endpoint de menor umbral ni con el max-return endpoint | +| 7 | La evidencia CRPTO fortalece el paper sin cambiar dirección | A35--A40, apéndice condicional, A25--A34 | no venderlo como validación live, causal o universal | : Escalera editorial de claims del CRPTO {#tbl-crpto-claim-ladder} @@ -112,7 +113,9 @@ bien confirma que la dirección del paper es contemporánea y fértil. | `alpha` | nivel de tolerancia a no-cobertura | cuánto riesgo de cola acepta el comité | | `PD_low`, `PD_high` | intervalo conformal por préstamo | rango defendible de riesgo | | `u_i(alpha)` | cota superior usada por el LP | PD prudente para decisión | -| `Gamma_CP` | prima conformal ponderada | costo promedio de comprar robustez | +| `Gamma_CP` | ancho conformal ponderado total | incertidumbre disponible antes de fijar la policy | +| `Gamma_int` | intervención prudencial usada por la policy | robustez efectivamente comprada | +| `Gamma_res` | ancho conformal residual | holgura que completa el certificado policy-aware | | `V` | no-cobertura ponderada del funded set | cuánto default realizado quedó fuera de la cota | | `violation` | exceso de riesgo ponderado sobre `tau` | incumplimiento del límite de portafolio | | `alpha01_exact_pass` | check exacto de bound en OOT | semáforo para promoción paper-facing | @@ -188,7 +191,7 @@ paper-grade, incluida la nueva Figura 1 IJDS, generadas o publicadas desde artef | Blueprint del manuscrito | estructura paper-ready y mapa claim -> artifact -> test -> location | `14g-manuscript-blueprint.qmd` | | Figura CRPTO limpia | Figura 1 IJDS | `crpto_fig1_journal_pipeline.png` | | Figura alpha -> `Gamma_CP` -> funded set | puente teoria-metodo | `crpto_fig13_alpha_gamma_funded_set.png` | -| A35 frontier y A36--A39 pool93 | evidencia visual/tabular del cierre pool93 | `reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv`; `reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.csv`; `reports/crpto/tables/crpto_tableA37_pool93_body_tail_risk.csv`; `reports/crpto/tables/crpto_tableA38_pool93_body_cluster_bound_audit.csv`; `reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.csv` | +| A35 frontier, A36--A39 pool93 y A40 point-PD | evidencia visual/tabular del cierre pool93 y costo comparable de robustez | `reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv`; `reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.csv`; `reports/crpto/tables/crpto_tableA37_pool93_body_tail_risk.csv`; `reports/crpto/tables/crpto_tableA38_pool93_body_cluster_bound_audit.csv`; `reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.csv`; `reports/crpto/tables/crpto_tableA40_pool93_point_baseline.csv` | | A12--A34 | robustness appendix journal | `scripts/build_crpto_journal_package.py`; `scripts/build_tail_satisficing_challenger_audit.py`; `scripts/build_tail_constrained_reoptimization.py`; `scripts/build_distribution_robustness_diagnostics.py`; `scripts/build_multidataset_external_replication.py`; `scripts/build_price_of_robustness_cross_dataset.py` | | A25--A34 | réplica externa Prosper/Freddie | `models/crpto_multidataset_external_status.json` | | Status del paquete | trazabilidad y fuente canonica | `models/crpto_journal_package_status.json`; `models/crpto_multidataset_external_status.json` | diff --git a/docs/research/crpto_full_audit_2026-07-05.md b/docs/research/crpto_full_audit_2026-07-05.md index 8d237fa..e041918 100644 --- a/docs/research/crpto_full_audit_2026-07-05.md +++ b/docs/research/crpto_full_audit_2026-07-05.md @@ -1,5 +1,11 @@ # Auditoría integral CRPTO — 2026-07-05 +> **Cierre semántico posterior (2026-07-09).** Las secciones de proceso y +> refactor de este memo conservan valor histórico. Para métricas del +> certificado, frontera A35 y baseline A40, prevalece +> `pool93_certificate_semantics_v2_2026-07-09.md`; ese cierre reemplaza el +> atajo lineal que este memo llamaba "Markov cap". + Auditoría de consistencia de claims, parsimonia de código, skills, docs y plan editorial del paper, de cara a la submission IJDS (target interno 2026-08-10). Método: tres pasadas de exploración paralelas (código, claims, tooling/docs) @@ -38,9 +44,9 @@ canónica: `docs/research/active_claims_2026-07-04.md`. | Retorno body point pool93 | $184,832.48 | OK en qmd, tex (`\$184{,}832.48`), A35, gobernanza | | V(alpha=0.01) | 0.035350 | OK en todas las fuentes | | Gamma_CP(alpha=0.01) | 0.162616 | OK; notación Gamma_CP vs gamma de política bien separada | -| Markov cap (alpha=0.01) | 0.345084 | OK (A35 guarda 0.345083740; redondeo esperado) | -| Endpoint budget upper B_u | 0.245084 = 0.1715 + 0.4525 * 0.162616 | OK en body, supplement y A35 | -| Alpha grid | 8/8, violación exacta 0.0 | OK | +| Umbral exacto de pérdida (alpha=0.01) | 0.345084 | OK en la A35 policy-aware corregida | +| Endpoint budget upper B_u | 0.245084 = sum(w q) + Gamma_res | OK; en el body point la cota por fila está inactiva y el atajo lineal coincide numéricamente | +| Alpha grid | 8/8 screen pass; exceso realizado sobre tau 0.0 | OK | | Return floor declarado | $170,464.54 | OK; en gobernanza y supplement, no como headline | | Frontera consolidada | 50,010 semánticas dedup; 27,508 all-alpha sobre floor | OK | | Búsqueda terminal | 37,068/37,068 passers; 296,544 checks | OK | diff --git a/docs/research/crpto_publication_strategy_2026-05-12.md b/docs/research/crpto_publication_strategy_2026-05-12.md index c275f90..c13e3ff 100644 --- a/docs/research/crpto_publication_strategy_2026-05-12.md +++ b/docs/research/crpto_publication_strategy_2026-05-12.md @@ -66,7 +66,7 @@ The first paper draft should be written as: - title: `CRPTO: Conformal Robust Predict-Then-Optimize for Auditable Credit Portfolio Decisions`; - body: 25-page IJDS-style manuscript; -- supplement: A3--A39, proofs, extended tables, reproducibility, +- supplement: A3--A40, proofs, extended tables, reproducibility, MRM/fairness and external replication; - review mode: anonymous by default; - companion: GitHub/DVC/DagsHub/MLflow after the anonymity policy is handled. @@ -77,8 +77,8 @@ The short paper should keep only the strongest body material: - PD calibration summary; - Mondrian conformal layer; - robust portfolio formulation; -- Markov bound and conditional tightening caveat; -- pool93 body metrics and A35 finite-grid frontier; +- policy-aware decision certificate and conditional tightening caveat; +- pool93 body metrics, A35 finite-grid frontier and A40 matched point-PD audit; - A36--A39 selected-allocation audits; - regret-auditability frontier with one concise SPO+/DFL comparison; - data/code reproducibility statement. @@ -100,13 +100,13 @@ In scope for the current paper: - promoted pool93 finite-grid return-bound frontier; - frozen upstream CRPTO chain as provenance and return floor; - calibrated PD -> Mondrian conformal intervals -> robust portfolio decision; -- exact alpha-safe funded-set validation and A35 finite-grid frontier; -- A3--A39 as supplement evidence; +- exact funded-set validation and A35 policy-aware finite-grid frontier; +- A3--A40 as supplement evidence; - regret-auditability frontier in the body; - OCE/CVaR tail-risk diagnostics and robust satisficing margins in the supplement; -- cluster-aware dependence caveat/proposition with Markov retained as the main - distribution-free bound; +- cluster-aware dependence caveat/proposition with the Markov step retained in + the main distribution-free decision certificate; - external economic replication on Prosper and Freddie/Mendeley as A25--A34, without reopening the Lending Club body claim; - reproducibility via Quarto, DVC, DagsHub/MLflow and guardrail tests. diff --git a/docs/research/ijds_claim_concept_audit_2026-06-26.md b/docs/research/ijds_claim_concept_audit_2026-06-26.md index 7c46ae2..2258b59 100644 --- a/docs/research/ijds_claim_concept_audit_2026-06-26.md +++ b/docs/research/ijds_claim_concept_audit_2026-06-26.md @@ -1,5 +1,10 @@ # IJDS Claim Concept Audit: Alpha Grid, Robust Region, Bound, and Exact Frontier +> **Historical only.** This pre-closeout analysis uses the former linear +> certificate shortcut. Current claims and corrected endpoint thresholds live +> in `active_claims_2026-07-04.md` and +> `pool93_certificate_semantics_v2_2026-07-09.md`. + Date: 2026-06-26 This memo audits the concepts that are easy to take for granted in the CRPTO diff --git a/docs/research/ijds_claim_maximization_analysis_2026-06-27.md b/docs/research/ijds_claim_maximization_analysis_2026-06-27.md index 1507e99..6002fc4 100644 --- a/docs/research/ijds_claim_maximization_analysis_2026-06-27.md +++ b/docs/research/ijds_claim_maximization_analysis_2026-06-27.md @@ -1,5 +1,9 @@ # IJDS Claim Maximization Analysis - 2026-06-27 +> **Historical only.** This search diary predates the policy-aware certificate +> correction. Do not reuse its cap labels or endpoint thresholds; current A35 +> and A40 evidence is registered in `pool93_certificate_semantics_v2_2026-07-09.md`. + This memo evaluates the strongest IJDS-facing claims for CRPTO after the pool93 claim-governance and local exact-refinement work. It is an internal research artifact, not manuscript prose. diff --git a/docs/research/ijds_corpus_claims_improvement_plan_2026-07-07.md b/docs/research/ijds_corpus_claims_improvement_plan_2026-07-07.md index d6c60fe..0333702 100644 --- a/docs/research/ijds_corpus_claims_improvement_plan_2026-07-07.md +++ b/docs/research/ijds_corpus_claims_improvement_plan_2026-07-07.md @@ -1,5 +1,11 @@ # IJDS Corpus, Claims, and Improvement Plan - 2026-07-07 +> **Superseded metric vocabulary (2026-07-09).** This memo is retained as the +> corpus-reading and editorial decision trail. Its old "Markov cap" and +> preliminary frontier language must not be used as current evidence. The +> policy-aware A35 correction and matched A40 baseline in +> `pool93_certificate_semantics_v2_2026-07-09.md` are authoritative. + Scope: analyze the current CRPTO body paper, official IJDS submission PDF/source, online supplement, frozen metrics/artifacts, and the local `Papers_tesis` corpus using the global `academic-pdf-intake` skill outputs. diff --git a/docs/research/ijds_literature_expansion_scan_2026-07-08.md b/docs/research/ijds_literature_expansion_scan_2026-07-08.md new file mode 100644 index 0000000..bf11190 --- /dev/null +++ b/docs/research/ijds_literature_expansion_scan_2026-07-08.md @@ -0,0 +1,514 @@ +# IJDS literature expansion scan, 2026-07-08 + +## Scope + +This note records an external and local literature scan for the IJDS version of +CRPTO. It is designed as an editorial input, not as a request to reopen the +frozen champion. The active certificate remains the pool93 finite-grid decision +certificate documented in `docs/research/active_claims_2026-07-04.md`. + +Operational guardrails: + +- Do not modify `EXTRACTION_MANIFEST.json` or artifacts listed there. +- Do not rerun protected champion stages without explicit permission. +- Keep the IJDS body focused on the implemented certificate: finite alpha grid, + robust funding decision, exact violation audit, and economic return. +- Use the supplement for adjacent method families, diagnostics, and future work. + +Sources reviewed: + +- Local corpus: `Papers_tesis` inventory and benchmark snapshot + `.tmp_pdf_intake_benchmark/run_20260707_ijds_lit_analysis/snapshot.md`. +- Current bibliography: `book/references.bib`. +- Current manuscript anchors: `book/chapters/CRPTO_*.qmd`, `paper/submission`, + and the active claims register. +- External web scan on conformal decision-making, contextual optimization, + credit scoring uncertainty, IJDS-adjacent work, and 2025-2026 emerging papers. + +## Bottom line + +The paper already cites the core CRPTO neighborhood well: conformal risk control, +predict-then-calibrate, conformal contextual robust optimization, conformal +robustness control, conformal robust optimization/satisficing, end-to-end +conformal calibration, and credit/P2P decision papers. The most useful additions +are not dozens of new citations. The highest-value improvement is a small set of +strategic anchors that sharpen the IJDS story: + +1. Add one broad contextual optimization survey to show that CRPTO sits inside + the modern prediction-to-decision literature. +2. Add one credit-specific profit/uncertainty paper to show that economic credit + scoring is active, but CRPTO advances from score-level economics to a funded + portfolio decision certificate. +3. Add one non-exchangeability/source-shift conformal reference in the + supplement. +4. Add one post-selection or human-decision conformal limitation reference to + make the paper look honest and current. +5. Keep 2026 inverse/decision-calibrated and decision-aware conformal-set papers + as future-work comparators, not body-level foundations, unless reviewers ask. + +## Recommended additions + +### 1. Sadana et al. 2025, contextual optimization survey + +Candidate: + +- Rahul Sadana, Andrea Delage, Alexandre Forel, Emma Frejinger, Thibaut Vidal. + "A survey of contextual optimization methods for decision-making under + uncertainty." European Journal of Operational Research, 320(2), 2025. +- Sources: [ScienceDirect](https://www.sciencedirect.com/science/article/pii/S0377221724002200), + [arXiv](https://arxiv.org/abs/2306.10374). + +Why it matters: + +- It is an OR-facing survey of contextual optimization, prescriptive analytics, + predict-then-optimize, decision-focused learning, and smart predict/estimate- + then-optimize methods. +- It gives IJDS reviewers a clean map for why CRPTO is a decision paper rather + than a classifier leaderboard. + +Recommended placement: + +- Body, related work or positioning paragraph. +- One sentence is enough: CRPTO belongs to contextual/predictive-prescriptive + optimization, but differs by producing an auditable finite-grid robust funding + certificate rather than learning a new end-to-end policy. + +Priority: High. + +### 2. Xu, Kou, and Ergu 2025, profit-based uncertainty in credit scoring + +Candidate: + +- Zhuozhuo Xu, Gang Kou, Daji Ergu. "Profit-based uncertainty estimation with + application to credit scoring." European Journal of Operational Research, + 325(2), 2025. +- Sources: [ScienceDirect](https://www.sciencedirect.com/science/article/pii/S0377221725002048), + [IDEAS/RePEc](https://ideas.repec.org/a/eee/ejores/v325y2025i2p303-316.html). + +Why it matters: + +- It is close to our application domain and explicitly links credit scoring, + uncertainty, rejection, and profitability. +- It supports the narrative that predictive uncertainty in lending should be + evaluated economically, while CRPTO moves the unit of decision from + application-level classification/rejection to portfolio funding under a + distribution-free certificate. + +Recommended placement: + +- Body if there is space in the credit/P2P paragraph. +- Otherwise supplement literature table. + +Priority: High. + +### 3. Xu et al. 2024, profit- and risk-driven credit scoring + +Candidate: + +- Zhuozhuo Xu, Yishun Dou, Gang Kou, Daji Ergu. "Profit- and risk-driven credit + scoring under parameter uncertainty: A multiobjective approach." Omega, 125, + 2024. +- Source: [ScienceDirect](https://www.sciencedirect.com/science/article/pii/S0305048323001688). + +Why it matters: + +- It gives a credit-scoring reference for profit/risk tradeoffs under uncertain + cost-benefit parameters. +- It is useful if we want a short supplement note distinguishing CRPTO from + multiobjective credit-score design: CRPTO certifies a portfolio decision over + existing predictions and realized returns rather than optimizing an application + classifier under assumed parameter uncertainty. + +Recommended placement: + +- Supplement, not body, unless reviewers ask for more credit-scoring economics. + +Priority: Medium. + +### 4. Farinhas et al. 2024, non-exchangeable conformal risk control + +Candidate: + +- Antonio Farinhas, Alessandro Zecchin, Andre Martins, David Grangier. + "Non-Exchangeable Conformal Risk Control." ICLR 2024. +- Source: [arXiv](https://arxiv.org/abs/2310.01262). +- Local PDF already exists in `Papers_tesis/supplement`. + +Why it matters: + +- It directly addresses a limitation readers may raise: calibration/test + exchangeability, time dependence, and distribution shift. +- It can strengthen the supplement language around source-shift diagnostics + without making a new guarantee in the IJDS body. + +Recommended placement: + +- Supplement A23/A24 source-shift and calibration-drift discussion. +- Do not cite it as implemented protection unless we implement and validate the + method. + +Priority: High for supplement, low for body. + +### 5. Hegazy et al. 2025, valid selection among conformal sets + +Candidate: + +- Mahmoud Hegazy, Emma Frejinger, Pierre Pinson, Alexandre Forel. + "Valid Selection among Conformal Sets." 2025. +- Source: [arXiv](https://arxiv.org/abs/2506.20173). + +Why it matters: + +- It is conceptually important for CRPTO because our grid search selects among + candidate decisions and alpha values after evaluating constraints. +- The current paper is careful because it reports finite-grid denominators, + exact violation audit, and does not claim a new post-selection conformal + theorem. This paper helps articulate that boundary. + +Recommended placement: + +- Supplement limitations/future protocol. +- Possible sentence: future CRPTO variants could study formal post-selection + validity for choosing among multiple calibrated sets, whereas the present + paper reports a finite-grid certificate and exact audit for the selected + champion. + +Priority: High for limitations. + +### 6. Hullman et al. 2025, conformal prediction and human decision making + +Candidate: + +- Jessica Hullman, Christopher W. Zamecnik, Yuval Rabin, Fred Hohman. + "Conformal Prediction and Human Decision Making." 2025. +- Source: [arXiv](https://arxiv.org/abs/2503.11709). + +Why it matters: + +- It argues that valid conformal sets are not automatically useful for decisions + unless the downstream objective is explicit. +- This is useful for IJDS framing: CRPTO is not "uncertainty reporting"; it + operationalizes uncertainty through a robust funding decision and economic + audit. + +Recommended placement: + +- Optional short citation in introduction or limitations. +- Use sparingly; avoid turning the paper into a human-factors discussion. + +Priority: Medium. + +### 7. Djeundje, Crook, and Andreeva 2025, dynamic loan portfolio profitability + +Candidate: + +- Viani B. Djeundje, Jonathan Crook, Galina Andreeva. "The devil in the details: + Dynamic prediction of loan portfolio profitability with macroeconomic drivers + through multi-state modelling." European Journal of Operational Research, + 327(2), 2025. +- Sources: [IDEAS/RePEc](https://ideas.repec.org/a/eee/ejores/v327y2025i2p703-715.html), + [University of Edinburgh](https://www.research.ed.ac.uk/en/publications/the-devil-in-the-details-dynamic-prediction-of-loan-portfolio-pro/). + +Why it matters: + +- It is a recent credit-portfolio profitability paper rather than a pure + application-level classifier paper. +- It can help if a reviewer wants more credit portfolio literature around + dynamic profitability and macroeconomic drivers. + +Recommended placement: + +- Supplement only, unless the introduction is rewritten to emphasize + macro-sensitive portfolio profitability. +- Do not use it to imply that CRPTO currently models macro transitions; it does + not. + +Priority: Medium-low. + +## Emerging close comparators to monitor + +These papers are close to CRPTO but are very recent, mostly preprint-era, or in +different domains. They should strengthen future-work positioning rather than +drive the IJDS body. + +### Zhou and Zhu 2026, inverse conformal risk control for decision robustness + +- "Calibrating Decision Robustness via Inverse Conformal Risk Control." +- Sources: [arXiv](https://arxiv.org/abs/2510.07750), + [OpenReview](https://openreview.net/forum?id=lV4tqcVIyx&referrer=%5Bthe+profile+of+Shixiang+Zhu%5D%28%2Fprofile%3Fid%3D~Shixiang_Zhu1%29). +- Value: Very close to CRPTO because it treats the robustness level itself as + the calibrated object and reports finite-sample guarantees on miscoverage and + regret for robust predict-then-optimize policy families. +- Recommendation: strongest 2026 future-work comparator. It is too new to make + it a foundation of the current body, but it is the cleanest citation if we add + one sentence about future calibration of robustness levels. + +### Stratigakos et al. 2026, decision-calibrated prediction sets + +- "Decision-calibrated prediction sets for robust power system operations." +- Source: [arXiv](https://arxiv.org/abs/2606.02081). +- Value: Closest phrase-level comparator for calibrating prediction sets by + downstream decision reliability. +- Recommendation: future-work comparator only. + +### Chen, Zhou, and Zhu 2026, learning polyhedral conformal sets for RO + +- "Learning Polyhedral Conformal Sets for Robust Optimization." +- Source: [arXiv](https://arxiv.org/abs/2605.08506). +- Value: Decision-aware conformal uncertainty sets for robust optimization. +- Recommendation: cite in future work if adding a paragraph on learned + uncertainty-set geometry. + +### Wang and Dobriban 2026, optimal decisions from prediction sets + +- "Optimal Decision-Making Based on Prediction Sets." +- Source: [arXiv](https://arxiv.org/abs/2602.00989). +- Value: Decision-theoretic framework for downstream use of prediction sets. +- Recommendation: monitor; useful if reviewers ask for more theory around + prediction sets as decision objects. + +### Huang, Farzaneh, and Simeone 2026, OCE risk-controlling prediction sets + +- "Optimized Certainty Equivalent Risk-Controlling Prediction Sets." +- Source: [arXiv](https://arxiv.org/abs/2602.13660). +- Value: OCE/CVaR-style risk-control extension. +- Recommendation: supplement/future work near the existing OCE/CVaR diagnostics. + +### Baesens et al. 2026, foundation models for credit risk prediction + +- "Foundation Models for Credit Risk Prediction: A Game Changer?" +- Source: [arXiv](https://arxiv.org/abs/2605.18147). +- Value: Useful for monitoring PD-layer baselines in credit risk. +- Recommendation: do not add to IJDS body now. CRPTO does not claim a PD model + leaderboard. + +## Already-covered nearest neighbors + +The current corpus and official submission already include the most important +methodological neighbors. These should remain the core comparison set: + +- Patel et al. 2024, conformal contextual robust optimization. +- Sun et al. 2024, predict-then-calibrate. +- Hu et al. 2026, conformal robustness control. +- Zhao et al. 2025/2026, conformal robust optimization and satisficing. +- Yeh et al. 2025/2026, conformal risk training and end-to-end conformal + calibration. +- Bao et al. 2025, CROMS. +- Zhou et al. 2025/2026, CREDO and CREME. +- Yang and Bi 2025, cost-aware calibration. +- Liu et al. 2026, online conformal portfolio selection. +- Yang and Jin 2026, multidistribution conformal prediction. + +The key editorial move is to distinguish CRPTO from each in one sentence: + +- Not a new PD scorer. +- Not a generic conformal-set method. +- Not end-to-end training. +- Not online rebalancing. +- Not a multidistribution fairness theorem. +- A finite-grid robust portfolio decision certificate with realized-return and + exact-violation audit. + +## IJDS-specific venue scan + +Relevant IJDS-adjacent sources are useful mainly for framing, not for core +method positioning: + +- Wiberg et al. 2025, "Synergizing Artificial Intelligence and Operations + Research." IJDS. Source: + [INFORMS](https://pubsonline.informs.org/doi/10.1287/ijds.2025.0077). +- Morucci et al. 2022, "A Robust Approach to Quantifying Uncertainty in Matching + Problems of Causal Inference." IJDS. Source: + [INFORMS](https://pubsonline.informs.org/doi/10.1287/ijds.2022.0020). +- "Rethinking Cost-Sensitive Classification in Deep Learning via Adversarial + Data Augmentation." IJDS. Source: + [INFORMS](https://pubsonline.informs.org/doi/10.1287/ijds.2022.0033). + +Recommendation: + +- Do not overload the paper with IJDS self-citations. +- If a cover letter or response-to-reviewers needs venue fit, Wiberg et al. + 2025 is a concise AI+OR anchor. +- Morucci et al. 2022 can be mentioned only if a reviewer asks about robust + uncertainty quantification precedent in IJDS. +- The cost-sensitive classification IJDS paper is less aligned than Yang and Bi + 2025 for our current framing. + +## Recent credit-risk papers to leave out unless needed + +The 2025-2026 EJOR credit-risk stream is active. Several papers are useful for +background but should not crowd the IJDS body: + +- Distaso, Roccazzella, and Vrins 2025, "Business cycle and realized losses in + the consumer credit industry." Source: + [ScienceDirect](https://www.sciencedirect.com/science/article/pii/S0377221724009688). +- Ballegeer, Bogaert, and Benoit 2025, "Evaluating the stability of model + explanations in instance-dependent cost-sensitive credit scoring." Source: + [ScienceDirect](https://www.sciencedirect.com/science/article/abs/pii/S0377221725004230). +- Baesens et al. 2026, foundation models for credit risk prediction. Source: + [arXiv](https://arxiv.org/html/2605.18147v1). + +Recommendation: + +- Use these only if a reviewer asks for macro-credit loss, explanation + stability, or modern PD-model background. +- They do not change the CRPTO contribution because the paper is not proposing a + new PD learner. + +## Exact manuscript insertion plan + +Keep the body lean. The highest-value body additions are two citations: + +1. Add Sadana et al. 2025 to the contextual optimization / prescriptive + analytics positioning. +2. Add Xu, Kou, and Ergu 2025 to the credit scoring uncertainty / profit + paragraph. + +Then use the supplement for three limitation/future-work citations: + +3. Farinhas et al. 2024 in source-shift/non-exchangeability. +4. Hegazy et al. 2025 in post-selection validity. +5. Zhou and Zhu 2026 as the preferred emerging future-work comparator; if space + allows, add one of Stratigakos et al. 2026, Chen et al. 2026, or Wang and + Dobriban 2026 for decision-calibrated conformal-set geometry/use. + +Suggested body language: + +> CRPTO sits within contextual optimization and prescriptive analytics, where +> predictions and downstream decisions are optimized jointly or sequentially, +> but it differs by auditing a finite grid of robust funding decisions rather +> than training a new decision rule. + +> Recent credit-scoring work evaluates uncertainty through profit and rejection +> decisions; CRPTO instead evaluates uncertainty at the funded-portfolio level, +> where the reported object is an economically realized decision certificate. + +Suggested supplement language: + +> The present certificate assumes the calibration/audit design documented in +> the finite-grid protocol. Extensions to non-exchangeable conformal risk control +> and formal selection among multiple conformal sets are natural next steps, but +> are not claimed by the current champion. + +## Bibliography action list + +Add or verify BibTeX entries for: + +- `sadana2025contextual` +- `xu2025profit_uncertainty_credit` +- `xu2024profit_risk_credit` +- `farinhas2024nonexchangeable_crc` +- `hegazy2025valid_selection_conformal_sets` +- `hullman2025conformal_human_decision` +- `djeundje2025dynamic_loan_portfolio_profitability` +- `zhou2026inverse_crc_decision_robustness` +- `stratigakos2026decision_calibrated_sets` +- `chen2026polyhedral_conformal_ro` +- `wang2026optimal_decision_prediction_sets` +- `huang2026oce_rcps` +- `baesens2026foundation_credit_risk` + +Before adding all of them, apply a manuscript budget rule: + +- Body: at most two new citations unless a paragraph is rewritten. +- Supplement: three to six citations are acceptable. +- Future-work table: emerging 2026 papers are acceptable if clearly labeled as + future work and not as implemented guarantees. + +## Citation synchronization note + +The official submission `.tex` currently contains several compact citation +anchors that are not all mirrored in the source `.qmd` files. Examples observed +during this scan include: + +- `angelopoulos2024foundations` +- `bates2021rcps` +- `zhou2024` +- `sun2024ptc` +- `boosting2025default` +- `yeh2025training` + +If the paper is regenerated from Quarto, these anchors could be lost unless the +QMD sources are synchronized. The safest path is to update QMD first, regenerate +official submission artifacts, and then re-run the IJDS compile checks. + +## Do-not-add list for the current IJDS submission + +Do not add broad or weakly related references unless a reviewer asks: + +- Generic LLM/tabular-model credit-risk papers that do not affect the CRPTO + certificate. +- Generic conformal prediction surveys beyond the already cited foundations. +- Extra IJDS venue papers solely for journal signaling. +- Additional classifier benchmarking papers unless they directly affect the + LendingClub decision framing. + +## Decision recommendation + +For the current IJDS revision, implement the following compact literature +upgrade: + +1. Add Sadana et al. 2025 and Xu, Kou, and Ergu 2025 to the body. +2. Add Farinhas et al. 2024 and Hegazy et al. 2025 to the supplement. +3. Add one 2026 emerging decision-calibrated conformal reference to future work. +4. Keep all language explicit that these are positioning and future-work + references; they do not change the frozen champion or its claims. + +This gives reviewers the right signals: the paper is current, aware of adjacent +decision-calibrated conformal work, and still disciplined about what it actually +implements and certifies. + +## Post-read implementation note, 2026-07-08 + +After the missing PDFs were downloaded manually to `Downloads`, they were copied +into `Papers_tesis/supplement` with normalized names: + +- `Xu Kou Ergu 2025 - Profit-based uncertainty estimation with application to credit scoring.pdf` +- `Xu et al 2024 - Profit- and risk-driven credit scoring under parameter uncertainty.pdf` +- `Djeundje Crook Andreeva 2025 - Dynamic prediction of loan portfolio profitability.pdf` +- `Wiberg Dai Lam Kulkarni 2025 - Synergizing AI and OR.pdf` + +The expanded `academic-pdf-intake` inventory now sees 100 PDFs in scope: +97 under `Papers_tesis` and the three active CRPTO PDFs. The post-read +manuscript update keeps the body claim unchanged and implements only narrative +and boundary changes: + +- Body: CRPTO is framed as a contextual-optimization credit instance, anchored + by credit-scoring uncertainty/profit literature and the AI/OR IJDS perspective. +- Theory: post-selection conformal validity is named explicitly as future + protocol, not as an implicit property of the finite-grid frontier. +- Supplement: non-exchangeable CRC, valid selected conformal sets, + decision-calibrated/inverse conformal robustness, learned polyhedral sets, + OCE-RCPS, and recent credit-profitability/explanation-stability papers are + mapped to diagnostics or future-work boundaries. + +No frozen champion artifact, manifest entry, or protected DVC stage is changed +by this literature update. + +## Venue recheck, 2026-07-09 + +The current [IJDS submission guidelines](https://pubsonline.informs.org/page/ijds/submission-guidelines) +still require the Data + Models + Decisions + Implications synthesis and now +state an explicit abstract sequence: problem/data-science relevance, method and +results, then learned insight and implication. The CRPTO abstract already +covered the first two elements but ended by repeating pipeline architecture. +Its closing sentences now state the learned decision insight and committee use: +the return-bound frontier makes predictive uncertainty actionable, while the +certificate separates exact funded-set accounting from its weighted-validity +assumption. + +Two recent accepted IJDS papers provide a useful style check rather than new +citation requirements: + +- [Robust and Interpretable Policy Learning for Manufacturing Process Parameters](https://pubsonline.informs.org/doi/10.1287/ijds.2024.0041) + leads from a concrete decision problem to a named policy method, robustness, + interpretability and practical deployment evidence. +- [Using Operational Data Analytics for Planning Decisions Under Uncertainty](https://pubsonline.informs.org/doi/10.1287/ijds.2024.0051) + makes the estimate-then-optimize gap explicit, compares against several + decision baselines and closes with real-data effectiveness. + +CRPTO already follows those structural signals through the funded-set decision, +the exact certificate, the A19 regret comparator and the managerial implication. +No additional venue self-citation is warranted: the existing Das et al. IJDS +credit-risk anchor and Wiberg et al. AI/OR anchor are sufficient, and adding +more would crowd the body without changing the novelty boundary. diff --git a/docs/research/ijds_simplification_cleanup_audit_2026-07-06.md b/docs/research/ijds_simplification_cleanup_audit_2026-07-06.md index 7c7bf52..209815d 100644 --- a/docs/research/ijds_simplification_cleanup_audit_2026-07-06.md +++ b/docs/research/ijds_simplification_cleanup_audit_2026-07-06.md @@ -1,5 +1,10 @@ # IJDS simplification and cleanup audit - 2026-07-06 +> Certificate metrics in this historical cleanup memo were superseded on +> 2026-07-09 by `pool93_certificate_semantics_v2_2026-07-09.md`. The cleanup +> decisions remain valid; the active paper uses the policy-aware A35 frontier +> and matched point-PD baseline A40. + Scope: body manuscript, IJDS submission `.tex`, online supplement, code refactor posture, and local repository weight. This memo follows the 2026-07-05 full audit but focuses on reader-facing parsimony: remove technical @@ -89,8 +94,8 @@ Recommended new-run shape if Carlos chooses that route: 2. Write outputs under a new experiment path; do not overwrite the current frozen A35--A39 or model/conformal artifacts. 3. Compare against the current selected policy on: return, `V(0.01)`, - `Gamma_CP`, Markov cap, exact violation, table/page simplicity, and code path - length. + `Gamma_CP`, `Gamma_res`, exact loss threshold, realized risk-tolerance + excess, table/page simplicity, and code-path length. 4. Promote only if the paper becomes materially simpler or the result is materially easier to defend. A few basis points of metric loss may be fine, but only if the new claim is easier to explain. diff --git a/docs/research/pool93_certificate_semantics_v2_2026-07-09.md b/docs/research/pool93_certificate_semantics_v2_2026-07-09.md new file mode 100644 index 0000000..02f71f2 --- /dev/null +++ b/docs/research/pool93_certificate_semantics_v2_2026-07-09.md @@ -0,0 +1,132 @@ +# Auditoria consolidada del certificado pool93 v2 - 2026-07-09 + +## Decision + +Se promueve la semantica `certificate-semantics-v2` como fuente activa para +A35, la gobernanza pool93 y el baseline A40. No cambia el modelo PD, calibrador, +intervalos conformales, asignacion seleccionada, retorno del body, grilla alpha +ni denominadores. Corrige la lectura del endpoint para policies no lineales y +reemplaza el comparador Lending Club mal rotulado por una baseline point-PD +emparejada. + +Tag activo: + +`champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2` + +## Hallazgo 1: descomposicion policy-aware + +La formula historica + +`B_u = tau + (1 - gamma) * Gamma_CP` + +es exacta solo para un blend lineal cuyo cap efectivo liga. La frontera tambien +contiene policies `capped`, `tail` y `segment_relative_tail`; en ellas `gamma` +no basta para reconstruir el endpoint. + +Para cualquier score efectivo declarado `q_i` con +`p_hat_i <= q_i <= u_i`, la identidad general es: + +```text +Gamma_CP = sum_i w_i (u_i - p_hat_i) +Gamma_int = sum_i w_i (q_i - p_hat_i) +Gamma_res = sum_i w_i (u_i - q_i) +Gamma_CP = Gamma_int + Gamma_res +B_u = sum_i w_i q_i + Gamma_res +B_u <= tau + solver_slack + Gamma_res +T_Markov = B_u + sqrt(alpha) +``` + +`T_Markov` es un umbral de evento probabilistico bajo weighted funded-set +validity; no es un cap determinista. El campo historico `violation` mide exceso +realizado de default sobre `tau`, no violacion de la identidad +`sum(wY) <= B_u + V`. + +## Punto seleccionado + +En `alpha=0.01`, la asignacion seleccionada conserva: + +| Cantidad | Valor | +|---|---:| +| Retorno realizado | `$184,832.48` | +| Filas financiadas | `314` | +| `V` / default ponderado | `0.035350` | +| PD puntual ponderada | `0.082468` | +| Score efectivo ponderado | `0.171500` | +| `Gamma_CP` | `0.162616` | +| `Gamma_int` | `0.089032` | +| `Gamma_res` | `0.073584` | +| Endpoint exacto `B_u` | `0.245084` | +| Umbral Markov exacto | `0.345084` | +| Exceso realizado sobre `tau` | `0.000000` | +| Pass grilla alpha | `8/8` | + +El cap row-level de la policy `capped_blended_uncertainty` esta inactivo en las +314 filas financiadas. Por eso la formula lineal coincide numericamente en este +punto, aunque no sea valida como formula universal de la frontera. + +## Auditoria de A35 sin nueva busqueda + +La reconstruccion lee las estadisticas suficientes de seis evaluaciones exactas +ya existentes. No ejecuta HPO, busqueda de policy, generacion conformal ni solve +de portafolio. + +- filas crudas: `51,678`; +- policies semanticas deduplicadas: `50,010`; +- policies elegibles all-alpha y sobre floor: `27,508`; +- seleccion del body: sin cambio; +- thresholds con cambio material: `10,423`; +- policies tail con understatement: `2,866`; +- understatement maximo: `0.241324`; +- antiguas filas `<=0.50` que exceden `0.50` exacto: `716`. + +Los modos afectados son `tail_blended_uncertainty` y +`segment_relative_tail_blended_uncertainty`. El endpoint de maximo retorno sigue +ganando `$223,458.14`, pero su endpoint es `0.597056` y su umbral Markov exacto +es `0.697056`. El body y todos los denominadores permanecen iguales. + +## Hallazgo 2: baseline point-PD A40 + +El campo historico `price_of_robustness=-10.56%` comparaba contra un solve +rotulado `nonrobust` que aun heredaba una restriccion `pd_high`. Una +recomputacion preliminar a `tau=0.175` sirvio para detectar el problema, pero no +era el contraste final del body y se retira como superficie activa. + +A40 resuelve el contraste correcto con: + +- los mismos `276,869` candidatos; +- presupuesto `$1M`; +- misma concentracion, `tau=0.1715`, LGD, solver y controles operativos; +- point PD en objetivo y restriccion como unica diferencia semantica; +- outcomes OOT usados solo despues del solve. + +| Policy | Retorno | Funded | Default / `V` | `Gamma_CP` | `B_u` | Threshold | +|---|---:|---:|---:|---:|---:|---:| +| Point-PD two-stage LP | `$196,369.14` | `225` | `0.118400` | `0.526736` | `0.680579` | `0.780579` | +| CRPTO seleccionado | `$184,832.48` | `314` | `0.035350` | `0.162616` | `0.245084` | `0.345084` | + +CRPTO cede `$11,536.66` (`5.875%`) y reduce default/miscoverage en `0.08305` +(8.305 puntos porcentuales) y el threshold en `0.435495` (43.55 puntos). Ambos +funded sets quedan debajo de `tau`; solo CRPTO pasa el screen tight +`V <= sqrt(0.01)`. + +## Limite del claim + +A40 es una auditoria emparejada sobre un OOT historico congelado. No demuestra +causalidad, significancia prospectiva ni dominancia universal. A35 es una +frontera de grilla finita, no un optimo continuo. La probabilidad del teorema +requiere weighted funded-set validity; el draw observado audita `V` pero no +prueba por si solo ese supuesto. + +## Artefactos promovidos + +- `reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv` +- `reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.tex` +- `reports/crpto/tables/crpto_tableA40_pool93_point_baseline.csv` +- `reports/crpto/tables/crpto_tableA40_pool93_point_baseline.tex` +- `models/experiments/champion_reopen/...__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_frontier.json` +- `models/experiments/champion_reopen/...__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_governance.json` +- `models/experiments/champion_reopen/...__ijds-certificate-semantics-v2/portfolio/pool93_point_pd_baseline_audit.json` + +Las filas financiadas de A40 viven bajo `data/processed/experiments/`, no en el +directorio de modelos. Los artefactos antiguos se conservan como provenance, +pero no son fuentes de claims activos. diff --git a/docs/research/pool93_tail_risk_closeout_2026-07-02.md b/docs/research/pool93_tail_risk_closeout_2026-07-02.md index 7b7ab88..b081f6c 100644 --- a/docs/research/pool93_tail_risk_closeout_2026-07-02.md +++ b/docs/research/pool93_tail_risk_closeout_2026-07-02.md @@ -2,6 +2,10 @@ Date: 2026-07-02 +Certificate terminology was synchronized on 2026-07-09 with the policy-aware +A35 audit. A37--A39 values remain valid for the fixed selected allocation; +`0.345084` is now called the exact loss threshold, not a generic cap. + This memo closes the post-promotion caveat that tail-risk, cluster-bound, and bootstrap diagnostics should not be cited as pool93-specific unless regenerated from the selected pool93 funded allocation. @@ -28,14 +32,15 @@ At baseline `LGD = 0.45`, the selected body allocation has: - weighted default rate / `V`: `0.035350` - realized CVaR95 loss rate: `0.276211` - decision-time CVaR95 loss rate: `0.218140` -- Markov cap: `0.345084` +- exact loss threshold at `alpha = 0.01`: `0.345084` Across the LGD grid, repriced return ranges from `$188,367.48` at `LGD = 0.35` to `$179,529.98` at `LGD = 0.60`. ## A38 Cluster-Bound Repricing -At `alpha = 0.01` and `delta = 0.10`, Markov's body threshold is `0.100000`. +At `alpha = 0.01` and `delta = 0.10`, the distribution-free Markov increment +is `sqrt(alpha) = 0.100000`. The regenerated cluster-aware Hoeffding thresholds are: - period: `0.395502` @@ -43,9 +48,10 @@ The regenerated cluster-aware Hoeffding thresholds are: - period-grade: `0.281247` - score-vintage: `0.348546` -None is tighter than Markov. This supports the manuscript's theory boundary: -Markov remains the body-level distribution-free statement, while cluster-aware -tightening is shown as an assumption-priced sensitivity. +None is tighter than the distribution-free Markov step. This supports the +manuscript's theory boundary: the policy-aware certificate remains the +body-level distribution-free statement, while cluster-aware tightening is +shown as an assumption-priced sensitivity. ## A39 Fixed-Allocation Bootstrap @@ -73,6 +79,6 @@ search. A37--A39 are selected-allocation risk-profile audits. They do not change the pool93 body selector, do not make CVaR/OCE the optimized objective, and do not turn bootstrap intervals into a conformal guarantee. The paper-facing claim -remains the finite-grid return-bound certificate in A35 plus the exact funded-set -audit; A37--A39 close reviewer questions about the selected point's tail, +remains the finite-grid policy-aware decision certificate in A35 plus the exact +funded-set audit; A37--A39 close reviewer questions about the selected point's tail, concentration, and empirical contribution profile. diff --git a/justfile b/justfile index 336dce2..0112faa 100644 --- a/justfile +++ b/justfile @@ -32,9 +32,30 @@ fmt: type-check: uv run mypy src scripts +# Fast type checker from Astral. Daily active-scope use remains advisory while +# ty matures; the clean full scope is blocking in the final submission gate. +type-advisory: + @uv run python scripts/run_ty_advisory.py --scope active + +type-advisory-full: + @uv run python scripts/run_ty_advisory.py --scope full --fail-on-diagnostics --output reports/ci/ty-advisory-full.txt + +complexity-report: + uvx radon cc src scripts -s -n D + +api-docs-core: + uv run --with pdoc pdoc src.optimization.portfolio_model src.models.calibration src.evaluation.backtesting src.evaluation.fairness --docformat google --output-directory reports/api-docs --no-browser + +hooks-check: + uv run pre-commit validate-config + uvx prek validate-config .pre-commit-config.yaml + # Fast smoke: paper-final guardrails + Quarto book guardrails smoke: - uv run pytest tests/test_crpto_final_sync.py tests/test_quarto_book_guardrails.py -q + uv run pytest tests/test_crpto_final_sync.py tests/test_quarto_book_guardrails.py tests/test_publication_integrity.py -q + +publication-integrity: + uv run python scripts/check_publication_integrity.py test: uv run pytest -q @@ -69,6 +90,13 @@ paper-ijds-supplement: # Render the current submission-shaped manuscript surfaces. paper-submission: paper-ijds paper-ijds-supplement +# Compile and scan the official INFORMS/IJDS LaTeX handoff draft. +paper-submission-official: + @uv run python scripts/compile_ijds_submission.py + +# Final local IJDS gate before freezing or uploading. +submission-check: publication-integrity lint type-check type-advisory-full smoke validate-champion paper-submission paper-submission-official + # IJDS-oriented manuscript body (local HTML-print PDF verification draft). paper-ijds-pdf: uv run python scripts/render_submission_pdf_previews.py --body-only diff --git a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_frontier.json b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_frontier.json new file mode 100644 index 0000000..34a0363 --- /dev/null +++ b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_frontier.json @@ -0,0 +1,358 @@ +{ + "generated_at_utc": "2026-07-09T22:29:22.353666+00:00", + "source_run_tags": [ + "champion-reopen-2026-06-19__pool93__ijds-claim-expanded-refine", + "champion-reopen-2026-06-19__pool93__ijds-claim-micro-refine", + "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", + "champion-reopen-2026-06-19__pool93__ijds-claim-bound-closure", + "champion-reopen-2026-06-19__pool93__ijds-claim-bound-floor-closure", + "champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal" + ], + "selection_rule": { + "eligible": "all-alpha pass and nonnegative return_floor_surplus", + "dedupe_key": "semantic_policy_key", + "dedupe_semantics": "duplicate semantic policies across refinement runs have identical metrics; one representative row is retained for the consolidated table", + "body_selection": "highest realized return among eligible finite-grid policies with exact Markov_threshold <= 0.35; falls back to the legacy balanced normalized return/bound/V score only if no eligible policy exists under that declared threshold", + "bound_semantics": "Markov_threshold = weighted endpoint budget B_u + sqrt(alpha); Markov_cap = tau + residual endpoint premium + solver slack + sqrt(alpha). The exact threshold drives selection.", + "caps": [ + 0.3, + 0.32, + 0.345, + 0.36, + 0.45 + ], + "role_semantics": "finite-grid frontier roles, not continuous optima" + }, + "counts": { + "raw_rows": 51678, + "deduped_semantic_policies": 50010, + "duplicate_rows_removed": 1668, + "eligible_all_alpha_return_floor_policies": 27508, + "nonpass_or_below_floor_policies": 22502 + }, + "certificate_semantics_audit": { + "status": "corrected_from_existing_exact_bound_evaluations", + "selection_metric": "alpha01_markov_loss_threshold", + "legacy_metric": "tau + (1 - gamma) * Gamma_CP + sqrt(alpha)", + "material_difference_tolerance": 1e-05, + "materially_changed_policies": 10423, + "materially_understated_policies": 2866, + "maximum_legacy_understatement": 0.2413235, + "legacy_under_0_50_excluded_by_exact_threshold": 716, + "body_selection_unchanged": true, + "affected_policy_modes": [ + "segment_relative_tail_blended_uncertainty", + "tail_blended_uncertainty" + ] + }, + "by_run": [ + { + "run_label": "expanded", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-expanded-refine", + "rows": 3258, + "all_alpha_passers": 3071, + "all_alpha_pass_rate": 0.942602824, + "best_return": 223346.5525, + "min_markov_threshold": 0.335785 + }, + { + "run_label": "micro", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-micro-refine", + "rows": 2949, + "all_alpha_passers": 2949, + "all_alpha_pass_rate": 1.0, + "best_return": 223369.907874, + "min_markov_threshold": 0.328799 + }, + { + "run_label": "micro_ext", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", + "rows": 4407, + "all_alpha_passers": 4406, + "all_alpha_pass_rate": 0.999773088, + "best_return": 223458.135875, + "min_markov_threshold": 0.316713 + }, + { + "run_label": "bound_closure", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-bound-closure", + "rows": 1653, + "all_alpha_passers": 1653, + "all_alpha_pass_rate": 1.0, + "best_return": 213593.775, + "min_markov_threshold": 0.298369 + }, + { + "run_label": "bound_floor", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-bound-floor-closure", + "rows": 2343, + "all_alpha_passers": 2343, + "all_alpha_pass_rate": 1.0, + "best_return": 213593.775, + "min_markov_threshold": 0.284839 + }, + { + "run_label": "bound_terminal", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal", + "rows": 37068, + "all_alpha_passers": 37068, + "all_alpha_pass_rate": 1.0, + "best_return": 213593.775, + "min_markov_threshold": 0.267672 + } + ], + "rows": [ + { + "role": "minimum Markov-threshold endpoint", + "run_label": "bound_terminal", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal", + "local_candidate_id": 10662, + "family": "claim_bound_terminal_ultra_low_cap", + "anchor_rank": 219, + "source_reason": "terminal_cap_threshold_search", + "risk_tolerance": 0.16825, + "policy_mode": "capped_blended_uncertainty", + "gamma": 0.95, + "delta_cap_quantile": 1.0, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.6, + "return": 170467.268819, + "return_floor_surplus": 2.728819, + "Gamma_CP": 0.095719, + "Gamma_internalized": 0.090933, + "Gamma_residual": 0.004786, + "V": 0.031875, + "endpoint_budget": 0.173036, + "endpoint_budget_upper": 0.173036, + "Markov_threshold": 0.273036, + "Markov_cap": 0.273036, + "alpha_pass": "8/8", + "n_funded_mean": 311.125, + "semantic_policy_key": "{\"delta_cap_quantile\":1.0,\"gamma\":0.95,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"capped_blended_uncertainty\",\"risk_tolerance\":0.16825,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.6}" + }, + { + "role": "body/default balanced return-bound point", + "run_label": "micro_ext", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", + "local_candidate_id": 131, + "family": "claim_micro_ext_body_cap345", + "anchor_rank": 219, + "source_reason": "candidate37_205_body_cap345_extension", + "risk_tolerance": 0.1715, + "policy_mode": "capped_blended_uncertainty", + "gamma": 0.5475, + "delta_cap_quantile": 0.975, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.05, + "return": 184832.475845, + "return_floor_surplus": 14367.935845, + "Gamma_CP": 0.162616, + "Gamma_internalized": 0.089032, + "Gamma_residual": 0.073584, + "V": 0.03535, + "endpoint_budget": 0.245084, + "endpoint_budget_upper": 0.245084, + "Markov_threshold": 0.345084, + "Markov_cap": 0.345084, + "alpha_pass": "8/8", + "n_funded_mean": 320.5, + "semantic_policy_key": "{\"delta_cap_quantile\":0.975,\"gamma\":0.5475,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"capped_blended_uncertainty\",\"risk_tolerance\":0.1715,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.05}" + }, + { + "role": "lowest realized V return-bound point", + "run_label": "bound_terminal", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal", + "local_candidate_id": 5504, + "family": "claim_bound_terminal_ultra_low_cap", + "anchor_rank": 219, + "source_reason": "terminal_cap_threshold_search", + "risk_tolerance": 0.1675, + "policy_mode": "capped_blended_uncertainty", + "gamma": 0.925, + "delta_cap_quantile": 0.95, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.6, + "return": 171006.195983, + "return_floor_surplus": 541.655983, + "Gamma_CP": 0.09719, + "Gamma_internalized": 0.089901, + "Gamma_residual": 0.007289, + "V": 0.031875, + "endpoint_budget": 0.174789, + "endpoint_budget_upper": 0.174789, + "Markov_threshold": 0.274789, + "Markov_cap": 0.274789, + "alpha_pass": "8/8", + "n_funded_mean": 311.125, + "semantic_policy_key": "{\"delta_cap_quantile\":0.95,\"gamma\":0.925,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"capped_blended_uncertainty\",\"risk_tolerance\":0.1675,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.6}" + }, + { + "role": "max-return economic endpoint", + "run_label": "micro_ext", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", + "local_candidate_id": 4041, + "family": "claim_micro_ext_economic_endpoint", + "anchor_rank": 96, + "source_reason": "candidate2122_economic_endpoint_extension", + "risk_tolerance": 0.156875, + "policy_mode": "tail_blended_uncertainty", + "gamma": 0.445, + "delta_cap_quantile": 1.0, + "tail_focus_quantile": 0.925, + "uncertainty_aversion": 0.1375, + "return": 223458.135875, + "return_floor_surplus": 52993.595875, + "Gamma_CP": 0.457438, + "Gamma_internalized": 0.017256, + "Gamma_residual": 0.440181, + "V": 0.069575, + "endpoint_budget": 0.597056, + "endpoint_budget_upper": 0.597056, + "Markov_threshold": 0.697056, + "Markov_cap": 0.697056, + "alpha_pass": "8/8", + "n_funded_mean": 239.625, + "semantic_policy_key": "{\"delta_cap_quantile\":1.0,\"gamma\":0.445,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"tail_blended_uncertainty\",\"risk_tolerance\":0.156875,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":0.925,\"uncertainty_aversion\":0.1375}" + }, + { + "role": "highest return under threshold<=0.3", + "run_label": "bound_closure", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-bound-closure", + "local_candidate_id": 992, + "family": "claim_bound_closure_low_cap", + "anchor_rank": 219, + "source_reason": "micro_ext_min_markov_cap_endpoint_closure", + "risk_tolerance": 0.16975, + "policy_mode": "capped_blended_uncertainty", + "gamma": 0.75, + "delta_cap_quantile": 0.95, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.35, + "return": 174552.51361, + "return_floor_surplus": 4087.97361, + "Gamma_CP": 0.120988, + "Gamma_internalized": 0.090741, + "Gamma_residual": 0.030247, + "V": 0.035875, + "endpoint_budget": 0.199997, + "endpoint_budget_upper": 0.199997, + "Markov_threshold": 0.299997, + "Markov_cap": 0.299997, + "alpha_pass": "8/8", + "n_funded_mean": 306.625, + "semantic_policy_key": "{\"delta_cap_quantile\":0.95,\"gamma\":0.75,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"capped_blended_uncertainty\",\"risk_tolerance\":0.16975,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.35}" + }, + { + "role": "highest return under threshold<=0.32", + "run_label": "micro_ext", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", + "local_candidate_id": 2021, + "family": "claim_micro_ext_bound_tight", + "anchor_rank": 219, + "source_reason": "candidate949_bound_tight_extension", + "risk_tolerance": 0.1705, + "policy_mode": "capped_blended_uncertainty", + "gamma": 0.645, + "delta_cap_quantile": 0.95, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.2375, + "return": 179436.119445, + "return_floor_surplus": 8971.579445, + "Gamma_CP": 0.139182, + "Gamma_internalized": 0.089773, + "Gamma_residual": 0.04941, + "V": 0.035875, + "endpoint_budget": 0.21991, + "endpoint_budget_upper": 0.21991, + "Markov_threshold": 0.31991, + "Markov_cap": 0.31991, + "alpha_pass": "8/8", + "n_funded_mean": 310.125, + "semantic_policy_key": "{\"delta_cap_quantile\":0.95,\"gamma\":0.645,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"capped_blended_uncertainty\",\"risk_tolerance\":0.1705,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.2375}" + }, + { + "role": "highest return under threshold<=0.345", + "run_label": "micro_ext", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", + "local_candidate_id": 512, + "family": "claim_micro_ext_body_cap345", + "anchor_rank": 219, + "source_reason": "candidate37_205_body_cap345_extension", + "risk_tolerance": 0.17225, + "policy_mode": "capped_blended_uncertainty", + "gamma": 0.5525, + "delta_cap_quantile": 0.975, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.0375, + "return": 184800.413581, + "return_floor_surplus": 14335.873581, + "Gamma_CP": 0.162562, + "Gamma_internalized": 0.089816, + "Gamma_residual": 0.072747, + "V": 0.03535, + "endpoint_budget": 0.244997, + "endpoint_budget_upper": 0.244997, + "Markov_threshold": 0.344997, + "Markov_cap": 0.344997, + "alpha_pass": "8/8", + "n_funded_mean": 321.0, + "semantic_policy_key": "{\"delta_cap_quantile\":0.975,\"gamma\":0.5525,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"capped_blended_uncertainty\",\"risk_tolerance\":0.17225,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.0375}" + }, + { + "role": "highest return under threshold<=0.36", + "run_label": "micro_ext", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", + "local_candidate_id": 3212, + "family": "claim_micro_ext_cap036_return", + "anchor_rank": 219, + "source_reason": "candidate1975_cap036_return_extension", + "risk_tolerance": 0.17575, + "policy_mode": "capped_blended_uncertainty", + "gamma": 0.525, + "delta_cap_quantile": 0.95, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.075, + "return": 186050.727749, + "return_floor_surplus": 15586.187749, + "Gamma_CP": 0.1746, + "Gamma_internalized": 0.091665, + "Gamma_residual": 0.082935, + "V": 0.03775, + "endpoint_budget": 0.258685, + "endpoint_budget_upper": 0.258685, + "Markov_threshold": 0.358685, + "Markov_cap": 0.358685, + "alpha_pass": "8/8", + "n_funded_mean": 318.75, + "semantic_policy_key": "{\"delta_cap_quantile\":0.95,\"gamma\":0.525,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"capped_blended_uncertainty\",\"risk_tolerance\":0.17575,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.075}" + }, + { + "role": "highest return under threshold<=0.45", + "run_label": "expanded", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-expanded-refine", + "local_candidate_id": 979, + "family": "bound_efficient_local", + "anchor_rank": 223, + "source_reason": "rank219_rank223_bound_frontier", + "risk_tolerance": 0.185, + "policy_mode": "blended_uncertainty", + "gamma": 0.35, + "delta_cap_quantile": 1.0, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.15, + "return": 198693.277519, + "return_floor_surplus": 28228.737519, + "Gamma_CP": 0.252323, + "Gamma_internalized": 0.088313, + "Gamma_residual": 0.16401, + "V": 0.0456, + "endpoint_budget": 0.34901, + "endpoint_budget_upper": 0.34901, + "Markov_threshold": 0.44901, + "Markov_cap": 0.44901, + "alpha_pass": "8/8", + "n_funded_mean": 310.875, + "semantic_policy_key": "{\"delta_cap_quantile\":1.0,\"gamma\":0.35,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"blended_uncertainty\",\"risk_tolerance\":0.185,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.15}" + } + ] +} diff --git a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_governance.json b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_governance.json new file mode 100644 index 0000000..7f7476b --- /dev/null +++ b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_governance.json @@ -0,0 +1,198 @@ +{ + "generated_at_utc": "2026-07-09T23:10:36.956639+00:00", + "source_frontier_path": "C:\\Users\\carlos\\Documents\\Paper_CRPTO\\models\\experiments\\champion_reopen\\champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2\\portfolio\\pool93_ijds_consolidated_frontier.json", + "source_run_tags": [ + "champion-reopen-2026-06-19__pool93__ijds-claim-expanded-refine", + "champion-reopen-2026-06-19__pool93__ijds-claim-micro-refine", + "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", + "champion-reopen-2026-06-19__pool93__ijds-claim-bound-closure", + "champion-reopen-2026-06-19__pool93__ijds-claim-bound-floor-closure", + "champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal" + ], + "counts": { + "raw_rows": 51678, + "deduped_semantic_policies": 50010, + "duplicate_rows_removed": 1668, + "eligible_all_alpha_return_floor_policies": 27508, + "nonpass_or_below_floor_policies": 22502 + }, + "selection_rule": { + "eligible": "all-alpha pass and nonnegative return_floor_surplus", + "dedupe_key": "semantic_policy_key", + "dedupe_semantics": "duplicate semantic policies across refinement runs have identical metrics; one representative row is retained for the consolidated table", + "body_selection": "highest realized return among eligible finite-grid policies with exact Markov_threshold <= 0.35; falls back to the legacy balanced normalized return/bound/V score only if no eligible policy exists under that declared threshold", + "bound_semantics": "Markov_threshold = weighted endpoint budget B_u + sqrt(alpha); Markov_cap = tau + residual endpoint premium + solver slack + sqrt(alpha). The exact threshold drives selection.", + "caps": [ + 0.3, + 0.32, + 0.345, + 0.36, + 0.45 + ], + "role_semantics": "finite-grid frontier roles, not continuous optima" + }, + "certificate_semantics_audit": { + "status": "corrected_from_existing_exact_bound_evaluations", + "selection_metric": "alpha01_markov_loss_threshold", + "legacy_metric": "tau + (1 - gamma) * Gamma_CP + sqrt(alpha)", + "material_difference_tolerance": 1e-05, + "materially_changed_policies": 10423, + "materially_understated_policies": 2866, + "maximum_legacy_understatement": 0.2413235, + "legacy_under_0_50_excluded_by_exact_threshold": 716, + "body_selection_unchanged": true, + "affected_policy_modes": [ + "segment_relative_tail_blended_uncertainty", + "tail_blended_uncertainty" + ] + }, + "claim_hierarchy": { + "status": "final", + "paper_body_candidate": "body/default balanced return-bound point", + "paper_body_claim": "The selected pool93 body point is the highest-return eligible finite-grid policy under the declared exact Markov-threshold ceiling and passes all eight predeclared alpha checks.", + "appendix_frontier_candidates": [ + "minimum Markov-threshold endpoint", + "highest return under threshold<=0.345", + "highest return under threshold<=0.36", + "max-return economic endpoint" + ], + "do_not_claim": [ + "continuous-region optimality beyond the evaluated finite policy grid", + "nominal funded-set alpha coverage when V(alpha) exceeds alpha", + "prospective live-selection validity from retrospective OOT selection", + "pool93-specific row-level tail/CVaR dominance unless regenerated from the promoted allocation" + ], + "promotion_gate": [ + "consolidated frontier generated from completed exact runs", + "semantic-policy deduplication applied", + "policy-aware endpoint decomposition applied to every alpha=0.01 row", + "selected body point passes 8/8 alpha checks", + "zero realized risk-tolerance excess at alpha=0.01", + "return exceeds declared return floor", + "A35 frontier, A36 funded-set grade audit, and A40 matched baseline are regenerated from retained artifacts" + ] + }, + "selected_candidates": { + "paper_body": { + "role": "body/default balanced return-bound point", + "run_label": "micro_ext", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", + "local_candidate_id": 131, + "family": "claim_micro_ext_body_cap345", + "anchor_rank": 219, + "source_reason": "candidate37_205_body_cap345_extension", + "risk_tolerance": 0.1715, + "policy_mode": "capped_blended_uncertainty", + "gamma": 0.5475, + "delta_cap_quantile": 0.975, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.05, + "return": 184832.475845, + "return_floor_surplus": 14367.935845, + "Gamma_CP": 0.162616, + "Gamma_internalized": 0.089032, + "Gamma_residual": 0.073584, + "V": 0.03535, + "endpoint_budget": 0.245084, + "endpoint_budget_upper": 0.245084, + "Markov_threshold": 0.345084, + "Markov_cap": 0.345084, + "alpha_pass": "8/8", + "n_funded_mean": 320.5, + "semantic_policy_key": "{\"delta_cap_quantile\":0.975,\"gamma\":0.5475,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"capped_blended_uncertainty\",\"risk_tolerance\":0.1715,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.05}" + }, + "strict_threshold_leq_0_345": { + "role": "highest return under threshold<=0.345", + "run_label": "micro_ext", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", + "local_candidate_id": 512, + "family": "claim_micro_ext_body_cap345", + "anchor_rank": 219, + "source_reason": "candidate37_205_body_cap345_extension", + "risk_tolerance": 0.17225, + "policy_mode": "capped_blended_uncertainty", + "gamma": 0.5525, + "delta_cap_quantile": 0.975, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.0375, + "return": 184800.413581, + "return_floor_surplus": 14335.873581, + "Gamma_CP": 0.162562, + "Gamma_internalized": 0.089816, + "Gamma_residual": 0.072747, + "V": 0.03535, + "endpoint_budget": 0.244997, + "endpoint_budget_upper": 0.244997, + "Markov_threshold": 0.344997, + "Markov_cap": 0.344997, + "alpha_pass": "8/8", + "n_funded_mean": 321.0, + "semantic_policy_key": "{\"delta_cap_quantile\":0.975,\"gamma\":0.5525,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"capped_blended_uncertainty\",\"risk_tolerance\":0.17225,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.0375}" + }, + "minimum_markov_threshold_endpoint": { + "role": "minimum Markov-threshold endpoint", + "run_label": "bound_terminal", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal", + "local_candidate_id": 10662, + "family": "claim_bound_terminal_ultra_low_cap", + "anchor_rank": 219, + "source_reason": "terminal_cap_threshold_search", + "risk_tolerance": 0.16825, + "policy_mode": "capped_blended_uncertainty", + "gamma": 0.95, + "delta_cap_quantile": 1.0, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.6, + "return": 170467.268819, + "return_floor_surplus": 2.728819, + "Gamma_CP": 0.095719, + "Gamma_internalized": 0.090933, + "Gamma_residual": 0.004786, + "V": 0.031875, + "endpoint_budget": 0.173036, + "endpoint_budget_upper": 0.173036, + "Markov_threshold": 0.273036, + "Markov_cap": 0.273036, + "alpha_pass": "8/8", + "n_funded_mean": 311.125, + "semantic_policy_key": "{\"delta_cap_quantile\":1.0,\"gamma\":0.95,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"capped_blended_uncertainty\",\"risk_tolerance\":0.16825,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":1.0,\"uncertainty_aversion\":0.6}" + }, + "max_return_economic_endpoint": { + "role": "max-return economic endpoint", + "run_label": "micro_ext", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", + "local_candidate_id": 4041, + "family": "claim_micro_ext_economic_endpoint", + "anchor_rank": 96, + "source_reason": "candidate2122_economic_endpoint_extension", + "risk_tolerance": 0.156875, + "policy_mode": "tail_blended_uncertainty", + "gamma": 0.445, + "delta_cap_quantile": 1.0, + "tail_focus_quantile": 0.925, + "uncertainty_aversion": 0.1375, + "return": 223458.135875, + "return_floor_surplus": 52993.595875, + "Gamma_CP": 0.457438, + "Gamma_internalized": 0.017256, + "Gamma_residual": 0.440181, + "V": 0.069575, + "endpoint_budget": 0.597056, + "endpoint_budget_upper": 0.597056, + "Markov_threshold": 0.697056, + "Markov_cap": 0.697056, + "alpha_pass": "8/8", + "n_funded_mean": 239.625, + "semantic_policy_key": "{\"delta_cap_quantile\":1.0,\"gamma\":0.445,\"min_budget_utilization\":0.0,\"pd_cap_slack_penalty\":0.0,\"policy_mode\":\"tail_blended_uncertainty\",\"risk_tolerance\":0.156875,\"solver_backend\":\"highspy\",\"tail_focus_quantile\":0.925,\"uncertainty_aversion\":0.1375}" + } + }, + "paper_artifacts": { + "frontier_table_csv": "reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv", + "frontier_table_tex": "reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.tex", + "funded_grade_audit_csv": "reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.csv", + "funded_grade_audit_tex": "reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.tex", + "point_baseline_csv": "reports/crpto/tables/crpto_tableA40_pool93_point_baseline.csv", + "point_baseline_tex": "reports/crpto/tables/crpto_tableA40_pool93_point_baseline.tex", + "point_baseline_audit": "models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_point_pd_baseline_audit.json" + } +} diff --git a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_point_pd_baseline_audit.json b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_point_pd_baseline_audit.json new file mode 100644 index 0000000..d51dbe5 --- /dev/null +++ b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_point_pd_baseline_audit.json @@ -0,0 +1,86 @@ +{ + "schema_version": "2026-07-09.1", + "generated_at_utc": "2026-07-09T22:41:29.886155+00:00", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2", + "comparison": "matched point-PD two-stage LP versus selected CRPTO", + "fixed_design": { + "candidate_universe": 276869, + "budget": 1000000.0, + "risk_tolerance": 0.1715, + "max_concentration": 0.25, + "alpha": 0.01 + }, + "point_pd_baseline": { + "solver_status": "Optimal", + "realized_return": 196369.14000000004, + "total_allocated": 1000000.0, + "expected_return_gross": 283248.8175, + "expected_loss_point": 69229.66551833821, + "expected_return_net_point": 214019.1519816618, + "certificate": { + "alpha": 0.01, + "risk_tolerance": 0.1715, + "n_funded": 225, + "weighted_outcome": 0.11840000000000002, + "weighted_miscoverage": 0.11840000000000002, + "weighted_coverage": 0.8815999999999999, + "empirical_coverage_funded": 0.8844444444444445, + "weighted_pd_point": 0.1538437011518627, + "weighted_pd_effective": 0.1538437011518627, + "endpoint_budget": 0.680579296391784, + "gamma_cp": 0.5267355952399213, + "gamma_internalized": 0.0, + "gamma_residual": 0.5267355952399213, + "effective_constraint_slack": 0.017656298848137325, + "effective_constraint_excess": 0.0, + "realized_risk_tolerance_excess": 0.0, + "sqrt_alpha": 0.1, + "endpoint_budget_upper": 0.6982355952399213, + "markov_loss_threshold": 0.780579296391784, + "markov_loss_cap": 0.7982355952399213 + } + }, + "selected_crpto": { + "solver_status": "frozen_selected_allocation", + "realized_return": 184832.47584455396, + "total_allocated": 1000000.0, + "expected_return_gross": 208153.285844554, + "expected_loss_point": 37110.41439596723, + "expected_return_net_point": 171042.87144858675, + "certificate": { + "alpha": 0.01, + "risk_tolerance": 0.1715, + "n_funded": 314, + "weighted_outcome": 0.03535, + "weighted_miscoverage": 0.03535, + "weighted_coverage": 0.96465, + "empirical_coverage_funded": 0.9426751592356688, + "weighted_pd_point": 0.08246758754659385, + "weighted_pd_effective": 0.1715, + "endpoint_budget": 0.24508386600030369, + "gamma_cp": 0.1626162784537099, + "gamma_internalized": 0.08903241245340617, + "gamma_residual": 0.07358386600030371, + "effective_constraint_slack": 0.0, + "effective_constraint_excess": 0.0, + "realized_risk_tolerance_excess": 0.0, + "sqrt_alpha": 0.1, + "endpoint_budget_upper": 0.24508386600030374, + "markov_loss_threshold": 0.3450838660003037, + "markov_loss_cap": 0.3450838660003037 + } + }, + "contrasts": { + "realized_return_cost": 11536.66415544608, + "realized_return_cost_pct": 5.874988379256577, + "weighted_default_rate_reduction": 0.08305000000000001, + "weighted_miscoverage_reduction": 0.08305000000000001, + "markov_threshold_reduction": 0.4354954303914803 + }, + "claim_boundary": "Frozen OOT matched-policy audit; it quantifies a return-risk trade-off and does not establish causal, prospective or universal dominance.", + "outputs": { + "point_funded_rows": "C:\\Users\\carlos\\Documents\\Paper_CRPTO\\data\\processed\\experiments\\champion_reopen\\champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2\\portfolio\\pool93_point_pd_baseline_alpha01.parquet", + "table_csv": "C:\\Users\\carlos\\Documents\\Paper_CRPTO\\reports\\crpto\\experiments\\champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2\\crpto_tableA40_pool93_point_baseline.csv", + "table_tex": "C:\\Users\\carlos\\Documents\\Paper_CRPTO\\reports\\crpto\\experiments\\champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2\\crpto_tableA40_pool93_point_baseline.tex" + } +} diff --git a/paper/CRPTO.qmd b/paper/CRPTO.qmd index 155ad00..09e3487 100644 --- a/paper/CRPTO.qmd +++ b/paper/CRPTO.qmd @@ -17,11 +17,12 @@ execute: CRPTO integra prediccion de default calibrada, intervalos conformales Mondrian y optimizacion robusta de portafolio. La superficie IJDS activa promueve el punto pool93 body/default: retorno robusto `$184,832.48`, `V=0.035350`, -`Gamma_CP=0.162616`, Markov cap `0.345084`, zero violation y pass `8/8` sobre +`Gamma_CP=0.162616`, `Gamma_res=0.073584`, endpoint `0.245084`, umbral Markov +`0.345084`, exceso realizado sobre `tau` igual a cero y pass `8/8` sobre la grilla alpha declarada. La frontera A35 y el audit de composicion A36 son los artefactos paper-facing principales para el cierre pool93; A37--A39 agregan la repricing de tail-risk, el audit cluster-bound y el bootstrap de contribuciones -de la asignacion seleccionada. +de la asignacion seleccionada; A40 agrega la baseline point-PD emparejada. # Manuscrito @@ -35,7 +36,7 @@ Los borradores de trabajo son: - `CRPTO_ijds.qmd`: cuerpo anonimo IJDS, conceptualmente limitado a 25 paginas excluyendo referencias y appendix. -- `supplement_ijds.qmd`: online supplement IJDS con A3--A39, robustez, +- `supplement_ijds.qmd`: online supplement IJDS con A3--A40, robustez, reproducibilidad, MRM y fairness. La version larga y sus apendices estan en `book/`. @@ -49,9 +50,9 @@ documentan esa cadena historica, no el body point. Body claim pool93 (autoritativos): -- `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_ijds_consolidated_governance.json` +- `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_governance.json` - `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal/portfolio/pool93_ijds_claim_governance.json` -- `reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv` (y A36--A39) +- `reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv` (y A36--A40) Cadena upstream congelada (historica, return floor): diff --git a/paper/CRPTO_ijds.qmd b/paper/CRPTO_ijds.qmd index a9a9429..be56261 100644 --- a/paper/CRPTO_ijds.qmd +++ b/paper/CRPTO_ijds.qmd @@ -25,41 +25,46 @@ execute: # Abstract -Credit allocation is a data-science-for-decisions problem: calibrated default -probabilities matter only after they shape which loans are funded under a budget -and risk appetite. We introduce Conformal Robust Predict-Then-Optimize (CRPTO), -a post-hoc bridge that maps a frozen calibrated probability-of-default model -through Mondrian conformal intervals into robust portfolio constraints and an -empirical funded-set audit. On a 276,869-loan out-of-time Lending Club -evaluation, the selected policy earns `$184.8K` on a `$1M` budget while passing -the declared eight-level alpha grid ($V(0.01)=0.035350$, -$\Gamma_{\mathrm{CP}}=0.162616$, Markov cap `0.345084`, zero violation). The -consolidated finite frontier contains `50,010` deduplicated semantic policies, -of which `27,508` both pass all declared alpha levels and exceed the return -floor, making the return-bound trade-off visible rather than implicit. Frozen -Prosper and Freddie/Mendeley applications test recipe transfer and preserve the -predeclared global conformal gates with positive robust LP objectives. The -insight is that uncertainty should be reported as a decision frontier, not as a -post-hoc calibration table. The contribution is a conformal-robust -credit-portfolio decision certificate with a distribution-free Markov bound -under weighted funded-set validity: it connects real credit data, calibrated -predictive models, robust funding decisions, and a validation harness that -rebuilds the prediction-to-decision chain from frozen inputs while keeping the -statistical guarantee boundary explicit. +Credit allocation is a data-science-for-decisions problem: default probabilities +matter only after they shape which loans are funded under a budget and risk +appetite. We introduce Conformal Robust Predict-Then-Optimize (CRPTO), a +post-hoc decision certificate that maps a frozen calibrated +probability-of-default model through Mondrian conformal intervals into robust +portfolio constraints and an empirical funded-set audit. A policy-aware +decomposition separates the conformal premium internalized by the optimizer +from the residual premium needed to recover the exact upper-endpoint budget. +On a 276,869-loan out-of-time Lending Club evaluation, the selected policy earns +`$184.8K` on a `$1M` budget while passing the declared eight-level alpha grid +($V(0.01)=0.035350$, $\Gamma_{\mathrm{CP}}=0.162616$, exact Markov loss +threshold `0.345084`, zero realized risk-tolerance excess). Against a matched +point-PD two-stage LP with the same candidates, budget, concentration cap, and +risk tolerance, CRPTO gives up `5.87%` of realized return while reducing the +weighted default rate by `8.305` percentage points and the loss threshold by +`43.55` percentage points. The consolidated finite frontier contains `50,010` +deduplicated semantic policies, of which `27,508` pass all declared alpha levels +and exceed the return floor. Frozen Prosper and Freddie/Mendeley applications +test recipe transfer and preserve the predeclared global conformal gates with +positive robust LP objectives. CRPTO therefore makes predictive uncertainty +decision-useful as an auditable return--risk frontier, with a distribution-free +Markov bound under weighted funded-set validity and an explicit separation +between deterministic accounting and its statistical assumption. **Keywords:** conformal prediction; robust optimization; predict-then-optimize; credit risk; portfolio optimization; reproducible data science. # Introduction -Credit allocation is a predict-then-decide problem. A lender first estimates a -probability of default (PD), then chooses which loans to fund under a budget and -risk appetite. The modeling literature has become very good at the first step: -calibration, discrimination, and backtesting are now standard ingredients of -credit-risk model validation [@lessmann2015; @chen2024creditrisk]. The second step is -less settled. Once a calibrated PD enters an optimizer, uncertainty is often -treated as a reporting diagnostic rather than as a constraint that can change -the funded set. +Credit allocation is a contextual optimization problem in credit form: a lender +first estimates a probability of default (PD), then chooses which loans to fund +under a budget and risk appetite [@sadana2025contextual]. The modeling +literature has become very good at the first step: calibration, discrimination, +and backtesting are now standard ingredients of credit-risk model validation +[@lessmann2015; @chen2024creditrisk]. Recent credit-scoring work also prices +predictive uncertainty and parameter uncertainty through profit, rejection, or +multiobjective risk metrics [@xu2025profit_uncertainty_credit; +@xu2024profit_risk_credit]. The second step is less settled. Once a calibrated +PD enters an optimizer, uncertainty is often treated as a reporting diagnostic +rather than as a constraint that can change the funded set. That separation is uncomfortable in auditable credit decisions. A portfolio policy can have a reasonable average PD and still concentrate probability mass @@ -68,12 +73,14 @@ conservative can pass every risk check while destroying economic value. The scientific question in this paper is therefore not whether one can build a slightly better credit classifier. It is whether finite-sample predictive uncertainty can be carried into a robust portfolio decision in a way that is -transparent enough for a reviewer to audit. This has practical stakes. In a -pre-registered randomized trial, conformal prediction sets -improved human decision making relative to fixed-size sets with the same -coverage [@cresswell2024]. CRPTO takes that committee-facing idea into a credit -portfolio setting, where the uncertainty summary must change a funding decision -or it is just another report. +transparent enough for a reviewer to audit. This has practical stakes, but it is +not automatic: conformal sets can be valid without being decision-useful unless +the downstream action and objective are explicit +[@hullman2025conformal_human_decision]. In a pre-registered randomized trial, +conformal prediction sets improved human decision making relative to fixed-size +sets with the same coverage [@cresswell2024]. CRPTO takes that +committee-facing idea into a credit portfolio setting, where the uncertainty +summary must change a funding decision or it is just another report. CRPTO answers this question with a post-hoc, reproducible pipeline. It starts from a calibrated CatBoost PD model, constructs Mondrian conformal intervals @@ -89,7 +96,7 @@ out-of-time evaluation set of 276,869 loans. The consolidated frontier contains 50,010 deduplicated semantic policies, of which 27,508 pass every declared alpha level and exceed the return floor. From that declared finite frontier, the selected policy is the body/default balanced point at the approximately -`0.345` return-bound lens, with Markov cap `0.345084`; it is neither a +`0.345` return-bound lens, with exact Markov loss threshold `0.345084`; it is neither a continuous global optimum nor the economic endpoint. The selected point earns `$184.8K` on a `$1M` budget and passes the exact empirical funded-set audit at $\alpha = 0.01$. The headline result is not a single lucky allocation. It is a @@ -104,29 +111,27 @@ out-of-sample and out-of-time splits. These replications are not new champions; they test whether the same PD-to-conformal-to-LP recipe remains economically usable on different credit products. -The paper makes five contributions. First, it gives a CRPTO construction for -credit portfolios: frozen calibrated PD, Mondrian conformal uncertainty, and -robust budgeted optimization as a post-hoc decision audit. Second, it proves a +The paper makes four contributions. First, it gives a CRPTO construction for +credit portfolios: frozen calibrated PD, Mondrian conformal uncertainty, robust +budgeted optimization, and an exact post-allocation audit. Second, it proves a distribution-free Markov bound under weighted funded-set validity (Theorem 1) -that splits realized portfolio loss into the conformal upper-endpoint budget -$B_u(\alpha) = \tau + (1-\gamma)\,\Gamma_{\mathrm{CP}}(\alpha)$ and the weighted -miscoverage $V(\alpha)$, with supplement propositions showing that Markov is -optimal under the stated assumption (A.1) and locating the cluster structure -that would tighten it (A.2). Third, it locates that construction relative to -data-driven robust optimization, P2P lending portfolio models, conformal credit -scoring, and decision-focused learning. Fourth, it provides an evidence-backed -empirical study where every table and figure is generated from frozen outputs -rather than manually transcribed summaries. Fifth, it uses a three-level -evidence ladder: the Lending Club funded set carries the certificate, Prosper -and Freddie/Mendeley test recipe transfer as external economic replications, -and the tail, dependence, online, and robustness appendices stay diagnostic. -The key claim is narrow: CRPTO maps frozen calibrated PD models into a robust -funded set, reports the portfolio-level conformal premium -$\Gamma_{\mathrm{CP}}$, and verifies exact alpha-safe weighted miscoverage on the -promoted Lending Club portfolio and its surrounding finite-grid frontier. The -same conformal and LP gates remain viable on two additional credit datasets, -while the paper keeps the boundary between paper revision and new model-search -evidence visible. +and introduces a policy-aware decomposition +$\Gamma_{\mathrm{CP}}=\Gamma_{\mathrm{int}}+\Gamma_{\mathrm{res}}$ that recovers +the exact upper-endpoint budget for linear, capped, and tail-focused policies. +Supplement propositions show that Markov is optimal under the stated +first-moment assumption (A.1) and locate the cluster structure that would +tighten it (A.2). Third, it reports the selected Lending Club decision as part +of a declared finite-grid return-bound frontier and compares it with a matched +point-PD allocation; the body point, surrounding policies, exact alpha checks, +and external Prosper/Freddie economic replications are all generated from frozen +evidence. Fourth, it packages the result as a reproducible IJDS decision +artifact, with tables, figures, governance files, and claim-sync checks designed +to keep the statistical boundary visible. The key claim is narrow: CRPTO maps a +frozen calibrated PD model into a robust funded set, reports how much conformal +premium is internalized or left residual, and audits the promoted Lending Club +allocation and its finite-grid frontier exactly. Adjacent methods enter only to +locate and stress-test this single claim, not to create additional acceptance +criteria. Read as data science for decisions, the paper's four components are explicit. The data component is a static Lending Club OOT panel, with Prosper and @@ -161,13 +166,14 @@ the price of robustness made visible as a design trade-off [@bertsimas2004]. Robust portfolio selection makes that trade-off operational for allocation under parameter uncertainty [@goldfarb2003robustportfolio], whereas distributionally robust optimization broadens the uncertainty object toward -moment or ambiguity sets [@delage2010dro]. Data-driven robust and -prescriptive-analytics work then connects predictive models to downstream -decisions while keeping the uncertainty-to-action contract explicit -[@bertsimas2018datadriven; @bertsimas2020prescriptive]. Recent work connects +moment or ambiguity sets [@delage2010dro]. Data-driven robust, +contextual-optimization, and prescriptive-analytics work then connects +predictive models to downstream decisions while keeping the +uncertainty-to-action contract explicit [@bertsimas2018datadriven; +@bertsimas2020prescriptive; @sadana2025contextual]. Recent work connects conformal prediction and robust optimization more directly by using conformal uncertainty sets in downstream decisions [@johnstone2021; @patel2024; -@hu2026crc]. That line certifies the uncertainty set *before* the decision: +@sun2024ptc; @hu2026crc]. That line certifies the uncertainty set *before* the decision: coverage of the conformal region is the guarantee, and the downstream decision inherits it. CRPTO follows this line but audits the other side of the decision as well: after the optimizer selects a funded set, the realized weighted @@ -198,8 +204,12 @@ panels [@lessmann2015; @ayari2026; @xia2017]. Recent IJDS credit-risk work shows how richer data structures such as firm graphs can improve rating prediction [@das2023creditgraph], and cost-aware calibration work makes explicit why probability quality matters when predictions feed asymmetric downstream -decisions [@yang2025costaware]. IJDS decision papers also sharpen the distinction -between an accurate intermediate estimate and an effective automated decision +decisions [@yang2025costaware]. EJOR/Omega credit-scoring work similarly moves +from discrimination to economic uncertainty, using profit-based uncertainty, +rejection, parameter uncertainty, and multiobjective profit/risk metrics +[@xu2025profit_uncertainty_credit; @xu2024profit_risk_credit]. IJDS decision +papers also sharpen the distinction between an accurate intermediate estimate +and an effective automated decision [@fernandezloria2022causaldecision; @fernandezloria2025observational], while replication-robust analytics markets show the journal's appetite for robust, reproducible decision systems [@falconer2026replication]. Work on fintech @@ -221,6 +231,7 @@ leaderboard. | IJDS precedent | Lesson for this submission | CRPTO extension | |---|---|---| +| AI/OR perspective [@wiberg2025ai_or] | AI methods and OR structure strengthen each other when rigor and interpretability remain visible. | Makes the AI-to-OR bridge an auditable credit decision certificate. | | Credit graph ML [@das2023creditgraph] | IJDS accepts credit-risk ML when data, method, and reproducibility are explicit. | Moves from rating prediction to a funded portfolio certificate. | | Cost-aware calibration [@yang2025costaware] | Calibration matters because downstream miscalibration costs are asymmetric. | Prices uncertainty inside a budgeted allocation and exact audit. | | Causal decision papers [@fernandezloria2022causaldecision; @fernandezloria2025observational] | Decision quality is not the same as estimating an intermediate quantity. | Evaluates the funded decision, not only PD quality. | @@ -229,20 +240,23 @@ leaderboard. : IJDS decision-science precedents and the CRPTO extension. Finally, recent work on conformal model selection for robust optimization, +non-exchangeable conformal risk control, valid selection among conformal sets, multi-distribution conformal validity, online conformal portfolio methods, end-to-end conformal risk training, robust conformal decision certificates, and -conformal satisficing motivates the journal-strengthening package +conformal satisficing fixes the boundary around the single IJDS claim [@bao2025croms; @yang2026multidistribution; @liu2026portfolio; +@farinhas2024nonexchangeable_crc; @hegazy2025valid_selection_conformal_sets; @zhou2025credo; @zhou2026creme; @zhao2025robust]. We use those ideas where they can be evaluated from the frozen CRPTO evidence: OCE/CVaR [@rockafellar2000cvar; @bental2007oce] appears as a tail-risk diagnostic, robust satisficing appears as committee-style margin evidence, and SPO+ motivates the regret-auditability frontier. The external Prosper and Freddie/Mendeley runs are a separate, frozen replication protocol rather than a -new method-changing search. The remaining method-changing variants--optimized -OCE/CVaR objectives, online protocols, causal layers, and hybrid -decision-focused training--remain future work rather than hidden acceptance -criteria. +new method-changing search. The remaining variants--optimized OCE/CVaR +objectives, non-exchangeable recalibration, formal post-selection conformal-set +selection, online protocols, causal layers, and hybrid decision-focused +training--are explicitly outside the submitted claim rather than hidden +acceptance criteria. ## Closest Work Boundary @@ -261,6 +275,7 @@ and conformal finance portfolios [@noguer2024portfolio; @kato2025; | Neighboring literature | What it already contributes | What CRPTO adds | Why not enough for auditable credit decisions | |---|---|---|---| | P2P/Lending Club OR | Credit investment recommendation and robust/multi-objective funding models. | Conformal PD uncertainty as the uncertainty set plus exact funded-set validation. | Leaves a gap between prediction uncertainty and a post-allocation certificate that a credit committee can inspect. | +| Profit/risk credit scoring | Economic uncertainty, rejection, and cost-sensitive profit metrics for credit decisions. | Portfolio-level conformal premium and exact funded-set audit. | Still evaluates the classifier or rejection rule before a funded portfolio certificate. | | Conformal credit scoring | Conformal intervals for ordinal credit scores. | A downstream robust portfolio decision, not only score uncertainty. | Stops at score uncertainty; it does not audit a budgeted funded set or economic policy. | | Conformal robust optimization | Conformal sets used in downstream robust decisions. | A frozen Lending Club credit-risk model stack with paper-facing audit trail. | Establishes the decision logic, but not the credit-specific PD lineage and funded-set certificate. | | Decision-focused learning | Training losses aligned with downstream regret. | A post-hoc governance path for institutions with existing calibrated PD models. | Improves training-time regret, but does not certify risk controls after a frozen production-style PD model. | @@ -328,36 +343,43 @@ safety margin the method is meant to provide on a large out-of-time sample. The decision variable $x_i$ is the allocation fraction for each eligible loan; $x_i a_i$ is the funded exposure. The optimizer maximizes expected net -economic return under a `$1M` budget and policy constraints that cap portfolio -risk after replacing point PD estimates with conformal upper endpoints. The -selected body point has -`risk_tolerance = 0.1715`, `policy_mode = capped_blended_uncertainty`, policy -parameter `gamma = 0.5475`, and `uncertainty_aversion = 0.05`. +economic return under a `$1M` budget and policy constraints that replace point +PD with a declared effective decision score $q_i(\alpha;\theta)$. Every policy +used in the frontier satisfies +$\hat p_i\leq q_i(\alpha;\theta)\leq u_i(\alpha)$; $\theta$ identifies whether +the policy is a linear blend, capped blend, or tail-focused blend. The selected +body point has `risk_tolerance = 0.1715`, +`policy_mode = capped_blended_uncertainty`, `gamma = 0.5475`, and +`uncertainty_aversion = 0.05`. Schematically, the robust decision layer solves a budgeted allocation problem of the form $$ \begin{aligned} -\max_x\quad & \sum_i x_i a_i \left(c_i - \tilde p_i(\alpha,\gamma)\,L\right) \\ +\max_x\quad & \sum_i x_i a_i \left(c_i - q_i(\alpha;\theta)\,L\right) \\ \text{s.t.}\quad & \sum_i x_i a_i \le B,\\ -& \sum_i x_i a_i \tilde p_i(\alpha,\gamma) +& \sum_i x_i a_i q_i(\alpha;\theta) \le \tau \sum_i x_i a_i,\\ & 0 \le x_i \le \bar x_i, \end{aligned} $$ where `a_i` is exposure, `c_i` is the loan coupon (interest rate), `L` is the -loss-given-default (`L = 0.45` in the frozen evaluation), $\tau$ is the -risk-tolerance cap, and +loss-given-default (`L = 0.45` in the frozen evaluation), and $\tau$ is the +risk-tolerance cap. The linear member of the policy family is $$ -\tilde p_i(\alpha,\gamma) +q_i(\alpha;\gamma) = \hat p_i + \gamma\left(u_i(\alpha)-\hat p_i\right) $$ -on the PD scale, clipped to the feasible probability range. The objective is the *expected* -net return `c_i - p_tilde_i * L`; the headline realized return is the +on the PD scale. Capped and tail-focused members transform that score while +remaining between the point PD and upper endpoint. For the selected allocation, +the cap is inactive on all 314 funded rows, so its effective score equals the +linear expression above; that fact is audited rather than assumed for other +frontier policies. The objective is the *expected* net return +`c_i - q_i * L`; the headline realized return is the post-hoc accounting of the same funded set on observed defaults (a funded loan earns `c_i * a_i` if it survives and loses `L * a_i` if it defaults). Separating the optimized expectation from the realized accounting is @@ -367,16 +389,31 @@ operational filters and caps live in the frozen policy configuration; the body displays the core statistical-to-decision contract because that is the reusable CRPTO pattern. -Two quantities are kept separate throughout the paper. The lowercase $\gamma$ -is a policy parameter controlling how the optimizer blends uncertainty in the -portfolio rule. $\Gamma_{\mathrm{CP}}$, by contrast, is a portfolio-level conformal metric -computed after the funded set is chosen: it is the allocation-weighted gap -between the conformal upper endpoint and calibrated PD, with clipping at one -on the PD scale. In committee language, $\Gamma_{\mathrm{CP}}$ is the realized conformal -robustness premium paid by the funded set. The selected body point has -$\Gamma_{\mathrm{CP}}(\alpha = 0.01) = 0.162616$, weighted miscoverage -$V(\alpha = 0.01) = 0.035350$, and zero exact violation at $\alpha = 0.01$. This -distinction is small typographically but central for auditability. +The lowercase $\gamma$ is a policy parameter; the uppercase quantities are +post-allocation funded-set metrics. With weights $w_i$ proportional to funded +exposure, define + +$$ +\begin{aligned} +\Gamma_{\mathrm{CP}}(\alpha) + &= \sum_i w_i\bigl(u_i(\alpha)-\hat p_i\bigr),\\ +\Gamma_{\mathrm{int}}(\alpha) + &= \sum_i w_i\bigl(q_i(\alpha;\theta)-\hat p_i\bigr),\\ +\Gamma_{\mathrm{res}}(\alpha) + &= \sum_i w_i\bigl(u_i(\alpha)-q_i(\alpha;\theta)\bigr). +\end{aligned} +$$ + +Thus $\Gamma_{\mathrm{CP}}=\Gamma_{\mathrm{int}}+\Gamma_{\mathrm{res}}$: +the first component is uncertainty internalized by the decision score and the +second is the residual needed to recover the endpoint budget. At +$\alpha=0.01$, the selected point has +$\Gamma_{\mathrm{CP}}=0.162616$, $\Gamma_{\mathrm{int}}=0.089032$, +$\Gamma_{\mathrm{res}}=0.073584$, and weighted miscoverage $V=0.035350$. +Its realized weighted default rate is also `0.035350`, below $\tau=0.1715$, +so its realized risk-tolerance excess is zero. This terminology matters: +zero excess is an empirical audit result, not a violation metric for the +deterministic identity below. # Theory @@ -398,14 +435,18 @@ adaptively selected subportfolio rather than a fresh population. What the paper certifies is therefore the exact accounting together with the safety level $V \leq \sqrt{\alpha}$ that Markov delivers, not a claim that the funded set attains nominal $\alpha$-coverage or that post-selection evaluation creates a -stronger conformal guarantee. +stronger conformal guarantee. Recent work on valid selection among conformal +sets reinforces this boundary: selecting the most attractive set or policy after +seeing multiple valid candidates is itself a statistical operation that needs +its own protocol [@hegazy2025valid_selection_conformal_sets]. | Claim component | CRPTO evidence | Boundary | |---|---|---| -| Deterministic portfolio identity | Exact funded-set audit computes $V$, $\Gamma_{\mathrm{CP}}$, and violation after allocation. | Does not require a new statistical guarantee. | +| Deterministic portfolio identity | Exact funded-set audit computes $B_u$, $V$, and the conformal-premium decomposition after allocation. | Does not require a new statistical guarantee. | | Split/Mondrian validity | OOT coverage, minimum group coverage, and temporal diagnostics. | Not exact conditional coverage for every borrower profile. | | Weighted funded-set validity | Exact alpha-safe certificate plus A23 weighted/multi-distribution stress evidence. | Assumption for the theorem; empirical audit after frozen selection. | | Post-selection robustness | Nested search chain plus a declared finite-grid frontier with 50,010 deduplicated semantic policies and 27,508 all-alpha above-floor policies. | Strong audit evidence, not a prospective live-selection guarantee or a continuous-region theorem. | +| Formal selected-set validity | A future nested or stability-based selection protocol could target this directly [@hegazy2025valid_selection_conformal_sets]. | Not claimed by the current frozen frontier. | : Assumption-to-evidence map for the CRPTO bound. @@ -431,6 +472,8 @@ and leaves online/live control for a new protocol. | Marginal split conformal | Population-level coverage under exchangeability. | Core interval guarantee. | | Mondrian/group conformal | Coverage within declared partitions. | Used for score-decile and grade audits. | | Weighted/localized coverage | Coverage under weights, local neighborhoods, or selected groups. | Explicit theorem assumption plus A23 diagnostic evidence. | +| Non-exchangeable conformal risk control | Loss control under weighted relevance or time/source shift [@farinhas2024nonexchangeable_crc]. | Future recalibration path; A23--A24 are read-only diagnostics. | +| Post-selection conformal validity | Coverage after selecting among multiple valid sets or policies [@hegazy2025valid_selection_conformal_sets]. | Future protocol; current frontier is an exact finite-grid audit. | | Multi-distribution coverage | Robustness across multiple source distributions. | Read-only stress evidence, not recalibration. | | Online/adaptive coverage | Sequential alpha updates under live drift. | A24 replay only; not a live deployment claim. | @@ -487,24 +530,25 @@ nonnegative variable $V(\alpha)$ with $E[V(\alpha)] \leq \alpha$ [@ghosh2002], combined with (i). The full proof is in Online Supplement Appendix A. $\square$ **The optimizer's cap versus the endpoint budget.** The robust layer -does not constrain $B_u(\alpha)$ directly; it caps the $\gamma$-blended PD, -$\sum_i w_i \tilde p_i(\alpha,\gamma) \leq \tau$, with -$\tilde p_i = \hat p_i + \gamma(u_i(\alpha) - \hat p_i)$ and $\gamma \in [0,1]$. -Because $\Gamma_{\mathrm{CP}}(\alpha) = \sum_i w_i(u_i(\alpha) - \hat p_i)$, the -endpoint budget decomposes exactly as +constrains the policy-specific effective score, not $B_u(\alpha)$ directly: +$\sum_i w_i q_i(\alpha;\theta)\leq\tau+s$, where $s\geq0$ is any recorded +solver cap slack. The policy-aware decomposition gives $$ -B_u(\alpha) = \sum_i w_i \tilde p_i(\alpha,\gamma) + (1-\gamma)\,\Gamma_{\mathrm{CP}}(\alpha) - \;\leq\; \tau + (1-\gamma)\,\Gamma_{\mathrm{CP}}(\alpha), +B_u(\alpha) += \sum_i w_i q_i(\alpha;\theta)+\Gamma_{\mathrm{res}}(\alpha) +\;\leq\;\tau+s+\Gamma_{\mathrm{res}}(\alpha). $$ -with equality when the risk cap binds. The term -$(1-\gamma)\,\Gamma_{\mathrm{CP}}(\alpha)$ is the conformal robustness premium the -optimizer leaves un-internalized at $\gamma < 1$. For the selected body point -($\tau = 0.1715$, $\gamma = 0.5475$, binding cap), -$B_u(0.01) \leq 0.1715 + 0.4525\,(0.162616) = 0.245084$, so the deterministic -accounting bound reads -$\sum_i w_i Y_i \leq 0.245084 + V(0.01) = 0.280434$. The exact audit reports -zero deterministic violation; the conservative Markov cap for the same point is -`0.345084`. +For a pure linear blend only, +$\Gamma_{\mathrm{res}}=(1-\gamma)\Gamma_{\mathrm{CP}}$. The selected capped +policy has no active row-level cap on its funded set, its effective-score cap +binds with $s=0$, and therefore the same numerical shortcut happens to hold: +$B_u(0.01)=0.1715+0.073584=0.245084$. The deterministic identity gives +$\sum_i w_iY_i\leq0.245084+V(0.01)=0.280434$, while the observed left-hand side +is `0.035350`. The exact Markov loss threshold is +$T_{0.01}=B_u(0.01)+\sqrt{0.01}=0.345084$; under Assumption 1, +$P(\sum_iw_iY_i\geq T_{0.01})\leq0.10$. It is a probabilistic event threshold, +not a deterministic risk cap. This policy-aware form is essential for capped +and tail-focused frontier points, where the linear shortcut need not hold. **Remark 1 (why $t = \sqrt{\alpha}$, and why Markov).** The choice $t = \sqrt{\alpha}$ is made for interpretability, not optimality: @@ -589,12 +633,12 @@ data/code-disclosure policy. The body-supplement split is fixed before submission. The body keeps the CRPTO pipeline, the alpha-to-portfolio link, the finite-grid frontier, and the core metrics, plus the compact regret-auditability frontier. The online -supplement carries A3--A39, the conformal finalist ablation, funded-set loan +supplement carries A3--A40, the conformal finalist ablation, funded-set loan audit, tail-risk diagnostics, satisficing margins, dependence diagnostics, the CVaR/OCE tail-constrained re-optimization (A22), the multi-distribution (A23) and online ACI-stability (A24) diagnostics, the external economic replication tables (A25--A34), the selected-policy frontier and funded-set audits -(A35--A39), MRM/fairness material, and reproduction commands. This keeps +(A35--A40), MRM/fairness material, and reproduction commands. This keeps the IJDS body focused while preserving the audit trail that reviewers need. ## Multi-Dataset External Replication Protocol @@ -622,7 +666,8 @@ address whether the method survives two materially different credit products. The results section is ordered around the reviewer decision object: first the selected-policy certificate, then the A35 finite-grid frontier that prevents a -singleton reading, and finally the external recipe-transfer checks. The core +singleton reading, the A40 matched point-PD baseline, and finally the external +recipe-transfer checks. The core metric table summarizes the paper-facing metrics. The calibrated PD layer is not sold as a leaderboard model: AUC `0.7139` is sufficient only because the downstream decision consumes calibrated probabilities, not rankings @@ -631,8 +676,8 @@ as discrimination. The conformal layer over-covers marginally at the reported levels (90% coverage `0.9297`, 95% coverage `0.9664` for the conformal winner). The portfolio layer then turns this uncertainty into an exact finite-grid return-bound frontier. The selected policy passes the -$V \leq \sqrt{\alpha}$ certificate at the tightest reported level while keeping -zero deterministic violation. +$V \leq \sqrt{\alpha}$ certificate at the tightest reported level and has zero +realized risk-tolerance excess. | Layer | Metric | Value | |---|---:|---:| @@ -643,37 +688,42 @@ zero deterministic violation. | Conformal | Coverage 95% | `0.9664` | | Conformal | Minimum group coverage 90% | `0.9190` | | Portfolio | Body-point robust return | `$184,832.48` | +| Portfolio | Weighted realized default rate | `0.035350` | | Portfolio | $V(\alpha = 0.01)$ | `0.035350` | | Portfolio | $\Gamma_{\mathrm{CP}}(\alpha = 0.01)$ | `0.162616` | -| Portfolio | Markov cap at $\alpha = 0.01$ | `0.345084` | -| Portfolio | Exact alpha violation | `0.0` | +| Portfolio | $\Gamma_{\mathrm{res}}(\alpha = 0.01)$ | `0.073584` | +| Portfolio | Endpoint budget $B_u(\alpha = 0.01)$ | `0.245084` | +| Portfolio | Exact Markov loss threshold | `0.345084` | +| Portfolio | Realized risk-tolerance excess | `0.0` | | Portfolio | Declared alpha-grid pass | `8/8` | : Frozen paper-facing metrics by layer. The exact certificate is an accounting claim. Here "exact" means the quantities are computed directly on the frozen OOT funded set rather than approximated by a -surrogate table or visual proxy, and the deterministic part requires no -distributional assumption. The certificate's `pass` is the Markov safety check -$V(\alpha) \leq \sqrt{\alpha}$ together with zero deterministic violation -($\sum_i w_i Y_i \leq B_u(\alpha)$); it is *not* a claim of nominal -$\alpha$-coverage, which the selected funded set does not attain -($V = 0.035350 > \alpha = 0.01$). +surrogate table or visual proxy, and the deterministic identity requires no +distributional assumption. The declared empirical `pass` combines +$V(\alpha)\leq\sqrt{\alpha}$ with realized risk-tolerance excess no larger than +$\alpha$; for the selected point that excess is zero. This screen is *not* a +claim of nominal $\alpha$-coverage, which the selected funded set does not attain +($V=0.035350>\alpha=0.01$), and the excess criterion is not presented as a new +probabilistic theorem. -| $\alpha$ | $\Gamma_{\mathrm{CP}}$ | $V(\alpha)$ | $\sqrt{\alpha}$ | Markov cap | violation | pass | -|---:|---:|---:|---:|---:|---:|:---:| -| `0.01` | `0.162616` | `0.035350` | `0.10000` | `0.345084` | `0.00000` | yes | +| $\alpha$ | $\Gamma_{\mathrm{CP}}$ | $\Gamma_{\mathrm{res}}$ | $V(\alpha)$ | $B_u$ | Markov threshold | Risk excess | Pass | +|---:|---:|---:|---:|---:|---:|---:|:---:| +| `0.01` | `0.162616` | `0.073584` | `0.035350` | `0.245084` | `0.345084` | `0.00000` | yes | -: Exact certificate for the selected body point. `pass` denotes -$V \leq \sqrt{\alpha}$ with zero deterministic violation, not nominal -$\alpha$-coverage. +: Exact certificate for the selected body point. The Markov column is the exact +event threshold $B_u+\sqrt{\alpha}$, not a deterministic cap. This makes $\Gamma_{\mathrm{CP}}$ more than a diagnostic line item. It is the amount of conformal robustness the optimizer accepts in order to keep the funded set inside the $\sqrt{\alpha}$-safe region. A credit reviewer can therefore read -$\Gamma_{\mathrm{CP}} = 0.162616$ as the conformal premium carried by the -selected body point, $V = 0.035350$ as the realized weighted noncoverage audit, -and the Markov cap `0.345084` as the conservative theorem-facing risk cap. +$\Gamma_{\mathrm{CP}}=0.162616$ as the total conformal premium carried by the +selected body point, $\Gamma_{\mathrm{res}}=0.073584$ as the part not internalized +by its decision score, $V=0.035350$ as the realized weighted noncoverage audit, +and `0.345084` as the loss level whose exceedance probability is bounded by +`0.10` under Assumption 1. The central empirical object is now a return-bound frontier rather than a single winner. The consolidated frontier surface deduplicates 51,678 raw rows into @@ -694,31 +744,61 @@ interval) was regenerated when the body point moved from the floor to that every eligible frontier policy must beat, which is why the frontier is reported with the floor surplus rather than as a replacement champion. -| Policy role | Source | Realized return | $V(0.01)$ | $\Gamma_{\mathrm{CP}}$ | Markov cap | Pass | +| Policy role | Source | Realized return | $V(0.01)$ | $\Gamma_{\mathrm{CP}}$ | Markov threshold | Pass | |---|---|---:|---:|---:|---:|:---:| -| Minimum Markov-cap endpoint | terminal | `$170,467.27` | `0.031875` | `0.095719` | `0.273036` | `8/8` | -| Low-cap balanced endpoint | terminal | `$171,006.20` | `0.031875` | `0.097190` | `0.274789` | `8/8` | -| Highest return under cap <= `0.30` | terminal | `$173,314.04` | `0.035875` | `0.115400` | `0.294580` | `8/8` | -| Highest return under cap <= `0.345` | micro-ext | `$184,800.41` | `0.035350` | `0.162562` | `0.344996` | `8/8` | +| Minimum Markov-threshold endpoint | terminal | `$170,467.27` | `0.031875` | `0.095719` | `0.273036` | `8/8` | +| Low-threshold balanced endpoint | terminal | `$171,006.20` | `0.031875` | `0.097190` | `0.274789` | `8/8` | +| Highest return under threshold <= `0.30` | bound-closure | `$174,552.51` | `0.035875` | `0.120988` | `0.299997` | `8/8` | +| Highest return under threshold <= `0.345` | micro-ext | `$184,800.41` | `0.035350` | `0.162562` | `0.344997` | `8/8` | | Body/default balanced point | micro-ext | `$184,832.48` | `0.035350` | `0.162616` | `0.345084` | `8/8` | -| Highest return under cap <= `0.36` | micro-ext | `$186,050.73` | `0.037750` | `0.174600` | `0.358685` | `8/8` | -| Max-return economic endpoint | micro-ext | `$223,458.14` | `0.069575` | `0.457438` | `0.510753` | `8/8` | +| Highest return under threshold <= `0.36` | micro-ext | `$186,050.73` | `0.037750` | `0.174600` | `0.358685` | `8/8` | +| Max-return economic endpoint | micro-ext | `$223,458.14` | `0.069575` | `0.457438` | `0.697056` | `8/8` | -: Pool93 finite-grid return-bound frontier. `terminal` rows come from the -conservative endpoint sweep; `micro-ext` rows come from the body-cap local -extension around the selected point. All rows are finite-grid points, not -continuous optima. +: Pool93 finite-grid return-bound frontier. Each threshold is computed from the +exact funded-set endpoint budget. `terminal` rows come from the conservative +endpoint sweep; `bound-closure` and `micro-ext` denote frozen local grids. All +rows are finite-grid points, not continuous optima. The table gives the manuscript its decision geometry. The body/default point is not the highest-return point and not the tightest-bound point; it is the balanced point selected by the declared return-bound lens. The neighboring strict `<= 0.345` row is reported separately because it is a different finite -policy: it earns `$184,800.41` with Markov cap `0.344996`, whereas the -body/default row earns `$184,832.48` with Markov cap `0.345084`. The endpoint at -Markov cap `0.273036` shows how conservative the certified frontier can become -while preserving the return floor, and the `$223.5K` endpoint shows the economic -return available when the committee accepts a looser cap. The supplement reports -the full frontier table and traceability details. +policy: it earns `$184,800.41` at threshold `0.344997`, whereas the body/default +row earns `$184,832.48` at threshold `0.345084`. The endpoint at `0.273036` +shows how conservative the certified frontier can become while preserving the +return floor, and the `$223.5K` endpoint shows the return available when the +committee accepts a threshold of `0.697056`. The latter policy is tail-focused; +its residual premium cannot be recovered with the linear +$(1-\gamma)\Gamma_{\mathrm{CP}}$ shortcut. The supplement reports the full +policy-aware frontier and traceability details. + +## Matched Point-PD Baseline + +To isolate what the conformal decision layer buys, we solve a matched two-stage +LP on the same 276,869 candidates with the same `$1M` budget, concentration cap, +$\tau=0.1715$, LGD, solver, and two-stage objective. The only change is that the +baseline uses calibrated point PD in both its objective and risk constraint. +Neither optimizer sees OOT outcomes; defaults enter only in the frozen post-hoc +audit. + +| Policy | Realized return | Funded | Weighted default / $V$ | $\Gamma_{\mathrm{CP}}$ | $B_u$ | Markov threshold | +|---|---:|---:|---:|---:|---:|---:| +| Point-PD two-stage LP | `$196,369.14` | `225` | `0.118400` | `0.526736` | `0.680579` | `0.780579` | +| Selected CRPTO | `$184,832.48` | `314` | `0.035350` | `0.162616` | `0.245084` | `0.345084` | + +: Matched point-PD baseline on the frozen Lending Club OOT panel. Weighted +default and $V$ coincide in these two funded sets because each observed default +lies above its conformal upper endpoint. Full fields are in A40. + +CRPTO gives up `$11,536.66`, or `5.875%`, of the baseline's realized return. In +exchange, the weighted default rate and miscoverage fall by `0.08305` (8.305 +percentage points), and the exact Markov loss threshold falls by `0.435495` +(43.55 percentage points). Both allocations have zero realized +risk-tolerance excess because their default rates remain below $\tau$, but the +point-PD allocation fails the tightest Markov safety screen +($V=0.1184>\sqrt{0.01}=0.10$). This is the paper's measured price of robustness +on Lending Club: a return--auditability trade-off under one frozen OOT design, +not causal evidence or universal dominance. The funded-set under-coverage remains structural rather than a calibration-draw effect. With $n_{\mathrm{cal}} = 237{,}584$ calibration loans, the @@ -726,8 +806,9 @@ split-conformal conditional-coverage result makes marginal coverage highly stable around the nominal level [@vovk2005; @angelopoulos2023]. The residual funded-set $V$ is a test-side, portfolio-selection quantity. That is why the paper reads the safety level at $\sqrt{\alpha}$ under Assumption 1, and why the -frontier reports $V$, $\Gamma_{\mathrm{CP}}$, endpoint budget, Markov cap, and -return together instead of promoting a standalone coverage number. +frontier reports $V$, $\Gamma_{\mathrm{CP}}$, residual premium, endpoint budget, +exact Markov threshold, and return together instead of promoting a standalone +coverage number. The compact reviewer checks below summarize the body-level defense. The supplement expands the same structure into traceability and guardrail references. @@ -736,6 +817,7 @@ supplement expands the same structure into traceability and guardrail references |---|---|---| | "This is only a classifier." | The claim is decision auditability, not AUC leadership. | Exact funded-set certificate and A35 frontier. | | "CP + RO already exists." | CRPTO instantiates the idea for frozen credit PD models, funded-set governance, and Lending Club payoffs. | Closest-work boundary and bound claim stack. | +| "The robust policy has no matched baseline." | Holding the candidate set and operating constraints fixed, A40 quantifies a 5.875% return cost against 8.305 percentage points less weighted default/miscoverage. | Matched point-PD table and A40 audit. | | "Adaptive selection breaks coverage." | The theorem states weighted funded-set validity as an assumption and then audits the frozen selection exactly. | Assumption map, validity ladder, and A23 diagnostics. | | "The selected policy is cherry-picked." | The selected point comes from a consolidated finite frontier with 50,010 deduplicated semantic policies and 27,508 eligible all-alpha above-floor policies. | Frontier table and governance files. | @@ -772,38 +854,15 @@ as an unverified shortlist caveat. ![External CRPTO replications preserve the predeclared global conformal gates and produce positive robust LP value on two materially different credit products.](../reports/crpto/figures/crpto_fig22_external_replication.png){#fig-external-replication width="94%" fig-alt="Two-panel external replication figure. The left panel shows 90 percent and alpha 0.01 coverage for Prosper and Freddie FM48 above target lines; the right panel shows positive robust LP objective values and OOT candidate counts."} -The external layer also surfaces a result a single-dataset champion cannot show. -The signed price of robustness--using the same convention as the Lending Club -field, $(\text{nonrobust}-\text{robust})/\text{nonrobust}$--is a *positive* -premium under frozen application, and across the four frozen external -applications it is ordered by panel default rate (Table @tbl-price-of-robustness, -Figure @fig-price-scaling). Within Freddie, the high-default red segment pays -more than green; across datasets, Prosper's `30.92%` default panel pays the -largest premium. This is a pattern across two datasets -- three Freddie -default-window segments plus Prosper -- not a scaling law: it is consistent with -the mechanism (higher default risk widens the conformal intervals, so the robust -worst case discounts more return, and discrimination (AUC) does not order the -premium on its own), but four points cannot establish a general law. The -measured summary is intentionally modest: in these external applications, the -conformal robust layer is economically bounded under blind application, with a -single-digit to low-double-digit premium. The Lending Club claim is handled -separately through the exact return-bound frontier rather than through this -external price-scaling table. -That closes the external-replication claim at the right level: the recipe -transfers as an economic audit protocol, while the exact funded-set certificate -remains the Lending Club object. - -| Application | Panel default | Price of robustness | -|---|---:|---:| -| Freddie FM48 (green) | `0.58%` | `+1.00%` | -| Freddie FM48 (combined) | `1.45%` | `+1.09%` | -| Freddie FM48 (red) | `2.97%` | `+2.37%` | -| Prosper final-status | `30.92%` | `+9.46%` | - -: Price of robustness by external application, ordered by panel default rate, -from `crpto_tableA34_price_of_robustness_cross_dataset.csv`. {#tbl-price-of-robustness} - -![Across the four frozen external applications, the price of robustness is ordered by panel default risk.](../reports/crpto/figures/crpto_fig25_price_of_robustness_scaling.png){#fig-price-scaling width="78%" fig-alt="Line chart on a log-scale default-rate axis: the price of robustness rises from +1.00 percent to +9.46 percent across external applications."} +The external layer adds a secondary economic pattern. Across Prosper and three +Freddie default-window applications, the signed robust premium ranges from +`+1.00%` to `+9.46%` and is ordered by panel default rate. Four applications +cannot establish a scaling law, so A34 and its companion figure remain in the +supplement as mechanism-consistent diagnostics rather than a body claim. The +matched Lending Club comparison is instead A40 above, whose point-PD baseline +holds the candidate universe and operating constraints fixed. The external +recipe therefore transfers as an economic audit protocol, while the exact +funded-set certificate remains the Lending Club object. # Robustness and Comparators @@ -820,7 +879,7 @@ The second concern is whether conformal uncertainty is doing decision work or only adding conservative decoration. The answer is visible in the portfolio frontier. Policies are evaluated by return, exact alpha pass/fail, weighted miscoverage, and $\Gamma_{\mathrm{CP}}$; the promoted point is the body/default -balanced policy on that finite-grid frontier, while the strict `<= 0.345` cap +balanced policy on that finite-grid frontier, while the strict `<= 0.345` threshold policy is reported separately. This differs from a workflow where conformal intervals are plotted after the optimizer has already chosen a point-PD allocation. @@ -886,7 +945,7 @@ normalized decision-loss scale rather than on the `$1M` funded portfolio. That experiment is not the same object as the funded-set economics: it isolates training-time decision quality, so SPO+ is the low-regret method by construction because it is trained to minimize exactly that loss. The -favorable price of robustness reported above and the higher CRPTO regret here +external price-of-robustness diagnostic reported in the supplement and the higher CRPTO regret here are therefore not in tension; they are two different measurements (a real `$1M` funded set versus a synthetic regret benchmark). The right-hand columns report what the credit decision actually delivers: only CRPTO produces a @@ -919,7 +978,7 @@ Lending Club funded set. Two reviewer questions deserve a body-level answer rather than an appendix-only one: what does the decision give up on the tail, and does its coverage hold once the evaluation is sliced by grade? The return-bound frontier answers the first -question directly: lower Markov caps are available at lower return, while the +question directly: lower Markov loss thresholds are available at lower return, while the selected policy sits on the declared frontier. The supplement then checks the funded-grade mix, selected-allocation tail repricing, dependence sensitivity, and fixed-allocation bootstrap interval. These rows explain the selected @@ -927,7 +986,7 @@ policy's risk profile; they do not add a hidden CVaR/OCE or bootstrap selector. | Reviewer question | Body answer | Boundary | |---|---|---| -| Is the point only high-return? | The frontier shows safer lower-return choices and higher-return looser-cap choices. | Finite grid, not a continuous optimum. | +| Is the point only high-return? | The frontier shows safer lower-return choices and higher-return looser-threshold choices. | Finite grid, not a continuous optimum. | | What loans does it fund? | The supplement reports funded exposure by grade. | Business mix, not protected-class fairness certification. | | What happens in the tail? | The selected allocation is repriced under LGD and tail summaries. | Risk profile only; tail risk is not the selector. | | Does dependence change the bound? | Cluster sensitivity recomputes tighter assumptions. | Sensitivity only; Markov remains the body theorem. | @@ -942,20 +1001,22 @@ the `0.90` target. Supplement A23 reports marginal coverage `0.9293` on its multi-distribution evaluation slice, while the promoted interval summary in Table 3 reports `0.9297`; these are distinct cuts through the same intervals. No grade falls below target, so the conservative marginal -coverage is not hiding a -failing segment; the thinnest grade$\times$vintage cells are where a future -group-weighted or multi-distribution recalibration would matter, and we mark that -as future work rather than a present guarantee. +coverage is not hiding a failing segment; the thinnest grade$\times$vintage +cells identify where a group-weighted or multi-distribution recalibration would +require a separately tagged protocol, so they are not promoted as a present +guarantee. ## Managerial Implication For a credit-risk committee, CRPTO turns a model into a decision conversation. The committee can pick a risk cap $\tau$, inspect how a policy -parameter $\gamma$ changes the funded set, read $\Gamma_{\mathrm{CP}}$ as the conformal -robustness premium paid by that funded set, and compare $V(\alpha)$ with the -stated bound tolerance. The method therefore supports a practical question: -whether the realized economic return justifies the signed price of robustness, -which on this evaluation is favorable rather than a cost. +parameter $\gamma$ changes the funded set, separate the total conformal premium +into internalized and residual components, and compare $V(\alpha)$ with the +stated bound tolerance. On this evaluation the matched choice is concrete: +accept `5.875%` less realized return than the point-PD LP in exchange for `8.305` +percentage points less weighted default/miscoverage and a `43.55` percentage +point lower Markov loss threshold. The method therefore supports a practical +trade-off rather than promising a free robustness premium. If the committee wants lower regret, the SPO+ corner is visible; if it wants stronger validity language, the validity ladder states the new calibration protocol that would be required. That separation is the managerial value of the @@ -971,8 +1032,9 @@ tail-risk diagnostic audit, A21 cluster-bound tightening audit, A22 CVaR/OCE tail-constrained re-optimization, and A23--A24 multi-distribution and online (ACI) conformal-stability diagnostics, plus A25--A34 external economic replication, exhaustiveness, interval, subperiod, and sensitivity audits on -Prosper and Freddie/Mendeley, and A35--A39 selected-policy frontier, -composition, tail-risk, concentration, and bootstrap audits. +Prosper and Freddie/Mendeley, A35--A39 selected-policy frontier, composition, +tail-risk, concentration, and bootstrap audits, and A40 matched point-PD +comparison. # Reproducibility and Companion @@ -1017,13 +1079,21 @@ middle path: keep the predictive model auditable, quantify uncertainty with a finite-sample conformal layer, and make the optimizer pay attention to the upper end of plausible default risk. +The policy-aware decomposition is more than a notation change. A linear blend +can recover its residual endpoint premium from $\gamma$ alone, but capped and +tail-focused policies cannot. Computing $\Gamma_{\mathrm{res}}$ from the funded +rows makes every frontier point comparable on the same exact endpoint scale and +prevents an attractive tail policy from appearing safer merely because its +decision score is nonlinear. + The external replications also change how to read the price of robustness. Across the frozen external applications, the premium is ordered by panel default risk rather than by discrimination. That pattern reframes robustness as a panel-specific premium to be measured, not a fixed toll to be assumed. It also -tempers the contribution: the favorable Lending Club value reflects champion -selection, so the transferable claim is bounded measurement under blind -application, not a universal free lunch. +tempers the contribution: the matched Lending Club audit observes a `5.875%` +return cost, while the external applications report different premiums under +their frozen contracts. The transferable claim is therefore a reproducible way +to measure a return--risk trade-off, not a universal free lunch. The limits are equally important. CRPTO does not prove that any one public dataset is a universal proxy for modern credit origination, even after the @@ -1045,46 +1115,48 @@ an online deployment study: there are no new post-2020 Lending Club retail originations, no live monitoring loop, and no end-to-end utility-directed conformal learner replacing the frozen PD model. -The next research directions are clear and are now separated into paper -improvements versus new CRPTO protocols. The current paper includes the safe -journal-strengthening package: OCE/CVaR as a tail-risk audit, satisficing as +The manuscript is deliberately written as one IJDS paper rather than a bundle of +method variants. Adjacent methods enter only when they make the submitted +certificate easier to evaluate: OCE/CVaR as a tail-risk audit, satisficing as margin evidence, SPO+ as the low-regret corner of the regret-auditability -frontier, and dependence as a formal caveat. The larger scientific upgrades are -useful in the present manuscript only when they sharpen this boundary. Table -@tbl-upgrade-map states that boundary directly. +frontier, and dependence as a formal caveat. Table @tbl-upgrade-map states that +single-submission boundary directly. -| Upgrade path | What the current paper can claim | What would require a new result | +| Adjacent path | What this paper uses | Boundary for the submitted claim | |---|---|---| -| Tail-aware selection | A20--A22 and A37 show the tail trade-off and selected-allocation tail profile. | A predeclared CVaR/OCE selector with its own exact funded-set audit. | -| Prospective selection | Nested holdout and the finite declared grid reduce post-selection ambiguity. | A fully prospective search/evaluation split after all policy rules are frozen. | -| Multi-distribution or online validity | A23--A24 diagnose grade, distribution, and vintage stress on frozen intervals. | A new calibration protocol that targets group-weighted, multi-source, or live sequential validity. | -| Decision-focused conformal learning | A19 shows SPO+ as the low-regret comparator and CRPTO as the auditable corner. | An end-to-end learner that jointly optimizes decision loss and emits a conformal funded-set certificate. | +| Tail-aware selection | A20--A22 and A37 show the selected decision's tail profile and available return-tail trade-off. | The promoted selector remains the declared return-bound finite frontier. | +| Prospective selection | Nested holdout and the finite declared grid reduce post-selection ambiguity. | The paper does not claim a fully prospective selection/evaluation trial. | +| Multi-distribution or online validity | A23--A24 diagnose grade, distribution, and vintage stress on frozen intervals. | The conformal layer is not recalibrated for multi-source or live sequential validity. | +| Decision-focused conformal learning | A19 shows SPO+ as the low-regret comparator and CRPTO as the auditable corner. | The PD model remains frozen; no end-to-end learner is promoted. | -: Scientific upgrade map for the current CRPTO paper. {#tbl-upgrade-map} +: Single-submission boundary map for the current CRPTO paper. {#tbl-upgrade-map} Optimized OCE/CVaR objectives, online conformal methods, multi-distribution conformal validity, utility-directed or decision-theoretic conformal variants [@cortesgomez2025utility; @lekeufack2023cdt], causal variants, broader asset-class panels, and prospective multi-period origination studies are all -valuable. They become stronger paper claims only with new protocols, data, or -proofs. In the current submission, their role is to make the frozen CRPTO -contribution easier to locate: an auditable post-hoc predict-then-optimize -certificate, not a universal decision-learning framework. +valuable comparators. In this submission, their role is to make the frozen +CRPTO contribution easier to locate: an auditable post-hoc +predict-then-optimize certificate, not a universal decision-learning framework +and not a collection of additional promoted methods. # Conclusion Can finite predictive uncertainty change a funding decision in a way a reviewer can audit? CRPTO's answer is yes, provided the statistical boundary is stated and the decision record is frozen. On the Lending Club out-of-time panel the -selected policy earns `$184,832.48` on a `$1M` budget while passing -the exact empirical $\alpha = 0.01$ funded-set audit, and it lies on a declared +selected policy earns `$184,832.48` on a `$1M` budget while passing the exact +empirical $\alpha = 0.01$ funded-set audit, and it lies on a declared finite-grid return-bound frontier with 50,010 deduplicated semantic policies and -27,508 all-alpha above-floor policies rather than at a single lucky point. The external -Prosper and Freddie/Mendeley replications show that the recipe travels beyond -the Lending Club panel and expose an economically interpretable regularity: the -price of robustness is a bounded premium ordered by panel default risk across -the frozen external applications. The contribution is scoped as an auditable post-hoc decision certificate, -not a new end-to-end learner or a live-deployment study, and every reported -number is regenerable from frozen evidence. +27,508 all-alpha above-floor policies rather than at a single lucky point. A +matched point-PD LP shows the price explicitly: `5.875%` less realized return for +`8.305` percentage points less weighted default/miscoverage and a `43.55` +percentage point lower exact Markov loss threshold. The policy-aware residual +premium makes that certificate valid across the linear, capped, and tail-focused +frontier, while Prosper and Freddie/Mendeley show that the recipe can be audited +on other credit products. The contribution is one auditable post-hoc decision +certificate, not a new end-to-end learner, live-deployment study, or portfolio +of additional methods; every reported number is regenerable from frozen +evidence. # References diff --git a/paper/README.md b/paper/README.md index ee051e6..e23cf4f 100644 --- a/paper/README.md +++ b/paper/README.md @@ -34,16 +34,18 @@ the submission-shaped versions are written. IJDS disclosure form. - `submission/SCHOLARONE_FINAL_CHECKLIST.md`: final upload/proof checklist. -Selected P2/P3-inspired diagnostics are part of the current paper/journal pack: +Selected P2/P3-inspired diagnostics are part of this single IJDS submission +only when they defend the promoted decision certificate: regret-auditability, OCE/CVaR tail risk, robust satisficing margins, multi-distribution diagnostics, online replay diagnostics, the pool93 A35 finite-grid frontier, the A36 funded-set grade audit, the A37--A39 selected pool93 tail-risk, cluster-bound, and bootstrap audits, and external Prosper/Freddie economic -replications. Tail-risk row-level repricing is supplement evidence for the +replications. A40 is the matched point-PD baseline with candidates and operating +constraints fixed. Tail-risk row-level repricing is supplement evidence for the selected pool93 allocation, not a hidden promotion criterion. Prospective live online validation, causal variants, new multi-dataset protocols beyond the frozen Prosper/Freddie replications, production, and package tracks remain -future work. +outside the submitted claim. ## Render Commands diff --git a/paper/submission/CLAIM_AUDIT_MATRIX.md b/paper/submission/CLAIM_AUDIT_MATRIX.md index 9929e23..2c1e2e6 100644 --- a/paper/submission/CLAIM_AUDIT_MATRIX.md +++ b/paper/submission/CLAIM_AUDIT_MATRIX.md @@ -9,11 +9,12 @@ overclaiming. | CRPTO is a data-science decision method, not a classifier leaderboard. | Abstract, Introduction, Related Work. | Pipeline Figure 1, exact certificate table, funded-set audit. | Desk screen reads it as applied ML. | State the central object as an auditable conformal-robust decision certificate. | | The predictive input is frozen and calibrated. | Method, Results, Supplement Appendix E. | `models/pd_canonical.cbm`, `models/pd_canonical_calibrator.pkl`, Table 0 metrics, E3/E4 PD stability diagnostics. | Reviewer asks whether results depend on hidden retraining. | PD artifact is consumed, not re-searched, by paper renders; E3/E4 are non-promoted T1 diagnostics, not a new champion. | | The conformal layer is conservative on OOT data. | Method, Results, Supplement A23-A24. | `conformal_intervals_mondrian.parquet`, coverage metrics, group audits. | Reviewer expects conditional coverage. | Claim marginal/Mondrian coverage; stronger local/multi-distribution claims are future work. | -| The 90% intervals are useful despite large raw width. | Method, funded-set audit, Tables A35-A39. | Average width `0.7842`, Winkler `1.1107`, and pool93 body-point `Gamma_CP=0.162616` after allocation; A37--A39 profile tail risk, concentration, and fixed-allocation bootstrap uncertainty from the selected allocation. | Reviewer says intervals are too wide on the `[0,1]` PD scale. | Width is not promoted as a standalone utility metric; the decision layer uses upper-endpoint ordering, funded-set miscoverage, and the conformal premium inside an exact portfolio frontier. | -| The promoted Lending Club body point passes the alpha-grid funded-set audit. | Results, Supplement A35-A39. | Body point: return `$184,832.48`, `V=0.035350`, `Gamma_CP=0.162616`, Markov cap `0.345084`, zero violation, `8/8` alpha pass; A36 shows funded exposure by grade; A37 reports baseline LGD tail repricing; A38 shows cluster bounds remain looser than Markov; A39 reports fixed-allocation bootstrap return interval `$167,963.20`--`$198,650.47`. | "Exact" may be misunderstood as universal validity. | Exact means direct accounting on frozen funded-set outputs (Theorem 1(i)); statistical interpretation uses weighted funded-set validity, stated as Assumption 1 with full proof in Supplement Appendix A. A39 is an empirical fixed-allocation interval, not a conformal guarantee. | +| The 90% intervals are useful despite large raw width. | Method, funded-set audit, Tables A35-A40. | Average width `0.7842`, Winkler `1.1107`, body-point `Gamma_CP=0.162616`, and A40's matched reduction in weighted default/miscoverage from `0.118400` to `0.035350`; A37--A39 profile tail risk, concentration, and bootstrap uncertainty. | Reviewer says intervals are too wide on the `[0,1]` PD scale. | Width is not promoted alone; the decision layer is judged through exact endpoint budgets, funded-set miscoverage, and the measured return--risk trade-off. | +| The promoted Lending Club body point passes the alpha-grid funded-set audit. | Results, Supplement A35-A39. | Body point: return `$184,832.48`, `V=0.035350`, `Gamma_CP=0.162616`, `Gamma_res=0.073584`, endpoint `0.245084`, exact Markov threshold `0.345084`, zero realized risk-tolerance excess, `8/8` alpha pass; A36--A39 profile the allocation. | "Exact" may be misunderstood as universal validity. | Exact means direct accounting on frozen funded-set outputs (Theorem 1(i)); statistical interpretation uses weighted funded-set validity, stated as Assumption 1. The threshold is probabilistic, not a deterministic cap. | | The result is not a single lucky point. | Results, Supplement A35. | Consolidated pool93 frontier: `50,010` deduplicated semantic policies, `27,508` eligible all-alpha above-floor policies; terminal endpoint search: `37,068/37,068` all-alpha passers. | Reviewer asks whether one selected policy was cherry-picked. | The frontier is a declared finite policy-grid surface, not all possible continuous policy values and not a future-selection guarantee. | +| The conformal decision has a matched point-PD baseline. | Results, Supplement A40. | Same 276,869 candidates, `$1M` budget, concentration cap, `tau=0.1715`, LGD and solver. CRPTO costs `5.875%` realized return, reduces weighted default/miscoverage by `8.305` pp, and lowers the exact Markov threshold by `43.55` pp. | Reviewer asks whether the baseline is incomparable or labels entered optimization. | Only the effective PD semantics change; neither optimizer sees OOT labels. This is one frozen OOT trade-off, not causal or universal dominance evidence. | | Prosper/Freddie show transfer of the recipe. | Results, Supplement A25-A34. | External replication gate, candidate sensitivity, all-candidate LP exhaustiveness. | Reviewer reads them as new champion certificates. | They are frozen external economic replications and exhaustiveness audits only. | -| The price of robustness is economically interpretable. | Results, Discussion. | Lending Club price `-10.56%`; external premiums `+1.00%` to `+9.46%`, ordered by panel default rate across the four frozen external applications. | Sign convention, selected-vs-blind confusion, or over-reading as a statistical scaling law. | Lending Club is selected; external panels are blind frozen applications; describe the pattern as ordered in these applications, not as a general law. | +| The price of robustness is economically interpretable. | Results, Discussion. | A40 reports a matched Lending Club cost of `5.875%`; external premiums range from `+1.00%` to `+9.46%` across four frozen applications. | Sign convention, incomparable baselines, or over-reading four external cases as a scaling law. | Exclude the historical Lending Club `-10.56%` field; use A40 for Lending Club and describe the external ordering as a four-case diagnostic only. | | SPO+ is a comparator, not a replacement champion. | Robustness and Comparators. | A19/Fig. 15 committed regret-auditability artifact. | Reviewer says CRPTO has higher regret. | SPO+ optimizes synthetic regret; CRPTO emits funded-set risk controls. | | Tail-risk alternatives exist and the selected point has been repriced. | Tail Risk section, Supplement A20-A22 and A37-A39. | CVaR/OCE and tail-constrained challenger tables; pool93 body repricing has baseline LGD return `$184,832.48`, realized CVaR95 `0.276211`, decision-time CVaR95 `0.218140`, no cluster-bound threshold tighter than Markov, and fixed-allocation bootstrap diagnostics. | Reviewer asks why not select lowest-tail policy. | The body selector is the finite-grid return-bound point; tail-risk/bootstrap tables are documented trade-offs and selected-point diagnostics, not hidden promotion criteria. | | Fairness/MRM claims are limited. | Supplement Appendix D, Discussion. | Proxy/intersectional diagnostics and MRM scope. | Reviewer asks for statutory fair-lending proof. | Public data lack direct protected attributes; no legal certification claim. | @@ -25,7 +26,7 @@ overclaiming. | Objection | Short response | |---|---| | "CP + RO already exists." | CRPTO instantiates the bridge for frozen credit PD artifacts, funded-set economics, exact audit, and reproducible governance. | -| "CP + RO is a direct combination, not a theory contribution." | The paper's theory is intentionally modest: Theorem 1 separates deterministic funded-set accounting from the Markov step under weighted funded-set validity. The contribution is the decision-certificate bridge: frozen PD artifact -> conformal endpoint -> robust funded set -> exact post-decision audit with reproducible governance. | +| "CP + RO is a direct combination, not a theory contribution." | Theorem 1 separates deterministic funded-set accounting from the Markov step under weighted validity, while the policy-aware decomposition `Gamma_CP = Gamma_int + Gamma_res` makes exact endpoint accounting valid for linear, capped, and tail-focused policies. The contribution remains the auditable decision-certificate bridge, not a stronger conditional-coverage theorem. | | "Adaptive selection breaks conformal validity." | Correct concern, and exactly why the paper isolates it: Assumption 1 states weighted funded-set validity explicitly, Theorem 1 separates the deterministic identity from the Markov step, and the exact audit checks the selected frozen funded set after the fact. | | "This is one dataset." | Lending Club carries the certificate; Prosper/Freddie test transfer of the recipe without claiming new certificates. | | "The AUC is not high enough." | The paper is not a credit-scoring leaderboard; calibrated probabilities are inputs to an auditable decision. | diff --git a/paper/submission/CRPTO_ijds_submission.tex b/paper/submission/CRPTO_ijds_submission.tex index 7ff9bec..f469a88 100644 --- a/paper/submission/CRPTO_ijds_submission.tex +++ b/paper/submission/CRPTO_ijds_submission.tex @@ -75,27 +75,28 @@ } \ABSTRACT{% -Credit allocation is a data-science-for-decisions problem: calibrated default -probabilities matter only after they shape which loans are funded under a budget -and risk appetite. We introduce Conformal Robust Predict-Then-Optimize (CRPTO), -a post-hoc bridge that maps a frozen calibrated probability-of-default model -through Mondrian conformal intervals into robust portfolio constraints and an -empirical funded-set audit. On a 276{,}869-loan out-of-time Lending Club -evaluation, the selected policy earns \$184.8K on a \$1M budget while passing -the declared eight-level alpha grid ($V(0.01)=0.035350$, -$\Gamma_{\mathrm{CP}}=0.162616$, Markov cap $0.345084$, zero violation). The -consolidated finite frontier contains 50{,}010 deduplicated semantic policies, -of which 27{,}508 both pass all declared alpha levels and exceed the return -floor, making the return-bound trade-off visible rather than implicit. Frozen -Prosper and Freddie/Mendeley applications test recipe transfer and preserve the -predeclared global conformal gates with positive robust LP objectives. The -insight is that uncertainty should be reported as a decision frontier, not as a -post-hoc calibration table. The contribution is a conformal-robust -credit-portfolio decision certificate with a distribution-free Markov bound -under weighted funded-set validity: it connects real credit data, calibrated -predictive models, robust funding decisions, and a validation harness that -rebuilds the prediction-to-decision chain from frozen inputs while keeping the -statistical guarantee boundary explicit.% +Credit allocation is a data-science-for-decisions problem: default probabilities +matter only after they shape which loans are funded under a budget and risk +appetite. We introduce Conformal Robust Predict-Then-Optimize (CRPTO), a +post-hoc decision certificate that maps a frozen calibrated probability-of-default +model through Mondrian conformal intervals into robust portfolio constraints and +an empirical funded-set audit. A policy-aware decomposition separates the +conformal premium internalized by the optimizer from the residual premium needed +to recover the exact upper-endpoint budget. On a 276{,}869-loan out-of-time +Lending Club evaluation, the selected policy earns \$184.8K on a \$1M budget +while passing the declared eight-level alpha grid ($V(0.01)=0.035350$, +$\Gamma_{\mathrm{CP}}=0.162616$, exact Markov loss threshold $0.345084$, zero +realized risk-tolerance excess). Against a matched point-PD two-stage LP with the +same candidates, budget, concentration cap, and risk tolerance, CRPTO gives up +$5.87\%$ of realized return while reducing the weighted default rate by $8.305$ +percentage points and the loss threshold by $43.55$ percentage points. The +consolidated frontier contains 50{,}010 deduplicated policies, of which 27{,}508 +pass all declared alpha levels and exceed the return floor. Frozen Prosper and +Freddie/Mendeley applications preserve the predeclared global conformal gates +with positive robust LP objectives. CRPTO therefore makes predictive uncertainty +decision-useful as an auditable return--risk frontier, with a distribution-free +Markov bound under weighted funded-set validity and an explicit separation +between deterministic accounting and its statistical assumption.% } \KEYWORDS{conformal prediction; robust optimization; predict-then-optimize; @@ -106,14 +107,17 @@ % ===================================================================== \section{Introduction}\label{sec:intro} -Credit allocation is a predict-then-decide problem. A lender first estimates a -probability of default (PD), then chooses which loans to fund under a budget and -risk appetite. The modeling literature has become very good at the first step: -calibration, discrimination, and backtesting are now standard ingredients of -credit-risk model validation \citep{lessmann2015,chen2024creditrisk}. The second step is -less settled. Once a calibrated PD enters an optimizer, uncertainty is often -treated as a reporting diagnostic rather than as a constraint that can change the -funded set. +Credit allocation is a contextual optimization problem in credit form. A lender +first estimates a probability of default (PD), then chooses which loans to fund +under a budget and risk appetite. This places the paper in the broader +prescriptive-analytics literature on decision making under uncertainty +\citep{sadana2025contextual}. The credit-risk literature has become very good at +the first step: calibration, discrimination, and backtesting are now standard +ingredients of model validation \citep{lessmann2015,chen2024creditrisk}. Recent +credit-scoring work also evaluates uncertainty through economic metrics. The +second step is less settled. Once a calibrated PD enters an optimizer, +uncertainty is often treated as a reporting diagnostic rather than as a +constraint that can change the funded set. That separation is uncomfortable in auditable credit decisions. A portfolio policy can have a reasonable average PD and still concentrate probability mass in loans @@ -122,9 +126,11 @@ \section{Introduction}\label{sec:intro} in this paper is therefore not whether one can build a slightly better credit classifier. It is whether finite-sample predictive uncertainty can be carried into a robust portfolio decision in a way that is transparent enough for a reviewer to -audit. This has practical stakes. In a pre-registered randomized trial, -conformal prediction sets improved human decision making relative to fixed-size -sets with the same coverage \citep{cresswell2024}. CRPTO takes that +audit. This has practical stakes, but it is not automatic: conformal sets must be +tied to a downstream action and objective to become decision-useful. In a +pre-registered randomized trial, conformal prediction sets improved human +decision making relative to fixed-size sets with the same coverage +\citep{cresswell2024}. CRPTO takes that committee-facing idea into a credit portfolio setting, where the uncertainty summary must change a funding decision or it is just another report. @@ -142,7 +148,7 @@ \section{Introduction}\label{sec:intro} deduplicated semantic policies, of which 27{,}508 pass every declared alpha level and exceed the return floor. From that declared finite frontier, the selected policy is the body/default balanced point at the approximately -$0.345$ return-bound lens, with Markov cap $0.345084$; it is neither a continuous +$0.345$ return-bound lens, with exact Markov loss threshold $0.345084$; it is neither a continuous global optimum nor the economic endpoint. The selected point earns \$184.8K on a \$1M budget and passes the exact empirical funded-set audit at $\alpha=0.01$. The headline result is not a single lucky allocation. It is a @@ -157,30 +163,26 @@ \section{Introduction}\label{sec:intro} champions; they test whether the same PD-to-conformal-to-LP recipe remains economically usable on different credit products. -The paper makes five contributions. First, it gives a CRPTO construction for -credit portfolios: frozen calibrated PD, Mondrian conformal uncertainty, and -robust budgeted optimization as a post-hoc decision audit. Second, it proves a +The paper makes four contributions. First, it gives a CRPTO construction for +credit portfolios: frozen calibrated PD, Mondrian conformal uncertainty, robust +budgeted optimization, and an exact post-allocation audit. Second, it proves a distribution-free Markov bound under weighted funded-set validity (Theorem~\ref{thm:funded-set-bound}) -that splits realized portfolio loss into the conformal upper-endpoint budget -$B_u(\alpha)=\tau+(1-\gamma)\,\Gamma_{\mathrm{CP}}(\alpha)$ and the weighted -miscoverage $V(\alpha)$, with supplement propositions showing that Markov is -optimal under the stated assumption (A.1) and locating the cluster structure -that would tighten it (A.2). Third, it locates that construction relative to -data-driven robust optimization, P2P lending portfolio models, conformal credit -scoring, and decision-focused learning. Fourth, it provides an evidence-backed -empirical study where every table and figure is generated from frozen outputs -rather than manually transcribed summaries. Fifth, it uses a three-level -evidence ladder: the Lending Club funded set carries the certificate, Prosper -and Freddie/Mendeley test recipe transfer as external economic replications, -and the tail, dependence, online, and robustness appendices stay diagnostic. -The key claim is narrow: CRPTO maps frozen calibrated -PD models into a robust funded set, reports the portfolio-level conformal -premium $\Gamma_{\mathrm{CP}}$, and verifies exact alpha-safe weighted -miscoverage on the promoted Lending Club portfolio and its surrounding finite-grid -frontier. The same conformal and LP gates remain viable on two additional -credit datasets, while the paper keeps the boundary between paper revision and -new model-search evidence visible. +and introduces a policy-aware decomposition +$\Gamma_{\mathrm{CP}}=\Gamma_{\mathrm{int}}+\Gamma_{\mathrm{res}}$ that recovers +the exact upper-endpoint budget for linear, capped, and tail-focused policies. +Supplement propositions show that Markov is optimal under the stated first-moment +assumption (A.1) and locate the cluster structure that would tighten it (A.2). +Third, it reports the selected Lending Club decision as part of a declared +finite-grid return-bound frontier and compares it with a matched point-PD +allocation; all rows and exact alpha checks are generated from frozen evidence. +Fourth, it packages the result as a reproducible IJDS decision artifact, with +tables, figures, governance files, and claim-sync checks designed to keep the +statistical boundary visible. The key claim is narrow: CRPTO maps a frozen +calibrated PD model into a robust funded set, reports the portfolio-level +conformal premium and its internalized/residual components, and audits the +promoted Lending Club allocation and its finite-grid frontier exactly. Adjacent methods enter the manuscript only to locate and +stress-test this single claim, not to create additional acceptance criteria. Read as data science for decisions, the paper's four components are explicit: the data are a static Lending Club OOT panel plus Prosper/Freddie external stress @@ -210,15 +212,16 @@ \section{Related Work}\label{sec:related} variants are especially natural in credit because risk grades are already used as business and governance partitions \citep{bostrom2021,gibbs2024,zhou2024}. -The second foundation is robust optimization. Classical robust optimization frames -uncertainty as a set against which a decision must remain feasible, with the price -of robustness made visible as a design trade-off \citep{bertsimas2004}. Robust -portfolio selection makes that trade-off operational for allocation under -parameter uncertainty \citep{goldfarb2003robustportfolio}, whereas -distributionally robust optimization broadens the uncertainty object toward moment -or ambiguity sets \citep{delage2010dro}. Recent work connects conformal prediction -and robust optimization more directly by using conformal uncertainty sets in -downstream decisions \citep{johnstone2021,patel2024,sun2024ptc,hu2026crc}. That +The second foundation is robust optimization and contextual optimization. +Classical robust optimization frames uncertainty as a set against which a decision +must remain feasible, with the price of robustness made visible as a design +trade-off \citep{bertsimas2004}. Robust portfolio selection makes that trade-off +operational for allocation under parameter uncertainty +\citep{goldfarb2003robustportfolio}, whereas distributionally robust optimization +broadens the uncertainty object toward moment or ambiguity sets +\citep{delage2010dro}. Recent work connects conformal prediction and robust +optimization more directly by using conformal uncertainty sets in downstream +decisions \citep{johnstone2021,patel2024,sun2024ptc,hu2026crc}. That line certifies the uncertainty set \emph{before} the decision: coverage of the conformal region is the guarantee, and the downstream decision inherits it. CRPTO follows this line but audits the other side of the decision as well: @@ -250,7 +253,10 @@ \section{Related Work}\label{sec:related} credit-risk work shows how richer data structures such as firm graphs can improve rating prediction \citep{das2023creditgraph}, and cost-aware calibration work makes explicit why probability quality matters when predictions feed asymmetric -downstream decisions \citep{yang2025costaware}. IJDS decision papers also sharpen +downstream decisions \citep{yang2025costaware}. Recent EJOR credit-scoring work +similarly moves from discrimination to economic uncertainty, evaluating uncertainty +through profit and rejection objectives. +IJDS decision papers also sharpen the distinction between an accurate intermediate estimate and an effective automated decision \citep{fernandezloria2022causaldecision,fernandezloria2025observational}, while @@ -271,13 +277,14 @@ \section{Related Work}\label{sec:related} multi-distribution conformal validity, online conformal portfolio methods, end-to-end conformal risk training, and conformal satisficing \citep{bao2025croms,yang2026multidistribution,liu2026portfolio,yeh2025training,zhao2025robust} -motivates the journal-strengthening package in this paper. We use those ideas where +fixes the boundary around the single IJDS claim. We use those ideas where they can be evaluated from the frozen CRPTO evidence: OCE/CVaR \citep{rockafellar2000cvar,bental2007oce} appears as a tail-risk diagnostic, robust satisficing appears as committee-style margin evidence, and SPO+ motivates the regret-auditability frontier. The method-changing variants---optimized OCE/CVaR -objectives, online or multi-dataset protocols, causal layers, and hybrid -decision-focused training---remain future work rather than hidden acceptance +objectives, non-exchangeable recalibration, formal post-selection conformal-set +selection, online or multi-dataset protocols, causal layers, and hybrid +decision-focused training---are outside the submitted claim rather than hidden acceptance criteria. \FloatBarrier @@ -363,29 +370,35 @@ \subsection{Robust Portfolio Layer}\label{sec:method-lp} The decision variable $x_i$ is the allocation fraction for each eligible loan; $x_i a_i$ is the funded exposure. The optimizer maximizes expected net economic -return under a \$1M budget and policy constraints that cap portfolio risk after -replacing point PD estimates with conformal upper endpoints. The selected body -point has $\texttt{risk\_tolerance}=0.1715$, -$\texttt{policy\_mode}=\texttt{capped\_blended\_uncertainty}$, policy parameter +return under a \$1M budget and policy constraints that replace point PD with a +declared effective decision score $q_i(\alpha;\theta)$. Every frontier policy +satisfies $\hat p_i\le q_i(\alpha;\theta)\le u_i(\alpha)$; $\theta$ identifies a +linear, capped, or tail-focused blend. The selected body point has +$\texttt{risk\_tolerance}=0.1715$, +$\texttt{policy\_mode}=\texttt{capped\_blended\_uncertainty}$, $\gamma=0.5475$, and $\texttt{uncertainty\_aversion}=0.05$. Schematically, the robust decision layer solves \[ \begin{aligned} -\max_x\quad & \sum_i x_i a_i \left(c_i - \tilde p_i(\alpha,\gamma)\,L\right) \\ +\max_x\quad & \sum_i x_i a_i \left(c_i - q_i(\alpha;\theta)\,L\right) \\ \text{s.t.}\quad & \sum_i x_i a_i \le B,\\ -& \sum_i x_i a_i \tilde p_i(\alpha,\gamma) +& \sum_i x_i a_i q_i(\alpha;\theta) \le \tau \sum_i x_i a_i,\\ & 0 \le x_i \le \bar x_i, \end{aligned} \] where $a_i$ is exposure, $c_i$ is the loan coupon, $L$ is the loss-given-default -($L=0.45$ in the frozen evaluation), $\tau$ is the risk-tolerance cap, and +($L=0.45$ in the frozen evaluation), and $\tau$ is the risk-tolerance cap. The +linear policy member is \[ -\tilde p_i(\alpha,\gamma)=\hat p_i+\gamma\left(u_i(\alpha)-\hat p_i\right) +q_i(\alpha;\gamma)=\hat p_i+\gamma\left(u_i(\alpha)-\hat p_i\right). \] -on the PD scale, clipped to the feasible probability range. The objective is the -\emph{expected} net return $c_i-\tilde p_i L$; the headline realized return is +Capped and tail-focused members transform that score while remaining between +the point PD and upper endpoint. For the selected allocation, the cap is inactive +on all 314 funded rows, so its effective score equals the linear expression; this +is audited rather than assumed for other policies. The objective is the +\emph{expected} net return $c_i-q_iL$; the headline realized return is the post-hoc accounting of the same funded set on observed defaults (a funded loan earns $c_i a_i$ if it survives and loses $L a_i$ if it defaults). Separating the optimized expectation from the realized accounting is @@ -395,17 +408,21 @@ \subsection{Robust Portfolio Layer}\label{sec:method-lp} manuscript displays the core statistical-to-decision contract because that is the reusable CRPTO pattern. -Two quantities are kept separate throughout the paper. The lowercase $\gamma$ is a -policy parameter controlling how the optimizer blends uncertainty in the portfolio -rule. $\Gamma_{\mathrm{CP}}$, by contrast, is a portfolio-level conformal metric -computed after the funded set is chosen: +The lowercase $\gamma$ is a policy parameter; the uppercase quantities are +post-allocation funded-set metrics: \[ -\Gamma_{\mathrm{CP}}(\alpha)=\sum_i w_i\left(u_i(\alpha)-\hat p_i\right)_+. +\begin{aligned} +\Gamma_{\mathrm{CP}}(\alpha)&=\sum_iw_i(u_i-\hat p_i),\\ +\Gamma_{\mathrm{int}}(\alpha)&=\sum_iw_i(q_i-\hat p_i),\\ +\Gamma_{\mathrm{res}}(\alpha)&=\sum_iw_i(u_i-q_i). +\end{aligned} \] -The selected body point has - $\Gamma_{\mathrm{CP}}(\alpha=0.01)=0.162616$, weighted miscoverage - $V(\alpha=0.01)=0.035350$, and zero exact violation at $\alpha=0.01$. This distinction -is small typographically but central for auditability. +Thus $\Gamma_{\mathrm{CP}}=\Gamma_{\mathrm{int}}+\Gamma_{\mathrm{res}}$. At +$\alpha=0.01$, the selected point has $\Gamma_{\mathrm{CP}}=0.162616$, +$\Gamma_{\mathrm{int}}=0.089032$, $\Gamma_{\mathrm{res}}=0.073584$, and +$V=0.035350$. Its weighted realized default rate is also $0.035350$, below +$\tau=0.1715$, so realized risk-tolerance excess is zero. This is an empirical +audit result, not a violation metric for the deterministic identity below. % ===================================================================== \section{Theory}\label{sec:theory} @@ -427,7 +444,9 @@ \section{Theory}\label{sec:theory} What the paper certifies is therefore the exact accounting together with the safety level $V\le\sqrt{\alpha}$ that Markov delivers, not a claim that the funded set attains nominal $\alpha$-coverage or that post-selection evaluation creates a stronger -conformal guarantee. +conformal guarantee. This boundary also covers conformal-set selection: choosing +the most attractive set or policy after seeing multiple valid candidates is +itself a statistical operation that needs its own protocol. \begin{figure}[t] \centering @@ -489,23 +508,25 @@ \section{Theory}\label{sec:theory} \endproof \begin{remark}[The optimizer's cap versus the endpoint budget]\label{rem:endpoint-budget} -The robust layer does not constrain $B_u(\alpha)$ directly; it caps the -$\gamma$-blended PD, $\sum_i w_i \tilde p_i(\alpha,\gamma)\le\tau$, with -$\tilde p_i=\hat p_i+\gamma(u_i(\alpha)-\hat p_i)$ and $\gamma\in[0,1]$. Because -$\Gamma_{\mathrm{CP}}(\alpha)=\sum_i w_i(u_i(\alpha)-\hat p_i)$, the endpoint -budget decomposes exactly as +The robust layer constrains the policy-specific effective score, not +$B_u(\alpha)$ directly: $\sum_iw_iq_i(\alpha;\theta)\le\tau+s$, where $s\ge0$ +is recorded solver cap slack. The policy-aware decomposition gives \[ -B_u(\alpha)=\sum_i w_i \tilde p_i(\alpha,\gamma)+(1-\gamma)\,\Gamma_{\mathrm{CP}}(\alpha) - \;\le\; \tau+(1-\gamma)\,\Gamma_{\mathrm{CP}}(\alpha), +B_u(\alpha)=\sum_iw_iq_i(\alpha;\theta)+\Gamma_{\mathrm{res}}(\alpha) + \;\le\;\tau+s+\Gamma_{\mathrm{res}}(\alpha). \] -with equality when the cap binds. The term $(1-\gamma)\,\Gamma_{\mathrm{CP}}(\alpha)$ -is the conformal robustness premium the optimizer leaves un-internalized at -$\gamma<1$. For the selected body point ($\tau=0.1715$, $\gamma=0.5475$, binding cap), -$B_u(0.01)\le 0.1715+0.4525\,(0.162616)=0.245084$, so the deterministic -accounting bound reads -$\sum_i w_i Y_i\le 0.245084+V(0.01)=0.280434$. The exact audit reports zero -deterministic violation; the conservative Markov cap for the same point is -$0.345084$. +For a pure linear blend only, +$\Gamma_{\mathrm{res}}=(1-\gamma)\Gamma_{\mathrm{CP}}$. The selected capped +policy has no active row-level cap on its funded set, its effective-score cap +binds with $s=0$, and therefore $B_u(0.01)=0.1715+0.073584=0.245084$. The +deterministic identity gives +$\sum_iw_iY_i\le0.245084+V(0.01)=0.280434$, while the observed left-hand side is +$0.035350$. The exact Markov loss threshold is +$T_{0.01}=B_u(0.01)+\sqrt{0.01}=0.345084$; under +Assumption~\ref{asm:weighted-validity}, +$\mathbb{P}(\sum_iw_iY_i\ge T_{0.01})\le0.10$. It is a probabilistic event +threshold, not a deterministic risk cap. This policy-aware form is essential for +capped and tail-focused frontier points, where the linear shortcut need not hold. \end{remark} \begin{remark}[Why $t=\sqrt{\alpha}$, and why Markov]\label{rem:sqrt-alpha} @@ -601,12 +622,12 @@ \section{Experimental Design}\label{sec:design} The body-supplement split is fixed before submission. The body keeps the CRPTO pipeline, the alpha-to-portfolio link, the finite-grid frontier, and the core metrics, plus the compact regret-auditability frontier. The online supplement carries -A3--A39, the conformal finalist ablation, funded-set loan audit, tail-risk +A3--A40, the conformal finalist ablation, funded-set loan audit, tail-risk diagnostics, satisficing margins, dependence diagnostics, the CVaR/OCE tail-constrained re-optimization (A22), the multi-distribution (A23) and online ACI-stability (A24) diagnostics, the multi-dataset external economic replication tables (A25--A34), the selected-policy frontier and funded-set audits -(A35--A39), MRM/fairness material, and reproduction commands. This keeps the +(A35--A40), MRM/fairness material, and reproduction commands. This keeps the IJDS body focused while preserving the audit trail that reviewers need. \subsection{Multi-Dataset External Replication Protocol} @@ -635,13 +656,13 @@ \section{Results}\label{sec:results} levels (90\% coverage $0.9297$, 95\% coverage $0.9664$ for the conformal winner). The portfolio layer then turns this uncertainty into an exact finite-grid return-bound frontier. The selected policy passes the -$V\le\sqrt{\alpha}$ certificate at the tightest reported level while keeping zero -deterministic violation. +$V\le\sqrt{\alpha}$ certificate at the tightest reported level and has zero +realized risk-tolerance excess. -The results are ordered around three reviewer questions: what certificate the -selected policy carries, where that policy sits on the finite-grid frontier, and -whether the same recipe transfers to other credit panels without changing the -Lending Club champion. +The results are ordered around four reviewer questions: what certificate the +selected policy carries, where it sits on the finite-grid frontier, what it buys +relative to a matched point-PD LP, and whether the same recipe transfers to other +credit panels without changing the Lending Club champion. \begin{table}[t] \centering @@ -658,10 +679,13 @@ \section{Results}\label{sec:results} Conformal & Coverage 95\% & $0.9664$ \\ Conformal & Minimum group coverage 90\% & $0.9190$ \\ Portfolio & Body-point robust return & \$184{,}832.48 \\ + Portfolio & Weighted realized default & $0.035350$ \\ Portfolio & $V(\alpha=0.01)$ & $0.035350$ \\ Portfolio & $\Gamma_{\mathrm{CP}}(\alpha=0.01)$ & $0.162616$ \\ - Portfolio & Markov cap at $\alpha=0.01$ & $0.345084$ \\ - Portfolio & Exact alpha violation & $0.0$ \\ + Portfolio & $\Gamma_{\mathrm{res}}(\alpha=0.01)$ & $0.073584$ \\ + Portfolio & Endpoint budget $B_u(0.01)$ & $0.245084$ \\ + Portfolio & Exact Markov loss threshold & $0.345084$ \\ + Portfolio & Realized risk-tolerance excess & $0.0$ \\ Portfolio & Declared alpha-grid pass & $8/8$ \\ \bottomrule \end{tabular} @@ -669,31 +693,36 @@ \section{Results}\label{sec:results} The exact certificate is an accounting claim. Here ``exact'' means the quantities are computed directly on the frozen OOT funded set rather than approximated by a -surrogate table or visual proxy, and the deterministic part requires no distributional -assumption. The certificate's pass is the Markov safety check $V(\alpha)\le\sqrt{\alpha}$ -together with zero deterministic violation ($\sum_i w_i Y_i\le B_u(\alpha)$); it is -\emph{not} a claim of nominal $\alpha$-coverage, which the funded set does not attain -($V=0.035350>\alpha=0.01$). +surrogate table or visual proxy, and the deterministic identity requires no +distributional assumption. The declared empirical pass combines +$V(\alpha)\le\sqrt{\alpha}$ with realized risk-tolerance excess no larger than +$\alpha$; for the selected point that excess is zero. This screen is \emph{not} +a claim of nominal $\alpha$-coverage, which the funded set does not attain +($V=0.035350>\alpha=0.01$), and the excess criterion is not a new probabilistic +theorem. This makes $\Gamma_{\mathrm{CP}}$ more than a diagnostic line item. It is the -amount of conformal robustness the optimizer accepts in order to keep the funded -set inside the $\sqrt{\alpha}$-safe region. A credit reviewer can therefore read -$\Gamma_{\mathrm{CP}}=0.162616$ as the conformal premium carried by the selected -body point, $V=0.035350$ as the realized weighted noncoverage audit, and the -Markov cap $0.345084$ as the conservative theorem-facing risk cap. +amount of conformal robustness carried by the funded set. A reviewer can read +$\Gamma_{\mathrm{CP}}=0.162616$ as the total premium, +$\Gamma_{\mathrm{res}}=0.073584$ as the part not internalized by the decision +score, $V=0.035350$ as realized weighted noncoverage, and $0.345084$ as the loss +level whose exceedance probability is bounded by $0.10$ under +Assumption~\ref{asm:weighted-validity}. \begin{table}[t] \centering - \caption{Exact certificate for the promoted funded set. A pass denotes - $V\le\sqrt{\alpha}$ with zero deterministic violation, not nominal $\alpha$-coverage.} + \caption{Exact certificate for the promoted funded set. The Markov column is + the exact event threshold $B_u+\sqrt{\alpha}$, not a deterministic cap.} \label{tab:exact-certificate} - \begin{tabular}{rrrrrr} + \resizebox{\textwidth}{!}{% + \begin{tabular}{rrrrrrrr} \toprule - $\alpha$ & $\Gamma_{\mathrm{CP}}$ & $V(\alpha)$ & $\sqrt{\alpha}$ & Markov cap & Violation \\ + $\alpha$ & $\Gamma_{\mathrm{CP}}$ & $\Gamma_{\mathrm{res}}$ & $V(\alpha)$ & $B_u$ & Markov threshold & Risk excess & Pass \\ \midrule - $0.01$ & $0.162616$ & $0.035350$ & $0.10000$ & $0.345084$ & $0.00000$ \\ + $0.01$ & $0.162616$ & $0.073584$ & $0.035350$ & $0.245084$ & $0.345084$ & $0.00000$ & yes \\ \bottomrule \end{tabular} + }% \end{table} The central empirical object is now a return-bound frontier rather than a single @@ -717,24 +746,23 @@ \section{Results}\label{sec:results} \begin{table}[t] \centering - \caption{Pool93 finite-grid return-bound frontier. \texttt{terminal} rows come from - the conservative endpoint sweep; \texttt{micro-ext} rows come from the body-cap local - extension around the selected point. All rows are finite-grid points, not continuous - optima.} + \caption{Pool93 finite-grid return-bound frontier. Each threshold uses the exact + funded-set endpoint budget. \texttt{terminal}, \texttt{bound-closure}, and + \texttt{micro-ext} denote frozen finite grids, not continuous optima.} \label{tab:pool93-frontier} \resizebox{\textwidth}{!}{% \begin{tabular}{llrrrrr} \toprule Policy role & Source & Realized return & $V(0.01)$ & - $\Gamma_{\mathrm{CP}}$ & Markov cap & Pass \\ + $\Gamma_{\mathrm{CP}}$ & Markov threshold & Pass \\ \midrule - Minimum Markov-cap endpoint & terminal & \$170{,}467.27 & $0.031875$ & $0.095719$ & $0.273036$ & $8/8$ \\ - Low-cap balanced endpoint & terminal & \$171{,}006.20 & $0.031875$ & $0.097190$ & $0.274789$ & $8/8$ \\ - Highest return under cap $\le 0.30$ & terminal & \$173{,}314.04 & $0.035875$ & $0.115400$ & $0.294580$ & $8/8$ \\ - Highest return under cap $\le 0.345$ & micro-ext & \$184{,}800.41 & $0.035350$ & $0.162562$ & $0.344996$ & $8/8$ \\ + Minimum Markov-threshold endpoint & terminal & \$170{,}467.27 & $0.031875$ & $0.095719$ & $0.273036$ & $8/8$ \\ + Low-threshold balanced endpoint & terminal & \$171{,}006.20 & $0.031875$ & $0.097190$ & $0.274789$ & $8/8$ \\ + Highest return under threshold $\le 0.30$ & bound-closure & \$174{,}552.51 & $0.035875$ & $0.120988$ & $0.299997$ & $8/8$ \\ + Highest return under threshold $\le 0.345$ & micro-ext & \$184{,}800.41 & $0.035350$ & $0.162562$ & $0.344997$ & $8/8$ \\ Body/default balanced point & micro-ext & \$184{,}832.48 & $0.035350$ & $0.162616$ & $0.345084$ & $8/8$ \\ - Highest return under cap $\le 0.36$ & micro-ext & \$186{,}050.73 & $0.037750$ & $0.174600$ & $0.358685$ & $8/8$ \\ - Max-return economic endpoint & micro-ext & \$223{,}458.14 & $0.069575$ & $0.457438$ & $0.510753$ & $8/8$ \\ + Highest return under threshold $\le 0.36$ & micro-ext & \$186{,}050.73 & $0.037750$ & $0.174600$ & $0.358685$ & $8/8$ \\ + Max-return economic endpoint & micro-ext & \$223{,}458.14 & $0.069575$ & $0.457438$ & $0.697056$ & $8/8$ \\ \bottomrule \end{tabular} }% @@ -744,12 +772,47 @@ \section{Results}\label{sec:results} body/default point is not the highest-return point and not the tightest-bound point; it is the balanced point selected by the declared return-bound lens. The strict $\le 0.345$ row is reported separately because it is a different finite -policy: it earns \$184{,}800.41 with Markov cap $0.344996$, whereas the -body/default row earns \$184{,}832.48 with Markov cap $0.345084$. The endpoint -at Markov cap $0.273036$ shows how conservative the certified frontier can -become while preserving the return floor, and the \$223.5K endpoint shows the -economic return available when the committee accepts a looser cap. The supplement -reports the full frontier table and traceability details. +policy: it earns \$184{,}800.41 at threshold $0.344997$, whereas the +body/default row earns \$184{,}832.48 at threshold $0.345084$. The endpoint at +$0.273036$ shows how conservative the frontier can become while preserving the +return floor, and the \$223.5K tail-focused endpoint requires a threshold of +$0.697056$. Its residual premium cannot be recovered with the linear +$(1-\gamma)\Gamma_{\mathrm{CP}}$ shortcut. The supplement reports the full +policy-aware frontier and traceability details. + +\subsection{Matched Point-PD Baseline}\label{sec:point-baseline} + +To isolate what the conformal decision layer buys, we solve a matched two-stage +LP on the same 276{,}869 candidates with the same \$1M budget, concentration cap, +$\tau=0.1715$, LGD, solver, and operating constraints. The only change is that +the baseline uses calibrated point PD in both its objective and risk constraint. +Neither optimizer sees OOT outcomes; defaults enter only in the frozen post-hoc +audit. + +\begin{table}[t] + \centering + \caption{Matched point-PD baseline on the frozen Lending Club OOT panel.} + \label{tab:point-baseline} + \resizebox{\textwidth}{!}{% + \begin{tabular}{lrrrrrr} + \toprule + Policy & Realized return & Funded & Weighted default / $V$ & + $\Gamma_{\mathrm{CP}}$ & $B_u$ & Markov threshold \\ + \midrule + Point-PD two-stage LP & \$196{,}369.14 & 225 & $0.118400$ & $0.526736$ & $0.680579$ & $0.780579$ \\ + Selected CRPTO & \$184{,}832.48 & 314 & $0.035350$ & $0.162616$ & $0.245084$ & $0.345084$ \\ + \bottomrule + \end{tabular} + }% +\end{table} + +CRPTO gives up \$11{,}536.66, or $5.875\%$, of the baseline's realized return. +In exchange, weighted default/miscoverage falls by $0.08305$ and the exact +Markov loss threshold falls by $0.435495$. Both allocations have zero realized +risk-tolerance excess because their default rates remain below $\tau$, but the +point-PD allocation fails the tightest Markov safety screen +($0.1184>\sqrt{0.01}=0.10$). This is a frozen OOT return--auditability trade-off, +not causal evidence or universal dominance. Full fields are in Supplement A40. The funded-set under-coverage remains structural rather than a calibration-draw effect. With $n_{\mathrm{cal}}=237{,}584$ calibration loans, the split-conformal @@ -757,20 +820,24 @@ \section{Results}\label{sec:results} nominal level \citep{vovk2005,angelopoulos2023}. The residual funded-set $V$ is a test-side, portfolio-selection quantity. That is why the paper reads the safety level at $\sqrt{\alpha}$ under Assumption~\ref{asm:weighted-validity}, and why -the frontier reports $V$, $\Gamma_{\mathrm{CP}}$, endpoint budget, Markov cap, and + the frontier reports $V$, $\Gamma_{\mathrm{CP}}$, residual premium, endpoint budget, exact Markov threshold, and return together instead of promoting a standalone coverage number. \begin{table}[t] \centering \caption{Reviewer claim checks in the main manuscript.} \label{tab:reviewer-checks} - \resizebox{\textwidth}{!}{% - \begin{tabular}{lll} + {% + \small + \setlength{\tabcolsep}{4pt} + \renewcommand{\arraystretch}{1.12} + \begin{tabular}{@{}>{\raggedright\arraybackslash}p{0.20\textwidth}>{\raggedright\arraybackslash}p{0.47\textwidth}>{\raggedright\arraybackslash}p{0.25\textwidth}@{}} \toprule Reviewer concern & Body answer & Primary evidence \\ \midrule ``This is only a classifier.'' & The claim is decision auditability, not AUC leadership. & Exact funded-set certificate and A35 frontier. \\ ``CP + RO already exists.'' & CRPTO instantiates the idea for frozen credit PD models, funded-set governance, and Lending Club payoffs. & Closest-work boundary and bound claim stack. \\ + ``There is no matched baseline.'' & A40 holds candidates and operating constraints fixed and quantifies a $5.875\%$ return cost. & Table~\ref{tab:point-baseline} and A40 audit. \\ ``Adaptive selection breaks coverage.'' & The theorem states weighted funded-set validity as an assumption and then audits the frozen selection exactly. & Assumption map, validity ladder, and A23 diagnostics. \\ ``The selected policy is cherry-picked.'' & The selected point comes from a consolidated finite frontier with 50{,}010 deduplicated semantic policies and 27{,}508 eligible all-alpha above-floor policies. & Frontier table and governance files. \\ \bottomrule @@ -819,50 +886,15 @@ \subsection{Multi-Dataset External Economic Replication} \end{figure} \FloatBarrier -The external layer also surfaces a result a single-dataset champion cannot show. -The signed price of robustness---using the same convention as the Lending Club -field, $(\mathrm{nonrobust}-\mathrm{robust})/\mathrm{nonrobust}$---is a -\emph{positive} premium under frozen application, and across the four frozen -external applications it is ordered by panel default rate (Table~\ref{tab:price-of-robustness}, -Figure~\ref{fig:price-scaling}). Within Freddie, the high-default red segment pays more -than green; across datasets, Prosper's $30.92\%$ default panel pays the largest premium. -This is a pattern across two datasets---three Freddie default-window segments plus -Prosper---not a scaling law: it is consistent with the mechanism (higher default risk -widens the conformal intervals, so the robust worst case discounts more return, and -discrimination (AUC) does not order the premium on its own), but four points cannot -establish a general law. On the \emph{selected} Lending Club champion the -signed price is favorable ($-10.56\%$), because the bound-aware search found a -robust funded set that also wins expected return. The measured summary is more -modest: in these applications, the conformal robust layer is economically -bounded. Under blind application it costs a single-digit to low-double-digit -premium; under selection it can be favorable. - -\begin{table}[t] - \centering - \caption{Price of robustness by application, ordered by panel default - rate. The selected Lending Club champion is $-10.56\%$ (favorable under - selection, not blind application).} - \label{tab:price-of-robustness} - \begin{tabular}{lrr} - \toprule - Application & Panel default & Price of robustness \\ - \midrule - Freddie FM48 (green) & 0.58\% & $+1.00\%$ \\ - Freddie FM48 (combined) & 1.45\% & $+1.09\%$ \\ - Freddie FM48 (red) & 2.97\% & $+2.37\%$ \\ - Prosper final-status & 30.92\% & $+9.46\%$ \\ - \bottomrule - \end{tabular} -\end{table} - -\begin{figure}[t] - \centering - \includegraphics[width=0.78\textwidth]{crpto_fig25_price_of_robustness_scaling.pdf} - \caption{Across the four frozen external applications, the price of robustness - is ordered by panel default risk; the selected Lending Club champion sits - below zero as a favorable reference.} - \label{fig:price-scaling} -\end{figure} +The external layer adds a secondary economic pattern. Across Prosper and three +Freddie default-window applications, the signed robust premium ranges from +$+1.00\%$ to $+9.46\%$ and is ordered by panel default rate. Four applications +cannot establish a scaling law, so A34 and its companion figure remain in the +online supplement as mechanism-consistent diagnostics rather than a body claim. +The matched Lending Club comparison is instead A40 above, whose point-PD +baseline holds the candidate universe and operating constraints fixed. The +external recipe therefore transfers as an economic audit protocol, while the +exact funded-set certificate remains the Lending Club object. % ===================================================================== \FloatBarrier @@ -881,7 +913,7 @@ \section{Robustness and Comparators}\label{sec:robustness} adding conservative decoration. The answer is visible in the portfolio frontier. Policies are evaluated by return, exact alpha pass/fail, weighted miscoverage, and $\Gamma_{\mathrm{CP}}$; the promoted point is the body/default balanced policy on -that finite-grid frontier, while the strict $\le 0.345$ cap policy is reported +that finite-grid frontier, while the strict $\le 0.345$ threshold policy is reported separately. This differs from a workflow where conformal intervals are plotted after the optimizer has already chosen a point-PD allocation. @@ -944,7 +976,7 @@ \subsection{Regret-Auditability Frontier}\label{sec:regret} decision-regret experiment (A19/PyEPO) run on small synthetic optimization instances (50 items, budget 15, five seeds), scoring each method on a normalized decision-loss scale rather than on the \$1M funded portfolio; on that scale SPO+ -is the low-regret method by construction. The favorable price of robustness above +is the low-regret method by construction. The external price-of-robustness diagnostic in the supplement and the higher CRPTO regret here are not in tension---they are two different measurements (a real \$1M funded set versus a synthetic regret benchmark). The right-hand columns report what the credit decision actually delivers: only CRPTO @@ -979,7 +1011,7 @@ \subsection{Tail Risk and Distribution Robustness}\label{sec:tail-dist} Two reviewer questions deserve a body-level answer: what does the selected policy give up on the tail, and does its coverage hold once the evaluation is sliced by grade? The return-bound frontier answers the first question directly: lower -Markov caps are available at lower return, while the selected policy sits on the +Markov loss thresholds are available at lower return, while the selected policy sits on the declared frontier. The supplement then checks the funded-grade mix, selected-allocation tail repricing, dependence sensitivity, and fixed-allocation bootstrap interval. These rows explain the selected policy's risk profile; they @@ -997,7 +1029,7 @@ \subsection{Tail Risk and Distribution Robustness}\label{sec:tail-dist} \toprule Reviewer question & Body answer & Boundary \\ \midrule - Is the point only high-return? & The frontier shows safer lower-return choices and higher-return looser-cap choices. & Finite grid, not a continuous optimum. \\ + Is the point only high-return? & The frontier shows safer lower-return choices and higher-return looser-threshold choices. & Finite grid, not a continuous optimum. \\ What loans does it fund? & The supplement reports funded exposure by grade. & Business mix, not protected-class fairness certification. \\ What happens in the tail? & The selected allocation is repriced under LGD and tail summaries. & Risk profile only; tail risk is not the selector. \\ Does dependence change the bound? & Cluster sensitivity recomputes tighter assumptions. & Sensitivity only; Markov remains the body theorem. \\ @@ -1014,18 +1046,21 @@ \subsection{Tail Risk and Distribution Robustness}\label{sec:tail-dist} while the promoted interval summary in Table~\ref{tab:core} reports $0.9297$; these are distinct cuts through the same intervals. No grade falls below target, so the conservative marginal coverage is not hiding a failing segment; the thinnest -grade$\times$vintage cells are where a future group-weighted or multi-distribution -recalibration would matter, marked as future work rather than a present guarantee. +grade$\times$vintage cells identify where a group-weighted or multi-distribution +recalibration would require a separately tagged protocol, so they are not +promoted as a present guarantee. \subsection{Managerial Implication}\label{sec:managerial} For a credit-risk committee, CRPTO turns a model into a decision conversation. The committee can pick a risk cap $\tau$, inspect how a policy -parameter $\gamma$ changes the funded set, read $\Gamma_{\mathrm{CP}}$ as the -conformal robustness premium paid by that funded set, and compare $V(\alpha)$ -with the stated bound tolerance. The method therefore supports a practical -question: whether the realized economic return justifies the signed price of -robustness, which on this evaluation is favorable rather than a cost. If the +parameter $\gamma$ changes the funded set, separate the total conformal premium +into internalized and residual components, and compare $V(\alpha)$ with the +stated bound tolerance. On this evaluation the matched choice is concrete: +accept $5.875\%$ less realized return than the point-PD LP in exchange for +$8.305$ percentage points less weighted default/miscoverage and a $43.55$ +percentage point lower Markov loss threshold. The method supports a practical +trade-off rather than promising a free robustness premium. If the committee wants lower regret, the SPO+ corner is visible; if it wants stronger validity language, the validity ladder states the new calibration protocol that would be required. That separation is the managerial @@ -1041,7 +1076,8 @@ \subsection{Managerial Implication}\label{sec:managerial} multi-distribution and online (ACI) conformal-stability diagnostics, plus A25--A34 external economic replication, exhaustiveness, interval, subperiod, and sensitivity audits on Prosper and Freddie/Mendeley, and A35--A39 selected-policy -frontier, composition, tail-risk, concentration, and bootstrap audits. +frontier, composition, tail-risk, concentration, and bootstrap audits, plus A40 +matched point-PD comparison. % ===================================================================== \section{Discussion}\label{sec:discussion} @@ -1054,13 +1090,20 @@ \section{Discussion}\label{sec:discussion} conformal layer, and make the optimizer pay attention to the upper end of plausible default risk. +The policy-aware decomposition is more than a notation change. A linear blend +can recover its residual endpoint premium from $\gamma$ alone, but capped and +tail-focused policies cannot. Computing $\Gamma_{\mathrm{res}}$ from funded rows +makes every frontier point comparable on the same exact endpoint scale and +prevents a nonlinear tail policy from appearing safer because of a shortcut. + The external replications also change how to read the price of robustness. Across the frozen external applications, the premium is ordered by panel default risk rather than by discrimination. That pattern reframes robustness as a panel-specific premium to be measured, not a fixed toll to be assumed. It also -tempers the contribution: the favorable Lending Club value reflects champion -selection, so the transferable claim is bounded measurement under blind -application, not a universal free lunch. +tempers the contribution: the matched Lending Club audit observes a $5.875\%$ +return cost, while external applications report different premiums under their +frozen contracts. The transferable claim is reproducible measurement of a +return--risk trade-off, not a universal free lunch. The limits are equally important. CRPTO does not prove that any one public dataset is a universal proxy for modern credit origination. The Prosper and @@ -1073,17 +1116,16 @@ \section{Discussion}\label{sec:discussion} because the public data lack direct protected attributes. It does not assert that robust conformal policies dominate all decision-focused learners on regret. -The next research directions are clear and are now separated into paper -improvements versus new CRPTO protocols. The current paper includes the safe -journal-strengthening package: OCE/CVaR as a tail-risk audit, satisficing as +The manuscript is deliberately written as one IJDS paper rather than a bundle of +method variants. Adjacent methods enter only when they make the submitted +certificate easier to evaluate: OCE/CVaR as a tail-risk audit, satisficing as margin evidence, SPO+ as the low-regret corner of the regret-auditability -frontier, and dependence as a formal caveat. The larger scientific upgrades are -useful in the present manuscript only when they sharpen this boundary. -Table~\ref{tab:upgrade-map} states that boundary directly. +frontier, and dependence as a formal caveat. Table~\ref{tab:upgrade-map} states +that single-submission boundary directly. \begin{table}[t] \centering - \caption{Scientific upgrade map for the current CRPTO paper.} + \caption{Single-submission boundary map for the current CRPTO paper.} \label{tab:upgrade-map} {% \small @@ -1091,12 +1133,12 @@ \section{Discussion}\label{sec:discussion} \renewcommand{\arraystretch}{1.12} \begin{tabular}{@{}>{\raggedright\arraybackslash}p{0.22\textwidth}>{\raggedright\arraybackslash}p{0.36\textwidth}>{\raggedright\arraybackslash}p{0.34\textwidth}@{}} \toprule - Upgrade path & What the current paper can claim & What would require a new result \\ + Adjacent path & What this paper uses & Boundary for the submitted claim \\ \midrule - Tail-aware selection & A20--A22 and A37 show the tail trade-off and selected-allocation tail profile. & A predeclared CVaR/OCE selector with its own exact funded-set audit. \\ - Prospective selection & Nested holdout and the finite declared grid reduce post-selection ambiguity. & A fully prospective search/evaluation split after all policy rules are frozen. \\ - Multi-distribution or online validity & A23--A24 diagnose grade, distribution, and vintage stress on frozen intervals. & A new calibration protocol that targets group-weighted, multi-source, or live sequential validity. \\ - Decision-focused conformal learning & A19 shows SPO+ as the low-regret comparator and CRPTO as the auditable corner. & An end-to-end learner that jointly optimizes decision loss and emits a conformal funded-set certificate. \\ + Tail-aware selection & A20--A22 and A37 show the selected decision's tail profile and available return-tail trade-off. & The promoted selector remains the declared return-bound finite frontier. \\ + Prospective selection & Nested holdout and the finite declared grid reduce post-selection ambiguity. & The paper does not claim a fully prospective selection/evaluation trial. \\ + Multi-distribution or online validity & A23--A24 diagnose grade, distribution, and vintage stress on frozen intervals. & The conformal layer is not recalibrated for multi-source or live sequential validity. \\ + Decision-focused conformal learning & A19 shows SPO+ as the low-regret comparator and CRPTO as the auditable corner. & The PD model remains frozen; no end-to-end learner is promoted. \\ \bottomrule \end{tabular} }% @@ -1106,10 +1148,10 @@ \section{Discussion}\label{sec:discussion} conformal validity, utility-directed or decision-theoretic conformal variants \citep{cortesgomez2025utility,lekeufack2023cdt}, causal variants, broader asset-class panels, and prospective multi-period origination studies are all -valuable. They become stronger paper claims only with new protocols, data, or -proofs. In the current submission, their role is to make the frozen CRPTO +valuable comparators. In this submission, their role is to make the frozen CRPTO contribution easier to locate: an auditable post-hoc predict-then-optimize -certificate, not a universal decision-learning framework. +certificate, not a universal decision-learning framework and not a collection of +additional promoted methods. % ===================================================================== \section{Conclusion}\label{sec:conclusion} @@ -1121,14 +1163,14 @@ \section{Conclusion}\label{sec:conclusion} the exact empirical $\alpha=0.01$ funded-set audit, and it lies on a declared finite-grid return-bound frontier with 50{,}010 deduplicated semantic policies and 27{,}508 all-alpha above-floor policies rather than at a single lucky point. -The external -Prosper and Freddie/Mendeley replications show that the recipe travels beyond the -Lending Club panel and expose an economically interpretable regularity: the price -of robustness is a bounded premium ordered by panel default risk across the -frozen external applications. The -contribution is scoped as an auditable post-hoc decision certificate, not a new -end-to-end learner or a live-deployment study, and every reported number is -regenerable from frozen evidence. +A matched point-PD LP shows the price explicitly: $5.875\%$ less realized return +for $8.305$ percentage points less weighted default/miscoverage and a $43.55$ +percentage point lower exact Markov loss threshold. The policy-aware residual +premium makes the certificate valid across the linear, capped, and tail-focused +frontier, while Prosper and Freddie/Mendeley show that the recipe can be audited +on other credit products. The contribution is one auditable post-hoc decision +certificate, not a new end-to-end learner, live-deployment study, or portfolio +of additional methods; every reported number is regenerable from frozen evidence. % Reproducibility/companion disclosure is kept for the cover letter / non-anonymous % version, not the double-anonymous body. diff --git a/paper/submission/IJDS_SUBMISSION_ROADMAP_2026-08-10.md b/paper/submission/IJDS_SUBMISSION_ROADMAP_2026-08-10.md index 72c144d..82049be 100644 --- a/paper/submission/IJDS_SUBMISSION_ROADMAP_2026-08-10.md +++ b/paper/submission/IJDS_SUBMISSION_ROADMAP_2026-08-10.md @@ -50,10 +50,10 @@ CRPTO should be read as data science for decisions: | 6 | Related work | The closest-work boundary distinguishes CRPTO from P2P OR, conformal credit scoring, conformal RO, DFL, and financial portfolios. | | 7 | Figures | Main figures have single-sentence takeaways, readable axes, grayscale-safe contrast, and no unnecessary decorative elements. | | 8 | Tables | Body tables are compact reviewer evidence; voluminous diagnostics stay in the supplement. | -| 9 | Supplement | A3--A39 are organized as a defense layer with scope caveats. | +| 9 | Supplement | A3--A40 are organized as a defense layer with scope caveats. | | 10 | Reproducibility | Accepted-paper package has code, DVC pointers, manifest, raw-data instructions, and guardrail commands. | | 11 | Double anonymity | Reviewer-facing body and supplement contain no author URLs, names, local paths, or private remotes. | -| 12 | Official IJDS template | `CRPTO_ijds_submission.tex` is manually synchronized from the pool93 A35--A39 QMD source, keeps the official-template compaction, compiles against the official files, and is rechecked after body edits. | +| 12 | Official IJDS template | `CRPTO_ijds_submission.tex` is manually synchronized from the pool93 A35--A40 QMD source, keeps the official-template compaction, compiles against the official files, and is rechecked after body edits. | | 13 | Data/code form | Cover letter and disclosure text acknowledge IJDS accepted-paper reproducibility requirements. | | 14 | Acceptance-risk audit | A short list of likely reviewer objections has body or supplement responses. | | 15 | Freeze discipline | Protected champion/search stages are never rerun as routine paper reproduction. | diff --git a/paper/submission/README.md b/paper/submission/README.md index 7e01923..701874d 100644 --- a/paper/submission/README.md +++ b/paper/submission/README.md @@ -60,11 +60,12 @@ INFORMS class (`\documentclass[ijds,dblanonrev]{informs4}`). The narrative source remains `paper/CRPTO_ijds.qmd`, but the official `.tex` is now a manually compacted IJDS-template surface. After freeze, do **not** regenerate it mechanically from QMD; port substantive claim changes deliberately, then rebuild -and recheck the 26-page official PDF. The synchronized submission surface should +and recheck the official-template PDF. The synchronized submission surface should carry the central IJDS body: title, abstract, keywords, core sections, the journal pipeline Figure 1, the bound-claim stack, -the A35 finite-grid frontier, the A36--A39 selected-allocation audits in the -supplement, the regret-auditability comparison, plus the core, exact-certificate, +the A35 policy-aware finite-grid frontier, the A36--A39 selected-allocation +audits and A40 matched point-PD baseline in the supplement, the +regret-auditability comparison, plus the core, exact-certificate, funded-set audit and regret tables. The `informs2014.bst` + `../../book/references.bib` bibliography wiring is already present. Journal figures use PDF/vector exports from `reports/crpto/figures/` @@ -77,11 +78,11 @@ PDF crop box cuts the right edge under `informs4`. > `informs4.cls`, `informs2014.bst`, template PDFs, `.sty` files, or generated > LaTeX build artifacts. -Current local build state (verified 2026-07-07): TinyTeX/TeX Live 2026, +Current local build state (verified 2026-07-09): TinyTeX/TeX Live 2026, `pdflatex`, `bibtex`, and the `listingsutf8` TeX package compile -`CRPTO_ijds_submission.tex` to a 26-page official-template PDF. Section 9 -(Conclusion) and References both start on page 22, so the body remains inside -the IJDS 25-page initial-submission budget when references are excluded. The +`CRPTO_ijds_submission.tex` to a 28-page official-template PDF. References +start on page 24, so the body remains inside the IJDS 25-page +initial-submission budget when references are excluded. The only LaTeX log warnings left are a small `\maketitle` overfull from the `informs4` anonymous title block and font-size / underfull paragraph warnings, visually acceptable unless the final ScholarOne proof shows a layout issue. @@ -175,10 +176,10 @@ These protocols are compatible but not interchangeable. - Use `SCHOLARONE_FINAL_CHECKLIST.md` while uploading and reviewing the generated proof. - Recheck the official-template page budget if the body changes materially. The - current local official-template build is 26 pages total; Section 9 and - References start on page 22, keeping the body within the 25-page limit when - references are excluded. -- Keep A3--A39 in the online supplement unless a reviewer-facing argument needs + current local official-template build is 28 pages total; References start on + page 24, keeping the body within the 25-page limit when references are + excluded. +- Keep A3--A40 in the online supplement unless a reviewer-facing argument needs one compact table in the body. - Preserve CRPTO as the coverage/auditability method and SPO+ as the low-regret comparator. @@ -233,7 +234,7 @@ updates the template. 4. **Recount the official-template page budget** and demote body floats to the supplement only if the body exceeds 25 pages excluding references. The local - official-template build is currently 26 pages total; Section 9 and References - start on page 22. The Chrome-print body preview is only a verification proxy. + official-template build is currently 28 pages total; References start on + page 24. The Chrome-print body preview is only a verification proxy. 5. **Verify anonymity** against the checklist above, then upload the body PDF and submit the title page separately. diff --git a/paper/submission/SCHOLARONE_FINAL_CHECKLIST.md b/paper/submission/SCHOLARONE_FINAL_CHECKLIST.md index 14551d8..34fe81b 100644 --- a/paper/submission/SCHOLARONE_FINAL_CHECKLIST.md +++ b/paper/submission/SCHOLARONE_FINAL_CHECKLIST.md @@ -17,7 +17,7 @@ template files have been downloaded outside Git. ## Official Template Build 1. Download or refresh `informs4.cls` and `informs2014.bst` from INFORMS/Overleaf. -2. Synchronize `CRPTO_ijds_submission.tex` manually from the pool93 A35--A39 +2. Synchronize `CRPTO_ijds_submission.tex` manually from the pool93 A35--A40 QMD source while preserving the official-template compaction. 3. Place the template files next to `CRPTO_ijds_submission.tex`; local gitignored copies are already present. 4. Build with `latexmk`. In Codex/PowerShell sessions where `WINDIR` is absent, diff --git a/paper/supplement_ijds.qmd b/paper/supplement_ijds.qmd index 3bdb326..3c6065c 100644 --- a/paper/supplement_ijds.qmd +++ b/paper/supplement_ijds.qmd @@ -26,13 +26,14 @@ execute: ## Scope This online supplement supports the IJDS submission body. It collects proof -details, robustness and external-replication tables A3--A39, reproducibility +details, robustness and external-replication tables A3--A40, reproducibility commands, model-risk material, fairness diagnostics, and evidence lineage. It does not introduce a hidden selection criterion or unreported claim family: A35 is the declared finite-grid frontier consumed by the body, A36 is the regenerated funded-set grade audit for the selected body point, and A37--A39 add selected-policy tail-risk, cluster-bound, and fixed-allocation -bootstrap diagnostics from the same selected allocation. +bootstrap diagnostics from the same selected allocation. A40 is the matched +point-PD baseline used to quantify the Lending Club return--risk trade-off. ::: The supplement is organized as a defense layer for the main manuscript rather @@ -47,6 +48,7 @@ external and robustness evidence. | What is the theorem? | Appendix A | Markov under weighted funded-set validity is the body claim; stronger assumptions stay explicit. | | Is the selected policy a singleton? | Appendix C, A35 | The frontier is finite-grid and exact, with denominators reported. | | What does the selected policy fund? | Appendix C, A36--A39 | Composition, tail, concentration, and bootstrap are selected-allocation diagnostics. | +| What does robustness cost on Lending Club? | Appendix C, A40 | The point-PD LP holds the candidate universe and operating constraints fixed. | | Does the recipe travel? | Appendix C, A25--A34 | Prosper and Freddie/Mendeley are external economic recipe-transfer checks. | | What can be reproduced? | Appendix E | Routine reproduction rebuilds paper surfaces from frozen evidence and excludes protected searches. | @@ -55,15 +57,16 @@ external and robustness evidence. ::: {.callout-important} ## Journal Strengthening Pack -Selected former P2/P3 ideas are included here when they can be evaluated from -frozen evidence: OCE/CVaR as a tail-risk diagnostic, robust satisficing as -committee-style margins, regret-auditability as the SPO+/CRPTO comparator, and -dependence-aware theory as a caveat/proposition. The multidataset layer is now -included as a frozen external economic replication on Prosper and Freddie/Mendeley: -it tests transfer of the CRPTO recipe without reopening the Lending Club -champion. Optimized OCE/CVaR objectives, full multi-distribution or online -conformal prediction, online DFL, causal CRPTO, multi-period portfolios, -production monitoring, and package extraction remain future work only. +Selected former P2/P3 ideas are included here only when they defend the single +IJDS submission from frozen evidence: OCE/CVaR as a tail-risk diagnostic, robust +satisficing as committee-style margins, regret-auditability as the SPO+/CRPTO +comparator, and dependence-aware theory as a caveat/proposition. The +multidataset layer is included as a frozen external economic replication on +Prosper and Freddie/Mendeley: it tests transfer of the CRPTO recipe without +reopening the Lending Club champion. Optimized OCE/CVaR objectives, full +multi-distribution or online conformal prediction, online DFL, causal CRPTO, +multi-period portfolios, production monitoring, and package extraction are +outside the submitted claim and are not acceptance criteria for this paper. ::: # Appendix A: Theoretical Details @@ -82,11 +85,13 @@ A does not turn the selected funded set into a universal conformal guarantee. | `x_i` | Funding decision or allocation weight. | | `a_i` | Loan amount or exposure. | | `w_i` | Normalized funded-set weight, `x_i a_i / sum_j x_j a_j`. | -| `gamma` | Robust-policy blend parameter in the optimization rule, `gamma in [0,1]`. | -| `p_tilde_i(alpha,gamma)` | Blended PD used by the optimizer, `p_hat_i + gamma (u_i - p_hat_i)`. | -| `tau` | Risk-tolerance cap the optimizer applies to the blended PD, `sum_i w_i p_tilde_i <= tau`. | -| `Gamma_CP` | Portfolio-level conformal risk metric after allocation: `sum_i w_i (u_i - p_hat_i)`, with clipping at one on the PD scale. | -| `B_u(alpha)` | Weighted upper-endpoint budget of the funded set, `sum_i w_i u_i(alpha)`; equals `tau + (1-gamma) Gamma_CP` when the cap binds. | +| `gamma` | Blend parameter for linear members of the optimization-policy family, `gamma in [0,1]`. | +| `q_i(alpha;theta)` | Effective PD used by a declared linear, capped, or tail-focused policy; `p_hat_i <= q_i <= u_i`. | +| `tau` | Risk-tolerance cap applied to the effective score, `sum_i w_i q_i <= tau + s`, where `s` is recorded solver slack. | +| `Gamma_CP` | Total portfolio conformal premium after allocation: `sum_i w_i (u_i - p_hat_i)`. | +| `Gamma_int` | Premium internalized by the decision score: `sum_i w_i (q_i - p_hat_i)`. | +| `Gamma_res` | Residual endpoint premium: `sum_i w_i (u_i - q_i)`. | +| `B_u(alpha)` | Exact weighted upper-endpoint budget, `sum_i w_i u_i(alpha) = sum_i w_i q_i + Gamma_res`. | | `V(alpha)` | Weighted funded-set miscoverage quantity. | Prose writes the conformal premium as $\Gamma_{\mathrm{CP}}$; tables sometimes @@ -114,10 +119,11 @@ endpoints live on the PD scale, $Y_i \in [0,1]$ and $u_i(\alpha) \in [0,1]$, with miscoverage indicators $Z_i(\alpha) = \mathbf{1}\{Y_i > u_i(\alpha)\}$ and $V(\alpha) = \sum_i w_i Z_i(\alpha)$. Write $B_u(\alpha) = \sum_i w_i u_i(\alpha)$ for the weighted upper-endpoint budget of -the funded set. The robust layer caps the $\gamma$-blended PD, -$\sum_i w_i \tilde p_i(\alpha,\gamma) \leq \tau$ with -$\tilde p_i = \hat p_i + \gamma(u_i(\alpha) - \hat p_i)$; the deterministic -identity below does not use this cap and holds for any allocation. Probabilities +the funded set. The robust layer caps a policy-specific effective score +$q_i(\alpha;\theta)$ satisfying +$\hat p_i\leq q_i(\alpha;\theta)\leq u_i(\alpha)$ and +$\sum_iw_iq_i\leq\tau+s$, where $s\geq0$ is recorded solver cap slack. The +deterministic identity below does not use this cap and holds for any allocation. Probabilities and expectations are over the exchangeable calibration/test draw conditional on the frozen recipe, partitions, and allocation rule. @@ -165,26 +171,37 @@ and the choice $t = \sqrt{\alpha}$ gives the body statement: the endpoint budget $B_u(\alpha)$ is exceeded by more than $\sqrt{\alpha}$ with probability at most $\sqrt{\alpha}$. $\blacksquare$ -**Policy-term decomposition.** The optimizer constrains the $\gamma$-blended PD, -not $B_u(\alpha)$ itself. Since -$\sum_i w_i \tilde p_i(\alpha,\gamma) = \sum_i w_i \hat p_i + \gamma\,\Gamma_{\mathrm{CP}}(\alpha)$ -and $B_u(\alpha) = \sum_i w_i \hat p_i + \Gamma_{\mathrm{CP}}(\alpha)$, with -$\Gamma_{\mathrm{CP}}(\alpha) = \sum_i w_i\bigl(u_i(\alpha) - \hat p_i\bigr)$, +**Policy-term decomposition.** The optimizer constrains $q_i$, not +$B_u(\alpha)$ itself. Define $$ -B_u(\alpha) = \sum_i w_i \tilde p_i(\alpha,\gamma) + (1-\gamma)\,\Gamma_{\mathrm{CP}}(\alpha) - \;\leq\; \tau + (1-\gamma)\,\Gamma_{\mathrm{CP}}(\alpha), +\Gamma_{\mathrm{int}}(\alpha) +=\sum_iw_i(q_i-\hat p_i),\qquad +\Gamma_{\mathrm{res}}(\alpha) +=\sum_iw_i(u_i-q_i). $$ -with equality when the cap binds. The selected body point binds at -$\tau = 0.1715$ with $\gamma = 0.5475$ and -$\Gamma_{\mathrm{CP}}(0.01) = 0.162616$, so -$B_u(0.01) \leq 0.245084$ and the deterministic bound reads -$\sum_i w_i Y_i \leq 0.245084 + V(0.01) = 0.280434$. The exact audit reports -zero deterministic violation. The slack $(1-\gamma)\,\Gamma_{\mathrm{CP}}(\alpha)$ is -the part of the conformal premium the optimizer does not internalize at -$\gamma < 1$; setting $\gamma = 1$ recovers the tight endpoint cap -$B_u(\alpha) \leq \tau$. +Then +$\Gamma_{\mathrm{CP}}=\Gamma_{\mathrm{int}}+\Gamma_{\mathrm{res}}$ and + +$$ +B_u(\alpha) +=\sum_iw_iq_i(\alpha;\theta)+\Gamma_{\mathrm{res}}(\alpha) +\leq\tau+s+\Gamma_{\mathrm{res}}(\alpha). +$$ + +For the pure linear blend only, +$\Gamma_{\mathrm{res}}=(1-\gamma)\Gamma_{\mathrm{CP}}$. The selected capped +policy has no active cap among its 314 funded rows, its effective-score cap +binds with $s=0$, and its audit gives +$\Gamma_{\mathrm{int}}=0.089032$ and $\Gamma_{\mathrm{res}}=0.073584$. +Consequently $B_u(0.01)=0.1715+0.073584=0.245084$, and the deterministic +identity reads $\sum_iw_iY_i\leq0.245084+V(0.01)=0.280434$. The observed +weighted outcome is `0.035350`, so realized risk-tolerance excess above +$\tau=0.1715$ is zero. The exact Markov loss threshold is +$B_u+\sqrt{\alpha}=0.345084$; it is an event threshold, not a deterministic +risk cap. This general decomposition is required for capped and tail-focused +policies whose residual premium cannot be inferred from $\gamma$ alone. On the promoted draw the realized weighted miscoverage is $V(0.01) = 0.035350$, which *exceeds* the nominal $\alpha = 0.01$: the funded set under-covers relative @@ -224,7 +241,8 @@ The phrase "exact funded-set certificate" has a narrow meaning throughout the paper. It is an exact accounting audit on the frozen out-of-time funded set: given the selected allocation weights, observed outcomes, calibrated PD values, and conformal upper endpoints, the audit computes `V(alpha)`, -`Gamma_CP(alpha)`, and violation directly on the funded loans. It is not a new +`Gamma_CP(alpha)`, its internalized/residual decomposition, the exact endpoint +budget, and realized risk-tolerance excess directly on the funded loans. It is not a new distribution-free theorem for arbitrary adaptive portfolios and it is not an external-dataset certificate. Its statistical interpretation still requires the weighted funded-set validity assumption in the body; its value is that the @@ -233,7 +251,7 @@ transcribed table. | Certificate object | Computed from | What it supports | What it does not support | |---|---|---|---| -| Exact funded-set audit | Frozen Lending Club OOT funded loans, allocations, labels, `p_hat`, and `u_i(alpha)`. | Body point `V(0.01)=0.035350`, `Gamma_CP=0.162616`, Markov cap `0.345084`, zero violation, and `8/8` alpha-grid pass. | Universal conditional coverage or live adaptive control. | +| Exact funded-set audit | Frozen Lending Club OOT funded loans, allocations, labels, `p_hat`, `q_i`, and `u_i(alpha)`. | Body point `V(0.01)=0.035350`, `Gamma_CP=0.162616`, `Gamma_res=0.073584`, exact Markov threshold `0.345084`, zero realized risk-tolerance excess, and `8/8` alpha-grid pass. | Universal conditional coverage or live adaptive control. | | Finite-grid frontier | 51,678 raw rows consolidated into 50,010 deduplicated semantic policies. | 27,508 all-alpha above-floor policies; terminal endpoint search `37,068/37,068` all-alpha passers. | A claim about all continuous policy values or future searches. | | External exhaustiveness audit | Prosper all-candidate and Freddie capped/all-candidate LP solves. | The reported external LP values are not shortlist effects. | New exact funded-set certificates for Prosper or Freddie. | @@ -254,15 +272,28 @@ the return floor. The terminal endpoint run evaluates 37,068 policies and is fixed at `{0.01, 0.03, 0.05, 0.07, 0.10, 0.12, 0.15, 0.20}`. -| Role | Return | Gamma_CP | V | Markov cap | Alpha pass | -|---|---:|---:|---:|---:|:---:| -| Minimum Markov-cap endpoint | `$170,467.27` | `0.095719` | `0.031875` | `0.273036` | `8/8` | -| Low-cap balanced endpoint | `$171,006.20` | `0.097190` | `0.031875` | `0.274789` | `8/8` | -| Highest return under cap <= `0.30` | `$173,314.04` | `0.115400` | `0.035875` | `0.294580` | `8/8` | -| Strict cap <= `0.345` body proxy | `$184,800.41` | `0.162562` | `0.035350` | `0.344996` | `8/8` | -| Body/default balanced point | `$184,832.48` | `0.162616` | `0.035350` | `0.345084` | `8/8` | -| Highest return under cap <= `0.36` | `$186,050.73` | `0.174600` | `0.037750` | `0.358685` | `8/8` | -| Max-return economic endpoint | `$223,458.14` | `0.457438` | `0.069575` | `0.510753` | `8/8` | +The policy-aware v2 reconstruction reads sufficient statistics from six frozen +exact-evaluation runs and performs no new search or LP solve. It preserves the +51,678 raw rows, 50,010 deduplicated policies, 27,508 eligible policies, and the +selected body point. Relative to the former linear-only shortcut, 10,423 +policies receive a materially different exact threshold. The affected nonlinear +families are `tail_blended_uncertainty` and +`segment_relative_tail_blended_uncertainty`: 2,866 tail policies had understated +thresholds, the largest understatement was `0.241324`, and 716 policies formerly +labeled at or below `0.50` exceed `0.50` on the exact endpoint scale. This audit +is why A35 now reports $\Gamma_{\mathrm{res}}$ and $B_u$ explicitly. + +| Role | Return | $\Gamma_{CP}$ | $\Gamma_{res}$ | $V$ | $B_u$ | Markov threshold | Pass | +|---|---:|---:|---:|---:|---:|---:|:---:| +| Minimum Markov-threshold endpoint | `$170,467.27` | `0.095719` | `0.004786` | `0.031875` | `0.173036` | `0.273036` | `8/8` | +| Low-threshold balanced endpoint | `$171,006.20` | `0.097190` | `0.007289` | `0.031875` | `0.174789` | `0.274789` | `8/8` | +| Highest return under threshold <= `0.30` | `$174,552.51` | `0.120988` | `0.030247` | `0.035875` | `0.199997` | `0.299997` | `8/8` | +| Highest return under threshold <= `0.32` | `$179,436.12` | `0.139182` | `0.049410` | `0.035875` | `0.219910` | `0.319910` | `8/8` | +| Strict threshold <= `0.345` body proxy | `$184,800.41` | `0.162562` | `0.072747` | `0.035350` | `0.244997` | `0.344997` | `8/8` | +| Body/default balanced point | `$184,832.48` | `0.162616` | `0.073584` | `0.035350` | `0.245084` | `0.345084` | `8/8` | +| Highest return under threshold <= `0.36` | `$186,050.73` | `0.174600` | `0.082935` | `0.037750` | `0.258685` | `0.358685` | `8/8` | +| Highest return under threshold <= `0.45` | `$198,693.28` | `0.252323` | `0.164010` | `0.045600` | `0.349010` | `0.449010` | `8/8` | +| Max-return economic endpoint | `$223,458.14` | `0.457438` | `0.440181` | `0.069575` | `0.597056` | `0.697056` | `8/8` | : Finite-grid return-bound frontier (A35). Source file: `reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv`. @@ -426,16 +457,16 @@ future journal hardening. # Appendix C: Journal Robustness Package -Tables A12--A39 answer likely reviewer questions that are too detailed for the +Tables A12--A40 answer likely reviewer questions that are too detailed for the 25-page body. The block structure is deliberate: A12--A18 report tail, margin and policy-family diagnostics; A19 isolates regret-auditability; A20--A24 stress tail, dependence, multi-distribution and online interpretations; A25--A34 test -external recipe transfer; and A35--A39 close the selected-policy frontier, -composition, tail, concentration and uncertainty profile. A35 is the active +external recipe transfer; and A35--A40 close the selected-policy frontier, +composition, tail, concentration, uncertainty profile, and matched baseline. A35 is the active frontier consumed by the body, A36 is the selected-policy funded-set grade audit, A37 is the selected-policy tail-risk repricing, A38 is the -selected-policy cluster-bound audit, and A39 is the selected-policy -fixed-allocation bootstrap audit. A12--A34 are diagnostic by design, except A19 +selected-policy cluster-bound audit, A39 is the selected-policy +fixed-allocation bootstrap audit, and A40 is the matched point-PD decision audit. A12--A34 are diagnostic by design, except A19 also supports the body framing around regret-auditability and A25--A34 support the external-replication defense. A20--A34 are journal-only add-ons: A20 audits tail-risk trade-offs as a @@ -456,16 +487,17 @@ frozen funded set or intervals, never as a re-promoted champion. |---|---|---| | A35 finite-grid frontier | Promoted body evidence | It is the active return-bound decision surface. | | A36--A39 selected-policy audits | Support evidence | They describe the selected allocation without changing the selector. | +| A40 matched point-PD baseline | Support evidence | It quantifies the Lending Club return--risk trade-off with operating constraints fixed. | | A19 regret-auditability | Support evidence | It answers the SPO+/DFL baseline objection. | | A25--A34 external recipe transfer | Support evidence | It answers the single-dataset objection without new certificates. | -| A20--A24 tail/source/online diagnostics | Diagnostic evidence | They price assumptions and future-work lanes without promoting new guarantees. | +| A20--A24 tail/source/online diagnostics | Diagnostic evidence | They price assumptions and outside-claim lanes without promoting new guarantees. | : Promoted, support, and diagnostic evidence hierarchy. | Table | Role | Scope caveat | |---|---|---| | A12 tail-risk OCE/CVaR diagnostics | Reprices the funded set under tail-risk summaries. | Diagnostic only; OCE/CVaR is not the optimized objective. | -| A13 satisficing margins | Expresses return, `V`, `Gamma_CP`, violation, and frontier pass as margins. | Thresholds are explanatory, not a new policy selector. | +| A13 satisficing margins | Expresses return, `V`, `Gamma_CP`, realized risk excess, and frontier pass as margins. | Thresholds are explanatory, not a new policy selector. | | A14 dependency cluster diagnostics | Documents period/grade concentration for the tightening caveat. | Does not prove independence. | | A15 leave-one-period stress | Reweights the funded set by leaving periods out. | Descriptive stress, not re-optimization. | | A16 bootstrap funded-set metrics | Adds empirical intervals for return, defaults, `V`, and misses. | Bootstrap interval, not conformal guarantee. | @@ -475,7 +507,7 @@ frozen funded set or intervals, never as a re-promoted champion. | A20 tail-risk diagnostic audit | Ranks tail-risk alternatives by CVaR, OCE, return, and satisficing status on the legacy diagnostic surface. | Journal diagnostic machinery; A37 is the selected-allocation tail repricing. | | A21 cluster-bound tightening | Reports cluster-aware Hoeffding thresholds by period, grade, and period-grade. | Transparent caveat; not tighter than Markov for the observed exposure concentration. | | A22 tail-constrained re-optimization | Selects the max-return policy under a decision-time CVaR cap computed from conformal upper endpoints, tracing the return-vs-CVaR frontier. | CVaR/OCE as an active selection constraint; reports a tail-constrained challenger, not a new promoted policy. | -| A23 multi-distribution robustness | Worst-case 90% coverage by grade and grade x vintage cell on the frozen intervals. | Read-only diagnostic; the worst fine cell motivates MDCP/group-weighted as future work. | +| A23 multi-distribution robustness | Worst-case 90% coverage by grade and grade x vintage cell on the frozen intervals. | Read-only diagnostic; the worst fine cell motivates MDCP/group-weighted only under a separately tagged calibration protocol. | | A24 online conformal stability | Per-vintage and cumulative coverage plus the Gibbs-Candes ACI target trajectory over the OOT vintage sequence. | Static-OOT online-control diagnostic, not a streaming validation. | | A25 external replication gate | Applies the frozen CRPTO scoring/conformal/LP recipe to Prosper final-status loans and Freddie/Mendeley FM48. | External economic replication; not a Lending Club champion rerun and not a new exact theorem. | | A26 external candidate sensitivity | Checks whether the robust LP objective is stable as the OOT candidate pool grows. | Candidate-pool audit; supports the claim that reported objectives are not a tiny shortlist effect. | @@ -492,6 +524,7 @@ frozen funded set or intervals, never as a re-promoted champion. | A37 selected-policy tail-risk repricing | Reprices the selected body allocation under LGD alternatives, CVaR, OCE, and decision-time tail loss. | Diagnostic risk profile of the selected point; OCE/CVaR is not the optimized objective. | | A38 selected-policy cluster-bound audit | Recomputes period, grade, period-grade, and score-vintage concentration thresholds from selected funded weights. | Sensitivity under extra cross-cluster assumptions; Markov remains the body-level bound. | | A39 fixed-allocation bootstrap audit | Bootstraps funded-loan contributions under the selected body allocation. | Empirical contribution interval only; solver inputs, model, calibration, and policy search are not resampled. | +| A40 matched point-PD baseline | Solves a point-PD two-stage LP on the same candidates with the same budget, concentration cap, risk tolerance, LGD, and solver. | Frozen OOT trade-off audit; no causal, prospective, or universal-dominance claim. | ## External Multi-Dataset Replication @@ -611,9 +644,10 @@ to make the audit path explicit. |---|---|---|---| | The predictive input is a frozen calibrated PD model, not a refreshed leaderboard model. | AUC, Brier, ECE, temporal backtesting, and calibration diagnostics. | `models/pd_canonical.cbm`, `models/pd_canonical_calibrator.pkl`, paper-facing metric tables. | `EXTRACTION_MANIFEST.json` and champion validation hashes. | | The conformal layer gives conservative OOT uncertainty on the PD scale. | 90% and 95% coverage, minimum group coverage, and grade/decile audits. | `data/processed/conformal_intervals_mondrian.parquet`. | Conformal validation status and regression tests that check metric consistency across surfaces. | -| The promoted funded set passes the exact safety bound `V <= sqrt(alpha)` (not nominal alpha-coverage). | Body point `V(alpha = 0.01) = 0.035350` (above alpha), `Gamma_CP = 0.162616`, Markov cap `0.345084`, zero violation, `8/8` alpha pass. | A35, A36, exact bound-evaluation parquet, consolidated frontier/governance JSON. | Exact-evaluation file and regression tests. | +| The promoted funded set passes the exact empirical safety screen `V <= sqrt(alpha)` (not nominal alpha-coverage). | Body point `V(alpha = 0.01) = 0.035350` (above alpha), `Gamma_CP = 0.162616`, `Gamma_res = 0.073584`, exact Markov threshold `0.345084`, zero realized risk-tolerance excess, `8/8` alpha pass. | A35, A36, exact bound-evaluation parquet, policy-aware frontier/governance JSON. | Exact-evaluation file and regression tests. | | The result is not an isolated lucky policy. | The consolidated frontier has 50,010 deduplicated semantic policies and 27,508 all-alpha above-floor policies; terminal endpoint search has 37,068/37,068 all-alpha passers. | Table A35 and governance files. | Protected search/evaluation split; no continuous-region claim. | -| The supplement strengthens interpretation without moving the body claim. | A20--A34 challenger, dependence, tail-risk, multi-distribution, online, and external-replication diagnostics; A35 is the active frontier; A36--A39 are selected-allocation audits. | Journal robustness tables, Figures 15--25, A35--A39. | Scope caveats in each table; A37--A39 are risk-profile audits, not hidden champion selectors. | +| The conformal decision has a matched operating baseline. | A40 holds candidates, budget, concentration, risk tolerance, LGD, and solver fixed; CRPTO pays `5.875%` realized return for `8.305` percentage points less weighted default/miscoverage. | A40 CSV/TeX and point-baseline audit JSON. | One frozen OOT comparison; no causal or universal dominance claim. | +| The supplement strengthens interpretation without moving the body claim. | A20--A34 challenger, dependence, tail-risk, multi-distribution, online, and external-replication diagnostics; A35 is the active frontier; A36--A40 audit the selected decision and its matched baseline. | Journal robustness tables, Figures 15--25, A35--A40. | Scope caveats in each table; A37--A39 are risk-profile audits and A40 changes no selector. | | The frozen PD binary is a faithful paper model. | E3/E4 T1 diagnostics show negligible seed-level discrimination movement and stable expanding-window validation. | `docs/refactor/SENSITIVITY_RUN_DESIGN_2026-06.md` and Appendix E summary. | Non-promoted diagnostics only; they do not replace the champion or become routine reproduction steps. | | The manuscript is reproducible from frozen evidence. | Tables, figures, Quarto pages, and status reports regenerate from frozen inputs. | Repository code, DVC metadata, rendered book/paper outputs. | Pre-push test and lint hooks, DVC status checks, and manifest validation before release. | @@ -631,9 +665,11 @@ The selected body point has an exact aggregate funded-set audit at | Weighted miscoverage `V` | `0.035350` | | Funded empirical coverage | `0.9427` | | `Gamma_CP` | `0.162616` | -| Endpoint budget upper | `0.245084` | -| Markov cap | `0.345084` | -| Exact violation | `0.000000` | +| `Gamma_int` | `0.089032` | +| `Gamma_res` | `0.073584` | +| Exact endpoint budget $B_u$ | `0.245084` | +| Exact Markov loss threshold | `0.345084` | +| Realized risk-tolerance excess | `0.000000` | A36 regenerates the row-level funded-set audit from the selected body allocation and closes the composition card for the live manuscript claim. The @@ -669,7 +705,7 @@ to `$179,529.98` at `LGD = 0.60`. These quantities profile the selected policy's tail exposure; they do not change the body selector, which is still the finite-grid return-bound point in A35. -| LGD | Repriced return | Realized CVaR95 | Decision-time CVaR95 | OCE theta5 realized | Markov cap | +| LGD | Repriced return | Realized CVaR95 | Decision-time CVaR95 | OCE theta5 realized | Markov threshold | |---:|---:|---:|---:|---:|---:| | `0.35` | `$188,367.48` | `0.205511` | `0.149497` | `-0.118852` | `0.345084` | | `0.45` | `$184,832.48` | `0.276211` | `0.218140` | `-0.075978` | `0.345084` | @@ -713,22 +749,52 @@ calibration data, conformal intervals, or the finite policy search. : A39. Fixed-allocation funded-loan contribution bootstrap from `reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.csv`. +## Matched Point-PD Decision Audit (A40) + +A40 isolates the effect of carrying conformal uncertainty into the decision. +It solves a point-PD two-stage LP on all 276,869 OOT candidates with the same +`$1M` budget, maximum concentration, $\tau=0.1715$, LGD `0.45`, solver, and +minimum-utilization/slack settings as the selected-policy lineage. The point +baseline uses calibrated $\hat p_i$ in its objective and risk constraint; +selected CRPTO uses its declared $q_i$. Neither optimization receives OOT labels. + +| Policy | Realized return | Return cost vs point | Funded | Weighted default / $V$ | $\Gamma_{CP}$ | $B_u$ | Markov threshold | +|---|---:|---:|---:|---:|---:|---:|---:| +| Point-PD two-stage LP | `$196,369.14` | `0.000%` | `225` | `0.118400` | `0.526736` | `0.680579` | `0.780579` | +| Selected CRPTO | `$184,832.48` | `5.875%` | `314` | `0.035350` | `0.162616` | `0.245084` | `0.345084` | + +: A40. Matched Lending Club point-PD baseline from +`reports/crpto/tables/crpto_tableA40_pool93_point_baseline.csv`. + +The realized return difference is `$11,536.66`. CRPTO reduces weighted +default/miscoverage by `0.08305` (8.305 percentage points) and the exact Markov +threshold by `0.435495` (43.55 percentage points). +Both allocations have zero realized risk-tolerance excess, but the point-PD +funded set misses the tightest Markov screen because +$0.1184>\sqrt{0.01}=0.10$. The row-level point allocation is stored outside the +model directory as a processed experimental artifact, while the compact JSON +records inputs, solver status, both certificates, and the claim boundary. A40 +is a matched frozen OOT comparison, not a significance test, causal estimate, +prospective trial, or claim that CRPTO dominates every point-PD portfolio. + ## Why A21--A34 Do Not Strengthen the Main Claim A21--A34 are designed to make the paper harder to over-read. A21 shows that a cluster-aware tightening is transparent but not sharper than Markov under the -observed exposure concentration. A23 shows where weighted, group-weighted, or -multi-distribution conformal methods would matter if CRPTO were recalibrated -under a new protocol. A24 shows that an online controller would have little to -correct on the frozen OOT vintages, but it is still a replay, not evidence from -a live stream. A25--A34 show that the same recipe clears useful gates on Prosper -and Freddie/Mendeley, that Freddie's full candidate universe has been solved, +observed exposure concentration. A23 shows where weighted, group-weighted, +non-exchangeable, or multi-distribution conformal methods would matter if CRPTO +were recalibrated under a new protocol [@farinhas2024nonexchangeable_crc]. A24 +shows that an online controller would have little to correct on the frozen OOT +vintages, but it is still a replay, not evidence from a live stream. A25--A34 +show that the same recipe clears useful gates on Prosper and Freddie/Mendeley, +that Freddie's full candidate universe has been solved, that subperiod/definition/segment sensitivities are documented, and that the economic cost of applying the frozen recipe is ordered by panel risk across the four frozen external applications. They still do not create a new exact funded-set certificate or theorem for every external portfolio: they verify candidate-pool exhaustiveness and economic -viability under a frozen recipe, not a new post-selection conformal guarantee. +viability under a frozen recipe, not a new post-selection conformal guarantee +[@hegazy2025valid_selection_conformal_sets]. Together, these evidence files protect the IJDS claim: the submitted result is an auditable post-hoc conformal robust credit-portfolio decision with an exact frozen Lending Club funded-set certificate and external economic replication @@ -741,25 +807,34 @@ The main manuscript positions CRPTO as an auditable post-hoc bridge, not as the only possible conformal decision framework. The table below clarifies the certificate landscape that motivated the journal package. -| Family | Certificate object | What CRPTO uses now | Future-work boundary | +| Family | Certificate object | What CRPTO uses now | Boundary for this submission | |---|---|---|---| +| Contextual optimization / prescriptive analytics | A learned or estimated context-to-decision map [@sadana2025contextual]. | CRPTO is a post-hoc credit instance: frozen PD, conformal endpoint, robust LP, exact audit. | End-to-end policy learning or stochastic-programming consistency would be a different protocol. | +| Profit/risk credit scoring | Profit, rejection, or multiobjective classifier metrics under predictive or parameter uncertainty [@xu2025profit_uncertainty_credit; @xu2024profit_risk_credit]. | Economic return and conformal premium are evaluated after the funded set is chosen. | Classifier-level profit uncertainty is not a funded-portfolio certificate. | | Split/Mondrian CP | Marginal or partitioned coverage of PD-scale intervals [@vovk2005; @bostrom2021; @gibbs2024]. | Upper conformal endpoint becomes the robust PD input. | Stronger conditional guarantees require new assumptions or diagnostics. | | Data-driven robust optimization | Feasibility against an uncertainty set [@bertsimas2004; @goldfarb2003robustportfolio; @delage2010dro; @bertsimas2018datadriven]. | Budgeted robust portfolio with exact funded-set check. | New robust objectives, DRO ambiguity sets, or portfolio-selection variants would be new research lanes. | | Conformal robustness control | Robustness probability or loss control in downstream decisions [@johnstone2021; @hu2026crc]. | Used as positioning language and audit inspiration. | Not re-promoted as a new CRPTO selector. | -| CROM/CREME/CREDO | Model or decision certificates for robust optimization [@bao2025croms; @zhou2025credo; @zhou2026creme]. | Used to motivate A20--A34 challenger and replication diagnostics. | Could become a CRPTO v2 certificate, but only with a new protocol. | +| Non-exchangeable CRC | Expected monotone-loss control under relevance weights or source shift [@farinhas2024nonexchangeable_crc]. | A23--A24 diagnose where such methods would matter. | Requires a new weighted/non-exchangeable calibration run. | +| Valid selected conformal sets | Coverage after choosing among multiple conformal sets [@hegazy2025valid_selection_conformal_sets]. | The current frontier discloses finite-grid denominators and exact audit results. | Stability, nested, or held-out selection must be predeclared before promotion. | +| Decision-calibrated prediction sets | Calibrate the uncertainty set by downstream reliability or robust risk [@zhou2026creme; @stratigakos2026decision_calibrated_sets; @chen2026polyhedral_conformal_ro; @wang2026optimal_decision_prediction_sets]. | CRPTO reports one audited credit decision and the return-bound frontier. | Learning set geometry or robustness levels is outside the submitted evidence. | +| CROM/CREME/CREDO | Model or decision certificates for robust optimization [@bao2025croms; @zhou2025credo; @zhou2026creme]. | Used to motivate A20--A34 challenger and replication diagnostics. | The current paper uses them for positioning, not as an implemented certificate. | | Decision-focused learning | Regret-aware training through the optimization loss [@elmachtoub2022; @liu2021riskbounds; @schutte2024robust]. | SPO+ is a comparator in A19. | End-to-end retraining would change the frozen predictive model. | +| OCE/CVaR risk-controlling prediction sets | Tail-oriented risk control for prediction sets [@huang2026oce_rcps]. | OCE/CVaR remain post-hoc tail summaries in A20--A22 and A37. | Promoting OCE/CVaR requires a new selector and exact funded-set audit. | -## Scientific Upgrade Map +## Single-Submission Boundary Map -Several natural extensions are strong paper ideas, but only some can be used in -the current submission without becoming new claims. The distinction below is the -operating rule for revision: use frozen diagnostics to sharpen the boundary, -and reserve promoted claims for separately tagged evidence. +Several neighboring methods are scientifically attractive, but the current +submission is strongest when they are used as boundaries around one promoted +certificate. The distinction below is the operating rule for revision: use +frozen diagnostics to sharpen the submitted claim, and do not promote any +method-changing variant without separately tagged evidence. -| Upgrade | Paper improvement available now | Why it is not promoted now | Evidence required for promotion | +| Adjacent path | Paper use available now | Why it is not promoted now | Evidence required to enter this paper as a promoted claim | |---|---|---|---| | Tail-aware selector | Use A20--A22 and A37 to show the selected decision's tail profile and the available return-tail trade-off. | The body selector is return-bound, not CVaR/OCE. | Predeclare CVaR/OCE objective or constraint, rerun the finite policy search under a new tag, and exact-audit the selected funded set. | | Prospective/nested selection | Use A3, A9, A35, and the declared finite-grid denominators to answer post-selection concerns. | The current frontier is a frozen retrospective audit, not a fully prospective clinical-trial-style protocol. | Freeze all selectors before a final untouched evaluation panel or a new dataset, then report the search/evaluation split as the main design. | +| Valid selected conformal sets | Cite the selection risk explicitly and report the finite-grid denominator. | Selecting the attractive policy among valid candidates is not itself a conformal theorem. | Add stability-based, nested, or independent recalibration for the selected set/policy. | +| Decision-calibrated robustness | Interpret A19/A35 as an empirical return-bound frontier. | The robustness level and uncertainty-set geometry were not learned against downstream regret/reliability. | Learn or inverse-calibrate the robustness level, then rerun the credit audit under a new tag. | | Multi-distribution validity | Use A23 to show where grade, vintage, and fine-cell coverage remain strong or thin. | The intervals were not calibrated by a multi-source or group-weighted objective. | Fit a new conformal layer targeting multi-distribution or group-weighted coverage, then repeat the funded-set audit. | | Online validity | Use A24 as an OOT vintage replay that documents whether ACI would have needed large corrections. | A replay over historical vintages is not a live sequential guarantee. | Run or simulate a predeclared sequential protocol with online alpha updates and decision-time logging. | | Decision-focused conformal learner | Use A19 to state the regret-auditability trade-off: SPO+ owns low regret, CRPTO owns auditable funded-set controls. | The PD model is frozen and not trained through the optimizer. | Train an end-to-end learner, calibrate its decision uncertainty, and require the same funded-set certificate as CRPTO. | @@ -775,6 +850,8 @@ purpose is to keep the strongest claims aligned with the available evidence. | Marginal split CP | Coverage over an exchangeable evaluation population. | OOT interval audit and paper-facing validation tables. | Does not imply profile-level conditional coverage. | | Mondrian/group CP | Coverage within declared partitions such as score deciles or grades. | Frozen Mondrian intervals and grade diagnostics. | Small grade x vintage cells can remain weak. | | Weighted / localized coverage | Coverage under known weights or local neighborhoods [@barber2023beyond; @guan2023localized; @jonkers2024wcps]. | A23 reports where reweighting/group focus would matter. | Not fitted as a new interval method. | +| Non-exchangeable CRC | Expected monotone-loss control under covariate/source relevance weights [@farinhas2024nonexchangeable_crc]. | A23--A24 reveal where source shift and temporal weighting would matter. | Not fitted as a new non-exchangeable calibration layer. | +| Post-selection conformal validity | Coverage preserved after selecting among multiple valid conformal sets [@hegazy2025valid_selection_conformal_sets]. | A35 reports the finite-grid frontier and exact checks transparently. | Audit evidence only; no stability-based selected-set theorem is claimed. | | Multi-distribution validity | Coverage across multiple source distributions [@liu2024multisource; @yang2026multidistribution; @bhattacharyya2026groupweighted]. | A23 worst-cell table is a read-only stress test. | Full MDCP would need a new calibration protocol. | | Online validity | Sequential alpha adaptation [@gibbs2021aci; @liu2026portfolio]. | A24 replays OOT vintages as a static online diagnostic. | Not evidence from a live stream. | | External economic replication | Frozen recipe transfer to different credit products. | A25--A34 report Prosper and Freddie/Mendeley scoring, conformal, LP, exhaustiveness, sensitivity gates, and price-of-robustness scaling. | Replication evidence, not a new universal guarantee. | @@ -791,6 +868,9 @@ decision. |---|---|---| | IJDS credit-risk graph learning [@das2023creditgraph]. | Shows that richer financial data structures can improve credit-rating prediction in an IJDS setting. | CRPTO keeps prediction quality as an input and moves the contribution to auditable portfolio decision control. | | Cost-aware classifier calibration [@yang2025costaware]. | Shows that miscalibration has asymmetric downstream decision costs. | CRPTO uses calibrated PD and conformal upper endpoints as a governance-visible decision input. | +| Profit/risk credit scoring under uncertainty [@xu2025profit_uncertainty_credit; @xu2024profit_risk_credit]. | Prices predictive and parameter uncertainty through profit, rejection, or multiobjective metrics. | CRPTO moves the uncertainty price to the funded portfolio through $\Gamma_{\mathrm{CP}}$ and exact $V(\alpha)$ audit. | +| Dynamic loan-portfolio profitability [@djeundje2025dynamic_loan_portfolio_profitability; @distaso2025business_cycle_losses]. | Models portfolio cash flows, LGD, and macroeconomic drivers in consumer credit. | CRPTO is static/OOT and certificate-focused; macro dynamics are outside this submitted claim. | +| Explanation stability in cost-sensitive credit scoring [@ballegeer2025explanation_stability]. | Cost-sensitive gains can reduce explanation stability under imbalance. | CRPTO keeps the predictive model frozen and makes the decision audit, not local explanations, the governance surface. | | Lending Club / fintech credit scoring [@jagtiani2019altdata; @albanesi2024credit; @zheng2026twostage]. | Measures predictive and scorecard behavior on platform or fintech lending data. | Uses the PD model as an auditable input to a decision certificate. | | P2P investment support [@guo2016p2p; @zhao2016p2pportfolio; @babaei2020p2p]. | Combines borrower-level prediction with portfolio-style investment recommendation. | Adds conformal uncertainty and exact alpha-safe funded-set validation. | | Profit scoring in P2P lending [@serrano2016profitscoring]. | Reframes loan selection around economic return rather than classification accuracy alone. | Adds a post-allocation risk certificate and finite-grid frontier evidence around the economic objective. | @@ -812,13 +892,14 @@ decision. ![A22 answers how a tail-constrained challenger would be documented: each point is the highest-return alpha01-safe policy admissible under a decision-time CVaR cap, but the active selector remains the return-bound point.](../reports/crpto/figures/crpto_fig18_tail_constrained_frontier.png){#fig-supp-tail-constrained width="88%" fig-alt="Upward line of realized return versus decision-time CVaR95 cap, with a high-return point and tightest tail cap marked."} -![A24 answers the online-control objection on the frozen panel: per-vintage and cumulative coverage stay above the 90% target while the Gibbs-Candes ACI target alpha_t barely drifts, so live control remains future protocol.](../reports/crpto/figures/crpto_fig19_online_coverage_aci.png){#fig-supp-online-aci width="88%" fig-alt="Per-vintage and cumulative coverage lines above a 90% target line across eleven OOT quarters, with the ACI target alpha_t on a secondary axis rising slightly from 0.10 to 0.12."} +![A24 answers the online-control objection on the frozen panel: per-vintage and cumulative coverage stay above the 90% target while the Gibbs-Candes ACI target alpha_t barely drifts, so live control remains outside the submitted claim.](../reports/crpto/figures/crpto_fig19_online_coverage_aci.png){#fig-supp-online-aci width="88%" fig-alt="Per-vintage and cumulative coverage lines above a 90% target line across eleven OOT quarters, with the ACI target alpha_t on a secondary axis rising slightly from 0.10 to 0.12."} -A19--A39 should be read as literature-aligned stress evidence. A19 places CRPTO +A19--A40 should be read as literature-aligned stress evidence. A19 places CRPTO against the regret-driven training tradition; A20--A22 translate tail risk and satisficing into finite-grid and tail-constrained audits without changing the body claim; A37--A39 regenerate the selected allocation's -tail-risk, concentration, and fixed-allocation bootstrap profile; and A23--A24 show where multi-distribution and online conformal work +tail-risk, concentration, and fixed-allocation bootstrap profile; A40 adds the +matched point-PD operating baseline; and A23--A24 show where multi-distribution and online conformal work would enter if the project moved from a frozen historical panel to a new protocol. A25--A34 add the external economic replication layer on Prosper and Freddie/Mendeley, including @@ -846,7 +927,7 @@ frozen champion. The governance boundary for the current submission is: -| Topic | Current submission | Future work only | +| Topic | Current submission | Outside submitted claim | |---|---|---| | Fairness | Proxy/intersectional audit on available data. | Direct protected-attribute validation if legally available. | | Causal automated decisions | Observational credit-risk decisions are reported as predictive/prescriptive certificates only. | Experimental or causal policy evaluation would require a separate design [@fernandezloria2025observational]. | @@ -959,7 +1040,7 @@ The active IJDS submission surfaces are: | Surface | Source | Role | |---|---|---| | Anonymous body | `paper/CRPTO_ijds.qmd` | 25-page IJDS-style manuscript source. | -| Online supplement | `paper/supplement_ijds.qmd` | Proofs, A3--A39, MRM/fairness, reproducibility. | +| Online supplement | `paper/supplement_ijds.qmd` | Proofs, A3--A40, MRM/fairness, reproducibility. | | Long companion | `book/` | Public companion after acceptance or journal-approved disclosure. | | Publication config | `configs/crpto_publication_targets.yaml` | Venue, template, anonymity, and pivot rules. | @@ -980,6 +1061,7 @@ boundary that prevents overclaiming. | The theorem is a Markov bound under weighted funded-set validity. | Theory. | Appendix A proof, Proposition A.1 sharpness, Proposition A.2 cluster sensitivity. | Assumption 1 is a modeling premise, not a universal conformal guarantee. | | The promoted Lending Club point is exactly auditable. | Results, exact certificate table. | Appendix C A35--A39, funded-set audit card, governance files. | Exact means file-backed accounting on the frozen selected allocation. | | The selected point is not a lucky singleton. | Finite-grid frontier table. | A35 finite-grid frontier plus terminal and consolidated governance denominators. | Finite declared grid, not continuous global optimality. | +| The return--risk trade-off has a matched baseline. | Matched point-PD baseline. | A40 point-PD audit with candidates and operating constraints held fixed. | Frozen OOT comparison, not causal or universal dominance evidence. | | External credit panels support recipe transfer. | Multi-dataset replication protocol and results. | A25--A34 external replication, exhaustiveness, intervals, subperiod and segment sensitivity. | External evidence is economic replication, not new exact certificates. | | Regret and auditability are different outputs. | Regret-auditability section. | A19 and Figure 15. | SPO+ owns the synthetic regret corner; CRPTO owns the audited funded-set corner. | | Tail and concentration evidence interpret the selected point. | Tail risk and distribution robustness. | A20--A24 and A37--A39. | Tail-risk, bootstrap and online/multi-distribution rows are diagnostics, not hidden selectors. | diff --git a/pyproject.toml b/pyproject.toml index 550b6d1..c4a24f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -263,9 +263,22 @@ module = [ "mlflow.*", "duckdb.*", "loguru.*", + "torch", + "torch.*", + "tabpfn", + "tabpfn.*", ] ignore_missing_imports = true +[[tool.mypy.overrides]] +module = [ + "scripts.run_spo_comparison", + "scripts.run_spo_real", + "scripts.run_crpto_vs_spo_stability", + "scripts.experiments.run_tabpfn_tabprep_full", +] +disable_error_code = ["import-not-found"] + # Strict mode for the fully-annotated modules. Legacy code stays under the # global laxer config. Promote modules to this list once they pass cleanly. # Modules in this list have been made fully typed enough for strict checks. diff --git a/reports/crpto/figures/crpto_fig13_alpha_gamma_funded_set.pdf b/reports/crpto/figures/crpto_fig13_alpha_gamma_funded_set.pdf index 97d681485f230a94f3c0d7623ac894cd6ccba5da..370e4a93e97b53adee29095f01c526a88328cc39 100644 GIT binary patch delta 24 fcmca|lkv(;#tku0C>3wbpQYW delta 24 gcmeyjlkwk9#tlMIoMwh521W)3CKj8eqaHH>0C+_RYybcN diff --git a/reports/crpto/figures/crpto_fig15_regret_auditability_frontier.pdf b/reports/crpto/figures/crpto_fig15_regret_auditability_frontier.pdf index 3d541b60e974e2fa8b8bbcb2eff9fe39e78e7ad1..7e7d9deafa1b0df2aed7ed4216930627a29f4e61 100644 GIT binary patch delta 24 gcmcb$p7GXt#tn9%oaP1=Muw)AMuwZ+Lnku<0CQRh7ytkO delta 24 gcmcb$p7GXt#tn9%oMr}=Mn;B)rbe6HLnku<0CO1#5dZ)H diff --git a/reports/crpto/figures/crpto_fig20_bound_claim_layers.pdf b/reports/crpto/figures/crpto_fig20_bound_claim_layers.pdf index bb4b0f04bf37faafceca287499a4f058b1ef053e..91625292a9f1e8a29b369958196e5131124e1fa8 100644 GIT binary patch delta 24 gcmaE{h4IA}#tmn}In50$j0{aJ4J|fb4WG;e0D_hYWB>pF delta 24 gcmaE{h4IA}#tmn}In4|#jf@NpO$|0*4WG;e0D>k7RR910 diff --git a/reports/crpto/figures/crpto_fig25_price_of_robustness_scaling.pdf b/reports/crpto/figures/crpto_fig25_price_of_robustness_scaling.pdf index 790c211ffa915152069f30ad9266c63f6ec30cb2..2c0bf99cf84d949dd4b0d80a8c596e98c57f42c8 100644 GIT binary patch delta 44060 zcmY&;Wl-K+)a~11#ogVDySqzqcXyZK65ORgad&r@q7Uv+oZ|4{?t1y|k3099WHM*X zT6S^o9NbBNmh}+4J^D@-Vkdh-8@Kzzb+x$ z_2|hT$Xzi#>bZ>eY`IQj3iX2%W%TsfGJntTrf)g%R5LYSlX>CEv$S?UPb@>N2?hKD zc!+pSCZth=qS_)8ZrKC&a6%j3grl}A`*NxaOmxj-_HOChoMa7w;pgvNO! zEqP_9mPqS`Ti|Dut=cb|&f4q-Rdu8TXtm}En@svtHz|bJVSqzVd;ioOci|^)2OrM*6JS0_!bW-=o@10n;S7Z=zKDv;_5A%Y{BhD zChIoCoYyQ*jgqkFeoEF#lW-k$z$cx8K)C8P`%E)?x=1hNFiO52AeDCK=5kWx6eiXkHSq42Y1#aWu$d~90LRX7CF1g0}80CACr z4iulLu`d}$fvIFCwwYEz-4>tfr{X(-w6AnN-E?8X9M%+i_}YV_A?~P`3>v!C?w5a6 zi@U|ZqF#QSYG989uw5Diti48F3rVe%mKeqLNL)>ucWahBt=)6TP#H%jc0D&krM=Z| z((G}NyWT`2p;}Gi6qusoL;J#>9L|^}F226GR551}Wes$AIaEt)K#RdwW{J1N!YW(R zSeO0!-Z?X#dzdXzUZ@(LiuGq5Lp~3a)+Uxf0n5hJ{7hLVhwfol|?+3Ys~+ z;`ykRZw47eiE~7a#ptYBG^$Ptn)C7DKDH33h8n;6Qm~!qYOZVGJE-98qy8$;{2{6Q z$CXi7!+4g(1x(t-&#TzxHYM~?2Y?TAr|a^q`1EUXoHo|0tXER$l?Tjq9Y^7dzRXI$ z3h0t+x6epcxJsI$(!5egSVcU8v$9BZ^~eqN_Vt`wvi&WwI?W5;r<$7?9oR@o zcPYKeqq9;IRfNE+$7{M7`D?K~D>2xyVc{jPwgMp&0H?DNGOnG{sU~?jKEh(A^2&wA zlT);>Ka8Sg&v3gii#h!@ z+V0Ld)x}qvzxl4$&)WnyO$QI&AlBT)Q`NOgF7B}r!ghm&IrB3jMGVbjf3YIYSWB7` z6r}7Zcxe3|Ps0>EfMBXrqGwVn*3E}%xy1_E0%XxwC24$|jMctSCqqDGe_iNjoTQZN z$S;9AiOr}Bx<`_;t-EQmI)c^i1BdvB@R(Kd#U(`D=It^gEjAs`jFd}!>?vo3li1yf zfXpuqPiSni)t>mo*xg{cr?CIdWU8`fvN(#6NLtcA9y%YgT0vGt9)|R`xBmS{72l5f z0|+~thJ~*tva5%@SB-V6X-00%lR&oevlVAT86D63Rek8DE6+F+L3A+znPE33iQ$2Q zm(7>GDIktq*mxUfM~6PkLNYxNDr=;Gp3p^j2wnAyxmS(_J^3n-Pe9{EzgJWVRV#wN z119UiVsU)FQJC9^ZW^^>Ns%Ns{Djvh;tX3Htl&a_z62 zyW9@zM$#)!Yz}&+Djv};cQhe0gapzRke(CoU$|Znnc&PF%v^uEm>b*w&&bil29AZ5 zgqh_34!Bs@lNP?Qp|WvulQ7AdJ6Qg-B4Op?W@bq`A0`7%yI?&r)bF``9n{^_@4Aay ze|mKAWuMb$7mrG!k(K`nr8FwAgT;a6lm10Pzx!284~McWZ zVM0_rNo6UKK`0Kkm|fnCTc7i@|Hgpsho}1spBxW|ySIOTjv2x4=R6D9|9$z7F6h76 zt7f-5qac?6KnmNa#?M!E_wQL{4xFhs!+$G0{>Qh@n+~WwkjvSDM_EJVP@$L9p?#D~U!AD`n`oUj6l@KC30#6G@~(UBezMhg`UBs_^(BfJnpwZ6%=GGd zpW(b*EAEVgAsSqB`X4srrPq92x3%+2a7Sd>1f13@X~OUv@#ajc0TR(|)$kLg3vu z7nVOZPwRBN6ejkA&mN|$z|;CZJpPB*kyb$V=9f!MLOP8aS+oD91`7w9x(aQLrGRbw z)~_Kd{{b;Q(7YZhI{YW&y63s3Rj)ekqr(3#1aAc;2I2+JJi+1WVqEEr$xQu484IOI zVl>QpFl_Rd0~HsB9QxiZdokn2Jp1c{Wg(ivUy+t10u8sz6t`W~)36@V8lX%Yj=3tx zy3D`hUauLWT9VSnp?f9g?Rb(tb-6mM{>Bs97zdCw%N_7)s3lYa`29f^ap>)saW*mdG?eNHAz`>QRC^a$6^7jx0>XA;krT}tbHZaMQ3xN6oHmA* z6UiE_!^K0@&4YEu()<00V8PY}$tYG+x;a-vPMo<${rvXqA*>x!O z@K}@z{jypyZBg4cT0w2lDP-?oQz!*P=x_{R&muJYTtJoE*C3d>lx^Na;aAF^WHnO0 z#2xwYS9n^Ho(xr@T}T?#3cYuA(9RIAU#~Qspmq==hPNf#(45KjVwVS;0Q4&PE*P#p zUQp(CYD6pZI>Nt54`X<}U{5-|XixkB!vVL5)`7doo_@l*FINhWm^=DT5G@_}4xl+7 zoOftWvJLo5l#AfZx+zCohtN#a);@-aHPp?(7i0LHi!BWh%oZmI26Z!x@X{8P?Zmeu z=0v?S5aHNI?u5Nl=)|=HFt)+mc(A<+tVi97+h0J#I78baAc#7)D`*0R)05yz-IL52!ys6Qta}Q@C-4y(fg-Lq^O725im(|ZMBa_4NNo`2f%A@; zBjSslBjJmj^MfF4_mY1H*7M7ipC{@S1ov-GSV1#SGy~DsZw8XDY1`CJ6wcoSLklBH zw=hA|IQ~erlm;PcOj~Xskw^HQO`Lba4PkynN3wT_I8uLDN78rn4TY}%4Ey*Z4B1|x z-=iKGx>7xfx`THkHrZaWvxHy2Y$)=>2(c0bA)|(GQRb%EASrq;k`1tMoTpBYkWS;d-vwN$EQwsERzo8Eun8n-mpEXzO9`8oVkXO{I&x_gWKsq5u_gZR{arU-C; zeu1$S>BKJ`Xf{z1;qu%E#=rNow5{#nk?&O=ATxc>v*i9R(rr9BTHq@|kz z{YQ>(xjVO}G4t`CriZxSaCZMcAU|~E$f^J&IX`E{%x`|48+!Ma>oc~G`wemE2r$li z=iR+s!Y%EY;(i0%hK_1k6o6RgXWN*0L({Bx+TB~tPi~C+P5qN%RscespXolS&oaP$ zQlEJ~%YZp_B=zabJj?&HNXnn!)3I@ue~9xl>d+Arn*xyT{5<{{^jRR*-P?hfd1N*P zp$YCc%Auq1PpL_k|7V#@)V$IN9zk_NSK-?l-8W{ z6*mNFh~9R#dFTa1&Unp|1rI3o$+Aq9=2?8&DN~LWUn@ty(I(4as8_JrH|tc#oS7>F zvGa~fw(JEl<_#n{;9d(BV_kS8*6{IetLsW- z4cM@0U(RV)*)%WN)E^??RSp&{=7344asI33v|_euZb3~wcG6O%=ZxO4SUUO&++$?U z7B#L^b=(l{g|i8l!tee)CLo_m%CK=QTCSGlYmskVBczJn*p{9>ns!x6OM>XgiyKC9Ia=H}=A+zWRssALgIx0dS6Lm1eOc_HBqX#_0rF77wz43IMlQKNm^-A4$u zZ*iG$^0MDn>a`G>?)%$eQ@3)=Pn5}E3BugR4sp43v9#nPP+#HUA?hIq`1tmma==@O zH0ZtCX*)MO1W)8ORb}7#h?xux{B3LmEFM`^Zf+V?{(*V4nD0cL+Do9nGHPx|bQ=BC zb|5tfR1PJY4oyQHxy8WR_v!+-v(BY6FE9EHXg1U#8ocJE&yCj6)NJ6tGT!mr`3K^QY(;mZjH5J=edU?W zJ(LHb0n)GUJ$~daIjR;8TFniWji!V9<7=tr7CQt9)w!Qt7w*`dlGhPJ-_2 z-?iEps6v9#srF`nIx$*Iyva!FPQ+{93Du(CwAS0a=Ps$$9TeQy(2iz|tz>}qYD31l z^Z@-CVm>|m+u_lw=nNKrSNpm_VtlhJ8v5?g!8;se$hyFusrI5pVsiWU4z0HgN&z?f zX3tK8>JlogMx`Vm75)z%gJd&KG*1xW9}r7Q-8XPq<>9{YqG_o~dEaVB8lBb;ACz4& zH&<*>2BS?)r5nsw2CsG&6*eJmCg?tA7Sj;*K)n03G%fU`Wz*LmM^;8jJzO4n;F31j zgJy}J@Imkg26F_v;FcN7H1fyw)wJD;q$yXKuJ+=Rr?6ZgrQ<1w7px*Ae}B8a6>r~D z_EcK@C5nN;Gad)K!ttbh7-lm>b(wh~etL(qiG6qk+BYvZE;x>crz5nFH)GyCU8n>p z4xaI-y$TVVs+k%Z+YtBghrBvc-gA}?_#%Cg00C67-j~weoU?&(*#0=?pSt7m)yZg{qBkEdN|tA$(Q7~yF^H?hwg0f;48SxXCA;AXrI zuLCQn+63N7AZ6$;hrK`v+yvL`J&XDX#_fYIV%vLPACrF$n+JsyLV7neUa^Xc#!TI- z96weh$i<~BK$@eiV0pNnUOzFllWJ-(h>%JMnj7$&3326XEiU+vS_ma?ypyJ;mtPLl z!DORQ^2R*<4@Op`h;pcTHZhQmvgN3$=tXXH<-lI#_wFCNfi)AC$J1e}2HlesHBiJG zpwh*xy}`_}>@-_t0q85PjjVIZy=c>F8@RXFQAj32ac{GbhOlw4L=<>jRas;N604U9 zY5}cH_Fo-1PE99t<|jJaQ_NyqZ%#G7_oQb1VUuS+Is6myYh(`)(Mwbg&_=pi8kVz1 zPh2j5g?#tmjG|VU*P!QKvNMybj~bbtH}y|>*M9gZ!0)d0%@uLp;_lYUUqd6_Y++uN z_;%@`>24yHx&sXLR$?^$r90j6{HV;9uMv2?0`>pnl0#A_sNO5Vh-A47g?LW)OGj^03to&fa}AfYLsyw5DVIX_f^Tm~ z>0s=9#tV1VDPLS*p}Id|AvRQj9=5RqyS|3tfa$Im)AZ?a_{WEkfHDaK9m% zPh+EDP3(vC_cnlqt?S~`_K5@l3Zd?j7-|LV-6UpX$$eFeNHU?>B+S|9Yye^Xn8*&_a9nl zuH1l>D=BZ5mPb-g_26b$W~koZ82kldRpUd$5KBKN#(yO8qgU*tn#Rnnmb5KUdJ$M2 za-u^z>>kFy94>J5m-kMo+pPf=gc8o--hcDxr;(_qCS^o)yJrI|Ec^-3k4F)ZV8$Yw z5weOOxxfubUoSr9R}DweFN) z^xs%EZA0H@zTWw_{B`}8)-RJLt;*n8XeQwnp#i!-MUP{MzipAtcjW<28_5!@=o>p5 z*q9c7mPCOBagJg@+)}t$bA9Mz!!qtANu_Y|pd)pqiuI;up&@yE<1w?B$foj_i6F{M zW2Ci=Q*McK*aZAc|Bij8NNs$^oU@_G$pO<69f z=O+R~_2yV1So>F+(ms1CgYlF=;XYz!(2D2g|>e@@s<}d*HJuYW=b2q-wU{dGK zavwXVX3-RS8-SG*?S)x;@*oY%fd~aUg~;{V&Sc&x>45~`3xHK;!E}2x8R|IzejM%% z8d#gYz^7SRa%9F9y_0>n*c}BYLyA^_hV;62Hx765f<{ebj$t zJOv}!If~N@_k8^%zfPSNruD>-eJ}&Ke!$cwXcLJOf$OTrVmlb8#N?50LUG{IUvA!M zP5RjY!TJesl)S#^Ya$%zlmjGht2t8F%rC)A7>wMD;+r zF3eY~Y5`SC`vwJ~9;}n9L0;SXDV^svK7UFQc@cwdt zs6Qf$eP8MJ`FH~rT(~CH<_pmNf_kwz!5AQf+rdNIS@_r2?wGHeBJZMFjKLKnl=t)0l319^dCQOtp(sX4pkczbU04R_c-KTL_(1=U}}2YH)eIm12|L)ZVR z+*w<~_lHpw>t2Tvp4YUyk+0^S#-cbDx`DQLe zm@XSHGe)QP{NtA#9Jq2T52aRo0i#y(X5EQ~@>S50lk(l3ahM0&`bb@rahOS3(T7Fi zdwkMuPe#!+=vyk(f|e8CKtMtD;zji$m&QqB<((%N45aY%3QiQpC>VjsEUmH#@^%yE z_55Vbe{XGn*-us@u+Y1u? zlDms=sznF<^pR_TZ|=Ms=wA1F{Oj}LQVS}5hjv*}y*M5hh-~jUAO9E8XYbd3hGTaQ zKZ|6rsF}W7$(c0xt8cdBh7>fLBS+-51KH)u7q*)ed-CjJU8(aNz+HX>AJlxRDXC*p?0IzI0a@X~Va%}nm_{`ThP6lGNq z_+1G@YaOJ#mncMZ5HYcR9;y8VU1NOAjn>F6JkaD^KbsH+wEb@5*1)ok7?p3#^nq0*lpO?pAU41coMx7Dqam&~R%~%o zFE6R_RMMMH?%y?kK(9(NyG-$r3%My=v@K^Ns%E~(68Zh*tD)2w*@=W4!Z+DHiY|iC zqi!vzx&Lno5D88>fY_t0X*@T{P^_u2=g~!+Ye%YrPG-+rcqYx(2>v-YT_JC=pky$A zV&gMEpOlgvJpeVL#lT{MjncrS=`8?4#-b#rvm)nfEd^wz#!-Zq81yc~R?f4P+K;Je-8b(1=S5}Vxv zjlDR81!?>C#})jMx%1TI=}^~#eXt~Nd!AiOUv-vQIZ+KiRc(K?+1_2UZ8^Q_d9TfJ z@-aCHEY4>x%vhe;J#q~qHhEI}X+ZqXzNlbM?V~X;xGOED>ROAit7F3`6ZV#+5xiD_ zcsAhdt~#vMA~gh=)$@}W?w(6>l z?l*nps^zNRQ?OI8lcO1pVp^zrs5%MFCgC$Ub_g669Ooz%LF)MxRTYGyB6TEcztro* zE}{HL4O1M$>oVF?GAssf&Gr*y9VGUj$0n24Xdl!n5?4~F=i`f9UFS55Y85xLx)}gJ z#SUqBM(d4QHHRM`KD<46lnGU$nwsMsop@reGkX;aKS}87J#g)hk=$A{xXRy-%XE~v z%GK!}?_Mo(9g7*SItLkc+?D@Gtdlw0uOb2VttnjAZ>wb*vz*rIf|38l*;9J+N>yrhVkS{6 zJ~w6P5Kk{W|8=R3S1|lx=2leF&P#Yu=r5;~It@YhO!7>2t2$e`H62l#|AF!*Gz;%U zxKleT6wsY}aOvf1knL*CkR6=z&0K}raG98ahCbSq+e4W(iI!hR|-u)2M;`&4^HkHm)y|SzWS?8xGZbTGeXcnDy71u27esjFSPST)QQ?I7=c?}vQL#c0Oqg4-|dVocN}CXBV?Q{ zS(k5Mwup-Pfe}EEf9kkTM9m4g3o2C@+ZU30X3intAp+_`6^78tIE7k)l(RQB5<=?- z1ji)|xAf|$+V|c^NJeZh8Du6p#8yBRXdA>L_l6}%h8t;+7EmAG7vm0qyxTD4FUGe9 zLGib>$wox{;){u@W$Z0|RQXzmGW5YLb;$3cc+8woHEM*a7h=s|4bK4X5#i4+3Hl>B zEiC+%!cu?i@~$Ax*7XzKg8QWy4r2hkF=L7Wq9ZCD?jT*{1gH%pkoJP&J1Ws;=>_|O z8W_n8;}G`Z%()SeR0ix?$-q`PbY6OgzRJ5Hye$4#xlMaGlyd=U>D>MgS3=SmNzIps zz?f5RxM8vmB6ky6N(RE36k<>D6-oVP8*M^j9i^ZQ1qn<>efZjz;$uv$G{(sm`r}Xk zG)G#iu)TZ5-bdop|rZqm4h_X8-2M=9JKM5B*Qr&X87hbahwt&Z72# zROFsSD`~yyzl^9a$*}E82!+c2So!P>1MhBI3sAP(c!xlsu9AU6@LAc&@V1QSm-nw1 z{n%|^g}*L?iyv?Izq{d&FnInGb59aVupEKk^Seg;)AqBgp-rVGs&f0deQ+JR%K;qc z;^>rnP2?}wV8vyi`}Fpz>v{Z5R8&1;337pIL2QSV=0^s}WRl^tv(tCWGV{Yf#yARu zIRd`m%iJ(PNM=G>FY*mpglNxH(XrQHBbc=fqE<>7?O~wt60K|W`I&M%0*-P+Cg?k2 zdz+2A8GnTW`Y47As=@qKhK$8FN9qLmk4qiJj?v~lUKfG(^!2`)PG>v55ppvjgZPAC z&$mzlYT~5nkyHXWmf=G7@HV-c0J`CXMW@|%$#WB+3(J-6@_4JGjiXHz+4*h#%Q@o4 zSokkT@OAd9z$@`%>EqJV1(d=FCR~WDaaf&Jn*ujk7a0M%ztWSMTLooBbw#Oos}{Wi zHWT^<)Tgog+6I((qr^IRnlYFDshsP&=p%D#>JeeT2h)+k1R!T)&RE zP06@`SVyqqwxK!$S*e$hSLW^Ik0pwpcVaQBWN zs+}30N0b?CYvOPlrY~Yf#i3d)Eax`wLTOoV4=m<7FKe&BfZ?!V>LXK`l0D_Hx^BB> zFH_c5l@o!-9pbwF@}ao{O)G3hsL29y9R?V9LtVh+-ZX8;mqGgIsKXT830@bPwd+FH z-epdwFmpO-IXj^Kjrj5NW2L4YTn?2{p|t-|jr2m+UiQrN z#e98{Ax`|eAF%?=)?2wJf z_zT+KR2^luj9|{wuYdco3iq2aYR${`tA+nqAvV7I2R_d*DU+W_>VkFt{&MK8gsEQ9 zojLD8=P2w{0#@TVcOc5D&mm9Twa4WOS} z7i#m*t~k4bJ63$02%0O=%X0mE9qO#E+!BXpEv&Y>CE?wRk8N1gUp2gC-t~fQ3K$oZ zEbO@MwBb~Smd~Ae^{3l^@Xt1-0s33heaA88+lq}14UVJx*QZ|AUYcOS31sla#79wH zg^?xD#IA##jhBtDkNReQTXOhkRzEKYHWEDeRAJs*_@0hbn3thPyoO?}5ri6XxlX>L z@Q=JiN|eu)IFbnE;RGoSDdZkZ6mjg3#iXz!}oBV3HpHrFO2N#McBE)4Xnf<$lO1Af`@!dN-$IYa5c!+>g<- zdbSyx;R-Hs)Zt?ZsGwD)Gzb<8E|EommJbHm(J+<1hrN#Q&d(PXOl8m|BdNPNf5$@# z{q3a9&83FR8VX?y^U}`EbS%1|G+U;^q-Gg&mw5ecXvoZT!luQGRDb^_QH^1U-zEk~ z#=!8FQO7aciChRF!bXje%`Tdcbg-2iHp;|g2|>Slg+N% z%6t2}(8tJQW+{J%&Cbx!Q0#7DU)SqrUs;kJPl@40v+_6)z4d)Np3Q?vf0eKCG%8RY zU8+<&s`T`2_jf`5xaFC_`paWgcnZVYKuqG?v(Q646P`f*MI`jIqKVP!C4~b}eT-En zDy9;4&Mstj;9tCEb-?aXIqAfe{{b&u=)e4Hg95Fd&wRCb>RZjt7D|mBr5d6JVq8>1 z?r`79fu%j;EovK{l(J{HN zWe~q!{42;-(H%`!(Ei~>NzaKcO<7!wmF4(*zZ2V`+wpOvRnsvO&55H*ongdU>*v}f z(>v?y4s$!d+5?aF{e+>30H1a)Bk_jBc^EihKBZC6yv2zjuJo*Hb@l+@J>FS>q!3{q z!&-J!{;8ASZ*{ZMVc*4|;DS*-%{@GVt3UNZWR_X%Hl{@9kkpry;ces`NHgUsqLN^>;{v7@%Z%y5tW5=V!I z;{}@Ir~>6XIxzit@mn{*;hb0{Y%$-Dv*?KYy&y&NudGC*PC0FQA=TPnpTAk&!RHN? zhXn5K5@bz|g$3t=DB7BJ*!gThvD4H_d1_#=rl!y!bB}O7fXz&EMe~tn1Di$yv5~KM zoJsexD}Jh4ZpS`zqPhzz)B0PEqS2p)<^b=Aocj&lm~E1CLIe#kM9xgeQs06jq_Dmv zzM%lyT#BmNg1J! z_BA=?Pc-RG(+{sl*vXYzXAK%O%Y

!G(k^VT(o@Dd^1xJ@7n*FnY}QD+djj+-C20u z;>_ZN*1e<{R7rdTQnN1Xh-{ODC?0v=MaEb+BMg(VZ)j9;wbxvS6qli*gU%Tt8^jwc zyB(EIx=ViWE>uq3a%qD$3~l9G;}Z2P+cDGeSF*%x79N!%WK?gVn}yxdvdc#6rSre{ zX@)SCMX-NqY6;=RL$#*g<8pPwLz`oIn? zHr6QHGqHDQoHsItqZ;uIyRO4Sa?hs~2;w+B-zUXJ4^Pmh(}D~4 z;vjN=-$)KOn~xse-=8YgXAYA{40|)-=31UgFf6GJuivXJqV54vcqQOMJ~p7Bpk(lJ zESig5xK=tcCZJ~!tY1xkK1JBks?kjVQ-_`1(g?Lx2dj7hf0T2)Ntvppggo~>I@Y@% z<|@i;Ogp()i{ru%POF7=HL7a*{_dtAeGWom!xb*B1mZ3giofe_40C3doy1snPE%7l zYct3dfKN5p>N5C@)rQ9*f~;NzqhTZyr1Pe1#N+ToGv6}47(B1Sn`*BrM z@Jvf~E$VWpww+7Uu`Y4<&I1(&B@PhNA-4;FJ44|z~e zyQ_#*fl#YFG^7`h2osb5K+_f=X-1a#rXavTJdgFzWEXRAClMc#K_15a>*IdH;tdJdk2Iwo@3LXNv1#iHvWQT8?-}?nxpYXpu0Y_y)W|B zlwk{f{KcE@MKVNv<}~TH-2FS@c&3vEU8i;u9YfUa5uAw-)`@+l&pE56C-ABZHO48S^W560bz9i^>=9bq#!&U6#T{w`Z9XOp-8+PkFLO^j`B$ z3CpjN1@Y#D|5(mn;R3DlRrOKFfK}o1R@4-J=Zv_fO7JM<15RL5wnE z_5%zzW&`i5{(Ml4NZDMzu_aEaoXN;O_q46qN4h>p^ZtCHfsXenU7rbDjva5FUN?r{ zZOljLd0!=RnA;#>yMf$`8F7kEyr?eZ^4ajM;JAR?ktCYZgq=eR89E8SxGJIfe6CeO zMLo(Y7UvgW)oW@neEC}Cj{SD?^RacLoOSaZIhVx7CA!I6C>ON6M`agbMxVJnv400*$NjM z^fMK~^cwbuC>6a-AQ|1s64g%=#7-<77kjY^gpwIGqdrf~=)Q$RnHWhpZ-$b=1?Z(* zfKcbxBqLjl>fAF-Xo1IDvI6F{BPY1(IU*~Fu^pDF^xz)%{+VgT2RJ4tVL8epX9Xst zX{pLfj=;^3a1{(;4&AAo#cbZJD8~Dm_`~P`*A}n#T%k16)SdBu>E2aG1{y z4Vz@h%zm5?JzcxywfTbX_9N}UX<&a>EMplexuW|=*{SXYl#H`w)KrHo{2dj+9*Qo= zny7u-x_aP5ihqG|9)Zgc6fIt^XQyWNWp<8Fe>_*tVe!%|P+nNqO1ri!XXpBNOGiX4 zPI&!tHf!;35@%EEd+F5Ri7>YE400V_SA(;|?Lb@H8sdoZRL7oa^kAk^LQzpeYc<7` zjfwaQWBmI3metTr>6@g-YWa$BqAy2~0ZAG?cKl>+=)828zc?0zwb8mL*n>3p&~tFd zA$(d(XF+B6!Rf`8X{l)jNo$>JeH=EC!9qC?zmya(Ycc53P5Du-@Q#17NrR9zj@(Y@ zLotM@B7-0DZ(0){3Pp9-)pM~_cfOWde4&==aF|MT{F#`mtY5Mjc=(fneYejILP;!& z2k$iDPD#y$#1G}e^_S!?-CtDpYlqtKg)_vP`j2;biYUM6NhJQJdf1!FsmXj&my2Ak z8N+==`l?N{M1xbeG?5{QM9YPPk|uaRu1=}xM0wQcejtx6)9TCO**-SHfs{u#wi#CE zve)dm_ebZ&O|*FdJg&e%?htd?2SF8v=zMKoF-%~N_xQfw=3F*nmB{rA4=<_q{TL8E z5;4UW$6X&xe)dMHfW>Oae`KjDb{-!mwq3OE1zj06Mq4#Kf22(^q2a)Q%mkxuKgcg zEtG+Pr0#LR*VD>f?MLXC?rf2j(jr-8& zD&Ejld(FvrAee{sM={w@Z?1VhBX*Lk@x-3f++WIw*XPsr#lR{QY zi0Hi43+&8La(o6?Rc>i)RjD++588bf3q`WSeiDq@`rd&fK-(%WK=Tsxq%oo3aMz6< zFg706qbgvz$>nas%7nWE-{xz_83@hEwaT_VI_R`wj26K#<-GqQHqE5L;PIO`gr^@$x&;C3B3aE_J$v;_`-^m&oT{bPFr>6Fi-@(P zn5titk=((&Aa^-{6{7t3w#Ro8i-H#Gv+iyFU^4-*AmO`LyQmG>!*$>LljB;5zrh*U zVsN;U66-y~H7(bE+Z16r0&ToCO)KX;>3vmJ(C$tqwpK>v`tcj{P6V%EC~hQ^O^K$y zj&)OS^V&7lA0uUXCbJUlEQ3^5Yd~Ljf=GPeRgeQyk~I?;6H>Wte zR$V+?3P$kT!?iDwS2HN$MDQ!egrlYH@ljD%hI7!@h-_k7vnsDX0kYkEyiF?}m2j@G zfM?c0*Z%r>)GZ?gu7%nHkA)-BJI5cSDkFlTll;^yKv`NjxZg#PWvbatc6+i1h4ycv zvch|pLj2859~>?`o6^FD!=_NQGQ{JUfK$h}NG&45k5;u#c-fbiM3F_4s)h#gZtw!; zAD3zW%5d6vH*r|(X!|bmxJ->-x^%UV`_#Omwv3*mxO=!0kVYNX9aW`tHg?{2Izq8s za-YP4I4)zDaKxTezuZ=Kw=ibNExfk)UfwEuVrQ;{f?{nNc6W1pX?~X@<1w3F33xQ+Qx{*uq z`Lh*Odg=(47QyiTfB+TF9cVnv2G)i zQ@a#@{{4m?bHc^sFDM0jO1&h315*Xzn~30iqq zu@bl=q>Qllzv*jScU| z!-OeArLvt z7yl3b6#f_=P+&h4#dbc^L-YNIYC#3j0gu@9Q2d8_<0*$8ZCDV3J5-I6hfn^jw3d-g zVx=xA1*DKedwYTYrBftj!d>$j9==-kW}4xM;-xaB;ZQ$HMm_De$03010$kPlfwD=K zW!&o&Wu&z!oj%ww2Y##=sajDlEiL`KBVO|WJ^Y8I7F@Hu=AQkR>6lr;uqQ|tX1qUX zi6sW_<3^jyei<=lwb?qTdm%5iVf}vL3P>D4b)8n>Si2Z$TU1xn04E2K zE0G=siWb7b#rbbL&_4~hH~?J#jF$@l`Zo^1%lj`|0N|a-^S^U_4}j-?aFG8h02c=* z563@%TmVkae;NY;+yKx&aPPFDcka;tb^-)&a{n`EZa(gJdeZ;Lz555emkG%IPshOb zp#K~A9~?Iy2=GrQAZ{+uKb?TM_;~*Hgt$5X6*P#88_f6b?0ElgY5uD+@1bz~%P|*- h1NdK={3k#Vkc*cmkr)n292p4Y1|iYXiYrPW{U6gyYg+&S delta 44604 zcmY(IRZtz#mWD$J?!n#N{ouhNSa5fDcj*Lo4esvl9ymAzcXxM(Aj7>cGgDpFwg3L= zU()@udhfE}FDKz&Y6u`8I1)%8TmWr*3~r391J6_D8ah(SJWx|ib&cJn6B34-SDyrI**K|N zU!iRvvOKI+ZFo2xg(tA&zz8c4_V@)=ML@G_=+2fp?Gt@G?)u*6aWyb9~xv-mdr>D(-0kV_x|Li zF)_+VbIATJuNP!-pyDhaj01?)QD#DFu+1IMFIU%eyjFCuL{?xP6k0D73kHVp&t_kh z_n|{+%5#E;Xe$j%T*9fdbml^+?|Kjz^FW^2x`(@nX6LG9@rf%TiPYV5-C47P@9eHx z)RqDFIJ!96IV{!OSb+uj0wyWndh%_vgd3`_r$OP$GtlqbyboTfSPBMmCxKPKS5V-kwL$5hqNY zjLbLjU!u{WZ{)jY82dgx1LDR!Y&xyPZV4AaxKDFjo|G7l9ym>uuYi@kT$ATL=Yb}* zlo%R4ZfL6-FSn#DV+UTk2cNJr)^=X71|R;2-MLkJsl56JSh_0fZoXHgZEj;=Z=8#U zc}YS{j_*p*7248y#S$$~-W*UGuJyudrI<~@e#s1>wTxs_Q*7{iDtmpIvFq$grxkGo zg+@V{LpBjw7FI+z##MMOD@kex-d{dqs9su9+P2F!thlw~PXS!<^H*F+T>4oy?#z$k zMXtyVsj7r~D1Xv+_$nS$>bQPt*QtF!(ZrY59#4a64X>cO6&JEnyf}?iqdW1RqO^u= zp|mb+$ZCvh@SIn1XLfnO%4~7N;ATF^wobYZ+@r+Jggw-W($tN#E0} zQ+SRQO^xADH%@09Y>Kc+;G-c%dX@X`=5W%$m0*n+a{#s4%O*TfS{YxArJ){kGn zR`RM+glcv2S0Y|O#&@yvTT9A(zHqdzlempS-@wC(phwZL1MXLcFEMU<`5RHcN#->R z4X~UZ{CgHvNq8K)#>Ex#E`~33}k#VkxDZLOE6xt9jxf3SEEGcyz6ZIiS;X+Q&15!zINn!Odz1$(wIyk;2SMWO?SGd0 zO+Xs<9X$P8VH@4U&rwblnmQ7rBFoG<2V*FO`oXKIg$~?TrOcR?(U_BTK?@8esnr5l zIHJbc=0+Eb?+tjhs`l$ii0MXx-hTuh=0apw%LMKzCEh#rQCxEhRw*$*iW5=UOpXJ9m2IV`{9>>Zki$LobEi;m1N18;%-SHUy2uD^No~9=o%0v z`0I6hp(F3y)HNRCoV9o8ne6+>?|n%j^u#l2xe57l2e0P?SKU2+=6-__ps(i8yRFu7 zxv|R5^|~GIX4Dl%G$fK5#XBB9mrq^?e=BqXCs|Kce&1#QV@#U|K2h|qzmA^Xi#wGu zVGCuOsSGeI{E2#~Nf-_l55!ZAE*Q?)0ryDMEzG zDyI040&&O@+E*$xtzjRanobg)$(v|pCGf|Q4IV)Q_|a{>7dj$MC$g`K6w1yRA?|X+ z0Mo1I>h`B?G=#=XZAFuXJCrVOM`Azl` zO1v`Qy7fSY9Sb<2I^q5coZv{t@!RAL#C@~Fk|2>hq7A%Ebi5>7c>kQ5%Z&(cYG>l? z;$&)Q`=6D)krg~E8wm@^e-|Kj_5?#%c2o{lE)r&0Q#*4P3lcVN5Ld$H06B2l0ppQA z_r&d!ZEkfEXTm5`N2Hm=zl&MD7#&7D=$3?TGgOBzo00jGw26H&3YWGI;_E$)oXAoj zR~`G0&y2x!q!f`NDR8I~pPDW4X&C#wE}VBO-d8T#*X@CYy9-YrKXW1T^={{E^5Jiv z{`?~Ob`=;raFz27>d#I;&Pf_DIn96O=P@3%74|8^vbO%qN>><`xS*Y$4KXf}hDn7aBEQg?Ji6oiIP#h<-Y) zZC6oZ>(1_{L&vVe8CYnu*^6U)kOtnBrs!)L-ddd%}oc9RB2yNv@%y4tLt!V`xXG(yiH($?qd zORT+zE1NgV^`UFlp&hp+gSjc6%k&Az3B`wq>UO~w%e^c7#2eQG zgC7&u$^lgbcy@uB@GlM5+4i&t*|I!_Mhr#%D{!2bR6~EZIxH?_PVdkv)cb1wBqu~2 z&V)ULgG4Ceot_wLQtHv;b|~~P#Gj$3k`0P@TfmeHe8!$-Z;`2yiw#Ah50H&adF?5uuJfq@gG*lV(H zR4EkSuBlxHjlCe-lrIJT9tt5;rk4BTx!4|NQL6qzdZlWJ(%8>VnHYmHAT261B%tmJ z+)BUfTo4IN+A#zC@tGj`B7<35oa7(sks%aH&^)so0)I-rjDui>j^lG6b)9ey`hHp% zSbQe#Hbh1bYp`LC0?*S{)8PuaU1c9 z5_`nIe-SDmhsH7ciNXr&Cx&??Y2*5d72QA7@CRBw>N^C021_bIiq4vDh-^X${L~30 z*%!uOgd$o z#_cbYsMKsg#uwLNA+QNf$l{l~P~{v5M>-CwkWCw$m_mrIM?CP+ zWFQE#h@v|vl?Brg3#?`#VH1GWx8xvyjh_O(JPo6^DX+PUTTmg!`e2aism2b5063u2Xt z2c`?@+vhg41rk5_&dC4|&Wdjx&}w9Q2<7o1o4guPEm%y#FO)jr9s=9KEf7qiIxy-a z_pCYqiaM3hFZ?aKzVuD(P;U0sPo2XthU7X?9tb)C9`xXqO|8C_t+riB2bc=+4s^dT z4{V*7!t-E!mOG;0hhE-3{T{Erf146xH`c-Fn^b*&9oVe`VRu6maPcNuU-6b=-v-!tSg3~> zM-WmQ4Q30j&wQ(S*nN{H7zo@T6-06D!$#c*#?B?aqFjuAWIO%l6SP%c!~TTT0o@Lu z>W5IHJ&1h1LR!r9LBH=I>U%}p2rNe3@GnmIw`KbXxy^@Iiy3;+Yv4rO0W(2HWbi2X zf_4SzA@~ICGjvJr0dZyQfp&%NLEDOr+&dZ?vWeNJu%+5JvDvtLa>)Y*U-E8aUqWvu zBX;}y4lit4FUEL$Sr>Z=-fF&dyH*7rk=KO@{Dny0Qn&Osb^G!!iMI8(K5cJo`u25$ zVTRQ=aeP7_1;E6cfz)qNl@scLh!r4XsA$6@o@5{dGO$tbDtq7G9{2Xp7||g z<~`l{xjAyC+ce`1Y3J4+rx*Z(UhxNxByoy~4Kw@#9iP$uS1A5h>9fnfr#U_^e7MXq z-Y|D=t3O=*pjY~VBYigc_jt$W5r<1_w;$snUE z+3{I#;K=*KHx7E`95|}}C@sqI8L*C=(KpU`!`r!C_(=Ev*niMR*>pR%>L0cd&@1ML z?IUQA<1_sSdb964s(p8O20*XfNW~jBK-kR2KOl6b`Ss(=>TkMy^1@cPoIUwOmS|R54u9V*G?(NdS8WU2#HfM&QJ-QymF>yk5NL7KbqxUQNc; zm{Wt2bgZcsa=_>3yY1Ay3Nv&AOT}GQVovBxy$OD@qy$g$a=qB7`144okuWF!9Ajv{ zie%Tv$!8Z0`C^&GnVW6P0W(%(9)z=wEZqEcgV-Yk3Wgl?zr3CD@#2|^Tp4<@E&z=h zsaB)6voSanaGK~FPd&s ztPL#2&2(xBP{f(TWa+M!I6+ZOAV5}4(>)E&V^w23z?fBOJa-X8YrKM6+XX8#B8mN& z!P0E+)`)lR04j4l#5h+u#c`R?Ey6#h$7+t=t65bsQlfLtw)RAUn^gZf%!$iP=bpoc zjuwz1b0tU4hi)UC7F{u@ZoZubsVAgqkk+w7I$a%waodB-d9*457 z&L=hNTCiawbjmQYS=kYUNGr$N*cBt(ez)j?rMMbnI|uPN@LO(IkB-=tw#VKp}j za&~g|O?XpYwlmYzTv_!t@_HIn*Mdy5v;nmMFzft@D=2U5_bz51Afo8QA6LS@UC~Qp z{|A>5WsgU6G`BmPWh&>-cCY;Rk@zKe7t@V6)wS&L=c)T(iqg9|d^`=Pv zOS88X%b^=Ko3omwsihsMV2^+At^e&64(8Z*W`bnpq;tN%3>j`*9-T8Hq}9!Vsn1i! zbT=}Wa8Yzu$&V8QK*^7$ZWANnLLrAV+}5OnsUp&^`|vG%kaJ>F)Fcu=F2*F!>M%G5 zyk?J!a@?8_IWtsC+I6xl6{wbUfiJEqY2`SLqv);sxkVz8 z>5tC_TdYUtEiYYI>nZIO#YVSQKG-xzu^yA2W1_-1>{ zVEbdIK-4xeM!&P6x;7s|f5Dz(Hsk~w7ydeT!$7PW&DYVrr98W3@az88YSi2-!Qt3q ziHBE9F6%iG_as)|lhjT}&{dMN7CydD?hkEYzl>RC0Cplq)I(<#&sZ}jOBFr&c!#ZA zLRZ2Y@?`GqrKbxY9e(U0G!-itdz`PDb9}-&-*wSQ!(MdN_sMXgl%#7_Z^P^Ls8q14 zda7Qwu&1l*WbW}Jfr0{(qK!4!y{~oXF}mwY<0=XiC^w^05uv>@k4J4?E0TC0a8Ene zF1`gw;LSMx!dnO#3`)M&o&1HKu#O$#%#46kP;A|)nn=mniTbR^gp{Zd2sO-7AdeAZ8i zb#3{niKsX@m>3;KYr`HW@*WZNbvPerz;Kqw#6ME*;B_sqGQ@ZJj3A zUOnL;vFt%n-qm$w=<1^{xzQn3=kBMW+Uwi?q}J6~U0%D>cS{}bJ^yce$KD1k&p3W-e-s1oi**8J14O-09^&V5|UjwoW<2mZyVp(ikP1HUBKQkw6{s^$6 z^Dv_C3EW=`nvR8`;=&nzdVk)Q8AoIiz$H*%v@410{rz*(%gmY`qx%o`g-t}}7#s6C zpZ*zGTt`mbiIblH*D^B3DcsdM)J;ib{vG?ep8c*{u`{IQjfu-z9I>sez}AlmP$_KuC6Wz{R{(i@*P(8E}mwr~|Kxj+n7>p!JAcHkD2& z4C^j4rACD{(l8GgZc^UOf^E&+bBPmTKDXd?k4AxM`qZ4t3n_c>-usI68Flx43`H2J zO%zgzBxSREW&8lj`XuAU@NhHckJ%dnC{*nelrhHbT;7=o@U=39U{`2ueNiKiaxw~& z&7Fk0*@RmTN9rN@CF}jS_vbp&P&61A9l@mQ;Xf^olUVUdVU27bwO- zpqE%oE*kLXh5*pF}cG3BS|FqH*cJB4YwJ#WI z$J^H3^N8qjG05V|9TNxyG0AniQ@jx&woi6?_jMxd_#Uo@86K~*e%1S1ZM3Hr-J<=7 z35hFQ$OJi~+R$&|dAkbaS8Pru)~i42UYGU^Ofd>SjcbROJf}ltIT}IttdiZM-K_58S2xO{XPrpA{pa)oUttd=5JF04q1wP7 z{SThN9gkR&K&(BxUq7S;?Fz6)D ztSZJTI2#H3=v)1l1qbU#&R#r48atAE^T)3iQxHtjeEOd_c^5G=v_&PV*OZ!*vK3r; z#h`4lm!mA8PS12A)t_#r#`_&pBcgbTcOcMMfP$R0m^abGaGW|LLHhlAqY6ypN*shx zjzTC~nB}TCl_QgpFgA{N#Wa5|d}T%)>mEHK5&rMK7SzT=k)-UwV=2}`5A01pgjJ@R za`rS=NgF^wDD`x>~@3`R_gDdD$&V>iB>a7^8m5vA>Z&+z)GKc8JYL+FaOVX`3$}M*-SZ&jMhdm;@{}n5nx`q1D=P&;V?w4eQ(lxsoT32}UUTMw* zN!<0n4uoK>V9hksx~GugeIZ${cc(fYd;uNgS;O8lMA5X`@s9lQRUPy3O7HKNj6ci^(6mgB?fN7=`@ zi6O(hE8bw7fqWDyAk+P`g`!^5!+Iu$3Vv1gaJ$cJ;lZAdD#Q*|ZQ^eg`!kzR4vqcV zH1HD%ft%Q+2Gy}I#sC2XgB=7*n5#+M6pt$`GQKpCrJ%WVYk2=|_zC27^q$Lam&Y%~ zmh78kr&qGFcB91j8A|n>Ai;Q&l!`e6(=<{I!gDw^Ll;wj+2t-y5FNd6x>DrDVjF!* z2&V*i@i$@5H7>iv#ny;0Re6JtXCc3S7Hu|pOq|T#XU#;&IzV53X2~^tJdgbd@f7m2 z@Ziz6e;h9^CTr5|wvT-Z^XEq^6eBeP%nI&g5)u3upyf zKBeuKUlA7>DgcNcS(xA*{!K02lhVeRM%5_$NA5{h|61h%biOr@&pX@hTyoZgG{DQ% zpOz*MxEtz(L;G-r9Bk?KRy#XSW3XLS1g~vn`|zHQ`%WuImd^U58ymiflWq!n*`(13+Ej!5s_**0t7D9x$L ziKfbpou)wOtRa-Xele0AY7XJ&jV~~0VB|+O+O*V}ceu9i567fe|Dn-H7u@2M7$ROJ zwuK~T9UfEsAj5f;HqbIZH7RlEV=$mrX=LYNXJR+L8WOm-wHLR?GfMn54g5E5KVv^6 za2as0E#NA?C(WhN!Jr+X^^$@O4u~_5za%xpWVgMmOuLr-+l6Q6`8O>Mi8I~0X)d(X z;vakJKepD^)KT^Y6T6|+j8;bFJh+3#^JyzC^Eb)_{UG|d3bC2RKWXjox-a=XiRBiO zY6d5Dytlf~dIHsls=l>)GtGzndoOr?kP`sR6IoZ1{rO{>6*rl*|O;fkDC_?*TO ziijg$YV7GJr&+ZcM~Z3dW>Got@cj7~P-jo@{9PWGjmnMgDqT}+KHVAnxA{PTCWC>g zq`b^vwLl@VuBdzG20)VjPh)6vuJ5NaH#{)*B~( z9STQ7+ECnvGAuV_va#MtoB?`P$*~`}3)D2;m8FHQGs3LUfM{u(?2%<~gJNt7Z=*(} z`ZUVCjz}W^i$qI^Z%+7HxgliAPk^82ss7`TEQDE#Xq&Kw@YSvFv)HEmBBG|l?@0dx z161yk5XBJdUn_>6PXlZXevJ-UgR%@&GKs|B5Vqa8h*re{y*GV9C|;Wgpg?NijOsCP z@k$_SGKhMtY!|JMs)e)_-x)WD$PdO_iaHfq{4jfHZ%0GSK-39!H8x>2o&RU9J{X&V zehCGA>Vb0clAspat|$Ki>eA}k{9by+$0JZjIyz^15#l6>EtlDoH1b#GCG~V*SE)m) z$G2|uxh(BXh;6EX;eg@lO_UOVf}xjN7n0i`0>&Vge)puGF-HVr+`rzCs7Mdu0v!go z-+K6Tw`ErV=O64#RD>h*HCq!obbX(Ndiw=Yk#X;71CzZ9P?Hj6(*Yas){!96$ z(_w#-P^4-0&ovVa-2PU@tG$|#Ru;$^u0Nt0?{13b9axXFLG8$cYHQTV1JdmgG7&sNNzpc7(d8y@!o7q z0BNI>$lUVN*T;^C&IU99(_2w`oa!K+(gdqRY@(2P+|Hd$gS?|9VaxiXz_95E5cg()DenRS<*k%qv zK?3+*l(2_3Y~xXGCq55^>Px_|E2Da+_{R1H>E-NY z^aWoerbhMgTL#5Adh^IlFf)-K`_k`@=u+LUP4xKJxrh=w&zh0tMp*Ki)tdcTBv1}_Q6KgFUpdE(Cdp;fgt?wa|WlPbiT4ARB&92G4*HJ3b6|J6BD%|3hmcJ@Oyu(>h8)j4%)M(*z+@CXI7hHm*Gg6e@2xccVVOX**%Ym+w!6Jf#sN+MR)~>z)IWtK!pw zbt1CpOnbSgU$nQ4oPMhh!xY5m59p`({3M=u9b%?tZ#9>q zBE~3x>USWg7*;!1KF4`T+@zRkC8v;V3F04SJOECvO2Q?4v>o6V!u9h%zZ0oAtH%kS zL%nIeE$?3BYORVPho`r23u7WXB7;Jedkn$I^(InC#*b+QL}KG15tXx|eXz>{3DI-;)#n#92u5rc-Z%kq#gA7>iEi zR|mf2sEqkgn`a`__mFvu0?A5av??B!8?38Y`K08&GR&?a=Mum^u{XTsu!3_pUEKZ%+mU%Y=r< zbHeg`mTOhC5Hmae_NE&%A!0H3F$|YCDEr{2h2eku5yHJi_YONgC~)9hAw+0Z<6FPw zM@p<+NyhGO9QH4m+JC^W_6cumlg{Qw!EExx=Q^VwZuEJhvNi~wC5smf@;*a?^6=EWAeJ}R->4`>! z{}M^C8+;#wne8vQC9pfbyc;zv^b38E99_~v@H{VMi|mm-dPU3+^?S&~{&T|46Vqxg zaxc%YWW2D^&%$AOll9^)Qcbb*BOnOzid3ilB&c79wTT;BrvTP?0-gKj;tIfQdsFYa zxgx&#zt=naPVZpUa}rD=4sn*Dv*OMWI00_%BH~9gt>~IDN7icD4*)m7Yh1IWKT@>Q zoZkvv`NxFSfH|HZ`(@-@F!jaF4}B}izZOhchS+&nOJht=iTU>`RzY|Wq=5SpBppIJnE$W*gZmK1!b8Y z3}9atTz$22sP45ciReVO|MV(%yW(;G_(@P~?Gzmpx;#fZJJH2Ir^$cinoxW>HreMJ zL%oQ(jj2=)=;S2a=Do3kiRH4jGUjVKp5Zg<|HtBw5{x{Ks*7IlsQ*9N45(OubX?MpQ z06~Ytq>Wd@dGyhwyNR+67l+q^Dct+IQoc8q#@omK`fQEq_t{fPCY}5C)IH3d_ZDA~ zqeE#(1@0-}`14E%wX=FScK~FXlJ%kd3OUz#a*-?7xI^RRmBx5S$*Ep}4 zqU?*XEq)Zj!PLrM%el+Biv2d5P3SQDz@Tuh=vJheWUCXD1RrG>n90mxHPM}M*@Kw& zX)9dLJY$g~u!2AdO%}B0CO-!D_rXu?r5T~ZDp`rdj z0PZD=dQ&rSy0(6kcBG|=g(99#gWf-y{SpT)HYH71&m$mQk*1rCtjIWGITd}FbeLO$ z+Bjt~szY(fSpyNK`NiG@WjN_h^ z7eYTcCYeCdx)0~hV@*#UfQM^(Lj65`kidcPtV4BcismYKb%c5v?kwnm1WOG^8ne{V zQSSSimloT-G({V3W-cPgHfhPV1IlTbd7`P08K8A&x3MHZmU44x5TR@_atT5*r{z>w z-NjcQKX17h(Ote?I(!-^I}cC94eHlaZJk5fLmzFC(@)q3oKLvASvtExsf9tvSU}O=n#Qj;uPO_vu2Ho zw91afwt=>(wthN5qw`!q2aAWF?WH!YbY#+yPKC3PldH|3eDhy!_V7b3PQuH%+Xgb( zh?l7UyG@R=eIBH0?=GW~m5g)i>iCZ*jj9Y}WWw}(6Q?0X=XI6^#Ur+%xZ5GnR^dOO zt*e+VAVRF=PswO&jNODXyZ_=ppVL>hQ(x%TfV(%mCF2Uf$@|4T(Uao~Gn94kXd?Z> zP{Qt@iL~+oY`v@VYP|sVvD&iQ@&)_bvQx|%+zxt@dY}3#v-i4t_XSVEoAP7v$`J)b z5B+n7Uh)Fr<&anNlriFOPil{aR_>Cz!BNL^m+E2qR8MB=??l{HeoLe&=q9B_3)YrJ z%X$J+)Q?KQ--6I1{0@_P_w?kO^DP-iDf?x4kg|mTYC)>-bHx(tLhWkp04eCJ*gV1Q z=)h5ql~whKdx^G+{GFiqTbOris)g3FjvY^P@X8+n@YxQs6R_oy!QB$F>hm27d7+>n z*Y~*AZB^Y-DLA^>ePNd9;yn3~+cSGLhkD}h;fM#YxUh>LEZXheM{L%Umo{B;OnjhX zE^5}as#5PO&uZRulWp_xQ(j# z@4crVJl{NV<;y8T=#@GsWZ0-fzSGf3+-M)X`Gk&<9~VVn+eJ%s{PVk;9PCdZeml@$ z>vo<89$mCnTe|> zH%@ox%%}`$C9}jKq%>n@j)b}Ly!-qYpt*CeGq>V;a@l->g%Q+o)FiVO*U|4DG!szE zH=&l!tyKp~K`a zxP@qc$E!s)<1;B+Vy*CNz~|a#Q-)Cu@~XJ`FLoGq@I!k#o)*wTqM4X&ligY!z!)H~ zki32X`~+0GDSc>cf>>(lxZ1%-PAAi21QXPu`t_x`9EsSoKiVcHMu~*`7XPlOa_%k$ zI2~;frVZOrWkWn@(tP<`x36&-2gm2V$LNwZ`~aFT%nRb{)xp7@6mq9sO+tm6O1AJ) zRc@lm*#5gJdQKv_SU_wi$*Z~ySkp|F3PqAd`mHW429mccrFQD6CJ&h#2~D-C6AV&te-4+ktQfn`=g-_)(mb=}8-@KGjxJskN7 zCabizGGOL8B8$o&w0EVulx`=b(^AguecSLAkcCUUt8U#^n1f<77yp}sXvi<`GH)LEWR(H{BbmBRM&br)N)ApO;`4&hWT#ko>G1zzJ< zli4#O)KaC|H7hC*CSE3>_}5N0gr6~H=9>4qX_s&W{ko%6vzNkw26;~Lbdms{P@A9* zobLU-@ey~f229=th0af*Oz`_Ymx#c{$!gafhcdgb7DCc;>_PJO{-+RlLb@rG2X87w zaUEmB2Uc@3*)(*V+OM6Lm`x7W;kqV{o9t(opDsqfjd`)W`CTj!=wl#REZLCPcJ=hF zc+O{k48+*s(ERr;gg zLiH2hO!j8>_8rwanZvIzAADx4BMj{%lv0J&;V0N7NVqP`WUuO!odaoqzLpuR2zoK; zOIJSb6z+5$XMg7D@afxI?X1AxGzvELaXFRq154AK8Ge9o)G?`=!X()92SF=tmULVY z0#4Y#vUf`Gqo@M6yfL?3*kGJojISo&?OH8q9wH&WlAa-YF|Tht9A|V1>v6c0pf5y9 z1}ty2#y3PeXNb&-xPYxWb>-pXo~3DqvSW1&p~1qBjj`^O*zUEdql1~TNV=KZQn>QQ zLazQe-zfmCs0>-)aWU2GFe$JmEh^r0BCkR^#ydZVJD{gle;s&*QLiLw``4zQx+4bZW`?^3a4eJdjWP)y9qC0B(pfRy(77&utIB6`zupCc!L~ zpBzA^7|+xq)u0FoNH=87)eDz8n2S$EW1)rTCjUbQdBnk1Q|hojWHR`JP(c1l!`oc} zsoeoI#-Y47aW-;=ml2${qas77=;a`hn0xV!$FEmBu@PUPe2q2RY~*iLxRFY+vm~7S z!G@dL*X?K#5hA?9NQuRBoW1E)qN0U1S6ZNxYasAu?5Jsqhp90Lq19nAiT5HoZ+cYz zj@;be!E;D>W*G84Wy&U6Ov=p|EHnnq+3hiVCD1>UPgGADdd%^z?hNX5oPr{Ae_1Ko z1MmG!AP2K22X?qAP_TXP%(#$!3fah9NfO=T#5RAI&hW()G`&7n{zghyA~u@Q&j@TW zZ9UI=&koM6m4%s54gJ+BH8t!DyNt`=bAo_uN*BCOh`^Fhe*A6$ zWb{rHe6fSM#h!wG(s(53bxB#KwLi07rC-JA%>SQX?Qb126#Gb(GWsWkg_0-Gi|4Yb zkPs{5p)9?Q_(WAID4O+Ij%Y%~I1-BayZ?YVmFh^_++FzfGovRNyP zhjgL55}Qb`+Z3`;Yhjj-5G5$Dgm;095W%Y{f@`b!f9EK5 z>l1>pH?~Q;3%$Px2*VSYeoOlc@?bPU`;Py>`E2Ew^7>&?ER2x==C6NZgr?}07&&y6c)drabX6^p$<1_Extq!VW$(1 z7WwynX*KZ zParPCqKZ%Jd)*7{(t3x|*Q+2Ig;-iU?}Q@#B3j(kTmz5D*wox;;8$$;AfK0bXfe`2 z!&=Gmq)b6dwGiq+!o&FZw{HhyGD862p-T~~qKi|W`~+6;IijM;=ahSW@o1))1b5q| zs<^-LWktKffQ!}ARk1^6!IMK~Z1=gB^-c(L%yB8b5O>XORoH?Yv+Y(Dj@}%8pQ{qC z&ak_Z{>+~}l#6CrfCcCL!nU(~Sa@NqU>CJJdIOr$0joR9k$G^Iqfsp%Qn!5S$>Djo zH+0-$*Gz0NL+p=AniSDoq#GZ1{;YVt!D9lUOWLHftn+j!ovEixxyk+rl+CoN02VDn z5F#mecgtA8(N!T8BWjKXm_nKCDLP4*Jyo_B59O*x17&mzP$a{3lI!{;UScjt9ML!v zCtBZp2ok{ULSK;yXZdsA)zcGWTzin{ejiOYkl^36dU!##XhPxIvuiP=;6#= zx&p6tl4F=7Q2E9Sc9l4BS!iTp(%@@OI+;uq(Udc^6JAp88+^N4-emGkq#e{S&GS{L zW>S}Bq>>jsWVxbqR!je)97P?+pRV_?l zhxvP_h4S$#O^yW%87tO58uoXZTu~hx!DF@~bOCN5pfE*@{Vj0bqSRlCWg3482d!af z3U&dX6fMXsoWFlA0C9YG^IOeGDv3-3qQ2rBjb^<{sZ+!%{Y|)R^hg1ed+T%U9d@z2 z*P4I;tBOLf(NL($xK;F^)$iFGQNH7>!DwPT_24$^5U)!RalPskf31e%cL>5wxi51Q zxwBnrK);E-$P*5r;SyO@QL;=TnMb6?UdVUd z)bAssk#61F)Jd{|R;2?9n6r3p@?w9wHS~xC7~+Yhk=6?E3+uXEbLQm_bIG6uJmwc! z;l0D@uP1-@X6PoBP0`&8#J%9f9MHYm{+!a-0~DiF?Hc^rmS>_=p)*FNcxE|stiP$p zRCOKiCccDPg8iP|y!b_eOBqOk&iWY>y73>A&H3mD?A*ER7l#a7@SJd&emTl4(IxuS z+@@a2p4&L=ZFxp6#~?bMHB0IkE*HRUI)b(+m!j@wix3|n{S=U;0Js!LwUt&~-~V!aZq}Q!M=0|Jt*8u97%-!8jF|$;{lQ zF#d(>3SGys+A(o+*ZSvZ6G~(an{XRr9550=bDtmXOHMsT)8#*?Zos1zob>k_676S| zgCsK#P}`fOv)`&P`0(i_fn&rZT*4+&DV7(y)c!)s*IT)HMevM`@;lLQH)*~-z~0{< zr;@u_y<|%D#cC`0<*pa}qL#@oOf}cmVB?%gPF~rw!}V*^`Io=s5jrC-CI)x7PRJs? z9lGb0b^^km1o3xfm5A2S|KaK#fHdc#ci*$+PqDg-WU|-d<2v_wLHl-Mc$M41~nyeRISdgd~xo-A2 z%>O`He9GO1+E`ncJtfg?Zhq-#{R)gOd8Xan#i?;?q*>}S;!%pdMj4V@2T;*X$*pA7 z)Gn8MVG3uD0|7rHQlp1EHr5eXSErbhRf}uM3`dEjNns)5KfAMvmHF9+VsHr`2 z-(qLfg?7iA=KNBWJRP4N1YAoCe>(8gHjTB3hqTGKk<_7#M(9RLr=>d)G9C&TamO4) zYCJlLms1DR%EMn3ThpQT`I?52z_r|n+%4+5Y{NwFIg-8|AHK6aPED2dJAsW;Rn)-d|EICOgv27 zOvAZ*aD7zq@_zBXPsYaliOCHc3)=YelaWN@Dd{)nE#kiN9pe@4L2B3No9gT9(7lGk zhn;Fm<;mp2B4)ijyx*kLcN#3K0xH#XG z!9=*pn0zd5+fz>+^s3|Mwb)_YSUuLjZo{u>uH!G)`)Pb#v8nY}fH*N8j{pr$saRCu z)Z*XhvHNSlP1#%jczMr4JP%FysHY&sj+fkgTA?VmZdiL_7_FeL6=!1!v{l~_#f+&6 zQc(s=W^kQ$=H!O&XXTJDoeokV+-pm8KT__10F~{WVE6;o& z1qOP-ZxXHPk@9<3Kx0X?*EF;c&vI;u0P7tO?b$U9BJB*)4u~L0&+v&+^$9WsxY4iq z@+ufLYGQF^m6mdbv_r$3pJ=6$BGLX>cLS>KFvRn)QL;auo&B~pcd$-GaztV?PUO!d zK5`if>Ov4!IyH?YMVAtXk`?j?bX#g{n#7jEEn@gc9nYy10IXxEo!5n&W-bLvi$$|x zpke8*-EKk`t>~S>>?~&4o+m>yr0GC7x3h7~T6Jz69h*yKT2+=|(_34Ywjbj(<|1dK ztm_l3co1{^W#zp&71jPaOYB~75q@Q0rJH%g1rV5qQ25xSh?pj+qzLT|+J)P~N zKN1fkBb*_ofCXPCp>`F0jb5_@QV!P>4|#6D#R0Q;ts5}l`v%nfK7!gTrpjxQDk zDoy$2Esk3E1<`Yd zM~L7+K4Rz9{9>m&(xtQz_@X4*lx=@Do{d6e96V(A%T$~RVJ@0xv`d7gWjkwk~0M?dD3 zBk3ZSxa7z#wVd^q+!6rUR+F0(>)!TULqIXo)_V`{9kx+$-oI5d&rjWJ!%MaQMVVE% z5AfhkA%{M+bPDm~VHe!S&7P87o<86~adC%x2FaIS&g!>O0%qNv2fR)Mg72EbDaSIFAO z;{)ZY>lIJmKhy(^*4N?G)FA51(MMAnZ)7`k`oYpARGgr>q3g|XXQE{yHF-k$4ah;F zN(jFP(VrQ?tNpIG7M|IC)>asTY6VT#KQWW&E4oSN*K{E`~S z(CKHWk?+Y6S_0W+JKl5QrW(0XOy16Wzixnny_HkwH65iL6by&Uw*c1bRc zf`We~;+1O=g)r7v}=H1OqRvh4MJw86& z)Xi&UxjaG5BsTq13P0K14badI|2TM3euiqEIcq-lFld#(_U@PA0@~P_`KhVP4Efl-jV3$ztF0)?Cnykf4;-Sk z?ylM*+J0xV^5lLa%A*?_AUEHRC~r^KcWPZlZYlp24qyt?&S~|+hV)b_aJlkQEyvsB zF@HT#ovr1>35MHixjN^;`S(7ox87E8nl|a)c73xkpN;4i1>oM`=BZ2Wo@3!K|Nbd+ z5<7KUQZ=Pi!c#-cA?_o>-d?x_N!LTnvI@ENru<#1eC<$t>e`KD6gSh!!hPk9#1&&% zs&!^1*4g0ZY@&UT0YxAgFUzfM-0e|4Se==1)M5{B`->N4Png6sJxWw4UH#rBTZU}@ z+FNFc!ODpb4bVPDYe`Bg=#pvN9$P(IFcMlfna8K42K{Axs7j+%5hgcHe*aX*CLk%y60PJx!D zj&|2QKjyN=&Md|0rMFp7%s5Scv1P^8ZNiwY2vjB>2;d|QtI>7$S`EdW%brhRPH*BD zA8nY)PIrmc#mzsS&dfoUouYD;yT!GgLdSJ>gr_}Y&-{0!!M4Gf8*R?6w=Z5nXXNVI z{^Y~qcn^{D zoi-Es)iRxZrs`(8>J>s8uSm3*lvqkE70!^fD`57>!jP1ni%4 zEu?8si@zCF=5e6C;grhqCi*!EYqjaa3o) z$!(U?FFn;e7AEG(3vgy#dBJssn*FMM+O@Jc5ufYU(eBn%qr!|z)h0HM#@(I9y=HG- z1`sGl(bW?A5{z=$QyK#2Ru|87EmjkcbRjJ#QNwGv=aQ^S&qm<0_Z8x6YnngLp;=_e zBh5>)FVP2z|7K{+#Qfg zm`JeiEA1hm&cW|@czsy)Zq-@8)tTs*1`ylF=qkN_2*aV=1r-xJ_|ehS=|4VWYf4O?{iLR*x@#^&k69JxI_KdESsJR` z)wPc0(@6q$<%cNl-(Id12BU&`r5w{P8xY9qNq9CtCvX+ z1kuqF8Z8xNlblny^i+&kNz%)Ze^D;lt0D-3vr2n-2AY^8ZL$`Dl5?g-Vad+1CE`(8 z78$#ps+&vFk@Dq}^~Iu-3GeAD7ZQWo)`14Wa}fp-?u5{SYQwXEx&~z9IH zf2$pYtJdp~WipI{UHAOD{!;;`tf-Ug-A+7>R-JvHk3U^A6aDPw4ZXK~f`4V4xPvqUQ<9jLd7FE+D|a7z(LXeTbK{@poaB&d zn$a(Tvl6JAX^yF^aosr@OJ}9WjcY57`S5U$drc@+nk7SY?6(M3j=|e?7?%cdE(Qbu=-sl<+h+c{$}IGH z@WW}44Ud9TsV9*{fCus;M-;MPPg0DRp6noq+XnKYjEUK5q23Fn$06+&Sb7$hdhIQj zt}%4!Cqzxc5JMs1(wjO}@0BD53H=UJD5*d-E*XJ&vg}Z#y$yJqDYqC`{={^me#})P zE1&du86}7KBP-?uz;Bq>(ZOM$CeFgVxajbd&TlYuT+B??PeI-lSPkm_BaUACz4ErX zoT;z(k;U7AEn5~--35o>_tfF^{4{jpS|Y-AiQ?_vWKDyXc`{yQI0EX%sfgozz~sjp zH`kYK1oqNx7m0gC+VV!K8VD(w_m9iJO7ROMfHy2V9+3Z_J(YuOcLf#lXJlU7*ny4|DR>@%4RQ z);jS}k{baE%%w}9pE~V!xJEfza*o&X_+qJMaDiUzxSUQ^xXQk=AO%nHg&>naqjv z29AKW6^J8;)Xf}c2Nkv?TsYeE)pHyUz=I>k$ktQ`kU%P*)it!X>DW!?s7(q9m-zMO zs%|n<;Va$-b5%^0%U51kuszD0O#li?1Y^;reC{4uMRgY(b1P#9ogR zbWgmdg5_=^r{xvvhcm?Samy{mDcY zE_KQz@kXyVr}$2U{79tUJLvX`>A#_y3>sqai<0|fL;ot8)GMPa72+14lF}3}?BSN6 zA5$RN5Wxu;%_du>MyG(Kp$YHG#BPO$N74fUzDEnOcY<2O?&5N}{#ntb9Yz=d9;ycp zOEHJPPMR&sfU85uv^6QRx!tcC$15b;^&uvJriAuJYLE2&J^0f&Rz0*(akNOf>5+i(j09PQR4qWp8E7%sJVkMx_2>9N&a<@uEIN#hjWJzgidVAja0_!qRHMWu>a z?~HjxF>#I!PeP_~I#xo;qjDNUvD-JF##E@07GwtcB3Tn$$AlX?l11ZWNUPgiK@~B% zJesn{VEvKkTp-H=6lT9@EM%JG8^b9BQpHr&hFAp)^j>TDiduWloI*SUy?GVD?CNQz z8{>8R4fryG?nLo9?rnIm5V>k~C<@#LkXhK=ArsYDi82e}5Svo}d`Skfa^{Kc#9n8f zoZl@Qw!M>!RrUmV{Bi4;GC};oJD~`^u9=S-#f4Q2YzEXm5J$dp-c9mC;IW?PRq%`q zVR=4VgHGB7nm`Iu?N|@lih{1Jsqh zU;sty#Fl-K&q8{>Qx8uHwc9+GKUfUjXzbxbW5m@^-2}nIo+hzF@KR$j9USx%{%tBl zu!bn0Q~l>i{dq8{SSx*?kF29D4RPU?`>gnyu7J(82-#XHt8$j+`ICRdv%bBxhn%SS zF&e84x#fBM_Gha3kDH?|TcY+ygBrFTjjaTDcA_`}E3srJuDcgELRJe^S|D$|)^O+T+28Q9lf-D@@IrD_D`2l3x6k zO`M|Y2-dt6i6J_`zl@8V3uOxg^6DL#0>;YuXhm)li2{0EEW9(D+jzOV%CV^5oTVaL ze&zv7R4$9`%~rM+i^QB}i3@8C^^X2aIQfJu0)iF>qw-g53mtalLrtMtZ~}YkrDA9B z5ycq=Al+R8^z;jsHfUqz;|Gmg5fM&_=Vyv@4mp)t#a20hf0wcYx{ zIH_Ag=Z4e+OYIL!%!aDUf}+iS0%k>3f>8!-KFadAwEUJV3oTln`8!JMBZ6Ii8TdM_3{1r3;Jk+~K!={93 z=Xmi~;t~{Fj zs^(_^QhJUqb3$I0WK4W4v(hm0P%IW21m*$%+v&HU%>fke5wi?ye}t{Nsl6b8j4^#0 ziQ*Rd+P3IvYmo-e6%nDx-$Pw=D7Af-Q_J3XDPc6hpvw?o_j z_jAD+GGQkvtiO5LG#M?jQ$H0#D~&*s)`2~B&8ju@VRhzFB+E#Ai(WeT>B%Y#;2)+# z5Z?g-U#gmRzdx%K- zw1wz4;64x$aBR|*kc!cIxJ7V77ASxQo|cb;3aC8$h%--Vjbi91mLq6U)G;(qRMWW} z6_`^dNgxsRq0iPn1Jji?8twQ~3F%W9f}dP-!$T(4k5Lx7YR8iJp1`wFXP3lIpOCcC zu3FsUCrc~ft3u1Q0J!gx-+bmRR!q?fsgzib94W~} zaai#Ak7$2+r1i662PF^H9Wa0qYUDvxgNrvYe_|ftJyGdB^Q*Ea`r#o@QW>2(A{(MY zJKmBqQ7(c(y8T|&(WlAA3r4n#b*&>vTjq21KOBAlN{8G5(+&8JNRC;xkmU!i6*7$mpYo>_IY{0NbKz_S!#$ zN=FrNUxn`*-~DC6-!JQsv;pfL+2nlP5uWtVB?s)$X4PGOFE#>@(NO=WVt2fxOroyu zyoR>z!}Zs8%W-lGL$a9l6&L)~2Hdda{OJo0rU`A3h+o@N+mE|(54CM(<&g7*m*Z&C zqfv$|6OH+)n)uZfb-)l05-(oKQ{r`_F_whp0%R&amIdb`^Ud#EoFoRA3s~;9qr228 zn$zq%{e^LQq{|%u8CNk<*#Dy899cv{MI8fW1H8B4pfv@zM=Qz%XCvH&n0dI(>8#;E zH@!HL`rdTvn*Ymb1=P=ezPB&`*Yl>+an*D0TLaG3=hD;~t$NPK;Mcp|+9==SD~kN4 zv*p__z}@om=Ur{U*EHZGWeDJR*W3MF0X<0b>E!t9UDY2T3d)+ELv6sgEx!LrZN$xq zxX<43{mxR{r*@zJaK^w#GY2?OqH%E>!^zoYsQths)b zfO#Ik~rZ@Mv)nEo46QLC7f{q z%#JFO`ms_(`b)BzGz~0(@l0Ayp@m*hGzV<4%tDl=$x4CbNF2=+_=9&nhGh5?!ur5t zaR>w|m;7y>$3mVUQ7`;bw`a~Yg?Q3*ca0?W9WNRbTj6p>8RiRSF;0R2Ao`bI@Ie+J zbRroj)=Y|g`HIHI=@~2598EutjdTx10jU&HAwgy=mAB*3gm-N{0u9osg5$|xiuoqM zW(P|=yIT9=isekDDOO0-S#KR}iiGes%*V}kxuPf4yzf?Aj1mzXTlR>RBiI4^Hp2NC zXg5!Or>~Q%Ze)q5j1ectN^T`P8AlCBEu9gdhTT;rgal&9g|5UleXUG)o`mTsvo~~3 z-9b>0Ajf-d-UDbe`Byx41Rb7s#U6Ua4>xhI91s zJzm<^QDgI-f4-ihvm;>jt;`Ejsx((on zHGg2pg^~}=s9%nFJ02bh;7;9z3UfsEm+pfi4ZN?*Nz-q#IAH=)O-v+mxcuAJYGS*|`{M->_+HY?I;U#}ErZ7`_f?zKvdvkVF8Og+hx+DzzY^^QaKBbsZRBE=nb- zO=*sBB0qA-v3c6rb?N68-TP)TQ{2TCs+n|cC7#-#I!}e35jg86`3m{b*rIqck{^** zCM}*;>Q+t}ZSnw{JZD|mdmYU=XIz)+$@ePyBzZGGn!Q~X&adm^(r*El9PMuvKPc`g zxbf2A<;6>j5ZrmRoTN33M3#0N#fSE&ZMP+$71>Un#ex`V2`KrlJal}7zGcpsEz4SW zrqEhZTh*Ucucx+keP-_Sl;u^ojkcpZ?u68FyGI8dLDCq5%pG@Oy&71e21@5@hg&d6 zZXJlD59{IOV^OW>AtV8YS<6{+?K#;dGeNYm-`rFw>z#6u7NQqaJ^hLRw18OUNRgBD}tf*5-uT~+?R zoz~ETVdv&~<-x4|A`NwcZ6$PE_64r@>Ry_n(*zO~ghQ^(zAmG}1972)r z$rJgKO&dw;CbZ9!Z(h2;`~i6f`LE7-;{}uAQG9HHMwvT8o{4-lX==LN9E5o{V^rqh z_7A^&&V@t3jtU_lfrxZ($R$ece<3^GInhCQeWwcfkBkUZnEK7 z04mP!QL4GgS%)!4`(AN!ZbbYk5$uP2);To=tAAW=J2%yJiYmTp+Im&AW4~H&`f0mj z3a@@IcbQ8et0ZApUxdE5NrGNy=&^5%z_}VIqjh{99n@0;04tfaZ0L-eu*xuDK|kL+ z%DQdSN0vE4>-`o7=@jl09b^D?nmHH65}0oXhjuyZDsZ`$RK=gt?c$gv7skRD2IjE~ z-DSI#8uBOKw=8=6byp57sP?3>ljA!5Ui0rges>ohNBL<7)r4-kr-~tUKIi+?*e#u& z=G#8~t#=QA4(xtRp4Y+I@Y}hW8tW78XB^R3QKJ1Z%_ntJ<~vs;Lwm|>9^?G4WO^}U zfgCz35C?Fdm7^j_0@H*ZHJXUH8x<;@L~@B56uvTrd>ewQ0h6PJz&xZ2kHW(bqdxtd9DV4a^y~*tY6n^-07tZkU?83bD*tC zdb6VIme_06q&XJDMRQql)0@^7T^>EUZo$<2%Jo;sul@bwgZsM;kMXcyeTbg;?FFEq zwZnW}afWZkany0v`oDZ=IjXu#Zpn>bb{V#=)~>dW*N(s1VLBWHSXJ0C63CcarJ->Pg-Y$9~P5_>5zoqCjl(|Mu_hbZg1swI;EIeECPlzFgqYduas$HSN zqd7!H2F*z@EpU#^UfrkhhutLe?{P}VF6Pn!)Yc^d0#dii;niz3reG;5HDMykyR>z^ zdXvAGsl=?j>YH0X`4{Z3p$^;Y=>5zG`qhV*j+~bX%CETlHqv3Ynkl9E&PV*!y8HM- znI^W-ZZKHNyQM8U+_SxEE${9AnRsv)h3FsbWJ>Yk0Gs?95-vxqjM*YDTya-$9q4Dx3mGWHCi&I7MuFP5HKuk#ba?O}KGCDUVZ3I8%M4vv|pk9*X- ze&0)7#Z2I||0)4%bnM~<^o8JJZ8;oVylPqGbje{gT<*A6ao-_u6QmF*5iH@$@g{px zaO=Y4IPhR{C75o;GWw+Xr1f*+3b9$6FcBXKf+Y(}=`1OawZ+~IaU_b!>quGLa!{xu z@X;Q?5|TV;mS#$xgV1xlxbPiMA3JN3WHEAqhy6qY1^%`OXidBTL>ZH$%D7c0>ngSQ zvgQO-ouIm)u6Myd=;jo0d*59^q+vLd5+G05cY0}p-ik>=O^j56sW4X;F>WR)g1|mR zmH%bTth#U=FVIVdiNveP=kbQYA1ae}a2tsIBf<7l8lbnTv*Mr8xST(2gsI-=)ct3G zrRT+5$$1Zws)!B_020(K)!0fKGIQe^HwlQycHOW}4yoIN+MqLvVO5GgxMk6#7N0uzWClZo+?FODi~RM3oTP7PKt-q^7lzFQBu2*UqiL1d!SQYDB2Z6o3>{gH=D)z4g%5 zepN-pWol&68ao49fJtObo0#@i<%Ro@)^CZU!-ca7QN#N=d0fksClG}J@p9u?(BMFg zlzTv8jvQgeASixEscR;n@gd6tE9Fd!7lv7SfZONLhiTt%4DF#p zP`K_}5{ftj0LwMle-~daO560?Zd`k3`^I=*hwP8PpZBJjqqd_SYt0D89d0cO+ivP! z>g228Jy&Ex;l?Nd2D|1j%e)E}0tXDzp0Q)6bU1L$h}#vN ze3LEX0g^w^GtAt&PlC|fM7zFoND&EKSV#xalnQubS9fRPG2y2I1fzEXgmNO8&6*9X zQ<0A3#|gzOnMJ?rc15LMpH)T(-sxQ-T-KJw%|rIqG5grKqC19L%4~?s%y>ckeEd6_r&4Xj#nsn+E zRQhZNX~)6FJ&apDAAlE5=PHH{cp~(zipOeAaS{#1qsyyHtr8LQPr5J8n(6dPkzZPB zoP_*$Q_(`h;}FW@z~@genNrxO(PO39b$zRwU{b=Kd^rYjY3g0>Hdz{Q+f%Mf8-q(= zDw@Sj!H>TBePUa*s$(y;q!h~C?5JyAYtL|YU5;?o?#laVV2!s!1h%gQ2&!dto@L#I zv>Ai+T3+$;wzbx1?^TFDV70!%j{he886d2PY~*-X?ZSxgxN!jzV?h9N(z9gwXIfIJ zG99lLoFO{}nLW$&X z5UlY$dNwhOKx5pUM4N?y1ufUE17IS|^zX+j0WcgqY6 zL$hDB)PCjF@HL?pn&MS7=QgR2f|?nxpNRD-f(~MVujyF@@etb_&_eVg5d?;fQrcrY^+}O z-tM5dfT_kX$h6H}J%0W)@29Ph_Q4Ni65C?y9R7BefTc|kb9Ny*N?wT1E?-Zb&XZPN zyE5Jv-OpDFUwp0(n5nujm>(i|ay zMg;#|1N`+iz|^0jC7y)A)XzdD_(t*2?@3vw5f=%Om0*i%9mz1#A`vVvij z1={$BA-~#iL<|-3hvqJx~e#(MW#U7ntk^*dcIzx98q<&>-_T_9Ye z>D=-I#J|*2@jrcAQm-^Sj<;>HdAQMYdtmLyqEr?j+QxxoQFjr9=`;yM0{GFTx^GOz z02lxpg(|pm9rQ1cx77Iu-*^Jyg249~Uz_>_dWAa&{`}?{uk&NDbcY~f;#oe8`g$B+ zk&0PaLVNR(LnCRqxi?aUg(aH_S{a? zFh2>3{4=Qo;_$d)?d!3dT%v6us#*QU;`j4Uqk%Ns50vn%OLH;yeAF`5549ZRMQ2|z zceDgtBh9={=tMe!4qT|VThn6KTM<^1ejNFwk1fjw_SrbpQ?uWo;6=8~NqVkipp48+ zNxMI!w&^h|Awfj&nO|hs`!;(vEK^_K7;8KtjYuIjI@zGCE`qF^WBiMDo7ySc%5_#n zgW?fpeDhMBE;%mOTXR)T-_s@zH3MPvFaO-du*?%J%`(gt&Bi}WQe6F-D9dc!R*w}r zj9VPN&dh=SujK}N3S*iT1e7W1%at62gZ-yB#7UBFzW{&2!hwd1l~60B=s6z>=5h;h z!@ET!Vg%tiZ=Q#8SyLjDqs`n#xE)hn03MtxcBrRe=dQ9v7Tngjn{S=-q*E@ZHi?J* zE?y?+J&!F|i7sr*^R~t3fB*P4Zbm*h(`|{dM}(Qr=bbVl2^C*Y#h+h@^7vd^OFa5w z`khF*p#siLT2VsRUfY(BW&3ZJ?LR9c zNl=jj3qgS)GXN7qbB`jkfusFJgSI2ttKDg9l1Kt#+v#tk-#>|-YKsPl#z0vOI?gz% zC|xgH*1vwHJ@-8KT>D;smPNjIuD73aK5sZvLH}?DqX~lwXCL8;`k>VOQ{8ADXP+pHx$?Y=&5ALl^nqCl3d zlUf^_!UrZKZmbaJ>t?!LiRxd;Z3Kkx)<9QV?Y#Hs?9&!`agWa+6ovO4>JR=nDl5Gr z_Svl#lN-OS$%lbpoKrrcZM}2h_9X9X_aND01S!-;AD^ zCh7)#mOh54)!DlywgQ^rEk8u1&0UvMV!I*VJMr&5qdL`}N8PQrMc=p?r|D1ic+CI7 zZEk}`4?NcM@=B{$qfT;zaOB0SY&ZGHO~*cf>iIvk0CCojQl z3!-fR!`Rpc3#A9zU=KQ)rl&QS9-zkuDfzW_f!#^?5AUzoGNdsgaSn`Xw#(ESnO59R z@!3wg@}%B4G<=j6C>96bCjf-YELTkL9OtA2E6NE^m7tk%>$8YqOD2OSEP)In9!51&8G`+VzgmtMKFU!Ap|d%H zy^BRug0K}qv2Rf=1f$A}$|3Ozfp0N35I~KvMNu%3Cd2uQEQ*P={^|u=fc`4PyCvT+ zq%K6c2&ap*a|r(qc5xsw^T(M5Jq_0z6*xcwfDw2gG6#r11R*f~x+OsLml-5{L29B5 zB_#X;;zSggD8UT18RUVA*cRqHAVe3SGa?!yhKeFDNZJ`A0~{2gI)d$O=VSjvw9dy> z^ye_5-D_n)rb0T;hiSkUBV;4Y^;c8~;=l$QWJJX&Q2T%w?w1%OD;LIvd<>tQA%ySu zzzpF479;tiko`lts53;47PTo^fshSPGR%HK`)AF(2aftvfY%7JnNThyVlRy$X3r%( zcm=v9978C}1AuGRXEI2a9=HOBK{zGC1JRU^=g1}_nhiRScuedEd~d+D=W`;WBj89h zff&~>Iup_c)q#YetOLS4G;K&ZA#@S28b1Dm2y{xPhVO$(4{J-@qe_qSV5AY~0;5Cb z4_ZWB?js(QHe}p0trz(~r7LSg_d=}mHw`~W@Wk0W|h+69ge#qB_7M0t?t1bCnfWf^wA*gO2viTEM*!Q84v zX6;%HuJ2&q3a@}H6K}yU6Kw(G)Gy{Z@GfnM-rB8%dmz{dbz#^EWWG~uA+QtMg=8VN z{s0#N`2O06fM30WK8SXrK2Sf1JWMtcK9E)-KA2V#zR(vw9(Z>`yD)Af_dZ>Xo}@jx zHc07Qe{UprWtl$TaPJ+zTi%nPn()thI7fOO|271_s5Sz85uZdO&+(T226nIWhUTL0!Ehzwbo(!WPKICn*3$#z7JMbfS=okDC_*-8O_@7tO^ZE59bu0Ws{oCFV9gN#Cy_J1Q-$U{MzeNFf zpxr8a$bNwOh&clJ#l3KL1$ZR?u>T-$;W!}diSff)5qpFABYVT@cI5ZYZ&3$nnd$zByLT z$0_IdWuHCltCSIxaeZR`oGO>&7ju34nWfg7&-MB1e3^b>^F{gpi$cy9^v}`%67iqM3Aw%@{q*F= zz_@ry@Z&Cwbv#81tN7GKqgKh0szfI?$WYx$CUDgNs9mQ`?Xfb91n0AQvQVTFhp=R{51Pv1@hzMm->q18o_t~j$~vR3ad&T! zJnWncv+TCeJeSP7&x*gj!$jRNN^z7uKq9)s!_Qt=o!6~Zyr}kmc~bHC_%D77L=v{l z?~AWo-a$QCi@*l-1Fz0>NDLv({s*LO;#4uaS3aEyH`g+O9fv)BYH_wH$`v#oFFs`7FgM489MUehEFwnnnr&^bO`QF?e@~njUmo;bXpj8x$3{yf%EQ)?OlKab z{VP-o?2BJf;&*Oib@zk?vW1Abc|Ve&+{kLqoa4A3Jb)fNAYPFcu(j1(PE?xOW+!Z? z@!2UHDyX7N!nid}oawHe2%Y#vjeCYf3pu_bb61AruS}I_<9p|i)=4Q**6OkHb1uJ$ zb`+atzfxcjynH{u^R>Ri@bUJMxU&!Ytp7}wmq$rrwu^EsSBSl3|Cv)Q9;Fc=aF)F_AUpo8mHf! zm)28gRhp2W_Pfw(oc;VN_x?=gW$|+kYSx1pV0jendI!nyNv`-H?W5TMbD?e7MuiFsH?Fx1@|>-Xhy_?Ur?(OAr=eN#&4esv*USrhUgSdnIPO;CCuHUR<05Sf5kV*L>Yu1I?s z2%`U%Il&tX@q;ST;tTMk=GPBNk%?QAJyG<^rzM7`ZC&BA%u|FfD+s*7c&K~?(fab6 zCy!f3s5F330%Ltc^5ZAx+&e;7hNjTfsJ3;a^dyl<&xN7+p;VVq?&MSl#D>Vwkg&E1 zF3xxyjJpYku|AIq8iqXo!X>LvowQi@6v)z=n)YW(R{CrO{xj^TfF;->zIfjuZ`9K> zou%1OeWCpiEP`l2%Uu=R@kp0n`*93{m`Ijk#cNFygTW6qL}`{wN{7rDvoBqoFMj@5 zD5g~XjPAj0Sa(IxpStu6kp%h8VKbjK>B#t53!KewNu7kKYpB9p0?CDX^4A#ZJvua z7Zv+(B8Q32kC1`&cK2y)<}i1AAf<#>17tN&oH9jCu>`NfU}4G*li!e}|6a-(i)E|I z1Tt0>Di)5igPAa^B#1QjGjaM)02LeE3YdewuQ87ODhUk0swJdK?fNNB2?44rM(_Nk zEeZfP(*L^s=J}&52eC!glOhy_;Ud`sHz=G4z9ngCjCZ7UXt2JzWyx%bQ)5XnuQ}DE z&6+~1M^@jg(oxtZgHU9DTwpc)^~+$y`8=<$F6q>w5R51-&`YLJ_$ruGj-u?-7q~Ih zryYybn-Zr9s`R!d2Zq`~;Pg)9?2|BQKf}G`)P`r12F7F*@0&|bQThB}?_}>Bt1Z$H3(k+iMIlx>`^#oK zk2PTH8Ha8|@B-7W%UY}Lhl-K|mpX1mnPEYMFSK|Hqp`6W%U0$b5Z@yWN8~iYfwL}| z`fbB9@2(=wA@xUp5CPp2Z%AvMtms=ZsxD2gU{V6(9#$0HRAt$0^=X7%vx@{e0{$ud z(cjcV<;u0fM>X@3WC7W;;k>A1Se^G2i-Ol3L54*90FsS`1HHhpv!tJzxJ?Mp)1SZs zNTj>l?fP$3S&kTVA!lVWU+}k56feII{xn9TWmMD#u8wXTG;KWLT>6E?RabI0_In|( zAUwA6Cs8KlO*~DMQr6^7crm6>raX}xpPgN!;~2?Ky7M!TUHE9WPCXUs$5TAD*}-pI zv}1}7$6gE>O5Tgax>nBw1c^f+mqF`1{&!xSg^y=rKSk7bAf{&@N7qa2B}$&(t&^RY z) z`!^G5VlZ;coay+E#wD#~cozPuj+2`R4tJ7}IJB&2KuWFZvVK+LKCypGj zagm!Tq$P^xLv&#(ie_4LN^Fwyn(^D16pSy5A>RxVKi`aEny}j#QyMFFTz7q>dS^so zV49v%Fdr9IZLbuMO+z)@BZH(eQhUo6xH>DCh81EaI3xY_Qn-X)V%EHM@a2m{_w0Ka zn=ho|8T#d7!0a+U2@zOdo$U}>=W!(B!uqe|?Px2z6UeVOD}A77A|)5izM@A6LUSW(5Oal{!LngaLp`DTqD2Wee9>0Z7ol zCP{h5V{-&_qK#>1p@oCTqp^WEC)ex_wVMtJw zlKLXS z(bv3H{Ji|E2}ADFD4v#LKCw^Q36~^Czgm3#StDlbKPK+@t0s;_5Medx5Ih|7biR5f z`->*VkY_emtiqsLs!heW_<3=bP_z|l?Qvd3QRTc|WmbI;%k*F0cu#A**vxxi0>XUS>uH`;8tAcK7_JMr`YX}^JX@7cerO3+r6We)i?4lw}n zqJQ@2QD_`!@Z)8S zF_y&sbg8H4jQ%j`7~DvJFs#%3fg@TyeX}cw*-q6Mkeh>fQ2qW)RJIM-t~2BU`&nc) z&tF@Rad-Cof|4K%?hRi3RVYv7ibrZS?n@q;{-Oa zVz5CB+x4MU=}TUVKDRcm;?%>SE5vr0D|GF1^ar=_q420xaa-!UqN}??N4Yk~qg;y9 zSeMs~x67}z9AU#2cSx0c*uC$4QKQr<6IlUG(zm5=8~7d*xk{S8G=)h0T|wZ#5&DF$ zY%!~^Y)*Fy+QR%NLHP39e-I)5F`Gu;W4c*)Pt=@cHhNeZsSz#{3=^SjY?-5!P7)YR zuiMi&o*|uGviMIE@jbhsD-I86t622r)4K+JO(=-0Q&c2grsEn8_d!^B$?5)sh9OxG}U+34M_MShf-Z3wMFPL|t z!Ms;P&(*$+%;_{Onyw;b-QHIw|qes?~6h{JQ%Gae&xiqA^ig+pCHOPkDfg~mz)o&HY%BgL6 z2RMbkERRD2vYKKDqo$eo5-xC{*(CKt8_60&x#kKwAsjX+tzlO)f$R%Cf{3B#f<(`v zt%$dnJXu9t+dA%e^9j_B1QV0Iri9n|C;0QBSdU0gUm?R|3cODWjxF0RDdT|(%c7mC zl{w_lC~Zw)_kuSC-8v_~2QA3K6l(S2(cBr|L!!rA_gziD)Tn*O9c!sjIG0I^60JE$ zJbKY)*KYTxvA3;YwF43leNFY}6`{Uf0wWx{~!IWu&W?$*@wt4C0)w6%0f zq;Z6P=tJOW?~-xXD{B@}x`}%<4y~G4>gkELGuC=C|9}szmFZnTB(Lyrik^BFjAqQb z35@u*FQ(M?*DIgcEbym$iL}$v`?GO2cfId^V^E08NdKSK^8-#oo~9%nMmzitVoipQ zG+1!c)y^uSde3Z5usino&$U=7Spf5@8%uJpL*Yramd~HmR7AqjhFvdp8QcDuxF7}# z;jt>>eF-v*vrIV9P38smBgcdwum$<|sE_;G{1?aBQ&UsCb>Q_7C3`kXXyiJATQ2*v zOg-eye5dE`-x>Gxv(7}ZKNqoYAOdZi@Dtm5L)5%3Mk&Y|Jhj_8p18)<=hcve-x*)h z=XUY;32$kZbRS&8sn1fv6?974wUuBG(q7Or!WW{KBwCGc+pM>4C$FdYMz*qmm0PFr zzbwRBIX1-dtff1hGcIaW^Ow>IB?wkpF@Yc$OJww>l1>9xksLFGGeIt;YTgu%}#tzUpA{gNfD%?zAh@C2wqEq>QB1G0s0BJTGx_n{t~Vc`oSlq zqtcr~z9BP2wd!SE-Ff#09RtcrDaI)8Ge=Bj)un_5OB5DLywhg=P-9|nV-TN=?V%7A z0d6ZrUdCT9ezfXQ^Pd^KB9io{9^2<%iA79NcB|CXb=6$X?zXskpaZ8q;XkoIc5;5gGi^NdA0+Hqj|LEVqF#3b%i_p@Ewo4fN}saJpvv4206UUT7PuA# z;kC%m2q~ZX)i}j}4;Q0(?x3*X;K#|<_Hsj|RCNw2n+U{SImbf7&INFU5oM~ZFV(re zl1>s*X9n)!s79=*V$m@$)iB`v8P#XhR4v$4q*KzckmxmK5f+p0QfEqJ#6yErcX+M$ z7bj_Ef?7Cgw&suQ7fstdhXNgXD4o*;*_s=(U<|sZO_5$`G{hwpm`uZv;OgVLb084DICoxJr_&4yn&0tq#=& z$U0%?r`Z>W+50`qH#5@O7&k*`===OPoxjcO2Hw|@pjUGK>OfrzKN&+Oy=)34`xMAg zw6-;O$i$tw4&8%Yn2dned>N#2r{IM=GZh|DX$io)cVnyRB(ESl&Y4Rb@KXim) zrtl;C>~{G`Q9&Y!9;vGAt}Tq?^=a=CBNqw9CYJ1)W4I&Aoi!jMpdUicO_z@FLC(cB zIOd1x`KG>1c*igf23DhY~v0M;O}QSkhl z*fnIV&YKNnyfmW+n|4R&aK`943OoK9L36XF`sS0%w?Q*!&}XTP}; z2e*7uMw_E#t{5*zU+c8KvKd|P;!UiZSovXc;~(KLl;NmW&zaZAAow1#P&TIkWB}UHW+J`))n+p0WQHK78!G*$KQ{xtHzZ#DNG){0N4|}HvTgjJZ|D*e5o_k)x^xKg^wp|1L zd;64XTJm5FaR?}3J5Y#Q6pLk8{qEU~x!vLzeS4+WT>-#1kZwe)oDMVuVl2pMaCAbQD*Y%$Bg<063TmD6Q@2~=4ydg1i?4|stUC555W@~HY9j4x@nR7y->o|p;esA5^ z5{5l{BCv%AARde4%I_RQQ1FBPME$i!`_48aPit^6yjf{LB|MOQ_K0S+_q3F4WlZI0 zF!$hxq;lKH2icd8=(!>Ua#@Antv*cfWyKezKq&VZGN-qM!+#1-z~aef6FKzem*@^R zFC43%BRAG1LJig3{tVF1YiqjR%1aFpnq$Y-q46bW6-=kHX|WAhwN77lr4q?nRX(1u-YteR1j|M;ro;?f7yK}D^O&X1%pKu=y877~UOo~d4O^{r zgrKC>+B@nZrIJ+!X{&k@;e+dh2>54zOV7N&%E)tr=Y1NvIBA&-C&ZFO6P4*SaDH!` z%}!6KRO0wE90}(5Zm(E!f}mVssfZ{ytDv2*P>nYW1u^Et^#~b>R>&XJt}aZq4qSZ)CIZ zMe6slFcZl0Ezfc>ji(vlxpn6TZS{G=?{ctjUyyeU6i^V}J$|tXfPRwUo~KLBKFf2f zL&ims{YFKX!tN5~2;R?A&o4{9@+K&>HQrS(e7Nzb|9iCP6mD&-Vj;PgwOgLT4cT8V zDGNAX_tc7CaJmErXhvmpC z$kyO_;z-=;cIEF0X$Ei%Z2rDygpkEdlwIxye<#arTL>O7sSup&u}m;_CSNfT$NHPV z6(LR5MS3m1pw`MJxwA?e1&^C$6SKLe#8tC|$8@!NV!%7D9kZ5D(!Se#UHT*OEu}dx zmKm=niD$CGr*kJJ3LRTJi)bTf;pQm41#|w_B4ij zEA14NxN25Zi4vUHHtWYrg+;C?O$Vwu$SB=|GwN9_a2EyO@(d}NuJ8LylYdr?@ zz4e%ldISgTA{wa(NdG!?hrp|qqAgF$x&O6*!aQW+L?>&_jng7O9M8&oDu@DqCj!+g zhAdfAOFu*nVOOU`njzuCFye(5O_H-s-cbYMdI}n(BazAw1y2=HPpD%a;~ik$(3?3n z`T^=yv(_k?L<@80w7}S){l|Cp%CIH7K1K3m$%T3eK(o?s^RNO<$lOKI#RgzKo*nuz!E{Z)`6V88id8Z=VJEP)8wk9~k6m5pD zcFkfwH(SNsv{t#_pNg0D-k-vYg1!PqiWfk|WEOtctPV&^o$ImHi`&0&e7hT{$4jl} zPgI;_@v+h^TSa-Nk- z>$PQUYyUI;!Bw^|Kfh!fI;Y3G--Y}w^gVsG7{09TGWDGaq#hkaCDA;Im!>?sjM@wq zBeHTi0{a!1J7{X!unyFF%cCa^O~$girRE3zN~ zz8+#P2eD*e#fvUz2rB4PV7?nOH7J$qV=YoMuFSz1 zV+!wOz}{SYDe`6Jm-R@M+A{Gem(9^&B|74dT=?#>ckcpdH@_liA9+#+|wQvUYF|Q)evq zM$w|KMjzx#2;qC^)H|B2!Razt%}4&2z=HnmxVYOXjCF1YA;EkLiR4$3NFIjIUk{pL zf)Imo2s=xUWo__k_xc<*(b;00E$|UeMNTXef8DjSE2yd}@>a(C+ zvt#c+j>0RPG3<)eT?yljKVWoPxjs3ouIctJx5hw6&EQqbg0U|sXU6T;z(sbqWS~Zz z5jL&q&sKa}$XKSJZIcN@oSL z5}qN7@~mNY3x8M+KY7-lqIa}XD(^cSgk1X#Igyzrv(&77XZwb?OdZd%7{itec}H`C zWtkKiu2QrVGAakIZ+epUt~VnIw}?hRsM{wd(HgzuBI9t#-*0rPZJP4%a}0-#jl$}d zYZ=i7hWBn6Q$-(1iMx8s>@1@kkB=v})`wX$W~g3>qGDxqWIk4+2R0LA^Stw?p?(R> zL43dX!O?v^v-)qajoVOuLdx8C2#Jhj1Obr~m0yPc2p*cGgu6q%S*zr|gJ z=NAz#nPg90Eozf%k$Os$)itp3S9nyIXKhNrb-F3JvC`%dMo5&~BN_CbgXC9?So_g= z)+7E*g{u^Ss*p?_ckNf>FHNp6R6i20N?a9XW0>GQq*FxTd_ZbMNjQlTDsbi%%0}o| zKX%m8W_ANem366pz;{RrTxB*|7rd3+l+Q*n%%EK0g3+dn`X|LW_;YgjeSur({%ya~ z*prdLGpPFW8`k>q?Nb8t5TqfzS9=m7S@^6O(VynHZCuW0n|HBBOE;*C zl3;S8B{LZ=V0C#K>7Q{yU2FB$+}xe;oR*F-c6`wC$&~4wd~|aS2jn<$ZNy9js>~9~ zoW=>OuX3MS={xDqJw8}(Jn6|n)_R}$-rJ^*60kk!YBf^&2ChwBtsW|THT4>O+V*owUT)=}8m~sxE#j}!C zKm`KMN91KO*|Gb(Awb>Wdt?_8D5X}Aw3I;3C+*NraSJ0)jJ)SNZAY;&oOrPJDiPr- zq{i(_kw z=xwl0mNUOSWN;hUV-8VbI4d2NbU=tWRPicr{<(n}5kZ+52r1=2SOlu&cZq?CypLWMLA(M#HkTrYvoE$imxo{Y#CJrLa9=W~}zSuRMTi z`i&hBL!x9G;Cel-Y~)l85P^@IQY(sk)(#F$zQu2R`gW1A8U4|4!kO4Aktnq*r;2EM z=_U4JI;=*Ydzu)BRk$-+Z?fbPes6k*!#&5=l3ClXgc z^hg9oyeAtmiNVg@v;jB_zh`KCT*;m7LlK512$0eZ83*8651pI62ukQHWS`b2&Ioai z*)dm1v{RiKybaKmmgr^P8%bdieZPC1UP25EuOMnTPGiM?lzzDF4D~g9D0Lw1zxfkS z483B0X|1*YV0q11-cs8>sHWmM!_@tZz*YyJ_h+1eS}laFWL<;cxLtPK6kl?BHTa23 z%9Y9;61x3n@UmJjI2p@R31dhOz<<0nbf$anSE)hhkWyfV_Wi>|sc?=VZ#L)0)Qp`^ zU)FZ0%WqI)gwGN&sg76*m8gw4dpLq=M~Pbd+c;>*)C@fqd8J1<8U6lS&21Tv=MjyICZ#Osmi{7y_blf#8BlL z&VI4LTgGFMn#SlpeTeQ3M~uJ-Y}gv2d{^A8KE!9QcFMNnZHY%RUZhdk-@1>qPq?2x zrZwgW^u+S7r`*It78ms0FmxzukVFs;a0g`ET3rfcy%am8oey4LGcmzK zbu?Yu1y#oqld4#F9_D%c8dI8vpf02?!Y;Fb>P@9mJ?`F4sh{#6_mo~HN07ahTA`KK zcs_iuwXq%k>rWT2CM%Xj=Z>ojdY>xA z|G{N?qG10`F~WCPi0B%Gp_t(?UxW`3Y&pZbeVw*uA^D;?0K)qrJEO5Zfg4$#e%8>6 zXCLX*p9ycR^m}pNz(g{6)F4J0#Iu0{`Vk_x_{x=KVnGHQb*Uh57l z)yY+o$b(P6`$B8z^3iA6BfH{L)qx1iUDHU;fmGnz!>ax*M61fbDvS6r-Yk7S_??W> z6Wz`j^Gy6eJP5=Te)@dvg0`^G$s|_nU@D65<`YxQj_VFsk-TQ}r@?YnN`lXbc(G2< zwYGz)&a^ILww1;W`M4SL8U~rDp%3-Ec>bC*=eV9x@sIo-hP^}Dl1f)Ror=Wt2yZ=q z8|X~Dia03!j4Woa&|^B(Dc;rWtxZGVnH`l4e~99!UmpZqP`gq?chv*rmKmA3!d0)% z_+zte!JMTNqTJzlZ-=Y4$t=EV*|Tc69367mbDSD@6LDzEq4+op?nq9%hfF2&SAZ}m zP7K~=u1y2I;;%bq2XLCr7{*IWP#5GQuk7n%lrhX!8Uz@2|61bOOzXLZ)71Z!z1fO> ze!`IYI}Vxgcl`-|#?M~fA~L-{7aY3y3oI1nX(w6Kuu5iu8)@BhVul;}d6p#gOD}Re z;FNCfn^Q8&+dX??e;4{uOrpFE-DE?iygv{g><@4sM(KLnetDh3Q+>#6baQV7Oa6$A zKKQxQv_FyBM567iJaMSkYMwsZ`{gwC5}zrv?pG`JB^MFHp*rjmH#>)G?(dtMETq2) zH+R%sws0r&P zeZQ`!-;n^Z*NzS}B~YeXQr!daJYj)d9-63PzwBef`;ZWaPW!vlW6Dh9P&kjR9ZFUB ztEvNk_28OWse*O-X1CtG^Zhl3$gYTH|JV$o1VzmAgCRew?G04m;k@pepsF49UxaYF z4>TK_Ymb-u{*e4A5fz7vIfUL*+RrnI%@n`Y@Ng*fuXZamEb8s>l!TS@pM*7Ff5%>L z%jgpvSB!T{`^byMdrB6LBND-+Xw8P<*bZ7Wr$&s;-r%E%S4!v29h8Mx$?9A?->tsz z-4Q=9JdwPRT<=D(+9n!=83g-yDsEMo@mc)1nqcYAPXW{{Jru)X{0AfSA0W{GcdyBO zR(OZ>iZlWMfnX4O!rfnKJT3tHC%}KMPqv?s|NFoJ0((5C_Nq%D|le zBjW&olZxNc;Qng?z{bYGm9+ns3c|(?{$DNjf7=6D**LlWX$J&w0YLxCSXtTrM+RW~ zAA7IK_0A zvVvLv9We;N#rm%kb}lgIzfL&Vxj_H6<6sB9V$1$JiU0p>vU76%&kVBz6v6+7AM^?( pWBXSb#P%Aee+{zzPp3IRTpaAsWw^KBP&ioGfXGx--{eG*{||N{*Bt-= diff --git a/reports/crpto/figures/crpto_fig25_price_of_robustness_scaling.png b/reports/crpto/figures/crpto_fig25_price_of_robustness_scaling.png index f36d241d62d08f3a81998bdc5f832e60cebd9c33..966ab67a4cd4e716365b7ca2cf905b64ae09e1be 100644 GIT binary patch literal 116119 zcmd432UJtt+BF(PL`A@kB1NSt0!o*T1w?6Lp$8D@B?yEPs-mLOM0y92CJ=fHkYJ%m z=z)X|(jh=-AwVd1#dFSk-t&Fq-f#Tl-usU~V`z3r*=w)zJoA}zuKiv|OO5#``%wr4 z!mNJpjxGdp7#z~SKYR#0z32Wu0G?#s?&`Zef?2tFn7dd)G|k1o(=$<(9*!Wn3f-{Wn%$ehq#ue8~EXMDc z7`=JK9$-hj9}VcD*8E3^N;lLarEDX!)G?_uN1hLPySg-0XH$GTs#W+SfANA7|Laj- z$UsH?>jmvOZS&sWUtWZK{QFB~$VJe;za9Y)<{QJwp zkiRzsavbvi{fTR@N=xhcZO;jum-j4s{rYutih}n6yHL_l^&>8?$rHK8)g>Bn*S`@O z=*C`|)VQk?kS2Wm{HjYeBt`uBmi)3Z2`_T5-qb55k&hqF2;IMP=XIGwzb=>e!i}4V z^_;?BM$fxeR#`q)Q&Q$MY`C|xrftf0YB^5WsG|8c zs{(JiJs~E7N3G=(-vj*`2;}0$H`{+k`tUV5_l1u}u(~HK+=>qCMXkSntu(t33XB96 z2P-MfB|2i%Z*9_}4#s8*FUIfG7p$UvsqEB^EX~B9&&9S-GLWIu91errw*pg?4L+;v zV?}B-MC#Ucx3-Ce!XzS*j{eZO3`&wL{5}#jf0gG_npU!`qOhoraWI=`%uI@R`?)Q? zn>XXnfYo)t@Za%IN1?*n?m77Zc};EYSYPVKd4yUViY6j! z+GUh%OS&(>YJU?i)A6aQp>2Ar1)(TT-57*T&F2+bF%2pX&8K`4ejp<A`_~{=*E_yD60}Q!T5# zdZt?rHq(&;+U!S;38ox9bM@f*)qMTp&ry6FY>=_hGRMr8SWy^naX-#IHthR)6Sx1p zTen{Iq_a@Z{Db1 zWMXBFEqOh00J1{>R~KSrb{enB#LOHWEP~28_~1}fk}@n2O!e!3rhDyGnW|0AcKgCQ z{A}jaPg+SbmkR4P4Bz`%r>*`})JfQ~dwtY&tO^3z*M?ko?V)lDG99RM1hb(2PX`N< zoiMu_C-=2Dw@m|+Au=M6i#i-ienYEC7X?8>kH`Iq4gsdJE|d4tf0XDUjr%=Q5)!Uj zccchAv)fNJg~!H=TO*3f0_;`dhlYlHNU=hCT34Uu+pLX%HztRwT}4Q^l=>GhOMdp> zR3%+-^e4?e%GOeU`ZVT2lgOC8fOblmJnPY;tG2Yj4Dn=~-~^#A?Qu(U^T)T$7i4#? z2?(g!^?rRaHqjzfSf;VRP}(7zG!2$;u>tasDYxIIdAx+JE?W5`nMYh4XInkfo-8Ci z>QgFVk8eFG2)-E|a$a0j3N~Uq2E&t5CGBI5amaLL>5YK(^}MghywUMEyQCd2hy<1K zu#Oak)$dHA_Y$QXR|`s0{r3zGF|+eZ+VxyN@Nl@sBNm)a#W-F46bZABHuhO*(5-QI zd>;}L6&H7IqSE9=!J@x2(HI)4Y!=C{Iho?U6p<|F zA)5rPOwHI`-=mT`qzlk`eSV~S&z^k_;ZiVUm#{HAwwu3p?7Y~|zIJtWb(e`y=@vFq zpWhJ^&p_X;js>!^52|2r8mHJsyh#dk1Lb+eN^`Q@AUfuo)I2rLgx8nZ;zCLYg{Gr@ z(vmCbU{yD+IB|}xPNIsm`#i_r3-F4Fv{!|@XWozgqE`rQL>Sf~$~BT@pE%W#pI#AB z*h}?URnL7|kl?e?ZyiJNSm-k^^_O%Ucub4xH=W0n)3HM(mccg)xCEvx=iO7}y-Iot zjP#yaSWIOlI(=GScMDo0Qb}kk(ikjL-E1vRi`^#$hP|7+2;ynABYJqS`uRKL*Fqjn zEBbEhk4VRBtnN}qFTlf`I%a2WCOms~8y=dhfnGgN4O_B6F_$5G$%QlF-)CW+zBypJ zZoNvA#gw{YL5%wZGe8VgBNk$^;A3HdjI8Way{_+^>RecV9E4gsQ6-rT8XsoMd%>!BCk!l!+ZBLtF(5!rcQKgh$cna1eQ~m9B9jBbT zMEjOup{>am^+%u=B`YFQ2+3WJNVq1(Z#Tpd+2o3i5uO6Q5@YJW`vhNTDoP6jpS#J= zO!Nc<1mt!#xD(8*G>Ptx9&SW@n@wPx<4{#s4t-fIYQ)V~)^(xpMn>{yYB)aH(0Ufu z?YmDVm`W>(hTy9=EaBvIM|$h^(pV_sR}RJwpKsgGMy}i?`R@xAB}c}_S}#yh9kUgQ z*Ueu|m4X?M4HiWUg7hOj05jAJW0!2nsZnUSFvFcnOBG&$gyvu zlHddr>m;>xs+)h;1SeRfJ=bZl1X^bH?LefITw6GZ_+6P>QjP=9?K;z5{^$T%R|8yS zxovCQjjVU?4ku0#Etbc=Win^>b9m6vxtS5k<(|-TGeu& zJoyZ@zem-ao10^ZQ72Sy)ueb$pP#5>RotvE;h+W~G2u#kedh6Q==NkeHbiT_DP}~% zciRmYtp~LgFIa-L)7F=2{MX01&k!JMg!-=MW~YdeiG&WT^QfWETx^J;Fm9!ho-gXB zf-t{91ba~xx_;u!8B4>G=jxZd1AeVjM*R~F%WTZQ8kA0nTc*bS`+Sn^<%Uk=E18b-xzm2(MR70wWXSMB=c zVRqjOq+ADqPtx?l?Lqx%-|st}1Oct2qAe#!QtfJsP$R0dg1{_2WVV9_@x9B+G(0p^ za)Vy4qkKuxjZ^M3RSQID4adO>Nd?M~YZho_huf$xCH|@&(ji`N@WQ~^(ySRVM>Ras zWYBy!ZN_yN!x?>d(9NjA;l`Pu><-ox*^zRVHLJn0szpb86gnxHenAr;KbgkzVHu|( znF%=%=zsaZtA!lTK_CI2Zv8+Fu$F2j11g5p^spf8YmStyfRKnpE0O}L)1xD7}AQn0v zs772GF)*_L9jyv^l8sY#a&akXn@P|~CwA3d(?klsKL}yonJ1NC#*pD4T*sk2NKVGh zl~6*Mh@kzli6{$Eb`vDoHQ#WgD6?6@48aTsCbfwcq|+=&k>Hqx?P6LA6LFKfe?aHT zl`B$fr`=j-8y24y7@kRh6uC^@x2T)lkau6uM|bRR6GgFp&pFrKDOH=Z8Q~#jIfs5O zhEr4hwtsDg4Z<+PanQ(f5;k_6&cVUK;;#zZZFKI2Q_^n-u{Q-tGDGZiKf)pP57RPo zh~E*^S;d@hjJpLJ?n^_qJ7J&sA5>Re)6jE^$dz-j&PfiUMb}Q}Sooow zRuDq0c{OSC(bVKKw&!B*9v9V@yowdAyppJuH|k9=@rW-eDREH5S^Sh${xWF0I8Ywb zAz79Et!>b$rfpizEwy(-FEEE->>65}I*Yzx(HP?M&4U?bQr2z+IDZNE5jX zd8M7@+vtw+JqgC@WIo7{YllMPOoD@zwtp%Afz#4-%8`Bf#Q76Es``GZ6$%T))z_~N z^1RE#m~`agiI0ex^o%U$$QAHs0bAz2twdBWqySwAkFNN^Q67Hl_HE9J@P#=K`{Atj zrU-}G6$evyUr?{eNm1WbSaUeuX5J*uAT$O@j0tAA3^mq7av8`}*qMapQoG)(8MPba zK&}i!RvVXK{sgo_;9UQd&_^ryy$T5|97j=kOJabNFb@T zbT9K+-du?h0;aKMrDD>RR;gv2TG=Jp4AF-tgd6c&%~zjE3aPFm+zYtgM*n z7iq?bd_wfb+uq<(qe`dp_2a#ew}*sU&k7qpKO!}obEw@$)SMXHtY_+5jR4$!(w=zYG$_)o?mu|2HjC<1)>2Wp z-=+{k))$&T44Fx0iFJ0Z!Wp;&?Cv;-!+IAnO%%se(`Bx3z|~l5NSol5%XI?a2+Fz5xwIE&mlnt zQGLdXWNF(mHSamu`TkrwA!2}uN}W;=EBDgsUOqJf9#DAEkg>{TA?7Vy=Ug%Y<>%_| zkZ=mh!7x-{8*ZGedMZOJMc%+nd1$9ILQ=k*m3YH~EB6-tTDwj6%R|})mEpeMBW}Lz zs7;m2P#BlgSrVjE4s4xXi}3C6vPUQiX&neq7!E3+FAa+$l=EMj6+D8+*$-!D_zRj~ zrcaozOV4`^XwVa-3W>y zAbx|Z6q>bhxRj*%m2Ipbs2(5Ogy=HLCe(lQHS0@ZVRYQ(Y!}%xG`&u!kAbt^vu9uK zo@6Fk+PwpEbG#j%Z8w{PeNLc2u(x!b01}MXk`yJ$Tokj z6XU$WN;1)5waC2v-YE8ww*|t*B+k2<&8>z34nWXZ;#tUjiY;;Uv&v~}avuOo4v6mg z+upYaaP=R_66~H`ui2=+y?c1B1pQ1Lk#hQue<^vDlOdvvF$bP90edQ+2+9 zQ%)t9F_3wC&EoT@r5S@hG86N~M(lVM^D@R7a*4#tmJ`I+*O<|=pA|@L3c-?N#1LO+ z(^IrbGwO!ZjPc5c(M`VdefEi;UxrEIEdFWN>+5PqAneEBQ?)sRHD69a@`rsX!!dK^ z9ENO0xHL?KW`3lv#E};HV|8IUI>(boWo-)8-B9x03GaR{$dB7xAX+T+6|;DE5vvX9 zDflC=!e-rtuGzqn(=ryXY9=_}p&!O$yWJn#%F+TEMRl+za?w$5P(pS*d{_7^N7uni zCi^s2F`%1QpZ0W8!mJ*$13(YcB<_)#v?HFERkW zg>re0(Vf5X?Ddd0sg7$`I6;RBkP~~yU?ogcEsW#E_Ti&POX5A2hN`z>*iN0YSVPSB zi*QG|ahQbTK+kIJlf0)a*4H?&w4PK{Z|iCqA9=DWV&UZQds^*c37|ZTS6+T-HDzh-{Q2|aj?<*g(fvIET4^Ao3F?p`=O4^mR4w9e z2>_0|xw?EoxHo{&0Y&`LAyJRT0Y0VO^~jZ4I1$glWse^;Zi^Rh${>E1-yd%Xww!8- z*7DzS%9~`41oRdr2a~1WpRnG zL@6%->>U(Yj~ux%&p;fNvF+yl)zTuC!pbg@Q$I!A0X0zC6d6FRsd56;wF)LD7oHQY zxZRx9egRZdKh`!tg5d-4Hbym+{rsqjj_+AcP7@nnD_B(Uh{Js6?ysL4|n%kTQ#x=p8>k(?j9GIp|GMl7aITNX^<=npBxVs6>(-~?jth5 zJYKaR(jNxqnxNcni1b-nTVpk)hj~Gsy&>|<57b-Zwh3lwDh!|y_BhX~2fo}QmFE9*Zv)&aV^2UAd zWAtWvmg=LU;}R-ox%{@fh~;PowFMC0&wBI_BX~#_4+1SbYn(=BqVP8Q&8!&_57<3M9KNC}NH;=+~7*Y1moQVyIqko}LGaJ1#RTs|ApH zLj#HkXphum1Ol~!*j}GgeU=TsPI$OH3+KPO^_2L?L$VM0{)<#?exm)DQxUuh|F02Or`jS zLNupIj{6(AH3QR+l8m<(Yy&zTlH{1TJCbhVsy1%|x&v?#3UUuV)DY(vBl)wKJk4vAoPR(s4D|9SkK9x{9Z@&JP)+Zs26|CWeFqrK8@$$Lu(BSk>V3 zGpYVg>zDCk#ws(fG9SbmzIrQZk9V7swjnuaJIT2ocvVBjNEYlDoPtQK%+QF5FmcVY z&JKx7_lPG8*cQ}?#n~@kXexF~edxZ^07%r4S?0p`2N}rjqUuXX*y0Pv;YS0}JJQJa9;3u$`82DRZGT3;O0yDvnvSln6b@aPk& z`htJMAD|I0c9+Y4=LJ?VQy~UE>%#e4CGY^k;q=*^wX@8EhKvs+oGm!VuC#9xc+->% zX2#e#6>gvUStebJw`9-Px_|%b^^bNYFjQpi#1+)2T}gO0(O=W5Gp%l8Pg+0*P$&&D zN_NizSgMhgZ;H8&4l8!4(0?7s>+|uZQ{bMhMLd8c>%s0m(hsM0t9nluxmx=g3Cz!J zi(I2qCYsfm1j9RaDn}4l+Y7oBhBXx4w>ZBSEE?_)F13t?-c%aVhL86e?XfU{NvHoR zO7Q&b_!iV8G5TJhi;brSqddTK{gJwh20f4jsDw0*2|nS;y1@xoBCMNkN*9DdQf-nVUUqzyR&{qC1zr_CGLH%hGGMT8JL(XxoZjmMQ3 zSP_fnaG|Nv#4D7*NsiiYkxdB#_=b#d#jE~1D{5$Hi>wA{`5o`WBF0rZKHtzmUj-2H z2?Pv2V`(UX;pJ`0;-%Yf%4W|;)B64K(9NE=NN3PfH3qDA!=n~yj|j*A|}ydbZ@ z`-a@t{qk%Kl96f@X;ZHYv;Oj9#~$WpJw7oJn%+=YyLvy7QXazY*}IkMy;OY?(-wwF zf|+{$X0Yf(h-)?$Inl?IyC-J1vo_ousrwc?s0n-UdB)X|YB-9zOGxHtvD0m!sfj<| zpW-tV0eI>AAGJ3?S3S^;-9h9&49+=jBx;x-Iz)6?LQ|bq_}7l3U*4kMla*gegrRMZ zRiZ$>S-sxqVbHA?Kol1%NJIW;msu%N zPj2K6fMmJ}UgP>}KT(}s2PE!&et&Rd&>3*9-57uW3G@h`u#^;jx;*bka?O2M!K;!f z8im=O3ow;Hk1EPUc>fCN;D&RYLuH;rxLRibz7J*6-z;-{xbXTeN^{iA56 zyX{Rib#0Sud?Q=OSisWYm_;c_QS?QrVLA}!Th=S!^!9Y~SyGAliRWugoKjpUYOKkT*W)vfj2gO}erp zDBV}m{MhB1>!ha8!OS63q)!cBN@dq~ixyBl)=8>lxlg>l%P?Fs)PFNAZQC?ll5^Xi zh%abdbXf>mB-}xLAGpZAL)gAGD9Wx;mQ^WS4Fuhv+)+QbHbC9pIIA_iovE97U-jhO z=g*(N95wf+p7S7zYlq~FKX?P65Gdqs+uFkHUvhJV@93py5bi+TLyr|*0D3a5O3Rs zwrmxD&haOraZd}O!$cl|sMB>yd2q-*Z}%FpFD1L34GhBsoT<=LE}hI`02?-13uION*!^=`p9)ywHRc;8IwfydEEayjZvgfx zk$NI8Cr5Ci0y41XKiiouHF(>bIfj?`8(*%LpGO&Whws{p&V+;+q-;fBume!Qb?}hrLk)83tKs%d^&TGo8P~mhG_1EqE2k6-RT30~* zGIF*V3Y3E>u%vHn93c)~ogEcq7+wX!a~f|1nA;Et=3`Ql!#c7Q#yk{lcJfT`o>lr z>9aKAFFOvDMRb3y`iI~(hTYfza!;~fEI)(oNgWLhja)}1I!HiC@E_dZ8J&$#9`P9U zRXv^#7WI~*5yr2s3;Wo6FqBI{)>M?UPOUP2HbeLaS>A#S#|*lo#F_RMGbNacWw~goG@=1jf^_39VP#=! zM8)Q?ke7}iluTTeZNI-h!jfG)(sAn4DW+p_`lb%@akrbYJ|!DglCEx8 zM!j$*JtdFEu00?|>-8C%5e6U6ejH%vO7WBp-7TJe4}aLXV>kZuJs7N{J2~IO0zvs` zxFM>iy6imFcOIbgHj*bW--L2iGSF1eE}T&$^5{V0$O4~fXUU?Hkd2{UxS9f+0jmd8 zC*Z3@e&BcgRMw;HjljXF z*kiuUwCzXTHpLQW{pL{NY^ObYQT3>$vO?Av5fi)8ltmMG?-MD7VugpP`h}t%K|&su zCd7AwsKwSY3fvr?Mc&3wYFfi%@w-4`$eJQIu$f*1!m!3mZ12I>fZo-I4yK+eLlcay zoxW39%74-rF_#HdAZ)GfjO9)YUm!MIpm7&jsC_?BYG`59B99eNJp3`1Ae|iMD7z1q z-Pn^2wmy@&^ye_ZG&I(^k`L4>aI)(MUF{7HM2Nzy+4e+%3WX&Spx&A^y-XLT?RxS1 z_JMSGeJ@%dpC$e4k2mwnm<_i+XW<~|mpb5xS@m`Dzx2pkxLWQ@9(36)I_#ZU$M#;e zp_bZ@SbJ7$D%00#szjk-`PI>n%^M0MK+fNAlGz7Fg7_i53u-w|i$KmMFv z{Cq?-457s~)z>wW#-kcibR@~9B-)8yt&+Y|_7>5yU}k}I!_idWG(H%&$D!Mae!f4r z0Q>zr|JWN%dtZ4wNSoiL=kUJL_AC$2Ia`yUv!`iVK-zUY#&Tn!UsX3qa(F|wMv2~AVZ>_FRGY;S^ZHCWWxL_|&h`75zQYpl&fLtP4#DOQP*SIf!scOAFBg7ez zR^%QJrqCSIgCrfAeFd05`qooK1(gdjO5)U-x=`3I=e+@B2}{{LqU*%TqfAwH$_t`{>sP zb?|mZP)43TMVnT4xqa6opy$iv5#;%))mj4sgDmS*U;Jqf4hNv;e!9ZL^J97qNUaI7 zZnI|BC;)qjI2!=HBT(}ujS(^|w~PK^8w(bR_#*I;S#D`yX29exi{JYl#9a{&rCrH?SS~1y{bE=*?n`paE(M*x=e_ZKg zf&;0-F=h)GOa?S)yj{Wvdl0Flu(beQnIw?vU(#o7lZi&t_5)?Pb3k@y$?hG^z&*~@ zZ7B1lx@Th@fKi}rwZfr)+!=K{h$S8j$8!yu^nO;q9n_Y6JBq_FC-jyTZhw!WgnJ-b z_gr>=XD!rs0i)>ZXU-Kjafxq8nn-W^ z0U=zIk(H~pGHD#iq`8KjM%UO<^pA z^zN;KSO6s-JVGE4=~MT#w4@}#svF&=skJ8lyW}i^6wmMaFq%#~5)Pz4KxxPhjQ|%= zajPlX#|t-y%TD$qg5u>k&s|e#k9E&G5 z0nbz7hX>&kfU+FS$8h`a6xVwJjox0oZpi}KxtL9CluzYyMKLeTDFQtv_8e|j(P9+; z$kx_23-tRSM^^}5LAK_v<2E#OOwqm^likY6;XV(9{n z`Qu;ys-nRBlS9tE)k?OmFCUAm&(Em;ZQ?O(2^gc8HhIK~fvAh2Y1P6g2J@opai9_K zD3!-6N=p+23wKu+xe+&XMZXvV55$Mjb(_3RbFe;TK<7hWqf@z zU=cYFxwK+$z<5kVw`&$fuZ{vKRaNI2A78`Nt~&j5lnAgm#lzyaI~~6aB0!1AAYu(_ zk}P1#8^{+OW^83chI1=99gKw65EUx`&^ZvB{c-~s;SzW6pE@nkU+;5ir*hO!9A6$| zbn41_-S;^9wlFSs4q&ZIxRLC8Ai&QZfTDihs<^%bXjxXMlz)F#(Vhj`4p{SbAXO!R3#|ngd0a=S)Re+rgc*CqbW{QV z^=W~fAuy-?7=1VdH?K2sGr}x?XKGZmKr%_hWsMf5DlOO3XN{NanRIn^HQpv3TPw~O z&237)DO0GzM%}*U3lN;Dm&A*2e!jMlCqz2yZ#+aq_TAvKZ;<}NSfojf!?fnTd)F&+ zGk94vC%Y{O9Ipu$`jj(NNT{NOO*WkXGe7Z09vypuePhUZ3*fsjM%RDjZ?`eYfDgVu zB@i;RK?nDpMVyy&&;J4JV0jt4J+c8X7KwfVh)q0jW4hy}+Y%o%$T|nb^vjyr?so>H z;$0EY(tQnjbA(@MHrdpi8tyj(JdP95zWT%~#I-E0A3$lZ&o2|{{CT$kM;0&OPAXz!CO@vQV#lPND2Hvfa2R@=bs&nYLhUt6R`(;8V45v*}V~s^j1@s zpLQ*&z_`hpr~&7OYDxU=UOuZr_?*`auvoaqO+GzgvzN;t)q^qK-7MT-Aj{%->^^{2aPJpa?by2nzqhsj6m6A_4XP9NT^w;fXohoUv~orMr}6 z&qzrEV%xSio}3W`9dqw-ZIjKhN6R>-nwc#E9|qj>#qiJt_r0C1t?6P`h>UliX)XTZ z6Z18j18=&aL2oqk>MplQEKknfB7=+;3SF?mztm^G*H1}bSd`D`%V&(wT%YGU+0-J_Jm8Ml@jM}5CNDhyW0aVYde73^3%ui<^vAoDu(q>%n zggJomjZ!WW;Ye#uoflbE#P1KEy=~9_bz@pqTkU%spg(?{%EzeDqh+_>@(a!q=X>*v z4j8j<_Kv+N4M@rub=-HhJw@r~d5Nbs< z^IkT9sz1APt_-W@hgKp)?dwR{BlJ*`y74(jC~?<{%+&?_%o0nKJjz86H}2JL4HC~j%?Pfm z`{{9x&bQ6Q4T@whc@?0xd;0ou#-K*qT5?=qT~Db9J}dkQP+u1YACSl#+}xsU#?CK) zc<-W0c0s{s2|$R;`s~9atgO;Y)O-pD63{&nEjg4He9%auJ~oR|TaKPt$u}qs@r47v z|Kyo7Q*JjAqZdO=!hBY1TR`#`;GR1QcR7Q1s5IY4GTcJ6xpGIr?ROnQN&432=U8^yjrIeZ3$`LH$#6t|7Z7xHjt`fZbeeX?=2 z`=a5*CWu%GFNT0!)cF1fJi&i|x6)yx)*y`I<}-&%uwCFBCCFpOCySy&K$!tEc%#cY zbR3~X2H@cW{1aEko8+J}uC6UOXCg@x2{+i^+YN=EI=nuVJ~K%32Ei-#kf6udS&4{t zHaG5`Z4KhnVNqbPz6jop0)p$6w%Qkd%lMTgTfY90_O>=jd>&VZ_r}5v{6pYrFZR2S z8Ud5V`OF{C0D){i3;+c{J1dwp&eIqtu;b{jM?LHxEYCmx@Yg1v09`>*(GJ302qfxO z5;u5P=lDN&@Cd#EOqKnbtgvvKwIv-SitE3=4YU!&6P0(-?1f1Q z>qr41z;li_AtlAhi<65>3-sVT6QoQOQXjtJ!9_a`z+`ALMKQangPixT$3@y+kTC`o z6_t~~U%bZtuixJP@4x->(7w=TYlRlio&hTGXSU@59odJbkTh_y`^q)U5HbU#ezYr#q2&;m z)uPwqH5%XdY)e9Pz(+tk_JT2Mv}4E)r=ww-!x)X^|LgI;e=FNRPyOFcoQ@WO9DnyJ zFp%%+)t2f1yViqUdx9?`1^>FX(Eqr$G$XOUuIzu@#P??+9=^jLsSd2p!iBeQ-J*Go z|A)@)9P*D1JzL;mZxE>(1343w#0U(p%=8QL*qQ*NM4xxZfcZ_NOEE_wg{@_%s- z{QtPBUxR&g$`_A<`~jkHA_4Nx8N4a_pELL%q@L;3t5>`dQ9qbyb8UbPa$Imzj`eTz z>)w=&!K*kE;&&Z+j~fqv)X@V`Bw?<0pm?5fO2cLj%qHuVrcapUV<-9aQ#xR(7@# zG$<%262w7(92S`%%N9GyjI`r1k`3;FCB4a~zcu0iU621+b$^XmK>6!OL6fDNc~ca6 zqvzOoi>;?x&N?3-9}fLEA93c3W@dtm*RD)8`UT_r!#HE$Ty+#8#1K%% zB0{2ddf#8&c2G=ISU#)HvHebC);XO+G4`_3(u1hP7(X6I>$`V&hRS>{3@-f_J{QIudD8uw` z?Y9$Rd+mjLFJq0MBn^ct;_d~a>7VE_S^t?y|NQ)pu8z(}iT6mmwUL2HMxUo#^@Ll# zCScQ=w$!Cdm!t~m{?*~8e|30^ZlV;gpuS&U<8b=AQ`NsZw z{)zDE>@TX()0UU(&wn+y{i65^g)K%f`FRWM*Gku2##QX|i1LX<)!AleS~kqDdM2V> z;2G{L{;_@pHaS9zw^0l>OI(fb-m5SE1zBRbxQn(Ai&2)S+{D;lRptKoT<|xs-l-c^ zPgSMmw9j()UkTq^%X3h4)ks-%j(Bl_;owG<-_hf~EPG4%p5uU9& znk}yMVt#pepX8MhyqD#MzVrR|t2fP@wF0y2shTiUvS4I; zc5QU=EZ(a*LHFZNko?Yw?X{0R*YeyQ4|W3cWn z&%MXgmyJIbf|KoqJ0xYCIP(k7aoKty>Gn-2uunvGkB@c{XB=V*Z_kU~EP0Scu7%F) zq=XGl0TNTQNSNmV_$Yl;|7q-OK1!2=r1w&U&hgXYZ4hH;~(VcW~&x&FV z{#Y)+I&g5y0>c6GN=}ZoKW{tsc=~2GcFCX~re+4L49l;EhKvF8Qd}GM{(Z#9$r!EVaAjY&(%tvA{lOadgTaEBuw0~^P9D&K{Xu? z8tAOd38MQK7K>=!CtV@k3+cVSzu~gyzv)XfIv6*nv+r+;NHu;Yy^d~orL@FO0g=&%Dwk$*h|aRYitN3WEmX!Esr-}*Yu zOv9KNZDfk@lgn5vj2;#(UBgm*tf`PCGrM_OyvbPPNB_dLL@w&ot$pSJ>G#NIUP^>0 z0e{Y+X=gv#FSq_)PNiwoRDcz^7PWBf*Dq{&Z3WjWQ~Di{U>v=6`@{Tsc?1SM{Y;4| zB{rQVx|h~5Cl0)7sL=CQVM(TvHoEEUr7PtzMeY6_z2d~ZD2FMJ}RYS@H#-cI(`)iy{WY0B}bb!4)41oGbFvKhv$ z_BX%{2SQ{&I&T*li_P#Dc#j8gbli(9bVo+t5VC4n6h@s{9;`Nj8>sxP;0n+c=!g;+ zT`&|C5bGGI-6tc-pAsD9G%!3&;R|J6bvu1rze15k-K#Zjy-E(U2x{Ng zz^BR!_Ip*Yt`8^)laz3NX=R1FO{<2nE1Ibhx_p;gR!4Sz$bG?y!tRj^1)Afcm0AQ8 zkkPY6&HH{-^TAQ{X0wQe@8xBN_^R`F;$xzscvn_dVhHcwzXxvd8JrlytE40jbuF!m z+OW{j=+4glrL6;y9yW-LDI~!EH%Xp$4Dp|VIS#-)3S&*grNSwQzT!dB#ECV!L8La zOq;t{o?y#!`#UdMSM&T}MXK@BQK@PC3?;W@wpj#AB;V!|d%I@p<&zrGd7Rtm5!VNz zFhOdM$tXrihRco(u5f-G*Vy&hn za{UV3UiiTPv9N;>(sCuJ7e>({Lv!7mUgb%2`}EPQV?8UdNNCGB*ft&@_S0$;szfBB)f3Fi$ztDr{P{y(#o>bk?1v*)A^t zb1=NNC`5=zXC6=1@{p2ygoIu<@l&e0D^ftXzX4Uy`h&mrw$ekO+o3+y_Q^g3kEqAT zMD_Ald6N|{dmuMQrsmi^+f6lD$T$m={SUS82^ZdDkXA+(G>``E?7&+Y}`L z{`m9|!Ctt{FYfXLVeb-#yYKRv2pVU5vVQ&))$;o`W^zDv@yK4R%Fuzd*A9afnps-O ze{OTK0~EYZ$;l?Vrp$kh+rih9`L3%A+c(ARO`pBQ4O1xEs>`Os!>b35upWtCl4SqY z*48dQlIUua7y@0@EN=M}lYHgJSs#(Pn{S$SUDj47)1N3k*jG~g7-;?I`PP{CWUol9 z+_Alx$*Gn_EJpg({`~6p3th>Q+9bX5lodpOv4R*J>k>2?333U~SzkU$>|HE%#`H=4 zDC7Q}0|6Jn4IZx1Zu?MX)AfDItGl~0iEmYl^}Ii*DcNZuQ{(-D7?eA=0~wdgl3M#XtI;Al7@073LvB3iRKYoh_zp_*Pt^%WsNP#h@Y9Z3f;Wy&=cLf3>{+Y7y*Yq^0lcpx62-gN!3e#rID;H_0W&v`kNCh-gwl zES{=SIMTQlG>%ZOX(?w)Ql#vA?_e{clrWfw0mH_}Y+t_#~th=<8dPk=h|sq2R^= z=lXjID<3 zeO&(@M9^mHAR`vsI^ZI#cIE935IIktK0Uem0XVl|G=E9s8Y7qI#nBns4Sc|^{_`0} zhTI1o@AkX@)q{|uF92+shBp-X2leP(=|BG|A^G>JtbBC52(4ftnK)-kDPF;iQzm(T zs!M;^g8Pm<-@!Qaiy4$We~-ofo9!FZH_l9uDy-z?F7@KHWxGhIuNIQf@&7SiIJJs+ zA(n4mb1PdZC4ygxH2U0YINR#fR#u1LmSuh0NzSvEh+3Rcw90HSW(s4u`Hr8{a)r+W zYe_c5{$HHEbzD?y+cvyF5DQTZ1Qb-d73mhFQ$&$g1nDk^9Ak?}w@3&gT}n47iol2n z2uOpJbeF`u=R$FNKlk(de&72Je{DC+tXb>2t}~9~JdWWB zt{4;6r|0L|YQKjcn$GZ(zz6M}2gpCQx$fR^qiYLCF_z0Y_VT?E_AVJSwQiVX5sb^C z{hboG78Ut|CkO1W*YI&?>v{PUQf(n<^3VV1(WZ-d(1T~*+nRuBI1B0R%##6*FsX8N z_v?VxRMP#GERz7?EnV*!Rl%WqKenM78HVh;uI__@d;p8;T{vB7Ew#=iM#nNmhXg9v zw|gz}`llHlDlvHWkGD%5wY;LyvN4(5&DRzeYy49J1-2}0dvEaNh<_{PHXxL^oonx4 z^!{azUo(-|xLwxBRGNIsR|aJV$*@NY_YYYFb{D!N2kaJ3rqv z`-|oB&*PgNeAZn4FWGPh_b(c+zY$${|Gaadul;J^hh9jC1P z!3OcOKs`q?6IV5OrB*At1CxxeEOIR|O*^TjAAra6Sg6n(aq-vLD9aRglJlJL3jNAV zVO735)aMV;LQgj(NyDF=tA?sD9B;Lu1L2E)sAKb-jWhMcipyKCAn0j>X4Y#FF7|kX zH-^%Jo}0zSmEY+RlvI(M`7-SK$nfx~z;{H%#Db@fo&)AYG2SNw(ieYk-dKakp~D=$ z5%t>o`f>BN;Y?nkqOtwq%i7PquAQ}J@;ug^#xw0Unp>=`X)CgRSNQ7eo7r{y{??yw zU1`}lj4E*(#c5UaC+3B~SPS2R zv0GE8U_<|BRlyQ`gifT+&9ZXEf}atCKO3ve(8!u2)y>`-VEOg!xc#Rp`G;dnmzo+L zW_U)B;I})?)_K<|7ryDQzuMq!NirARU!;@MUPSXH_bV&K9eG3`)+xJsQgo%;Y&&6k z!Th^nNd-oeQlso20Mx>i5ch>tGwV1fzcQkY9I8*D7v#}SC)%PQ(O|mNYUO$D(Iy3R z_URkh)lSx1(JCg%IDf^va*NN>xwGZTqSyJfdY!AoVy)aGLbv-l$W)Azar1sWcZdit zeOCQW;U!)I5E$3%?7As_+slq5lDEDnNRlaCa^Zf6gjM{Mlzqrzdi*GIp@XcwJ_(X# zwcNj}W?C}qkZCg}|K+1+sLoyMI+U*^D~Q1?sJ)wAm>jgSzZ~wm;M9LrJ?+Wmn{9dK zd!jqa)bU%^c}Fs6`Zy9ZO3QnMBfPj|FSQuee{hl4kbBl>!bsAzXxVpS|KaaKj%Q)p zE51Fwc!B!%?Lv}R(MA>SyjkAE_#`R&n*QE!Z(7Q5{d>XM#U}$t*7Fxnw49x&`W!PcxwU^`dE38sQ9N)aDDoIZ!|>u> z+@r%3lw=OFwi|A$9lo`=5)?o;@@4m<6^S2dXfjvk0TgQjcLa^_uf!+~jA0vav_o zdtGK9UcEVs(G(E64KhKc;Tmsk>;qu)ew@E@=IW7&eY-zhuk&&1IyHV(`^~6NJgVv~ zp}Bkp{dvo)m|b(y&w@RluDJGl7jMMy7m*<^e*!^YrcPQ_YSl9lY@=R`DS4M(-6%0# zFYVYju3vGxa=p;1fHc`gPHd{(*sc?6!e#!+Ajcim#YdePb2Uu>bZYXKh zj+wMoSZEd0oPwV+)Q#75!4?=IJ9&z{l`iu7D9q zewq&n6>7e&;`>K@K3&}CTa->BdB*g(c7$KB*4eOs=-$f!=6y~?zkjYYe|hW_;4C<% z8^AR2ScinsH%`EVc!ksh;@$!Ou1=tJ651*VYFy_{|KXPYtvTuwS7|%2`EP zES)@klcQdXvL#j0c)haUpZvs8tpS_*MqFsHZ&k!V{Mo3!Nmty3?UjC)>qE}m6V;4S zl@9*UxW3k6pEr`SCwhC!BYd(h?+mBoB!9_6=MtAdCe2_78qhn)weA%fS_iHnR8_!1 zSFC+L3iJ8Y0zP=HZ^3cnzPcFu zJG!d~v zQ8r%=lU2E<&F5LnP|DW32U;Ks2en1`JwZZc2!4&zXL#P$9E-Y!5Rw5L1`zTbS$;R# zRs0RH-h#uecq$j?cl)%xy}c1oO^EmQH^McH`oFC9W@-j$wrICX0H*&NTKSD-Rm1}| zncIEMc4!?qMhw6p9N&kqf}$r3YLEE8{F62zNW(sIq$emk+M+&)^%H2dalo*8NO(8{ z%uY6}mK-L-1C5cY3FUG6`1?DhF#q+Z-_8B9=hmvMd zIt1|I&cmEYII88jhfL%wsyBCtiboFRhwdjO<;Vodqd%|XPl&LRhiJumel&`M+E4WA zvJ*&JjetZbo9*`e&_~?6e-1Lx(_i5PHp@5+-(~6)slbr)xjd7m2oR4rdE@U^^8x9O zMafOj@>JIq`cx(M=R-gO6NP&Ief@tod;SZ(Jo~kS7$Lk0EfdgT{Z_tr7q|5Kjay1+ zS+pkKHK}{!U=|V{$u{`*oO*>WvCm;&_WL1(%!g5g{>w2_kd(Sv^qWo*cQe*Bkm}7y?tvc zK!$q%@uQAD6C&M9$W)>V}p&=0x#YM=_u8XdN6Y;bX3uhFp4VPF0lF8uWe%yfKA zwBa;){o%Y!WcM6PGE`3o&E)l8j3BFaDJF(fr=+ALBqYQTbLQAFx^w3ax$W8AFPguh z$Rqo#y8OLY>lqZ2>j(XKa@;>AKo&hWMv}V}ad$9*MW700kfntv8C1Yb5g^ z-EDH>zwb6VBg3QV5;eC53I?j2$gA59CM&93wBbPFP5*966ZWvl`x@0m(J?IOxw_qt z0)d2RMGV1PKL{;d9ue*YS}dgx&> zMsxro4jqyArF(bVk1W40l{pK|fpwr$6dfH=H=gOB{ydC~G!ix|=;UI#|3NtvJZmpt z_z-i3nwl*k;k5NGr|1t~DXH;|LmK1&M^X02m zPm8$8=&R^5(^~7XjM?WU(A0t5>dZpn|FR)tpQp`EL_&QXJ1(3Wpfv{ z%Shh{6Rc02|D>sLtjmCzgY^d2o7p{cTz`=VudfjkLyPkKXMAYlUoLotj zlWIwY83m=t*qu(>(I@wG#?qZe-tc*pZ;K)ef0NOejmcHcgve8mqLA%-dd8x$9IC7O zG*4$4T#CviSRY4*0{F20YEg}{eCnWJNHAlcx-hFu<)p*IM|-{RO$#Fef49}u6f%V> zoAzzu%f-{=7CWGZTdJ(e!f3M4p1i%NwUv5vJ)zslzO8{$U!L&G0KUg}Cg^cy5Yxw~z6E3TKdYp)jLtypvuZYPC@hy&r~8A8xNwU-{6G zd)WnN5gpm^XxU5NSwH)DxaZiT6$+}v!xyH+2i*SNHw?@GDpgH&(;quF+Awj9H8b7QX@pW!W*Ldxmj++1)y@1SAs!AMLua!!Im^cr}WmGiX(Q^sv9J%FJypPV2 zC=#!#DpB=~|#ohkn>C7OM;7}lwkQT9(@17 zZ#-BxI3)bsthE+n4g+Vzu_%*X8yAHj-QL$(epVd@UDvaxo5NM2PclSVjOBLIjg>d-3*x=@lR&5xn4 z4E)Ms5bQ)&eFvfzN{v(nJx}%=IyyQA0fA>uq`RT>+V9XA@0=VH!~&F@{4*I0!dxBG z;b=GWj<$0o$&jnUvVAi{sk#|TtlE!YBUN64x@ttn;NT#9fnCbmsb5DqHrEPm3ZA=N z>0V$F3#E&7lL1S4jL4CWYcp1DQmoQfe7L^+zTMCyG)4Y+E!Tb#$@%+0d7TS@QdIu!xJ~C<4#Dk;v8hjY7(j}Poa85{K9>@! z8U7WlC8A8Yql^ZFW4Y~WbfdiP$p*K}0__B)bqqgFj%7Ii=)!4ih-~AY-Rr?}pIO5X z4V;WBz8%LdUYH_>Gu3)i>Sxt$kZseWFZobPL|^7#0y6qn%nl1m7;X!j5G93YY!<3C z>@D1|FgS*~mm}FGih_d%ZUsgjMS7(FT_8wBvJu`4P7dk^cKn2=#&6qXevyWP0|XGigk4IEe8^pSq9tEO<`7Mxw_07e0^0 zI&m`Th;>#f==T&$th`X>Y~kk2Zk4Pppgt>V&a-fD9jOle`zBSR*z;;pW{_s+$@*Z2 zG->VH6U#&X)m|y-PNMo^$Q;Th-F+Ro5OS%hUKq$!lhG$}X`dX6#>fsI2(h3phy+54 zynB+iJ_{Xk?;xxe3fTAZa&k}pJL+RwW`Ukl9-RMv-*$?TNwKJxsoLwFFZp>Q&B-|c z3n&wkq6}5wc+L@$ZfmA%_&6Q>y>9m%`Mquxa)1N1pzvK1<289ezNy~XZTm@(T1k=? z1twFO)=VuSmxvXmVE#>;-2B{^e{G?q_Me6MmFsW-4XtcmO}K5Z>3eW^aIH_BjkT7| zm-34_F`d)u~7{OlErc zKtz%$F~#sy7WFiLcTYQVa&UfuK*9|2um70{pUbq^O=4gCPGZ#-BF^-3pfO4Our)`^ z=5oWwh9Rq;4ULVZawT6buuSgMmFUBHw!=x;`Y%Rv{BW1v;$a(p*G(>w+&tI-H>^2tjR5%10BqeMV{bPeiqRy3_5J%ZWKRo6vgR;x1iKWmhXOqh_8DY$&C0?! zc%BNO&(bYVAhyT;g&h#)h=(RkbvU1cBPerC*SfHKQTFkp2EfSbfgke<_*8{^k>f}I z{)-TlcJ={Chpp3ixf}Y~-HVZst6t>b3iAH$rs!|qY)OnTGs5oY+IRSH1fY`t3pcp1 zke)V_f_$_-vaDS`o26;G?HIMb%8W&?)Qm-&RQi<&ljR8H_eNABWV-q%Y*7CW8(vn> z9AdP2#?(J8eUNhUtub@#@ft_FgOb>EF2UzQk;Xfcn zb{(Kdy88P1+6@o~i-gKw0VH7<#KiPr%KI!Rm4(8{bC|~bzbo~9`VQ-$WC74HX>Hkp zh7WbnAVhIQh4t6_+Akq%cL*V^3(?BETfa2kIs)B`Flf(vx581TrH_Xx;9C4oeHh&u zd2W#%3=S6ZN=knGzTo-r`z7oP<+=}S$r}@@Y8bF-OA7@L9j$;Z=xSeT$%;la9 zl^GCahqx>Tv-=qE%`ob(^r&@{UZ-*}#O}qe&U&8!rUZ zUK!`7X(*zn0SZO^iiXm%=+hK&J&0wwmRyIns$eLZoRX5oso* z(bKUK5pd}APj{XTJvNG~Z7n>*qt{kqt_K;aL`l9r+y2An=M1RO$%)5ms>X-Ra=b_z zz_7sp^^G8!z(sNy3>)7_Rl6+YJY_24sD&yIEx?Fu9S|U18B}?qbJ%k5Z=&;mo(|Fw zHU&}MeoPz9h%6Na^20f!J|JQ$2&&ePil*lRj&9cc@}IC* zU_uZDgONSZO1PN=x!*`sz8pU z?ni`{s`d^l3a!~p10oTQ+VEt9-%6TV!h@!UKfPtO=P{oI*wACt#!8>nB$vAMs zGJNCAu+x}0?qs+n`5{>K5CkP!fMLsX9_yZ)?kwEg>XqxG)Gs3byiKuFwqARh9$ zZoO!%w|CoyRzyi!CkD+ZcbY~Guz`&CBX8fn6(ie@z77s89WXlf`!$!gWmdnS{+)2$ zw7Wkf7)>UCm_HnFvUeCrOzJ7_(rBDldul^4uzzD^Iur&qucD6=xG3STUtdVmD=UnF zt!dwOT~mI2VZ=l4%I-U3)h$dK_5(P79}Qr+i}t;7;zI0_wfR$=kVS+|jJ&K+mq8@3 zObGN_8&H)tbk^Tan7|E{X%>MQ?4894lfxeHd%s`U7IR3?qsk#YuWcI3_3rln!B?)Gonv{xzS7qr2}I&!g?K5RV=|cW0E3hZfVW<wmhrDywe3by)br zApONJLwJhY=!pj0Y>E%WyPq{za^mAk-Fs38XBGPOUZr=7g2s$+=FG7xkd>ulDujJ0GxT@(Bzq zDpyfb3M(!aXxF?remgvRqu(P!Qd06jZwEn`GQf^7Lsl*R#TuoWRYy(-{Uhs4bsxX( z?E-0nKdVLr6CAnJ&z&Ql9C^`uhDuBZJ>8yO$5p}p%5@l18Nx$gD&(P5Bn z@%48-vPOYM*CVJaz0)3Bf+Gv&(qC7bI!yZ>)OI?|xMH+$(ovzNbWkg@{5doSJhVy#xuK_e%2f>aq4uDKCn%e-l~pHf1Es2s`#_RT4s)x_$4}aNJC9f-32BM=it#**|8yK zNaG>0sHF75LAwcwNAB&Ky@JkD&#p*I%L5)c0=6w7AVV)_YWikI9V>Z(5bH8LJS_Kn zD2;uCdl!$`9dC?7d4jVxTIX`;IYJW0)P3{)*d{*Wik#wZH z@%=-nQjoA86~vez-eUcPK^mAJxPXeI8Gqk)5R3t1Bn>?N$$&(ba! z1`X5*Rk8Iepk-|V34lMle)-LY=OA7zK?PI7s8_-!u=sL8-DVxeq?2F-TZUW$oWOE? z_wBBz@?oef{d5+~c-E{#L<(leiXP$z&|Q6IRW8~XtIEpq2`nb9o)#wN-p%mrT_6nRp` z{mK1A#?*dNVtDO>%d4FJfiaLWQLigT# zVfNF5S`_!Txc%^xRx^;}HNw!O`W)H)ETUZhl*1@nZeco(G{l4(Y9ye1qEl{uW}SM5 zbcFbodIXD5!9Ial!3pE;wbKNyOFzzmuTQa&v9Wyo1+w)j$8q(PiVc37KFdu5P&+V# zw9F?0oe|%89q$E3?#NTj3;FIB`r(vi9dP4vv^cf(1!vVe9hLYdZ)AJ*bz$0x&Y6-k$CSaUyQhFK0_YKOHr7ykJdw z;r2VC{RbEOkM)D>3kQf0a=WhQ06q%O8O4IRLukH&)Dt}jB}MkYfWJM+ZLgPtiQTVP z^?pD&p!|=78m8eT9lnq7o1SuFMg!W{26{Gw$gB6WUk)M6bPtqV+gTqQc&|k{c(0%LH5cI5IIL>qqx7~LBI4Y z$U8>>S63U2i*(~}uYp1UAnPW&;1)h-|q{T zX@DyH{Ojb!N&B7z<7RFx91eGhCVOGnre#(c9cR#dWB9>+WU;9K_>*p*{`bzTq#LMr zQjC~Km-@*pE=j3__&q^GX+}k%zX??-%OAj3kA0Rx!$3`T&04tR;q-eTbu8u#)wj8R zd*KPgBKPb=k^kQt)(xgdlns}d-){eX3dXzFQE zqUfu#$329JzgLd=_YY5<9%-torv!=|3m38~s(FI#1Be1iTNe-Rtp`&+RJ@^LO();< zTsWI%1ZYnUZPCdFpjL;UVg#?P%=UAEOH43``*x0Vqilsc`PgXdQu$J+Nk&zX`9~tl zU$1czODRxSOuq$-%wcfb3lnu)IrVlO{P+}LQ2C;{B*5sRKRsO8h$~_x>>zvWBFaA4 zH4a;G*!G332}!`yr`$B{j-Hwgx0E~5w!~g;7=Q#g4mUS!^As3vY4rIInw=(_s#uN` z9jo%0)}UGu6}>-MQ_3DdMY$pnr{FBiG9kJmYB1~ev&y4leuG4l%OiUr2$DvEn!OAxmgf*P zat+G2BycnM9w?8107eL!G{PP8BXwUGmM;NONn=nOlF20^4D?_~&~icsGAq9v6GVzz zDxU-d+?6K42C9+mfo6EaW3;bp-sFm5F$=a-6^7(OSFIw!4^WFznKs*uRh9uOR0-g% z^CkiR83EhseeDz{H#I>4!(c<_VLCDXG}kZr8yv?-m`bx&ch zF!DQ;-kn}2#Sdd|r!OoT1Br93eeEB9z2H$N@pC-TP7@pkX%ruvSABGjkhC6>iY78*)a{ou|9d@{jIvd@=0WyIR>9&frZL_R$jZ z6BsU{II{Y7-+MQ=xgBcfY1OYsi4!@;Hs1?IIukj)uQIJ^;T*Fe-S&KGrEy`~dCZll z3vg__)hVVDR;2a;M66waU}AeLMeQEN?B4u#EHpLB?5nxF~} zax{A&Dg`k*926sGp}`SOO3gRC;Xd0}(E`>?ogmjTuWe&@!SdT{axSJnT{~>1D=t&QhUNTL~-ZZXtKc;7uV090p z<>IloUdxczox;0-4dURKkotpwr(YmG!+`=P`Um*5xx;5_>40st&?UmUcNZ7i_W6a2 zZBKtI(ZImqno@7Bu6o`j_|`D{CBr|>9#}Narfit*n&gKq+UcekG9js`WiJRWS`+0% zym5haF)&8ZK?Y?P{>=2}7tt z<6rFnPl3|I1>TEb1L99|1)})~lw%+W6-N+UqzzCpbk329!lEx#&%_F6)$ zdp^);WUE69cYQ>v##jk=a5w?Lf<#@OZX#rWzd?pxkBR$RCUl}8*pZ8ZFR0Xi2jlH1c!>fO_k?ycdogC!sX%5SA(uX^)W zO{8a7&N$4lfmonT8nq-I6^MU~7q=b2q887CJZR(uRY3i{hfbY^RG4F)dv{?T*@qIo zJ2;gNiY!!;;M=gnEP(+PhY=hnI$z3ilyZ_+f+n@zRsPg6* z0`{x^gHt4jPcx%KGAVJc3aHkm&(o*ms%WO-x~cl?!exl~#)sa(r1kCxV5`GS8bS?L ztU`SDI~W5+2hfpdg#shp?roSq(b<1nLm!bq*qjNmU_lxM9?6oZi!NF+1{2wWd?jcN zNuegP;EL2aAh~j$6Mb@pG_M*Tl zJtHkk3-{*?ZB?{Y8QE?Dv+6wop!BgO5`x4-p=W&PBObCks@3w2jqXW0H0kTe;X=1E z4YXV?S|3wOkRL)u|#%qAQ`8DZ3 z<$2N-;7GWaIAoOvT`(w2I?dI>?b>c?M6^xPZf*UZ1<+PD5wM&(Pe(TbOEMCsx9Z_I zGp8pq{o)DdX04R>luM-GT0gjGqgG|QAX^{D&b|9JnTfeUv8DA4z7AEO=*w>f80nva zXsCdh>qLI?}EUlLh+0!Jvd*4;gWi-B4{G zr~dyk%)7e>3HwnN@ysa9Ewva524mWPAyj5xRRE!4g4v!GK&X&8eufa_78f0z6m+3j zfdCr(^{WO5p+!QkLk?ni61X-o=*4jV3sp+U3q{%8=$Hoac+P-=Me zuHG=vhhANEJb?yG7&i|67adjOn^F|!;9}woLlHX45<%bJtgb2T zEnW94qIcwNaU3u-d7=UG`i)l&5L0c!P{;4lXtl%qyKrmOFSr#u3GjVzJ@qKAqpU+d zu6}qWJ>$5(3eA&A78n*i7k>Wb&1J0jPV6}!vU?BWzwL4!XjiP+TQ=M1*y?KLD@L*- z#uOcI((4c**zh;8`Tl#8x8o=UqU}2r>CXd_sTzy^B7=L{DseB zyI08#?ir;VU3;-ZchLV&x`Qn=RO6~CPwc*V@#!`ER`p5fz5O7$C_sZ0FU(PLCJjFxD;GZmE!=3yg$*z?uNTE$9s# z?w@T*@@<@i$gQsrHY4?)KINB#m>=&t^xTZWQMr$karbSFZ)iIwmsc78Qy!8p7Z9fi z#EtA!TCSlk(jb-VlPFA##D~!%O5jBZlCl)INg!$0U)R(#4n^iZU23X>66AvscjqTJ zkcle+?8F#!DO*5*+lFHIfnd2A1Q%n<^Dt56m1b{Z8kAhzwt=mWDxjd*-EPYaNn$1# z05{tps0Fnk5bU@Y9NjD`yxT~H2I+YbfW(0Wk5&zknK!^)@+W#AJX0(MG}>~#Kke6s zM?;im3ot~b>*OF<-yR#X?-VtF@)b0HF^G2XF*iw#Jbbrwg z6SEG7#9z+6H*Yu&J>w$ZgZHig2jqQiZP$NMq=}wN zjhhU&y_a6tZa)D^7Sk)6zzDNf%;Q|R84c;Xq%m9o)@5h^G3z%l_5C_bOsq4l55UmJsVJ zRLEUGp%qphr5STzIiK#=fj-tfKtalVcUqyMk|k!Xj3v+wKJB>zYVg`|lR^0B%mYtW zUWvKP#B=~W^y?*Yw##|6SgBLMAbIvConW$;?2wZzsu$CpUIzRpMHqKjMb9WKEdyf8 zcNH2JHNgaq#a?*;fC|K*<6dat)I+no(2(QSLuobKf{-benLI?PR5`hG`19*2AZ=hk%yhA2W4u?p>Wv`Jn-zYiMZYCG=8+u*7po12%1jy&{n zvhIF_r`5907PncANK^sW8z57%Q*Z>nUyTsfj^R4-?z{-levpANfaEsuV5b93Nyb_{ z^zK5?JO<#`JMJ_`kDjxsSdOT{jD3ot1Mu=f9Q53Jx}}Ll82{p&oe6kEuzP5F#rN(7 z7Ee+yBO%>pRLbN%KHlvAQb;d6*|5%9$|8kGGr=SsK(D8Rfn-{Uo}`41Ev zv(ZY7iK}?%xh)c}lARHv&#C;Gp0rw}(g6`4cG27qc^*;GUKleLiGctDPaajVpdH#F zcr*Ji32l_ubo*E1U(%HUgDM$lo~Xc&QD^`;&G+RX0)%#_6VG7J=LPK*bkbHvLj2sh zw?CG1iuRrYQDIbZ^KKX1$a?6x(1RKu;0#y&nSjOt_L^^FJ)PP0# zM&S%)zcuzy8Z8r~VTH#LtVGSdKQjzrhJqjr0{SE9Qn3_t$dtns$Gb{V_Gee=!UU*Y zw;rmeD!2R0k@({cpBjVBgm2)&fTt-utpT4kA?Lag zTTn$c>nVQYcUn-r)OmVh(s1`e)eKu4a)%SGyl{%t6b)~eF2V7dw5(lUT)5|z&hp-! zM2fj)Kf+b96jHNF>o#=}w+N#yXz$$k?E_-IRwWpbM18uVL@>EUzPq(8RyE1UxDM6t z%8>URzE`USl$VgRCF!@RK_R5dF;)E|TG=ATA_QmI*c5ln-7#0LJVLWDZq&-gEW2cs zDoWg|vS!`DeO}t^+v|iL&yDH77Cpnvf7Q&L?cYo!pNAedHMQLeY*n z6zPiapH^ZoF5Y3Y(k+`cLO*DFhG+s45pbIfF{8T&9zI`Bu79YcKZ8nDW-Nl zAXYIPI`p`*=txCvmE`V4Z({GdAIY)kxKon6()FD~c8vlKxJ~dfQ_s7rJcR%8!!)`H zMuvvrkbg~t2Vrkk*an44DfN3UnS2g|_7DzgXs9mF^>KpB)vuSn(|~1ec}TIGSWl_? z=aIVrKo|i4(Ow>nR$ri6vAlBzc`djSV)xF>77qouLmvRD$_BLCHTIln4q*GS=xmTO zZz;CBgZ90P{!m-69a9QGA7^9_LRq;m77AI|IUdOwYfPocaX2``a9)jPz@7~594NL1 zgIel{n}Hi%6}$|{H);7SpFyErGswMkk`OE2>>&i|mH#E}7%%WYsYL5{YH;-)S#EV6 zS*~*7bqkh<5NSyiUWJ1%Ui2sQcw%R_ph87Qy6_xVOjDJ>PJM6_e32r-hff}6YarNF z-{Dzn5<~feSf=Jm1nGZC1FZe~FTdGN1fk${Wq;mPrSG1yMd?RJMt}xr?}9r>&nOb& zkHp_ZQ}c%%$B=wYJSrVg)ax8gL_$J6YXx{pZ6K}9_=&6+V5RVBa8oF@rF-{5AZlG( ze^%Lqlgw28? zKxpLvsM`A)t3<@SI9N3_qf06kLF>y%@*BTJ_Pj`81LrkcaG zVgqKZA?4dCcUPW&)Iez)^>cs*unRmw1uhkA2(eUP<9G2uHGoUZ0L+j>bU=Y%ur-$h z5|L#xiUlw*G09T#S(=LM-X~@&;La{ARw~O3cs17l&r)|Cuk!6 zo^6Uz#U0-VdY$M<+=8}j$-e#P94K~KKRLfxKkQ?V{zqN=R4~U}6g^ONwR6IWNR9$^ zmV2lC^6`VikN3G|B9r>$|J1*{|C`7Ilup&clo;lg1LM~d-}nItzqj3*^YZ1(nHmVg zQ#4~(qbC@EB+q`Jn!pT9IZ$2dg>6`$et768O!@vfKyT5}0YkTUJ`kO*no<_ldsX0Y zdwLXVLa0MC;m{tq&cIiw2A!mG0CrkRoh;I>CO&qY_$m)7w3f-8fCr)R-F0P(1t1;P zw3oXl^Q|FJLqn&$sJ!qGdMXU>20y?{O?<3RSDXzM=PHdI5C#pu`&>2Pq<7;_%=E7Xkg)GIy>ez*m*uy?Zwd7A6tkqEZLOtn}c8 z_wO1DyUKLw^Ul1C4vb1fWgdcqfB%7qU}Yf`can#Z29!yHATO8@w0XNTfQkX)9ZF_m z1;gEbFTBV7MSoyY=wn7s03|OBtOI2$f%5ckX029xT3$66WjMR(+(5@B;4{b$2ctmS z$F8Md5GpK3KLe`{xjLj%)OfIh&;^+ub9#bTNlEpnK+KPFP!W#=0}x8wGYDK?a}4^s z|1ea-2sT<&a47rI3(5|dekvN{q9A8<$^XF!c~!8`2pcp-;X5R8LZ^BSy39LK1-Rbx zaqIYZH_0}{0ctE6PmV6Xq4IlwNDOob+e;?Xe~N5!$qt4^STq)R*f+>c+bNPndv%uv z1eYAWL)ig?#thtLuZ>}XJTOYZ6dV118*HxO{}191T$va_JaqI?&6J|CZmGy=akmqCODMVNXtJOOTL z?N4P3>>mYZ=fZjIPz22*KB(`;M#B34ENxfu-^$2-FsD}r4pmvYl2y3uV>q{TSp9^w z##F_^z6g?oe8;B$8`mNo-mjW+iohpxi~)$J_j-D^as#Xwisg9Sgi+0Rr@|~ zT;o{p*wSfyjBoRCn{G(FW-FyU&8MzeFwRNl>y`u)2kF!u`B)2EHtuY{)1U770@9-o zt^K6V*Zp9f1GBTtV79%9hqxIY5ph?q$Y=-;)LLE`ww0x5HiiJ99r*AZj2Qzs)%Hhl zn>p~^OG%F+ewo(9^9A#31ZB?JtEI|t5sKPQ;UD~iMWDiQ$OxJD_1n7zTn1g>Xnwg9 zD0-J~4b_lx?^^b+^}5rAMfVbHfb6H|cNap;70=uEE4z**Q8ov6g}A@Z7Wv6*nkHN3 z-|}oUR?+_k43-XjN;d*({(%DrmVj(&I^L4>^yyPHCct#d@l&T#p~TIw>8~sWqmr)> zKhxPGaE1{7e!V`QFFI@RVp=;Vs@XT6ZTguW2AJ0>*Wmr1FixATLI1HpCEO2D3n?or z>#VjB!FDw~$+-AHPr-A>=lsry-{e8WR=)?8c23pTTF3mf-Jo?9yX&)JOy3>;5z{CX zCACFn{x!J;M}mCDqK{hxgJ84#1jkW>t_nan+Vqac_xRP~hljnauK>Ahc<|~_l{J!1 z@N00~_ONad`sOs_7(rK)N%O|1dg*jpX3isab~xL-k6t-H8`&z{kpZRRWMW+@f16<4 zyu1GT9?REvZy{1GO+RKdzFd65)fi`grl~$(u;N3bGOUPfZ#o&~a+PD=XlE^+G5rkz z0waufXYr^TQ>pLn8>ABYbdI|+IG6N2(MkJSZW-O`R;=YHwbA#y#i>|`h~HKY3RuKE zm-Qg>Ow_GVY<>7x-c|hC{LtM!UYIeu+cS>8Z+2?M;k(JEH$JIXlvx#UFD_cD%udfF z;2nQtkdz+2mI2O?Fd%|pFWCaQK}O~=XYl|bhhAyEMxrO?%iJzeqd?QW%TAOw??HpRU?=3%Z+0=Ml)alr5Cx3PC zzDCY_Nl5+0Y*+l|OUdX~63kCi^|8yPOjGP_tMe|u-Xj>^0~l%++spIw&EQa<>C0Iv zhvIQn_$q_2>Hezz)1si7(bfE6cRdMw8y)iIp%YSf7)sPIl_h!Y+UHryKgHhLX%-|| z?r~-(SecB@Az0uS=B?r($~uDEeRo(Pq}D$n!^;4He}PPB7yWvD)Q)@>Ki(Yt^W3@i z>?MDXJx*S0YRXdr5(II5;b9a&3d>LMja8^QFwT zrUZI|%ddCXt_Gc23GA)>xq-vVf%>UQ-Y(tYvK!SuT?GsuWm|My)1Ntvrv#WRzWuep zPg2N&bHF_P2e{JavTcD&g;Tqh$#vI~-R~COJ%_&|nfVjH^!&5dT%4Xi{`NWlrI9Zd zCtT|HlrrRV`9s->@Y&z{@hUPh5}rc_I0>6VV~z@p>6OQazJJdLzQ1?)?!vfL_28YP zM;^cSrI!>e6Kj{t7JgjMDM@ywPV?1%T>pJ^&eH7fc~Bou_JHY0fLO>#ZAEUv`a64N7&d`arM$m zx&jV{=4`V1>c?5jrbSn&dNUJydV1F439x(Yln)l~^(H~?-MjZpO8mqm=!+?j`{(Pf zz3n{pLTf0G?DxTq?Yh5#Drw^DSn4|P)0@|tWVl-5mWG@?1b}53Z`LbgZaU<=xqT41 z{%9vyl>cK74IOPT;wms?q5L!LYGC( zvlG;$2@ncB$6(0OQ!}zl6##e4L3wvJHa1%95!~L+3W|!YaLzh`T!tx_RRsL$jK)72 z5{z=S!r5088vmm*e7CdQrl*&^G}($w^z=o^SShMEHt5+Qg9*%I`EbVZw zn<>@R60xvkTnFxxID*-?K)#=ZF&jI(1|$qlogY7bJmgA%w(xbKkK^!Nj10@xMj@Nm zvZOx6T|rr-FupFxm$I5)Jej;wv8>iHXilNge-4@i#iN1;BuMum7htX9ufpUxicmUe z)9t&pFkF9_g<_yC)b#G%WT#FX&KE>Me}BNSA@l>jQfyobS$;Rl;OezO+rV0vCs+|! z=50+nokQ}@J55^Ri4^qxY#SYpV;3COiiDuqHCw(IfacHFN`Z2!2lOljRn?AzT;&6= z0dCxRVcEK4ot2JhdS(?GG<~(OW3#m0B%7OSe>hlk+-qH5Me>4xZ>wT zMk(lqSUx9=cbaT>1^OJX?SO;_FzQNwy!1V-y_+K1GT7D2@?3HauI#C}N-X^kUkcg$ zLYto6-pTCG53e_U+_@Qu1ZfOmd}?Z4wqrs<0{EI?Q6zF2q<)rxtGAT<=Nf&4Jk?qg zPfkp(HL`|GSM|29u5@?|N;%1<+&{3eVE1d0Jlum!g+G6u4lRnLSFe5mb*sgnzwc)6 zpK-;MCE4%#%7IV625uC73$IKw2%_0Xo2(t;JLYW&2?jJY%JQ# z9Bm8-07K^qDfajpsrSBiv>5?sYG2J8Ny+r zY3F6`RfC7q(#+NI)s54gcQk3vJf!|Hm`&%XTo$*GU##hhhCH=(_V6ezaA{kH7O*+w zgr(a*Cl&qo-V!Mq8=RcX04FvbG@beM@^>he$hvk@BXxGQHnk}*XQI4c{B_2Bl|?M$ zPxEh1PtUAkt*5?t*LuMPFntf2e-638k{5o}3wB!2O!=b9iIXQMz;yBkG}0Tw`%Vc~ zfBYy5WJ9$VVjjnj92t0G_~%p9)NZDh%wK(e^hDaT;Tq?W<`mC^%9DHwcXS*hK4htH z1i;0|KNWC%mkiFx%PhGK|js445S-ZvErSsEr%DHTTc~k|w z)Z4PW(`x40nsAu{wBCD>Q;aGxsl*3Q3jDPCb?=aBW23RL@uAqsrJm6bUD2NJvPC|d zQKhH#%1mU|PP0O`f*`3$1ju9j)~r|V%~RY5vTL4%IlH)U6&(7L&wlWr!9G_vFd82D z(Q9SF!*)0AuyImX_(y4u8XjK5S3h}7wbG3DAYUzYTM4rhfO%tSwy7#7_XRj5U!g4K zHT_bZFhY3xLvek^wUoocr+#PW@eu=JWD-st6f7lLu9vLC^#k;hU&S1XZY`@<*32JI zmOU@S{L;D=jf(N3L(fmzl3piA)M&sdss%<*vVe4OGciaekR}UsPg@V|{r!y*&Csye z1kED$j!8SVu#jjg2UiziNW5Ys=hMJIE?z2$9Q}%XXuC{W zfFE`12~FLE@^X_ZovON^9PtoJ$l$AdE&#jBv-ddbK~K$ zu9!aD_iP=Zr8X#I(v)wB?&NX^=P6_kARe=U>ieUw&!f$$8EDd8J*A}FlGp6Jv!AgqD`p1CpkcO&frg4v>)*9p3P25?%d)qJ0G_w{IuF*_8r(&F6Ic;_abedHyl} z*~JOTn>SCDl{|nlA@e@R(UC zC8}nd8ts{{ug6j)m=o=TVGdOS0)li7zaQzSTQX{ZyRf+E)M*@P2C$`)-S0*F)jlKO zy@l(NBv>kB=#58j5Kq3%u33KZ#dBeKE^`TuP}L=ZBH$3Tv7!yMo9yE6Pt}h~qr_qA zRd}wz;_#;x-_9TRPut_S*Wu11Y}YNoEuUYw{ebcm^!E<+LXUl6&UWqg?+;W{_Uo6T zpzTUCd3n-&eozk%Ccf8kGy$g)sJoU!B897%;871Oo%T$+2cu8p;CIz5I2Ln9491L{ zfPiBLHu3F!_?+^Er-EPc^}u?~DVr(9xmy$M7ub-lyx4I6V&i3Ki01=`?jLV&O_yEN zS746+a$*WT@1k#kkX?=*#__Oqs%)wM*XvfLxSDq&jp%VSieT$QnI>V(r~^nGDS+ZU zCqO=k;{6ZaN+j6?W|*q{D){?O1PIT38LaVcbOKXTCKEo8!V(2*TywCf0`5uU+27w3 zDNwzAHP}-uUzuz6W>#aGER_Fly~)qpt-?61PjLEnr^TkguVT%CPY1OH^u0bwU{Q7| zF){JzzXf9CWi_pm^r#l`d%Uu>ZQU3UunVFx^HZ$G)UDKE+ zY`!D(C~-@?dqvt@$2CL`vt;u{p%uO*6v|n2spy6qxy}$OO-w}e16*cXA<@c&!6uBG zTLwJeh2>>Y<6e}8=>GL2N(w^ld>2cO&Rw`wjh*299Tu?)VLCxMsxDa_pMFSp#X5h= z7J1BXZL+dP+xJS86iwGtI|&d5w|R(#WuKg&wWpzT>e^|g=REZ>3)SD8?*OL%>s26i z!!u5-XWKh2Q0t`!Wj&O$`gtnPyR2UA^ojUYVV75ooCHVK-X{c%I7xs0QYID_cwgzYNlr0p$y1tXcR!s1Z-Gkq%<|Ra+ zQxoSp!Qz3+E6V_zt8{rC&WubT>zl*6?JBlQBtLfS#Oc$S(DE5i@TTETxssZiC>!{E zy~_}tJdIdfTx|SUb#au|!No;-!F>6y>iyjJ}>suhYEHy-o2*rhJ&CCWbZw!wn zXLLDs)EuQduAkzwPUZC8AaykrCn#?Bnd6|-m65MUO-hqTTmzeEI^67^ZZM^gA{WGN zfj%vSpblunp}liKE4%j z{FaCxUh_S)$C8*{xaUe@paaSxC|IIPTPox<5wqY5n1M3wEpc)2#j7E~l_JK>EFi{4 zLz5)_xwEsg;S)RF|E95ZXk?^jT8G~fD2CXNTBZAc^s)Lb&=%GdXE*bI82bvSD)zPS zL63z37O3P=js*fLjnokZ1?f`Spu43pQ7M&@5Rh(=Zcr4IZlqC~O?U6_`R@(hbKduU z-}>fVcdh$gH#4(m&-~|ke)TZl5eu0-)&IaTEMb#Jc50?Qak8#wH05k9Qr;`{ny2ro zc-j9&w=S<(dNA4Ty?$_|JN)fTyN}d>F0OxP1A3*)&82HIQ^4z2k3PjMpIrG`BvtgL z7~RaVv7#mS4S+aY$O4O|CorFo2zgnE2U(8-W5AEqqHs^)3}briK){++u z`|2qeKBtz=9YG;Hq1#N+)6)a3BnFxOv^X=Hpq{QQrLf8FsOrZhlbuFCjm_J#SWeCN zE^f4ZW?S}UctV23iLYm4&_jA-T2-(2#wNb-nD2F*^MpZn%`n)NcYkEzSI~FLB+Qh& zyEG1#tkR98-c)E2Btc{Xq(aiKw-HQG713=nxEi(`D=sds{Qfo#KW@gdFYvql{b7dHdzzUR`;BQ*TKa}3L(^v6Fy42qOQg@8No|WRAsNL zj8t9PA*`hpo^`V1RH*Z+z9dk#wZ9;NuL+G8rTf%(OI!Np%}Y;q0;pXEFk%b9iUmDL zTnn{(=J^^B(W9~4xLH6fsvGthl<^<0R{jh(eZ|sCz6hh{K@{(60l(4Av6Q=ctiu{l z*3PDQR+blCDG|$UG`-L%e`GASX5A}GYQ8?^hk`-s2?q>x0_;X;5bqeqG|bad zUe^nR*T-zp!%1E`b80V z;}F^J`wP-z{)T7+B{01w%kKtjT{|77^~d25lPD6^@UM0kvqKMPG3@LZ(Ja87ZdFUhv{~+7IcW-V+B~%a?e9M{z1_yWZasZQ4;mWh8!xg9 zZ8;Xnns@#zP-V5V%~*IsZ~pc}Gq=!} zDskFkHd~NYROepx&g|I2E@`|fis%fF`9-@c_c?X7j4*N(XGG#Se!Be^98jmBqh7&=>)dlNM*ow$FUx$a#ct(-IQ zcfSyP(4df-dxtQvgf^31@i1ck1~d`OEs;I#Pu<)=GI}QnSSk(s(bJ$$kb-Q5^I5f! zyczE6do*LF#09_EqNGGP9v05^4SK-wZW>O&G&|)EPgLHQDQiV#j3_rwPpT6MGYc4Ql@HZG$K|f>|_jAn~oB)>!~8 zowBVg@#|r+e)3RW*d#S&1Rpyuxbalo&cmSNlhLwJqO2O8xU@|5YC2-}wn z4Ueji7R(;+jL3))^^n6qPbT)I!pXrq!gJ5M*yR=!F!S)-hP&-D&9}3XK~YFr(GClL z<2E`Qm)qQBRFq}yrvZ@smvLeb8IV1h06k=aSdk7bs+yWfp?|E8mp==+-@4GO0FQ?{R{W3Mq<8ak+YKZTMhxnI`ed~ZR2NI^Qk|dB392;rRD<(23Pr&ySwGU_Jq>7 zXEw2PO_mMr3p2IWmCP5?c%59`awbPOgni1~65g1Gqjv?|-UCOm)6jX1!5g6oN@=U< zzQ3UxWP(}%tSkz!rds#d4POE}o3Qmo{CexH2Pgwt(@9^E8cQ+!8cZxGIMAxD{yZ)v zCycn)W(AY$uqn=nf{q}F$2@}S!tLo)9624I0vjaEf{js;^17wIsIch6hq>0g+ZdI_ro>iqVkbYc~qR^zTZ*TlBqEulMH(=%-YX&>?< zn}m1St;FS{++G8wT3J`axVML}&Gk8HL@C3HqceDqx#3#9HU@@& zMvsZY1fsZj9x;-XPd{0F!K6gZuHyk{*WC#c<8?ZaQlOq4EsCJ7hxiXZi098XYQN`o zI1T1JcLTbs!)DOWM!c@B9iK*MGsnM~z*6+{6CJc$eBoX(R&*bN`R3o`T%o&!KK7(6 z)dK|H@jyrzrXt!8hu@sSLy0lW9)rxPo6G$al3+-HwoeY!C@smV++JIWynSChdiQ|E zvBqdw(~hGjg`aOXcRk}FMr-R%`rjZKraPph(0Y5_FC2=@SKfHsy&k=FO$ZzT(zMc# z3nb=|qMmZX!miF>0TQEXLKGttP>}hwM~+0MrKQ2IMf^4f0l+cVNc_0RwVo=J-J8OH zYK)_y*@V$rKXGrK>IFZ@Ujv|k%WQcs=0?H5#pnQIHhYkdm3xzBePd$*(jDRfd=bBK zLk3LMbSg^W=|`!kn66yu%pdRXXWY*ngVdmAaQ&s+*Z4~XcRhO+M8gn0D!ZRDm_R%> zn|?sV;}0tJBzce42SB{K2EDR8+DA=GkJUqfg>ifOby!f9pn|9w?aVPX6@;lX3Ba%R z0Psv}ez*n%#Hc8B2Ug1Zkf3SWRxF!E8p9bUK9Q@sg#&^CDmmOBL3MF!cJy?Tzn z_pc)%(tHe>6FGQQblV-5CjnnxoLtqGDkt1R8=$vHI>QBk-o(G%$Y%C#sNpj(_;Q0x zzsn!^ELf2;F;`G+jsh>!2$+2hc-94s#2Jnqi-onaox_@VwK3NOpEm)KwmHcSN$;Zh zeEkfR>5pX{V5iw!)A{1JZ`a6e%5cCAp;N&kT0F01RoCiZ{bdwOb&OM3~OTgwS7&8R)yB6Y3H zvThWt%-pem}60*2@-Sl901`t^sbd)QV&oag*$t(Qe=u ziq)khUmpM9^@)u4BJ5UJaxZoqo}={ibR+skJhZ+SmR3cl@Nfw_4s4wvCgb$~1+AHB zQ<7pAsfUE1HRT1Na{%u)Ip#X{=D(n2)~c~2`D{jUK`VQ%L?N9A#&jIJ2)0f`Oc>

%1Wpq3qs6px!O@xpy5`^P(Ak`m`Xw=9Vq={}!4u-W^=E~b+$cER z)+pxZ;|cf>W(Lq7>J`aXwrIFg-ekVRsQ~bdwe3`ezv*JTjowEy{}5PUh)xO(4b2DD z=Y)9h`}fHZ#Q}gMD49fcG=z<{9AhMQmmjYpg#B*O>mRt7n3x)WiCh6q!B;oic0aMs zQhW50G|_95(3qs$jQ^+vVBS_aDKn@%2=55{xf<>T81ci*%38ac-*B6niYf;FUrjW! z3IuJ+E!TH4dCu6S06y$+^q=@}R`O4Lz=nAcJ!L#mMPX?sBG70Ym15AOPERGehXuVo zWaQ1gc0fz?#{8@1F4Fz`7XYpq%Nq`r*U$*Hi!>L%yE%E<_~UJ>iLACUfEg5^UrdO2 z$jS`EJs>5sb@1K0HkB5*+J;E6-^e*3gb%+GrFCp(bz0aiB@WVD~^f($?vi4pGz`^AKleKc=}^Ez

t=vCT0klO~6YSe+O`KePj977Q4k2NL|rsE2inmec#^i^3Z3I zV;y}0tk1pAn?bf$zM>30|;{~_G z(i_Js``;?5FWmTYVXmf?Jh5+{$E&c-1oPZ>GSqoq($l4d0xX00L1_GFCa&xazy-J#{nFFCqkRY z4c6I|a!k}gL7Dh_&F9JZGHz8wTX4(tOR|EBhF8s~K-Y!*k4~m&30&MR1hK3OSZO`# z#ha+h;=mK9YGPvIx2@nQJwaD}3$66_ww(srL&NPMJ)PYm_|bZ03M;x8B@%05HHl?$ zV1IIOC|tU9DIcUO6XHU~9UOy0LkV`4Lv+M@*%kLn`bx?$7h9{?l~$&l>9F`cRaL!Z z5;?Zi;ot9yw0vRvhIm-$Hmi1GkfKIl$&Q?rQh=hp>OCtcK>;RjKpuPw3JRzet3M2G zsVfX3CeFVMIzbm-XKO4&pbDPwJ6S0Qd;*OA;BVCi27^LN-`n=YLr~Rsqf&0_IY{PF z99o4r4XbcJ%vUm}o&)7R?WzmdH^I;TCURTJ+C!9K;XeS?;l-U>3o@ph#(D|b8c5e8syY^gmoG|={LtK1bpTh1mSa$Of(9!FFzH^S z5pQpAq5gc)vXTZ|Z+*n|{TuM!)j7D;ILQRF2o-k19w$n<3L10fDH=pT)#gv`G}ynT zYQtpa6mMd#1aQxlWTzF_7{IZq*xKe!vqtu{Xp^i>)1^qKCP>?h_}IZxDq22ucR$5s zJ=CgMP(&8Y>M`8b^tPh;VA#3n8{pu#(S^QG`0j+pE000FsRv$JL=y~E2S$SL$FRbS zuClrsWpP{;5kc*kWW0ITR*B2Hxi((Ocp>Ty$TgywRcw6< zzRHaf?vg&ogwucJf_`ggYpFCl4TagNj%j9$*mb7WFDJKT9@aF``&Dh;eD!>e_)|>r zIF@GE#)=A8FqpRbo|oq^|8-5QaWW$TNz@yOb#r6hQFu@PWFnmJfFUl+>m`_aV_i&1h3&W9Iz|a_AYGP9j+&+L$B^ zOtO1A^2OP{vMS!Qc@D+WKund2Z3`6bEkzpaRFMF-Y^}mpy+f#;p zCk9ncGN|E4;NfdVkUMf|_4bx@je0ih#B}8q6cm{BzIJ!N=Ux5Hi5*|E|@TZr)vn~_GuQg*7W4e*{;IN|dD+n7hfH^ae)!`;$F0&ZE**1)Mga#ue;X*KqCssI2Mh^U&GpN$`hkw> z#|Ofr3VRh-v9wB!5o2qXUi&rPaY=IH@x0cuS$D_Buyy{z9|Y-_ff%epz-iS3EvzTs zU-AiLRtQ9=Vo@1L+lDH?HYu@QNmoUhwa-tkjCY*>m!3oUyR<3#3R;S%ZBIfJKs_YD zDcrt&&i(5>`3@)|M*#`9z1>dymVF%_&+)zFM$ow^(|hJHdqYz&5ASpJ?rb-HmPumU zh@JCh+AjhrkWl8f!8;Oo&EiIQKM}SzB@k`c?NhCOj$f!)1pM&Y`!k5CVl}2bX-jhj zs!r%IwC~eiCVo8E7j$Sfp5Be~anD{!3f@zucN)v7kL(uJH*#kMUqa$MVze<=f#1$B z2?VnJPoYIirb#46g8m)??u9k;Y`p}raHXo+x+ZwHj^)+3`@Mc+1S?6fb#wEJH`I-} z#?i>Y3J?1Jtrr6p=XZJ5Vtz84lb{RJjcz9>DB!CIlBp=!ub>Zym+!)k9z8lz!`<5q z{XeJO>;rgKlPtt{^IWpEwSD&VsjmuQh7FQh&HWOr%ZnP&8THKdv*)!5^cqCv3uwmD zCApDiOdZDsav)g;Ks>e~6KD~Tx&bV0J&whIQh+2g>T=~S9!_pkhdP#*yNH194XLwOG>f}Z zFjojF0!x629EgOjzamvQ-(PUoanXHcdHLJDk7h@IucA=Wa*B+jK zt&290x&&T-)G^eg@#?K(-pU_idaFfu9+!b5Qw8NJ?^Xd; zJqvu?HDN-ULL2iw;G~O&sFZW(&)&}9 zmel5C<=|(2{P=O^y|1KD*raulAv)!ihetD%xHbHQGsXJWE%whYjHWCEohZ6d(BuUx z$#nTdH9fv8I5YMq1NsmxCG50pnMNOaqDYj|3);9Q(D@I&-dfnh4IT^*k3N}j8Cj`u z&k|z)3d%6U)|;TWZw?nVCL-{d3=MQv;St~}f47B#l-`UU)W*F}PF%P*ewpsOcm}gK z7z+qhOQQ4KRu9;G2Ht?v3kxm3zPh(ScVE!xP0c9|vrOaWEr=JT)Ld%d!59+4TUU|( zQwVRuweHrUKTfRh30i2({c8lgDwWQ;BG~eGo5Uk}&n6#9#<+heF|n{!49(M}rlxr-;!)#2D| zi!bx0+{Q4ob}iZIQia*5)fsCfd4qb(z7^z3a-{uptbzgd6+@Xg7YY^!zuLA<9?`_;0^z8va>BEfS)DHR+A(}1J=2aJzEGyAE?)mlpz!1 z+K^v$Xda>$(=?n{wAnYlyM);+qhYHzbuiX1u)#ny5DJbChOpppwf=D!=5-YlbbxsM+|ZWlE?!hhfW zP~1+FJSzLU>-?aUJSGezL0#|DHGe(&!4Y?149$EEPg6m zP6y=xK+@T4jq9sx+%u6fpMN)}kulf0w@Irk0y_u{0cf59zqOxY;D1;@Igetod3BWl zdtlB|Bi;xpOtD}I8owQ@%grX|xVjHW!j{Y@0JbSbW)q<-*Tg?Cm_ew<;5hw=m(118 z&DeAaz_Sq$&Rh;#8f~ih`!w;3nUNt;zG)|Q(ZP&sd)VcHJ~eW$4WhmzzCn|nuys=z zJS5^35Y?oKj5nxtfmGQk%_dMC=tDQX+Kb=RWnCI;kpRE0xz!d~Y7qV3*vBj)`{+=l zv$YhJ?%RR9bq2dhAoq|4E$IlZ^&-`}MJ7W8D*{EpRAVhHRSO;dl;X8D>BO=ubK(P7h~g}7Fm659eb+$M3&6`wh;eF$9V8-m zv17=~h7rwby>&Dtp$6Cw!X_gK@Y=i_6lYo1S&eg+Mm@%%P@HuYd(*!6~uL*-;0A7s+K!{l%D+G zP)1AUTA^eGIg)u4zwAvqwZWMm!2(?!t{CXdA_ASs6XoHWn4k~jC-xQbj2~2>&r#-m z`Fj<~Yw=wx<`7&puaxb4~mK1+w_S+T13Pt7J)*Ex@*^& zX6wlTE~1kWlh606m!w<$*7-i{3A%pa*}uGqOhNc_u64YPRxpxf_=)v{Rq?*rPSinw zh~cC((W(f4Ad^7^P~g8kIZ8!2=Jxb9auBTEUz@08jF38l%#8uruU=o7^G5BOYt^6N z;FMwWgas0p4yHZuNFip;plJ4i^78D0N0~6QE3(!d>_g)2^B;xgQDRw37 zD}mmLNcN&#Vf|ts1_mDBEP$)w=U>hTD;`lY%q-S3JV-Twn^UJjm`(>Ucgpm-oCb!l z-WYT|dhQY)G*fVJOINa6!unXLAIjt%l3W2*jY2n2AZSy(a79Sk^lP$$n`vmvvY+LI z!8!xDi2UE%y7_zMiRg0gkDeivR%*2BZxg>G2?|JhlkEIws7?V>R*`+J~XBhsXs5S5s2XB4b{(OB?~m6&9C`rKZks ztGCAl)4x0(pJE*hcM0Kh!!S;Ok{P%cWX=JK{rsg%9}{}Fj{Y%f{6NuR0G^H1<9)lp z7>LvhP6eSshc8_P!^9Rh1vIprpD7h3Wi-kef^tgnJ)d`M+G2JQ&($0S08f&wfV z3dZxOCVmr2P$Cg5{cS_y`nAYUMe`JED2K+~bSn|mFM zck*DDPblCa9+2d4qco~e7e0AqRh~xziS3@xjY3hrM9Yh?h=uFy!q7|$q(iC7G65QU z9J9~mfdTGR>kHzxDaChD)Tb~w)HKgd0# zHeY&5Cc85X^CD1-;X}JHpvu^Eb~>ATyJ%`^!puw@yqmRSf`Ok$a4TEMx*YNVX6iQ$ zRC~lzK-v>*WFd>KBhx~@(dzToAyV8}^@)bYV2j|K{qMa~}nsk5^n-yJ=C zfIECzjoQfnYxc-97aMq&0U+-OFIDKN&)6wXz@8Ce4A~ez~=Ta-XVth`FR%K`! z&T_Ss_kRIxtwE!1YVL)>Waec-#X9qdp8gEw_hAB&&gWK6Tu3Yq_lhg-f3V-{NX5&# zP)nyU@zye&6m30?i*sTxxp9C%5QyeCVQGzaVpy?%7@DmS>yTN|0T3<=hAGw$z?arD5d!F-yWFYJZU) zzH^oQ^S4AF@oK-(A6^f1bo{S)S9Zb9zyJl8q=7BOF4&;o{9Z7R6z5=uuG&-IBt|LQ zH|Cd%r4?O>y5uF@DXX?4y8la+e73)$p?Ar&Np&AC>&#&IctZ!vu(oH?7)mv-1YS;?O+qSEBXl@fa&R5Qv<=KoiqZhs}a<1Eg+P|P+h!ne)vuTFnNx0-ytfhGrBN$C>}+a0$9P)lD4x=>%e!} z%S;umKk+2F8cwpI8hA{tGw}JzofwZOVsE<(?noQo4zD@?PqDvV>&^J{@u>3TJkn;o zEg0!~Mcc)>g2Yiz_Z@+lfN1FEf>N~~JUDmpq71WAQa#Xp(l^1@h7m_ysJY)3((MVN@vacnU8=|4{jXq%9DrH zG=tR?LPyj-QPp0g5H$|zd_vGn_pe=}ljRz-00FdRo>#cn^8U6owE)Z1yZ8-xaNQ4$ zw@00Ta$JI!$>E<~f)Zmh>2FS~ZD}E42i8rjbk?bq7FR|es7=i{G41a?TMTK==>HP@ zj14%?V5JxDec3w!(@lVI3au4f8)x5*t%I;*h;|XoO*5qO~quMe+My>^THu zZY-7HTRb4(Iv=at`L zIs~YRnaMiqA6w7Vhsqt;s}p41DB{bS7c%TmoH(K5?b1)IcVyZRXEZHXfJgUGj-0+7 z^5~jnj8szn%un3FvKCS!A#@lGK!?j)9zTn@`kW~0eV-L8g#H&Wmykb*g8YEPQ1N)C zlg<+J1HeBz+EXiX^nrr~;ZthNSa=k;^yA?_v-9&wAbBTjXa6S}GP)s9Xyp(~5=6_T zAi4gy`uqVOA0Ik7qV$!?{kq}<$T>!4Tcn3bRxeo0#9h!VR)gG+{KXE#ENIOWn23g! zxYH0x+ag&t9Jj$;H}>8*(HKm;i?a=Y;}J}}^}{q=;L2URe0zFpABJww*G@lym@|5hCAm5Iaw z7N#kVszisb zGX2k!o(nf}-sDGEbLXkj_An5Z4A+hm5Q{jhoI;O}1^9*@H@CdJEPjC)-x)UwO~-<@ zVcmf4O06?q{6%SYJ74u(Di`OQv;4HMXv`z;cFrx*&1ax8nzsVTkl>ths zkt%f>4o3So*|BFjvO0eQ9v|f1Ni3ms5&+7$h--X;UUlbnKmFqfr2WI+)5Qpxc&d; z!ndCv-Cxt-V}UHucqq{z#`-3#^LZ;gK5sqo&d$!OK#I1&@13v}g%!7>!t(M0L>NQn z0qN;s9njHq*F=C|qTct#p)uOVE$s_n!$^&B_t%$~=w0rDU(qH=loS^lF9#2(OnOQ&xaOZzvB@0`a1{QM1Zx?nPY(e2*R4|0~e-&U8 zz=w|EjD`@E2O^4nIQqD4ero3f2(ffJAtu?I=6rIani!Tb8og z?4g<$Ud;go+zItwt|JH8G`qXC(Pv7QZrys5DASyUY(MTj*j&)0)^N2 zr^}g%rnZ>|$WT#ImG|oKwbAR(R-2hoQMd_D}$7f4n{a?E~( z?~ndew?D%q_r}fUho2bEl@HQ@o|=r+R~)cdzPI#Tob4<+FDxbMDhY;0+Wq>q`D(H^ zQ{M*#K2`D|YY439%=1qk1Cn?B72(%5I~7W`WRa|Eo}s4{_A%e!`*6ZO{tHxY(M~S) z+CjNvG=oP643)z6CxA`SJWLQ@ssvb2ro$i4fzz*+rAsmCEVD0UPRA3DEDRv*UuVv! zm~Q&KeJfQ^ivSF0rdo`?q@FK2O87{v;outq9n%%_^uW(G$r_7DW8(gn4Eqndy= z+7pkk4xqpw%8Lz%boTB*m_-%HNP5)!BH{3^cUu}BTJJT`>ORpgvKHu4^3uCCinA9d z(<;=ohregTNTA_g+Dztq;}c!qw+aemy*zTSWc}f(-h?@4e0Z~A@{X}{ zo3rt;Nd;^I?bT}#8fMMUEuXxO)4Nwe64b2ynpfp>$sOm1IVKu2em2o`8O1jD*Lqg9 z>Iad@lD?Zq6Y95gvEEe{kdvPCG;X*K5}=^`pUN9-f+}tqaxHw}DJD1`S1BSi(ERi~ z<{@?mX~Cc)Kr9DZjccavdi*(!Gc|XBk$R!C0PgmL+D~Lzc3nt9QQYhP*B!!`IF8OCy1o*lvE`K!eGh_g%Q>_tj@h(96A!^`2bqcc|&f z_@1&vxD7CT+}nMNCIu5*2nI>Xgib=Ri106^Uk8s->GglyNnJ3_IKkQ=vhhH4YWXdk zk@dn`wq~xhlm$+1pcsGY4B5>@NLQDS*gRU8hwy}f!AbBcgc+yk77E%;6v~P8X_gpe zx4W9=3OT;xz3O_2>{HjzwnUAW!jF6u|1utaEvj>WC2(9)D8>9u*gF#Pa;v>g)R_ve z1H30G@P-c_wHI)Y;cb%S?LrFi9t~9E`s+)}4qr%MVS3+vq1kAiyDM*1)v|>8jLmrJ z(B?^A*tp@sfUJS( zl*?^dBdO1a?oAo1Mgq5>*MHq-LBYF0$8-)DM5XXpZ=I>0DC?VR)eF114eryODJk|Y z;eG4+)4iL?IZGR>=#eLy z4t0)%If_W!@|j-Db}31fs;g?K`dFoKs9)u|;lz)Eqp~?K^NeUylfR24GhT8GIB417 z)MxTB4s2bN_5|?}CiCe$R%r;4*Bvm2Q6<|b9?J7hUoP;BZLDr^hB5YC_@D6n+O1K( zuJKEz^P*wdcZ;(ciuuu#3yCsAmXW$H=2M%$LTA1Pt?UXAAD;Rd4Z%rBl=%Gq8fVf? zhmCh>(an4S#8;|&y^3Oy0KRPV7=v}k#Prg9x*%Gh@>pQ;l%Kr;Ht}FpOc;RbPrh=FpzY@*D$g+{smWyLPoM2`UxhrPev;U9_kE%NZ|22T)T2MV|Q0o3eNB zo-6{}8f-Q*GtcBhYl2^R2Lr3z#DvHRf;;5_nu?Wm9f!3 zdVuS=#<=ApkoBZhb8L%-ij4hGNS(7Zn|ndI+hR-VS`L^Jb5~R#axY6~TwOaxqFRbA zOYOnj6~YqdYCwU-X_gctcywqe2GU&{VbUUOaj%`pRK{*7jg}9v&zx`AxoJh#UY@r6 zylC|7CMTP>Pt4+F`<^`xEB=YJls8qW*p?baR;MJT1^e?In!IrHB?KSCLL~KO$%LE0UpJggL%T&+GDN#p7Bnc|ElKI6P^=i6K{0PnmqXl z9uDa{x?**mXgo+Lof8_~XZIh?g@v+P^0SfLvVU zfcNvQd#$t_XSin~%`)hjN*E0z34i|29Qjr|Fad15$!&9VwrO%{<-Wk5S+>fb^C#X{ zbOYRBxHiNqfXAPi> zwugzg)(}u~o-uO1%S|`kpoOUjJj~>53^MItpg^IWPQUFe;f^54lVH&u3eaGcQ2Kk6 zccRMRYG|{aS^W7OJ)Om(w#h3R4vk*=E~4r5z01Lyw>o0s?g_+~fS|u+ZZRVT51gr` zhC?I?KD(t?&qYT(emy6KHcK$|hKk5{jkP-~(K$1pK$PHbs8@=X&s-LQUW_9NL`7MH$e{JN*3!C^?uLAJ;cI%1lrg<0DG(3Rl`-7I6MGN}u%YD`-4>C29+b6m~Axt!0lye3MSX(UkQ&sC3ozOh9RpH9u{F%ZE6{ zg;rMyP+Fx>iWjUbpR^8JdP|BgcCz&e=2Evx1y@>Dip%hrY0b|K=7>DawDRncB-`xd zkHu6}jW-f2q~yY5>-3v00OvW(vfSrex*rBd9%Q*PMV`faY2(xLs0}_`TaNFHt&H0t z*FyuNk@bDMhgQEv9y|WH$Z=jRb&m666V~zvA>I>g)K^mUN!t1V0f6WnJqVj{DFTjr zB_~(5zHf*L9Fk1$h9rEN?Hvx@66;IQ&htJR1hxh1h96@8Qskenh64sQJVnV&39qIy9Fm^sA>BLbJQ34&*8v;ahBr z5V=A?a(D|R7WIzZmIiIhx2hU-?i3#DF+s?{qVft2zbn}h#Z;a&3kedk`r#caNe;{RfFuS1qLJ&=uI9=HE|W;;mhu94 zP?~GrKVME`8kd$PL*;H#@Aq(0iYhcC&9u%~7P-d!uJR>NWTHsG5AOO`Ty8BI9NEa!pt1d2<_Z^fUmE1sew z5GN^M@=W8Uddd;xILOJ(8y({!I1ZlumhfSOy+I}-vV8$$I+BS;;+EzT2v!4;7B&Av zGwmW4!XBk?dP+f{=u6?wN!c<_u}tQFP-pwMc?#=;h;{@zg*r^~&gDS@;Y!YfKZR!u zCOz_y$OSsG}85ANgJfChQj6%n!eQOrU#C4ZZoaH}r8XEkLXD zzAAAG0E860{Ld3!YQA1rKw7G-s^VsO0a0ATr0$jR4fPZMdR!EEus)DDU!;KmXD5(M zu1%F_!jBj>*2X3<-v{%6IAG>5;%hC3s!xJbC1}R!;*Psp_GjU*D3{oerK-8egx%AS z(z$#AU{dSubT&@ix)`}sHlg$i#|wO!@>Asb)iFyNP8=Hdk2yy{UEJCRLBc^ri{L$% zG%W9l7A<}h-I#Rq7Pu)uNgc{6Vr3g}GGVx1bIfJm=vk|ZAXCn?yM|D)G@hJ-SQ5;a z;XfjeGzE?)rzQM-Nq1kgUkvM7Hq8ZHViv3_8OoWLs7oPEkZH9Q-72msti0S3(k83w zy=TAp=Jh}XRxHP6018nE%5b=!*W0G2z)g*3pR;WFEOgDkBsQCwD$Mz+x9NIvv*EMr z&r||3rM=8hwnM^j9{eTEl-)*pJJSWG#$jD`7F&sDrgMbK)%8f?G}}%P$LV}|-B9WR zqx*dGOu$UV2?-E$yu_BuwNh)o-o+YXRlAn1(iM-H(-jPzce*%Yk~nOT5ti}R{&YR+26hs&hoXnH6EAZBsQ;pTBDsp$`%9Qj5%TP% zGZrj};@w-$q-jJKauc2qv!9qNAW4BPsYQz&7RfF^lmZEH1GKG!Dnh)xcR+xhAVUl& z$qHyVi2vEgY$g5U&d}?1K<79FGsAkQ&2Gjoc=;W#EV%&gN|cgIKV|N{5^$h{-oudnc7!Y*Qsx5KHujJiI}Zz0P^?9Yoy(jBQ4+^qGK z$pxd&7mJwZLkVXXZWA~C*C41BhI%0k!q;#Ep2T-9D4r%|K{@I62Nc^3x&X>^r)9Fh zsO$HBL%I$^1R3ree|2Qb_Um@MjcGkIRkJ0gpgi>*K> z|ATR%5Jx*CROAliB=|*yA=U=BKm3*5?JmW-PEEITxtc)4Srqm^#X2ZD2CKE9$~Ivhe5?!^zd+;wJ=?VZ1EZeo8jEcl9;a(K zERrPt`it?9XmEmEX9SHHQHJ`-xReEyd*#+1KR3N0eFmB#P%Smk-X(lo#-Zq=_OqTJ z`Ln)lbENxgN_rNDa-IL6j={cW4h^$#q?Uo()=5gC~X%&Z_k5PC~B8oU8^UzT%fsamVM*5r`!N zq5H)Sku$5eAlA$;=&iAQC_p3kH=PnH^Z8P`AMBs}z;a9|oXy6#Ig(}=7wcrM;n38?dTH}8k$lMNJt80)?CtY4mQFc1Rd=Ku5|zKdPON<2o4`1;DtJx;XP$UHSaf&@2Sav*S5bZWvav)Oj#~`|ndTe83gYxj< zQxx8>v<+{Oyng#OA~Q3yl8K;4E4}_;Xn0JBc{qsdC&PjKuA%w(ymos_96{ptvi$^C zEO!iTZW1xux*NyYv=isdp~+o&A7O*`THVrosdvt9DWS-nK2S2uvw0=c7Jw-rh_UkkyeX z9ZY5%?+ERB?5JKaF9B&?qx~n9{a1Ti2O?XCdt9XVY2D(MNlJ^maYfFjIp=RkTO$^e zxmsB1z;_?!qM2T;0Xt|=M?dXhNSZGsoPAc^*(V{k z^>&F5-E3nEPKKm~1V0zLiJMnaJRYkB_ck^CN(t^nRUhZXrvDRT?cG)Z-_DR{-X#jini_uo=w5f@@ejE8hdTy}Dstf zH7|W`O0je*(9qd@UUo-+Nb~-eNB559kSYVNS~EqA4Np2Q%X~$20J9o$VrQ_ zH0MV-K}NG$?_{^pW~!nvT~VuPNX-<{zpCa44xH%!*jPfy7k0VMMijdy0`A@d%R)e2 zEnQ+sL)gtVJZEasV@Ict4w34Ed#9)6dUvz5#QrS-FTHe&g@mx8;nb!42+xTkyioT_ z>`=LwzJ1+XGq`syw>d#QL`r57e(b(~LX|DsIWWlPJYQ^T5~x!I?O<>(O{|!}f$+kA z`Bd5)CP=bG{~-<{-G9Ye<0g9M%(xQ0%MI-nqMUmebd^_ioIlzB>G;lM;>VPSz%=33 z#|#dmtRUE$sYN~38p{giZz)8bqL@fy_HHGy2KVv}HV*|M!D6j5(0XTaazT6nPQy806KkaStulfV&xA)Wt8HC)B zVBMZzvxYK+${<5%{l8=giI4;n=G(w5-*J=)xs&)^U8$J`fVeSHO}=wU!kUMp&aCbhH#WyKh;vAc*HG@r#C+-z4JHV}qb=^7qlVw092PvERnxZ2kw5D>bEU z4t~O7`-_gbDtlV339pg; za)_bKyRE+v2ZdHclGPv;)w`lmn$_Ls>YFO=I_Lmm8rofWJ8W`Yeua>|KR@3ZRr|zR z`o}?NxrhT(;J_hd#EaMW7{u5dx=UwR07e12x~(=dO1@Pe;ueZ<+-+&)6IPI@MLL`C zP*}M7B_u}{-O!w9kNO;OD!8d_VX;X-{rL#kN8`P6G@osmmSuvsWb>>KR4ZkX#v6?~ z@mTM=+{XEMhet(-K(JMBK7+M-K`7$itq+*!&8?+Cc@1$_3+h)r)CWxlXyVq&o{wcq z?L68yYOG9@d}CR3Tc^BZcK?1v=-XQ~O#x4l##?mg!Zi4dQIQ12jgdgJbuRyfNqmqA zaI7epAFasBd5|(4JjDrJfujKbL?w|LELGe(*{rV6%=`I{GrB4+MlR3hEd3wMlDph-)>_^2t>;0C4wzHTbC>(CNE|ZvC&dCwG_q<_G%#43OvD?6?WUPY`!Fi^ z>mbnhJ-#E391TPm%dxe-$etxPx52JovU_9pIp9-qwI@B3$nZi_-ry0(-?){=+zSjL zXBL#D?J=r(qQ|iW(tmZmaJCAcZXPAdV;+r-txUZr$KMkz7Tv6>EfAscW`M|yH?R1d z>r)Etn~h_@#-y*7c=1lx-tg-+W&dJahy|7~%Yy?grU%xh;?Qn3fA-eNLFRz^D8O&p zT71{J;^wGQthx2EJ#h#B1tN!ECkX!x-tZQ1pJx=xQK{?DQgjobI$T7 zC^~>i*uG!By)7$9SlGDoM@t*@E-jbCrLz3D?kUPU2?d19;95DS+qh}a`<`Qy_)iKH z#D_QB|3*zsD?DljEM>sng2hzXaLYNaMoGNOWhDO6iA_9nqq;V3@)TijSngljfBlW+ zf{f&A`PZ7=*NOjRc60lJ9|Yqi-rZYHvTtHi_CNo;&O|r^sc$3NCF|czHu7B(S6)c) zCIm%1@^-O4Mm%Qe>8YnN9ObN|;&UvuYK-Pak%;-S{O~``Z(wSgbz%}B5n`QEL zt*Dqk@OHjLy+L8r;y=KcFlygs`kzDJA)Hw$>p!(H#M^(?|_8cq`Q_7;Aj~Wwb^CP!~>@2NKw0+5+c4W}y%qG1?e!j8y80+r7gIOQxclRCtPfQ=Yh-gb|Y9#6D9gE;WDCPVE zQY3+>2GmMLb1prXxwNS>U#+cEdg93#4)Rj0n*I-6-vQO+wsjjs@Sp<6ihzI(kuF_2 zHc${ydJQ6-P^1e1g4mU&Qlti@*H9AyLsSH$_Zo`Q0)!ALp@k%G$K$>Czwfq5 zBBL$uhG>NWCz=j+#B}+uZmyhJWpinUTur^Lbfp2rJ!{m8n>9c5Kc7Zoz?Qj8E@>L- zZHHX88k0yFa4DAXHP-)OUw%n~M13S>3{rIkz)#rss5JjJnK9Zv-a&V`yEow$IXDBf zs0WlkzK4?FuiekYzD4)EQ_N}{b7>bnQ?K{Dp#OI-(6+a*C}Y8=R89QXHX0-NQN-g$ zLcPd8Yg6~EzX14-3j_KLm6+X+W;o4}r)2AVjZ<{x-T{fv&l+rG7 zxz%!4&HV!_ujmftvssvckS&nDuZ?W&uQoW6K2)65R5f1=`Vx#IantAZ@fYn(XJ0H6 z@*VCsh{6A13PaLfg7-8ac4ddnaW861*OrLtc*DvXOf!61F zJ`Yn_3}X5%fJt;w`H%I>2l%Z@EAT0SbU@W1Z*uEm2gNdDrlgS}t+mwNw;i9N?SI4) zoS9h|)As9a?Lk=`y|U{71%!Z3sHAdLTYFKmU0J8+;LX@Tp&WJ%mOS-X4M18?6c};H z5a-)wpWmHJ?+_dVY@bA{&1gyS_4%erT?HFv0`ugFd@neutil~JzWdh(Koz!QDIlZM zeawJqj-nV+4q*C%mh(%ief>Q^At(S?zBzg6gg{G!OQ{;CwNnjGGTAKJ3ZzUCc$6Z1 zmnU_V&^5P_=+EQ{Ies^tTVwJ@CaTu}d$FY7g;*PiU7yKi+W{g~fKHe?wAXtbw-xUx zg%;i}tBHQaO)_zZqE*<0ydAA8o736>t&K2zQe`)BFhRs^AX--Z8pM(HGq2}TYaTW8YqPDFG3 zX6Jp~Lu3F83j2C5`8R}P#}|Ph?;sFpRkb#U&GtMnRm#en;v~Tf06oPs177>dtr?UD zY007pbvtEqkjuGjE$}_x*;9Jzf1@oK^HuI+tmcmVaDN~gm@G&NL<1wZ4)$%o* zTXfdlaj*QXn^~Ck2c|zCphVk>H;omjOoR`9VvdFOdy^M|@RRbb-8r94Q81ePzjBs7 z$SFajK{Jv^YS&_fE!a6ZI-@U(=aV*bv({8mS;^9axC=C9txH4#yO z*|FiF*Hw+o=j*Vk)yt7}1H;naQ#JtEr%w`8Qf8@B#W1GnQHZ!#&a%x~y2eC}f!|Z8 zi>+71JRHnetjXwsZ09}&Oj-JE zoa{r#U-QH98lQ=8r#ga!x?qzXTB_h6;%3{ckzV(Mm8k&CdrpRtW<3B38Gcw7I1+_^ zNRjs$7|QD)ETYv-Zkyy2lRD7#$Aew-=9XVDeIRFl&XrbuItXS}4L}9tsN8aRaGHEV zqj{au!qeHxj#G~Z97K|-Lt8d{!A%*LJ|ttC=kGedoFh->0s)WYVBUv+$NVkWJ3RVB zGv6K-T*IK2q?N1NapNvO3J-4BJ*cV+jy0P1FlXM9W+wV)E$4r#x9nEuioS!k_c|!7 zxPmFI5FpN41tdBuKU^m{p2UN3e&QnslEr1H2558IEh+#e2_VoMn_qBHl;BlokIo2b zYZ|jxH0j@~?Tzg|jI|e6^=vtx_d%8~^ENhL`gZF7;D1#<+`1Jtu_DZitw2FnpNz88 z`_AI^y?ikrlY^n3=8u4Kh3#E+rZi~f>2=VpNhR{4456xBDW$*2>_4WOx~L7$lF#lv z(f4<|Ksa_W;jg2YEU)(la=AGw`XgH=)~ES+wqNl6xEs~o3-b20&_KX4fsw&MTdy|6 zqs>i$C573upWn^9YWIX)>2}vn$H18M@GkDq!3{7)3e(_!vq97pXSgWR`7d*7Pvd@W z7_c%MA)+-SOhgYv0ilaD{_VYqR*HO#q>P}tWiI}E?Dvk!?O9#6SSxYn#+t2umngl+Kv}Xg;TcJ7SOo1H88L1&Y;{< zpf0+}buXu4Q^l)SG(p(`kiUrI50gZHuDXqeuFyj2yDT`pd+TkY8I2%c8~_lrLpwz0 z;;O=`(!{=e(nl{~2ly|3{X-CHRKbufQ_|2NU1ed=+Is+u)hGa^SI!d`wSZkvUuyzm z_r!tE;e$0^j3+(sE*5oJSVrCSfk9&TM;%=8cAFIPPacVLkk{mMI)!(FYtNzem&U-_7>xZ#- z3ThCPc%`Q#Y31E*OJTmK=v@_dr1HDMGj5#1{8+<{{@f>nKi9k5*A451AJt7vpsi+C zej#xwmybN1npkP1=FN1kH|7>Iu5VvQuyJ4O_jC^%2gt^jD6fdjBM|0SqHzf!La(eB zrXBa%ZNWoQyQ0_hZ)!JGAKH5-G)9V7qFhd)pq;cr`6W?p<#YHb&l);@{O@qz zWbcJ;9Z)DVw4Qt0=CqVIMFY$V3n8*ak@#^$p@^bnS!rca)aT=He-1ZC_J`L%uI4bL zRzTLC$mS5B7G3|m@N7$3dfMVd)~)rzA)5sCB}S_ZqgwEv5od^rW-bt@1v4+*pqm0^ zQci(@!X4l&YKDTAq@|r~{PxWi$agS(bo{@t<#D|@khxl{R?&P?3Sm=pzmkyKgRelV zqJmw7ZDdNo)XnGXz`vJC;5*2n2`Zhitgzt654Nq~IsEF>SMW`q%_wQ-b|1SpGHP`) zv#)ww>wXT0bHaL|n&&-4t79awnc{EI5=`|%yU>0&Ov7Yf+fHahIqMuc{@9GO2f0g_ z?^|@3xd$LPxBrmNI$_K-G^pWoGLX`-X)@5V0@zXxoK5rH^<;t^8f{0=6p}k*$ZKx|Y757m&{l z7D>l{%Ccg+RLT>U`Z;8-YCkafy2maPe{g?!f{z+3S67)Lw<*SZ<=uL}y>4W&N$G(MyuXHO1**U8xXEui zu-+7Mzy%^Q*hh#Qj?S2uu>wk)vIKi*>PJIBBI$b-CS6tf7APHp+7Yu1BJ6lBl~XDS z6VrV}y;F4Hfl>QA)D9ptcT8l9_*NX{v$5NJLQ`m8!<8$Lb^&jBf-ICK`zdgD*4WT8 z2&g|KV`NaPD`U5+S5a9n1opkVD!{W@D9N40(mE`pdG1?ELntWPTEUZ!E8RbF_V^eO z1Y;&R^vZjs%FX^QHRYQ&8_$(Pe^CMU2deZ=$7dE04W&bIht<8J?Msm_UiLGEi0vy? z|3H;K)ii@(v@{S3E4AIo0e*qT?g-DinbXXX7Ua^d@ybH;!1>J5j74Rur0Qj}W!j#P zDh^_aeklR+@cTi?O{1E=gr%DB3eIM33nV`!M#e5w#=&)K3c7hynRjk-c%~tzms*>& zYzs{U{I*vzlb>Wl!^?eMX`mJ3_y_K_yk-SxGUXx6W!0ru!!Lt+Pk7C9PNlP(9t?2>g8F=_@<)AV*n~H0O7VqDnYBb+n@lQJZSz|Bc z((|57960^_o-@D{Q^F3ir<>_r4x~?SAn4n9y}iBbz`TknX%ZvG^`n<5OxFp7>B_qb z&U)I6s(d{u8}e;8NEa*UHx6G(q4-Wlr#58{L9>(Cs`~*#b0hLSWG!&`SG$)^tK>ii^=kzj&g8p-zNZ{ zdhWa-=rFWVeK0@kI$ zQOXv@dLcFTS)QeNqPe9yDLNwSp0@Na<%UvmIcbOWq&F{3&S88FSdCv-If>SAvQO{& zTjxCtoK5gI;yog35MM)ofy)!PFgFXWC4|4%MJmNO=a`2;w62>r$y_{|<{y)GtkNXr z2jOP{3#XN6#kc2sn7W1Ea|KLnqx`1utkMI1*e)fV0gDhv4DYRbd-KwYL6ZY8Ew5bV zuO1ligkSjHmS@%adjtj&v09E;b&FpBE!l3M*b=NejgVo^pL5b*@kl$z?n#iqm1*(E z3Um)Y5lY7Pc72;C*}jp9T^-P?zm5M~o5ROHCh6sz0opL%Ja)Z#EV|Md!zcJ^9z1ZL zrnJBamVJP8mBQJt&07EE1Q-6q>Cf{W%0XL{oVjZn0MzjLtz@Vb``rZeS(wd?h;zgv z8G$Q65u-zBJv#Uqr?x{x&w&cH1`E5|c}9B^qCvqE{w|m|It<8)VoEo|u0)v$MN9kP zfJ&S(*~2z#PWb89Kn6U2F5m@6ZTPmw?`rR6bW{HM@XNp>!jAJLIH$9Ph55`p#x=4C zB{GNyQpH4?4Oc8TFE_Y1>?=W-cJjgvKfD`GP7yA$gb z{B@j7^Jj$kn*Nd!W}~<5n4ZqN zh-(*R@p%0@smKt_LKh{40`k64+Z`}g)qcqRc$(bhW>m11&_OZv!&mj~)A&UK`R=ke zdgAVqP0Y9!`_Pkoae~7k zqZD1Q=HxV+AwJ`bG%w^)A{1j;Fgughdv=fG97Ex-u20$Zd0ZN-@QQDz9ijhTlk(SP zD7{>-}H$u7YF zjtpvV$|=WN*1i_~+(3=OYO9awZaz>SaVj|rv?)J_N@$P+g`sQNpiI!|@!iVgLlB~1 zHSrx!i2l@sF24Tlqa(T-dlW-r_iP0*Cj32-|M0gbf9fdh2DYj&)#x1vr#y_?{?E$NEBTX(?faYFLMCGSwQ zA6EHTw2XUfRuy+KD&{h0J1qLhkVc81aHO~*@{*0%-_tYiDQzhq{wbV8^8X?%#2D!b zXwrwcQ+_)LeSlOOMC4&Q-3EJ>SSG?I^kxlQ)|WKj{ywt1URr(t^ua1A!qFLPe%&{| z^pv;Ce}*4&p1DiuftAzf`uKu!6y@V}q13BC_@g$Szll~C@s;fm-PbuYlkvVr_^F#> zuF-`89c%?_`qMOf96u*Tn3A^nQd!`t538<>)OxL!hR23o$zBrC3$ zD)N=+i50o|8xg*Qob}47jtTar@~y$cyodi@tE1C5G>-!P7a&h~k%0hZnGp9r@?5_l zbF(bg2C5%hTFAniA+l%)Lwh3+SnHht4mX>hRgDBTqGCpW90Q2=+Lq7A5m&d4`E|V3 zlkyxGPm=a z21Vdl!416eVLdj1qtg6X269okwx61IQr>j+WBQn@oiE;Sf{r*Kg-@^w$xdL~hp|TD z`U#On)R~zDmO0^Xi+$kJ%t9b6kbw)pc|lnjpIX|lHoFXK5Mq_L267!Ka8;rCBW{vBI;s{hkj`D z^_E2^3XMBRdFyMq*T=pNw|=25?6B;erG#kdnrPWpSl`fe!tvvceUbcgTy+L6N(=W# zJLG$jUex7zwfy#JlTWe52K3IvExXr`>J1yIS)T&f0=_NO7-7xdq8IR6uCO1*<;cy( zg_v%Zxp+PXT3^I)pknoxpwpaV8&MwHmAbQc@exZBu_?i>;0IZ7Q=1jDa*;U^`NUx$1yt_<~`PmS;!I(q%nfSfZL_pwU?`E%;HeXV2_ZepaqY4n|!{9zuYqm$jGNxVz9S1^rh4|+r;NbF^b^};sPVg2(C5{xP!oyova z-dblKkW4#d`J)*~&U#$|R{r??To%xK)*aJ<_w$-CL`kKZg=g4QJ(_n|Z@56)E%UOZ z*g+x9+wOKJTXctF;;&V&Fkt73cX5x{fO-Am(J?K+0G%gb%b8Th?U8qt@VDCxCP_P5 zVCP-e+X}gL{6kML&tzG!slKwgE}hVL!tH$4wTo@J)B8Gfe1fGsw!2*S4xvc}$iwLQ zBAkx(gCIPgpS+NR$rBfNOpiKo&T>d$gZcwG*^N2tr6&A$iQ?cPk0Z?m=+KQ$6K>Tt zMWC$1^a*vrZx$5qM3R9oU_G$01ip!$Ch0?P7Zn|!@TuKz+cyn-7$(rY5R)vX`tmNk zQ<;!&jqw7DzXUVU3xJvBdo=*d!GwxUDYxy)I-4+1l^k7X*;reXW6CG}krHYbJ*L~* z*$Jkq7oIj0j-HoBA)fLoomlAW(^HyL!7C1|Py4T4fBo9KWi?=>H0xW`K_V;mUj~|z|D_v$Rjl8sT z-@JNbNYhC7|2qCQ(59u{g}#*TMWvH2rSI~J)!+>1$g}l^+H_&!^FH>Pw8tt_>ox@w zMI89`m6&ShN0gSc;U|DMN+Ai;CHA>N58jy?+T_VJPcPu77fQdpE;vyTHxlbcdCuWJ z8{`i8GOtpy-j+#RZ^{U;@{{;-lG5-FHRc+DmGh5SjoLX!L~7$7vP4rFKEu{`(X&>F zskl$TkfZSW$J_wE+!Z3OPg(yelaRb-zgGhIi@j_9{$SU7^e5+NzG1raqB-t~b~T4w zAH7!7z#*JbW|AB&xPT20F|DNIuyv5P74D2n;gxj~uMdtP369{Is2kY{6ZqgpkX z6e4+W=`)wa?L$XyA3YSGxV+rCZHIg05*irj9Ha=?Nj0SpM_#yOsI9g8O%gOnAtTDs zDz(kaZaobui6}QzJFOwNc=E0MK%o(5U17)Rz>v2wKaYgs(m$jdZf#7(%@wrW_C`T!bJEoMD1k+zv9IabjrmYY3#mK0NEcFg^w zo15JT&Rv~ol9N$gYSB=bDDl=y@TC#|&+=Oyc^@jsPW@awgHA7b+P($en`|3hiN6E? zvht1jwRvQ+CXIiVunjFHU#LfJg-=mi_BWFA9=wLc83NJW&fw=uD)M;C;OO10F`K4(Rn;ErWzUODzRTHWTCu>-5ZDa*nMfH_`TJYf z^F3^RN{(bKub~o&XBgAwf)^+W$%dyVp3Mlu=(_Xr_fihR-RDl!1ub$9datLJ9>hG| zw`E6gSXxcA&DxlXHt*X%(6?J6h|KHvIOG?eC21#b*)rVrYU7MEg*_9(tcnF4U{9=E z9NbVJ*9Vr8M>bp5>rEF&>nH|>L9!(?`PGTxP_BHdNsYxf^;5k>D)QYzi@!FR;(t72UzHP1UzV`e7t5=u-}N88Ob_=0VQ|uNAu1yiv)+y2k0ghOPsXvFKX_rBS6h zxKf77shz+bf=(CINxaUH+mPe0+LDR-#(FyqeDySAvsnZ!mIU8~>x-LIW+?FC-w*KP zFAr?$D$R7VMQ{3HGA-~ou1eWVtV4oc$)gFvHEIONDl5SQ;tIH$PXq{6T0J^@v^01x z-OmK-cf(8{Kbr+wY@{%F0!6co*M2YdlQ!~uE~P=r&Q{@*T2 zZ1E^vTOt%&&S|&d?M6g`#`54X*wbb`Xq8h~%4XrW3hA}UEaxNCkdbx?tdd}Yv7Yra z-D=OTwK>b=7Ge^<`BiQ*eglZ2c%#>G0M+(AsZ z9?=+;qku9wtJfj|d5QWk!NvMry$ZPAMZP3o$b4{$<&;o6Qgd>-mMEqA>oY3}$Dnza z>Td!Eq)KqIDC?1vfmWaZ5R(3h3y?XRp=DAG{Q}&i+gC50s5x}-m04gpFsWV8U2%Er zBsPY3Yg?|mdA!b8HUsxV)@^Jy(lyX{!ooD+!n# zU8Z06tR2ll#J;3r{V<+o=-Jk1jEUnHTCkE;svWZ9{q&2fv^-1$!j48rLaPh67z`Jc z6sg=NY@tfvr1L+2Ahhvr-IESWE4-R`C&^k1d@bj+M#dH4zHm0v{M$WZIwr=y9A<5X z=Kii**?j*K(xZ$+co;4rq6DCMH67zsP3n2R?cuy&*@|C7Pn19N5|{9vW-S&9>#PI#hLt_a{|UrQXkBdhGdh!KrY9GaCu&?;*0G z9wRdPJgQs!UtddKl#7MeKdC{5udwB?`^o?gGZ(QZ>sd#tQdc;QVqRn}9n|{9We9MI zOMs}>PaLpAbp+BnEknV3Di+@czg^lQ_r^$%6&mSNcfP+WzP#Pe-%rS;Ej5*GaG~&H zPX)s-w^vZeor2yjwDlPNhD%o76rmj(cT-)OHQ$5zNYTJ04C;xU-FrvTB#Hai#<89Y ziIk1$UW8u%kd*P443cl8z+{mfVtiq~y(kA>jdnZ`@I zSXG@cTIqNl=`p!CR+BW|L%&ZY*JJzC5))?o*2S>_b)KU&F75YC*(?x)*u*#S9{3=V zf7f<)tgmPRdbu4gnf!(ZiE}FaEE_&>k9pdJ)!a{^)%u2S)rn!?+d-&-Z#(_oZec`B z#oVzXxpRNqEMKx6hO7>on%g>|>URohKpA>7VA@64?l-`zHq=s9|ObTuhRr#@@F`mQa9VsUH&Mt~0L#6KH z9jyrx3ConMq>l2sCT@(Kc{`@JMKsWB&0ynmco=DhJn4VzuiAcBS6i531*`kgF~O@U zb!3|FPZJ`wqwNlXx-BI#+9+A~m=W%>@qp(X14cQ#8ANQYUTQVlH?p9w(j;)fZHtao zMD%6&q&$5Nyz5FGe%ak}u@vN&`L1(>i**@-J0|V;=O{-d(bli{pYjyCn%SXz|?XGZMjNBq=#1XS=-oMZUQ~wC()$jM>Q`F-#`^ga9J}u(G7)M*iU*AvW=+R z^5*%)aPRTAu64S4MI-M$aTX`?1L2zPY?c9~LsoBJ_Ik#0RN}wFF84tJmHy64(+A-| z?1u!dGJ3oA_JwPxoHJMOMs`3V<>3d{z8KFz?{95j6OC+GVT?(SwMoUHx}paxzXvW& zc%}w=VMJj`BWzg{7~Bx5EdBlXwt5>I7l|$6>o9b_l3{}#ekz$Z7qY{s2+bhrYVo5^ z2Xhq+I2v~2!u4R51?LHEkX8N5?%=a|{MTpu0yKvCRad?pAptdXa1_)@%n^YJwVO!V z{YWfOAUCU#b&2Rn4H^K8uZgt?Gp*KDv23s85 zxL6uA2ZDj;##7UcFJri+1YX<-mb!ty>aNs5wq%EVyGbZSlvqx+?r>XQ8VEPIE6sQW zQwVgTQ59X?O2h^}B>mwZ#p$1eocj)xOC5puT`g(2yn(5^7Sz@byw?sZs6yiYR?tnt zylA#{k!H|R!?3Qkl-EHvL0+o9x zSKk^lhF%_Z^AsDZVBo0{@}>r7Bu73?EDLImJ(WsxFT!}D3v*hna62WkyAs|ry(5Fu zMTDg>XDYsJ+@L(NyAAPeidSkL)k}Y+B$Atis~0-@%iNHW5s+&6K?5E%WvYN&cl8H%_FbcS2hQU+ zvC`#c39y{tu3Q295;LA%CTHeY^<-Ma3b@QxxSwN3JVYpEpXxf%VQ&8wI@Hn34+DZ{V^`lqmt6~VPE|bc17|(7JZ1m*Ro{oJ3?j8b#gVifV zJeH`R(xdA~yeecxg&WA*T_|KN^Jv#b#}tk<_L@8rmqEzg zF6M6v{~5{f^?x+TJf^UV)n&-Uj=&cOhv*wMVIf<=cL-~%c29>t=o39maBU@_Ek=pv zgWi+1c}a{9Zc@K+u4}cALk+MM3B_bh7Vu1bQ#6n z+NRl}CuH$=sJ)PFNBTMb+Pg0)ux^#zPls7Q=`!NWQD%i6Tjk`$_9zu3-<(zh=fU|0 z*#ft--V4_zF=nduY|9YCU)>blN75csRyFg@&JwM03F72Q^!|UxfKPuuYjou5#QT7h zxl1fR9%%_b{0^n$yk)3U!h$*<-`^9lhkE+J6QrKL0oNqi1baEf?USs7N6Ov^+lN{^ z%ZS|9Y4JzHB?_*+@L#)xnC7cnN{TIkb!Z3THJ(|Y@fa>2sJB&(}^s>FK#&7!vJg;R&{5gZMq%57th1`x_WUNNn!a^k7`*!VEfezQkmZj zB-{8>#4}$bRV!K15pXcf(#;ljGq1WUl%tO82W^^rjmOU+lq)7C7V3pM1-E1LPCi(R z>+3srV5TaG5ntW*m4(5?QUqls;i4o&hjwW(&WmbNF#Xix&rP^}F%C?@-LR_je5Eck zD?+?o4Snih2r-mM6mq#lNu z)|l09c35D(a?w{UdE$B>bW_c}e}p_=%#G-St9SZ3=sIe>`uK`PXeX9uh9QS#Snyr{>CnyUgZoBBd=EGOjp!%=I)hg6HeXID1xt=ofn z;YI#kl9RPPR+c65IW%)G{E>b;oI?@O*6t_Lml3;+sG`;0t1lU;Z`4SmCJA87t(l+? zUJ4!2-I%BA?| z9S?kvjxF&O{r4_oI|g*{US?P0F~boYlf%j#Do*c!S#-x?>f{V^iKf*upH}O)J0!Iw zJyfOqjS{7nZ{`5*X(`cX6t?wBk@h$hWJoRYSbfUJH{7*DUWWy_cPbKXZG|7k67;LA zn>Z_VHZ)Z_T*fbM8~|}Ai@z$6V-Fu{C(o&JysDH&@}sW*TKmvRn8h6|zn(f+4itjcWu4 zTkfs1xNwJ z?q&GHuUI9RsPC4kOL?6?vxpTISJ2wmiZG=?$S=OQLTyTs!PSl33N(M8$PpCvChz(c5ZZ^UZuoy0Bl0Z5Df#DBt0;=kvaYx_e$@7aytL zm`s;Dt-t3pOb)?4DUE!yLx!Iz+v}uER2D(2EeV#D#s`wydx@J>DRUtUxX^AhPoLTdVjoW5vaWs5ixEUmH7!oe^^6$`aM1K4E~G+vEE5oitD!qo^E%h{XU8kz zXuV|J)EfI73S&Wl8cIJ}SJucPCHf;Tr}9W|o82V0t-HMcWkx;4+7c8YB?|U(yn@8p z<6FpTkd+Kr4-^8~f1y4rV(K9nE!fg=XLq>XOr+*iPNigsu@sNiwiG5vxezhl)tY9K zFNtB35*e@0v;2s&(qAgN-ZxH5uvW_1wu>po1l6t%yTpbz3v+!6{=VS$h7EE`b$NNX zZTw8fc-bf);sK8TgaebB)i?LSY~B|VqZwRf8%N!+e_T6c(QGh%GUK~A})albzf-fHs zOkK~KMO!(ES&%~{nYaUkG37Lg`UmYoZp(uw45_(s`78?X<95g2$I;Nf^^}6i)^J-H zO?^Xp)C7;^`yV~D*g&#fW!S5*OPkWFUMRGp$4`?@8*tPwfgkCv?K~5q-_?{#_e2G# z4uB*wvBYU_L}cMD-J41WTMIo?$0y%+<3NxE0&FTn!);I3)s#9Mg6Jy=oZJU7zFw8I zN_oRJj7Qm;4%VAgG+0gDBp%5?Jv7$A58A@Tj<=709E}lNc`5uMiScl8)((|JT$n7k zvi*11yM7Xcy*R!8y6}-i_34|3 z@}-x@mmLn&ipI*OC63~`=jg}ib>6f@v>j!~GDV1up>XhP_amvYL$6#uo-2{&(Z`JN zn?136>yvV2u>XRs)X86(70?6!{?LsFIUrhl@?9oNZ5c5`sX*jQSeAqRESw!2^{jc( z2Af+kEbz3(-7@xTj?Ln-%rW@HQm6YA2eI4(CNQ0^JcKpORlc2;WhI>JDB%u z>?)kaalCW9Zs`M3vnqDZpbDg|!`9Ik3!qXsILZiJ0@eXhneiTqE6&yqVc z?)D8z25pKbN_YaDI1RnE;7uVreD@RM5~rM$LC;}1GzlrlsA&LAPb~6 z=l&vi{-;{H0@CR?$bLG?woY+3Xtp&$Qus-OW#mv=~BWoz@`lwgMt(p!Rxb;m`V9z}F_*Nfoy{^~U%fgL@2x)#Q_0D~A zv!*|xNP7xwXt%d%k^LIi*A}D7m(L@m-`(S;QLy7w!(f_Rihq?gW2=`9Kjb|*_lC`g z&THjF&}^~q#f^mXmhJ$gGm?M{pIT1RH7JkeQpNMR$ojv0Qxa~W(g(6!XJlup#{osM zGS9D@duwv6+<8G0;fQLkk1DY*(=_+RK}f{dmtcMfGpyP-D@)#1`}_6Y+xg?;}8JP=Aw`Sb(U1IoL0wzJeO+31P|){f!txgzVLG5 zs6PheO?AzQ9#C)d7~ZK!n7)`vB`Zv^idhgD*(hbAi%#C{(rC>&J~4C1cVl9Zhu+%e z)8{xWM-flnHhg#S`cG^wJSRJzqFLtBgDjVn)SmKBaNdDxE5$HT_ zwNAPX_M5ge5(bj#N<(;t%Q554gS>xN0gML^T{Ya8&la*(h!nRDsAL2#IVG^Zuv(mD zO!r!XJoY`ibFrP#j8z7n31t?W?{xUxCvtZ(ykL$kBf#X zHa~4gI*!AX%2P)LQs%;R*cGi})a`ns^f`4$ED}Oc<)^H$SgxOVIHX zscU((>X32;F}ndla%dDNnWi78ZCm0j@)(XRC=FwTw@BbCU^-VeqGk@?9IP{cOln&;T-A#wtRH7O*Y}n?EzeriNyVSvXHtsJ| zy@cusiq&KI_Rv<|nSYcCz~bQnhsNK+TatXLDklvF$j-q49)kGzMe4OgzTlo_LIc5GD+);)F&cQk*p#1cdxA)h@MmM@3F67QuIH}?o zX;ksqFe@UAaQLr&SXIERI+o1iF)HV^2Avhd3rp5!CqJbNUJ9rZmlKfaB2S)OY_vkz zjQW&ijnu>P(pRRfOUEch;|A4-?}V%(d-Jj5>(txkzVL-a&gJrGRrE*-$$Bh|E$2)T zJ4EZwhifIs#LhrdYY_F^`-X&blMo60vZX<|NIG_rL@KQRv+A`z{Lb16)2y177w(@xu6zy>E?&Nb}g4@MJeSUv$yK1i#+*!@(j2NuvYKqgwNMsAh^nq z^E8Q}jhKIP5Wt!)Ius*&ve;+}R9F=Jl!^mc5R`RBU`rIJ2TY=@!psLjNicD+W(Nuo z@jn+7=c4=g${_{Ro#I;=Y9pg)PkuVQH~Z!wu-yqw&--wf(3y@7B;nLC;^w6uqM|eS zqt{qLnTZIe1lspm}AYytVtJ#PVD=h_H5Na%S!F}J5D zJy)SriTF@-Sp zmw>9qXugx5$Gb@StK1P#2{^o}oM5Dfe;u%iW^MxjR_}3@|3eLjK(=4}9`HRXc0jQ- z;7d4X(kf{x8SO(rgL>1m!T?)t1RIUJJqy&k|2(fF=O7S{H~^^<#LW%Xlpvs9K0=<= z4B56ZAGN#8XyI6zMLZ_XPWFZj!Q2A&TSy~1r6H#fJzT7rSgG$rU|qb4wgM(kWg?h9qu&CzwbSK*> zwN@P67_XC15FK$(6!ks^I+>B9ZHy#P|7?5G=Q;efV8c8zWZV3LWh)n|Z)``=)d@?i zF58(E6PfY*=N>{bGf%TXZbSglF8p8#W8fsXtYn>W5Pg9iF79% z%Enbrsrc4riu18T>%2uu31|Q_xgTaS3XH>hj(bC0>?G5??4(kh8+d5xWl*}O_)wtw zE)^6;H5F>+)8~A{Ss>f50#86lY>>>d>9pe)1Yb@h+{ugxec?6nNqINO-CYda5?ATC zvwcI7m}U`%<0BWEJ!nI8{i@d8-%T1LAdvl>)G?@5Xc+}bi5UI^Ll@?&8D zm{JLV54g70y!Z-cYv3|NgJNa%X!SZSNt3_>iNm=*1D6F>rfR3#!{13CBx27k7Rs~p z;ohw0L#K7;T@v1~vFi&tpxRHsf&wmm^>Po&&p9VdSX7e&b@2KfkspkXDPD9S^|;<+ z#RS+BRyt@yNWJ-2Q4w-V@|pGTB`iUpz-!>lGIdXKo` zgUA*A*^Wh)xt~lgBqJR3S+(xS1dwD^cZrwL57u^U(wE8NJfoKz*him3d8~_CtEa*z znKg3|Fl@RJTT{3rgKaod0N^p<;yk~=;x_ts2#oj$&Y@F!|KqXGM&96aP%*09KTu*! z*%{0VG*-pl)G2zWdj*`W)da+uNJd6((L(GZyykbe5f(_AAc2k*$Tk@T8_k8^)>oxd zje0`{-Mm0^vkDy0bNRA=UR64@<X06e|69Y()?(4>=35I6ohdu$`c+n>!_41NG!$134z^;lOTl;U8aUu#I_v zE`_5-q!sq6-`T(a`++M8XRw^ou0AGgGb=3YCj52c80@1UegBq_QPGP81JUHTu7BN; zU8dRV9evw6iPHca2QB=Mr)WWi{GIvUtisqc=hTI_HoaUWef>Lw-m{T({eR;KNN^1E z@8NWjGh1Z0+7x;MHKQ%O>yeuOiPK-9OUV?S-#_ zV@Nv$1dT?f*`oDOE1N;oA;#K+Vk+ZPc$fdxhtwX8vr}|U#xpx1B3%su{xItxt)u}ygnnED$`@h5SVMxT@+ZR!~31UyY#trN+W9+xK&Kqj3ErV+LH_+s~ zo|6*1Q=?_?DSdQ)BE2Lc;NGL5m3pZ=NS=z@S1`stAE0z~cct+o>!_jL6!XH@la-IJ zgoE&PosD$yf24Q-0G(O_OSk4UWc7^(`9;fFQcbByg>YQw^8HMuGjUkB`O;nAhnyD8 zLw#zM)Rgma7lhX5y~FscDk?+^iXHz1;R}yJ-t?mwU*E{ywR6^Puqm~&Y2<1;@QLW} zZ489vn{SrhifkF-P$8UqknZ+bMf?l#Qfmo08CCT#JS2cM7&f^`;rOFl0?|6nY-|w5 zM~-;(b&|F#6xVB{ch#FZJu19A_Y97l7*d6^5N&G&<%Re6kG6Mlm zG)*Y6vvo+pr(sxnj1v-v&=vM`|Lf0hJmEDDezXXrGGvj28`K?(&8$yBU4A#+$aP&^Kf;aPj4Lkd zS=R?R)RWjc#LY0Egh9X)vgUuk;Z^1vez=b>u|E;C?9X7;^2s6Y2Xt0WMtVw$E@989 zB;kb=Z{3$vO&oZ9pUq|*K67X-udvJ>WF55pE7C>yfq>kv^cMe$lM=Dpye*7&YK9gL zhcr3EFsP_l#5c+x{@U%7${iNs82!`Xx~piXwYgu@ z3F2l|Mxw%fY!%~SL*nY9?qwPU*EqxhG7tCWo6(9L)n8L!$%L1j-`*HuK9@Lr;L@=D z+FXZTU*9pvU7M?WS1)kf;dovv%3E&V@YJlr@n)MixbPA>zw`(u+#Nw z;(+Rys;c%cl|WEvLB1y(Fh(u6=kNXimWK@p^HQ6Ugnd_5{H)$`1&$AQd4lB>Odpvw zG9s;?7g*c%V+O5LcW~7%9VxT)eUR(~q*KeoDW>@K@~3RjNe>PrA^&29fRifpp~Zp4 zI-6d$mz6@#V`#NiEc5x77Y~M5yGTC}IP|?!+tLrL<3^nvkm#HBg?VS87q{#RO+=IK+o3%yn`+r_7eBf3UcTY|1>T6l4_eFImG9p_>tFE( zZaT{{=}Ggu+1k;3i9BF)-DQF!fLrkGoZ@Sqp7JP*ak|EeGnzf(9w`OnR^oC^qcLhH zQ?9=-H^(Sn>H^}c;NM|^v0k=7!^;ftGCkABAau90$I{k;A<<3nvrL;BU$YV`g;d~9 ztjGR$I?@CMRX2=LUJQlYS20B)QLqoj6m|~|NF(%>t^%o)OJ`;wK7HzSRfLmFaA6k#;o)Y`ygsvUwA67fM4v@r_JO8Ds31d%U$)2@90w;>OJ zu8rk1j%~DJ?U4IdAb2fTS1|rwk=bGZP$T0UVtlxCACW1hqV@Wa1dZIrAS4BVPElQ@ z$S}8_%Lu>*+9mY`g=ssUxiG&xrG@XmfE4)A(>Z#10)eI+5K++=++AOP#WIpk7qdB4iO!J4aY&6Z=*Fcef}` z6(s0R`@&bM#_7%G=(Eor!oiCz$gaDk*4Znhf{5qH!ORZM2@6{Sle7jUn(c}Fs$pA! znD2dEpTL8Rm%mzWW`RF26Ww&IaD<6QfVI}!>AjNRWwvuB6V`X7vjJ=LPsX7oVLT*N z>D7=?&cqdqx;tUBSCrJ!a=9Tj!BC<%awexBsc0w6NrXf@*M_gUX|xWfZVj=jB)Ny68Wkk&$V4)a@b zp`S0SK57D?@d`;$*U<6JNdN>8mn3rG;}&L*d_N5Iwel zin%_R!^Gv(1~CFCox-q05trwN!$cq{Uimc? zRNg&66B-*fhj#p05>I!4-y{h`eDiY?kc7L>hklHjgGp9lx_D=}!Zte=w0S|MIsvBW zgOLKFs0k=q#MQ7?!Q!*OgCwe8j?gCd6i?G{fz*}T<38hHvd3mh^1$MOp$7pV#wKVW zfWT})c}*yEuLh`ujtV^=u>ywF6-nNE^E#3F#`S~~CtH`}6A&1)7)%~P`1y`~7AS3d>TxU{lD4|iu%wt$5l0eAoxnEv?goS<@w3)IqB#5A(7Y|y)ZVb*=<80S^= zqW27B>Lcb5^p0NjRR9_^64?j8*KlPhbj0WQu;Tb=zt$Hmr3C_E?R1<$ zK2UzXORq=o2|Ui60q(38G9`6}$^6X_^s1o=kmxVRB0^AsU&5J02;dtyh*Sf{s(-@N0|)ahU{_$iTYutc4ZLVusD!5?U#u%VeSETw&}NPhifW`A_@xuZEi{wE zO>FSm*ZX-rZUMh9n1tFhQta~iUgowjbD>>^v%M;ZlM6uAzNgnid=F&*mLXCRYDYgQ z$s*%7-PE2i>!%8R2T$m7Ly_YTEltg@mC3-Xws>5CK^g)ht;tm=X)|o z*HVVWwQ0d>x*Af(Wd&qD*unSI<3eC#${ohV4OS3>@4)pFG~}6LpDv}bnx2KQlF?=u z);inothB{)*S;J3%E87t6njOaSVLr+VzNVj@}#75Y-|LvnB-8CVl+Bq(wrg(=SolK zHwBdG3LsHm6{=tnvfpFKXel4D=oWj?W2v}>M_AbMCkIqT!-fxX`wCcn|J$!wJz401 zC3X}aw+w!Ud7-b1l3G`toO=z>e+rYF3D)OQSZ=G@*x2|P*E8Y`q&4eyXFlj8%rnmA zW+yLT?UsN!^|{?p6ZX8ix&f>VgjFJ@v#G%aX!{QvIMiEQMR_`YvO&GQ2idg(W}bzc zZyWj$r{yPuzn={4yRY)4{7>J#b`OWt##*0*n!_)y07bg*IAnKH=zl2owy41iflLiY zIGBvu0w0tQPCWM{6kh%Tv(d#O-sw+2P{ZY>+SfByZ?+G)!n*|Yb(|QS0Qi_vLN`~F zt+F8yK>VUlX+MAResW0o6O-`EjKj4Smgin`uQvT(0k7%K>CAstUO$>Zel*YZfAYL% zc7ZSgAgy{&PfnQG=3zaf{rgj@CZc1bQ>2ShL}PH&dDvdavh(^ixKj?@kFizBr^DO8 z&54+3Tj=-@bt&~8SgyCqFAN<3XWOcMt*j@-VC3*EZ*wIGd+j;7^gg;h-5y_Amuk}B zZL-6{aMH=4Gk7>uVI)$Z{W7?p!>CXpTmqN{p@t83H;BUyjlA0DAi_f}>lpIA8vCJ= zA71Cq;0wCM`_I*xTjp1R_%ryROLVfW!^rUm-E-u_G1qiuE^rMehwmRnwxWl#tiBSa zYZuzp#_zs=YAM(O8tP#Av3sSpgQz*<%ObS-@l|Iri}%i_?xILw1x48Ck~s1G<<`ch z`lT($XsJ4$gJ4gq>(M|ID#_M0z<(7ezPq!03>?v0E==4-mT`MGluPk-%WP#bQ)3;T8khOlq{X@ zWcLNm?er7n5SRDvgn+i5_7LnpLW`*;nyZZm6T>B5`!B{&yYOR{Ux(bo8_Dm!2QJkB zVnLdY-2n($`9KWcdGKjqbqi4Tp8}8l{>$c2z!t#%a{G0^MGq9`qvh&G*Xlu-vIAdX zKC1u{7Ja?1I^*#h*fJxwG52z8a&Lidk+Jny_89OvKB`)jOvmfw>*1&X%Vr-WqcrDE z_+ozPSEeTdEDN4#wl9DcUnQ_&7FT&sa_tLsBYBH*WXg?;7Hw<7w}6d##(tVeKcxYeto53zfdInu;CHVLK4}VOCJ?4^qK&+q^iAs zO99uOe|9G=RPXAh>Pq*@6;=Y?Q#UaJEAU(70`rS6WY5RSJHYsTT1hPfijBBYkbZQ) z7fn#)I)8=d1u=tb8p~A}?$gndaNjJg!I(H{5pQ%Q{QUf2&1AU9h){on5Ac3}=2o|N zlI$#cy)&$>4#z-x8vv%gIYoH!V<5Kg^SpL3u~vK^e1G&X7rVWIPUAJXQ2M9xOz0R7 zWRfsdRS)WX(I6ad5KzvRv&0;IDWq*U&WUT_T0OJd&G4vg8c~LXVgAPz9(FZ zB-gnN5pMD0gIBucQcd0{6-mTtC(Yc;|8}Bo$6?fu2oc%7Soko9BvvvQ^9;Ca*VG1N zPT)Wa8jXxg#=NR&NH-qi3M>4Q7ZBDL8C{c{ftS~`yhs_8YpV=X-5);3{&6B%cfxMs zi=TCX0bG<8_w=Eac0_n_$Nhx(O!n?i%= zcn-k`Spq~N$Kl7roLbp(WLi@;U5p z*!dH(>Ty_c1SaoSHWG>9q# qz@a2ugDF!(J#sg0h?1PgmIr^qWiv$we zcv}9*(MUI(Fvcg*(f8MMJ7N<8;%*Eav4UCV?evS2D$PE164~O~4GiCnKI<)N4a*G6 zHM!d79Ubluzv>c;f0np#Dx1m#DTG_gCmsXi&l+@w*M!9JZiKB_SwoqGwKwTwcnY=n z&KFfNQs1gT`E!sUeekC_pEfzLcjOVcJ8CfpHjM&uo-aI%op7?tgX-5Uch{#i@c!ZF zparB|*p4j|wjNiU<7KSKmP`<5nzh+38-Y`m8I=?Jk~uF)`YaCitTFQum0=YQfYF2a zH~`lv>GW8szxvz^z%4lLOZvH>$Fw}s+|)@hV#=O~$lvK9phBcT>Ud>s#m2R7*jT8_STnXLKlL@o}xjMNm2(gKbT78-V4xCc5(o1h)>oa7w~ zst+8q*QUnRQzt|Nqgg6aq}PPs8AD$Kzo#TxqegG9#4dH9bfxqTwp0)5n&k#eYLm@- z7&h1Pv$!&u1~i+tReM4aZ%wDJxvEnCf$pFdM6YjRYyDEnQf|C~uL*LZ8z`YOg%& zh}dG#Sh0Xo(Y@a8TQm+LCj^4GZHTL5eQiXYnEKAB#u{Ga%-&CS56wHX2e=4Ry$g6v(jr5jujUs!}7GA(fE6vw^%Qi*JB7_x@ z-z|=vI>U6CaE3NthtK8Q>DP#2>GnC1uk(ZI5LB;j$?It5vivleb4=Sx7c^E}M*9kP zb~g~#<(^7+R%89jkFAFneL8Asw6{iGx*~b7g{hw`WjA)ep69E|@Y5?R;&7B9^CFHT z&)7_s@y)yT-A|vo49%8hnuvtvd`1HHaD;?d3z(Wzk-*0oD$9)n~{2`+uQHSaYJEP`-B_v zhXOZ8-_wvck2S(O$^4WcBp1)o(}GnNG*VTl1nJBk6{DZQ^R+cVPv1dJbnUe*eL7UU ze>Kmsjl=)0L_FvE>spSbhw$CtYc%eE}s98Q|qzeJN_a{KgjuoN=sG*WFE%A`EEf8$2Fa4 zEHhNXybWA&E8t3y;x(vnI$nh9e&(qrCyCQ%N~Y6X^X%5b<2+v=@okt`D$Hb??NFI_ zIhE|WD=2X`vqHkEIj?<6L{M)7#;4E z2Xg;Zcq%9|ln%Prax+d#IQeF~ZTY!F6M=r*aMg_Nez4m5IZZSrw4ANV$bYj6F0q$! ziT54#o?NpizlacKACr}R*j;ps?G;B*E6QbHLrL!v+Z&Ez=_Wg9ovfb|DonU)v@Bw; zON-ir6uP}pXq{)jy0Q^;)7adAHp5G)1zlDlAA7Fvn{inmqq2+Kw_D4nt+oh%mSZ-WpkXlUcR$(Ci;r~o5ymnwt>s`AvJ7iUVc zoiJ%@VYF4c(wQwW`ndtJ>A@Di6-^UnaiR&kyqVQ8XgHmv@L7?@xzRY z>lH1jSj-12&2X4$wPzJud72fx^RXb?3zhZxdYcfAbl4NESDUuNu=5hGa6KqOXTN+9 zrC->l)Z38FXXW*2dh6gvG)_E=l=Vy4tD(`FhuiBtcdsh@7Nd(YDjNP9M=z)U$v)$W zu#e&VZDn=}uJox=Xa5+*+;^9Vwo%ghRz_AVxC8m*1iAF-9%@*;Tn>>pS(+xTc@*_od%^j5b<;g3GOreLB#2D;a8IdsHV zl$P|8DvgU;w4ZlG^u{n8w)P42i)T;w-Y;aqaam5Q9r>4udY?UIe;$eibfQnMSftyE_c7b>u*G`x zES4X5Uts^chEo@qgB!o_L>u|w$`kdbqYnk;jVSui;R>*}#hMQ+c=71MGuWlF!r$|+ zJTGd|kzt&O(qaU3z09}AT&(m{9(Xxp{6(X{HNSe3F7@Mt!flz;6&Spxwx%K8oIm~b zDU}EG+P#*GgxIUXS>-FZ!k)dT`eAgK65bD8&;DrHbrp?+cXIgVlje$K=LuSMX-4Vm zV!pEY?3?q6NzF{u&rwF{x{`t~&#D_D4Q}0UP|^JO z7%zG&P}0X$G4sA5yfQCS+u=isTbc% zmbZb8V+CKT2#PG(3{GcowMT<$KcNR2R=!|frPlaQAZ$Q+ia+ltgv}|VP@;O-uWBt@4)hKhw zE8FOMc5X+#_lK2nE6lcI=E}L`Gny5p+`Mp5&Roe%)fnE~3pXv^ape>SjQ9-Y+`w&c zS>A-joS|6Sd)XfpH-n%|KYrpBg}pJL7|(~BeDF!ePsc)F_jO=vfD*esY<1zgN^iKK zYOH?`yyqp?hL*)M0dGxF1|PV7AABG_ZbqLnFPfZoLTe^HS?i*hUV}XQ9M}EKrMq5! zyL?<;pRB#+L+&JG=A`Zd4&n&C7_t(ET|BSYq<<`%%Lija&I!Uq_aqBd`RO7D&eeGf zSXF!8DtO0IueK;3bViDCTzO+f?ZAK0Ak)K1GXssC4$tTL)PfjOM8$Xyz4SOT{b|M0 zE1q{a`L~7hLLdVEe&Adfe`dMEVrf~!p`v!3zzXBG(y=W|kIH_5_YP*A9J{`@A`ZO;@lmjowLu}LuiDY{$yF()(~7azsW0-HB9M8dTZN? zES{2Q_g8w}y^byD_MV=XFOCk^PE_1TJ;fqc#PS3e4cgM+ki?}e-e=4)ujJ-;;TJe|e{-UlS|eCi9BP`ewvTI|9$RvEV5gw| zL$FiL(kr-f2BR*G?w!Wt!=lAeMjjSsHp27!@cI;3lH`k}o9Qy0EN@Wj#RZ z4vlW}i^Mq*5lsWF7Qg1)GvSpM0x89qXuj$xn)r*UX5s@4 zkvMqU61=oA{Q8P>i0D|EF-DAF#&mUIZaUBCJK=7FS9@DSeA1SD=7CoWuaTP0WB)%W zWE^WpQK6%4!_8r((akyszQa1pZ2R`5+8d=pvMshlJkEBZ<;PHHHiLe7Dl0%K{0MND z4I#(DpQ=NUPG7TGGpXzZYmzY*T%Xce%S$RfzZJ38VvJxpvZu}a&2+-v@!`_9b6m<9 z(^@YpvnX6R9V-Ss+ds&4nUgRm6SdMm%kehPx0c-5=H9KAE1EAfr;)V2R6?mRa_-jr zv38j2e%A)-_4}k&S8=z;?bzhmh1_NQd%kaJVg5tvA8)Yf?L^Dq+|6vO%XkGS z3q*YQvl>aVbER`P+xe{}HpGSx;VRz`EN&CTv-tRl?ARwal(=wvz2d2SZec7_?SU54 z8v>2gVw;iY*SW9t`aE4IrK`H)L&K2qPKX$R3#T}qJD^+s7wO1s{CB2;sQWXR-?-_;l-KD2wD){S4g;ok(R1 z%bP8aF0oW;K~$wGT2DV7QWl>p7#oZttDKwZf*X8PZ}u<19IarBK0K zKbeLkq3vlL>P7F^q26Ar`kR>rN@O-mw_4PLe(*(*`(l|d(muJ_4c}Ts)#GCCgWJV)c$?a`kraNsAGb6A~lqaH2ufpYu@!5j>Y$ya$5>XZ@ z`}e&?8&L~YSD@I~-+8}el787R-@+^A6R&@yF7PQZi*4$)++#_4uW==he3`IK8k=y; zu+Olzu7$>{NxXInm1o{RI;FvGk1H{ZZB65G85ifUVyVt&%zDg@hg10vYwmxJPKmir zXD`O_&6s^A?NGlc2(kA!-ul+53$gVJC8JRl<37u++e_cveDuA6WUBaUTdQ$=AGw2?UU^>N3Wu3|Dwh|R{rwH#AZxq3L7cq_eUwy9*SS_SWZAgGGQzSY@+ z=N$;Lj$gWqTM3cHk2sAg;c=!$wgW<%XT-*zJ{eYtHSK@O<2nkjgcd0_m!`0lXer-& z_R05!D4j-W$wy>Mb$O__Fs6K(3BP>AZwra(sU>Y;`%fgHX>@~#2@RC`Ue6k;#M?_gPVQ{vG1Gwm};3u+l8=8lwxx=G3&8w0B`|5$!e6xS1} z)m9b-2GT12VDY=Wl*RZ1F=qI4uOEJ`H_ixXzm8L3=Z=y%>)(Pl-!i^;gT0E{hFQ>4 zPA-VqM$74!O=x=l$1+>ygH>wAty?8bJ;iBlkfycZ7@pM0hsd7juC`E}H%*lW@PUJx z-k%mP40+jX>c9r*T^w7&i*%M-T=PX)SR{CGE zDO>D2yN+%m&TMFQ!8=ODR)o)epDqWh+pfc+aE69-<@`nAg$defy9Gz=qFPPo8t7%V z)CoJ5u=x92Pt z9=T_Rf6&uL-(ruw9aEN?e5`c4d$k;TV<%}$O>TtZh7Hxi+c@W&#(_R{11jKYVYUwXJKj1Jz!_dmccrEz^ba_v+cj?@2;2ID*0^DI+siV)w+ zna-Nnmew)7vdeZHafOw7XTs@oWq!6~6?OQ$xf*2vKMh?~h@g1vN-ooTu?#r_cDX&8 zoz-vD-x!`p`KbxIx~r6>u9sM}8-7-yR47}zJ$_N%qiE;!qmqLl_r1LD!4jV|Rt+iR z$T6@5PDsIpe2?Yo7K>jeumcV5tKUJMM(0?BT`}1VtB<~nd2cTc$#2#p>uKeCfq-{c zpS9~ZoE9S8j2}i%$^TKbjQc3rg}Oj#u;p6^G%!FLu^iG z8IQnh#0k9&m9-tnDNe!yM7j6SfU zr$Ds7HKcCP_L7@EHuH)bD43A|>S3{-pGesj=abdvTodj+eVn+KaC6r&J;v#xFdt2| zCs#-YG2Q<9kii(crj@wUfh7oyk*B$2H3-(ky)pa z9ml9Ni)b5(zT(2e!;{^8{r6D;)GW(eT@+N0z7RuT2YEd~B~s-}K!8iWM&>f&((3*I zXadq2A45v`kO1G_D)N^eFp)Hn;UC+coxHg>()&SX#r^{ehHP|%i(!QmvgEU!Gbt@l zqr(A)-Sfnjeukz9hN9y9bR%7@6TapkD!uPsZj_m%ZFOPwOyY$+im8~wdIh=04O3{B z88pOeWw5jp&b_>CJ%kohQ@jkTn>meMqA{usz7nbRvIT4<(=}X4NkU($E4;px`GLU* zr8Rqv0q$f~^IOP^PRr#&*z z)$RXc3*R#g)5>VR!@n<(Y&=~|p7Ndzvg5emKcoN;>hW}$?_oPg;IHz$l7x9B9dN;M zQA!G(bZ>wD`?oW;2xn2G^s@h#rY`EcA(&B|lG3hQ-?=vd<~tMq)P@tt>F>an9f=PS zk50)V-)i#B0GD%dNln#rLVH+=t>dtYlZkZTu2*FP%@%LP$=NRUg6bbX|72OT1WW+C7a@FmlK- zvVZ*;d6$^3r2RHFDf=ZNYVk=6u;>}kcwxMm5Pn}pYmsZ6etS^^L4FWq$m3q5%6OpS zlV=nh`{+^_mz9F~ybn&OyF$H|mRL2grEw=Kn$C$lFMqU^W#ve(Pdt6=F+#PBNV_93 z9ix>-1@zV?tQiEt@Wvjz3lSWUwU7Y-@?WR=3)OXv)odTsSGhd=KsXK{%u0iEp_W$l z6Ap)Z^|NMcWPB{zK^eAkjDg70RM+X3>wf3@W58yX0!~Zq&_&^tiW>eQTI@0Y&fWUW z9;e;`v>2*fTg=w0$P!xENA|c$*@#*RxAtCq&s?=@gci^|h zNq`Gdc{iB47y!%Kg@q@>-DS^T$8H3}zkY{dtXusRuorXfv9{l%lFU6P9t^lnAFZ`o z3{(-1+(%R%j!qRy&m-gD>0;n9-p%puN9exsncPEv%7BdPD>AL$t(w5RFfm+wjGybj zr0hhLs4^sMEqF1HRGM~-pDf=kZFaTE#iF;Mjnt>1+uNaA{;u0(1!sWFwp5>9N$Cpb z&nDCdT{SzF($$A(<{pwNqT{X5(hGr_yE%)&HO>V#nS{8SfSi|W@EL8Wc5g@p^^e*H z44(hcb1~~nDV#reOIIw**J(;1jm`fQ1ray%O*b&Ce6(y_u0d@-9UID_c)NtrY1K98 zV-}4mtxNY2(VrN#pPcM-u-0<9W@Xnc$jsRYxt=%mmikYc6u&J(FvB& z>7V#+KuKxP&HhJ0ZHFN%hm5uduaZ9QNSc-aQAjZ-wAdUV^p zhMzk=(VZU9e$f%iR*mjgKi$F@ z98Ya5e|})$73~~Cx~O?vORj*?Sum!_*TvkLSL22L^t50EEUTH(M=7kdIT=@jxT04^ z!oHlvVK15sGJK0?(uy4Sd7s--q1x%WD!@<13X1NyUt9NCJ`P-cde2QtkrFw}DD;Jz zEs8;6JnAPs@(gTe;kDw=^HF4`DT5b@Rh!crbR)r)Fj@2RpjNX7rIW~wbURJ2XR9qA z1BBFR-qOGh5|rUhnj#Z{rElZrepi0j!n-Po%jTK0u9-XAc>N$y5wes%1_`DlhkIBk zbK#^Uk`Fdz?86<@1il9n&O;Al?IMTnT_T)8_vE$RCLA{c48fE5ztkn)uM+?=tM@ab zuAnWmtF}gE50yBmMg|()`qDgdJtP{^9JS`>A(W5SJ?;9`{#!=gg+2ThANUZY@6U?H*i*Jzm($FD%`)opBjO$JZL?FAk;I zrXJN)MT?ga(Y#%{XZr;?eZ15anF#^O1Le%Uq~h#k_cmxnf37@UaNJ`>ES@%%M4J!! zWY4&Czn~M{M9lvVH9Fpkb@NZT8Zr=pojUE|E#J1>kiA%xh%&P67tdPsI3ms2N`ADyD zOR6Q5oP?Y`Vl2tt`l;1|omee}<>%D|nJOuTYeWZC6o)??)jgxt;jqfsI97BhIKY1@ zLPhMV@Q6JRQjKuz`8dk-A+lvbU~ByN=-~mJD9wmOZsa>l56hm@Ca4C7>^@R;9J;^!wSmS zRB>?;Bk}%S^HW;v*WpdQNBN}74h*a*kc;mhO>A#lwUlQSymLx9OZ$r_Y@`$E18-Uz z*6-~ zDBJeY!jJqGI)xBfKTGJkr=XnCq(wxRg*?6 zkkV{fYo4bszBtOFVoFrKM}IppU`F510Y$0phXtUx5%lUhnOwbG`z%F4OveH)?F&u{IDHR@ELr3}@6^nXF(!3#<0N6bnW9=cy`eQG4;d<4S z?9Tg2r_>+6rbaHnz3#adB{wI^eYUHop(*?5o-*JF;AEo>>8zgvn(CHH+IxPbSIB=x z`mWFw*f!OS73ldlhG+TOP0jxJb6E=@yJnrbrk%#MG+$#rDRGL%FX>tvv@_M9emu$#|yQLMX;Pd%4+xO_YdHVjNqDjGZ#)i0W+uB#zZd)@STWy+KC`!Om!0heX2`+z-3`#>x$I z?u?S%YD-=O<=!NM7(CNUox5n=*zH|VaVPmHKMa%aqI?R*097Z+UK;X8lF28T%qgo* zD!Rd&wET1UI#>mTY+lo9YnkOwWJs*`({q$6Q2NiR6`F>z7s@HKS<$*~QUVL#!n z`Zw6c+S6XRa%Ma)^tQzgv{6B zl=Zl3=Kdq`8cao7Xn%aDsuTK8>MkxQo!^fHO`YA8&7O54pT}o*MfSZ)3?C?u)7lS1 zvFxExkN0l26L;Gc&wgjSYj6$nYPV+eO_*f%HD7bV}~T`O_DpYK&vwdCB;hkEY=Ur@TGaQ(JAI z3`QY3nlhqJVmxtHP5Z8#8d5*c)+`rvoppZJ7UrcUrA6e%1;AVHLcf|Jc1UH$50I4r z7H;ut)=8yTSL|$0&6=Mn-@yH;9}7doB8Qe?=G@AL=kJ3MKW{|SFksF-D4t`qm1{|3 zxp^y3dsn(LU%h*T-C$a0IIe%?G^n>cFAMj{WDpthANVARJ%Al?SET+{GNrt4*sTs0 zkclkWMt;_)uhO(jTF*YUv2rT&>NO*$93y zltD+jxwICxoF>X-b}V3E0wy^29AVnh$A#oGue1L(Rxnd`B!MdAE++4@K`K`inP zwBwe4UGi8SAGbvm=L6*?V>VU}Gc^Q7%#SN-yJ(g=oz#JC-+^^5y@J#n8rm{de_G#x z?h9>^Aw2rb8Ef<9RJ3Qk0N{M&Y7KdsO^hJWb8KSX5Ffh5Z?Ztq28%mw?RNMRGo#CX zS-fgFrMD!#kE_L+q@puesH0-351e)MIW4|%amtZ77Z{-E9%mND zJes&{asa0a4~x0f5g5+m(>o4-Mw@_NqrZ+<>B{^5?r3{hz2zVO9a4g^ydj2I zBty}-Tqa6Lak?N&tmH4RFYjbAU9Uxq6%ZL9@W!FhI}^pbzQUF>|$SQqRHwodu; zNEHpu<}S>Y8&Z_tz)H%{SwTX#1&@o{{B4>0uL-A5)R4bVlz^`G+3Ia%d`S_1s7J#& zhR&bvNkAbRk;YI8x_kY{IR5dN0(yP;9m30;z6kPA*duIN6{Eg=W$O(nt-8*HM*H;( z(cD$JUA;BIJWc7Es$lQnr_H|2u9#Oijnrn{*i{DYfgOIOuief%5mu;6kHtnv%!enK z-GEAm=Upf{enJaBOx0Gu%qWH@UDE!ur7O?E)51fXL#;Zy)tC<~Ts^vahL9+c4+MlG#Ru&VwXLr4Bj-CJ6L`vW0TGZUgvvV)!M(XZ~ zGNngN566%d@f&%w^wuqql-h6}ea|=?e>%8~KhK?dUlLv3$Q7kX#8gUV zGe~vp9?3g)u%g#n&F~NFDOeoMp>6qy@@eYJ4P1agN3KaA#uaW%_G9Ry@GHY?{T~-@ z<=I~~XM3EnO=}AS?e#|PM+vdT&FA5y0L1(iXw-xALbG?E8VIj1T960n#dYVoN%RBs zTF3^#wUTJGJoDotjZ5D(<4HZbJ3lh(A(5~COFE;aYFIb_?Edugo@Lsve}oV^44J4T zEspy)QYIwQaQ9*vowdx2TMUj+Op*s{so_txZF#DWR*sLl?DyVWUM9c zZue-(3nDDk18cX(JR%U}28 zMLU*UJLpn7={sn0+Js3J*fF~I-U;A4E?VD|^k#hc{3CZr5zW68>)jiG(evDXy!Q^X zOOjDfT1c$&uFFoF>m12FWvpo4%3U(!3NP|6(7)V4fXYaB6XSr+ zH;t=uee`lsU?bC~`go&c)5kF_6bEQ*0cvo+y8PueKr*R5ECg)%3LxtLF#LpF)}=T2 zH$StEm*;5rCNH|DiS^~C)Iad3JFK#+IBa<7*P)lv5bwMZe4-3-dZ1qC@;}NDi901< zI1Ui{rC7(w=E4htu$1ScHPzSFTULL6(#wiR{pGj;G`flWb;sA&UgbKBHw=_nYYF|`RDp2Zqy5!b zWpUNz9i>PGA-Ld!-Ed)&4wvu`o0z+nbV@XT2yyJ8-1#xT(HO8_rkDjRa;jfdGd*eNLN`6_$WC}zAg&{Xo9JI7$7=k7}v zPZrk+zvcUXZ;LPHNs~AE5d^}DQvgicSM!Ro&R`OcL3!2OUkxwTcIS zL~pN)vT10pPigs@W&skXHI?Zh<%#yrqedXV6Ef}w^iyDg0B`ML^c=ADmRG{AJwyVq z_l8FWB=rlcG1(MTQlKkb0_Ub=3_8;a5T@^dmHBj(9BATv0a8A+mkv59Bt0}-$qneZ zv_F^I;%gd7jfK5{t-;l3NeO`&YI6I2AF8l7*zuHc@{ZO$_vrS2E*KIV8cfF>z*m*G zxeTIAKH$(5(nuPU>VVGmk6!RKjJkZ1LQVxja1(yQ8 zO>OZ{G3cFl(HvK(rtbWp0Cd82(K!|wa!Jhm_jQ~+OL^^bO*^BIozp3wSC~Q1(|M1Wu8IS?Af<99I=gxrv zbYIda;*>i8lN+u!0tVOC$g8f*pK0#katbuk<+e6^)rPr?>Y`i!J6-`*o8m;I{)CFw zDYD|U%~!vvsVm2(kePM2ZuPMXDi#vQhf2=T|0l}+{aydO;~TXEphVff+6HafYa~Ih zy-mmCBK^{4%6~P_{+=r74BU($Cp1&Qz!)Co(=Gg1Za2igut=zMaL81OeYp(~rqeRN z{((^aeM@QrD;XbrzIx!j%8_kh~umJ^nzQw&SDl1J6bfwHO*Iz!26L zk4IHJAGHCY>SvFKQXG#pAm-oCz6yI&BFLARm6bHyiH9r+^X@zV~qk4fMpS#f?-R)GkW3#c*CGITlb9 zge>(O*fhNIs3APZarpwQTtl1@p{uM>w&(sQvs;zf-1_!cF;=(u9d0=p?o`Mx!gW3{ zQrdrSw7RvG8pe@0dudO#XWMFZ|5oxAM(7W!?2<>f9POykO%bbGB?EHi*-SH)&CCY_ zgZ@HkJBAT&E9+Ckz-v=#!rMQ+a^FM#z8bN=b+a+OETS2nJzd${xELygsgm>lA#Uc? zV!kIl#f@#y=&=*D9r)EF z0C!%p0v(d`DSfA0IzLGVKHI;Y4Y+RuSVUeA_o8=HizBsmVEfH zb@+ul9@hc6Yx!nkgWW^B!HeF{*Qb6~nE#9h=K@8BfpmFl7gR5$choV+S3(|JOW^0G zKZ5^~wz^c3;Qc_kEv{gG;dTb9(OVj8UkY9`^5te=8wOhVeyP4 z&iBY=INZVn-GKWYOvj__fWGPP#?t@Z`lF4vty9IIeLei=N66^mM8vu);XL834^D1L z>Y7H`@Z0>fuvYhLQT@&d7fjm!GYLSHxO5Lt@N9dtR7Q%_P#+kDul(8! zK{Pj=fA;n}*ZBVlv*5&M3P3sxM#uGf#ZrjbWp&po^XvbFfwwJ<6ke=&0Wjk{eZZ$N zG&qwM`hV{O!lQ1u_=yA5`3UGGzA&~mP*K)S`JYgp-S|K<3ZJ2P_Q0~U3~=-A)^UmN zPQ6>zy8*c<<3Yx1{NEz;Um35jxqxt}?=A*rcsD3`Ru4fEx7ZGAWN zf@J=mL;V6sq$XS|C!{napm-Digz!~>ygGEkAg&yB{%}HL9snE#nXdo2*i?FjpDPR; zRDHe2V9s>@Pjv*c%~S3FeE7+~sb@ir^54Vvrtd$M-@iW!_WegQ0}FVpqiDR5_0j1Q zb*2-2zTZE9$Tt`*|4I^aT!OuMyeoG3zn=wRwWR-_i-P=LpM9d!^WTGm9Prcrd;EW0 z*#DnzKDm1km_H&P^2H?KYzZdtaIU&S>?ED-|hBDP8 zxIV}@yqrNFK`H6`bq08SR$RIzHwc1c{z){hb#dUu=BRNr3(8 z<*)lE3B{VR9&lH#7rmd8hIFC*5`RV64j@IRAT2<;P4 zuap3{`#}VQMs50K5(qCq{fdl3vdIU!H{0=Hw(}~o@;T8x=y1KA4@5JeZAKF9{#<&* z68Hlq3e@+e%2g>)SkP~{iEX#A_)N`Wa1}wZ1JWxIuxBf%nBcE{K3lXX^VORJS(lLayoJnnP96KDS%OgE3|36Y_xu(k zKLvJOA!!;SH%|0lK>810!AGp0Xe}H<#*57YliX(B-%UqJERaEh98bV+%vMEAWMwLYQ_P_J!H0OQct}c*_E)-{8yso4}nIJISB5{wbDrxC&D7j zZj^$rKw_!gSGxP*k)vr1J}0dMAW6&>6%{L7*G%)kovHbM+I#bGs@L{?cqu9rC2gq; z4Kh@UlzC__%1|4b?I>hO=9wr>+f+hiY#>8R$}CcfqR2cCkukF@T88($?frb8@9`e* zalC*2{`swco;vmuYkk&zU)On^*Lj^+gOHZE$;Z{Q-GA@@39jS9W6BMsvtE@H^|b>k zmEm=3rn6WFYZ8O#M%)gYc==@wz2_Yzr(4iT3h~QjQg0k!d~>-jc~*Wig(X5walUrN z>Wym;%m3x%l%>P!u{nm3S3 zj(FwT{UA|Gqd^17D-Gr3Na088^5ke)=e&5v&;#0q9e2F!T&%?xavWj2Hdb6>P_JVQq#E6MtHnzFrIx7xH@Yc zg>upiZ)qC-C%Q{Vh(g&BChCz)3l&XTKs{ey;5s>GuiPIaVZjkaQM#5>&qxu9u_>Ir zWZ9JQG}`X7FiKoyS|jY1E+WUjL@ZM1b-q6~&&at4_iY`_?%vb3tH>SC9~+q!Yqm=u zJOPBHO)pD%H+%goq>tF^xZ@EWTE3Fa;UbOGA%Uhy=}E)KZ6wZKme~25!|@g;r1BC) zz4OHZ2Fjfv5)25OBv^HGvdG4tib(1D$9&8?&OG!~WGaffJ}Ma3xt)P>OFjPZjj?%S z=11nfmD;K2#kA|67Mha~1DmO6y8E|;V+TPQX5s5!6{YA6Ubw-)3Guwydy#Hbznaps zJmCIsWUHI*Dr#YZ;3IY(!Uq(OcEg0n*Wb$n1aWmltU&7psUYudB$cGUkgDfaF)0TK&Wm&jto- zLXPkE$7AUU!bNQa%HL2yn=b9`ashq>-10k$~72=V2J)KN=3R zNDK^fG|2T}IU-&&_ z{k`P&G>)VmN`&)bl;BEL+2Pk?o?iJ3WuG#48kkMIR|*p~kddocMWHk~cHQ98CAx+A zGSj+L4@Fn-glyVxm#N#5djhm|LSKILm!H*0Ywieoz)!6f=V_H~e*1+#O}D#`+EM|I zeTcXz#(Tkav$*Lk$DfTM8v#eZrkE?yNuNvvF`ff?6j zq;U2YW{S^+|5>g{T31MtwRya z`1vTTsOU;r=`>g?{G}TkClAjkAY^_cA@c)Pt!^j`o~%(*JZ5kLV9I`Gf#kiRAq|s} z(+|Zz*c6N(CztL+YgPP_VT`q>Lo`*b3Z^ZBIw@enKyft#t0bE4Je=8((admE^YQUD z&Z2^-AM=S{>U+yG1dGGiZS^K|q^T;@)6GMo$#jcxt7Yxshb*K%> zH6^O?WwEHQf>G(wcs?DkxhOP2_72MYUaAB|V(m$+b5*@Y2Yl59e`#Htq|qcgj?AGg zd7RxR;R<1Z7eP<DVS81 zyfgv4KMbdpQ?xn2=3$hQwlpG;^XFus^xT0nP5vm?uW3bq#A*oyk|qptfe^ZxK{cX5HzWaRoTUEt*hy4D8yH;NxNoyF{ z<2~r2UNE@#htGGxT}9mzCEZKj1&hQ}jA1bjNV-Jpg}w!z<&xvmA^X)7q04ri2^1Jts)h^Dp9(poRPveiViS!dYzU*Ps7;mc*6b zs4IB~X}+wYWurPw>bJZ~+rUXUmGU?@LX2d=E;r_5X+&ayHgg6d(}>$yu(uQD7TB4- zv|X3XedHTGRo>7rK89K*3ot&H?G71Yyn9S2OXG!p?&@95`rvv=T9Z8OQ`q6K=kS9U zoKz{;Xtmi7?qiO#1r<h@Vsv356%*CjwV0tGD@+S{G%0%nerY$$w zy(yvhwqQ&;iimQ9AazF{G#Wn!N_}5-kVU-=SN^{*rFCyRzyJ5I6ix=@=KsFb8UJ7Z zZ>9fDY*3DY{{COD|ND6H3^GVT9VOj{%$e zJ~g*yT)ayhD(mBqtR$Pye?0wgm#c;Bj>b{TLWt7?{YwnwrXgL|IV54%a}o)-X7_;u zb-iDsqdHzK-``T;`l`*7*5-tU&V#Y`2<=wGLAZ<_HXBQ(A}t zIB=1f)-CnOPk#b*^;scBe7NKZ!@e>S(_1X5Z~rf5kR+L87`8jF z37arcjWWU-NjHe~8T-b4ZwV)fV*XXv`=q0+v+x635J?xCz@tp`yjoX;o7RF+qD_QdGJvBMo=M&G7O?nPaK&o^Gyirc zxd9Wa{yF$-4>4qewQ7Wgp8{8B&_w{|EVo~mqvTvO-hkO%B8%_|h98>62>&rOg>TRC z+J{XCF84H%uMHBP~w>Pzmdv)59&mz(`WVbkBXYyoQQc#SVd{q?Nxw6QrVIYo<^__Nx00J6i?&~Oj|SN-MrSA-7+0M4ZY zhF07cpB&Ulo%mi&Dl32w4nq+=fknbb`DP%~tWivy_{q`;bXz|^U$S0pX&pV&HQ%@B z2;0$#rSzD)FnZw4_7utwvUx+<`jHG+<|3^(#}jd}eznL-kN<8XEd zuYLd0UZ-5Yf2i_c2Bt1_4Y(Ha!PPkd3Fr|rz@#)#G;SOORGyNaot@o`BPpFWbHqti zt);l`p}J>TZgFH<)O6+FI2R7aTwClryyYCRS+z>wanFe{#iC{= z&Q|LucZ8Lf<>E3JR1*(tpAzh!ATMzZ)WWA5zJV{ijLYR2Sz24;4430$Uh~O2ybX7u ze-`5v^YK9Vc@rqGjXF?d$!f3Jy7%*C!OQfL&Bmf-EY0FSn={M5!gvwR_R;c~tQ9k& zq0I32%fRhFIjIlswsjHAu+w`6Q&Ze$r(!0qocPlduNpkPUb6}{wwydd5fUCCW`=br zJ5e>R_~)(!GyfV3ys2Eh(kP|6jg(p;Mn%P+S29^g^@I^RuxiDD?luL~b%gLNr{ zQ0khl1dfzaQ{0SqAf&qVM#us=pyn1}$eITWqJ!JqclF(P@}z(+o$|5)cvjEcFLK-_ z7I{^s;L#1p7*_}|w2`?}FZ8!e3RUYE#FcHBcg9#XzZULwDEvLR%YF7o_%I%JhK%Fj z`Ky*(>SNg<$>+)lu`sPoK6@@G9P%nVhGOsJlHfw8=te7!*7_KibR+`Qh9-G>JqwSE zx?K&lo@dW&zEbel4V^>!{@`F+t@)UAfp7XFs0g^ak=O5^B{ac|X&9WL8>43T4(hdlIhW~tWPVR!cb*#>S@h?u;A|$1O2oO!m#u>^l z72kZ&7;$JRtSBkl{(U9jxd)-S!6~;&l_wt`!`=NTuR4pUeu

llmkDAVNmF4L(2U zHf3JUj=w=i>{hntrg9vqGi97dtf&POci^JG`-~+>1*dn+g6DBu+A`p+TOKIgyOE@( z+J=?Ohl_cC{BAAl%&#}tafSv-x?Oi}3eAaRJkUW-p9I1iVd+@?W&y;2Jzc4XbQ&!T z6NkAghcYgJgR&{Qo%J*C80$&1<3mfd!H^F9f!Y(P`Kg*VCo7cm^S_6vxla0~&fI^X zY#_~}U0;mg;wwp7V3~S0xdLatpKuZ~SG4U^Lfd!sirMXGbmetxC3udI@`47*1MjQf z#VCb-A{J-AGsowc2G-QYp=ryin%3T&XzGs?E2C1gR1&n{4VyU6J)bJb8pqYKLEJd= z>Pk|=;nyZo?OO#2N0J?sZ1CJ5Kg-gLc(~G0o(%geXDf5q1Z&5;LmQ}`g(wPbn(`e6 zzTZ3Hdz0iK#n*TIOUd~QEu0W&;jUl@>|54f0sckXuCp zq8kv(5g=r>BwV*H+M12~_`kRV(QyApXXOZbcc+EmKixJ5lhwvEduOok8#n%3-a&@kMMbn9qLE$XIIn9M$vmS z>qAX1Cg@B;nYPj6<+fMdT>G5OqXx z{RX#k#4OrTE7HPJ!53{U-c2}~{lFh(V~#JUjjU5T6&#TR%;n5n;arQtXY6$CW$>F8?W%@4uRx z!#RKJ;%udNtEEcVZtiAuc_aJg77VF|Z<0#8fKa+(yfIOHnCV8Cbj*BBWl^0KG>gx8JGiPIbl=u9Q<{~{E<>CEej*~ zYg!~h#bD!IC#MK37|$#3c73V-jTr67bF}Qo(Xpp4RW35s)2uH@p?>|3-RIZ-nwL?i zeP*r$WU1e9Tmf{tMf36D7Bq`)NwZ(h|88;O2l@!?2W(m!c}l@&KGAC@G4#)8ryttp zKePTL2%kkvZBBpSXVkb@_+a0?KLC$w$7gSWXV+K6&z_ zs^s?i_m$S5EvIniZW3XRuX6FMM%8oRy?9dcOSnz-SHWbMgYEDTM*ktKdi?d(vPu?z z=bw#YuP=Rxl&Jg?A-4TbkUwJ%yskYVa;GR&;-R7j%}iOLjcd6(^Ly^8RHycxzm|f& zh(GC&-4pMvjExlye$g2j`w^Enrgs+IfE&jHw+z@QZyKZ4xJ?JqqZCT>CqD0bPvw!c z(T}tMLq@g9AC$H>^E@sA&C105pki1N9o$8hp)*^R_yN@5XzB7QdD<(gnkwN zHn3wqBn+#8kv_jMq(73p_?c6Z>ZZ>u6T->aAeP#5!*@qJBgOj4Fsm{p0)2-(#-O{R zc|vm0KtiQfB-?9wUHH1012XVKYbf2?R08OH|r`l5s*YrL0?&>G>&gZ@ggIDK(ZH~>S8mqZ535M_1xmU$V zVcKt5m;B?IBYJ`Xp2GeD^9cvM!BuWj07xH^aH z)1p|-I~;1x<#YQtg{fe_dCTAAjc)MrjDgmiukLR~Xa6bMKaD3>qU}F-?F^S}o zuEfVXu9YdXjz^;79_2qhbRGwsYd=FL1TJ?U?&x)=t!&vT5Y9vpWLF=|7h?WmryIU; z`ys@j^vO(@OL0D`?X$XS%#Tye&vZhTqFzR3JPc64B3yK3BNo#w42ljwDH?*FE!155&EH|Bl3!)x$QGcixaU?lhqH z9<#Kxn{?5LqI!DA^O&%=8R;nsSnhh-Suo`1RxRHf z<(u>}?95O$s-D~|EAY74@K)({q5r`IY^Sn*mFC;MLODi1>aVV27ud9#TfC$Vb*pCu zqS5M@WI-t6Mf4ah?9GY2*jur`@--TYxjoyMk{sN0p)$Ks9^z@`O zOQ+e*4)09s5oFZ}=Hn0nDARIcovrU9Js$O>a^#0rRe|*$>L@CT_L%r&5ko7z9rhGI znc3RRLoW{<`j2vk+h|`Zn$HS2GByNotR_#rC+79@Zt@48)xv9^@`el+r&7XvZPCgW zjnF_kI_<}+I5+0!2^kY9FB9Dj-;_Qu4%VTzoqrRypXHrRN1=S%Btq_e`XWSmw%IPG zlE#0q|7xU7KGg9k!eRk#1?uM&6G-~GdWO4kE9>;*Tq+{OoIY}!J;6`=>KCh_?tZb) z-*~j$xbt90=`lGnAZi<&Zmbzc|rXrWxe?PP_HimD6RR@L&7PCw5N zp-XQWi^VN$;XmU(uS|?~iflQ7*#L|3E5r>y=fd$z^jmK4`Rpsi4gxjPvZpr;uPhwA zo^;PAG0q`1>)dTUFM87OS2oQ{RdN>di=bFCh0i@}M4k`3Bt4~5$a5#PU|Vxx$OazY zCAII}apZ3rzkpDModdn&DavIDtdcMFglH7!T@my`dgt5*DS zso1T=70|wZws3Z`uZ76NfC7_usSFF}gx+WLF=iepv)H4bvrpg2K$DeI|L%#@>gn7r z>=$9}sNDT<+jiZ0!jiUWk2aAsSw6HYAF9%Rn~)!QniG0Y%Mbj0*jC|ga_WonRw>bp z=KYJ16jtt|Y_3iJ>-}y|Ra>v_Q|CV=6Ap+-VYI&}-4;BgoS?CRfv1BoQmk`#eLv`y zvLvLGdcNNApn&tI0I&4*Qxe79%Eewxx%l2Tc%9bV;h|>7V~+`MtJ;B?C(CcwyDRVx z##4Xw-M<@J5!ba5ejLk|mf%2&Ew;Sd%Y4#*cABh7AT7P z(%-mUJ+Y$FUL{>JuO!NR?WnlWObUO|bjnE) zn*^WPkhGhY+lQ%>*CMN)y-X>Z?cGcu@Ar|m*mErUuEi6Bv%(v#K7Y!on|)(7nCpc{ zl+V$od^7@--q6-mvEq=T(qo3hn?s)du$N(Tufo2EUgeHbk#6cBrI5w%-j;*_9_mFt zMX!P%iJdYU&u1%+PDcc#ipMjqGdiV24`i=cDXg@&Ikxt(ykO{)CoHS46pZ`VK`l0y z!@)o{m5}avR8X98m67IoyTjJyXn6N0%gn?qUbYlPz}q6A{-bW3V@u)I>|T=1-}3%L z*emg#^*PR<%4AjRN5gRl$WOJiVF}yO5b=xYP&$`wdZLwT81lSVdCgh+;4dow5GgR+ z_;>k4w~2RuIAzGg<(kH=@ZDf)_v@zx7&;6Tv(Yq)+%J|?DZRhC-<5}TG zS%sKRJJVpPjSk^R{EcYWm;nH>X)CI~HIna1JQLNcJlO%nBs!M**V;h2i86g1wl(_` z?$MKV*X{PCRu03o1HvxhLGitMSru!TgWFivC%eR5yj=a{HvR7!RoOk-{$ig}x&_V) zY|l7L?z_*U{~X;iFtz$hC*?~t^K?z3fd0gd>b-%v-_uG1HgoaH%I;@L7f?pgXD=Z2 zJmy_t!D|j?0*wUA4_emm9zvFE`O5ND9`ou=EM-5O)>7lj%e%MbbkN6J_CM~N!p5an zQ_>>gF=zyfxO#)%X_4-G)SbM|bSke*to}`>35!x+qVXy3fCI5Q399L*3ziMDc$|=# z8+&iy{w(Wgi&7oOVD0nn>0J*#jW)eXRH^>9mL>PpkY+JVRTo+W(i#0)C0lKeR6Joa zO53_0M7%*{ks4TZ!jHCIP3U4n6>o7{!4N+2mq7J?nUc?U9(CHFCjJWD<|J#yrH26p z19?r{`e<(W+FT>Y`LazQYm6q#MwIva=4fc^Jsfc74CRw^v3=M3DO@l-Wc!Jant!ye ze6)?pJqTnw|DdIBtsaexk34dL_~hN8Jr6P}1F9Rx0|V0RI&Wwi@^V4(q+0WMmf+_p zc~aJ?sFvM9y_`5JE4X2g9MdbJL4-a1QGe>>r@h~%XFI*_8GSlo2y*rvjTi+kqm8qz zt4>slO?rP==yoTl6X)9;pdhc7X5N#8-|^O+p{r|AcWtT67x75CqqIYFtNP~W%FEkv zOf7+jY-yUATO(%T;+M_YzZ4pKvE!jRJ+1yPGMS-)DUQ);>5e~yp9}9-J~;#a%5OH+Q)9Dst4m=DkOF% z7JDfkTr zGu9x#TUtAi>obVx79Z_L&0|VP=v8s z-OuJiPxo4W)!k>X`@7ymXR&BQKh8_xR#quWsDRs zibWGSFYfg8`J7YsKe2 z0>a2pNLpHYE|LhPryEJl(lwzsbh>~L2D+II&BEpc!HCCU@^5JB6_ zT2a)-pHr0<0Ba;bn*m1J?6?WZz$@~HF4`AD`Gy|jA0#Hvg6pb6TRXh>0HN`dpmMV? zl)PMY@?UA*$I{SbD@pWVSX@y7}uy(2bJE|w}O_!gGGlZ%SL$>f2O~$E-(XI91oJf zzb_l_Djp%(-!1{|rLF<8UQNU36?i7~n`}Jqs03cM796b>OZ0r~F*^~DIh143VOdeJ zs(A9PW`k0>Ais`axW;xkKJYQLQX*3j*4#vF4j;ZqN4frfIJu+J_x0+?m2Y|>ELEk= z;hPV)S*-b;*BfcsIu`>jazsiEM<<&H62MmyqeSI(1eu17sJ*|!cPL$_}*i>Sy(-aYD*pj-2mD$uCuHuNUqsg=DR>63AOY<9&`@?I#0ZW_A& zcrZ4}(kkMTTzO7?9*vw;Wbu}08OI|&rVLLdY_}bbCy<*dpRAk{H2csmfpFgZAb@-87Xbh{(is5$k}gzMBNX7>rgz=Y57`_Ft9d; zex%63+Rf85j?K@|RbbV`&6`5XErHqHSIEx-s-WYDu!(svH`Z)P&E+?>;p1x3Ss`F5 zloa#LOXKB*4^?P9Z2BxJz$PR#Nc-}$>US-Dd$#r^&ZG5o=;^M6GjhoN;V_48g20y{ z^!^{(%#z05NKV<`%gU5jvFsF{<$tX}RHP@lOzR855!Z}aBHQ;=VP@xD>? zY*20wMn-$)S_o=a!+kAPKew7`c$;5Ra%1Q600jo6H%*yF!egUvkng#_X1A{&xWQ$3 za{m?GMLy3djn;rGYzP$Va=Q@v|`iIF3LJWsOZ~cg^$FZBtj7xq| zNgdqwP3W7_=(%l2(%T4j2$oguk8S1EBHtKgZPZ8Q1)^@3`hj5QXyUZ8bYm=vDNkVQ z`h;Vw1d?x7li5qATLLTzevtHc2jSb+ShLAbMiHvzrT}`{ctUxoun7?%1nIpH8zp&% zC}6DMKKj`~SC=X}@|w)D@?KHZY&t_i}#3i8KA zgik6_ONk{u7Rxxm^Ti%L$u^fM1aD@=#ZQ8($Z^^M*WkH#IEa z{$`TN)}hJF)6F;0W*Vv21Myb#^KOXAS4-YsGS+?&Y}P=j#wCmLWDgaBG;ALQtC8=S zj~;hDR1_&VnEInci2jr8{uPXIdAEm-NNNq;#S80mo8wHBYw9MRN7ErvXubLy=eAkL zEc!v(tR{U56+b2k%k^83zxO!nOJWpLWw;f;WVe98hfG$$qgUvQt?oc%Ki;sy++bjT zofa8to|^*+Z-u6-7GhJwaZ_i$J5PQ|d(7Bo7xiz%x0b2bjy#+bu3h?Mfe&;)k6vs8mXy8IPuX1Oxqa7pRbHm6KL3t9iJne?p~XvY~$JhxDQ_a zULtXA?C-z%6x;E974sqS7{=#Noss+A;h6#!P*g{-y<`a+jh9E+{3}=c5bRq{iaz<8 zx}k!7>H;e?TZDEOFM^%>`XX;oBp*y+H`{^=wj~-oz2)LQw7ri^?qKo#^*2CxjEner zK8*j?m=yj)oW@$WU76%jn}1wZ6mtU2AO8jyPyuQC)K4vfW9oQ7t5hB{TH2N9WOPKC ze_X=BXPQbRd29+Z##++&@WK55k<#s`!V1Pgu%M_2%@^W?h8IP@FWB&xA1 zK5PGhDf*WvpLB7>QUOo!$Afw>OHkQ3gLG{T>G3q~a5J~rqY&z((UzZ(Orx+rStC$} zMpxc2`8xQo{-ZU3+F|>k>x-DG!Hd*5^F!%q8bryvXYpX*T5F@l0u^5ek-r%POYbLx zWkl)?B0F4gQu`4Y=4QyiM*YoPFlv+rW`0&)1wUFT1`tWAY+9Mlf1b{ zG%2BsV<4lNLWR$f^{XS}7$VZag)peQ0r=YvDME!v<#ICjOE!nT0QXP!_UZJIAI$qd z-#OoIxHSycreM!&v1-{=WZw_ND*+ qe{zNsTGs!*{(nYK)8CCQGTgbn@-R_x}Ne{AZQ` literal 122844 zcmdSBcT`hb+dUdYv7+aQic)QWsDL89iYSN>P*GYaDxDBQ3xQCTqv%0`g3_xrkrIIr zAP^7>5s?lF5CSNj5F#b?guCMVp6@$%jC;rLpZm|v7_zg?CfQkQJ!epN4`0;(xA?NBX)dNE>upgLm2Oecjp*0+Bz(`^OiU zryBs_gFuX~>fH%Uo2BpYx9z2Tm={MUVs0xazpc7?<_le7ZrOV!N;t4uXXKN_-2MYM zqVd=F_RvX2+D6@hIhTe9WYVXTn1iXcE9x>|Jt}Q>fKUDB zfp>2#vHRzQckJ*#{qNI1AlLtWdK~iN-=~)$$HBe+&m#hIdE5V-fG2zr+y8yK4YK>+ zr`sV1|9!d(^8fh_fvtHM6DaX5TSPT0=>r8Vx)KmUwNOFK^% zN2i6X(9sBAj!lV0MP^#y&u0e@9y}?k>gCBi$%pv(_u+G62{5mhuU;KWzHt8vTv+Db zojZ5FTDzu8`S|$w_S`aTF14#oK@2e{CGj+`+SVSLX9c0sEM%$EP|<5pZTFr%3u`P( zOUsr-hcMYuaxC)oa8+S(@hjD#K>hu?o8IQEzoMdyV_N9{uh>>dAI=&twynACMPge_57t!JdN=M+3|;rn zEcGG`@{Y?#hZb(-$VYvqc96;&dI?+rT! zp1A#mY~WA^|6&_hStVUHG&HucNvV0M<=J;Z$iJ83DJQEIhzu(s*uYy8;D}@bw|k^s zpjr65)jco#Ip{96wXt~Uwg@nYxLi23bYg;_+b!BB>=07s{)J*;Be;z_Qr;qeiEmsE zM!*N$+LkaHWbEF48k$Yo<+hM25Ral|BRVcmf#LLQGvBdeht3vfIBWx2S~DTCTTt+I zYpSX-+=k$pEavn1&#qcDa*eYx%p`vfnoM`K$%S9Wtjy7Ur}Gk17(%3`C7Fk}kt%5V z*i1#c&LwIxrnTQ9&Ip8_NTdWWhB~CD1SwU+VK52I)QRqEuA%SVy~}<7{wAf=?p(v> zLbMk6F8NZg2*)!oIvPa3zSUA-7v=G@5r!m5-MV#4ZW;E_r1m~%c#A7zC>e^^UPT(u zb{s*AYxuu!D4j4BNf z9y(+r!|0GhWP3H)ryMVoD{kPf!-b}nl}QHOxql)ahK{{7BzUm&F($KpWuNf7(EBu2 zSi=tXVQbDCaOM}+nV!O*$ZdH;A>cZv?@1T)CwI_~M2QDjRkUr*Rb)C3S31o;sCU zS^O*Jg#MG&kh_}pMKO}P_F#o0J=*m%O7-N4Qz!l8-r);mNp1V^7;{_>EJt{kWpZ+| zw0qUtw{O1$V|-0iK!)CR=yy8IYVDMNP-z@)xNZa6LQ9hKd%Zsy$zyLqBhCCKBQ(^(W>^mmvNn7>5jMqx|=VyLzIr~hH&*^ z?zDrIGdc9;9Hu+dQBGF2ur~}Af8SHfF48S7!d`#Ir{iYq9Rvb4qg-yUyP>}=LgCBj z&-j2p!AeUJ|DD6#;=8)LNhw}n78hDoht*J{@9gUAeZCx?2wzP!ikI%sgfociHiSx- zbB&?SjG8P`htE^-S`&ekuG`|=AG`=A2j~yLLWR%RY*@idI0nH*YYbe2NS#2Zy*$2U zRYDtcJqyvI(6D)%fS^e;p>WnB_;|U8zbgvfD5geF%cJK^H=-y9x+|5!98j zzl2OL*q!C~9v-lWjbCq?Bg71sOrDMY^3%sg@D6c*bPhYR_A<$|O5q`txv&LtQha}f ztLl2~WmqlOZ5sZE_@i&iKeFl<%S{$Q=ZvpNU4-+|rIxH#4QaQ8OxKr|L(nZI#CbV6-(=)NqrOt``*$6)Vv4YI7cZ+s{ zXT`sgu2%IqfIfs!=Wq9UtgYTUGEv59U|3=1<#f(G>IawgY-lirb5a} zRcW8*1*#aXrJ1S*K0al23*XMUkTkqf-MjOfAMTWJBNWMh8m#eyd0LC^DGg=!TDsW& z00mnJ(h>UWi?D;g=;sYK)gjNkz|>1Je2arlpVo$M^t)#E!tz0E+~k$v3f*aq0Z=l2 zSK((uGT%Nv-kS%j@L`Q4YG`nmE@O~ z+O@hjOS$!?IPyd=i{y&6D{Qh;f-@|fI-}rWl9j~GXq?#lve%Wjq zX^bSLB$@_;@0se{8)}~jtBMqOG})Ru*{kIbv-fS5a*+Vo#HGJBfa!s)1zEkvGd1Cj z-miiT%#EfPQ5kW~0HtRaL59D;SZ~+o|R3s#}tNuHB#=` zuhc>k?cC#J)>Sw@6PtKl*tS!r@2>o(Pxy>=QcFN+5j}V zJK5)_71plF((?1dcM|(5!vbb?NIIBJxZOK;nM5}xqbu|V13NMM@G4A+5#T3Qewh&u z;(gH5ke*qo!-G7ep(ikc=gXavAvc?_;^ztr3tck1woulk)j#*FF{_3}(xNhqCBip- z{KRI|KX-zNli^{xelzxJ?{KG|zT=cnDZ@PWE-a!te2eQ#M^TfB`IYYCdF(<7UdElU zeb-XN9!*pCTXxS#N%94GG}Td$nmT7m!mt&|GvAXhX@|A@Y@whKZ+e5a6rE`-Uhqj! zX`-%}DlXV7v@rwL&+DR6;ycri4@b)x>KKP<_P6PRO!@5WY?on6qC8uz2az}V8H5V4 zZDmQ7y&3rRL{#AG=bc@XAksG|umi}x_xCd=1wjcvSqviYvglHqCd-QLOIZ#Cg@=(I z27<%1n=uN|5NdtXVQW26Sy{Q48|31WzvelBcc*|zD`3qq>gu%MC{!hTgVj)#;rmUN z($R5qV=_tt;}+TS-ixlmI1wfGAYXSP$+5UyXQP-sp6EcZ2jGeF;>90}VS$0w?8n0P zZ_zq;@7|@rylTE|a?8jP@QKBr(F!)a0>2mvOE%S6bAsQ$pIxaA;I9>{ZqXDDfR(bm zd>aO=LzqL*_+iGWSgi-*DxOIkRgmC3IqM9ZcH1O)lhO*inPAVp_u03`Z2{F%d#ByT=#ZkvBwyH8-k?E&SoM@b_5|S~+XM}UtD1Ci>8ZGzAtcuNaIrwg! z+0~<$T%K1J7Z*2%j)1p(Z*c9JH5n@o&)6E?;$*Tjw>As9FNTgh+-U>W{y;I~Vd*h7nLAwP(VCWXLZ4{tU`@_eQDaF1n@81d~9+uRdJ>ctF!LLFgMe7CTvtv?cq>U zQ^FQ@ZYR5wW6V%WU(Sl$U#`Pl9nJqvwe^cpR#fb&!?om)AA8B5{b^|^inWW4c7T66 zoRi;z+rXEcCpK(?IwdHdN+AzXR&J~?b8}BKK*xg&UHuUS4GkZSQPn7k;w;nJi4VT) zsT!t^6KCEP&62Yu;6&D7*amYV7)wKR=R+w?U8v2mcr%T3za`Kh-KWH7#CwLV{@BUj zKcIDwk~4}Yo+l!w)xW;ZsN$i`U~y`hqag#-Zm*e3uhY{JI%r$m+)V7TOD;Wys#@Ad z&RcbisB{yrS4G#d;NrwE4hC-Swzf4N8?M&NBw*$M>|L?SbQ4!{@5;qYasZO*X9#M` zFRgZI+z4;?vBzM&D;>%E&5>?)qiQ8g4abdT=ENq`ovyPo=qD&&%Y9;RJarM%ER~q# zXwmX^Qi~SyC^E9Oer&oLg%0gaJ0frECax9uKC~ZX=`cTAyMTV@{6U-g`B8E(XG2O# zin3R}`h)FOTF^xD#DF37b%r_0o)BaUz|3;!@JrQ=$^r&4zsJYMi)7!yF^p|}($-?0 zZ*mcpG^!k>in<@lb}=gqBm0bD>tg2C@zdf?i&2mxvF z0;(vQaIbtQx94&O{N@$pHm{+&{*<`5+cMX4V%HWYQ$NHB3?Lqcn5q74*P48ZoRW~> z^5{>Iuc$n|5c1k|s|kAP$+8bw-1U(#Xt^BI+bf@o^blHI2djN3-@e@dQ*!wZ5;L=@ zR84ShIa1KB{66cU92vzd+>91hP*Dv<93VT_vnjB2o$e@AJ_k`jQx_+vp55{cLg(ez z0;2w+)n1l@ZwQ~uiN+8sfIT?^l?esLiSnqpgG<@8n=9AQLtv2!2thnu+S2;(W$D=g z*gCP8>P0MUbFD?QR2q?JJyy`-M1R$5xvx2~AdhspYkFL8q?X!Iwl*hqCtMm3;xvi% z8fk&0+5YZz&^%ggtuquD#nu%jiczEfB1xs~Agg(8(`VIhc%`>1bxa-2?Y#WBc6)ND zoiJ99ANKK4Ed;=skb4JaRh0QYB)lDww|d(Wu-xyengq#uW<-d1tJ;K#66E&G?}+I5Z{22f$?D#|L;JuC!Pg{16Lcmk?%Fdo zH>v6HE0HA`-Jv)6KVs5mf*LQ%eqZibEK6wa_9d#6!Bf&RHvQz{7^ zTLtAQZCk3%kTRT-^kICrOxisTShlTeMlG67U;ilh{0BMr56ksrN}^5d^rkH@dzelh zfd($2wnb3b1ZQ2)q0jV0Va`|6!ZN1RojyPPQ#xYKtCq9SqF_21PO|VDuilzbjv8eV z=Y}!+J)UXIQ60kBGj+vEfnKx&q{6+7B2{+UJ(voaaWr&Fz1ydsL0hq%#&mJ4g)p{N zdD53W6f^9Rwf2muAmi9SwPU9*PE@C{nLAs3uxx3*%kGa#nV3d%JJ9@&DE#K8E`N}L zxOMbZR=ymKxrcb>S$kje-UmoQY6RxgLjA_vrA&AP{m46 z0)4i@W!{5ak?LQsAUADoJ#qY%>{?9?Bsr^0z6uTr1?{!z!v|=kz@OD~qBGi`Xxk%r z6p2X=>2Qzm)o3i1&~C<$Dn&&}hfTYR;Fqp4qVuI8)fCBR&z?!dX zu>$MLQ%qT-Pj zN0BQ{g*%lK`+Yqijo{D_hN3i3E@E4L78XQ2oSt*Unu|t#wDF))!`6F2LW1jn~k)k z2BoY@Y=z&MVz5w?2YU7mn)^&4<6NcNdUJ@>LC%6SIN_WcN$^%8gd4NH$X zSdmwjuac2epEn3zgXveSOeJ=(&fDPMHnQN_Su~PtzFDRhZc!`dODWA^Mb2!gV!S0W z)u&Zk&fMGk{XS8Z@9yVUU!SwofR3tR(IzI(X>0q$HF`F&F%sde7YWrAoJ|wJR8>Qx zrsPRNvXXmTci+_dpo&CbC=E(}gw{GRdrrJ|P^Af_r}t1cR+DI}kzdow<6QId@+2lV znq@MR)~{2ILdg*dQ%O_0oTYemR!!G@H{$$}7M)mrWYKV&&W2~fN@n2C-4!MN#fx~n z(qaasWHKF{0hr|e!i-+OO)!@B1v;IJVfq_OQ=K`aflP6RCf$`+RsHsRtJ14|MrU!L z&tn9H;p*4l@kmSgx?flPLPwubim?8ntKS4w2k$k#uHuvy_;u>-6d%KX&*1UT6JkL; zDj77}b@mgk89@dq`e6CSs7D0{OCX^(G6ORejk)Ag@uK*Z}}`a>+;;ja?IIy#|L_$ zO9In(E$y7cNeG%s5Tm3jS6R)=lD;hR9-Ygm0a#hb!9wt_>b<^2qp`a7bD(|~tcVF6 zvbhlyRKr<|H`AH+A(lCkDXE7#kCrknM`~ou0=Cl>hYAm1gN2_o-T`?oaEGC(BNn+$ z-Kv93deVZe*Vfe9QM_2>0u-SZ-RTlvl5u!tHL~J_*fxct?9#Dd@(e!*D=s zbhO|49N?&a2GnYUvMeE#uXyPaRPmPi^t#4UhdvWG95g8&+wHL}g1Kpa=Z@2(%?vm3 zT=MNuz5B>ESgT$nF=Jbz$Sd@|M_&Eqc}5N6B5Ad=l#BD=z8v`Hg#x>ZAPJCE-31(20W!+bm9 zZ7BlrLnPZ~u0LA)-PCbv9E<$X_~Mk)Sr63(xth}AQ_w9ys-#RxhWvUMq}238P*jAz zU=F$mZ+v~$|J?37>CQU+$*WXQpH_>uYLs@95#vKmmeY!w!7)Xk?R>~_?7j%2z zw(S*`f5iiqr+uo8kGyBAl;Ol$Frd3lx))EFLYJ`c1@Qn%Q`hRHrTZVr3id*IY`FX3 z5|V@?>ZUYp@qZjG^Kj zkQs2s*g7WI`*mg}awb2)bX%F!nZCF@TcJN~-Q%1?l_HRg`7zMXUpj!CF3TaQD^-$T zQ{IgZh*13^RjIdQz~nE0wDErVd-fvd*QW!}*b`vJ5xf7JFGNDpFWi6R{%4^o8qFMV z_L_h*&M|+|Dk}?Q=4_UOTT)rvWg60yepB^o3$t`~09sZO?ZG@~7r#Tq(SaCB$$E8) zY142*M64$(S)jyI`}pd~k&Yhbj6%!1;aR8<yiL)1hsk)zYF&n2sG7FASQ$n)z z<=k3O)Cjq}fPfDA?3)V=p1!T^mdWwaSo6%p1q0p;)=&a^Y&=ZM($B8HPbUqHGu z?%25!O_grG<&!}_yvA&^hjq-ur^#<^;uaf`7rwb97NMIOhPZ zOycrLxq_13CxW~eaXMdv{#`XUdpkc8Zy`>l;Fpw9#_5ig!WWs3jfAxDG8Z0g0s30* zBoC}W_^$Dv60BT?L$TV2l=4Jxk;|MacieGAMzErH>^-7JT2`7XIVbpL;L%S^w|&c8 zJMHUm!d-q1(B$Mb%dt`ILt+mF1zp^YHJG6vDYutERH@QO0Mqk2-}O<#u16|6nL2}% zILlfN;#*m#7~=};9{hz@)AM5!mD1jB-q&HW9xtqGyU~I}b9)i}xQEvTR|jlv!WAP3 zUVu}Gg1)~YnGts+rO3SzP<@j{6Ylb^UtmCsBxvV$!N@NoOkvop`NOwC| zv@Gyo?;ghLkAWFWvJG`UEA`St{i;lXWPAZ%UbATFZb-^%Uh}SjxVu7Ho}?pgtV1bj z4|1G)hg_8IXG_IDPKkFc@}|u*j8mpYkD5?F)HB{|VwIY&*Je}M#VEY=lk+e_apyA` z=y;%92hKq^({``Y6A68+-miQ@9oo)CaAM<=;>H_nB(kk+J(OPHNrT3h;>l4>%V&X9 z&@r%is+T7~y2&U7P96P9x5N%t?nr~B$W>GN<~>3GII=WA@$JoA>n_*8o+}ZV(Wz12 zC!+M#Ke$MhWJiq@AmO9^1=d;~h3~ zbxf?t$XtQ66i3>*jh;cYL+Fozfx#dCkHoJt7U*{ywpV2*Y-LQYRYjtFboK3!H*?!C zmA+NQc1F#+l+>+`9w{QjY0yeexShL^&bOPgTCMq2suRpTEpd4=S6? z$%)nRcoa2ovZw4ZDPV62U+#}jQ7wS$OSQE?ye_4$!Hwtn_p5!FzJz#Iq%Z6S=~w>H zKZr8-@+#%Y8PdPr7#$qQD<^#Re0Jirt$S91x($8i*Ew7RmpJ|B7@)!e)=h+Tl&&Z(JmR*GOxd~-^=0=fXRlU`{c^!r?djZj6NPgs z6|U_nMmk{7{+os)4oJz!&@$oCc}?D)*$g#FCJ~0z3Wht>S_UBK^&x8u2C<{>RGn#8 zZhp&5=q(d|`2D*{W^8{6A5t*H42TAUm(fb7nL9Pyb;g%WH3`wfhYv3R)=q_1)(J*Y z?$>ZIOS?ih9Z-uXVkPCcpI=OPX~`>PZw??;DAVuk$%)i^&kET2EqzJz@5C9>umd6@ zN4LO>Je}5M*IwuwHzVx0WxR};SI>%e<*J3qSnph&a^KredTKVHb4?3f?(eb{gDt~5 zZmisA1JWTi`#RomTOOnPqO6|K;a798fx{{#TcU=woPgq5yV?y(*#yeOr{t7>gU-y` zoX?VKb1OEbO=-iS7keHN+5&pSj7+gD#o8CDrH`#XNQUKL^*^O7iUk8G-2$W~$8Zkd zI#Pzhz!(nf`H$5^Qbf%=i?fy)&Xr|QaVs;yA{{k=Et<3#&^pyQa4I`WBt0a{ zAVy?-$p)r}`Bb*C38>GG=O`v?)7@GXG2j$F=T+=Tm47WNRE<()etsrI9~#pE{DM~mQ}}{1JqH+Pwncdn<7|$Nkp47MdN!}!9L_cPf7`lI;Mzs z5~ud~1qOEksRLkjnruJv<6pZsLa@L<^-KA`)8RX-V~ZS|X}yYct12gp`8_v(VGfJb z6i}~}dZ${_to$N()KZl)PnKl$l)NJZE>28TI9d*3OA0rWb>)g2g~^~za!GF&=kF-h z7EsFXEjoqxF^3g+rTpthAHAB8=7r5N_3LC`#4NdOzy+T|QW7yFj`p#iB*Qm5-QT{LxB=<-ybU_=zH?;a z9Io!c6R{49DJ*`&hn%}p@HXQD-@pz6oYavqO@x2_`6Zg!9;-R0R?t2aI2vQopGo?G z85|0rAtV^0)La$l(f3lR_(M!>{iv?@)|DQwzo)9=jA&*;OJXZX=uqbgMUnFRp9K8m zr+|8@bj`TO2iwvuHu-BgSE{`X6EfwkK^PKWO7jMIT@@n<=DKY3Oxwxkq$6aIR1_aX?S8Ho%I> z0%uj1Rf>OBiKKXxL`T9)b|?Fuu}OzlFlyEZNk#ji9uIY}@J;0yeO^Z&xLGgY4O%eY*kR4Lcg;0uu4d9iVXwQpI|_S|dV(MHyCS`u1_rTM zp|kKJ*!$ELk=2j+(s{Xf89(xMeie!31UWD?(Tr=6{a7Q2$QWg;V!bWX z+1c3_$Y)e_d#%?vS(-lM4t40v9KfLlt)D6rw=sWw@#zz!*8}0!gM`YI4s?$_mQpvG zG+3oTJuY{PoR^=ELG7n%$jZo&xx{Rz1IQUDT6?$hc-o*XwReaD1-xkq(rp)0?{a$y z;k2N37=CW0AR`uJ#laK5AtEA<@WFR1P#-TVhFtYH{zZjbt||sYs0Zv|VNT7*8O!?+ zvkQ&*kif~6o|=k@zXjtC15obEBCVuY)M0rNztN2wzC_s)dK&na%%tsfME9xw9Lq*#|}NES~L>m-vSE# zN8g)UwZ$mGzoxi>Epk;o$5!a(_h2j6=&YI1)zpD+74=7AO36jiDsHzQ7hD`eg|ia~ zPUt*7cd|t?&>UasAJFc>Q_$LN{;4w<$`Mv6qo9WJSi&!3W;>O2;F#jo<<|m{GmN`e zBehZn-}nzxyy6gVa&n-LB62k6s`Vv+^FU*_1+h)(ObNp}SkHT(ea-&hYz76@Us>Aq z7r(g92ZP$$Dnu|^Zbh77iKY!w1{PoF@AuiW&pm5`C!$vDBidD}9umxAlrc-c)M5GT z9PYiXA*7z8TL!PpAu<^AI6hNEKNGwE?nu&|0G_{Si_0N#<*1YB=9sHs@D`dWFDv?eZ$=veS|Rl zPChbw)@oRpjOqNLE#_XHvivQ(glRW@v7p(BXlCcvs%*~`g3={;c8qQvji3dmpoL}` zZBv$&LLl)-2mBP-YXx;Z0njbqX8S7`CPf=F$6V9y+x*SANbiBaNO(Ic#&`gI=PM`= zPu~Fcz|^b9$)oS<$CQu^=(x>RO-6C&*fuLNSP6ZsP zF3@CerUwa%sdfC8s#!IY5|?m1YK6;FH4*QCfvbI56$zGdRrl+J1e>V+;vV_xpazs2 ziOw5Z(*e&+TP{5KysM%l^~Pjpfth4M&=zQVOUjGaYq8_in+q-4@}qz*a0=5rdGg~r zkm0GYHZWVA?)5|k9l!&hmy3z_lK}t&d zh5{r(uq}m0?YR$Vvg*2-{(h5f%8WL4Uk5wOx!Z9Rn1d=LtAN^cD?JGG7gMeT7N}M8 z5{$R;rW(yAIg7IA&*SH!f%l{NyZ*{CGd3kb4p#i~_16!H*s>T=)d7YkIjU_7_yts2vQUmd)BnD~W9F>A42=`c z`)C;0=~k5#0c%~7`|@yjcvZL37eI-+D0~8V&zyzdS|#hwcGm+K6|*j`%_XU4^nl-< zQepxP-gStI@Bt1FnC`~xr44*bf}CH9MSDpG&xcs=Ff+;%+*MwWB6sfG8CHv_6lcM1j5SPbT~R^1D=)kPfkA%e{DDFh8vzGaR+ zwHJ7G?qwRpf(WK!Z1+=x zt<4JZuolqQ(k61K$Dq6ydt@^cjB!z7xE%Z()mPIL-Fw$EdGSlEE_aI1<`+7(PT|~& zW%E2lX_HH#+LD+zGS*q;H&{RIozl|MHd}05c(Niu_#_H2)+J#O^FO~( zKz~qxC`Nq&w%(hrn#%b@J&P5icntmCBJfLD;ebz-QJ(~epYD*#{mTiAFf}01Yji7- z)d_$_Z6maQ`{t`DNnOmLa5D0NVbKloUqI0z>M7FGb-YZjV`jZWG*@IK47e>y04!x= zhD6?Wta;q*LS`$oF)pOesT=&)F!8HEM=f2Xb2%F@up)gJ4&j{ffFRq*t~m)C zX|m|poZAOO}--y2=7KvIv zdf{FKy9ifb9*)b(`Y=l-vYefqtXx}DdhZ=mevjkhH>26d8!KISXA9iGKY?@Xz4rYt z7pF>Mfvft9MmRTQ1L^*L=Ig0gJZlul|9&uOK?Aim8Q|I3o&_wECBFq5C8C}3u3SE4 zvO6ji4w(dZQQ<`=_7XyjZ-!UZOm5yLkZlk8&$MH;69Nz3k8GIJze$# zaWYNdh;knemCcy<|8m$eX|%agzI!|%%{~^U9RPBtE(5g62ueE`#CzKk3AsnfQ4;5{ zG!url;66s#s78svtkU&&eCwe<)D*M{OWArBw6!}c4LmEwm|nU;o5hnNRCjh-7P~Tm{!%_Rqg!NP z61Cx>Zs#7FMYE07rLx@K!{U$Ob%v<$U7R5xpxkyHve9K|bdHqFH4qiF!2(6)caK`H zoTL7wPQIGFE|GDtTB(`?DzBQ9vl``YqOI>UYcFj}_cu5Es6HydSmHv%q!eVRBG?h| z(6^Ie(5q!qy-YZ18dH+n2rhK1(duQ-%??zyoIWz`J1}#B93@>jqv1fppwd|$p~V~7 z_Cmxkvy}tT!EOXaY|;vf=@~ObQ&||h{N3W9Qq+WENERrQDQ_(uB|bx4H1sK8q1>SV z4Z~HiDC&sK@ZhN!O=OVAwTX3*%=FV)Gc2VgZL}N$p(@ROLW2M54#+V_dIrYpU%{LjIP#Q2X~e3U(S6W4!`=G)DKS4vujST8;Szc~ zhr_oJ+8eVK=>(DhN%?E`79*rm%E{3tlA%?LVpz!hkzFz+7MVt9Uvg*l}bnR%_uY#}C%>eO-d*S~oKNL zy_X_qHS(^I@ljdrez4nxTQG6jH-Dg-)d)UOdQyNfX>o^_ylYeuH0e#u{n>YrN?(5V z_uq<*`Rm)nZIJFr>C)j|X9~(=5m-yrm_;^{8kC@vklqdz>bQ?9@D)i1PA1hk2D^QH zC7ls?No{tY?+N6xGqvR#8wY_OwA#5i(<*Qi=H-W5C&`Zu7UU5VYbKEif^tOy!X#4ZFTBH4(AR{Xk@b#+P0)&YH55=)hkD8~W^N!u zLA08Zr1~X*?_(ZEy(Z#W)w|n9HEeA>k-aRy<9Vw1LPFjC7`%9I_M6>{!R}8N0|#wU zX{VrlzKQ#Co@HUFnFK}n!9vPLiq?pm0|Uyu_(9pn5Y~jVu4o!KnlWQWjjGM?P|7aP zW=)3;@>ZXi=Tw!y9qsh#(@JV2X-9zFR-=z|3M!*`VgY2}lDP0P&{|&>7oXp~d-qE) z0Lk;g;^fSg1gxeeaK2sADGYNUKa$`PmR2>bP_o|^pr5|?m1+zXVKaGV?qeYLZcmx8 zw-DcJMJ}L}rJ(i=0E|}A>VMvmd@-Js=Xi^;`VH*Gc+JzSrHP31KIHkoQJEb+O`<7@ zb7o-s&85*lX-%WOW&W{hKX(}!8!x^y0TAdlAQ1KA;^M~M=3wJMrZs~15m80QYGGCB zr7$}MAl2uB?IEhGbkJ3b8`_f}QCP^q>Q1pm;+FqZhaT|DPAYNz%%@kstZS}!IQRNw zR@(FS%M@CIeL^CY;U4td#V+Tm-l9Znd*o!9Q}e-!HWUZI^Dy7<7s)-x;OjtbjN^&j zp}qiDbPf7HpWq3%Y+RKt5DZ(x-jWUV{Rf$XqAE$_R2Ji+jh&sGU2T9e*}|CBLd0#~ zxzpIdz#y_)-j3Ka8$~p$RI(EWQAe^me?%Q5CYV#L@?GFMptADdhLpmINJ6SDvwOfxCbB8py+~^*%F&1#{-wLwAUo?a7x6 zfo1PFK-)@_um3z{kQAlcLA=_&8vv$DsOP}bW0IerpVBTYZ+nJ^Sa`y!i)fHBBNdp= zjZIClpTB%*=HZ{VJj}kzR}-b9{`9G(a~8!HCi5frYkWN)#WTz|_I%+nCf#8}VoM>} z8!W82rV{|Ksy1H?Q1&v|1TTPwr`LA z(lv06w=yFvuKjb-wkz)Lh20FW6fDmD_T=LQ{@H`}#~&{38O$XN&@M_Wj_`iZO;FFG zsCVFO!#>^_1fp~ce7z80peP3gF9>wMPQaUHnwbgQwu)h!f$9L{JA>IRM)EG^i`mXQ zfCt_hSW64Ue7~L$G+hBG=i6Vqg+gvi^Ime3`<<(KPkE0&Adg_R|9SeKpIOuYzubIx z9Bg=q_;AbKULHhbMZ+A3i>n3(1drtZ{cc$#uyL@}AaW~GK(M^Qp6Fk=PfYEF@_$E~ zcbyK{N;pD3e!BybJUkbLFd|6m9 zyE|;FwmppgodW-P^Z(OM#=i&w|5?tzf8j&=mo?Zpz0%KlVd8ap`GtbE$wocBgQ6lQ zreBsRsukv>rCq$)##Fy^^2)BrH=%pVZAvH<`R&_2l{>dM@f=W80IgpG{}dSD0V3G< zCwum6c>^{!Hu1%a7Z!3tkgY4)d=VPkAe(!suKyV|Jy@NWT=Afsd#1^Yv&JzTl98dE zJl!#VoGe!~#$D6B;f~4+YCwjhm}o61PRGBX{4FJ5vr``om&u~v(rM4&7dZY1%r*%1 zAY^2CRdYWCwH*kKZqd||-9K*Szi!BR`{KfMW}lwdF*Q>iou!bCjiPAZ@oI;mSLNEf z`7Q@v`6nKE1JEEWGOLacx?RN=cjTC3IH@$SAcPo}9eYDy^}s*D$-55kuBWHhbxyu_ znzW{=T4fUyK{+_!@lSB_u8=!@`ixcmK(&=u(prfp&CV%&tMPt z-nF*tFv-$X1zhPSEJ1kg#qr&7r!?3JhEb9-;4)@6sr;ux4VoK!`(_P|JEXZYQ!a7o zm~1xz7q2;C^6Kzo^@WZO$L|;R?Ao`)`(ponWVON^>bj@def#O-OLH47hEW-ot^;|s zgyzP0u2~qJU~%K!xv?7O?%b?5DmOFi%8j3JSwTS|^SCYr9lL`MOs$nwDAmx+^pn0(Fs0TXH1w&QIlFW0qy4A!eDw>YT2J+juy~V^bvoQ7 zM2`0D?_50b_5^eAyeo(MP^)Z}sxe+#T$oT1zI9nXh5O7u+UQl9pY&JJo#`hWoaJ8L z8g4jK;a9=zwx@*`YpKrUYAhxZQ88j`-J*(`$oNB+cfWqx8$OQ_|Fre;pRWmySglcEZ9Ni)mZ8Z zOM7J3vps_G5VMNS2muFD4VA72#+!>a|wgTa~9nr&O%p=zq#{^slSaowHS! za?I*RqOqngxcf_%dRI~tS|%!3iFJz-aekP6Wewr1&F1{XZHDMpj$VCVwbi4@M`O?a zBpU}_e>*LE@8$zT*>`6X7rConWz|nyY6|N2YcU>MXwXU2nGcRK-nDzrf`@nx;@^9u zFQ>k8xFs!FoGvrDHDAUdMQSd+aKP5wz7nbbopH}+zjs5KNB$sb>qq&>E7*tiY)XC35cLJbiywA4lxnB3Zir>C-N9x=v;=DzXRznE#P2-cF=?h(dH+=U) zaBxPuRYHZjzHc%9Mt@U)S<;57SNRMziW{Rw+v24PhNLZ5G&`0IyxX~kO79ao6VzCU z|DCW7fi$IY@x^dYtLoiPGy4z`X~Xa&vEG{4-TA?BMRV(?d4= z1hJX|455#wnC;3Vx<1{$>IIWGjiR5UQW^pZbwf8WL97>kQm0gm=Dd!77}70^G(WB@ zbg9ft$>pdR zPQ@??M}F=~afts^kgqM38*|oynshaI+Xn*u+HIS=CobOqQ!#w_4wlv) zv)Y1Vmb1%hEd3elfbIVSEqI@e7fsI@)ea-HoCuVdJNSz#za~BomUm`@^7480O)}L) zcyl@E?p#8ALER|_iGY`Baq3?g3pLUbhb&&FZik4^eJ}_X?+a^->`H6Fei*4?{e~~R zcLrNUFT4!4`Bp2$>S%2!n$f1SH242`FUOA4t}$<*K9t7@<+N2^YDoB&;(==s7CSVy zq#>(CG`(`gKB(~3UCSGe*PhIZcR?OL+Yb21gj62&NzjKtqdB)9XxhQSB!18);?uEOP?9qXK1u}zaZgY3K=Yj57 zyo%fKIr%T#u$bPHx=3s0@`goX#-5J)KcL$;kILNpZ)J}lJpNb3*4C^g|FN<7=ACaL zp{LJ5GoTU2$BicnzHbOaGsR?arHbuHG4H3FK7+~!MTu2W5%+w&dL1%LYNKb{V`m1b z+|4QZ;LM_(!kecL%WLo5tZ|Igs+v833*Y-N>fh@kti8;+E{N$YyjlzQ;9pc>SH?C} zuk4!6eBl*N%0JW24Y}4hWU-0fCxW!uqXX%+6i@|3-tqU`L9s>A)IpE}XrQKD6b;xguNYJC{G4K3s+K!*I}T28I6a z>a4!m)KiAfHqzXSCZSX6rT775<$vJ+e}!5$x&bbbe2w)XUd??MzCz*yBF!> zdA!UUwAFu0#^}2CC#}qNMs2zGg*_YUi0+qbnk^_M#<0z!C$ zk!PIv4eWWfZurkS1wZ>_GLcqic&YDB@Gcwpxixb|fp4mYwm+8_$w#0lk7T6K5wVj+ zT%Urar^Y@T+K5|QpMN@xZ$Mk13~oo(y?))Ac-v0&bC6%G=)QG+wVWDL(PIatBjYr_ zX>Ul%%Q&l%Tbj(mu5tV7hq#y@vQ;T_o30Zssee7udnP_kc^LJrUwdW0;fNdDI$NvU z`)`whx+?*%5|f-BJJv462QTYhQG5_T+ikZ-6WV0 ztTp#AQ)uf@nKOWmr_X%dxcC6FU1w%Dq~}k_h(omw|7CGLh}+(3FN7WNBT+Zo%>QJ1%SUebBTuhD5M#t!(=jnhX>Fp8@0j-I0GUi-_1e zYWuc|Jw zY0hX^kogl1D~O_-a?Ne=I}n8bd5knaoTh)lOkXNKjz5F>=YM&}|I;7;_yah&{`dNb zga?&2<2mfb@si#OXXpFaSJXQ#A3pErRc2p3+>e6?2elCSx>lNIJm6qE82ndV@K<&F zP*;OCgEeV*;dfe%b=OHpXA_$zgB@G<3KUzLTNA<+QqFdxixc-4S2}$@-MoYJq?sAn zWPJ5H=f@COK(B)*%@S4)s9VaIcCyG zYxfL?+F`we5(m#Z0=Uq!{#=R0DMwY>Dsn2MU+3k%yVADG43#Y^_3$1LiN5BQ&Pf-W zA7zy1;jeRbeqe$EdYZfBRLa`$y$K>U3oM1a0gsC~v$msW3SVASRIE%5;#jsB1UZin zdxaMi6K}U|jXF*rJuHJ02RmgF4j}gOfdBc^r>`Cq5=#Gn>XSO`OJUT*)T<`mBFhqdAf19&9lF`gMIV{Ki#0hvZOI@)9P1En1|*Tta2w(Th$q7cGq*TE_dKlI;BV=WQiC>B+QEYEN-9S|b0&3VRB8M4e~sT4 zR#szvOoxpFcZE0N$JF;JCG5%*RK~WMle>l0F z@@}5ak5srF&t|X_9_X$fF|uu&VX8{qy(zWE3a9#-d#kZn@0`ent$5?s%gcQA)t4fHLpH9*A$A(85=`yuthe4Rdv^a5ZF7Uy% z5$s1TruT~^$(f?N6XWb?_A2;mWhy^LQ0vZb;yT~T_il&^@As3`B_wtRa{il zrsuBY1}>g}83C>WEF(TU%uRrJd6ig&W9H_(e00br04XcfHY~~ZV8pGfcxM5!#qI8KE7D_rRVXev1@KW@~8k^&+x#mGi%4Yl%#Y*JDJU!1PO*u70!sM5;R!=FGp@nr<-IlSu=KsaK> z$aa`A0Pq2LM>+PM`!1xEZ#;@goKV}3$z-oxS}Anw)w?xTHbKNxX}DNSYT8y{{NRWO z$VRHyOmLV~vAAEWnfGFVT9e>fI(SndY<>T#d$*R$-A?;(*XFvzN9hN?@p5+N^$wb_ z*-y~b%(l`gR1K5nGX_Z|w9^Lcl1p-)g#9?mq9cgenb8NgRG!$Z;$lRr9a z6S6+Cd4@!1XZ{vVMfBUA1d>_Y(z+Hsy~gyK^me1fA#ZburBqtrBGdkunY1$@+Qw)T z#+Z&LEY2iRQoa`-t1UGm=7Q4dOO8545%I zBB?$woQ%OgE5H)IudU919J|SU=FGbr*xe-QZ85~t@soaD7431_ZapTvg9KenS6;=; zHw`Wovy>Dk8S=`zlr_r~YsFugFS?iI>d(wlu?ri{H10?_bY7RIGUGXKq4QdDv*1b9 z7zT&H784FCyUB*rm(6|HpHlBHz=lD=3HfHyZgPsFB>Gz5h@0>)B@$I2 zbE=XatasWw&5;^8>MM2NUH^&8UmiYX{QU{=Ukuf8s3`a6mhr=Px;&ivi6;>1bETG& zTw`CCcQb-qVQjP{RuZE}l)}FcUb=jsS*)L09;GEPo}6K{_1(-{=00~K+nbZ8d@YD1U_62OZGo=`;{n4SXR4--o^Hv>%}`ge;$@nd7DMraa!A?=<%v{v@TB-zIh1mVYZmE zJPIp|!wVgs`gACbr(UpOH>;!niwFA&!q4vB_KC3_?VqzwE__GWv%WA@{GyH&DMu9* z_>U#!o;{q~b2ihPhd;d?LZGM=I`3mq`B}CF3L3LP^edm_bMn$;tjEw-uFizOnxP`)?Tvx9%JZrR=Zv1(iiJhVT%+Ou?vMSkb)p6jFR|l zJ7S4B`|QvS#^S*OD17tW^6v(4*5la`JaF{}8_q1;n}we@sc%N7%OqhV&%oO)sPq=~ zRL9NfV{RI`!K?2#se=PjdWSlV&{eS_0y34#OraU)DQ!t)(lvEEDrIwMb>=0CoS%=2 z^}oG8DtpaRrL?8YwW^a>G%>X#-~5R^aq@1v9xkB`&Ek+#xsJ z1++uZCGTZpO)j0HTf3;~tBS0=e*7>$d#0nCL0(@!{?U`bfn zYfw^ASxo7_*=af}aW;iGr;Wnc{7+22*SnX3`w7aF1?Ucc9Cxw8+S0`6l9QaA+z_H@ zxkLh6iCdxEI_&MbuIV6nk^(j1=us}7zdN%W+qA>eZxclR^B~@@24(RV^E^55`cTJn zoH^qUmIb)@t1pJq3;KAgcf3zaQ$SoKgrHq(7<%LoFPYrOVSoP`s^4#ntXvu}^xF>K zROdl=Bd4nyLrE0;3LjwOMOwphtz?4qZ5Ws;frVmFN+>pL4@%2Ej3_teix)L?(VGqn zC0SXGFDIt04wr*|og3`lp!zOb9TAQtOm(Xx_Br9aM$Zw~1n4XH`yC|LFCRdqpE!B) z68LSE#t;Lf8PAiFlap~z51sB&vg^5D8BRjl_`}8BAMl0zlb)WGbj=Z(jL>5isJIwZ zc^(C7g@@?-r>fdXPD_bO2X~~J)bWw}Xbwc}+~9M7Y5IOq(_888SnTRZWZm`4D?6rG zE-Qb`mp1V6MIBrLJ^*L{PxSY-eLu14{vy~P%qUDC*q>x3^wf6*N>MP5yClCkn()UC76+_xImfxYzzz(ZEy_qu>jx z<>H)L7+T$HlfPuN_w&?e-t1^CJG@;Yo{%){@OiFvzWdn&KX*T2ufb=+UK3*Vg;S$Z zsqOk}kq$YB{wjA{;}M?g<_T*ybv9#mKi;s`z5Vc^Bh_|{%pZ}gY{tjOUtk(7?AeuXAt6i)ow*3l5esxF?#o;TYd%fPcI7-TNQ- z!50@A{bBP114c$g@S}@##|2j=zQX?AP}n>0LZJ zFCc*B+4PJwr>^~p#kGTx=^R53hp{!`I=gGWiAmQGUp+MmI>LQ_B4r z-J3!v{GJ>E<2bSlk<-1!@@i_KhQ`JCg@uN)Li5kaDoAEdfR$rOC1Mpe$wgvQ zj^V5X<*IlX4;z~_Y`No0(I7snNlA_|N0);09o@xycf9ciC5IdAMGsZ>(PKD{DDE}G z4Tew*{L9Hn)1AUk5ZlzLv<_dq-T%Py!lS{I;R2c^#&t#qib@&>igwp1yY2=d^`I3h zDyk{mPzQ^#woZ~{kJ57^eUTf|_e5_-od~+iwwfDyU5OgveD;8^R*7nJ8dB1#R5+lDMl8D!L$zAKaf013we-< zq2YybC+Lu`dwbSCxu>1sW?^diF?o(xNx^T*o;T9MR@#KERLu}t&6iKJt}At$C*fku z#hB%t4u!4{k#FP-J7{6r^`Wz9p?wp{9dS*L6!!h`kd zi(?l|fp7-imJxT-u z9vw*<92^`5pMym+x~6$SUrU6U`+b2P|8;?`HdW)R)Mgqto<#9+R3p#geutYsAxEFT{JlbyzKxx(S&qwwSz zpDE3NG<3Lm%f&L*mA2j_t5qAzU!S;35+|=qsWGay^o)sp-1|VSE03m{kQN z)xe+{rEcUCr`aZGF*PNAd+$#Rz8j|6C;$v;3*w*x~Y6_9@%-yS^p;s&7%~rS)Zp^tE*~ zNWb4>4`667&=TH2yS0D|V^Op!ztc1zu&tGNYC8v!r{bp z9f}kE=eHGC2T`1AFVZ+`12hXeN6zyJYKKmDar3LaSn+-y@uQ8K{NU_%okLe#&wOB) zBw8u7wgwm4Ss0m6mAWxA&hPU(I(s@9DX{7HfRIm)#LJ zSyIZizLcFC&l#6r{j}8@ep8&zb|P=~kkuC~Gdi-iCUyU!MI&1Eu621PNlIi9JcChF zraJw>M2p=Z8MAayIM+sp>I`#xR1%6(G(xW3)ijLuIRJPw%7AaV!|8JVZdtgOPN|FaDb ze#j3y(pfy$P8uDu3LJR9H#YR-#WONMSOzww(DAaC3iM<3E*Aj}>AJM(K{-ATRKrl& zs>Jh5s7o-bI;gTsi0T7L(k}#m`sbG0U}cGP_Dobo=YlQVPMa#{Pq=^I0zpClkVqEHnc!hVvh32YGn0 zyUU%nJy^-&0KFT=wdfMyjTIN??zLah7Z%WBaUzdhO`uy~a znc^mmNL6g59-9eq`BEt;1lH>&J&{>0{FRs?$a8sG5PoHMFc=3ZOj-nh3 z!5SeLNFSHz5dDg2Z^lNhoa49r$p#!0CabI334mWF5AQiZ9q{I?j-kEzlb5oY`&R(X z4TVf~u3(~#`s9uIiSJ=Jc?*+jsTUpr?DBQu4SbvzudwbD(we3hjj<*@RJ(#K-y+3r zwfutaH-KJkyC(kqbxq*ULjEb_5;>`}3daWEUMNL$0T9qxMX3H$moIm>I!x$zf5p*> zEr-#DDux|ss|#xWA(uINgytAG?eZ~h!2_aK)>sk>u0JDl#$hd({zL?fEC10KoE#n2 zG|KJGxBIKKDH!UBwvXuw9;jg|hvg4UX7j+0u+XpWYlPC38J% zGPc!B8RxNo#Y7JO=oTgQ-pMx)_+O*HdF+nxVb5R>>zP;K3-(&J^aoVp{#}X` z)lUP6x*F|BIQND{@q9z*m*RZcSKkI&xZ(!e{A8xs4VrnUdRIZ@Fl$B!zP8%}x98CS zbzO6>&3p)rqgH`lHNE0G-IoqC?>O9|NK4Fvvr*;wnfmZC`hgZ(lqb*jce1veA3#n` zI5io`;fsiSnQ_Z$gJ`W~KZPmduX(#Ymp!*_?F-MpnTPCWS!Jz~;A4|F9yY?kAx>yk z8EoOFJ9L=+Q($q`>qQ7=TE}NW<`D=xE;e)rYGdcNDEdzQ?yz-uba6_)5n9)ZiUf}{ zHZxCrE+vm^{U)D6Bbt<;beQq-2P%~1yF-5p&^zi@DFL965rGh#b51=?e`#DxZm(aC z(_p6GjTAbdtv`j5dp_9q!y0|9xVf@+gS(Z4&8RVuD)VM^m%Y7)J%{Rfzf!EjzJg=f z5+Z@RKuwAAWNM*`gc9<9Gc8XnLN67H2#W07Cen}b}kep=rrbhDF}4fHmuA?uiZ zATQ%RmpvF0x|vr&BvLXkGhggT9!7aCKmWUAo&rfFMREYpXYbxv%6`L)hrRgk5tE|X zf}I;a$&Mi(vSebzcA&0r4`33;d0Iv~bM<0e>$$Zgs)Yi%ki@SLQuk4Y97^e(>|L37 ztE~`Er_6}zXWAA#XHOTi*BCnXv4)34vhdmE7&+?ZPo|w&a1nw-9)5=eJ7a;=Kcs{J z)eE*LR0-VH{gvkC<_YiKRS|{7#dR-V{_uhPcdD!YGu4$eFykNsp`?(7w;zrT<4WCY z=U#T|=BfCp^tC*B5+GL0XHRvF4tv^c!8Y*3Bxc%Vl|TCakMP0s$&ZJMi8~=cH2@`9 zWAWa-XaIABc%Aink21tDb*k;I&iE(qqEy&42NwQmpu^&()fD7;9eP zynG|9c=OujGw#H09 zke#tyJcBGV>OW+4KeCI9d4;$=iF0HD+ESeLt(i!piw^6nO*Bw*9o6 zrInroq6msrD>Bm6Rc&Az=EjAeQ|}KtbhdYr4DD|JO4d+>4E1AMbTch*3=ON96xCKI zcJhh{YCEb>^LK(KGtMUw35rS23!-URw!006O1BA z0L^k9I{C-|g(|C{`%~Lv&H}Bfsh0H?Ifi7UBRhrkkEAN&0zmfkR;9Re2N6;#W`5qU z$wuHEcL<3YRMga=|7Uq-N~ew6Yq4WfNRyX?M|)(%DirR$*o&l(420`V`9FF?Bq` z|1IYn`bDV`cGqS*?abn8#Pq=Lk zjfU0J%2NPU>(X{buB5B@Zu5Kiz7Vu-K4(YaRdz1d8Jo4shBEK=$&YIxS81cfY1U`! z?>!?8LRWSie2pZuuVdV(_U@g`q@kf1JCB-sbNYdnXvrx}BzAd{pvHpy{W}s98+EsR+Hp{k7+Kwjsjcbm4k5*FK(UlEXC@SX z`JPEEm0Tlwyv zw}o+dxOR64Ry==(dwI2J21p+Gb&7(-W3xYGv7hP?RDJBgJpr}<7H2eEaf2GfcsC3-FN>^jNMP#1JVBN+$n2%#^>xWa6wYL zVk?exWm*|srG{`?9aU1NqN4Nt=YkVuqZQ(iN24$v=FeJjsY0A#eCNrn%vGNM!-Qbyi@$VhzDF#u8N;^j@3N%f!S{%L+9FyIJj z9PUcKxu&5VH3}Qx>etthl0b@s6l0HO&%mjYT(9`uDX;;hkD4)>fss)O9>FgW0@L7D z`AS4e&aZ)PKJYAzE)kEWyh~X&6Z+^aWvC(7E=Go)s+W)|s$`XiMjmAxS)BXeuh9N$ zLy{{Y32;zBHREi%^v|nFggfu&rL!YYMBaH%!3Y^j64#G-QEj~%lT@|M4K*u#etqMeg< zV9pPAsOB%6(O6cV#v~h|lUA`%D#6iT$G{mt`TpI^E9rm+ANGL+?99{}pe^}=`2M|a`QhzrkWfBJ(4$Si3%gXdV za_ig~4ej!TLZT53ymoBnD)o)~ykFZw&`|e6@driU*2a{S{|osvrUCU#Io+N=*?tEd z%2|tN4+%XMn=xMHf1NJ0*}Qx9fI4# zEq(cwvfYmwH0f_vi^3fJ`|ovAe_b93&e8Gn?l>oOs>}Yn4b_UD9PZopiuejdv=mwlZ**2|xVeHgT{L<5*tC(VtE8kZr#Ptw>iqqCLrz^JwFsJ0)i}g`_-A%28D~y4%8Xin@@L6ReGnw@OjRe%7@T9YS%qV^M!GncG%!#v zZO3Z#m>A->(p!|LBp^Y*UYH#sji6An<^sF4i701?Ei%p{d2?yikQ4frhqO*{ZFU1;_3E#%JdT~a~f9- zUj;E-NOJ^I-0p-LC{%5)JbnZ~V1{e)ymDPI5Zs6V`Ddub7QW)obw~r^67cfLOhC$6 zn3)s)2X)bFJp>Xh-fk!sI$a`bUCWYm2>%MdKEhD+}_YMX(dJ}@?S`A zosRUawp+LX9i*%|I;(dxcy!@N^x;EXNQn>lP@=Jn{*~Yc99%Apzry;jXE+;lN5OGC z3Vn#p4`TQZs`m(@1m)(maFs{irlDx)xre@bFv#kdlHf`aKz`t8G(9|1eT*K@G(8_r2aF^D5y}jUl8-h&s^0 zlRUv8;{^m!5yZP4G$Jyl_x2#S6H4iCNIaARO){V^QQW#QIW^TwT1F>W#dzCNI7o*8 z**Pit+zJkf=dM4QPh4*{6k6HLddAa~%M%8F?h63Oj5KU%UCZ^DK%v3%JK9Lz`Lk}4 zxlmOlGfjYc zb|;btLr0#e6x*FYEQ27h3|f_;&6i;VxYza;%GSJ3b<=v%7C@#!w0 zUrC>kyVNZZmE}sX4i~aZ0^b5DwR9~O7{?b3>gfd!Fop&Rhm5)TdoWLwc|y&=Zt(SG z05yZ~O1vrjAmr8M|MLb~h{@Qn*g@2;`E?1=3^4_Tnh{}FsK#IZnMm^9vr`WZ-W4NgcN?$?u?5$YYB2aYGdDNKrK<4rlTaQtpxmnY zNbib3e-`aTpf#Y*gzml5H;Zc^eD*PLch2 zg9z!XSHnqQL6Y#{g9eCIY7M~OAqR|_M=$T%3C>Q>OThHjwq9uTg8KcYTA<)O9?*Yl zg@%lYxq%LgU4M8$XfO06u$#X(s;K93)mkVF6rqFY2blBWK|tq z2G_iM`S1D5Fa?hb+;xNQ@Z=iu zAG!m=9M*t4Ym4La3k?!D;KYZ)MlcMF`{er2B=;am6ov49BJDr_bm+0MvPKvl`?CSr zU;~QorfWMggK-*`;1HAtWGjRn*Z1T=*+Du0Aa524@7yT61zI6Wl`E}8Tjt&hv7;@t>R?-R{NyDJ52}QU%LN7h30Hn+dWvt;SHgyDf;3(75Y7RS zg|3<_`aTV9N`aL~HmRvfs5IC#g(@7NeYG?own)(5N%;z05=^0d7zZ`+yU8X%b6Oz9VF0{y0-x! z3JuKnC{f1+Ed3Gla@fo~cZsX zXm@DX99sC+BPJmGHNmKB2k|GXg&ShX%81U{KVFcL5n zhW4m&{Qi?|-!wRL;_vxvL0CpRyT9n?tzqczL#I{^BzIa*UdrM8A=Lw*6#i&q#$go5 zZA5{PZO%1OHd6!!e;!6U<(boM{dTC9O+kxYdK|rphU))p#(MzJL1SkKeSgVqFgqqe zUglmF-l&*zr``!p(=*9*p~zE3|2xj#EI)KQMT zEg@*Da;$|y{8rkJ!U(1ZA_F5hoqF#3Al|UX7s=FZO6H#QxGmHnj+Qvk1hgN!x@Zw` z?uOnh8rs|`NI%0-*Rzf1ypYWS?*XK#;)s}DOU8QTS;+n8m8ZhBo0hVVFiJ3~rNERCHKhg+n*E!29!) z;)Yd!A%ULn-FQCBVa4-fgK(~gMz_UJ3&xZw2x6#qh5mKM#pRT z)PB7AD(I6Eii)}@h#(%|lGvE_c7r{c#O8@iL&(e$a`q=efP{1CA zE@#$MLv<@0vBWYwJc#8#&_t=5MO9<1kSRc&o+xdZ1DD3?Ry~9|cGNIElg>lz!u8nK z9V_RHVRR(R>1!_$QXf+xaTs`g!v|E(U|KdD|AMVnj!_fe#FLySh>Le>Hat zi-}s8n@?s!s9qU7_Pbd11CS~r{$0-Pks+%br(u9@8H!)Hm#MgZO}q%Je^M&2SmugM z>PzNh_mGx(qL%8)kGM+Ma0P#jpXVfkIaZQSQ0DPOnK1h9Dbwn5Sj-9hpF??h9U8yRbC$ z^p-{IBp4Q=fFnKu&!(F>xw(p&WCvYfcC3QMY;J?NHZogQ2qYTAUR2!vV0Q1N_8mEL zP!w5OXnu$Eq_8t75e>cU%D97v|KVI`bI2)}gaqKMJB`Y3FdQa(_(-AN!T7YokqC<+ za3l$p=PtYwgC@E>^YhCjh=BS9E4nuJ*7#pf7{Q@VeX$=k-jV(G3f!5D~_F z*+ixaJ)i}H9q%hwuE@BChJ>)VuPp|EQ5Klx7rDU>H-rqLIpTNYh-3mZ+KkK~VIfBW zfp_q?wE7oRTlj@9eN8Zdyn${A{;|M|iD(8Dl8DLn{mw zg7KXJVA@j+vkH$}=4A{Xv(B3(^8P zNFe(L7T)fyQs{!~{vgS@Cg@{qT2%f#V9R0fiH;yc_%Ni#LE9w=;9@G(g2GQ85vf@c z{#5l>%++Zo1uu~>`^M>~db6UDMI)EaWk#LZub>B7Rn|MC1~lJ zdTeoA^WS}#o+rn)Ta&;a%mImbk#C*J5+SA#iR(67(_WE*T5FSqqcMCF`9k?7*4EZ< z%W`1{BW8Y6nf%pKK2J@bvU-`4dvh>M$DJ=urau4Xfb^Psv@+-~ZwKAPIPG;u^p~=# zYSUBH=V0(!Z7~-QB907wHq{!j9s_d50&KF_$c6lv@6P6{Q&&nW%cL19=B_X_@wU5O z4H<48e<7XFoa5BzwEq|n>6ra%$^dn3E)@-(lW+$N<%RGKWk$VXCA;&ZNj=Y8r>qbo z`NW-2G87utC6QV*OPM*6otSq08gVwh#R;ZV&26*}rdAr+X})n`JVehfWFm1r`HPLV zxWtiL)1mhz%Gf#R10~v`9FrGcDlfe-Y93#laqQ+D$KTJlz0c5;PY{bls+H&hS>sWo zNYdzJkL9ww-rm3ybpHO_5IC#fiTOQ*RFRW8*g``BPk48_>#L*6hGLbYK7P=7@1pQW zN2Q#miMFh)EP+x)gLyk!Vah_}@slSH#L`;`3E>H#7Mn=sVyCb?&snw3j9`F)XW1!-ap1Dlkt@PD%h%+bDJB1ALe!JKc+}KCHckkXr-V2;@^~yU8TA~i4?7QY?HLx12 zn~Ko_Ff6g8ZGyjB@m*UVfqsq<#T)zJ|VT}fe}u?^w~O>wtM zz9m?hAQ&O`(W7D~ZPg)Z`nwNyG)gYQKuIpRitMA0WITXC|6ZqUx0e*P&=o|7hFIAs zQn~e-fu!{ODtLTq7jIQojRPOOqH@>8K->^&axLb}=mzo4iud&}HOjix-qB?VHEX?% zpj%q+jXrvQV(DU8VcK)~*uN?VJ7~KOn@Gu)Z4|jYuI($79f3w@`R9%omv{%L4xN0% z%USqh{3F`*vNYX4(WiRjL2e*NgG7W|FD_)yfc+)iUPzTs5oD%%+XJdG z7%RR3Q>b~JXYT~LBXi4NOK7crItcHSbS1R=?oZ0wND21_m|LlShF^iaH-O3Q8mZ}quHy5Y_Unz&1sQ5B`Azl6Q~zkxfu2Op?UiOZig3X}%9D2@?aZZZ-HuyNsgHxl-ty}+Cl4(j)F3PC=N?^jVkkb>oKbWv67}^z<=*kb z8H5Mu%}0hI(_*xJSkj^Sax&@8iTTk`&W%e+Ujm*(j=c8bO_~?a`ON&Xc9`~j?K_J)0wyvEE>)xo9q+7%kS|^%cX~MdAR8d zw#H9ettO+!c0AIbypSD?v{Za4g6d+pU{9d82L}i`>$IQnnn=vVPEn`aL>tbl5zl4*e zKe~QJ1u4w1PWNXH%}<{kt$Gq4Z96eJWB6k^&(?C(V}9<#!eq-WZS$dTs(iznOAED5 zr@f_yA7yL~Hyx^kb~K7)e4|kf4^K#P*yfh)2z28lx>^PzEsW4cd{z9m1YUe9qGdWb zW-i6!etmjIM(D?nm)9L084v<)OJn--0;XN%sT`{yfd%HZn;s7>RCVX*%7Xu4WI=-D zSgSQJJ3!@0jv>KM|6QPKAD$X4iV}{uvP_ojAs~u-BwE@!A)!)@!K>@Y5~G9AD4SWH zUB<+93SG3amIjPxGHgzeodkc`{gT?OP8zHY9ZHj~`|p>I^*awZchR;EPw)H) z_s@6e1*j{YSzUCtEj*MF+wGBxd4E|~oBt+sP^v^538%J$$A)jvC$wALcrnKdJEINd z%<#~PACHfqYrLiW0zg{UsiJj8nzZo*+O1BK=-i(p&1{7R^;cZ(e~INaTGv`#Sealx z3ms@q9D5zB0whYgwQlbr6YgIgzx6g`U z-Jn%H-RLo|^*l#TVcPPw0b0!$!+r6m6<>^etmz$bN}kfN2|QsOYadkd!L6uA+$4UZ z#41lgf+##r7p5Ppu5>kpesy8O*gaYLYt8$XnLq#`3-7lT%y)jhT0gsXd zJot6SFWJ366Nf3p#DT)dI={BIE)XbHc}<6q6&2I#S)GJD4zxQku)&}?dL`Gfe{rf+ zH|&{k{QV9>gbkkR^I$biXtfw3MR%{0V3cA3)=w1es%5# zKONLTAJrxB+>0}K{6;KHm*UMsLSCF$b7&crqjdmHatYT2gVwv^8nC^2prd<}Ym`U~ zSbQm;kELb4boem6;fPCD=1jyL`^*pj6qb=RTCNf{rOaKAHxlx`6~6)wjVjq)pM2%V z@})sQ7i*vmdG#)>+;gdT&(tKh0#LKla1xkPne~*S*D_@9HGV&MEy+j*G^0J@W7ii} zLAG4wYBBOu@HY4a>NnaZU;P@fWK5I8*5OjU*Z{qi-YW;F#pHlD9ZDkekOyjd@8MO;-!zNDM0nkRCe-LtpQ`LLQRV?H2@8 z|2r7JVVt%m*yk*@l`-&@!ysUex6*#gM(4>x)ALr$lB3(Yt|DREQ*G{ZAbu#|>dj@C z@hP1p<*!u=HajJvBNv1++t0Zee&?C5ZI&3y5kJks&8Z@;A<|$^mq9(<_^D6P1Gq6} z=_~RcSHs;f9uGH67E@Sd#LO1tS_V3dZ&$`Fj|yWwD9w`OU7aH4b&h14Ox|;1)V+9h zFS}EFeZi-`RiJuuCD72{TKJ(s#_Z|Dbuzq-mUZBfX`%i-E*49iEg~PkM?b*DB`EQN z@y)7$Yf8AashmkJJ^O`=pP6qXnn0Kqp`>>m~TfHUxu)lxsue+Aksv)d=*Y z0JQQX(%aRZPg3{vJ|M1vlvFN50n*H(q9y##Y0d#`Xi%itytGvdPc>n8F&TR`qDL)* z*GEXE2fAFPwBs#i^~`V{+Q6pcoFlop4Zc-AdJ;+D3BF?R)iu95{oulz@Hj;z*_|f$ zR$R4q2ZCVr<-E;?f~+{(oUiK$=K|XsV4pFD$+ov@-J&O8mEceq%+FTQG0m@y6a`oKn8-hTK^rd&Y#Llz4D}p zCzbU`hg;d*fQ>Ad$D0mDimx^coVwcZ8C@BqNLPk(YzU%y-r!pH#EOH@6=)6^-VPWl zXeEjDXRha`w<7=cA036!0mYSOz7j-dm#Mt=1b~kn#EX}^EQUV}`UW9w ziPXYlC-^RugAXAy=SzucD*>x6X2cC%_WcI{922GeiUbqEP@Hjr2wS3~RdMU@lU?#| zLM)&r#wRR{GPpLZlyLN%vA)>2K9}Lp_FXwvY5M%>PfMGEXWA<7(*^t9^jPxPEo(PE z*_?xKu{oeT#m=W(l8yZ1u#2VE&XT(^{D8W{z2DQ4BZ1q%o)Vr8*kg37> z=|dT9`bQUg65j44|ApYZvb*zuV616WI-p`Wmti%s3c8iL4?`I`Ll*iaQxT;mw{svJ z=zityDEn}q(lYl`eDI6_W}hz@lbrSQyI59l?~7}?{oqStO^z52De%cP%B{5z8qXPLtt>j~r`Do0ut9#7$<-#0bi=GAt=4U-5)wwVy&ywGa;imb z^+kfYFc+k9#7tQp=H^d$Oaao+g}ul|_AvFk2nC}hOa!NxIKG#dnw53VWwA;6i+;9@ zqobpD%LJ1NLMw1;WJv-9Is{L#qg{V1UXp8=sV25eD*wK1&Z{8POo|-=W_CD zCTDH7giVc;>SL*X2dBIR-aetRbII*tq}9TbqIsK^w-xhVQ??^EOjV$0)!j+b8P&lz zdeSa2ax>6BP*}N-V;34F956Dvp5f~OmG9xFN*yHhT9&YWM3hGk=GilMMQz1FG z5j{teZV4%A5k7NIcekm~7uDhU?)^A=2S6&n<|+;f_6d&0=7-*tY?_kdTF|6tSg9(VQFj zbSly2azwbUD^C{4Z-d!Tc&9Po|2yu8f7`9XN&Mn>5|t7}B|SYEgP`}gcEy`=Z5saudNj1bqk z_VVA8=*_$WDhY~TN|}|9A{NGxkPdfA?<^)4`0~ z294dd^>v|@v?O-)u%$mVCGn|ku8R3oEAF5UeVz4GxP=6!2BIJOO=vEDLoI__N)@@Y88{zSWWHRdd+M{Z%3lrpjBQy#|kowlkv`l6CgWOr8*2 zWU_d&W0rM=gc!RKF@6&zOn^Gy1an77f~J;TL4iJmf00xL?lCOb90gwe!^5pFFF|+G zCy)mx)i2^4t-@SK6hkE((`ZUv?>`_^%!7yHa7LlTYe@_@R@`1E;lrx zz_i0)MqX~;j(>iU9GLyWk3NAcWQuDY+|pzn%^rgCSx8tHsJIA8Z2l}$O!GG@UPRP@ zDi@hv=U;3F_P|nW9S;*tlJc-@o9l!yKmmjLB^qV;`T12~)-Bp<;_i+=lB8-Tl5a~s z6HrnL{)ln~A*6cunKT6=LQa5#5j}&B!7;{+9ebFG8qz@Fv(?K=_L~D+S=8@Eg^b}7 z=-p6`5!Dp1?l}WJpy1(rYIALgFE-kF;Mw70T|=?tT@0nNZ`tlvZfkob&N}tfBlxF@ zMQ5zllZ+14!fxnf0B64~g%~7|l>1p5pDouh0VR#H96bRaddEEUU5W8~UE69F5Y%nG z@bBOPNMBX_MFDYHybvQacc`eG)<=ZOFO|}*dbb3j*RDXL2l9@_z9vpk)4(h46{fj{ zy)zFKbD(1RZ|>;ge<;!6pG)$wpiIL~m_$2-B2ZV-6568Z`;+5CeXcVh;F6thjtJ-A z6>$@qG6B>X`e(M`EYIAk-1L#K0T9oTRY$X&dL)484M5JdJf4{c?I}VqRAdOjS7Dgj zWi8xIWigiO>grz%o@AsN^vD?P6frNJ7xd%Q)b&5DHaE4>4?R=Rav;zFjWWNP8|06o zmV3pME%feH>_ou$8U)H{5M6lLQ0(Fg9k|0Vx+V-}=&mpROmk2X<`$ZFfjW!gMAWE= zwg=&3)Ghz7hAB`(Zhb;svg~G_JNz|XFooAFv`s{^7;tJh!Ih4|+_Em2yA}>s<_IVB z@zW>m4Z)v_w5*DBP~o~K;VHmf6r0&-eg4^B@u&RwsirqQ>d3$Klx$9*@B|aSRrm#4 zp73Sv>oxGrdBSuY!@&t&-Y!>zGwBNn$&e)^^uVUWWE|Wja!H`Qg%x#Qxz{V|;xN2V zGZ^~~^GN&zcoG)iHH3;s{oNz*%Q>gOy()8lGs!3?F-->A&4v=JcMJvGtt^_lyjkWF z1lZcm(g?yD-*hM#;VieqbvFFwu>=kh0T8!c7QXhSdc1cxZgV$13+ND+>L&&r7KS&> zm~fu}qgnT%&pROhd<~IH>O`?E>;=qNr`z!E9JOYSMKN3*;yU-wzxj^Xx|IT6Vef0b z`UIItit-eQLd7A45T9|aKhnDllVn+gDDJ^X_v*9BQKtsKz6wf6NN|sFl9he70S{vy z=)M|SJ-3s{m`m7tH87Xuq?8Y}m3hMyE{ zk58~W)dd>Xe*Ci$8y(3dy|GdR;jQY?NogE|cO}lBB*xajxhYi!{#-<1;se=dFYk+F zP6q1OPSs`($1P6VBHpy-bl-Ks*tE8$j<@?J2%whFrm+*oe{3-xudRz_GeHtBwTs!& zA38e)ww?@zPf-bVJzCg-w|5TjPUxL)mglL4h-tzIhSPxo0Vk0#E{wV%xMBJhnQ<%{1bkP+w1Vq&F2&$mM6o)bGmieZhh zRbNV_TO(1kD+78H6eNby><3BoZ7E?-ASyLnpLHyHR}T~I#J zBlbVj@#G#n|D}`jbWeV`Qa~+^!$cJnmdu#a7=CM;6A)o2n$J|euUbgok{Elt)8^%7 zDR{H34mDCGbdQn*N)Mj_|6l&Nw9=iTqKp61wPz>W`#uk=YAi()tj}@0qr+CA;Ad<3faU{0Vws`oV$>(!l=9Zr_R=7|;U&*i zV#D4lyZwOPzVH#!W{|L~vrI0&jI2@aL;DWYENxVfXZbgM^d&|Bc}BABMwJjF2E|k5 zUOTA_>mvB|voGSavUU<-MA@0Q*Ior8#4=D9>|(CVmMH^dR`Y+Fj3U0aG@?cq|2>7I z_N5L^Hayx@m5;9q?<83PUZQZKy{(RPwe*RIdxp*u3G&$HmQOR_EcJ$RI(EEy6U$0F z8RpLoDEs7K1Ho6>Vec)b-o(rnCwo(`g=BYThEoQ^dEQC{^Fnf+yu@pvlU>}o%M|N? zz;+8SB__scFEyK>0o{z)&-0g5Yz)q$l2 zlZp>j#^NW6r<0@n6U@xyTLi{QPC6&e0`2Cbf`VFtT}AJo=Pf2&f5xf9OY@RYSt)#L zIrC{J&oi^LkA-)kne@yKscH&G+(8>VU|F9vB6416b@1O?>T~ zB}-CLRdxd3NPhkZmU!wgs=2vkxPnEZCcM=!h3?o{Wxm841Dm_0?rk zXiUhDnt2YU5W%)hBT&oPbW0f5^5m7=jxq0&x=b$D9kV{s_lnneLd7M=>_F~q*?X)k zos-x^>4u#f2Fo=Sk7_^re!O-1f{Ji(IyVEZ{akx`e5#Cp?}eWhtsiy-Rl&UycRl@S zC)-V@%%E6;b*)|T@bnQ2TuIphxf?EaQ+o3s--idD48N(N$vaK2c?VZ7oTWOgE&7e( zG~NcC*Q1i`K=!K7RkUc4Z53Nb#W3w!I6OQrm)cjTTPZRzQq?^%DmUDJFTBwz3-dDL z)EC)LO!)I$15>@a1?g_zmXb~#zxn^Ddh56*|Mz=*fZz)iP!Lfh3`$Z-X%tZD?o#O( z9nu(x7=UzxNW%c>(OATQAvJm^F>2JPF~)w+>HGJ1e7=A32M@UK`x)1D&biL<5_K8A zf2Gb|FPUI4pd}>*MbelBt+S^f+~NDqDF;Z!#K>mH*)Mx3t?|p}zJ4@G5{3NyQ3BjUuvb4d{x*)8r zYiV;$fTFF9{$M3=V=0g+e?lTD|6bY-i)Gh$t1_({UvuAOzMiXyL?c!b?Gi#BbBnjs z0`R`xzy8z_9#QC;ok&ZnNM5m(1%Z#7=7H5>Z&QO+jIZ zEAJ27_#`dmfj$2lH2h#j<^V+51dYO3lsdP@HXR`lK zNdnr4du96U+#fICA#suASevqiCHy%h-*&NzP>Z@D#PibJT1r{gpS5{`n^RV|r1}Z< zrt2!OEk9JXtt>WP@2dYlm{?#=QI>qoGoA4Q}5VP)}&c$+&cl7mc;Fuke$@ z*Gccck%J#gtymO%lACUPhduEKsHoOb)>pVaV!0-MIT^k05ueaQ%V;KJdM1NM&N(0^ z{ON$nL8Y{puKs${_lq>x_qpMmrqE-F3!hycT`BeXTi$ZK1bY0tQR;>VCV7-f)p~Y%o1u3WiNEie3FGNnv-qcEBV@pQju=yR}URk&H7wRCpo zwMf{#G;f}9^p_AipBceY{fdNP^63xZ`#YBjq-+|s|!XLw`+D!jk!b;yV(}&twT8Ry)eu?KOD%zra z2d8{FR15j&gT_pG@hq0N!1qPK2Ba|y--&y5`c$ud(pt|O<5sO7z`c#RJp)_kh!zg} zGcYO|G9luS?M1L%8#>7<2)WPf`&1N;mL|gnZgHcBUN(`3(bX~w2(=EMc$({FlTxsmnF&t z1alI@wf!VGDhs@)zTbDsj?fBU3|^iL;HFr_VSj7_{=_-|&Y9>9uXIDGO1*k<$x3Ft zA}O5C*`(h5qp$4WmD7>+2o&^kxs1$^Gddya)gkqMB`v6pQo&d&lNxxMD`6-t$DY<9 zAnhcYwz2d;N;rxSW4&-xbWY{Ps zV!h7VPiF5`AD2EX$g>LGs%#KRX}+W-IiCRvfD-o;U)+sArxkI$cb9kC2+_EP{Ss6@ zOh&>i+d_K&SjN9RZunM@XukHqw>R=`jPk;yd7lKft&P`#JdKlWZW3v=YmI4V ztqwBk1sZG`YIN!att(b&E&521cjOt@-YRC=W?+2s)Xg?vGv3Pf74oUD~>J( z)lOPaePvy)JN9#m(wwa`J~D!GohXEP%WzwVO*Kpys(V~MR7t+Jbs&rChKB2itNo`@ z!RD}E$&u1D@PRkB+X70EmLq-s5T21wMK|D5{p~9BsJBH~cA*f@nH zdB+v6oP=muRO;HXG0F97Pw-U?VvvS?adlOzT5gsv??sOWYxM369yF2yng{ z=Vd?AzMTYD+?;WGald3`B|iP5d$1)HpG?5M)!rn6fUQgpzN9Tl!Anq@sJMqz>#yRw z(hL@3oqG&denEF4hjPB_k4bz*Z(sjDR^Df*Z@YP`TR5tB28L1);&iusOiA60%#1Vt zq_^X;HY-}}aDv&RZn1I+N~bN9uU}{J5$=u&`?>Dyxk9c!{7ndVeI1l&QmcxqG4Ovtxk`e?5lqD+>i{Ty5Uhc+yA3DJc!_Jh+NntMT;y?cxML0I9c z6ftCEgJ9}oMkWsEi0W!e&$%NbI^mlHChastasRu8J;~61V%gJug#5e0RG$hS38yU^ z`N1v5z+j@KL0t%OdYfnB9`alAnddRg-K@ts=mQESJ}7|VgNS#2PKQg%YjIc}eO6N$ zB<3=I8{T)nJ#S#!KuoZu?6~=%sXMMI4dX?3=nOHWs^>-Wq#xMMbaX+5kaa1f>+v`3o$ov7 z1?-+ren8UE&}V%sAJXdl>4h(gX`SKFA17`&jeg?MVQ~yqzk+I~Y~p!dR3HB6#l4lK zk4bI~sLZ=_Td8dUtM$|gc3%gK^o4J$@c1^X3Rmi*e+LRY-OuPa0>$@T-;17wn39ra zRF*3VeZxj*&%@%j-fCou1R6wyoO;C|bpnl%T9f;nV}+2ANKbeEvh&ARJ$~Y6L6@in zevP}?CslSOK(MUZrqTkG;!4dcf3X?}-!1aFXPL)1+L}~sp~Id8OwdyOk>+>cxtlMC z1gqoEWh-``iWpf;T0_)Y2aG4JC=Z&31NH&FXXND96yb4uM zB`hn;mb&GK30v7+MRe0htsSak*VXkT@#EDc1lSd7rb@Z*(O-rPBg=%DT_)lKdTCy# zmRf2lrC0gXg^WDOr44hQ3WJ%}`PQ7hymsU}u)I!b$x>5Z{bHn|rOBZlmep4OreWM< zJ!ZCv%Y+t!tVn%wHf$1k03$dKRrK-O$#u0E9!<$k!1@tAl+XAlq@wRH)Lm1>2 zZyRFFjwUX9$rK-ob^u+OD82;6P9J@vqEDtu>uLOafe)yFx$`x%(X#{=(zS>21Ksh~ z-?+xx6A_r&w-q-;YM1Q?YJ#*!^7A+k${V$L7)+9I-pC{OXG!{T#nZ zZ*cY^4Uq%YW+}0vD_N7(Zrv-Z74Y|LQ&J<_Pgp{diZ?1%MoF4&?gOZL(i=|D9zd;h zq&A&3AcKKfDQq)BS?fzkzY0ip@Uy6pdixUC3nP3M;5zE~#VV(YI{E2%%#(b(5||dg zJxEC%pp@Bswi_Ul_BewkNMW`%4veWqI;QW5Gvg+`R54%bU+lx_j?Ii#%qJ>Y-i28U zrN`OoFg+KiP(sp`#cWDpB5V%ay`O2@H%lI}c;7)-FI!66ADE|1U10Q77;jiTq^J=J z((a9M-`pff$*#*Hy8TVk=04GM<~WUgn+_1)u00O)%)m-d7Ib8f{#GK^27J!c(n!*m z<`OmaX8nwtchpC!TO0bF$#wA@1&d#R@Cp)MG$6U{KR-ki#|+Y=Ha}V^84DC7Z4@ce zy9sRB7H|!(C|Em>%64Jo8W&C*dQg%)QiKN;SFJ@17Uug7J2VNi(cVi#|PazsYIS%MYmTlItlmd zBHKahxSKH`J)nRw|0C3EiLn z)vM@gPC-|wnnM4DTYA0w6HHon$eDVV*@!h2X@#(vMeccX8lSGI0e$N)^-#`xsX!b+&=T`rFh{hwR{Y z#GtyJXO<~fGqX1@3)V3>ltG%6GI_MHA7bmc&6qC|wwGSxE+6n_diYq!`8Kf<7S0C4 z!tsNe_dbc^5kaw_Xck2^<*dIfq4q1k<|s27B{B2R+9?lFnKF~Lc=ERM_f8;YO(dux zc2jcPa%aB23EI!yuuF)>jXa#Z^5oY?D|*=R-en=T^t%ZYy^3PvmC+ZCQDm_;z^J~{ zLI8R06BrIMnAyX^fNY8EaUQ?0fFt3JDlDb1NV#W{m6BP?ry|(&qRsCn6-P0$$)0oK zPUr4hQ4Ar5dRJUxH>`mt%9^CfG;C6VVnFZ>)<;`=>p-6K+B=;d3vK~j#+B**1F7Z2 zH4Mg39>%rs9VRlzJzfS?&KP-zf795htGHZrXXArn z+B{AtTjO%osu!9LAY<#cfbj5gsLO@t-;V#>a&{D}h{|Q?)$ILP6 zi!lJfNiejdKH4K7o0y{9hC?o1%5u0@)KtY!*wY_%PXBk~=)Px!%!c5!@IbEuRYa8< zX;d_C$V8or0CCD@WbgGJ&veY*@~_X{;KlTU$ro;`COczyN+rEq#!+HY@#O2V*@@jg z>5?P~#kvBcU3_4X)k?Y+wWoEN=vi-qrCR@8#Y)MDb?4j5kMeBe#ATmL`wHG#TytE5 zRAV#;I^8j^*b6XJ1j&O9b4lKTdk6B4J2q)o7dB0uD}?$vou?(HFi|$SM`LYgW{q}? z^&Zs|tp2#DLjt;wI2J?7rFD28lY&q4J?cvMBlePb$5@usS}*)CvP3U!X9)D8;CK`p zM~heI9ojtR|KA4(Pn;dNbOqqFHu+C~yxjrW^BHsPtw{nVy;nV2p12bZ4^xv#Kzos2 z>po*XR%zqG_#b)F8w`Q_Vs2AhGbvW_v*C6rTjnnuJ$P|bwjVk5=c!vPRNk-8--7hH z(c*`UbVXV9@boMm)+^YJ^@-%yxfZZqbli9u)N=MSJ<3b_z4HVulk=GG>{^dTH}gwD z-RD-Fk#4t{X6k$?srlN;55eG1&?>3E$LDU;XApZ{D%C} z6Z8DrtC5RQc4sl)EAq}~WTp+;uXsI2b5eiOHk}aM{5AMGt2#8+MYgqwW%WB`;parH zp;_>sIe70$-(x_DW7;j4lf9o`$C%>*B8u%b7KpRs`?o1x(L!n*`r|=VGomFBmk-2Y zW*p&UZ6?vOP8Xorg4QjjtWxf7VPj(?n0Z}1nZyJ*em>yqHrV)e1@U;i8koKPhU7Ch z%xfxQG}2o0i#nH%tLu|Lt_-QMjhxnRuA|M9Z})y3*rKLFjo%)c!dX#dqDuQ-%qj%@ zmZR{PuUJSGC>?14oHTC)iwW_%tqg4c}r{c3hAkDV>*OUvI(c!Z5X(BHGl7MCVrY-4^?FVsZqaeIZp zdCo`W^P4j~w2u)bpO-M-YcB={{n#{HQq5@!hMg;&xL+Fi;8Bscgk2BZSzkNTPcy!n zgfDgZz}}R9F}VgKBWLQLBmJbA-AKl?qIB##*4Bx`OpX6kWY5R%eI?ManTDRelG%w0 zSuK5+x6^e_)j&#gDVqM=M`JxByS?Z>VTm;z4*y5r2i?W@Yi;S~GBxwMj_}4O!TsBb z>m&DiX2iFP{Ctfpu99LFOEvlji_h}h3>0X~p_pR>ZmXw4Jg8exk))#S8=5%^R%G2| zN+$_KV;dw3nms*HRuTdCMJ+W+s5ST?xi?7wu{v5dBdwV`K7@ST-rkPY*C$cI8jbSk z%YW4>mEDD@8#?YAe0I7vo$a&5$IDXVW-Xv+<(2)1Tb4C5ZTNs9uvtq{`eP%c2}DW^&M_eVIQk2B$E>nY?o=e+&j>jx*y{4}G-7ouDE`I`dTz ztk@V|DWK~OIn0g{)@4F1_q)1!5oM$}<~szS>n20tDk=2!@+2Z1u0I3|!m7f17^)S2 zCw@DEDINU~E?cJxi1ts~TP-P+XAf%&{%v=q)&2`!zWUaFm8D`p3xKEl$Nnf5`Fww4 zY;w1Jt&Ck@VV|1Ge&v^Z_4>H=mFhX!3IEj)hg&U*mKli+XVFNACxAwuQMNTu&LUv6kPJy^d_8okDa7sj zn+x?*lRGv{)Rw>A7JUEyH7DadWZEj)Di_~AhrR&X@q^=P|G)x%yA2V|9q@{3P4K|GO_qzyQZv` z^bB{O>28@M-cRfjEWN=gW3{q*)-Je>2D_#is=*&hDs>JBSHt(IP-s-9+1dd-g{5_! zj1f8wU751&ivMFj6aBJ2r?PVNK1XdvjLORuR<4TM&acGNB=zqTHPY;&rW-#QYv6U} zE9ehGahvKZyM(Tx-wmV$h>c=k(**w2%MIo=GRi30kn&?ap{IE)b&3;e?)0;~GaTmP z!KU}?w;aA<818@aJ8LOSEBDARXWq~z{bCV?cHQx~byTPq*Y^8$&(b$0O*t743=?zq z-Q&^-VhMFi9Fx}*20m;SB&Q&-dy~?FI8-NgqOGDj+{$lUSa0+3{mgH+51H6=2&M@e z-gQD>^q9fBlTv zEZ8`x7*@PP%e@s=vf{i%=)zc}^je*5PvIFp5%|bt9giR#T9To-kQ6Tk{2JZ~NwCCr)^)g{vY*H@oDa;D?o5^H@j`z%U*WV|NY+v0{$XDtaSa_#&+Z_gvv zUrurFPGRiV>U7dp4IHIL-R+kw!(C_C%zS-QGPTtt6wV0Xk>CI&;|k__!@( zY(-|uawo^P$+*d|m^;VaOmI!xPTO_xZr+P?t|JqMvr7f5`!DIK`YwxKdDDESls^dHp`{!Uzg`CQH45~Y54lpm%L{k?Z5J)b%+)aJd=oTtD_aC*D;x^7^UffS zM6hzz+_9)s_gjo!Y?qcR6x#gU7Tn_-y0SIZMIT#WpRYmWoc*|HBF_&U;C7PWGZ^5q zsQq;Znkqsjb=Wdxt-op{B`bi^|5Ox5(yB~3hKx9K$p(|M72K@@zIE{SDX_P#rwB%0 z0G*P0P50Y+^v&eTfb!cuS?!ZevDkzDTI&YHvXI6C0*mP1G$OL2MN6yYe$_Vb!Y26; zVOfKrRTDHsVmt3i{zOrZjp(EF2gc$?Gg-Zk90Eao;_u4stf^)(O2;KWpeafZMD{qgg}0y`~$= zk0xNusn5JM4!0J=2t*NBlWvKGTYBZB5f1OyQrCo*%_8_wVn+HCMin}faC3L7+rLhU zTN;%i*9Z|M_ZyDeJQB_wt_e)xB<>dWP_nitWL|dU=NDpr`E_Af8`TStAghsT^XSQ) zlmqB4k2ElhSfoh14Sd~Jcv)C*3Q|WyC&d+GNB6+2wz~h!+=3&)muSz^a%~&K`0&LsCAcn7 zKhP+oA@`cCafjIV_QcGlMZXt!A&^9+?OQ#CI2DHuMckjvM6=7Kkt(T=!ynQerj%WW zeJ*=c6<%_m^Yq_Za4YUq1PF8J6>6BiK~~$rXW>3YWLuuuiY0j3g@fm7 z>WO3dFRz7%mt4Lmj71oWSzQoUQyL}xh8hU1mYWm=i8dp3Qz}10i*Wi$X;awYxL;^4OOCHQV!C`YC*$R$Xn7_;y&2(iO1EVtAQn~>leYO-BNlrHMJ~>E-G>l}90l5fVsmlq6lAZ1ilG~fM6gjR9mOBv z8olNhfR}{lvy;x3rx$_#B0=x-gS0siLd;}eXq%m?b>hZpJppVBa8>Si<;N=vS=YEF+h0XngK;6yT%r+PLJl!^=02w|UR zK+hd&53G&iLA=lJ-xz@l|D8}p99;diddA?RqTfLiTTao;nYPKc`?Jsh`%fQ0(|E0U z<;_^;-D?`L8%g0$MX63?bvQ+Kvy86uY5r4Jm`I_Z2xG($-QU?ANaDFjeRVl7C+!1| zyCPOykEDSlz7O;yAUSw(44a9SP1Ag+F*lx1Ck5QfqFmhaR(}kTy@0^+KKR;GEwquE zf5{tR34z3ot|W3vr$2izP*|jP%SQ1Q=SBV}6th=-$F%)fr;%=pz<^r5dg=SU8e*hc zk+%!G)t;pQsc-dU_4-G6w*Nn^@iVEQXe!9tY0lI2;A_wj$(QR`w(%4yc^{WF6m~LH z07!{)Z}|ks?W&-JJPuI^gnOW*CR)@P!%Akq%r4Z}_-DrS>|{1Qu`hs5ln#A<$_|J7 z3&d93kJZu^#}cqp_%5z4mZQ7iluC+ObQw3tOU0IR6&xQL{0k;*{7GVkK;BxS-qz)t zKY9gd@FWYC5Z;^Sdy=FHSdyFb%1Ys+d+&zd&X0ZjouD;^A=uNbfB(tPF@uMTFF%}J zm;$Pe3OVUt`H)u4$+`y|HYa7%;F3M(F{H%`_^BUefL)zc!*0DO7DZ zNN)Qg!U^7^+@-#rpJK_A2H+~McnTgC1kQ<3cK~!LALO5T0|B9W6g~UfAX7-heO=+( zhGPsNmOr5uJmGBP{H+dQJYKhY80Rv~h?PAQ@&aQ?_9hDap!?d+258Eod)onH$fOGZ zElv!ng`@1A-vNaQ%Tx*iAtaJgl(Ba)9DF+f#~gI!#D_oK?*>frVoqvnwWvcoS7E`$ z)w{=`AqDon_A9!^0tDAo{}VqXx6Nb`rt*-@pIDGIVcLG+@Ir z3KIe}yE|Ya9|8Q$RtBz9Lf%G;7Q9#h^>;tO(JsQQieeFaK3xdVd5Z*DrORyrInU1U zES+dU7gS$NQ2oqsTB05JzDVPV_T6!7>BP&tk+26&3!oXZFVg_cKH~zX?(+QKmj;>b z2W2+M&`AQ9$;7m@30m*csvR-?fH#P)Nz0<1F6zzCFgzG*tH5=vcN0|X18>$)tRs#|q$2ee%UxLrN6uFLtNUZ;>pMA&Po zE)IE?WuzOX{rY}YzN6ivCuh#`2?21x#|dL|ov&QC>)$I-=V{=9_^yxtI*xKW*qE_D zKHS!?b1{nQ;sv2W!$mp`b$7u~5d5#Z&i`-}5pw#F9GIGOkcpUkyd)1q4qN_TdZKrJ z7gCcUjdSI{c3&VBfkufeHOYZmxwzZBuif>92c9&D+8@XS=IgPT4gbDl7UJ9Bn{`rh z+~V=M59<@2oNEs_;*$7mugC8!&0rtb;U|G+95p&*#uRS*`#|&w#wBgY=go@uQxD9_ z{1E;?&-?Cn?X+2CR!CUSwG|1U_aSZ2l0cX$kt1H_x4l&iZe&8`%FcmG2{S7`5FrylhsyU}O)f>tXr>!@lx$MV2iO z)54JYo`0HF2nW#J0&|#0Ki)F;`3=Kz;tZ#EyGs~ zZ6Iq>%q{ep9|5#9`3L1+s`n;~*v0w# zfW@%psQz6)x}uB_aZg3x-eJ`awYGu!ES@Dx|D1HcgZptV%cs9n=d6byBA5kWJj#OJSH|xXrv}a(rj(6s5>d zATr$fRoLDS1@gow>E5-JTaExp|IyXUo59k<)E|^eAboiEQKUhnLB7&>8Smmiw*)4B zhXRq|=mxbT6IZujOC~BodrHcFKi?G`d*u?_S3UTcQ{UZB*M&%#P3 zxl)+DcPy^-y1iTU`oB%Z%}?Xu=PV$@Jqa(cJ(pBVrN5mKRX;h|4jsAqEsdVWkW$-nl~zFQ`* zuD-MG>`;w<);n!T>Qj_j2fS2aaP!z?Nv(No9DIzboa>nOvgB$)7L2)bziqs2`Z~om zp_?=Qvczeb^{Gf#4KK%FWTrROvrzR*Z5ivx=!Am;M8WTZy9Ggtu8%?zekc;#~nlG`{Jz&O`Boes%m zCi3byKzk0!K2zE9X(Y9^%}o(7fZBvu1R!WKs?-J2#pfku<#6j;-*1CN!xVEIGgw;ZN0gI5$20jkM$v8bm+F zRkqx5e!}nQ)i>K9j(Vq}OLVN}qrK^`>?lj0{{d%# zV1U~s?_99JT&wqiViZ2#h)4n^b6oPokjP;W(YgpcJ#TjkwoU`hzka2)dm#7V{ z<{zzed!I6_MM=>iUxY_qo0~z4u^*=PW7GJRM$anRT(LlJ?003bqy>J)TDe_a1=@fI zS6v_+I?d&#-lh9m$nH4%J88qF(z5BkKYu%}WtIZ@;{u?dv6zq8iui8*drSKxGi!a* zOMoAhR5YO#z6r#0D^}|KJ-a*=_CK8`o`4zv?ptr zrqBr&(Q-W&vY2(fapQ`#Kkh#~cBgsq2Y@>Od365xU+ZAq31k0@wI`e!1SW@YFifN= zK?2%$ZUu#!rV{le{`wL~X6x||LNx67Spt`Cdz&rwMaV_oAD`&zj$FFVOks6N`j`~2 zRmR&;WKdwESIT`4Wje@@Bx4k(nq0zsAToPzn&%@k`v#gZ*bpp(`JGJIKz?RYm5&-l z)A}mKx5YY%8vFfESpBde0sZT@HBW{c%hH%yEgu&{;YoK0T8^n}u97KJYjw@tFzV{` zS|6?Fes}nV#r1x=%vYi&ReMfk$tGxkm2dENrBnhO(IOaOy=`;?{1L?u?CtDmQf}P1 zu>dUV5<%qD6b=w`LgXR%mV?>gYL`*`nM5#0*yNK6ncIdI&kykQl+K^_uA9y|-5|>a zxo8Z&S(VgZ)$fll3DN4(C~23;Y`W^*aarMj$z<0Y;lZpxQR!dTudLn&LaP#0y3&bh z&CgG%wZmr`)Lu)j+XZKbDvdr2abqSWvjUAtyRg*4p*hf~d9S_L%nx~7{2p!>hdnVn zUEgZ~M97m&r@043l!A;Z1S)G=?ytT|>EH9GN)Gf#%ggCKQ_B4dpbppw5D}*%nfI#x z9ZD@C6iPGXS-Z1;(|18$W&+>^G=Q(x!lx*X_+_WA{Q}_omXlvts19`epxjc=YpMUV z0ThbIK5muDTl__fnX(t1uDV#AEKH~jzYYSFJ={f^Xq0l*S()w{8ZhJbYbEPbVXlbd zMgxq+QnP_~kV|%UPTBx}p?bI_21pzfByL(2ZlZ74)(d`=U9C&d`tp!#e;vP#qKh3c z+1s8+gx?{oAZHIICKE_N9qx_JD~Z2;r8=K%%O8NrMLI4U;p>Lb9EUZi8Qzx*F6u9Ki{Jx@33{)<`)#G zSz2ad9kHpuF#+4mOIvM{Bd3|PrKfX-y;*&SQD06zXG)G_f54rR8x?>BWxT7IDbjfa zEAi+Bw|B3Jl?Q*~7nS7RbLAN0iJ|NDQ&MB@x9Yy;X-c&5c*@s!H@%;2QsF=KIvvtl z0B+zw4%8+=vWJYQ0ED@~WHD%|O_|n9)NMt)#vST43;3H)0E{)`A8}Pi-D%GaYS2wW=4cOR&pfNVuLgQ6 z5-D-i=Ig%V=ea=Tt1fh_QwV2iESsmCbmx;WXK;oJ4={E7IJ?6Tw>;FjH0=~x;jIr0 zJ)VJH!rr`6mTe>0c~mpa4>0r?+0}Z)aB2)sb~0I|GkWY%zv9{)-n&s3pO`%HMX88d zchc-ysc+nsc5T|KjfLwLERcZI+CtON2>=c_Q- zvBY?e*^CWPPUtb5?G_to{XgBe(K0Or7~@ullfp_Y8Y*7B&Sp@^y6g4T5;zMp1MB@n z&)LD(G84jjTqcOcn`y5kM<6X`gz8cIpU;KB2m%fn6MLANYo}d{=uMnib5lGho7A05 zgryg41C3KSsc*%~eGxY7L zUj9I$8V09ysUn)kPq!!W@W6K#_ZOG)06(TSYbIr4tr%c^6-ZR%8eCp;6&0vYPuL|0 zKBm5DfYhS>{CWhMFmDRhhw!%(e9yEn79*l)8I%Dk@PWO0w`h5O9YK=Tk)iW72C3D| z#q9?d7NHGH!}@Hc9Z=*ggX^fusJxWqY~j<4+V(UDS>}6dd(U4ry2Uzs=xixeS7`c)^yn7f!dE7)J^&Nx{aDE(ke10#Ysi#o1Hk`M{mM! z;j0lJgcBS*iYM-8TW>e3b;x8so16{+Q!c*NdZ0N&3`sy#HAplv@z0Jd_fFKJ=Gu&L zu72U_L9>NB+md@<5tB_`{~k5>GB3mlK^iSH$(Z{6`}fcn+lRLd3g2sH$w+i9lHH81 zNPFkIjaS{{2s?a29?=4fA($GJ99ew%HQ&sR8;D~DsZ)kg2vA&6@wr_9O*ZgV(LfaQ z!-Gx=$G?%AYWjl@q<{iIpIY*tPVs6)!*538TwBIUW8LHeRl+73^b{sqOcgfn>W*dK z<$4#hAvxNkc+h0#)aBHb{9_Fcu0R!a*r6$O7 zZ|aCY<$P1RXwWRmk@cDuWjaDlEkDp*Chn!9{3cWP!4!$IEa<+lJco+@Q;Ljx zGa7}gq}e0(g(PE>N0C0m6{ZfH(>&-+bAdH&w#0OBzuzO2EvB>f=esGGry*r)l^?ig zdRV(EeJGSxIM_I1&sX=n5(ve*!F`t;8JAhENsYPO0;3K${k|fW@O55KChf1gdXn<7 z{+8x~aYmd00rN3-_{h%lMfBWF;DG1$M19Zw5IH7yEK3OeIDiPR+l1RW&8xG)% z8A)x()@#X;BOKs=>_9VJy*Q#bMH~}#0VI+>_xCS1#+b>=VuQnd`iJGtjSiF!5bV{2 zWPS4q2@UCo>x-~>n=KwK>7NfWEERALIF>>oH-X+7p8FUKfutHAr%@2+%_xrnf5_2V z_L4}IZ|sx^c6z+|r(gPF;ycq=zJ!*%@I2=sO>Op9)2az>?iE?h3o8Hq`-@(bO2ESg zL^I37>5pW28BAk1PO@S9W!>gnkPTDv2&6uF(R zV2XZii-U0@eIc$JHQwloc=VYq6MA~&=;mQ28T|uYo2CAGA-%U0ws$*+$LcJ0@86qI zr_6bOu86km?BL)h<^DZ@RJ$Gg=<3n{?%slyjk?0{1;!LBDzf7}$bRl-%Z-gCTUez_ zd2IY2mPFAJ`Ldd$lPA{$ zosMprWeV4ix8nK}7_WZsH$4n5_L7i>Vm(GZhBvo<@vm>Gaw(j$>Dmv15qf=uf|w`Q zC?bDYfct&=<^Smpk64xggf0LA13rR=*bG>PdUawrrM*?$;ft4;jvGozQ}s}dEl%!a zz*(L;eOmVT93+z$@@zgJ8cZ%=N)L4I)~zZ!yXUBBUD6?s8`uYj!4^>aM42QVz2O*@ z;eEH;2DEUCscNIe5(k!(s~5IZvVAwGbq~oP|J$=+ycBNlZF&*JBlq zieJ5YmD;tA$(^osPS8xfeNTa~E|$Pu8S9*+1BM>q416V-;S^L53@s%4$m+%&3041$ zh&J23HlttRH>B^$rD-1|NHm7G#iN;);0FhL2_^C#2?N6qdpcj;ZD@_rHoFb5$)!Dr zhhZjoC2u(gE{F2dPgU|N^6A=q7wdOsBZpz1Q@ihYV6#M> zfOn)_wP@IrDfO*Vj<4aJVhdIgCfL5Wh=+(DD*nR>KXst~X~!b&KY~<+Z2w^SJGBm& z4v|C4EXl_pGy?z_H)n4$+^6jLfo#A5^DM+C<*2?zAR`kEIrJ<6spk}}%3x*U$rp1J z<9s#EA#!Fm$5VaiEJo*S!xnBkY4!RA;g^IUD==#l5~{9!b@P7!F3YknWOCm|faLmG zm3ds|UxlTs0&+A|alp#go*b19ESICBqdy)VJ_EDU32&JN)X9BYX<0=)aO+|ji_?vG z1Zli`q&LC^nTZ7iiDVLKvz^p-i2udP%ttT&91G06l7P=@FSCiI-pDr!NJ9vVbh7Wy zOM|F2P|#0?fn_~ph!5uH+`co48NlNA%&AU{lzRu`XVn1o-qIYF-BRr^rrLTy(H6F& zju2QVagZPXA8_ivBog_EDvfmxbr$k)QdIKVwT`~#&$U#qXUFm(1Y}%8Z8Ei7|DPZ4 z7D3KJ5->MmDFbKnxbuswNZhH2%z+C+_c)z3DofPUT6#+CTm&!u*G6$F-Z><4ByRML6R@YA0rNv7L zKd8@<;}2$hYH#!BmUL3QV78##wZZFEIak(WYpLtYWQ4T%77nA=0Q#C zI&b>VZB4x(U?%r+6t16U{KKUz`rTl~a%)ui#(L=iy9Gc-&)GNKWXS6bqdfU{BXHlQ zcntd59Z=GzUHbzFUd;%_23#!AxU0wSt)ZqSNi3(qCMw(s2R@99jEpxVfFa82^(4O2 z^lWipw^);1sK}lr&(6qqAR)TB)n0wW zp?&>sR-IqK%x&n5l~Iskx$>I3)UVBkhm1P1Mw>ZRM!TExKKlA%Lq81gZ(VR2D(P>T53=a01;DrSaN!5Csq@5!!OVj!11qYZ zU9p@}2%BJR-B7t%jogRPJhcZPxu?ub2i)W2!~|cx!HP4$Y{@x$)*}?TG?3-`V(X~s z=)}tF2Q!)XP3in?k4WwT^n$Im4pWvd$3BQ(X*_pnO(9*>{%xSmw{PDHz=9mAdZL_m z$M*HPE27`!FaU%VB{%{xMg2Az)p9^vtN~T)s2{+35G^bP>Phhy?jSoHnGKC0TbG!) zjFu+N?;K4GVk#`*L#KN7nllbo*UgF0w(v;3zN<_zCvy zFZ+|z0~~^P6$ofrX?Cfg93IVOFK6}~(0Ha47xVdOy4;WSpS&#Y6z97()?dB}1WA95 zxA#^Fw|$p&r(9vyt&{5+i-?qrTeo+`97x$qdt#=8^HkopYbBp|I)75{VB5>M6eKWQ znp$TD5&2fG3)_qBy)Zv>vYf5&7~Rc|Bb>PXfzO?6<2s2xouEq4Q5WD~58ElbuR9K3 z4ec@}23ZMl-B-smBY158Z6u}WxiM1K)jLmL^<_vQPq}+vJ#1Lm!$&k573+4^KIRm2 z03?9fa|OB%mt65%w|Y(~2(jKK2TuS`H-i&g4k;DVWPH9Izm&mq?Fd9&q$`#EICJg@ zk59`jrb+i`#!A*`)7>}$t~flGUHYjiDNN{6UF_RvcqDYOnpCfQ6>=-Y56gR~SxrYF zwKU}tjpgMLL3J*pK&{L1k!;84FwNQrMYpvEuFrM8+yPrR{f_f_gO~BFuSSV^1_8@Z zx@i%sCPVke-!x(mFbla~Y`fCwsnV~VXJo&ATBoZwwc$Z;>X#hE+Gd;3h3D7d9A!hH zM?JhbsgNV?6w9YCc373h?kE5N10I&+H{F!E(G1K@wKv)b?%R8NN&Oj82ta2n01HL0 zM_&gayTI_oVZeu#PsF4mQ!N$fNJKQhzG^$5V->HGSG<0`bGnk2ok8)q%1oQ-G&u?* zit7XLzfr>sWGw=vb_#$VGger3$N84AqdqX26p3!wJnGb0_Icpb27R&L1cR>}=SB+! znSFU0Y32x=bu|CHrupJ0+Z3?r9uqnfRoS|q4uVqB{MN=Lw(q?}129IiE5`rpAqcF{ zeDSKS%XkZoZ!7^(Re|kfS4W@r4UrVq)-Av9-$LX@=ihldxk07LXWU6GqxUI;0xkwE>^ z&5GZ~%|=+H0SxK8j~N_?B6FhDtHo~hm`&lPjCOOh)j*|c#=mU2*JkV4?qHe^7a1oV zAN+HW0ipyx3Z=3eMxe-~^>>jdH)g=_cz(L43iBN;waL9Z7*Q~!RK`4LQT6z_VTGj` zN9e8|)A-2a_MwO+U`1+I_M+a2@;S|~x@t>-&k}qtzaYglVtcr}=ZbXST{>)>B@RHK zhn7S&%_#}bQln1r4&%x0t{B5g>zOAltbC}57US0z%&4(LiQ6_nb;YD>!WywY)ldne zdQS9_ST3P3C3F?A2$ zt*W6SZ6+pl8MI|?7U+v{?KG$4yFZh5zG|g<lR zzPPKbremjQU>#$nZmv12K}FhGF0%|Xo5qw8G$G*h4fSzzf}mv{i0h)813Z|{p;k9R zFx1a|)+R-X66G#u>Uvq<<;OAHm;+s=`@0f#H(!NrwddBGIOkf~JnVA9+S2K|=9-Hf z&8{}#hK_7Mii zSAX}%oBfw1ZhbYc6!ySAY#pdYNB5Ytu10dvwKHtTDNHvhr8m7I)pejYjdWoVPeQLtv0&@zZRae7z6VvAT+%t z@1BtUM9;eYTLPe5b&h**ouLBb;T!wg3#Yd2uAdZuIt7aGIJ=cowk%@s7WdDr12R`a zrrV6Zv6c_!_ERy)#TD5Dv78HKwDCJXuy!CtG`r$)C7p_`rqCvd^X1}=CD;{-$5Nhi z9fE6BjE`%9J|8T;>?EEGPFv_~8d(X?Vk??mpMrb3T3mzbPoPQeoT;@8=Pf9JDeTG^ zl&%&?jI4n0+%aq{x0S{NyW-<}wT{?acI<XNkD z=Pnm#*w&}~%=bw>Lif4CY?dU;%)1P@cHLf=msqNwPMdfwvynQTmdqc#zmfE5_-C(e z3jJ+pu@$%xVK|uNQF=)3J$nq-tIQX_k4{`h+e2@gFo}snU<6Zv`lL3NNxJ(7vHtd+ z^u6YrH5EEk@UgI}rVOWoHsf}S=Wa7R7)UiS@%#il6+60OR-4pDGXk4ySB(`a7&c!> z=3r=ou@0h6{YRi@=~d?%H>fET7RKx zbPNZPpN4Er%2xK|M-Luv&bBUv$#2vt-vfnHAI8jL+f~;_F?fKMAxLh`b(vs`RZS5yZ%4I zzB(?dHEMSgqJUVK7#Nfwph!rAO`}MsiUJ;B!?P0MWj0h7!ag}8oKkY z?J+pt{qFD1KSz%YGkfnh*89|gqje>IhQVPJkJa_`rLDk%b}!_)3Ei9#qRJ6YGKyBy zjKwr45OOpFYZnrWajjCtg6$aQ{g<$+6m?Z%4T+vg$9<{fNfdKVQ7Qp>#+z8rn@JjN z(H}CC@oh5<8TfchV%2mwc|+$DEsChd#>P)dPwT|7W2p)TDY;Z?rk~x{NUBE8yi)wC zJ6|s~5xqkK8 zZX9ll#?1=hPU{#6JHN=p;YW7z?n$Pm4&&`V)*H!(36ctFG}@a%@#vcHpnPg%j}4;4`Zs|J!%F($~teubvl-bvvhhfg>`y<`i=3ay*{L6l9?*aE_z)? zUBYsO+w=GT6oYHv*8@IejUg{adk4lZ;Re&K5hj679Oj{yiG=05*%%Jf9@l0$fd|xg zxO5A<)>!kXF2O-##Ok=pV=Pr2C&SIQ%-PQ6FJi?;spPR59j4nlNZwwY8W|Q2h4L4o z$(-oeb7?e2GbkBET1iXLr=@oP@#IfK6=#toUn4~U=v_o z8rgg88e+v-H&|k%(yyGo{X;fvO`Ah;gY&IpQNe>&8JWRs-e=OX!PWc`Z}4jd*(O(# zINSK{-c0z!m!b{Tq)88j>L-OLvt+)C@#n5nx(S7RWLJ&)yg52WC5=*aDGCm{?w$k) zJH3ZbQabja#ep^a{mb5S3|(2`Dm2Edb+LOp)5V; zYjoN3g#SutszOMEM;6sr+fC!g1ryxFfncbXo4YbUQ8d`rMv4-cb{X#TvHtRes&zi3 zrQXl3FGWmAsLY$|LwRm;saug|l&NYZmmo7TK>S&InnVT9@md$!+Rx`E>*_un2`v5E zgST|@`|`fX9pycboMieH$DE(Gn?|Fg zU_3r4sW`sw?L>TSj_X^yhy*R~at+Z$?MqsFzt}e2kp$|p@h9$*)wvKQRU01v zRSDd*cIpLKJRH86E5tFke}=NcGu&?k#fbomLZnH&q-~@TUHiael_$n2+u3$yXy|x% zGwjHpYm(6xyWu@jrCi)4ZQK}tEolX3o@d64WXEUVi5L%6ju`H|aUQ}tdJdGlp0w6B zuin}O<1?M}W{47zPEIcyN^gBgTl+L$KvhDb$WnQbNtF=ce`L;$+?{7{pX0-koc-mF z*H|H*ePetdqIH};Xw3|3=iYEKJ)yUVMQfzrQ4UEV|B?JHq5Pw~a`dXS|G{xzQ;aYG zbz@cqLdsrEsiSj9&fO#>`g$$qnqi_6#cW-9E{pZ0qRqC#rtb3zg~5`YOMWOXuyJc)R-LH1oNWi^o-d z&8lcMCPXc;sgni{VCF%9gBXNjIuFog8cvOW@D;OKobu30_nHf<=!@4bOpAX*;t1!2 zVw0!!JHu2uR;!KuZW0^*s~2dih0c9A0RE#kj~O)jotB4P`Oq?JVoKKB9>vD;_-#awW~Ubow#{Bl)gu`(tpQBqtE zGv4<4O|<4vRAX>Bs^%I4yo$z z5YJlU5O4ys;{7e^VU%1JqeJIaT zegL4-k#t5If<~4Z$?9cYv&aCi9kD#4*|RlG92+Is9R;P&n8>?kpZ3j{evrULTeBZ| zETubH`pUg}SSGo8VY)!0#9S3;-jlp}v(IQ+-_vb_C3~_fvWIf*$_FMuudBJ3AHM<; z0n`qP`6#@ps5K=_J*5NKQG;F>)zxzyxT-oISBaf$T+~P%nYlAVFC25?@h(Eq`v{ScH}mOH}69;ze+wq7o} z6n8k-taFWV_q6~0ccBCo{!`lT`B0lqg${sAiH4kO|A@rCz1 z#F=#PD79is+~hh_hBxphb6v?j>g=(Vr^D9nx+ga;N>O~zo#dmI(44`X_Av7;(bzQD$x{o^qo3xr-~OoZCouUY4n>97T$!_Ejow&=&AXUn%t>TFTb#m z3QcS<^{NFwF0Z62-INB%4W1>BwcuMvF-c2p7dq-WlcSWAmhePFJynXtPUG+Qn2`+g zp6_>Mg|2xO+NDMLUS;|uqh|HQgd@r^tb_3f_tP(HF-;@9BU#M1f>b$}FPw7Q=fZ!* zI8rvS{iN=4?De6{;8zO4Gt(nY15@f9YNsUZ{U|@h$kfuurExUr?t1z#Io(G3$)kwc z*9R_c`0>$c$W&MH$(`4T)o3uSOHRta7?JgT^ilL=pq*h#d_UiCg6TA$2WFc(?zuKG zVUrl;Uc;lZEt{$l_9FA|v`G_L=J;ggo?LDm?ehaE_^V{DjOIYk1uIDnuSR!M&Sf`4 zV_;S9uTEMhJv+2iRllG?y)#U22L2jKV%IIq-+7c@)}SWj?ullJbQXr;F|E(gbh(e3 znwrHSN$G*1gw{wQn#BOfsES!0Df1jP7i{yDWF3vJ3H`10QFf1yoY(VkESkH7um8u6 zJ;iHmp>+nl=~imKwnc7Sr1E!BWJ#m=`>uc_u}GE27vt-b`MSMXiVb6Txa!WX{w%?R zq=fM|U5ppBdAiP?;=Hy}dxH4J{E6?YE-VTjA8Rkj$m60p%L%-s6c3XQ_h4K zDyn*a-Yq)!nlofn7OGs2cu76Fwb2$zY0Y<Q#Pw-$6esCrQVHN}^Qc~u5*m{5LD5ZqA#YB9Y~o#-yrjFFxFXqPY3_GN zJd`EEo>@jK8aw`ME?8MvTFPZ)R&vN{I<$2&!k$&8n&RQ0tP$h1!l4cwE(sM?aB1rZ zZ~M$1d;7uM(Mjp~D&?BO=2THsvcCTMC`57dS!ZF$%b_O@W}hhURPuqBWCsbSCro5nS4xgX1&;nn!b zH?Czlh1?9;czk{R!YIEs=x%tdhh+U7nsTfk(r33#4^MwuG}3dO>Ma?%l(Q${vbOhp z81n9ZlkLcK^s_T{2T0O{KFnol$R_Au6m;zM+^#>`68Gu8he`(~mO@kv3pD7{>5+$W zPSh&>?1Yz*oiL5F^seL7XCuHs#jXfaM4 zMdc?zsB{bxhq0=r?vY$aG6^fR0-|TuyIlL0vcsKyN=^69npl(irmgWryz9I2=Hro= z&10%#RZd0g{11}r7qs$7HBOpZ;u9GW9X_>p-*TPj&oixEx4u|6Ni@l9b`u^i53m)s zniU(gG&$rD#bX{Q`#^e|0o4*;wz+C07SDJ23^Q-<+MS;{<#4%*`?7K4QV9oer1Uy( z)7t*Kvs==X>s?t<4ix#+pJ5lKi;0S6lz%qp#&;ggB=2>|ZE2q`Yk3$kI2dA8fZuO*%v) za_FkW!dm~f^Zev#5}vd(v?<%>>QyIi5IW~?1nn}wI(OJpBU4l$C*R_LC%G!vZVbbj zPCCzyG?SmmLS`sz8qKsx-j>r!()2q-+|1w}XYOS-Xw?7$b=cD<)s7;RKlM!{0@G}S!=`ooE zbUY}7rKO*{EOb0H@EOZ*aD2x%g`Kt7TrAqOB`hKP6TxWn!Y23=cyRA&IKA)ab~ z0j${O?W%ebUF@$xj1aTe-Sutj2G>6878Tlzwn$Cn&&qccy5>PY zfOr`zq?~W7`_8~wdOq^@oh>HtNdIthSxwE>3F3D{l#u}nG;ZQdKYVyAR{xhr! zk>@_&CAaXuU06f5dp{*G^Ru7EVrZNq90{y%38xE(bW=rjiAsy}(>TqiSVte<`keVt zLMVPnFmu()t?RC1#TT7@to*FsZSPtzWYwzN0Guc)n^K#n#00kU_0*8%2R$<#P#b#i*e-Kdt0hX zfmhTJvY$AIZhvSeXV^MjI>TT>yK`rN)nJtr%wQ(o7UzWLWaiD8CatICkl{Op zpFj}Cm4@H$@m}%Mo!6-I@nuJDMuV>N%?SQ|I_xP8y1#jn-Y|j_WegBlXl4L0 z|L;~9D_A4;mDbnS+nt@5ZPYaV{DR!{^R3&CosQ%avo`6GO)R;_@8p(e?xdf1Jfo3) z309ZrDa)-=ZSE{NGQH{fT34SE@JbB=$$***&jO96S-#DIoNnma_3O`B-_h|YW{XUV zkS$LA4@D7asH)@#YIhu(S6oBGlu=R9AutU8mN=1)BUbn8!Ucg`z8I&R5CrbNTcGG* zLL2PBss7M>12VAJ^fOuXaLdjYC;}aJvOsv)$##83=-1_X>Fo}05l>17*Ir!oYaO@dp~f&x^75{=+OlsvfL{JA)mq9Jqh z#fqp&+rJ6z5GL$Njp({eKL!iA6s)yf&@3bEZYfo$xO*XK8r3GF-C&^h#&zStYT^i8 zTRuXQ2&>*=vVZNDR}nLlE2LzaJG5{G#W(<0)8?w~&RicW+-LD7X~ou2yWVf3^z$$0 z3Iv+h^+a&!;*0}xB1Ih2w5)6SAOQ)sI%vUR?W4iS$hZt=VY$m5gRo%-K{fo{W_yDf zIU<6kn}>K=%WAC_5b}A@vff9t&9zk1LtV0`&5vwOO2f2>vT-~G{ER+~*BMah)z1B% z8XkT1tXG^T@}6Zb?1mWtT~97l-v?Co%_q6HVL_Xt;+xC9Cl5vTZ-#~rLGjGWUg)Y5UCR}S_qfPRlG#aIwQnz-2wjg4`%#w+_nQ^aiR5Q)@} z)I2LP?HdmjOs}j@tT@h8g{k}bxw$#dG(TAh3{tI70$W~{3i90&zEp$W1|(S3T#JyG zLL(Oa!h_f$v-qQy-an7F?U`EShbiJ=&F#_m80`4Jg>kSGySU`X_1b^pTnDwDbMt~O z>MsIqefXeb*vS<09x+gP*g}q1_7QX_FE#3klezyIzEfO)4nS|$nG)R4w`BbEerKVlbAVF# z+jlO+(*4qf!bkKutHRm^>AE=?mx-cp*5t*qt zGnC0fee$B?y@851pYC6N#lRN>WjC;f`_Tbiv!BLSbRopFh`bEFj$apmAiE*o(PIu+ zUMAPog+P!^_#=AC8<_K}p4(5nhGpZS`UDW=f!i5f6n8O-nM7*F${#;FV&mRzi`V|C z1sc0>|Jy%B?n8uFsRIfk(~qF>+>MRN%?<45r!)gwKg#1pFmA38*+xZG1!5Z67n@-E zl>vbvwf^?)+p9re#SOzZwRAShWk$-VJ>}ib!{;S-9lNBLC(gulRXTTbW3h;3NjJ~V z^oXca2Ims?yW6U1oO=4FjwqPs*WUR2Qv0A+*8BK4a>vl{v&+UZ@n_3S(3gFLD1+Ns zX4=#=G*vTQ!V}r0TD^v_rJ`EIP(ml)yE}2WclmCoq^tk}O6~ZeM8GI80fc~B1D~4pkxu&-1k`iO|LZAq-5>4VS4^PB1Agq z-dm&>CI+@}B|g9gnxyh6ug4qIh8YxP_YsM#unAtwS{&~_)EHU3SDwI zPep%*r89K#A~!iX`PZHval$;f*9I`>a^BUd-!=Xj>Gp4e|Bl`QVq zXiJ&O`UdYP5&Uw(lCEyAqv#F&#m$y$LC79PMs8U44i!YA&HD;r88{nfH{O0XPdmqI z@L1H*y?cLH<*Z!-;QE-b-8qHoOGQIajIe{VrKVX3ElK3UcrRql(|%Z&1bNti%XTJM zgU^Ce#BMpnr+GB(wJOX;Kd5GN)~;9rYGgIs@XjSkHBVH_Y2@ROorOip6$>-iNLi#< z#CfF#KzN$wN}ax$hOK0ID_~xScRF+C%wQNM*>brZi17#Ty%xA=uZnBkkHj zZk=<>FeB|l7b^vCCLQPHU1lo+Io3V-3a9bT&BhPqUtS)$I8&h0U6dOYF_J25+L_^k zg|@AQL)jMd<-O264E|BnZT?c*`dIj$3vO#mF4$n}NBx4A>#3bm#TC#dTSJ%XcGvAC zTKZIc75tK3bi=}@*ZI-o$-SXVCw{D0g_s5)NRNorRmnTY?wWtkhZ}thamRF7mLF&F z#IIb7dF+ce0t64Xx%Xm3q1@Bj49lQTP&@8 zDz0jsp|^bX-WSsa^;j8}Lc;Ch{v_@*xoaTy(ev_iy3GCzU6JyRGcbhpTay%8Kx%qu zJ~zs3B@S97_q}g>qePV;>Kdi}K)<}KOpsD>UR=RSBT5Xfyb#azA+)1|L@(Mljvr%x zp;^?D8*k8NnWtfH7g(RGpr!m&xS7;2Z>(4^mh!Mj&hdxw+qp;3)U7v#Q!q&&Z?UrT z)A$0mFmv6>+9^tlR6gD|HoE55y*ltbSv6{m{c|K?@ocPV)9QG6ved0ZPCZsa3-VE7 zBr91sYddkzi3v7a=5&^K$O-rn?ko6mYGYYg1nuW5s#jw!-OyS(cGo@9LM(QWiug|= zdXtX*INoyx{!q5~kF1L?+^UT>i~1ghm8?WDfph21Ro{mp-NO*MAo}HqJI2B%WlGfR z=L`{-*{E+??PKV$t^46jY56LA58lFD>r9c>4F?)U9WTS=jovo;9yCILIndRktu;7) ze3%R}7Jk3j1iK=UhD=k<<_*n+S4lzrg4z%hcpAi>0@Z4ZqL9mQ`*z2}huUluU)|iM z$A|vy()#ba9ay7Yz8GnaqA)#@K$>8Y4`+kk=l7x0(`W*A0K~O{Xs8lMthksvfc@YG zg(Y7Wy6`)Kq%Yld!h?*WMobBIr400-vFz3jqU$QqnayO>0DYj}TkN^Zx{7EpO10|P z&R1Dtd}u;s_Y;;k12);YVa`=yPeo%#c7)@{jO(m8j+BMMBTswGg-sIG!ac69y-}xu zO)za*-beBmwJPmo(GKQQel*KJB19tiWOuTtlDrZhkW)~kn{M9g8DffEtZc;?Wa+4q zl+v7|v|#tJ%8?WD`TF&%h}_+KMX1V%hJMwwiSX}Zo^!Qs4bo+7zJFR^(jPvhzovE8 zR+2Sj=}#f;~H)2w&%+_Jb#AH!EZtCfuk!7r|x4@ZoRbKsoQ;1NLOje5nUfufnscY#0_ zeUK^hKsT(B_S6pnchK$M1@xjr3(zT>4ZYc!^`ItA06#}VR~vC;me#*ecVTk=mt_OG zG@19WF*mn9+b>c1&5<0Hd$Z{*xly5bcrpD{!2#e#-jYisC_jV~Dz>+%DVYqgS00C{ zUMWBldfcf-zxQ(S#mBS^m5N z+cA~1amM&$PS72k&k`{IZZgDT{{ZN$Ovhpr8yynZ{`ZXm$I95rBq-X2eAfVZRp`ISm65t-TYkRX-9G7(n3xW9cAPE9_x1HP z!eGZ5o-g>BRJ=Yl1JPpDa4!d-+zv(yDaX2ivr08Mv9|Pw^`{U28%Or@%M@dT5kqA2 z9#`kL4b(_5JZyNja&3LR){RX$J^=NJWSaL(Of9V~&(;P|10os#GvQ#zr7d4YM`W9a z$_z0mDk|#gwQE1a%$BBpKx~Si&bql#<2Tz?mDY3FZ;<)&*E&y&J=rLVRg8&WJ z5~p+n>Pzl8r~z*6KJfSU^vWqAY7G1)Y*4iZqvr`4yMd_8MX11%@`3y00ST?3)xbY3 z391UH?D7b|NytdZ-?t-dp|E^R0Y-2W9N^vSJj_p2R(y{NB!7Oo<%_eORYH~k+wi8M ztAW^^KQ9Nv?TG1(A}|uzC;dK9qyM;nXg{)mc2$Kz?hne{YH*RDO%B0f5aY585x)0t zj1aQ8o45Y1*Sp1zz^l&y4wr%8E2#o4G}lrWh`+1dH&=C0)pXv|Z30XVyr!Kk3$wp1 zOT-t^jP)5Ayo7paPz8`tsn|B?QHAs{Bhev!lb5!@c{8?)w25k|mo;$4(0@TC*;D7C zeW-vrSc1CL3zzz6?RV3$E%3HaPEMNVfkXE9Z*l1V&u>9JIxtfGQOu#zpVib;reLBq zAsAKtGVmIng!%++Bg-lu{#x!Wf9O?DG=CUuP8jy6d4@mt4erog@?uT%0%q_yvkYTf z*P%=eM3EB)KH~rqk9q#|6X=r&es|Z!N)T(rK-G^5z0u5+zb{{6+gSwdB&s>+4v31Z z^Jci$X3(zVLKR>$scj)c>wMv-|N0|;zRN?^0<#@;{JyPnmm@zcY?lbODxy;d^m9c! zC8K)zg^6ws@Q~UQ6+Q2r6&2Nni!l-3PDS+`i`AU9LmUo+>jPa8roSv++;VTNFifd1 zGKX>@dASLqg+;h^p@_FekMT#ZZ+G~{-TUYfCE=Po6$0;H;vz%J4!M_aNxKc!q({s^ zF=-tU78W*X`?b6rwHDD>-1&R8hyx=;6QOJpu*;mNHyqqiBHZZU>&P7yQIFXg%4~}NK8(<850V@EU`g%<7Z`nkPk1i4IOSaJ-99YSj%%Al^ zjoSv&9-w;{P^~uXD@Kol(7Lk;;ODii&%k5fsUB(sQf>UvttM-36Aln=$fsPJ2E z-$+qu4CtP5TjJT0k`naTP*}KJ z$5GMrAR6S-+tPkR3g}z&)eGqnF!OOm2;2Y60si`op7z}EZ)gSmk(r72?-x)ct|eai zMencJBJ>}S$WAf|l#cl{xb^=|^-_9xcpy2j)Xd^cZr}EUN?O*%#Ub~q4s{WvA$Xjg7LGq zqb+A^?vj-Sx$&PqbLNH9VdVLR|9xD|aK1bwrJxXu#+HMM5e+^LfO#baEf-0CU{yp@ zE3&>j$)F1$+E5e~;G-t{KQ9>a&g8`%=?@2U$CCZw+|b=s=M%W|HRR%<3nsY1glx7P z`Vi#WrESG$d__*9@*C5M88{n_!1T^AZmqJR+_8IiFw7MxS=oS)ki+zIQIJ15vZDc{ z$m$@i08~BCdF#^Zz#FxbGtb^1`1LAOQg5c3sCEnOIqtoEH$T}XZ79EKSSU>|hz9~t zC|Uc;RkBV1d;Yt={jwvV7c@1u3(<9*56mf?sp1&Sn=JHiXlT&lcCO{ahm0U_jyR2~ zBClS(nr#^)F&z`-nVo%!P~Uk0;7RPwa>{6jfAbKLpBT^ItO#yh`(zh53H-E&2-zA( z!XhH+O%+Qe?2*<3u=8q zFj?IJ(pRE<%Z}lZWeGaSKvkWyXc!`_YDA-)fdTy^VlLS)8Th2v*4AQf_?|F33VhC= z*Y{%hcyvC%3iB4(02G9KX$_v`GL#ZAfCuZraYYcSf{7hFYY7(5pm{{Uv&&aT))XuE;`Y<1}B?~x0> z>p`$K&U~d;2$9%?eJt)_Gh|;_5@pDA zSUEZ6P(BORt;`o#^LI6!WCD$G^84uNXY;iHy>rNoiD9yq{m?t&=s8OX`5=g*%YO}u zE?`&oITs#mzwJ z_8%kaMNujdJgN+|VQpLO1f&OyvkE@d>(_LHET_xJa=4&1K<4hmGcAjkp?(ML!>>obBZf-}G}1i&6&G2LG-cbb-3@6RWl z2uB)#c2!!(nqCFNak>b%`^ub=Iv~}W->CKrh>3}@NEqXm@i9l}kkWLc(>H*_0eS`P zK?^fu-m7}pQZd)Y$Xb?7VLawtd9qGM`r}x$Kbwq4EIL`hsf!NleMgQM#p}(E!g5%P zxAI^m2pt)1>3diT97aLl4Hr8ep zIv|bXO+ldmlkLDq8aIeV^9u-ILa{MzN|85qmatU&25{@@K(4)NOX;*m*$DX0mwx;& z=yZ{=YN1{Ux%E0I{Hg$|H}ld(>!v#iDxBr?K6QsgL1+MFjZ1RE12S`q@%9U-q|t~T z8*!00B-XlE^QYalCy9|@;=j)g#CwjTaj`V(xB^d=lCd0FVPxRJ5e4&zBFp~k*7C6{ zJp~z>S1m6oAZ(GSeH@75GBlzcDce@$k3d-LjuNn3nzBo9GVI7DWfHoU{XA}H7P%GV zaB-0}Lt}WQWh}MesZ*zJ!#H|>!mnJ=vhR`9c!LicoTD5g`HP$B5fRtlq&#HMK7)z~ ziU~0^wj;+>d4`zDcOh_-%`Ag?6{>sykIz^wP2GLwAvmC)RQ32Jf}jZ7$fU24_o=RPw~X<)LF3oce0wJ55ISh=?> zKobd23kC%Q>|ch}`~l<|8n`OQ-+2SUhg~M6uLQoc!Gs^OwN>xdsikbm85UparHBLW zQOmYf84$Envnwki-Hf}%mZIZ?+6Y3Qpwg9sDEk4>8`juGjq-Cr&k0yIFhcIbGk6{6 zBwh$sP`HnqyfX!Qy|DR5j~`3te(no~Vi)tuwRpr)J)2Lr++1ILOT<_xd*`l%O3TX!!RnN~GlgT-BNsJ} zg5nfzxa=O4gciun5uFV=pootXfL z8gpBoD&1ZCAOE5A^z<~3-MCuwT?u+hc9lfK$jvXb+##!cjm0!HM1w@>#S!$Ri*98y zTsdDU!-@cEzxa(c(&03IGM&ZP&_f%f)Gcj@8>GX!tk2`oFaZBYz47#3g^WTA=wYl@ zpvQ!~5rwGw>$%8!LDS6RG-sGz#OrxEOM#xE|Nz=n0mOdk#dLJ7sW7(=yHAe zX9uOrKTuPz6pp06Cyaz6(fy764nMv;-bIZpcd&w@aje-i#mAa%6vJ}^wH@Amp<^RV zqDRoW_FJ^}d97`md{UBG>i*)qw=TN}4~wOcK`eoVMb$19x<{c>9jPQIL8e;mAdU3e?_5j$^o#g&O?W z=Cj+LPeQ#w041d(ofjreHp{GA_O-J22k8VnA6uT&yfHm*t49KrDsIIYs)Zpz^h9?U zOI5{p606+Da2PiX7C&s@AcKVp6fI3~deYoZzDhKfXmi|f>NY4aEX!j-3{zCh!C8?t z0}u19-V~fdWgFD9$|J=TguRC`;=pfZp2V`;RC^}bW~=ipzP_2LNr=ZL9ryu-#>L8= zIrH#9Lax62>CX!bGmMtBz=$~$*{#71_GHXCX)@!fwT!(Uy1@PII$Syhkbol?uUD24Eh)r!Qvd@ zYicQPN%4?)``9O{cI38 zCpu#XVCXebKA+jQmPIbu*(8#sM1?$-Oqk>QoLpi~S3GvjJ6}URiByG?D^l^oI6Kj4 z>^^qse6>)bu|v|=%q+-Dw@#9xR3SL5BC!Y9T8&-kGL27rQhDP(=26wm<48uY@icXv z_FXAvn8%qcLu$N+lgF-R#%HI$Ez+ULm8NKg%8jS>hwWG!U*PCUXRa7=VM2o$-Tr5_ zR%y)X%rI6SnWQYQO2;D!g8&kwb+-*r$hBC=U#_V8i8fK9N1rQtprlk0s`V~_wlHDE zHh|R}^V*aaKE~#1K29;v=GH(eAVxZ;-N_xZT5%yRw!EiiQaSxxOu_pHgKRyW9C<^( zs!vd{xk|7^Qe3xl{fjc5=J-<s7$S1K zAGI0}DP^(Ohe1y0TJtM@c-eX_p)X+{| zMQvLKMS~9$A!j5CD|s&|H_Pz7q?zHp$C)|iU~VrBpO@CvB@~` zE|_g$QSo{-yU~1b0X(mf{o(!}AmG9HlWW6b%SHi&Rp?U8trVis0d||+_B|0F$;DY! zyq*;XiOO{~L>J;-!7?K4f1UT%b?=$vee}Fi-<112?8+Jjw#z9tJm3`C)vY5Zk&?6(pXy4N#VIzxJGPTcIi@H9j4aAkV*eKE$k!Ooh~_naf`7#`iN zN(hiOj|!KoZ4Jw^n~HO;Q4knh;KUSvMX#$+JHJWVj-$eiG1?rKgCH79x(3XAfwhH- zQGydmG1^muE_&2j+fupAbTfp?6Id*{lWm0>-O@P$6N>c7=?%9zE80bw{_n9AfdQb-b=xZLYn+aWZMKy^tWOqb9YTK`qO0Un#I% z1E8)g`s{jd+tPfy1&(e-Z1kZ-aH;}(!~Ke|nX+Yv=Jk(KoZzWAzgp`wfp+=Il|)+@ zsOK8|9*#)B)yIC2&`ry6GP$*5!P9|QoJBRD&0JtuZJ+c~Wx>;tBehsL<$K{hjo70p zy4VD1;|N`C@1{<9ZNjv$>KhKaLb0i03InHIaw1ERnwBndgMuWa5|Jk1o!DWTK=lUj z>dBOFmE|$3h1u_8YV)Z8pZK%AJJ8)Zq0O-f{Kz{qbKK` zGEs;nNhpgk-SuRba#^mOry@g$9o!Do1mtNXwx$m+-`8&?(Z#^PyQCmg-_M65rs05m7zHD z;zOT-C-Rcf2ZoZ#yM+S1#3qi7w~a1BX%a%v*fx2r0!3Y_Pi9Z3ek72_VM`TgCEdMy z*C5P$e7LZgX+McauRvaBcqOn^PO?0v5&;yw0EW;C{k4$Lo6;jP-IhajDgOSiC8c#OI`j)y(x2?^nGslw1J2wf}uS7`=Cp01$`8!WRRCs8+ioXesE-W)zoZ#!NKu@fkGFt+DMS8Itz zT_J}@6TWd?qq?2-nrtXf8Pfr^3@aV)UdN8>s?t(-%|xV- zYL#S!%%M)3**MI$bgCO*IX1Zv*v7DwVgAfaS$q>_bW^S^2DRIi-dFLsqglfW`h|JS z0bDYiczBoMC5PjbV#A`d#d^Zrpe-n8v2l@Es^1P9EsYT~T{&~o3&`aUkv4z9a!T#M z1jE-hTebd-+|$GLDguZz!{Ig>hwekLPZmD0zTq*Rc~-d14Ii_eE@RBjkk*BV6Z8zdnVIDp@*L}&xdkfsJd9gDzd2G4 zK#{30rs=ULrfwC8pGuCTG*yQ1O-QvK$RwCmTa6Dsj8dyt)mX<8K9^&;5_pZS>ZZf( zW|6w?yAEglYRKp?rREIV5uKIP&j!dCkWsayvKNOQb{stZR-W~4(dM7=CLKTfCeNi| zp&7`(XK@m)aE>q`J6lYE8ki%B&tfE=ITzKy85GXsSQEp*d@qROas2X#8qENyf+HF? z+;SVW>k030RT{#`@R*uuEuUecC%9al!$#d7Fvj56zu<(lxEWqTI?VuuDKS~O1vRjk zF7OSTY-{CvjYSPC(xyp6U|`8g#d{>Ftc_Xo-b_&1qsF-erEh1J`D6-br;&uNCPmHD z-HiMtI;4>3j6O~8AGwU;E5m0{+yhOOc&VDDoKT4#b|T*rQb)xa@D1xW^`eU*>q(Ud zg<8GLtf5Vb_56m$v!h`a)2L%4hKiJV9-N`5j?ht-q6z9o?1xnqib?px;>Z~6L+UEY zT4!NnH1Se0{1#w z2;zmVeDgLN8c_T|UJgh?GE;|B-#;UF-&l?f9lA`|)GYsWA~tt-B9o`z%|)#14QitP zE`<3g5V5^ov;exU9{C-J&ILrKrXogB+Nc);fa^9su4uK?__;AWHmZM?cgG^I6V)iu zkHQ!bxEh8et29Co&{$^M*aGiKk$Ycvgn}16AAmy^G70Z?hH=;7cTdyg5*y_y8UDfd z#!=-aN#s%4qsz0|B4AN%Uf*eI+~AP+Nm{%76KAV1m}i+4lS^6;a%zMiSe5o3P;9fb zC-UlHO~3OUHc{@>oP)-;hCJRZVruu(L}26lbjDMj{=88WlhQjZ)*6IXVbf|>!YT=Q z9zIImEuGU6ixEDg(1($!q^Gnf7&=mO9$=S?c96Nx^BA6ndx3)YyhfbaDm!*t-FwZ512dQL39D5!g@;bq~p=5rp ziB~5?JGC%Vx`aEJ1LyusZNTtTC=t_=M9!z>FOS^sjK~XfSJ&ishooHt9g8hLhQ>|W zM3Tx+ILl4|duS5%VOFIFGNYhzLZ(Z>H&Eu)xIl#4$EfuG53lC#b|?|l{{R9{c-1$b zC7&6CgE7cwJzcnWJ5+P)$;93M_!TeE<0y>e?}0=^RG<8F$imG)6p9yBik%0A{T$G} z5sn&V5fFbxY@7cBD?;BO<^qbJgbBd2{6{VW8Iu4; z1_R$L=(d&(MZfx&FJEYk|AV^Y$Yl_VYYCgP>9YFzF=&s6NT<=WSubg~JYVzvHcF2d zt$^2k+M)wQqCe9kUS3}M6J0rI6qpOAd&h&|O2{gsb%9<3Aa)f3eLu>Qj5egh0N{oRVcvA{=rIEqw(4=n=!pkmX5pWpdlg6#XRTPHn zRKu0sm8cR2ZAn}8^ggTpT4RK`e-@j)%bEY(7sRxG%re52Gs4=+k>*v^|)~0VPgA%R=8s*Mq8BT2ib6p%OX=WW(r8kPADoeR@}#uvSJ{(4@i^y&a(ap%2)z^ ziQ=RkDR;fWM}bTE50_nTy$ZS-pv(uM@Z=8$jZg)g0?<*1QcG~Rf}*3NzZ}~tgwXk* zNRNy^e)1$t9)Z@}piTy~eiummVXe<8v!_PbT>fLTOQ-&3LMw_F_Jap>fhyNMv&y}B(F$m@caFLz4G$fU22gPsAHtZ^PR$ajDS73usB zGt54)XvlhRqIslIXV5&-C|}M$nvnzl(Tp5Y+WOJG+u$tw_x1n#Vv2^;2vBxE74r~V z$O02IO}}y<^tl4Z8KBz1pa;LT>x77!TIrSrx|jo01uj+#7Z%1zzx;Ua~!2=-BwEw>c zb3@Q(Jj`^|iU14eOIVmWWmerE!XU7MttFILFyE$WP_^ajA!z6>h(>*Z+Pns9C@X3a z0$+@EQj3{J&0?ziSQI(T>9N%{QtFMS%HEH*4Aq1vNN0C1u7Gx z-0f;88(e&bLd07>^am>68nC0GLzuV!@AV8UiM?%_=7G?=u@ZD?)&(`R-1+EQkhx6= z_P}%SrS1FKwv6K`7g!athr%#1&_w_>E~7EC)-K6n-#^M6a)e1NJ0%W_jF#^)!P_pb z-9lT%3?h+5dAfSAJb-ja6?kZ0Uc&~7&P-DbISQWSxWKynQ!S8b5{in}tJCVmRuj7% z`d>|)cz>hu>py5*F{F@ai6vL=gKnsS@aKao?OX1sPB^mXzyZiYJ@*C*Q?vx$IUy(|?1mwt?s#efyJa=$FG` z<&MD8d;~dI$3zt1>uC+;l#3hW@;7g$tZT*JISZ={EHK6B0`zT3c-NMcVj5<$VpNLa z2)D*mJW9w`8rDa-vLc<==@95&1t0YQ6bV41(%EgP_o5B=A>l>`gabDm6E=!~fQ5#a zfA3P-RN%dPpcR6B^L9%qTP9kkzpVE-g{XdHjV>CE7i1JA@_1`#71TkkLVc^xI|Gl>kQI~E^IrbfU^UPYXC~7G}lc(s0T8}2@N`W%1 zt*GPM{ApLKZ~TFYi*kj0PMIERQZ_vjyiU})`H(EJ#8b(T@}ha7r95TRs%xs&`IceAD_HI2rV~a+DPbcVNDt3aHZd09JEwm40$StDu@KRyIlc+dNah3$xXd+AnVkAN zcB8W`L63{>*MR=tIsp+iBAl_L-5Kizb*{u7BBY*rpz{64e^kx}I`khq_v?nQxU9Px zrE1g;ZHr=EEK&$xytMG|m?~Kmh5H!Zle6N~`L4c`WnH)~mlXH^A9%4x=Lbw@zAaWB zA>2v(&gvc;x8wOW@?tbn13dAsP;)&T2d!|jpu{vXMblXfb~TjWKvhgo99rYg7kokS zB5)K%gnnSgTC495*s78`1qkoom&ZMjMx&+zRGbV!UY8ffJFH8H^$>4tLl*)I1NGY2 zH9{}_T2Ts|enZD#r9D(EyhVvW#k)Lq^tB9}MP{J>tO7_=&SM4e++a}Ih7dZXuO_Sm z>coxmQ3u95d328MGwjvHQczGlfNnw)$KK;_%E|HJ@mkh$w{~w4l=YfYBco9C^KyozBZsey?$2kXnEfEqNpGFo0E~yK##l@@D-W) zCmgm;7U5mfcsAdyQ21VE5lgVUdA>f%hl2oCA>W1TsBn*TDlAB^nQnp7_B77d$A4W;+zN zMS#ZZD8I>TaG?&^Z!K;E*p}syY_JwTfDLwN0ZMUwxN7yti_lQw*{fHzi@ks>fLN2Y zr+UkNu6TREnZ^TR+B>}FKnczjWlb>63N23}4$g1c%#p4xGCj!ent_vLC?EvO1N6c5 zXok!H#3AsMtbsLC3Z@Y=7+2vL)mU|au$TAvw!~dqJ_#v)rGyMKvx1!}7)7Rj3LeXS zYG*Nq+dZl2SqAL?*>ZmZ4lhTNpUj#l+H?bDRvkVK#8VI1xBLLo)j@VLi)z;~HgHAM zK-`~g3PGR$J;?np!=M_@ z@}*Dc{tt2Q9o5wObqfa+6%bJbu~952ML>}v9YsY!rAa5CAkupiA_Pz=7Q_Nr=tz;? zOX$)B6i`a&y#^9MAfe~3eeUlZJ@38W_l|GeZ;b0d$1`He-urphv({X5&h-YEBkg^h z;I9PiM|`C4&Ig;qcmBZ&O8*O1kRP!Eu(=ri(4j+404V|S+I_O=_Yy&=nLzFw101A@ zp89$vL{8EZ{>Qx<3Zh3r^^NvV{31F}C6-LrO@?lDAp z3)5Q+9f}d85AxwD(lAY}KC&6>>Uz9`0|WPBv4vP0ng@IT_k43i#{lzb_q@M+xD|25 zySewg^jaz4IbY$B{bmbwkoz9MR{r^uIRrQ!fEEbC_}x|@B(viiN|aa&!d8f!9d51XrbRlD_*+v?dXJLK3NsU$peu2 z6;i&_&~5qOAQo9T)P_99omO~!sortJJ)G<{NYUW4WzjgUk+&=MaazOS?n{jy8ea=u zB5N=_IkPV=?YjE)Al6W_m+=I3Eo!Uf-Fs>0xjrw_5}2cF=E|%)N1bnp9Gr`AhG*2n zB+(a=u?i*3=mat=D%ZC6pUX#~0@=Hzw)?ou4fGQ_)2)G8dxKJ1&*2`CZ`UtnQ1A!j z>GR+kAfq~6m;TGKp6HFJS6M^vdV#ZBl?iP z<~aS9+ym@f!h%9dkrGJZZDFDF$w?FQ+H3%s>x{YhC{6!Qm}Xaqh2BuGG${)hs&$~L zwwH?*t~CWP$qi($-GH;^n7ICNN!+RX|C(-k1PDWnOQ%nsEK=g0D+9c_M{+X46sm}( zS^hsZg@X1Tnk*RWfMoW4&UCAx+e0EEZ0e6XKL?WKchJ*VBhx?9b+f;CWpL7RVrI`_ zplc(Nt3Qh0Tmu={s=GNIb{qlzGychwF$qLNRbP#!dtbcKz^+sr6X2RdK$qND8J zzn=&M1n8(~cLPptyZ_XUEW~XLIW|55TMx<0cy>*Bp?>_xPxy3rZeLIJ4wM?eibk|3 z6Gn>pfB9HA>UJsbxBz08FSqberldlbfCKWU3<)0yH@ZMJ6v?&vvk2`B>+2^6%yP3H-*`f*?kwA~?Q$Z>0MylR+n8sA+GxPf*P=VO@1e(h8AhpAQFg9Kp)tW^EGgS0yEZh63 z2nQOLHOy6!!dL(CCe&Bo?LLr0|B#Ja29PmP@+9`47TYLB)>{^d zKdAreRD?XDM=`Xs^J&y5hn=x(Xn1e%$sg4Pl|OWd1O>$ZK%9K4FD33r0YYdB- z;0q(WBuf8vMRnVKlA=5tA>i9}CxF<%?$2$=?b;AVk3hL*Nx(V2xBe&ao>eVNgHj+X z13UnetYYve1z8#&3_U6kjigIbv;ajq3^LRR13Y*zaR>`dD<{l(RLxmV z0WqC{h6YQ(Zq)L2pJWGN8kFQq96Pe;xP^zx*#eE>o$0lTWVM+`M z7=Z^UEU1vhWDL#k{d5JMKtQ<DET_^>kN-u%E74UdW-uY1>evm7K^m0D&oObS} zjREke47NT0R`4HBo?Y?JJN3Uj+5d9!|F`ct6m_9z^hEw&@A}?q7}9h`@_iF4tUeTp z2gRK^^sgv&?q7@JzgsIi*ms(A0qAK9x{ED5=pFNFF8^}-=a@hM_LyL30MPXn(k?uM z6<$~21Eh;-Y@|Ha8y4E%=gGl;|HH(ZKkobg*^2!C^Pw~pm(ie(tNa;Mxb=}p-yBb( z`yDUX=^^%O&Q}29U0ikuWpWViG%UYB!M^A}@5Zq|t`YU$1(E-<)c)r_>jtR71`>!Z zJfx?EfV=)Ti@s&j+4zfZ=0Cfxu13GEvKB}*Y75Z%CX z$g#z|!o2NLFLJm`eRP@mpERlNUuja@PM@PAi>tHM%<_5rPF(E*Qt|izba&A0ih5mF`(CH`((39ng+xFb+mE)%T4sRh6^Uy^7GXj030UBF2 z=vNJ(eTw9p(@|L9E&&!G5x(64@R!v`RE+`~y(7Tccb#gjLdFsSz?=g<1!fC>E8`Vh zm}0>yX~M;M=1I)B-3!$9lv=P$Aot-}2rCL(``iIqcR$Yk-e&6EF_*(o;_F9-kHJ(k!!TRv|J>wHS7@lvxG$~;1+L^BU;t#@qUF_2 zd&eeh)VCSk)_@9<#EI(yd+MhvtJ~awEi#Y^;|+J+0P;P^13zn?rci-*YXeotQOo~) zsN^LYDv&68pf*Q9b)V`1WEn%bp5aab5>)+i$lEv9P+#xDM=0Z$0I|u3V!31zN|xsr z`m^gfXSU=aaVrCI>H9wA8V0RhjZtJyN%TcjpciDlknD07IAC`SIi#Lg!hG`XOKbw) zE}wbD+DR7{lp<>EQaJPU0`!&EqTzu`kGQ}@sd;%o6mf{;=TP6J8WaDJq!KvU-)Ul87%ed^FCj+~X4HVDWv~&z(1UDL#M$(;6uZv_3`%6jGpV>K53y;+n zS_-777L}0OEIy=l(W45tzV^C@2lzrOcz{|8jsg3U5U~(-6(q4}-^Lm2-}KNx+cn=_ z&8#~YQ-S=%c$@}hB2S|bWa-PKdmZ|%SAlK~eoe44x^rSqqF7Kv{DY8EwqZ|#joQ(L zWM1o^mRZ%2M+s?X=IF zHspTp)rcGYsMRzqV>Y+&J!RY>Du=Ri|DI%Q^o-af-%PN*n^3^w+p>Cz)Uf!05Y~!i zf6u`JTLT=nq{YJ2{zG8>Sy!fgU0z{1SB}#YHB=VwpQ0H8s#wF)Sz7+DzUU7LWF`Nc zJC|(xabjqBR5IqAtj3JC;NfAMQJY5>wAtF5zNUCJM$2QLQhu$;)sA|YL|;%X)+1k< zq5sacGO8k46qQ4jQJ+&MC!Z*Sizyl$m8Q(EHg`yg*%pOyc(_K7EXdHW*LcJQf8AIf z6GQ#Nz1UM$9?Bgua5C26;)~}e=@U9ST(7=)+;zHnzuC(fz9>RqYfH7*g+}Xj)|Zof zd)MC)&Bs@$Ki$lGy3mkXW;~ z?&`f#rHs>chuO>+QlfUvieDQQ^q-mF6?-1tNZD^nshfLH)4tDRRHoLn+Y}x`fOEH| zP8R{)+M86Fk-tbl9u5mEzsnr2R%a{$^MM5~LY;wTrGAwkU9O9vX`OC!k7;$$kGEyd zV`gh7msxufJc`!yp4&0IdXPCkmKu@c^sbi9BDVm&(XI)sE*G(pJ;aOLgjEqycBjj) zj$i3p@nj%j=xJWr^c9_(az!L2`6Q51l_=d$R=b~f=e}0>U0jh!E|aC#f*D0_Iw1Z* zZ<0e-($@>BZ_Dx?Z*f;I*DSx?*FHX6TMc_+pKQ5vZozEr=rZe5ucyx!S&N^mlZ&E$ zKAYjoDSaqgtI4sCRG8XIY)DJfy_zO?I;?;vHEh31?`y55pak27l<}IM1@}6fOe_s@ ztR#!`1z#n$G_Ajl{MU;tCKd*lDLQ(NnPC&2IbKX(-vaw-Jv!5{SY*V-H+|WeN8iL)cDp|`ec_M#0KS*V5rv%3 z3pZX|m;;e%s&2MPilU-c27uN&4@x?1cd zz4C{lG##Je+zFJL9s^L(!BrFbWa(h6*9i={aNW0uz$++tWpf!a8>P@Q%Wzorz=+}D zMgT#|G#@HWNnZ&Ph6Bh^G1b`R$CU!0CyR!c!CYsA1l;8me>nx3v;zpcwdvV4RG=fY zhjU>T^$%G*U5XzkU#iLyPu#P=yTBn7rlS{H-AH|wAS7n+q=HMAYt=6xCg>SCGzk`V zK|kmn;X)&LO6HpeOV-^yGP7h*M!Ug426n_9NDhb$u z_+Y6wR7%Wrz-w0X?M;e z@5Qn;LMnR3Nw_P?=P|CdSpZ2bfZ^zHG4xt{kA|QcP}|J2x1=j*ul5+xML|pH4j8J% z!`a@y8J5Xi>bFG*rUGZh(f6}MyNP@%E5J7?+UW#+(gcs5$8He<5P6zAjrQLr8{}Es z?f$IbOPkThm;7M>qcis^obT0T!S9QgE;T?90kHE{E-WNErfpGLZ*4wlQ`wCk+-+av zB_|W7A=dYUpv+!SlXP!ZuoG2To+(v#0$k`A4z`h)(w^ zM|{SME9mc&THQZ7TyK?tgB&`16udT_`_q%c0%JQg1&Z|au`eMvzY?KZrf2bBFOLS_zRGnHWBm*|9(Mi~C0L5UOR4bG z4|3_JOt;l9dxq$1eBh-ShVItun7{hE*VpsNw zj#Y$8foJ;)A$X!LV+A^HL}=Z*nWB+VP#H&FUX%E{hu~AHn%=aH+Vbp|r^+Z0Jv;Mf z**6D_OCKzVPD;+C<$fVsg>cZ|OSQ$Fq^HswQyl$5}ES+eDgK@L&T+>aZBL68H|X|*!iM8 zIee*gEIf`nC#clBw;6!`p@LNCs;S$)@#j+0sDeOghat~qm->9*4zT#P`$b{wnOBv^ zP(m=R3Qzn2+b?<|yc*!p{@OT_9{}p3VZ~qu9qt%6gtaqo{JT8e+XCJ+Bl_R?AB0Dd z>ck#zGYQa_PBqK%bV51#5tx`Hj#+7O30yyI#OTQ+_y#f|)@V$%a`KTa*v41Owjknm z!GU)luu2!ft&Ud75BH8Nhs1EUcO+E!Yh71?x}%Q@vT902;U=|3EYB?&fyEJf&_b{2 z*t>MB(OQ`_9yE_?C->%)TNn;w6^vK+QQ3h=2E8N^Lkx_dDw22`Sq4eSobb@G>WJv5146} zj?atY%kJv$?5%J#dP-?pUTI=kWV`M<&@V8MFKBH1YGm{H3;X;yC62=d$His_tnco0 z-{4p>y5w|M0Bg6BmZl&bUTV=hvHQk?cXoV$-tY+_Rp+hzH&*ZLsm(DImS?wfo%+3% zod<}EN?pNpd=*EPBhJ<_Xv?4P@L(qO-kljpc|Wo__Vat%jr#Wb(pTy719m0eSiaiz zgdWxG4K|JYlXOy9UQg1o)G5}v$;^i-p;i)d-(Bsu)|`5CN6J{|!X>>DdBSZ%2&L=d z-|}BXM}&=R`sKr}8QoHO{`B6cxp_}{{vh0_KD}pWvhNT^KM?hnKih8o@?)K2w{u8m zv)(e!JoCB3Y-holTMOP#jAnjDxZ|ToZD%MF@GgtoW7?nYI=Cwjr+Fp$u9CM8vhuL< znvS<@34Zr*KjYa`&{85%yI%X!>nB48DRg4Px-MzfF9;{&+CEM*{uU>r)jnR8+;k#d zh`G+o<6d9*&e3ui5mM75M`O-I71`TICBoV{y2wd9V*++Tt14Z-#3oznz{%X2dn!U7 zeozD|o$jje8%n-7 zao_C@zG9-6VjT0Of^@v2kDNqmozE)YR$@c#q|%qj+RD=TeS9HM0bOgQk$3z1}r4{&*rfkKgjSpYgXDXKg(xOH03Ux zb5RyE@~X`n?ym?w4;;<@Z?k$(>}+oG}7p*lyQz z=?m}JbeU&wy0-)~i4VWfu^H18)-P;M>KT%@bLf>H^G%u!;KDk+m}UJI)c$kn<&#{S zslcS!H!bBkFFqFNetum`TnZ)<%Xf0+L#~wI`E3uW``%6}b^Y5QHI4kzz-z-XU4ytb zI2xHGL_9JhL~Wg^*$6v2>~5UDn-hQ!Ebq|%-aKGL9-NG7F)r%TFPNS<%_&wY0L0HG=~0>QmA<57QC~sh2nwJo%}(UrN5*tG93Qkg=EfBd&v%jNWLQ zD^FQ-;qdLIqXS|amw2GK*)Id~(wh_gWV`-?egZz#R>!EPEdxhTRuMAfBf8ERj#G&M zla!IaF^g013Akq&?)nPhMLqjw^=g#hWS(vT(ybwgiscTw7XuUeh2Ys z)4FbjHFG{rn{faXi$&ZqhzT{-UUn{bSzuyi5Wi-`CEsmycs)uG@vS8GjFZs~+hXat zcT#K%AC+w78z0!1$jBBp-@Mc7KAX2=d8T_C^BxuF3W`0oX?7@frdk!f$euCn>DOu6 z@R}zBzdiq|wokit)kBB5`#LStrJGcRqMQOpj9vV8n$T z>sMC8>w4iqIN^f`{^wf=cfl|q2f+buW*!)oq8v8gEL`>g9l*E4%p z^9a9S4$)Vzyqao8k3Ti@Bg{WfwW|$iaA(*3uxlMqLwM zjf{PHo+n#F4h{QX>#DGQPhH2ieO5ez8BSQpm~=tci`Wmuz6&StQ@!W9@1<&Q+{YWW zwVYCY(EN*NV7X3t&vW36eb8zfNpN`OKz<*!n$xo~qiCpI(d<+xP9|{9$DqP~@PL7Q ze&Kj;AwQwZ{OT)|kIiTsdovaDwagl^?2Kp_>YuI(k?SI_uGrs1Fggx*KWxUS5*XWN>^t}e+xJIa&sk0k@ksz$p~RS4 zVp|g!Kmg)yd2TpV3bqRjts)I|~)^Fq#{@ zSFWN*fr29s8qvLwtWV;;Wojfmy?L0DhV~#53%u5wS?h-ZqjvDc((P2_vyk|etu!0H z%Gf_+XP#H+xc^iMQAnpCtt63kMO|WjtpcozE-t2afQgk$_EWaTo`AVhi)BK4R>K_D zDjY4_YUFJjSF(qVRr{=Q^>X6u>#YjY&DH1B!^Qa)-pN`vAGYM}eK#7YWw*vRX8sa} zy1_@`xUGkZpkMcX5s z1WsAcPucwe%`Iv6-g!c@LWMt6`|Qh_i6-lB1iGw8PvKBS`UL+O4ea!s%Is&Qod#u6 zV#JSg*c>^X0V_PyK@<6JdUrB|BM;V|lg=XnPM{a=oS8RjAXfM?snjANRW#qxzI3`X z)vf_?2E;cx#q{nNu!O8Nmp%HB)#>O55sdeuJ-p>hw(5MzKTaEWBNj|JDV^i>03Vp; zTX8TDHi6lbk??Da*u4!?2dlSIlh%#Fmquk9OU-t*-Ze zsrh!vSW53~1S!&Hpm0ua!Q=`tl)f#_y6aG)4M-}gJzeC#Go9ERc_!Qg3a4qmj|?{a z+!^OOwTw2#Yf>2>%B@dEhq_teRnRPtn=4qv5&4M3#43@vANdGRM=;_e=y2wPSvyd=>b_*t_QP0=C(`kB>V{S9OjYs#770!dt1lT7;P3* zqjeiz|8ULEBvJ~KnbWH`&A5B!UbWA>YM*2>qC_`o7g1AhOS#-oM0nlFFxS<|@WYI@ z2M2Ugq$j{sD<2SzuKdQqBF8uA2dg*9b06|MsUPHgkBo11P=cq{v=3>b6QmwOTP=g# zcP5iyDceGSYw55{Z@&0ynd;_sp2NYLKx1&3)Wcg1NoAC}#h|@v={j>(U(u|GxIMq9 zFDlu->sNFrwI}?1MLwArtU4)qVahw#0gInSbGgo*k3UF@3cR~!qM4XZ+4XG}dV}<+ z>9X*^*0bADAy58iOU@+J@9d^sQT3_Pwa3<48Dnn_ccgZG%#@q<8)Og{P@AN4WE=XW zUz=i9PAiTTj}J+Wz5Ag1rQ_Vag|37voMc6%PcQcmHTM+Y|`5VE4W!p_vvk3%pF?`{5PxR z)Kv-F0h5^?{>N*^q0FghgL7rH-`nJ$Bkl|6q)^!dq(0osZ0>$Pi3W#9Eht;Z(#G;wcx z{ZlihOt>$QBMg>P2 zn7=qI?(xgO?cO7t_)ISs@pY;l)qbtqOQRD@-I@uGCttjNGh?f_sQc8dt+NE@X`Y+O zshhg7_cA!7_wbmp35MlNa;SfKu2lV6Nij$aD7v3#KAS?DQD`_0MoetgO!e+ya&Nfe z3xc+(<~JLVU}YF-A5vE^#z(Ly3E>|4FyZF1V^;ZfQid9nSsnl(diJ&K+S0TsqGVE; z|0T2D_pHF9NxQ^L!MG_90Aamah5*lVAH=;9Yo&g(R05rWBo!@Y+1EL?TD<9rf3QDq_(_txXASxCjKxsDqAqqTIHt=y~?xUJjz{~ zjE}vOEXO>{ASd=w+@7hIUtId}k!QVcxrvAKnV zkrVpy0U9kVah+ofLQfkb{DuLpPvcb9 zA&^<lm|YQE z=&-JCIL&bwr#som7$FeIVOP53bnLnyQ}9Zr1k*AnxAK9X=WIZ!NG+8YAE60%IHt^5 zG}(M)17G}Ful+^2#DNmevloD6S8bH3+`6z-GIAVW?Q1~=W-USgNfXe4i7YQ@J5m+~ zvrxY6m@1H23>WAqGH?HR?ngmm3D=FnmWqu|S!NxoCD&5aQ7}Cu9jg%B$_)2u=S5}s z6ofzA8_W{gnK?KV?Ar74g`F<5Y=QmD45f63w>9H@hP>nR5-jiX*t(vndrozI4i7)q zB2R1#43k{(t$&oLZt~=H!R3a-h`vEUUWyA?>#lt%+B8;dwxv<&u?E=Y=I%QB1S& zftlaDFYaKzYe_=yCcj16o27ES8Chr&UPL?G?ia`SmRE*FFB1}uPU7@K6-|rndkbVO z`pjL+&JdnT-j2OQm+n;iwYqjkLFaC*A!^lxXTJ62qw!LCJT7-;=36^A^R2hSZDM>Z zL>X9Dgj;b~;~qt)iBn83_Y&fa`qP`KFQe>1vM@oOGSeyAlI_hQCNVf=bONollN7?XhnGuZOZDnXTc%A*RGHfhx|%t?S<@!se}JZu_zatwOFCDYq0xE8uz0N&&@AyZBRQbmA4p_t_yN5 z4}=`@62euIidI`&kL8v}njR6~!pAGz(Fl+bRpMUe9%5>nUz(a?2`bIO3_oG-@lqt^ zN#$>)QiRJqW0)|rl9<(6b2Q64&z+Ls1K+=w`PJJqjuu4$@>}vYm(DOoTkX8(`*?## zFuOALcvy%$PWhr~*69P+JWOAQymh;3>c{gqS_12E->_zNk&RIvd#NY<`lU-6k6o1{ z$SRl0NA6A*KF)J?VPRQ(V(jj;Lel>doivrhshJ@SMU#?5o#7?lpJ4|@229%*Pq95$ z(>_jpwfEZM+H1GOuBhbBNW%*oxc!StWhB&-N&Bap^DN64Nrb&Hv9NI9h%-`Qfp)lUMkYuEZI-e|~O!;yT^&w{7#?Z}}1h)J|PV zj2d`m*Z<^r+Iof1$DhWxJ~y4$2%qUuVRZHQx)3AOQh<{=H?q0M)uYDr7MjyZW>3qy z6XDnG;6s%0s`^Lkziu}&{O50hW>B%UY}T2Oj5M&6lSkJCRWdb;D|Q=(*I9Hvs3*xC zvxs^uij{V``;_{!>|=;-r9Af29I+L{O2xSfb7YaORfugLh~J*kdz*q;R9c`!V{yEs zDox|7m*z!&>K6K`S`~W?NcP+Gsk}^Fk@;+KQ{~zXj|#T-r;_&~=@>cqPwb@C4@gIT+XVkt1Tiu4X(A|v9oI8%K=XMiokA+k; zt8cQ`E+6f3-HH|Y@C#>z8&aX5-`wWSUz-ZqCawKpPGL;JHHEe_{qcdhcl^UV5taA$ z;U8T+Jh*6?)<-H@*2j<%=!I6Olg6oOHx79fp4!e6{X|8SSL0Kl>va3TsnX?gFY0K^ z-KLutK#E&&#Au?iX}zmTHo)`l3OS%y+JHlnOLddsyY16}eEsNxiOtveOaCQ>0D0J{Y1a-`s2g}%p|*kxoKA)nj~D7@Ei zP%-|@TqLU31&!5Pw(om{y~OW*4RP^Gq#Wl!Y)ML5V5T6~(iU{#2m^RBqRe4`0ow3& zN~TVxQTex@=jIEJ$*wQevKy}yo{+8ALHgSi55zcnpc#51YhcmFy0nfB7eX7#P9TD& zzI6caekCY<+LT#t*X&=Hn)p!!=8%0$NGG;^#t78qSnQHa->^^|t{_N}|BfHR&usI~4dO2wDEOl%8DD9p>ErZ+AQxSEIvL7+eh@J#F zVaoDDNEyq)5_}y9Gy9jez!YRG>^%5{Q87q~jLE$6?7+V8B-6K1%aapjEWcU6d+SR{ zxXEwt-^BG26=DUa+61yf=fec4F9ah$4tUN8;-65UeFg4DM$WRwz5%lU;nd+--1~|g z@7Aj^EA=a4%IF^cYB>E~&ZxM`dNuuAW_$u^1~EC4u<@x=3MJr{ z#(eu(B9j?f>VN#&3yD1#?aV=ebv->&e1B42ouKM7oXi|>5Ra4(LwB_hhCwTDj_(rwp?0!S z87|cwDmEfkc-~`ZUU1kyn-?ub9Efm-fq3j_mshFlaO1e#uKA0YZOz!?F`O3+6?wg| zpE~0&pE>q-XAb3aLDZ_97r&fGTSS=9^Od7hJg%-LlfUq?O(VW8c47R=MY3*8%xiKg zLgxSGNw)_)nM?qaaiQO5G%s+NLAT$bnrTbn7f|uomrJEG0ET(txi&VYs3?+bO=`b< zxKNrUb7sk6n+N1>W@@xGpj;sW?$^8^Jy$v!#pPJDk3VlIn z{lSfj1$FYiOt#F2k?_F7(6nzDk4HB3Id$rp7gK@fIGMHlLZVa&!2dXgQYBNsLWHJ2 zJy@XnH(7x7KiioH78dV;aUj$Y%Ni$1g~HRJX%@hjr!A#9doPFvNQ@~C%^!T#_|%^@ z(DmfsR36k`b5@5aFm*ekqIdBCoxK^j>TEe?Z{xE_IS}*Say2z32misyx2=8b_KXKz zh*)ASZ5zYA&rb^U^N=ARTio`Zec*xa)~>EpJbC!M)Q3A!em6mM_)^M(4@`NSq1iH5 z!6#<`ooVXoK-q9TagS1!ES+cmEQlFJQ#{_D+*c$=_bqP_F}=tvyHnU0Uih8e(7~6n zAixvwKfl+PHR4Y){csY32}!IMB{?*0>-mx*MvP#7-*m>49Nq?Q(N|Z-iBAPGvd%YO zsHPeX9h@_2GI|*{$@4t|`mlH3>_x__N>98q?OOzha&@pJC1;LK2)LZq;1ve9M@!sA z7HQjym_jK@!p^Ncd82(?S)0)v{XJ3|qGR{yYj^Ni=jw2R*GJ{G;hQbHEWx-=LdX7d z-+gu-=Lgqip`gmsjkCU4_KDZy3cXxxyTIdQ^T4#2Uzs9cH%3XAavcQptII#IT>w-wZ@qPVz zRr$P3`QV~J!4}2tD$U5QUcMAEV7uTaZIzull<=ApS{4@1*o6x~64nJJ^)zkjwC%-A zp|2er<+R<(CodqO-Rby`trW9mpix>v&<3}p-FTh{5d1g#)l)+Y-3L1S5T7NADh0x> zKU1?mV+{YHOK9uVR7$H*_K6=kpwC=mox@zrfgu7oD%*z6`^AD{@KF<57=gI!bbY6O z-TZL1<{5#X=hT|enO6$P)GpJP`JhNH5lI+|YD}H5g+W{uPnLljK#H=C1X-M$?+ScC zHw+&_&zOZq!nR$*5A&P}#6R_hkw!valToqkGTPSnh%G?QnF>o9GJ}>{KNX;?rJdV> z1vzW%k8krkeb)C_{@n|-%Tqt9K^`;~B@Z+Yw`R%7B3FLBNzj|S^?n+ATuB-)8-1^B za&0JI17Yb%mEL3KcmyNHd*#1_BMcffa3r|$qr3)s3_UqqP_5!~ExquSL)qbEna_JP zC7%|M5UW9K*Hef-Z53!v;v2$aATuZK+PMFzIp}hgZKhmsdxy6+ql|=dMBwEhA%0aB z-NHbEGLw6p>A?@&u|6Ka>NYIgh^@ZOyZTM#41I#sjfU;~6|qaQ3ApQu&k;YDi_xD? z??E>uMY?@}M`fW*(cewh=xViv7=h`N%R3mMHCvbCr-0K1gK zVU246t@{YVi=yVj^oJGT%+JWBs>S-v0n3BJA@iWM5YVwfpso^{qU9^rXJJ(=dx85$ zIncUFR`dFSh2aFC&d-`cv!(-75pa%|pgx3?VhN%5T6lp)gL^b+jJlA8xhCHxRc3b8 z^DRgg-N6sE$TXZ2u^h?-xYzCwD5EJ%oHAYMFbJ-H%N1cb+I;{@pG@tBR1~Ov=k4o3 zEbw+@5u)q5Ja8#5kOF0br^d9(5 z9ebmiXe|%pCkSEgFhgeriplALj}U(iG8>;18vQso`y*bdYc$Ojp}bve>MRhm>+A9| z0)F=(Nak3ViW@rs;6LQ%0g(9OCdAf@Z2+D18bfc7 z7h$KYfz3^yIT-KXZ*{MReA5dyDOq?CW@ynHOoBOMwFI>O3$eI5m~~^M<1PzGA7AjR zMl(1tHt|6^w`4&qLGA#GpVt7erUt=8x^EpAj$|-3!{d`wxeAT-)Ts$qdow(4Nll+L zc55s4g1fc3?$GawU!6(3_Lt+vbA)gKjqDka7YP3Wq@68o-#h zeMcYdqZA(Hw+=7Co~Sq`n)05uaQNH1bJcY8VlVx;80yMERD7}RfiJOA2?QE*&^=fJ z(&bqTSx;UIQBZkOHyA|wvDD3SNW_bLOkgdb?`T!Y(d7ecZx|u6dBMOT_ckM#fWp_A z#N7nefDb|VX#2D!ZwvG(D^VU3hp;i{=3~b_SY3o*dL*L>U-IY5+eIe>A;~eD+Tc}7 zUItp>*L063=IBM)rMJE)Jx~X+`IqsQ|Q&rN^FBVj3K;w&nwru8b(bπ9(lZ`4V+QpV6O2dJA+R_3%8*{+0G5cBDk0uV9F zq}_XjBb9i19t)}Z8r+OG{rq?zM-yz{0CW^>9|H17lcnl=NKjhgPNL9?rz0Hv?16(T z6m-024n@ZqI;IXTNQJ6O9IkH}Cw87P=2biFi6+Dn2f+TR>+(Aoie6;fDn~eIy_%u~ zgbSHL_)=n6Dupds$0R5?)-Si`r33we5xO45%yES0IQGV&_W?{p`?L#(-CV{Ct;RiA z7|GZj>D0(4bd38gA+mKwGEeT;Yeknnw${$-Jp8f4xC-xPuL;9wAS!CoO z6)2?(r*Axo3ILlnfesIF%dj0hSATRgxH#ij$LKKOta`DAf>8rIKf5o?%$<^JD^Iy$ z`JP|pRmMj(J7(+S!OQn z=e!*~{sfb-ee+Fk*PoJ>1w@h9?tq_}+8emu{SEDmA~3plo}T6BWSqhuYQQ z0;pSz9)KEY3r`p*1wnK%?xw<&Sd!hh4q_>ez^vmhR#%E+)uFT;o`c+5ug-Q^k)bNMVpLg?Rk*)2%wUiW@{mw#7`{wDg-7??4BU+?iBzL4K<++xl+saD}wtyfN$J&swQrXbdr5Xoddh zE!J6lSt{l0_aW-A322HDQ(p{Nvp8ckL-v4KNG|yC=t6ST#3r6 zQDjbgz%n}O+3lJRzXmOk|&U+JJO3W7!KiI+nmpsn) zJGX`p0(=u+whGmWIXG=NRu#0`)xy4YFM%_K2-tT1M>`xJzJSfA8-mM5z+K2n9WmFC z0ij3}C^eimfbNpb8uTL2FPRXj7z`EDH@f&+of{|s=aEB|7vdeC1~sVmr)&H0nVGO>bduC#lB{!_1kr z5p-R*@^ZaS+|djDYa^w_`@xBx4BYP+rg@B(^19^v@L>L%|>fp1x%Km zRd(`(dn&hXG=|xN*y5{YkYxGQzi*Yuo$dlAx4!lyn^sjWbKVuMYm3&B_1O8hoB6gU zCP8RYL+1&KYTbWnzW=!SU*(>OXJ%_A$(pCS*?s>xzE!lvB|Ti7Wb0X<$zb^08s3$ zpzg7@vv0m@4blzqJC!IMB{(6B)gudq%U6yo6%ZXR{j;sP@4t9Qee%6~UNtyPW#-U1 z`2xQ%I1;iE`w+U@Wd}-r$b?W3u;_0Y%1s#lLiefK1F|4nz!1q`W~G~kIN5-8H>LnQ z{4g*_Nib8l$RXC1*wcX+Hh6S{SM6-ziQp`ty8R@!4PU-U1@xd95>iVpuO)Tlra;!w#{10NacA z$a(8lMaMoyo;d0T+2eh<#c^N^XgwW8fm`F!pb8N?ZW9WN?i{axk7IoUDAZ^7aF?M< zEIOEnhv!upBlrv`v?GQn5la1xK2pc6Ul^{09--wvZRzz=IKr}}K(5{dAgP-hEgfIv zB_$=V3~340i$&_+IX;&HvWAps;QB0+UVRk8O&!O;gFbPJcZ>~kShp%FA`U_(ka?g% znzm_nt?b;VtCaNEUruw!f`WpCP%rv4V-#qt0oC=NAVRe`LkEzn5E%{~eHaUt`*3~F z=_S}iXX&VrzRqH|pMov*A89`MFWGQlTCUlg(2@0e)nbigBdTezEsW|i5zJzF0RW}} zh~sj-Fd<&QCR@1L0(3?_VWQ5DUpT0bdZBQi^X!H;z?0hj!uz;Cfarr}>|Qq`2(n0y z*;%O>xTkMkUH^T)nKVC^pX?6a#5kMv>(8yoPwE>jS8ih-BcI~lZ&J)g2G$7ir;4y^ z^{p{in41+2)z~*=$nrA@D<Vi3}0lv5!W``g4Km2zX2@!0sT?+QX zh%CJ;Cr}J!1)|fpWUG5Kl!{3C&dM?_X2D>eBt6q?^5b^e7zq`=_YZVT%r9}3C| z@&sz70(2)WgGW#z!Q21-T!n(JVr(WG2S&%A(_WCuqtnv(slvNqzF;QStHlzMM(3xL z_IwI*WW%_%9N^fNP!OY^@RJUZ5%kPe>^!ai5PZH7>2<1lygwUAXy}yYibnVJ=2>b! zH4{*X?<6219#F0cfLjDrJ*w1babb zr}gYpIy!7*E&W%(fkCg}yVMEOj>25rjmynjI5AL5rh#EI>4Js9OHhcEz!=nib6@BM zm?3-OV<+P^2tLg>jg!@mS@#uXBbsNv&TJ^0@%t5#DxvZrq&wmd=I;QC_(3}3bfn8` zFF+l`j0l|Tpw`F(oc+q~5I!|3a4!tN(zuq}0KGab-9(g$SnBigZ+P3ZkGK?G_*;)I={D+dJt z-V3uXQY}`#>AzrnB%k{?h;B3b_X2);-*|Vn3SDocX9!sa5+6eyLwyk&8Tn19pJ&0U zH=kcCrHH5#;|^d2UvMuBa~N-o_xun7O@(emU|xWpdF=!#9R7})-wt(A6R+WZk0efGkT=V<$ z^<_yz@qQUS#)2yK08|LOH?b^lI6$Z^LB-QPVYomjnt5O$&$hd*cnw}9LY=zj(B zS_}iPz^=5>{e0Kb?o5MGgBhaVo4ho+0nGetti28clGP0{4W%n+zxzAz4xY zm4x)@^YeY=2e&iwGNr=!A2C#>9_95(c1~znR@qfFzR4!=AA+c*V}_JmFplxn;`7BedL*`w%R`Pyxn$s z)2>xSYe!zDeo~_{Kq3My`d5n{@3zNkQ9ci}uWAcZmHi#R7e2d zu2k(324y;pR@I9<`BUt4bqE0F=V(QfexTpTy-B!q-cR<|*Aqg!{sMa)BRVdTV*@yy zXzb|xJ2YG%ow_)q)?|}^nE6xX0Mbij))!ElQdhev_$Xh9?lN112Dhf~X}VXeY8vx5 z(?7CplmJ(D6L?W;`ZQX^09x|_@6T*jdElKs=Z9s8@{rx1td))T)1y&MSIHB!`jK|v zbKEMcnx@Z+7xA0wL{uIn$Z|v z4Z7R$v8W>U3tCPWlfQ+@<{#8ybcx(X7jb3$vaf1>TIV^fvts)E)>c}X)>?waWiwan zFHx;AOWqf(^3yxF^!+*Evzoj^C6{gXd2Zdl8oB`s#Y~~fD&Hx*(kIJvwpEV*SXGHQ zYbE|y=$Tj}`n=4z%k*f9@^1ARciG#SprEfTu%N!Z5tGBukCmdV8TV#v4+V8@QA>M? zpTyPDf<;)W?$@`yN>1Z-yiDU2YR|E)!;xPF$Lac=clGk*wRV9gukYMe^?2(NV|kmL z$DG1dIvVHWEqeQu3zGo%ob4kPmH*M)n}<`qw{7E#tO}KqQZh9t5*n0Dp%PIt6lE?c zGnt2EXwW1Y(10j~jAdRdR4S#Ad8Q=uJkP)L)4lh-pZ9r=2mN}S^xgfh< zM_!#FSvf~Olym!uQgso@Rry-CjHpQ!Q$`Fmb|SH>)*LU@_K_9y-pHblOHtSi2~%Ub z?XTLR?=6zCSR2p(fKz=*l|YWk)=}} zPYkTX`@E>17i0_QL^QD;5-N=;SctV?{ERbH+ojG&mWSz%aFS>FVC4=Dfs=w1L6)CF zf@?&2Gp?;#tW^K9>IaLqurg;1q~kWzm7bZOSan}5p{gb2N4Z?&=s3d01FOF;S7wn3 z#_J&BCzjdb%TXUK|AHpTNxexQj&0?8DF4hpZm5l_dSYQtD_5f5O*=+CT@Pk%AKc*& zCL+7F4livgW#F|=;f&FVn8f7dmW9U&?VL?d<+}_HC|EPgB>9Zw>S}l#-dMLqtYe*f zP8n?vW07xgw0#H9Mga%G8_EatU7re{R(!zmtY8^6W!1oHYMGPGfs6yD6?#u~YvN3E zGJ18>yB6B+6VoVIsmn>#d{pMdlh~(G!ent;OzI;o_{2V^@C8NT#bNGS2F4WV>#n{QaHVQhu;p1JBs z6J3w#n8eWodKq!uc4FIC@9lGKaeTb_0b`+Igk+g(+g;b;tD3CF8r4ma4>%%BqHJbl zyEF|MP9zC989%$`vZRcwS-P%Qt9Ia;%i`+ZT-VQA3?+U$QuQ zR9PGuQr@XK=E(ChJ%B~rqxJJ5L)E*2lPzdhoa>iAR_B-UqB$>E>F|!~wE$(;=UKkG zCw`2@v^_ZomxvuMu#DtO+}1Inni<8BwCHAOyJ6Tbfty#B`L!5G!-Zg2toiv_Zm~{< z&#PIgqMZ!P_%zoItxxU|lIL3!dMu&GG-__pJyhLT|Gg5^SRMQ0f)|$ZbpI<=LGC%I zjBV;Uj#MWvUYk%~Hf5vk>+|Jsrj%m)aoNX$vH9I^Z3nVps8M&u*@DdK_nO7g%0+b%DOh?smmHy@bbU`~1OrRo8Tx8iMN+ zlfd?lNaH}?)9y^^J^l|W_9Om|pG z^;4a+QRgGEwBb(bRE5)}y(}_*yvGDg%;tj*KBLfSbIWArI%QYiO+Pl>k@m~~cyWb6 zF6Yb%_Wk0f{K?!0^%mx=E|2AjyB$nVO%$N&ud-U&_(?f<+o_+|)Re8I>AXVE+Nf!} z&mQEb4mw#2b|ju=M`_uv>(>1^;P`4y=|hnQb$&y-zDgcV`;#PiLh@Jl-c4j(D%>*a zyrMJ5byUy&i{YUzKBu;%ULF29bR_o7OI318bD7Gha>HmNPIDqcK8 zu-`LWNpX(0o>__tyZqgnUmjgo{fbQL(lIFA^}cABW^~LpS?2k$;!lBdQ5p4@{Dl$| z^D7^Z1K@DDL5Zb??%FUAsWR^+NvU0nmI;B>k2jo4jB#hw=9K7 z$l^I#;@2*k$a5zZaZPELwTBbslMh@L{kx?M{J^rMJ)@U%-@`~Zu}70H>1i_|`qEq4 ztE|q5Wk||tO^55a8r<_})rwvnsL_4Yf0|pFVVD1or1{F5)3Dk-l6ImUv$3^%6!^Ed ztgOTM_xr=LCcA~;4H(qeB$XFgcE(4t+n>o*=1A3P831g2bcbPvQoFKI&&K*nSP2a_ zX=+GII2uuwZms8K5PN0G(i_jh8OvS?r#!bA{_?czK?PKkZ=2J+v=?)8txpR#Uw+EU zkp9=>@|HRFHfF;l%h`?#7Hutq!47poL0zV<^`nJH_(h}LS|#Fw27zztqXhYy1#B|LwJ$IMP4sXW)edZ;oQhf<*JY!( zf!erB%E*1URZZz^((0MJy0^$Qe+=`|?Dk`_qYu`tGdN7CT|2{>W;e$8+TsN74IR-O z`E$;TmkhWYygxsh7G>-nH5?x9iP3o?tD{Ht(M?@W8+XYb7Bp9QuON801r{rJZnSvm zR{PU($L}1qU@UTudea`Z64&V4LQ(XZnQ>lGnC|s)K@iPn@aXuNVx%%Oo`0bdG_*zc zMO8`>s6#*V!gpz9yf_gZ;}?Pk%`fZ_eT|v)8_kyS%Xe0miH#>2Q&tE$sGjm@+L-;u zkl~lXe;6qOljvTCVf)q9-!wtqR*;$%1ypIy`#Ye#&@)TSdaJ={dtlK|c{BoC@Mtq4ozeOL5{t%tG-ucLFc)7#jF>BolopP0NAfcKBKeRvR&IHDvKX+f6`W{A4{u6w z(%S@Zf#&7b^;%w5;I%$GS#2y`<<75XQc8M*GgOgariDi~riBL-u)4Y_1gR5) zmR9v5Vk87k9NUzBV-)EPol)7>O}5WVY(sT(g+lFHb&<68THEv(w4K;QEKYT&xofv{ zf#~4$R@KQmF(TPlnPtj+rmNHwY$@AU(`&dM-aIH`X$t6Ju5}P_<|2VTW;vGBarV|5{ zTP+~N(faZcgz($93lDaDzPN{nRe&0EH7OoehN7@U@iNx`v|r&fy!%S&qJX(dFLgG= zjk+1v-c0ob-khWtJ$B3>!cX(mz!&3){2>Truct5R*egisr)q5ctX!*g?>+WJf4F7O zXsPiqhG^DWd>`XusKO0>(hT#nMu1!rXdEa zM^3ejveS+T6vUio53&`j;nf`*nFRAHC9TJ;o?Z8oyQ#uw3U8Q}z~W9Wt*kP)a166l zxyzCnUIpOn`N&c2;NF#>}Kp}V@*e%@u6+%;|)F{v{l!&jkj>I=#+bj?^D0<@{3Jw?OEEdEo9f#C?95MN57aAF$`YQ!!wV# z0t`i_76{c`1Wq}1Insifu)4F0rv-;mGQcANv%H`+{1#y|z+x7OsQS1B_-cM1x!oeL zUGVzL;J0;R)M}2XHt0QOZJSB7c))nnZc%`e*Y>O?KnXL)#sYxfswP6eZm zi1(^YV(13SiJ+M-_3HzxAO4-Qc&UpJdJ?mG?pX;u1iR*sDCqMn5>tkppM`xZzbE{Z zGUW}8335!uR@`Jnip4XGmYpw3$svIWcOvhU5@q0uq-l7<73YSQ<0!Ch#(5 ziO_m^$HcA2V_J-Ye~*88c1Owh)bTgI#pY4k{VqoI}ZDoR`}2N-T9>Q1ok)PYZccfb&cmb<0r4q?WtVEj%OX`K}K(ZKN3nD(vIG zwe4iGe6!sj3fP}}`#M2+hrVIx1ec7Sf&83fo?~U-B*$FsnjDR`Wx)jA6QH+39^E|v z!nc5+?z>;90lxvwpugbVI>pm}D zmt;5Hn0djZnQ)Fktv@pOD1DF-bX08VtCPB6)Z3);L) zFS2Zk_z+mtk1*}w6S0~f{xEzlk}z4XIrS!IfB3VxfP$MnN}%WJ_*1)FY(RA56Q2d{ z{u76~5o{(gAKCihywo9p#H_l-_;H?u$Xc(&6HN@mBoHx?Tp!aL1D{S|Rg4yfMBtuC zpGWHS}oZwj9^swF1#JUJk6f#+tj;9Fe-gA2eU5K!Bg4~=os^4+c$&W zk`aJgX@L0fjf2@Tv9k-E_;4DMxxzSu^1#k}x%Zq@Vr2cxZadsAxg<;D!==s?h6M{V z-H4&nA53#7jsD(`&86KLNO~>-{xR0-AjLvMD(%|->iZ8LcC3?fjFFy`VvBrgnMHCv zliORO?us|7;o$9?BMn+;KxE3e(}V{$=oShd#ouR```0PW(ko(%>fEUc@)PT35P^T`;0 zI_ml5x7Ea_H+p0|Ute6UXTNM*%Dd)e-)v zKiUyt$h>@Ag(+P(H!k{0KCgnEt2*1NuB?OagJk9=S}|KLj`Vn;4#VYw?N0vBM33hv z8S4oB+oWFhk4gQ8ZlN{vRi8Qa+TQumcuGWd=c4Y5Zk}M{-Yl+7qw%x#NX^+j8-q3Q zC~qp~p=~5nIUcD57zed9G(C{%#h=zZl+MOtTEUoD*bvgrg1dS4cK-0-|&b0SId)ce7{Mx>Ppj& zRBr1ZoA2J0=uoui9m?K-X)kmpo#9*P!|TrzI)Bw`6Q%6mk1h9(xJ|epXv)AA%_b}r zKu#&(cyG***i!Njz81KpVypJY+-`fiY|T%{qL7CVVol92iL~R{mk;r+Am+vC;I@v> z5OZdn{LyZy6+u&A#LXmCe#r6FyMB})z?Jc*%57+!n|0r{zX5@)9tG`;$=f)^y+51p z@JRO%dO7Cg_s)iH`wwIK5@od`Z=|1upBc_>wwmqYvsJt|9Bp~&Xwsy3Uhwlah9AVH z5U$0)`*{s-Q1ERtp4jF}{$$X~e}h4~Gnhz={LKwO*&(S7lBGLT-c#U9Ejd=WG2qwO zOYU?v>0a@W>w_3UWL9HErh{*)7r-lP?@=E3O?Dp<6d=f_!<`~ur92?CYH1-O1TCq5 z7+l@wFx@}IU4Nto>;!9=Db3U1P#hc@`t^T6nE8}p(%sa!XRah`tnIN}@263(`hGpF z^wLC~ocOeDdf)0!#8}fUBr<+ZI$V-w;13p*aYY#2JW1w+)p=p1HRI23efl(=FWD|3 zfA@Uv!+j@pcJFPHJzJmKm~Jz^tlq&mZKYadKFAeI+Prc>f zQmrrW8)*pte0hKU=hfri{x?|#@>5>=YD?zaCK05-t%$!mUyp}DRe{}qXkh3@ONPMp z%*@ORK|cSL<*clx=6slf+sZKs=JV$8ZTj5%jgfLtD9MYB;k6C)HgoN?MHO@{HMqrJ zKmBS6?4^S0pD^%W)A*-3AarT<0V9Tan5Sk{6HiYdr%@fq|5`qM*qrjF3XH1;`rT+) zpGpQh9{4x-#D9QX2gfL-Y>nU$ganBi)UmAIR=5I08$Z(wVl?OSb{-~paO=KnQvXj> z1OI`F=WoHPw<4_N(-d$7DEVG*y#$W_=9oJ3_jfrhLCmcW-SIa^I!<7uVA zZ?F62uV(N3>oLn;!&)sNm#jNusR_vP+(%&~Ttl85TG~myx%RUIMx-yQ`Hz0`E(F2Z!2u_>Hp6- zTK`+#|MK=Wwv?Zr@D_ay#SCcT)#(C;JtnA%`XCFahB;DbJ_a;gE>lNT1feNgvw~-_~MOoQ@1ZDUS|0Pyp)5M=g z%k-0wt$R^6j^rg`zQ0OPonI$IC`k{=oTch4gh2$Chu>X$7_N)_FG#|F{ZyXz={8H3 z2}1~yx~oKHSBAvKM4^eN;XC$GxGzDRh+GbNs~>Yq{z>5A`+p@1;J<%`{E`_G6#27Y zAf%w>V4^Q1m)-ZD|26Z29UL&R42i$D4S6ePm;Lu|;XfrV{P!Qk`` z5B`&O!mYN)s>T?#Wg0g7DO8T}P&#M>LuqK*h6UxfTFlUjctIh;TNllE0*cHk453ty zY$A+Aer;4<1uu_1wUKu2Nd`6aBr?Xf8RcHgqMsu3t-(c}A#ipnyU8JRhEQ8bj8o^7 ztTEh$#~{n$%eWHCzhx!bB+^$LJWOaI}&X z-{41qfy)PXpf=fdip{&`4M<+uJ#UKQQ6gs1g(zq8ffghH!RlMoN(gLlSryGj6Vk{x zJ@tGRz;SW!Q6}$9kaRMR&x=l@2ypON$xzVJY=-kQ1K3lhMxisx?8_u1T?w3s!V$!97(`= zy61dxD0eLz8BHU264p~zwhc%`1>Osx(WItF*>mv>q~ zeQ)HuiuTa9ri7J4&{$YIJ{hjh4llj;l5zjK9cQj|pE~H|Z__uA%8IAv61kbe?JSfx4=?fK=65S78LfqMq~OA$pT`f)AF9uB zO9^unNAysa1b6HEV3e{sFLMsi`;TPB%qst zq>;92>+!-y@B2lI@}JWb_D!@bAPvY&e|Nw*bc7YNk-4_b`v?`RXr`KPp5h%p3?ACa zM>|db09|rA!rLf=TT;B9cQze)>ovu1@AGVx#gj?{Vo5mUE<6-t?_T~wGdk=?9$WVq!rTOEmq1D73IlRQ&FWdxNO zf@GRFZGs7YU*WTaMS+}xgG!@|WD#~FpNB-+#9Q-_d0e3zsehaUmeXO#2LJ8ZA8eFD zE^tcYc$_LU5~<1VCJc72D*er7E4QdQ8VDEl)TWSKEc8+%U(rOe+fdC3q*otuH{}XH z7%_Tw@CL^sr2|DHXt88b;8(Iti-*BSJpp7??*BZg1C3;hz)Ang&6K^eVs0rX}hCW%keMTn@BDq0@Zl zK?joKzizA3q{!!}m(D>Avt->&2nyv-qnfRwqAP7&PwzH<0RKBFd}lnWU&jPX2aq|( ziF#s>hHy3YSGx$EsMW5qt5+F7z_u%^k}|ZRe<5%F zkE4$!LiFtqzbZGgs>Km8_31H{a@HR_Od6j7s=uQ3)a9eiW?hfWLE)V9R^0@Hd#lpr zxB-|nXB6ng2k$`@=VkBh&a!y7Q`d2cEB*&n4zbScD=U@YdbUu$2TGyB&1AP(R*Df< z;Zv?ZNEi#D?K_x$i)9TPQNQ<=C@?>^XmC1UdpbAvL(417I>sn(7~#%D5nuK+yh2>kZN|JUs4zb|;-dDu=o$uoAzmK6;^?VMvv9W6FY7A`MKeko|9Ko?=FZUZ-$;wOM zQe}@;>CKu z2D}m@8OZ&xPSXBtqRkfX_rn<*(G(r_nFUanwdNgiTTF<;gqQ~ky)1W8F<$VlA~#^a zAJ4Dp#n%nU%RAd%95&YuR^J`>ebk{{8+PpYj$`Z$(t?vA4mpKed*&(Oii6LZwL}Ez z^#y)V?$x2hGlBhl-z5f0;ONNn7+vkoPgU5-UvS5^&mZ|*heEU^c|l>?p9kiLqF)z~ zOb<@)6g1S?`!*FAKG+0rTZh0g5pwks7#dcX=h2R%Rar*i-XngkE7iPwMkJQp zIqWG61z&BIu;im^(QOmB0?Mk8AN?A7f-!Zh&_UNv%8j>O|ac zuUr@t9FGP#`Fx&8H&d+EJ@Anr{t?|I;jt(fafw6Mz>z6yQ5+>(YWcrE z2eC`@N~{`>J(Xc&P&4^vKbAv0P=P*4xrqf7y9v;Y^{gQ69V1scx!^q+_9!J!OY2;*R$1yq<@Zo+Cj(l6vegUvNJNX8wk5=wbNAzM8o=ym_$p3iI z(8NvI8{cWWlUU%BQ>`G72$nsl7>=nGbP{V{L{=yTaM$j^iX3#O0$(;QohFGV^ODU< z@)uEq7asaEJ*ye6)G@jmhVPy`D3^ze@y*3`;^#^rgSW_Nbk++N!{RG=NlXAlGg}x5 z!Mg{=YQWyG>Y!~DevfZJ2dR?_m#oQ;T-XASMoE{F&2n=S&xxqHviZ(UPB9ypZLCmQ zP6=F(LB7*Gnn>x6StOU#0#WY>FL}6Q<4;>^)A(I3IpRAQO69*s&a;iP8o&LY658dp zuyb%g>GN{RbHD+($n#~sCpLFHqJ{P#Vr7ZTY(34%QsN_<_XPA24CC_%?}L7hQs7Mv zSjgkK*MYBS;P7qtifAq0pTrjcP)G6i-UZ(8zh7Um(F)YqDu7W9O&pXRV-j@;ZEsxj zg$O|)zKfc;h|jwRSLFB%svqMuD=6~Uh^pN4X-wKY^On%SpTm&{{AfR}=p^WxqCq(_ zIEh}lJ3vDQN5>qm6{TP^5;?7+#NLwA;B_s)uk-i|9;NLhR>TvWb#3^H5=dI}D^Bq7 zdSq!5{<)CuKtG_~(d~9*oED#BH*z(xjfe&Miq^?rtb1dsS~^czW;E5*k7)Gi%Z34v zV2X4^p1y%cxFy6pX2%>bnlAx`lt5;}h9E#SMhliGU0J2SLD6ZB7R&y|Datao&dGI0 z5A4}DR%6mPnSA=ae!g5OfJiG+C1YdPiexn0JF2&YQrJBnb(1fKGvBURRz(^A6`28s zjl?a-d94b|z?tRbNV;iAA$?ADpuBALNKfsqCu2Z9>f&((V2aokM5wl$8d2D%dc)8;?C}POi=n%CDyNhE5N7^6}2|b zhViTOL_maXZosYp>GF`B07Oz^N6=^jk5a%wMZ2kGhil830#VG=GYH6)M?YUv(As z+f_(c?tP%8_PMH;(c2YXk!R;0##+4fc)wJl2Ev{0x#A~sUb%D}2Z`atuLui?pV3YmT^#C&X>~J1wRCAa3wrV_MYF|MKe22Ik3z97FVF&_$wOuk+ zJc#zx>U=vy1&Ibn)R34a&EB`a5x0|uVO#Oil@a~BW>X|&}(6wY)<)A97 zB`5vd!{fg*a~pzWB5v=a494P=vANx^EdF~O>2>ywPbk0sIC2QEx;K71^c>r94A(sE z;C_ny$4pj@?_v_dh`mmwh1^`upp@U`hXvq8R_coaB@L|D}xnLp#G?{@+Z>{7>-7|L_Oz zutE`?MQ4^zkGx18?f&zh#m4>8`_1POMy2I_S?#<;0rZ}iEo@5= zoCz+~ph~%XgsN!-6$dFJ@9>SIXQ@NSWQBjG6BfXy>VS?fKj42AXg%5iwUY^S$*%$X z*h}HldT;yd*F9TG$SQ!x&G*4s%b0{0;%FrTF+ZY*Yg#g$=RT{NM|b zRE!OCN)_;}?47X0HPCzJE3`p6%*hIn7$DbHTJd}5PD+Gf?A8ZEile8hmL z7KeK|J&}yoMCl`%DCi!xT>;iy2Wx=@U|-H?#zCf$J%@H4uVJG*tU6v30UF^Fq4xTg z?ta|q7-mtuz@XAmbYHkhp2-n3BjjtRG6Vc=+&=MM98lnF2{?=1gLFQ65LF(9gf21b zHy6nOLsEQK^4>9`$70p@hV~M%uVakBIZD` z?Ln*#vftAc>4`+Z2a?Ng_h|JuN7Mu)Vm>hoD4M$LT)wjpuPp=6+9@D=DYM(hU~q$% z+5Y9Vdn)<80xvM{n=2zor}vR3yUyQ9v#Kqd*DZmhKrIepD$Cv}LYr&(NG!I*pX$aU z!(k?Q?*b_HA_)snHi--q9CsxJf9NvQ$B61YlHCi(65mv|%>&9ZP*2AdGgnd}o6VR` z1dvHE(74duoR{~3k?N!RK=8+xSI9N|3PstY`X^2&HP@afngFMvo?r+ZPl=5}71*HT zGtk61CcHF7RhXC~^h5dm!YgB3%WvFc78mzpe^5~-9Zfcu*HOV!9WYw@JZLT7MO-ej zq55=);yuAe0gM+;_$oh3(f3d&no8(9@D)hqxFQT}VBQJ#c0Eu`5o01Vaf3PAR*#;N38s+|LB0dNB&Zmy}7RIs&v4hY>`@AVs;*WK&BjJK?I&~$f}VGi8R1Wce-W($-vbgk4_e8Qn~8a?24cyx}U#>4;4cq{?hZC9P4=Ss^WN@ma*@#uk9x*E2%_ zxz&)<+5^>b0@^C)%u3elvkp;BLcefEI<>9B50)SKmt@~tUmeCU5!KV<{p~WpE})Dn z87sJqqoo!yQZ1SNMFECNyA0f8(H|{O!@^3+L zb<;g`h#=Nd&OIIJS?JfaU#P5-2Eg4*sMrn3IPTW%yO>$e=Hz3Du{F{og9x4H_wZy{ zT0C*YK#gP#=+J4Jlw~-{o)BgH>1ehksf*GDtKAP=?w4|oOl?3!L*`&ZrF%Bo&nl2n zNWJ^*(XK5YMhL(mydW}wFqDo)53l{^*g1&yHp_mcP4nnt&Oo(WcWQ|j(VdhpK_SWn z>~Y~GO4w;MttmaU!j66Mx@lkjRLQ%&;MLLa$3Uqsv-M6yLBMUGhq!B;OVYmP^Lxbr z^UUtnewP4#>ino&5^k`9CN`p_MQ4|uV}_4UcEw5jvoWi5Qimt{>@`d=URa)zn;p3y zw+_F3TcVtkY?8x8wC8mITrYpQ=6-y!%TgujUj-OC8>~G+*F^iFQ<8&P_s=8*6ox0i zcM`?vlI{1;#(@s!5ip%svSU3 zh7aPnP*bMkVOI5rMYZ|J!xmCMc;b2=EOEiLyzpBJ?BSYm-uIK2CkA2rhcS{(DI#Xq z*kl5WqF9@h^w)R_xj;3hh{@=ay%(OwX`doCljDfA zs1?D{AD&B)*m1PC(+FwoZ7Lsg??x<%K|x!*QC^b^_SZCtkXS_7W)3iXy+n+7ZoDNvJ!L zh}BSz#S`euJEeiV%oHUU2&)d%rNFYQHXy z-fn>q{hwWhxMF2uY>wA9W!M@OSgSX#Ix4%BfDK8gPy3`U0h?%;BkI^(;9Zm5yTek< za44~^ePx8z=l!1N@57gw!qT#EVG#J(foC2QQSbaDR2(Zu9{HxQD0ij6o2(gIZ}usv zVI|y0%!oFo-B-n@{&|EJEWkVghD7w^z!~!dE;GlU%FA*I5bN%qJ;DR=#;&-UJyF21 zO!eRw7$=gVyikkW8}&f{;EAWFI{1ijFjjyfIFVGa>FDz}e6Miw z&GlMg%iUxWs1z{V@Q_`JQOIr6_6r2g`RvH(-L=zz6^upudS04{&*$?#rV@`wwSo10 z>m)@9_x^v4$$uF$<`dAMZHc#^+akjZ_FLjfm7uy0+_V;qrNLmYJ{% zAJ^t$OGTa3!N_B#{OZCodP}VDK8p)8ATL}o&@onZrEbBe3QNFRI){$@tw?uQT+%XW ze@1nm_Dz~CDk9UPlUQ00&Mj97fFY+)kK^~gX5bk!3X)+Vc4~WnFwMW&kXE(qNRzUN z9apd8U3SWbuy!-uIKSsRx1f-4pdWTE!m1p;D59G>0P5NYsw;@0t&DZhOLznua(?f- zoTw`BkQ<}y+y#r4T?sp6`b96;y|Hkyz%ADzb>R`S7}K=sGB21IikkJNb=aVOsi|t{ z+Py!aF{t8RH9LjFb=>`yy06yWt$aR*choi8ImYKOtsC9H;*R^zf#Bzxd`iPtKTCrE zI%7#tx3ZK=ANLN17R;h=JlymF#yWuO7>0ux_?}+299+*w3sO>`^ql~ z`fSmk<(mqNnvzaveKpLR>7@ej9;HBffteo*{y?D|w8dHief()H4#CE~?SRFy{nzaH z%T=$b zCiD@&4vUUHZ*%D^`!Megd`KvG7^Ls)3$MsHTVk`vUi5Dxa?Kb%5nEv8;2Dw&R}s8obo@Yj1wr%kdeD`DZJ7 z9VHdpj{s=1Af$r%S=0uFP0&P193ocKRtSN%iJu~Vb)8Nt640Gg`=RGzF;-B-3dVdo%1YD3Zzzr%79gN+jX81n!g)Sdjgw48BhT$!HY`i#&K4Wnp$ZX0HAx_dF&&a6c;jOP#nU{hcl1`L>gzykiLhn8}t(kXe)+*tA4lNbU*@~o)(l?K^ z4PHgv$v1h<^anOGXH{cfM4tayX6mzw4VoekSLJIi7ll!r=tH;(z7W66i~cv6ID(#4^`ibC8h>~ffjgEHHjT7eQiT}C#m8{F($%iJvkwRg^2GKX{~7FZ>A1`JEAHn? z_jFE>t9uMrbsZWuh7@*-uWm}tZSej~TQBEio`WT{uWCr{&EJY`QO z&E(UsWl2VI{TQRrjYIUs9aFgkifu9C*jBYed%1x~-+O`BUs3tmKO+x3yt2~O0IqlZ zE&wWK`(rtolLWF7vj~Z<$37=&0duwcJ}!&Mq}Fx;PN7E%?|10K!O9uRn3I4nHZ1%| zk6bUl>ChqbrnsbBsSa75`yjtm3M?|)doMoI@OXgx`>iqxMm8uE})Hs1V;I!@y$( zKJ8Vxv3}IBW@07-*t?eiABUG2%CDC6GGGe!Nc*{LY54h|{HhLi>t_n7(-AF6zB5~O zA~_yZ4KPHSG+C}$$a5@Bo=@CNJ#h=gwSJ9tpziwwCe25Sk5JV6w;8hE^&P|=y2PWW z!aA>d!9jO-pT%H0l+=P+J=5AKJoppWR2rwz#qVhkbV?|Z2UfUtkGV5_W>FDT?_W^I z8`cU2S+-OXQrJ(q(_>sN$H%KK@Svy%5X_QNw%Wkqk58Zc9j%N8&^B&Zet*UP@MGc} zDl~ z>mm(`&O39D$ye(;_;*k3)0CL&;>qv6s&IJOkB@h%b4y?gZm-p{LqliSoY{H@UtQ4^ z_JcX$Ovjyb@-OmJI~3avi`4Bpa9E|w=Ld}a(WwegH|uQ3$NMlRoqPQXq(~(hp4@rB zsfdh(l9m}3c~$NvQ3)^EW9+-!nYnY<$E6dP>blEct^E>B??Vh~RI3W#aoi~ZG`+TX zjv9fikMRLip9Jp(dSA|+YbF*=+`Se?33IE?qFQ@p=@US>A!V*W0#jRf(jIw>|X$dTU?_D7YXVOK&XUJ9)Z%{yWC6b*KLS#;8 zO+tUm5T3bgc$6tlsngw;$dbN`0Tub$amYj*{?cX5QoDseQICmHZ=<*W;Ux=JqaERI?kvT$*335jAH0?G;>rHjSS07KVsn6!)6gHJRRS`C0+OW zbtR9hVGy1QC_BEo;$9=)mCTEW0k~)Wx-t@jvb(hV>xkJ8jqTY$fW}bXL`#K*%&;eZ zT@-x3L1o`iC}tLO?d$ef+4)SH`7M@&dD;7AtdX01$;|HH4dU=*P_5Dg$AxPT)px1` zN!Ug2?fP+mzW9o!7}KkbmF3>;gusD9BdfhDZT&>0-jVU)+kxGU=gp3T4I0)-ohCFryao$=GC%)o2wU0kgOvQklgzBxXPZ_hi=jG(u#$40R>`FNXI*T#a*c11OU+3GD%9hYo$?5gaL zcz$38?yt(l5Ckv}a7c_z`kQq|<2lGouffFHpw9{9JL2?4<{FA8(#sC`^eK~Do6I~s z5-EB2*ks_Z{c?Viu*9w{yM7jxr!z6XG6|}zRu$)%VcRy{rMNyV>Jb^K;5G=yWU^~K z|H`tZ^Q2MWxBly3S*@{rbMgy|LojTYuv)$)u$o;;Jej9PMcl6ESH9`XRd&oT@->Yu z9h`1?k{koGyRSU8i+X^VHG;3;%TdItFRAVTQCrFYbyWjb$O{oLQ`ZNI=AQa z`p5Rs2Sf4`dyU*XUmW)nn80#xoR>=M`XW~Ww5-*A5FtRyca}Wg;W_0<*g`cW71BPs zCAYdF^P^T)?t%z`?NxilN_oMs1xw8%Hr>caA<(Tz-+ZN8f&X#0KppJ|aONc{-cs-% zFtFD)3mX7zk)5BH_};F4OCbA`oPw|%1Lu<2ggE9uPES2S{mE70bt{ldcscyi94%JCD6t5i>jdwI z+xam@D$-{2BQYmmKhyh!_0adH`6VXg{l^||U*EG6M$&6ijQ~TBnpU8hS)M8a&g}ph z94M4u1mZ@pC+blNV9TV>zKHfGz>9`Ed&h^|plzqhhypRgcFL%zdK#L*fXt&f-|R|v z{)tMVVYk=I9mQ<^A|XA_r;z1S4M@!8Vvad+$Q-W)J*IP=giWYd9Tv2tnI~A0BR~kg z@~r!YXv9m+*rgXSsT@3}1u;W2B6B=3*c(G{)tDELzdM19fthkT2zaM6X7STlx3&__ zZoe3!&0Sy;I91k-j&D$`pO6#oc5kTjEFINDsn22BN^%57`HWrv@Dt}_L#=eJTTEQ$(w9&UrG5PV5l`0#tJ8iT-$TScz`2^=iK@1L(^Dg?Cn={e zR7bM8q^GU)qumLI;JKLuAL;>Ix;EYmdhB?0oS`I)YfAT+J^rIRZ3k**d~+ze)d`y{ ztGD<2bHD83`}gl_h5A90FJ_jjXOs281r6)`V8y3tAwsm+KsDdfNs#^t~&P+Mt zZ$D-LZ%CewbKm0VLE!DNKW$`t0`37Dj}x5Hilc!Up;#Q-|5xDWY$>KO`pY%nDpKFS zr&@%&cL{M`hyAQcMUerfqjKBZsnW*)$#C8-pFSVv6imR`A+N;z1m9@gr@mc)xkq7d zUrIg2n4#ojB^HqePINQYXREIP%TKCWH2)pn2_K&V@TvTk_FdVR+N2H`YgiULDi<^* z>d|@-!#HOu+SjgFQ^>lccxy{S>x-5t1DD32Gmp`jNGT@wfL)~Q&c8(W0igCpBP zO$iZP?piz|2$#)%cu_jF&lAY8WEg`yrlZl6@ZyB@Q|-EM-=Iqq>~IJ(q8dU9wl;MR zaDie2#OKpAbHVVk*tK^0YVPCzaPj^Lt;A5~T>9vV|%Ud!?-T3ZzCu0Ar@@Lc<% zfPtz=fAeUCRQO}Lk&7=B70!NhA>8s!f%%pS)r3yQB|ZyyMyqF@aDC9w++r+LSauGl+^|UuHr&-Q>h!8ypmLohv=7|ZVIoe?g_?*dLF@1?*mhXFs*md)0=jskYl6Bt zn!0(09DvGyL&yr9%$r_h1jYLb8>*Lj2%5E0-8o8+dlye15s9U1_gX|gs!JD~h~u}? zt(U=DiLeGDm~iI6Wd*#d)j~1Py`3$(nX89(>*_{V56ib4NVu9d+eOW1y(iQXTIZ&2 ze-JNAYg-&@R8e)V#1Dh$in%H$bt0rn(K}t%o$QX(MgbFu!+nt_7+=wT)(SaLm~*DG zE`9du(?YS*4gpHc?(8o?jPv2R>Z56mkGE7{)B!Ps)EuB1=Zf}ik4;`u?6i$I*ETMq zK0*OF6T!>QxZq{ijrA!x@|pLXLB$VFbX&EidIyl<5y%xXcP2(H9z7e;tG;}%{(e7p z^_-^djC_C4+p1WT$J%h)>hpN;Aj*(EXJeG^Apgd{GZdN(Y65<*$wPpL~j~-@v8%9Oh@31cwf6Bg=v0H?^s#w)h zx$n#VOwuWZl3c=W7-`sq4v{>H9oY*$tueyB_e5c-op;HzyCExO-7%>xJ@Ci&wohioO@pmJrvY)TC1Z~Udw(}MbSd4nDbb+8$$m36_YyD8zLW5g zo(T24bRSmI*P&iedSBPu_{TPq2_jxx>BI=RUI23y>86^U1|cFKliUApLYvtWbb-Ae zpC`e~c77PV!_2{aIQ`uXe z7EYM=1*TJR`V-UYhMG<|O{7D=qa>S@sOWDgXPi$F#*e*LmE@_pRu=%Pv<;dtmaVrU zO^Pko?G7te0Ab{h=RYVP#r9u7Gib;Hh3=|wJ;pI$;@6$R&wu{d3skl-h(qep&h(9W zqTaF7nsj2yDnB16^3{u*<(Oq3QJ16Df1@H$Hg_`t7*Y(@Mk?K<6#Q0xW zjJA-F?KASQ3j*A|oQ`(Q6Gk0qj!y*Yp+%)|FZ&nf0RMMWBGDlXURVjpc)(CtM6A?# zYt}jZLK=&j9qIukS%XyihHNP6JFQ*N5J^vUJh$#Bjobqkf>raO8J%>ZH@)Zd>?k)` zU&g_^z8xTc0L}QWZxO~k7evYdeu=3cThe8{F^Op&JIaC3NDP1#YAl+6fe5I?=<8;B z-5ZB-`cEQ+`OOG*mup-$;Aj2==>N;v#JB9ZeER!nEyd}t$i z5SZzY&vi1-6_F7rt+6_gz3>v3lzdw^invs~2m3-4&)$xIF@h-k3qujrmUzqEzjhAu z3ElX|a9beLdSU}lz(taaL4`A{{rE*hQH=ptQQY0^$2XTmpFV$BTW4HK~C(Cz>XCCCA*c$puv1@7S9!Dd2%WVGZ&zH^eurjrV{zweL9AfI? z#|7^K+O66r_XT+U92nRxDk^%$&292l^sm9e9bKdDJ$-$L;wsB&zJ*o?p-MjaCNJz+DSy}%xuh$I!939=wa&^CBc4J=2z4A6` zY3aKq?Iuo6yMu#+fA;o@@bdDO8MjPLN%i&h{Tv$FX^~cUynUYe&~ptdYiq@!*rDIQ zH+%A-JDXpLpGKj4efaB_n?1Xfr0G+uo5jA<)6*T}(;A{7CcFZ}PeVfsDl1RWKF^2+ zsR$Go7oRaPY1R8Z{W&~5{EU~E+^om7;o;$vCr>^c!%Ewp@vnEd%WNC-2T z1nmS#3gzOxKOfHc+R}xWPwMGC8Cx@8DzWRAebz~R{nU5(Sq%T0XyI=adN-Hz{`uSZ zp9vY#JN@}O>;12P;(vF_|ND pd.DataFrame: rows: list[dict[str, Any]] = [] total_loan_amount = float(oot["loan_weight"].sum()) grouped = oot.groupby(["period", "original_grade"], dropna=False, observed=True) - for (period, grade), group in grouped: + for key, group in grouped: + period, grade = cast(tuple[object, object], key) row = { "period": str(period), "original_grade": str(grade), @@ -626,7 +627,11 @@ def _solve_exact_policy( alpha: float = 0.01, budget: float = 1_000_000.0, ) -> dict[str, Any]: - from src.optimization.portfolio_model import compute_effective_pd, optimize_portfolio_allocation + from src.optimization.portfolio_model import ( + compute_effective_pd, + optimize_portfolio_allocation, + solution_allocation_vector, + ) pd_point, pd_low, pd_high = _interval_arrays_at_alpha(aligned, alpha) effective_pd = compute_effective_pd( @@ -670,10 +675,7 @@ def _solve_exact_policy( threads=4, solver_backend=str(policy["solver_backend"]), ) - allocation = np.array( - [float(solution["allocation"].get(i, 0.0)) for i in range(len(aligned))], - dtype=float, - ) + allocation = solution_allocation_vector(solution, len(aligned)) total_allocated = float(np.sum(allocation * loan_amounts)) weights = (allocation * loan_amounts) / max(total_allocated, 1e-6) funded_mask = weights > 1e-8 diff --git a/scripts/archive/search/monitor_regret_auditability.py b/scripts/archive/search/monitor_regret_auditability.py index a3cb693..4b8e150 100644 --- a/scripts/archive/search/monitor_regret_auditability.py +++ b/scripts/archive/search/monitor_regret_auditability.py @@ -5,7 +5,6 @@ import argparse import csv import json -import os import re import sqlite3 import subprocess @@ -28,7 +27,7 @@ def emit(text: str = "") -> None: def clear_screen() -> None: """Clear the Windows console.""" - os.system("cls") + subprocess.run(["cmd", "/c", "cls"], check=False) def load_json(path: Path) -> dict[str, Any]: diff --git a/scripts/backtest_conformal_coverage.py b/scripts/backtest_conformal_coverage.py index 7b2f9dd..bcdd899 100644 --- a/scripts/backtest_conformal_coverage.py +++ b/scripts/backtest_conformal_coverage.py @@ -233,18 +233,18 @@ def _build_alerts( if not alerts: return pd.DataFrame( - columns=[ - "level", - "month", - "grade", - "severity", - "n", - "coverage_90", - "coverage_95", - "avg_width_90", - "rule", - "recommended_action", - ] + { + "level": pd.Series(dtype="object"), + "month": pd.Series(dtype="object"), + "grade": pd.Series(dtype="object"), + "severity": pd.Series(dtype="object"), + "n": pd.Series(dtype="int64"), + "coverage_90": pd.Series(dtype="float64"), + "coverage_95": pd.Series(dtype="float64"), + "avg_width_90": pd.Series(dtype="float64"), + "rule": pd.Series(dtype="object"), + "recommended_action": pd.Series(dtype="object"), + } ) return pd.DataFrame(alerts).sort_values(["month", "level", "grade"]).reset_index(drop=True) diff --git a/scripts/benchmark_conformal_variants.py b/scripts/benchmark_conformal_variants.py index 059d384..65741c3 100644 --- a/scripts/benchmark_conformal_variants.py +++ b/scripts/benchmark_conformal_variants.py @@ -9,6 +9,7 @@ import argparse import json import pickle +from dataclasses import dataclass, field from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -42,6 +43,48 @@ DEFAULT_POLICY_CONFIG = "configs/crpto_conformal_policy.yaml" +@dataclass(frozen=True) +class BenchmarkData: + model: Any + calibrator: Any | None + cal_df: pd.DataFrame + test_df: pd.DataFrame + features: list[str] + categorical: list[str] + X_cal: pd.DataFrame + y_cal: pd.Series + X_test: pd.DataFrame + y_test: np.ndarray + group_cal: pd.Series + group_test: pd.Series + issue_test: pd.Series + y_prob_cal_raw: np.ndarray + y_prob_test_raw: np.ndarray + y_prob_calibrated: np.ndarray + y_prob_test_calibrated: np.ndarray + prob_cal_lookup: dict[str, np.ndarray] + prob_test_lookup: dict[str, np.ndarray] + + +@dataclass(frozen=True) +class SearchSpace: + partition_candidates: tuple[str, ...] + partition_probability_sources: tuple[str, ...] + n_score_bins_candidates: tuple[int, ...] + fallback_modes: tuple[str, ...] + score_scale_families: tuple[str, ...] + min_group_sizes: tuple[int, ...] + calibration_size_fractions: tuple[float, ...] + + +@dataclass +class VariantResults: + rows: list[dict[str, Any]] = field(default_factory=list) + by_group_rows: list[pd.DataFrame] = field(default_factory=list) + temporal_rows: list[pd.DataFrame] = field(default_factory=list) + local_rows: list[pd.DataFrame] = field(default_factory=list) + + def _summarize_variant( name: str, y_true: np.ndarray, @@ -101,14 +144,14 @@ def _summarize_temporal_stability( "stability_over_time": float("inf"), }, pd.DataFrame( - columns=[ - "variant", - "month", - "n", - "coverage_90", - "avg_width_90", - "coverage_gap_90", - ] + { + "variant": pd.Series(dtype="object"), + "month": pd.Series(dtype="object"), + "n": pd.Series(dtype="int64"), + "coverage_90": pd.Series(dtype="float64"), + "avg_width_90": pd.Series(dtype="float64"), + "coverage_gap_90": pd.Series(dtype="float64"), + } ), ) @@ -210,36 +253,13 @@ def _variant_name( return "::".join(parts) -def main( - alpha: float = 0.10, - selected_config_path: str = "models/conformal_results_mondrian.pkl", - min_group_size_default: int = 500, - cross_cal_sample_size: int = 5000, - cross_test_sample_size: int = 5000, - calibration_size_fractions: tuple[float, ...] = (0.25, 0.50, 0.75, 1.0), - partition_candidates: tuple[str, ...] = ( - "grade", - "score_decile_mondrian", - "grade_x_scoreband_mondrian", - ), - partition_probability_sources: tuple[str, ...] = ("raw",), - n_score_bins_candidates: tuple[int, ...] = (10,), - fallback_modes: tuple[str, ...] = ("grade_then_global",), - score_scale_families: tuple[str, ...] = ("none", "bernoulli_sqrt"), - min_group_sizes: tuple[int, ...] | None = None, - artifact_namespace: str | None = None, - calibrator_override_path: str | None = None, - policy_config_path: str = DEFAULT_POLICY_CONFIG, - collect_local_diagnostics: bool = False, -) -> None: - policy = _load_policy_config(policy_config_path).get("policy", {}) or {} +def _load_benchmark_data(calibrator_override_path: str | None) -> BenchmarkData: model, _ = _load_model() calibrator = _load_calibrator(calibrator_override_path) cal_df = read_with_fallback( "data/processed/calibration_fe.parquet", "data/processed/calibration.parquet" ) test_df = read_with_fallback("data/processed/test_fe.parquet", "data/processed/test.parquet") - features, categorical = _resolve_features(model, cal_df, test_df) X_cal = _build_feature_matrix(cal_df, features, categorical) y_cal = cal_df[TARGET_COL].astype(float) @@ -260,161 +280,239 @@ def main( if calibrator is not None else np.asarray(y_prob_test_raw, dtype=float) ) - partition_candidates = tuple( - dict.fromkeys(str(x).strip() for x in partition_candidates if str(x).strip()) - ) or ("grade",) - partition_probability_sources = tuple( - dict.fromkeys( - str(x).strip().lower() for x in partition_probability_sources if str(x).strip() - ) - ) or ("raw",) - n_score_bins_candidates = tuple(int(x) for x in n_score_bins_candidates if int(x) > 0) or (10,) - fallback_modes = tuple( - dict.fromkeys(str(x).strip().lower() for x in fallback_modes if str(x).strip()) - ) or ("grade_then_global",) - score_scale_families = tuple( - dict.fromkeys(str(x).strip().lower() for x in score_scale_families if str(x).strip()) - ) or ("none",) - min_group_sizes = tuple( - int(x) for x in (min_group_sizes or (min_group_size_default,)) if int(x) > 0 - ) or (int(min_group_size_default),) - prob_cal_lookup = {"raw": y_prob_cal_raw, "calibrated": y_prob_calibrated} - prob_test_lookup = {"raw": y_prob_test_raw, "calibrated": y_prob_test_calibrated} + return BenchmarkData( + model=model, + calibrator=calibrator, + cal_df=cal_df, + test_df=test_df, + features=features, + categorical=categorical, + X_cal=X_cal, + y_cal=y_cal, + X_test=X_test, + y_test=y_test, + group_cal=group_cal, + group_test=group_test, + issue_test=issue_test, + y_prob_cal_raw=y_prob_cal_raw, + y_prob_test_raw=y_prob_test_raw, + y_prob_calibrated=y_prob_calibrated, + y_prob_test_calibrated=y_prob_test_calibrated, + prob_cal_lookup={"raw": y_prob_cal_raw, "calibrated": y_prob_calibrated}, + prob_test_lookup={"raw": y_prob_test_raw, "calibrated": y_prob_test_calibrated}, + ) + + +def _unique_clean_strings(values: tuple[str, ...], fallback: tuple[str, ...]) -> tuple[str, ...]: + cleaned = tuple(dict.fromkeys(str(x).strip() for x in values if str(x).strip())) + return cleaned or fallback + + +def _unique_clean_lower_strings( + values: tuple[str, ...], + fallback: tuple[str, ...], +) -> tuple[str, ...]: + cleaned = tuple(dict.fromkeys(str(x).strip().lower() for x in values if str(x).strip())) + return cleaned or fallback + + +def _positive_int_tuple(values: tuple[int, ...], fallback: tuple[int, ...]) -> tuple[int, ...]: + cleaned = tuple(int(x) for x in values if int(x) > 0) + return cleaned or fallback + + +def _min_group_size_tuple( + values: tuple[int, ...] | None, + default: int, +) -> tuple[int, ...]: + return _positive_int_tuple(values or (default,), (int(default),)) + + +def _valid_fraction_tuple(values: tuple[float, ...]) -> tuple[float, ...]: + return tuple(float(x) for x in values if 0 < float(x) <= 1) + + +def _normalize_search_space( + *, + calibration_size_fractions: tuple[float, ...], + partition_candidates: tuple[str, ...], + partition_probability_sources: tuple[str, ...], + n_score_bins_candidates: tuple[int, ...], + fallback_modes: tuple[str, ...], + score_scale_families: tuple[str, ...], + min_group_sizes: tuple[int, ...] | None, + min_group_size_default: int, +) -> SearchSpace: + return SearchSpace( + partition_candidates=_unique_clean_strings(partition_candidates, ("grade",)), + partition_probability_sources=_unique_clean_lower_strings( + partition_probability_sources, ("raw",) + ), + n_score_bins_candidates=_positive_int_tuple(n_score_bins_candidates, (10,)), + fallback_modes=_unique_clean_lower_strings(fallback_modes, ("grade_then_global",)), + score_scale_families=_unique_clean_lower_strings(score_scale_families, ("none",)), + min_group_sizes=_min_group_size_tuple(min_group_sizes, min_group_size_default), + calibration_size_fractions=_valid_fraction_tuple(calibration_size_fractions), + ) + + +def _append_global_variant( + *, + data: BenchmarkData, + results: VariantResults, + alpha: float, +) -> None: + _y_pred_global, y_int_global = create_pd_intervals( + classifier=data.model, + X_cal=data.X_cal, + y_cal=data.y_cal, + X_test=data.X_test, + alpha=alpha, + calibrator=data.calibrator, + ) + row, by_group = _summarize_variant( + "global_split", data.y_test, y_int_global, data.group_test, alpha + ) + temporal_meta, temporal_monthly = _summarize_temporal_stability( + "global_split", data.y_test, y_int_global, data.issue_test + ) + row.update(temporal_meta) + results.rows.append(row) + results.by_group_rows.append(by_group) + results.temporal_rows.append(temporal_monthly) + + +def _append_mondrian_variant( + *, + data: BenchmarkData, + results: VariantResults, partition_cache: dict[ tuple[str, str, int, str, int, float, int], tuple[pd.Series, pd.Series, dict[str, Any]], - ] = {} - - rows: list[dict[str, Any]] = [] - by_group_rows: list[pd.DataFrame] = [] - temporal_rows: list[pd.DataFrame] = [] - local_rows: list[pd.DataFrame] = [] - - def _append_mondrian_variant( - name: str, - *, - partition: str, - partition_probability_source: str, - n_score_bins: int, - fallback_mode: str, - score_scale_family: str, - alpha_used: float = alpha, - min_group_size: int = min_group_size_default, - X_cal_variant: pd.DataFrame | None = None, - y_cal_variant: pd.Series | None = None, - y_prob_cal_variant: np.ndarray | None = None, - base_groups_cal_variant: pd.Series | None = None, - calibration_fraction: float | None = None, - ) -> None: - y_cal_use = y_cal if y_cal_variant is None else y_cal_variant - y_prob_cal_use = ( - prob_cal_lookup[partition_probability_source] - if y_prob_cal_variant is None - else y_prob_cal_variant - ) - y_interval_cal_pred = ( - y_prob_calibrated if y_prob_cal_variant is None else y_prob_cal_variant - ) - base_groups_cal_use = ( - group_cal if base_groups_cal_variant is None else base_groups_cal_variant - ) - cache_key = ( - str(partition), - str(partition_probability_source), - int(n_score_bins), - str(fallback_mode), - int(min_group_size), - float(calibration_fraction or 1.0), - int(len(y_cal_use)), - ) - if cache_key in partition_cache: - group_cal_part, group_test_part, partition_meta = partition_cache[cache_key] - else: - group_cal_part, group_test_part, partition_meta = build_mondrian_partition_labels( - y_prob_cal=y_prob_cal_use, - y_prob_eval=prob_test_lookup[partition_probability_source], - partition=partition, - base_groups_cal=base_groups_cal_use.iloc[: len(y_cal_use)].reset_index(drop=True), - base_groups_eval=group_test, - n_score_bins=n_score_bins, - min_group_size=min_group_size, - fallback_mode=fallback_mode, - ) - partition_cache[cache_key] = (group_cal_part, group_test_part, partition_meta) - _y_pred, y_int, _ = create_pd_intervals_mondrian_from_predictions( - y_cal_pred=y_interval_cal_pred, - y_test_pred=y_prob_test_calibrated, - y_cal=y_cal_use, - group_cal=group_cal_part, - group_test=group_test_part, - alpha=alpha_used, + ], + name: str, + partition: str, + partition_probability_source: str, + n_score_bins: int, + fallback_mode: str, + score_scale_family: str, + alpha: float, + alpha_used: float, + min_group_size: int, + collect_local_diagnostics: bool, + y_cal_variant: pd.Series | None = None, + y_prob_cal_variant: np.ndarray | None = None, + base_groups_cal_variant: pd.Series | None = None, + calibration_fraction: float | None = None, +) -> None: + y_cal_use = data.y_cal if y_cal_variant is None else y_cal_variant + y_prob_cal_use = ( + data.prob_cal_lookup[partition_probability_source] + if y_prob_cal_variant is None + else y_prob_cal_variant + ) + y_interval_cal_pred = ( + data.y_prob_calibrated if y_prob_cal_variant is None else y_prob_cal_variant + ) + base_groups_cal_use = ( + data.group_cal if base_groups_cal_variant is None else base_groups_cal_variant + ) + cache_key = ( + str(partition), + str(partition_probability_source), + int(n_score_bins), + str(fallback_mode), + int(min_group_size), + float(calibration_fraction or 1.0), + int(len(y_cal_use)), + ) + if cache_key in partition_cache: + group_cal_part, group_test_part, partition_meta = partition_cache[cache_key] + else: + group_cal_part, group_test_part, partition_meta = build_mondrian_partition_labels( + y_prob_cal=y_prob_cal_use, + y_prob_eval=data.prob_test_lookup[partition_probability_source], + partition=partition, + base_groups_cal=base_groups_cal_use.iloc[: len(y_cal_use)].reset_index(drop=True), + base_groups_eval=data.group_test, + n_score_bins=n_score_bins, min_group_size=min_group_size, - score_scale_family=score_scale_family, - log_summary=False, - ) - row, by_group = _summarize_variant(name, y_test, y_int, group_test_part, alpha) - temporal_meta, temporal_monthly = _summarize_temporal_stability( - name, y_test, y_int, issue_test + fallback_mode=fallback_mode, ) - row.update(temporal_meta) - row["partition"] = partition_meta.get("partition", partition) - row["partition_probability_source"] = partition_probability_source - row["n_score_bins"] = int(n_score_bins) - row["fallback_mode"] = str(partition_meta.get("fallback_mode", fallback_mode)) - row["scaled_scores"] = bool(score_scale_family != "none") - row["score_scale_family"] = score_scale_family - row["min_group_size"] = int(min_group_size) - row["selected_alpha_used"] = float(alpha_used) - row["fallback_groups_n"] = len(partition_meta.get("fallback_groups", [])) - row["calibration_fraction"] = float(calibration_fraction or 1.0) - rows.append(row) - by_group_rows.append(by_group) - temporal_rows.append(temporal_monthly) - - if collect_local_diagnostics: - local_diag = pd.DataFrame( - { - "record_type": "local_partition_summary", - "variant": name, - "partition": row["partition"], - "group": pd.Series(group_test_part).astype(str), - "y_true": y_test, - "low": y_int[:, 0], - "high": y_int[:, 1], - } - ) - local_diag["covered"] = ( - (local_diag["y_true"] >= local_diag["low"]) - & (local_diag["y_true"] <= local_diag["high"]) - ).astype(float) - local_diag["width"] = local_diag["high"] - local_diag["low"] - local_rows.append(local_diag) - - _y_pred_global, y_int_global = create_pd_intervals( - classifier=model, - X_cal=X_cal, - y_cal=y_cal, - X_test=X_test, - alpha=alpha, - calibrator=calibrator, + partition_cache[cache_key] = (group_cal_part, group_test_part, partition_meta) + _y_pred, y_int, _ = create_pd_intervals_mondrian_from_predictions( + y_cal_pred=y_interval_cal_pred, + y_test_pred=data.y_prob_test_calibrated, + y_cal=y_cal_use, + group_cal=group_cal_part, + group_test=group_test_part, + alpha=alpha_used, + min_group_size=min_group_size, + score_scale_family=score_scale_family, + log_summary=False, ) - row, by_group = _summarize_variant("global_split", y_test, y_int_global, group_test, alpha) + row, by_group = _summarize_variant(name, data.y_test, y_int, group_test_part, alpha) temporal_meta, temporal_monthly = _summarize_temporal_stability( - "global_split", y_test, y_int_global, issue_test + name, data.y_test, y_int, data.issue_test ) row.update(temporal_meta) - rows.append(row) - by_group_rows.append(by_group) - temporal_rows.append(temporal_monthly) - - for partition in partition_candidates: - for partition_probability_source in partition_probability_sources: - for n_score_bins in n_score_bins_candidates: - for fallback_mode in fallback_modes: - for score_scale_family in score_scale_families: - for min_group_size in min_group_sizes: + row["partition"] = partition_meta.get("partition", partition) + row["partition_probability_source"] = partition_probability_source + row["n_score_bins"] = int(n_score_bins) + row["fallback_mode"] = str(partition_meta.get("fallback_mode", fallback_mode)) + row["scaled_scores"] = bool(score_scale_family != "none") + row["score_scale_family"] = score_scale_family + row["min_group_size"] = int(min_group_size) + row["selected_alpha_used"] = float(alpha_used) + row["fallback_groups_n"] = len(partition_meta.get("fallback_groups", [])) + row["calibration_fraction"] = float(calibration_fraction or 1.0) + results.rows.append(row) + results.by_group_rows.append(by_group) + results.temporal_rows.append(temporal_monthly) + + if collect_local_diagnostics: + local_diag = pd.DataFrame( + { + "record_type": "local_partition_summary", + "variant": name, + "partition": row["partition"], + "group": pd.Series(group_test_part).astype(str), + "y_true": data.y_test, + "low": y_int[:, 0], + "high": y_int[:, 1], + } + ) + local_diag["covered"] = ( + (local_diag["y_true"] >= local_diag["low"]) + & (local_diag["y_true"] <= local_diag["high"]) + ).astype(float) + local_diag["width"] = local_diag["high"] - local_diag["low"] + results.local_rows.append(local_diag) + + +def _append_search_space_variants( + *, + data: BenchmarkData, + results: VariantResults, + partition_cache: dict[ + tuple[str, str, int, str, int, float, int], + tuple[pd.Series, pd.Series, dict[str, Any]], + ], + space: SearchSpace, + alpha: float, + collect_local_diagnostics: bool, +) -> None: + for partition in space.partition_candidates: + for partition_probability_source in space.partition_probability_sources: + for n_score_bins in space.n_score_bins_candidates: + for fallback_mode in space.fallback_modes: + for score_scale_family in space.score_scale_families: + for min_group_size in space.min_group_sizes: _append_mondrian_variant( - _variant_name( + data=data, + results=results, + partition_cache=partition_cache, + name=_variant_name( partition=partition, partition_probability_source=partition_probability_source, n_score_bins=n_score_bins, @@ -427,40 +525,52 @@ def _append_mondrian_variant( n_score_bins=n_score_bins, fallback_mode=fallback_mode, score_scale_family=score_scale_family, + alpha=alpha, + alpha_used=alpha, min_group_size=min_group_size, + collect_local_diagnostics=collect_local_diagnostics, ) - rng = np.random.RandomState(42) - cal_idx = ( - rng.choice(len(y_cal), size=min(int(cross_cal_sample_size), len(y_cal)), replace=False) - if len(y_cal) > int(cross_cal_sample_size) - else np.arange(len(y_cal)) - ) - test_idx = ( - rng.choice(len(y_test), size=min(int(cross_test_sample_size), len(y_test)), replace=False) - if len(y_test) > int(cross_test_sample_size) - else np.arange(len(y_test)) + +def _sample_indices(n_rows: int, sample_size: int, rng: np.random.RandomState) -> np.ndarray: + return ( + rng.choice(n_rows, size=min(int(sample_size), n_rows), replace=False) + if n_rows > int(sample_size) + else np.arange(n_rows) ) + + +def _append_cross_conformal_variant( + *, + data: BenchmarkData, + results: VariantResults, + alpha: float, + cross_cal_sample_size: int, + cross_test_sample_size: int, +) -> None: + rng = np.random.RandomState(42) + cal_idx = _sample_indices(len(data.y_cal), cross_cal_sample_size, rng) + test_idx = _sample_indices(len(data.y_test), cross_test_sample_size, rng) _y_pred_cross, y_int_cross = create_cross_conformal_score_intervals( - y_cal=y_cal.iloc[cal_idx].reset_index(drop=True), - y_prob_cal=y_prob_calibrated[cal_idx], - y_prob_test=y_prob_test_calibrated[test_idx], + y_cal=data.y_cal.iloc[cal_idx].reset_index(drop=True), + y_prob_cal=data.y_prob_calibrated[cal_idx], + y_prob_test=data.y_prob_test_calibrated[test_idx], alpha=alpha, method="plus", cv=5, ) row, by_group = _summarize_variant( "cross_conformal_score_space", - y_test[test_idx], + data.y_test[test_idx], y_int_cross, - group_test.iloc[test_idx].reset_index(drop=True), + data.group_test.iloc[test_idx].reset_index(drop=True), alpha, ) temporal_meta, temporal_monthly = _summarize_temporal_stability( "cross_conformal_score_space", - y_test[test_idx], + data.y_test[test_idx], y_int_cross, - issue_test.iloc[test_idx].reset_index(drop=True), + data.issue_test.iloc[test_idx].reset_index(drop=True), ) row.update(temporal_meta) row["implementation_note"] = ( @@ -468,108 +578,161 @@ def _append_mondrian_variant( ) row["evaluation_sample_n_cal"] = len(cal_idx) row["evaluation_sample_n_test"] = len(test_idx) - rows.append(row) - by_group_rows.append(by_group) - temporal_rows.append(temporal_monthly) + results.rows.append(row) + results.by_group_rows.append(by_group) + results.temporal_rows.append(temporal_monthly) + +def _append_selected_config_variant( + *, + data: BenchmarkData, + results: VariantResults, + partition_cache: dict[ + tuple[str, str, int, str, int, float, int], + tuple[pd.Series, pd.Series, dict[str, Any]], + ], + selected_config_path: str, + alpha: float, + min_group_size_default: int, + collect_local_diagnostics: bool, +) -> None: cfg_path = Path(selected_config_path) - if cfg_path.exists(): - with open(cfg_path, "rb") as f: - payload = pickle.load(f) - best = payload.get("tuning_90_best", {}) if isinstance(payload, dict) else {} - alpha_used = float(best.get("alpha_used_90", alpha)) - min_group_size = int(best.get("min_group_size", min_group_size_default)) - partition_probability_source = str(best.get("partition_probability_source", "raw")) - n_score_bins = int(best.get("n_score_bins", 10)) - fallback_mode = str(best.get("fallback_mode", "grade_then_global")) - score_scale_family = str(best.get("score_scale_family", "none")) - _append_mondrian_variant( - "mondrian_selected_cfg", - partition=str(best.get("partition", "grade")), - partition_probability_source=partition_probability_source, - n_score_bins=n_score_bins, - fallback_mode=fallback_mode, - score_scale_family=score_scale_family, - alpha_used=alpha_used, - min_group_size=min_group_size, - ) + if not cfg_path.exists(): + return + with open(cfg_path, "rb") as f: + payload = pickle.load(f) + best = payload.get("tuning_90_best", {}) if isinstance(payload, dict) else {} + _append_mondrian_variant( + data=data, + results=results, + partition_cache=partition_cache, + name="mondrian_selected_cfg", + partition=str(best.get("partition", "grade")), + partition_probability_source=str(best.get("partition_probability_source", "raw")), + n_score_bins=int(best.get("n_score_bins", 10)), + fallback_mode=str(best.get("fallback_mode", "grade_then_global")), + score_scale_family=str(best.get("score_scale_family", "none")), + alpha=alpha, + alpha_used=float(best.get("alpha_used_90", alpha)), + min_group_size=int(best.get("min_group_size", min_group_size_default)), + collect_local_diagnostics=collect_local_diagnostics, + ) + +def _calibration_sensitivity_rows( + *, + data: BenchmarkData, + space: SearchSpace, + alpha: float, +) -> list[dict[str, Any]]: sensitivity_rows: list[dict[str, Any]] = [] - for frac in calibration_size_fractions: - frac_float = float(frac) - if frac_float <= 0 or frac_float > 1: - continue - cal_df_sub = _subset_calibration_frame(cal_df, calibration_fraction=frac_float) - X_cal_sub = _build_feature_matrix(cal_df_sub, features, categorical) + for frac in space.calibration_size_fractions: + cal_df_sub = _subset_calibration_frame(data.cal_df, calibration_fraction=float(frac)) + X_cal_sub = _build_feature_matrix(cal_df_sub, data.features, data.categorical) y_cal_sub = cal_df_sub[TARGET_COL].astype(float).reset_index(drop=True) group_cal_sub = cal_df_sub[GROUP_COL].fillna("UNKNOWN").astype(str).reset_index(drop=True) - y_prob_cal_sub_raw = model.predict_proba(X_cal_sub)[:, 1] + y_prob_cal_sub_raw = data.model.predict_proba(X_cal_sub)[:, 1] y_prob_cal_sub_calibrated = ( - apply_probability_calibrator(calibrator, y_prob_cal_sub_raw) - if calibrator is not None + apply_probability_calibrator(data.calibrator, y_prob_cal_sub_raw) + if data.calibrator is not None else np.asarray(y_prob_cal_sub_raw, dtype=float) ) prob_cal_sub_lookup = {"raw": y_prob_cal_sub_raw, "calibrated": y_prob_cal_sub_calibrated} for partition in ("score_decile_mondrian", "grade_x_scoreband_mondrian"): - for partition_probability_source in partition_probability_sources: - group_cal_part, group_test_part, partition_meta = build_mondrian_partition_labels( - y_prob_cal=prob_cal_sub_lookup[partition_probability_source], - y_prob_eval=prob_test_lookup[partition_probability_source], - partition=partition, - base_groups_cal=group_cal_sub, - base_groups_eval=group_test, - n_score_bins=n_score_bins_candidates[0], - min_group_size=min_group_sizes[0], - fallback_mode=fallback_modes[0], - ) - _y_pred_sub, y_int_sub, _ = create_pd_intervals_mondrian_from_predictions( - y_cal_pred=y_prob_cal_sub_calibrated, - y_test_pred=y_prob_test_calibrated, - y_cal=y_cal_sub, - group_cal=group_cal_part, - group_test=group_test_part, - alpha=alpha, - min_group_size=min_group_sizes[0], - score_scale_family=score_scale_families[0], - log_summary=False, - ) - metrics = validate_coverage(y_test, y_int_sub, alpha=alpha, log_summary=False) - by_group_sub = conditional_coverage_by_group(y_test, y_int_sub, group_test_part) - stability_sub, _ = _summarize_temporal_stability( - partition, y_test, y_int_sub, issue_test - ) + for partition_probability_source in space.partition_probability_sources: sensitivity_rows.append( - { - "record_type": "calibration_size_sensitivity", - "variant": _variant_name( - partition=partition, - partition_probability_source=partition_probability_source, - n_score_bins=n_score_bins_candidates[0], - fallback_mode=fallback_modes[0], - score_scale_family=score_scale_families[0], - min_group_size=min_group_sizes[0], - calibration_fraction=frac_float, - ), - "partition": partition_meta.get("partition", partition), - "partition_probability_source": partition_probability_source, - "calibration_fraction": frac_float, - "n_calibration_rows": len(X_cal_sub), - "coverage": float(metrics["empirical_coverage"]), - "coverage_gap": float(metrics["coverage_gap"]), - "avg_width": float(metrics["avg_interval_width"]), - "min_group_coverage": float(by_group_sub["coverage"].min()), - "winkler_90": float( - np.mean( - winkler_interval_score( - y_test, y_int_sub[:, 0], y_int_sub[:, 1], alpha=alpha - ) - ) - ), - "stability_over_time": float(stability_sub["stability_over_time"]), - } + _calibration_sensitivity_row( + data=data, + space=space, + alpha=alpha, + partition=partition, + partition_probability_source=partition_probability_source, + calibration_fraction=float(frac), + X_cal_sub=X_cal_sub, + y_cal_sub=y_cal_sub, + group_cal_sub=group_cal_sub, + y_prob_cal_sub_calibrated=y_prob_cal_sub_calibrated, + y_prob_cal_sub=prob_cal_sub_lookup[partition_probability_source], + ) ) + return sensitivity_rows + - bench = pd.DataFrame(rows) +def _calibration_sensitivity_row( + *, + data: BenchmarkData, + space: SearchSpace, + alpha: float, + partition: str, + partition_probability_source: str, + calibration_fraction: float, + X_cal_sub: pd.DataFrame, + y_cal_sub: pd.Series, + group_cal_sub: pd.Series, + y_prob_cal_sub_calibrated: np.ndarray, + y_prob_cal_sub: np.ndarray, +) -> dict[str, Any]: + group_cal_part, group_test_part, partition_meta = build_mondrian_partition_labels( + y_prob_cal=y_prob_cal_sub, + y_prob_eval=data.prob_test_lookup[partition_probability_source], + partition=partition, + base_groups_cal=group_cal_sub, + base_groups_eval=data.group_test, + n_score_bins=space.n_score_bins_candidates[0], + min_group_size=space.min_group_sizes[0], + fallback_mode=space.fallback_modes[0], + ) + _y_pred_sub, y_int_sub, _ = create_pd_intervals_mondrian_from_predictions( + y_cal_pred=y_prob_cal_sub_calibrated, + y_test_pred=data.y_prob_test_calibrated, + y_cal=y_cal_sub, + group_cal=group_cal_part, + group_test=group_test_part, + alpha=alpha, + min_group_size=space.min_group_sizes[0], + score_scale_family=space.score_scale_families[0], + log_summary=False, + ) + metrics = validate_coverage(data.y_test, y_int_sub, alpha=alpha, log_summary=False) + by_group_sub = conditional_coverage_by_group(data.y_test, y_int_sub, group_test_part) + stability_sub, _ = _summarize_temporal_stability( + partition, data.y_test, y_int_sub, data.issue_test + ) + return { + "record_type": "calibration_size_sensitivity", + "variant": _variant_name( + partition=partition, + partition_probability_source=partition_probability_source, + n_score_bins=space.n_score_bins_candidates[0], + fallback_mode=space.fallback_modes[0], + score_scale_family=space.score_scale_families[0], + min_group_size=space.min_group_sizes[0], + calibration_fraction=calibration_fraction, + ), + "partition": partition_meta.get("partition", partition), + "partition_probability_source": partition_probability_source, + "calibration_fraction": calibration_fraction, + "n_calibration_rows": len(X_cal_sub), + "coverage": float(metrics["empirical_coverage"]), + "coverage_gap": float(metrics["coverage_gap"]), + "avg_width": float(metrics["avg_interval_width"]), + "min_group_coverage": float(by_group_sub["coverage"].min()), + "winkler_90": float( + np.mean( + winkler_interval_score(data.y_test, y_int_sub[:, 0], y_int_sub[:, 1], alpha=alpha) + ) + ), + "stability_over_time": float(stability_sub["stability_over_time"]), + } + + +def _final_benchmark_frames( + *, + results: VariantResults, + sensitivity_rows: list[dict[str, Any]], + policy: dict[str, Any], +) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame]: + bench = pd.DataFrame(results.rows) bench["promotion_pass"] = bench.apply(lambda row: _promotion_pass(row, policy), axis=1) bench = bench.sort_values( [ @@ -584,53 +747,145 @@ def _append_mondrian_variant( ).reset_index(drop=True) bench["selection_rank"] = np.arange(1, len(bench) + 1, dtype=int) bench_by_group = ( - pd.concat(by_group_rows, ignore_index=True) + pd.concat(results.by_group_rows, ignore_index=True) .sort_values(["variant", "group"]) .reset_index(drop=True) ) temporal_diagnostics = ( - pd.concat(temporal_rows, ignore_index=True) + pd.concat(results.temporal_rows, ignore_index=True) .sort_values(["variant", "month"]) .reset_index(drop=True) ) - local_diagnostics = pd.concat(local_rows, ignore_index=True) if local_rows else pd.DataFrame() + local_diagnostics = ( + pd.concat(results.local_rows, ignore_index=True) if results.local_rows else pd.DataFrame() + ) if sensitivity_rows: local_diagnostics = pd.concat( [local_diagnostics, pd.DataFrame(sensitivity_rows)], ignore_index=True, sort=False, ) + return bench, bench_by_group, temporal_diagnostics, local_diagnostics - output_paths = _build_output_paths(artifact_namespace) + +def _selected_config_local_diagnostics( + *, + selected_config_path: str, + selected_intervals_path: Path, +) -> pd.DataFrame: selected_cfg_path = Path(selected_config_path) - selected_intervals_path = output_paths["selected_intervals"] - if selected_cfg_path.exists() and selected_intervals_path.exists(): - with open(selected_cfg_path, "rb") as f: - selected_payload = pickle.load(f) - selected_intervals = pd.read_parquet(selected_intervals_path) - if {"y_true", "pd_low_90", "pd_high_90", GROUP_COL}.issubset(selected_intervals.columns): - selected_local = pd.DataFrame( - { - "record_type": "local_partition_summary", - "variant": "mondrian_selected_cfg", - "partition": str(selected_payload.get("partition", "grade")), - "group": selected_intervals[GROUP_COL].fillna("UNKNOWN").astype(str), - "y_true": pd.to_numeric(selected_intervals["y_true"], errors="coerce"), - "low": pd.to_numeric(selected_intervals["pd_low_90"], errors="coerce"), - "high": pd.to_numeric(selected_intervals["pd_high_90"], errors="coerce"), - } - ) - selected_local["covered"] = ( - (selected_local["y_true"] >= selected_local["low"]) - & (selected_local["y_true"] <= selected_local["high"]) - ).astype(float) - selected_local["width"] = selected_local["high"] - selected_local["low"] - local_diagnostics = pd.concat( - [local_diagnostics, selected_local], - ignore_index=True, - sort=False, - ) + if not selected_cfg_path.exists() or not selected_intervals_path.exists(): + return pd.DataFrame() + with open(selected_cfg_path, "rb") as f: + selected_payload = pickle.load(f) + selected_intervals = pd.read_parquet(selected_intervals_path) + if not {"y_true", "pd_low_90", "pd_high_90", GROUP_COL}.issubset(selected_intervals.columns): + return pd.DataFrame() + selected_local = pd.DataFrame( + { + "record_type": "local_partition_summary", + "variant": "mondrian_selected_cfg", + "partition": str(selected_payload.get("partition", "grade")), + "group": selected_intervals[GROUP_COL].fillna("UNKNOWN").astype(str), + "y_true": pd.to_numeric(selected_intervals["y_true"], errors="coerce"), + "low": pd.to_numeric(selected_intervals["pd_low_90"], errors="coerce"), + "high": pd.to_numeric(selected_intervals["pd_high_90"], errors="coerce"), + } + ) + selected_local["covered"] = ( + (selected_local["y_true"] >= selected_local["low"]) + & (selected_local["y_true"] <= selected_local["high"]) + ).astype(float) + selected_local["width"] = selected_local["high"] - selected_local["low"] + return selected_local + + +def _append_selected_local_diagnostics( + *, + local_diagnostics: pd.DataFrame, + selected_config_path: str, + selected_intervals_path: Path, +) -> pd.DataFrame: + selected_local = _selected_config_local_diagnostics( + selected_config_path=selected_config_path, + selected_intervals_path=selected_intervals_path, + ) + if selected_local.empty: + return local_diagnostics + return pd.concat([local_diagnostics, selected_local], ignore_index=True, sort=False) + + +def _selection_status_payload( + *, + bench: pd.DataFrame, + output_paths: dict[str, Path], + artifact_namespace: str | None, + calibrator_override_path: str | None, + policy_config_path: str, + collect_local_diagnostics: bool, + space: SearchSpace, +) -> dict[str, Any]: + selected = bench.iloc[0].to_dict() + return { + "schema_version": "2026-04-03.1", + "generated_at_utc": datetime.now(tz=UTC).isoformat(), + "artifact_namespace": artifact_namespace or "", + "calibrator_override_path": str(calibrator_override_path or ""), + "policy_config_path": str(policy_config_path), + "selected_variant": str(selected.get("variant", "")), + "selection_rank": int(selected.get("selection_rank", 1)), + "promotion_pass": bool(selected.get("promotion_pass", False)), + "selection_criteria": [ + "promotion_pass", + "coverage_gap", + "min_group_coverage", + "winkler_90", + "avg_width", + "stability_over_time", + ], + "retired_backtest_role": ( + "Kupiec/Christoffersen are research diagnostics outside the IJDS " + "promotion gate; validate_conformal_policy.py promotes on material " + "coverage, group coverage, width, alert, and Winkler checks." + ), + "local_diagnostics_mode": ( + "all_variants" if collect_local_diagnostics else "selected_config_plus_sensitivity" + ), + "variants_tested": bench["variant"].astype(str).tolist(), + "report_path": str(output_paths["selection_report"]), + "summary_path": str(output_paths["benchmark"]), + "temporal_diagnostics_path": str(output_paths["temporal_diagnostics"]), + "local_diagnostics_path": str(output_paths["local_diagnostics"]), + "selected_metrics": { + "coverage": float(selected.get("coverage", 0.0)), + "coverage_gap": float(selected.get("coverage_gap", 0.0)), + "avg_width": float(selected.get("avg_width", 0.0)), + "min_group_coverage": float(selected.get("min_group_coverage", 0.0)), + "winkler_90": float(selected.get("winkler_90", 0.0)), + "stability_over_time": float(selected.get("stability_over_time", 0.0)), + }, + "search_space": { + "partition_candidates": list(space.partition_candidates), + "partition_probability_sources": list(space.partition_probability_sources), + "n_score_bins_candidates": [int(x) for x in space.n_score_bins_candidates], + "fallback_modes": list(space.fallback_modes), + "score_scale_families": list(space.score_scale_families), + "min_group_sizes": [int(x) for x in space.min_group_sizes], + "calibration_size_fractions": [float(x) for x in space.calibration_size_fractions], + }, + "top_variants": bench.head(5).to_dict(orient="records"), + } + +def _write_benchmark_outputs( + *, + output_paths: dict[str, Path], + bench: pd.DataFrame, + bench_by_group: pd.DataFrame, + temporal_diagnostics: pd.DataFrame, + local_diagnostics: pd.DataFrame, + status_payload: dict[str, Any], +) -> None: bench_path = output_paths["benchmark"] bench_group_path = output_paths["benchmark_by_group"] selection_path = output_paths["selection_report"] @@ -643,68 +898,9 @@ def _append_mondrian_variant( if not local_diagnostics.empty: local_diagnostics.to_parquet(local_path, index=False) - selected = bench.iloc[0].to_dict() status_path = output_paths["selection_status"] status_path.parent.mkdir(parents=True, exist_ok=True) - status_path.write_text( - json.dumps( - { - "schema_version": "2026-04-03.1", - "generated_at_utc": datetime.now(tz=UTC).isoformat(), - "artifact_namespace": artifact_namespace or "", - "calibrator_override_path": str(calibrator_override_path or ""), - "policy_config_path": str(policy_config_path), - "selected_variant": str(selected.get("variant", "")), - "selection_rank": int(selected.get("selection_rank", 1)), - "promotion_pass": bool(selected.get("promotion_pass", False)), - "selection_criteria": [ - "promotion_pass", - "coverage_gap", - "min_group_coverage", - "winkler_90", - "avg_width", - "stability_over_time", - ], - "retired_backtest_role": ( - "Kupiec/Christoffersen are research diagnostics outside the IJDS " - "promotion gate; validate_conformal_policy.py promotes on material " - "coverage, group coverage, width, alert, and Winkler checks." - ), - "local_diagnostics_mode": ( - "all_variants" - if collect_local_diagnostics - else "selected_config_plus_sensitivity" - ), - "variants_tested": bench["variant"].astype(str).tolist(), - "report_path": str(selection_path), - "summary_path": str(bench_path), - "temporal_diagnostics_path": str(temporal_path), - "local_diagnostics_path": str(local_path), - "selected_metrics": { - "coverage": float(selected.get("coverage", 0.0)), - "coverage_gap": float(selected.get("coverage_gap", 0.0)), - "avg_width": float(selected.get("avg_width", 0.0)), - "min_group_coverage": float(selected.get("min_group_coverage", 0.0)), - "winkler_90": float(selected.get("winkler_90", 0.0)), - "stability_over_time": float(selected.get("stability_over_time", 0.0)), - }, - "search_space": { - "partition_candidates": list(partition_candidates), - "partition_probability_sources": list(partition_probability_sources), - "n_score_bins_candidates": [int(x) for x in n_score_bins_candidates], - "fallback_modes": list(fallback_modes), - "score_scale_families": list(score_scale_families), - "min_group_sizes": [int(x) for x in min_group_sizes], - "calibration_size_fractions": [float(x) for x in calibration_size_fractions], - }, - "top_variants": bench.head(5).to_dict(orient="records"), - }, - indent=2, - default=str, - ), - encoding="utf-8", - ) - + status_path.write_text(json.dumps(status_payload, indent=2, default=str), encoding="utf-8") logger.info("Saved conformal benchmark summary: {} ({})", bench_path, bench.shape) logger.info( "Saved conformal benchmark by-group: {} ({})", bench_group_path, bench_by_group.shape @@ -720,6 +916,103 @@ def _append_mondrian_variant( logger.info("Saved conformal variant selection status: {}", status_path) +def main( + alpha: float = 0.10, + selected_config_path: str = "models/conformal_results_mondrian.pkl", + min_group_size_default: int = 500, + cross_cal_sample_size: int = 5000, + cross_test_sample_size: int = 5000, + calibration_size_fractions: tuple[float, ...] = (0.25, 0.50, 0.75, 1.0), + partition_candidates: tuple[str, ...] = ( + "grade", + "score_decile_mondrian", + "grade_x_scoreband_mondrian", + ), + partition_probability_sources: tuple[str, ...] = ("raw",), + n_score_bins_candidates: tuple[int, ...] = (10,), + fallback_modes: tuple[str, ...] = ("grade_then_global",), + score_scale_families: tuple[str, ...] = ("none", "bernoulli_sqrt"), + min_group_sizes: tuple[int, ...] | None = None, + artifact_namespace: str | None = None, + calibrator_override_path: str | None = None, + policy_config_path: str = DEFAULT_POLICY_CONFIG, + collect_local_diagnostics: bool = False, +) -> None: + policy = _load_policy_config(policy_config_path).get("policy", {}) or {} + data = _load_benchmark_data(calibrator_override_path) + space = _normalize_search_space( + calibration_size_fractions=calibration_size_fractions, + partition_candidates=partition_candidates, + partition_probability_sources=partition_probability_sources, + n_score_bins_candidates=n_score_bins_candidates, + fallback_modes=fallback_modes, + score_scale_families=score_scale_families, + min_group_sizes=min_group_sizes, + min_group_size_default=min_group_size_default, + ) + results = VariantResults() + partition_cache: dict[ + tuple[str, str, int, str, int, float, int], + tuple[pd.Series, pd.Series, dict[str, Any]], + ] = {} + + _append_global_variant(data=data, results=results, alpha=alpha) + _append_search_space_variants( + data=data, + results=results, + partition_cache=partition_cache, + space=space, + alpha=alpha, + collect_local_diagnostics=collect_local_diagnostics, + ) + _append_cross_conformal_variant( + data=data, + results=results, + alpha=alpha, + cross_cal_sample_size=cross_cal_sample_size, + cross_test_sample_size=cross_test_sample_size, + ) + _append_selected_config_variant( + data=data, + results=results, + partition_cache=partition_cache, + selected_config_path=selected_config_path, + alpha=alpha, + min_group_size_default=min_group_size_default, + collect_local_diagnostics=collect_local_diagnostics, + ) + sensitivity_rows = _calibration_sensitivity_rows(data=data, space=space, alpha=alpha) + bench, bench_by_group, temporal_diagnostics, local_diagnostics = _final_benchmark_frames( + results=results, + sensitivity_rows=sensitivity_rows, + policy=policy, + ) + + output_paths = _build_output_paths(artifact_namespace) + local_diagnostics = _append_selected_local_diagnostics( + local_diagnostics=local_diagnostics, + selected_config_path=selected_config_path, + selected_intervals_path=output_paths["selected_intervals"], + ) + status_payload = _selection_status_payload( + bench=bench, + output_paths=output_paths, + artifact_namespace=artifact_namespace, + calibrator_override_path=calibrator_override_path, + policy_config_path=policy_config_path, + collect_local_diagnostics=collect_local_diagnostics, + space=space, + ) + _write_benchmark_outputs( + output_paths=output_paths, + bench=bench, + bench_by_group=bench_by_group, + temporal_diagnostics=temporal_diagnostics, + local_diagnostics=local_diagnostics, + status_payload=status_payload, + ) + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--alpha", type=float, default=0.10) diff --git a/scripts/benchmark_pd_set_prediction.py b/scripts/benchmark_pd_set_prediction.py index 1fde082..c987848 100644 --- a/scripts/benchmark_pd_set_prediction.py +++ b/scripts/benchmark_pd_set_prediction.py @@ -5,6 +5,7 @@ import argparse import json import os +from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -33,6 +34,46 @@ GROUP_COL = "grade" +@dataclass(frozen=True) +class SetBenchmarkData: + model: Any + calibrator: Any | None + calibrator_name: str + cal_df: pd.DataFrame + test_df: pd.DataFrame + features: list[str] + categorical: list[str] + X_cal: pd.DataFrame + y_cal: pd.Series + X_test: pd.DataFrame + y_test: pd.Series + group_cal: pd.Series + group_test: pd.Series + prob_cal_lookup: dict[str, np.ndarray] + prob_test_lookup: dict[str, np.ndarray] + + +@dataclass(frozen=True) +class SetBenchmarkSettings: + alpha: float + methods: tuple[str, ...] + partitions: tuple[str, ...] + partition_probability_source: str + n_score_bins: int + min_group_size: int + requested_fallback_mode: str + effective_fallback_mode: str + calibration_size_fractions: tuple[float, ...] + + +@dataclass(frozen=True) +class VariantPrediction: + method: str + partition: str + y_pred: np.ndarray + y_sets: np.ndarray + + def _build_output_paths(namespace: str | None = None) -> dict[str, Path]: if namespace: ns = str(namespace).strip().replace("/", "_") @@ -122,6 +163,97 @@ def _normalize_sidecar_fallback_mode(fallback_mode: str) -> str: return "grade_then_global" +def _unique_csv_values(values: tuple[str, ...], fallback: tuple[str, ...]) -> tuple[str, ...]: + cleaned = tuple(dict.fromkeys(str(x).strip() for x in values if str(x).strip())) + return cleaned or fallback + + +def _valid_calibration_fractions(values: tuple[float, ...]) -> tuple[float, ...]: + return tuple(float(x) for x in values if 0 < float(x) <= 1) + + +def _calibrator_name(calibrator: Any | None, calibrator_override_path: str | None) -> str: + if calibrator_override_path: + return Path(str(calibrator_override_path)).stem + if calibrator is not None: + return type(calibrator).__name__ + return "raw" + + +def _load_set_benchmark_data(calibrator_override_path: str | None) -> SetBenchmarkData: + model, _ = _load_model() + calibrator = _load_calibrator(calibrator_override_path) + cal_df = read_with_fallback( + "data/processed/calibration_fe.parquet", "data/processed/calibration.parquet" + ) + test_df = read_with_fallback("data/processed/test_fe.parquet", "data/processed/test.parquet") + features, categorical = _resolve_features(model, cal_df, test_df) + X_cal = _build_feature_matrix(cal_df, features, categorical) + y_cal = cal_df[TARGET_COL].astype(int).reset_index(drop=True) + X_test = _build_feature_matrix(test_df, features, categorical) + y_test = test_df[TARGET_COL].astype(int).reset_index(drop=True) + group_cal = cal_df[GROUP_COL].fillna("UNKNOWN").astype(str).reset_index(drop=True) + group_test = test_df[GROUP_COL].fillna("UNKNOWN").astype(str).reset_index(drop=True) + y_prob_cal_raw = model.predict_proba(X_cal)[:, 1] + y_prob_test_raw = model.predict_proba(X_test)[:, 1] + y_prob_calibrated = ( + apply_probability_calibrator(calibrator, y_prob_cal_raw) + if calibrator is not None + else np.asarray(y_prob_cal_raw, dtype=float) + ) + y_prob_test_calibrated = ( + apply_probability_calibrator(calibrator, y_prob_test_raw) + if calibrator is not None + else np.asarray(y_prob_test_raw, dtype=float) + ) + return SetBenchmarkData( + model=model, + calibrator=calibrator, + calibrator_name=_calibrator_name(calibrator, calibrator_override_path), + cal_df=cal_df, + test_df=test_df, + features=features, + categorical=categorical, + X_cal=X_cal, + y_cal=y_cal, + X_test=X_test, + y_test=y_test, + group_cal=group_cal, + group_test=group_test, + prob_cal_lookup={"raw": y_prob_cal_raw, "calibrated": y_prob_calibrated}, + prob_test_lookup={"raw": y_prob_test_raw, "calibrated": y_prob_test_calibrated}, + ) + + +def _set_benchmark_settings( + *, + alpha: float, + method: str, + methods: tuple[str, ...] | None, + partitions: tuple[str, ...], + partition_probability_source: str, + n_score_bins: int, + min_group_size: int, + fallback_mode: str, + calibration_size_fractions: tuple[float, ...], + prob_cal_lookup: dict[str, np.ndarray], +) -> SetBenchmarkSettings: + source = str(partition_probability_source).strip().lower() or "raw" + if source not in prob_cal_lookup: + raise ValueError(f"Unsupported partition_probability_source: {source}") + return SetBenchmarkSettings( + alpha=float(alpha), + methods=_unique_csv_values(methods or (method,), ("lac",)), + partitions=_unique_csv_values(partitions, ("global",)), + partition_probability_source=source, + n_score_bins=int(n_score_bins), + min_group_size=int(min_group_size), + requested_fallback_mode=str(fallback_mode), + effective_fallback_mode=_normalize_sidecar_fallback_mode(fallback_mode), + calibration_size_fractions=_valid_calibration_fractions(calibration_size_fractions), + ) + + def _make_cases( *, y_true: pd.Series, @@ -164,253 +296,301 @@ def _make_cases( return cases -def main( - alpha: float = 0.10, - method: str = "lac", - methods: tuple[str, ...] | None = None, - partitions: tuple[str, ...] = ("global",), - partition_probability_source: str = "raw", - n_score_bins: int = 10, - min_group_size: int = 500, - fallback_mode: str = "grade_then_global", - calibration_size_fractions: tuple[float, ...] = (0.25, 0.50, 0.75, 1.0), - artifact_namespace: str | None = None, - calibrator_override_path: str | None = None, -) -> None: - method_list = tuple(dict.fromkeys(methods or (method,))) or ("lac",) - partition_list = tuple(dict.fromkeys(partitions)) or ("global",) - effective_fallback_mode = _normalize_sidecar_fallback_mode(fallback_mode) +def _is_global_partition(partition: str) -> bool: + return str(partition).strip().lower() == "global" - model, _ = _load_model() - calibrator = _load_calibrator(calibrator_override_path) - calibrator_name = ( - Path(str(calibrator_override_path)).stem - if calibrator_override_path - else type(calibrator).__name__ - if calibrator is not None - else "raw" + +def _predict_variant( + *, + data: SetBenchmarkData, + settings: SetBenchmarkSettings, + method_name: str, + partition_name: str, + X_cal: pd.DataFrame, + y_cal: pd.Series, + group_cal: pd.Series, + y_prob_cal: np.ndarray, +) -> VariantPrediction: + if _is_global_partition(partition_name): + y_pred, y_sets = create_classification_sets( + classifier=data.model, + X_cal=X_cal, + y_cal=y_cal, + X_test=data.X_test, + alpha=settings.alpha, + method=method_name, + calibrator=data.calibrator, + ) + return VariantPrediction( + method=str(method_name), + partition=str(partition_name), + y_pred=y_pred, + y_sets=y_sets, + ) + + group_cal_part, group_test_part, partition_meta = build_mondrian_partition_labels( + y_prob_cal=y_prob_cal, + y_prob_eval=data.prob_test_lookup[settings.partition_probability_source], + partition=partition_name, + base_groups_cal=group_cal, + base_groups_eval=data.group_test, + n_score_bins=settings.n_score_bins, + min_group_size=settings.min_group_size, + fallback_mode=settings.effective_fallback_mode, ) - cal_df = read_with_fallback( - "data/processed/calibration_fe.parquet", "data/processed/calibration.parquet" + y_pred, y_sets, _ = create_classification_sets_mondrian( + classifier=data.model, + X_cal=X_cal, + y_cal=y_cal, + X_test=data.X_test, + group_cal=group_cal_part, + group_test=group_test_part, + alpha=settings.alpha, + method=method_name, + min_group_size=settings.min_group_size, + calibrator=data.calibrator, + ) + return VariantPrediction( + method=str(method_name), + partition=str(partition_meta.get("partition", partition_name)), + y_pred=y_pred, + y_sets=y_sets, ) - test_df = read_with_fallback("data/processed/test_fe.parquet", "data/processed/test.parquet") - features, categorical = _resolve_features(model, cal_df, test_df) - X_cal = _build_feature_matrix(cal_df, features, categorical) - y_cal = cal_df[TARGET_COL].astype(int).reset_index(drop=True) - X_test = _build_feature_matrix(test_df, features, categorical) - y_test = test_df[TARGET_COL].astype(int).reset_index(drop=True) - group_cal = cal_df[GROUP_COL].fillna("UNKNOWN").astype(str).reset_index(drop=True) - group_test = test_df[GROUP_COL].fillna("UNKNOWN").astype(str).reset_index(drop=True) - y_prob_cal_raw = model.predict_proba(X_cal)[:, 1] - y_prob_test_raw = model.predict_proba(X_test)[:, 1] - y_prob_calibrated = ( - apply_probability_calibrator(calibrator, y_prob_cal_raw) - if calibrator is not None - else np.asarray(y_prob_cal_raw, dtype=float) + +def _benchmark_row( + *, + data: SetBenchmarkData, + settings: SetBenchmarkSettings, + prediction: VariantPrediction, +) -> dict[str, Any]: + summary = summarize_prediction_sets( + data.y_test.to_numpy(), prediction.y_pred, prediction.y_sets ) - y_prob_test_calibrated = ( - apply_probability_calibrator(calibrator, y_prob_test_raw) - if calibrator is not None - else np.asarray(y_prob_test_raw, dtype=float) + return { + "method": prediction.method, + "partition": prediction.partition, + "partition_probability_source": settings.partition_probability_source, + "calibrator": data.calibrator_name, + "alpha": settings.alpha, + **{k: float(v) for k, v in summary.items()}, + } + + +def _subsample_calibration_data( + *, + data: SetBenchmarkData, + calibration_fraction: float, +) -> tuple[pd.DataFrame, pd.DataFrame, pd.Series, pd.Series, np.ndarray, np.ndarray]: + cal_df_sub = _subset_calibration_frame(data.cal_df, calibration_fraction=calibration_fraction) + X_cal_sub = _build_feature_matrix(cal_df_sub, data.features, data.categorical) + y_cal_sub = cal_df_sub[TARGET_COL].astype(int).reset_index(drop=True) + group_cal_sub = cal_df_sub[GROUP_COL].fillna("UNKNOWN").astype(str).reset_index(drop=True) + y_prob_cal_sub_raw = data.model.predict_proba(X_cal_sub)[:, 1] + y_prob_cal_sub_calibrated = ( + apply_probability_calibrator(data.calibrator, y_prob_cal_sub_raw) + if data.calibrator is not None + else np.asarray(y_prob_cal_sub_raw, dtype=float) + ) + return ( + cal_df_sub, + X_cal_sub, + y_cal_sub, + group_cal_sub, + y_prob_cal_sub_raw, + y_prob_cal_sub_calibrated, + ) + + +def _sensitivity_row( + *, + data: SetBenchmarkData, + settings: SetBenchmarkSettings, + method_name: str, + partition_name: str, + calibration_fraction: float, +) -> dict[str, Any]: + _cal_df_sub, X_cal_sub, y_cal_sub, group_cal_sub, y_prob_raw, y_prob_calibrated = ( + _subsample_calibration_data(data=data, calibration_fraction=calibration_fraction) + ) + y_prob_cal = y_prob_raw if settings.partition_probability_source == "raw" else y_prob_calibrated + prediction = _predict_variant( + data=data, + settings=settings, + method_name=method_name, + partition_name=partition_name, + X_cal=X_cal_sub, + y_cal=y_cal_sub, + group_cal=group_cal_sub, + y_prob_cal=y_prob_cal, + ) + summary_sub = summarize_prediction_sets( + data.y_test.to_numpy(), prediction.y_pred, prediction.y_sets ) - prob_cal_lookup = {"raw": y_prob_cal_raw, "calibrated": y_prob_calibrated} - prob_test_lookup = {"raw": y_prob_test_raw, "calibrated": y_prob_test_calibrated} - partition_probability_source = str(partition_probability_source).strip().lower() or "raw" - if partition_probability_source not in prob_cal_lookup: - raise ValueError( - f"Unsupported partition_probability_source: {partition_probability_source}" + return { + "method": str(method_name), + "partition": str(prediction.partition), + "partition_probability_source": settings.partition_probability_source, + "calibrator": data.calibrator_name, + "calibration_fraction": float(calibration_fraction), + "n_calibration_rows": int(len(X_cal_sub)), + "set_coverage": float(summary_sub["set_coverage"]), + "singleton_rate": float(summary_sub["singleton_rate"]), + "ambiguity_rate": float(summary_sub["ambiguity_rate"]), + "empty_set_rate": float(summary_sub["empty_set_rate"]), + "default_rate_ambiguous": float(summary_sub["default_rate_ambiguous"]), + } + + +def _sensitivity_rows_for_variant( + *, + data: SetBenchmarkData, + settings: SetBenchmarkSettings, + method_name: str, + partition_name: str, +) -> list[dict[str, Any]]: + return [ + _sensitivity_row( + data=data, + settings=settings, + method_name=method_name, + partition_name=partition_name, + calibration_fraction=float(frac), ) + for frac in settings.calibration_size_fractions + ] + +def _run_benchmark_matrix( + *, + data: SetBenchmarkData, + settings: SetBenchmarkSettings, +) -> tuple[ + pd.DataFrame, + dict[tuple[str, str], pd.DataFrame], + dict[tuple[str, str], list[dict[str, Any]]], +]: benchmark_rows: list[dict[str, Any]] = [] cases_by_variant: dict[tuple[str, str], pd.DataFrame] = {} sensitivity_by_variant: dict[tuple[str, str], list[dict[str, Any]]] = {} - for method_name in method_list: - for partition_name in partition_list: - if str(partition_name).strip().lower() == "global": - y_pred, y_sets = create_classification_sets( - classifier=model, - X_cal=X_cal, - y_cal=y_cal, - X_test=X_test, - alpha=alpha, - method=method_name, - calibrator=calibrator, - ) - else: - group_cal_part, group_test_part, partition_meta = build_mondrian_partition_labels( - y_prob_cal=prob_cal_lookup[partition_probability_source], - y_prob_eval=prob_test_lookup[partition_probability_source], - partition=partition_name, - base_groups_cal=group_cal, - base_groups_eval=group_test, - n_score_bins=n_score_bins, - min_group_size=min_group_size, - fallback_mode=effective_fallback_mode, - ) - y_pred, y_sets, _ = create_classification_sets_mondrian( - classifier=model, - X_cal=X_cal, - y_cal=y_cal, - X_test=X_test, - group_cal=group_cal_part, - group_test=group_test_part, - alpha=alpha, - method=method_name, - min_group_size=min_group_size, - calibrator=calibrator, - ) - partition_name = str(partition_meta.get("partition", partition_name)) - - summary = summarize_prediction_sets(y_test.to_numpy(), y_pred, y_sets) + for method_name in settings.methods: + for partition_name in settings.partitions: + prediction = _predict_variant( + data=data, + settings=settings, + method_name=method_name, + partition_name=partition_name, + X_cal=data.X_cal, + y_cal=data.y_cal, + group_cal=data.group_cal, + y_prob_cal=data.prob_cal_lookup[settings.partition_probability_source], + ) benchmark_rows.append( - { - "method": str(method_name), - "partition": str(partition_name), - "partition_probability_source": str(partition_probability_source), - "calibrator": str(calibrator_name), - "alpha": float(alpha), - **{k: float(v) for k, v in summary.items()}, - } + _benchmark_row(data=data, settings=settings, prediction=prediction) ) - cases_by_variant[(str(method_name), str(partition_name))] = _make_cases( - y_true=y_test, - y_pred=y_pred, - y_sets=y_sets, - test_df=test_df, - method=str(method_name), - partition=str(partition_name), - partition_probability_source=str(partition_probability_source), - calibrator_name=str(calibrator_name), + key = (prediction.method, prediction.partition) + cases_by_variant[key] = _make_cases( + y_true=data.y_test, + y_pred=prediction.y_pred, + y_sets=prediction.y_sets, + test_df=data.test_df, + method=prediction.method, + partition=prediction.partition, + partition_probability_source=settings.partition_probability_source, + calibrator_name=data.calibrator_name, + ) + sensitivity_by_variant[key] = _sensitivity_rows_for_variant( + data=data, + settings=settings, + method_name=method_name, + partition_name=prediction.partition, ) - - sensitivity_rows: list[dict[str, Any]] = [] - for frac in calibration_size_fractions: - frac_float = float(frac) - if frac_float <= 0 or frac_float > 1: - continue - cal_df_sub = _subset_calibration_frame(cal_df, calibration_fraction=frac_float) - X_cal_sub = _build_feature_matrix(cal_df_sub, features, categorical) - y_cal_sub = cal_df_sub[TARGET_COL].astype(int).reset_index(drop=True) - group_cal_sub = ( - cal_df_sub[GROUP_COL].fillna("UNKNOWN").astype(str).reset_index(drop=True) - ) - y_prob_cal_sub_raw = model.predict_proba(X_cal_sub)[:, 1] - y_prob_cal_sub_calibrated = ( - apply_probability_calibrator(calibrator, y_prob_cal_sub_raw) - if calibrator is not None - else np.asarray(y_prob_cal_sub_raw, dtype=float) - ) - y_prob_cal_sub = ( - y_prob_cal_sub_raw - if partition_probability_source == "raw" - else y_prob_cal_sub_calibrated - ) - if str(partition_name).strip().lower() == "global": - y_pred_sub, y_sets_sub = create_classification_sets( - classifier=model, - X_cal=X_cal_sub, - y_cal=y_cal_sub, - X_test=X_test, - alpha=alpha, - method=method_name, - calibrator=calibrator, - ) - else: - group_cal_part, group_test_part, _ = build_mondrian_partition_labels( - y_prob_cal=y_prob_cal_sub, - y_prob_eval=prob_test_lookup[partition_probability_source], - partition=partition_name, - base_groups_cal=group_cal_sub, - base_groups_eval=group_test, - n_score_bins=n_score_bins, - min_group_size=min_group_size, - fallback_mode=effective_fallback_mode, - ) - y_pred_sub, y_sets_sub, _ = create_classification_sets_mondrian( - classifier=model, - X_cal=X_cal_sub, - y_cal=y_cal_sub, - X_test=X_test, - group_cal=group_cal_part, - group_test=group_test_part, - alpha=alpha, - method=method_name, - min_group_size=min_group_size, - calibrator=calibrator, - ) - summary_sub = summarize_prediction_sets(y_test.to_numpy(), y_pred_sub, y_sets_sub) - sensitivity_rows.append( - { - "method": str(method_name), - "partition": str(partition_name), - "partition_probability_source": str(partition_probability_source), - "calibrator": str(calibrator_name), - "calibration_fraction": frac_float, - "n_calibration_rows": int(len(X_cal_sub)), - "set_coverage": float(summary_sub["set_coverage"]), - "singleton_rate": float(summary_sub["singleton_rate"]), - "ambiguity_rate": float(summary_sub["ambiguity_rate"]), - "empty_set_rate": float(summary_sub["empty_set_rate"]), - "default_rate_ambiguous": float(summary_sub["default_rate_ambiguous"]), - } - ) - sensitivity_by_variant[(str(method_name), str(partition_name))] = sensitivity_rows benchmark_df = pd.DataFrame(benchmark_rows) if benchmark_df.empty: raise RuntimeError("No set-prediction variants were benchmarked.") - benchmark_df = benchmark_df.sort_values( - by=["set_coverage", "singleton_rate", "ambiguity_rate", "empty_set_rate"], - ascending=[False, False, True, True], - ).reset_index(drop=True) - selected = benchmark_df.iloc[0] - selected_key = (str(selected["method"]), str(selected["partition"])) - cases = cases_by_variant[selected_key] + return ( + benchmark_df.sort_values( + by=["set_coverage", "singleton_rate", "ambiguity_rate", "empty_set_rate"], + ascending=[False, False, True, True], + ).reset_index(drop=True), + cases_by_variant, + sensitivity_by_variant, + ) + +def _slice_reports(cases: pd.DataFrame) -> pd.DataFrame: slice_reports = [] for col in ("grade", "term", "issue_quarter"): if col in cases.columns: report = _slice_summary(cases, col) if not report.empty: slice_reports.append(report) - by_slice = pd.concat(slice_reports, ignore_index=True) if slice_reports else pd.DataFrame() - sensitivity_df = pd.DataFrame(sensitivity_by_variant[selected_key]) + return pd.concat(slice_reports, ignore_index=True) if slice_reports else pd.DataFrame() - summary = summarize_prediction_sets( - cases["y_true"].to_numpy(dtype=int), - cases["y_pred_label"].to_numpy(dtype=int), - cases[["set_contains_0", "set_contains_1"]].to_numpy(dtype=int), - ) - grade_slices = ( - by_slice.loc[by_slice["slice_name"] == "grade"].to_dict(orient="records") - if not by_slice.empty and "slice_name" in by_slice.columns - else [] - ) + +def _grade_slices(by_slice: pd.DataFrame) -> list[dict[str, Any]]: + if by_slice.empty or "slice_name" not in by_slice.columns: + return [] + return by_slice.loc[by_slice["slice_name"] == "grade"].to_dict(orient="records") + + +def _slice_records(by_slice: pd.DataFrame, slice_name: str) -> list[dict[str, Any]]: + if by_slice.empty or "slice_name" not in by_slice.columns: + return [] + return by_slice.loc[by_slice["slice_name"] == slice_name].to_dict(orient="records") + + +def _promotion_gate(summary: dict[str, Any], grade_slices: list[dict[str, Any]]) -> dict[str, Any]: gate_coverage = float(summary.get("set_coverage", 0)) gate_grade_a_singleton = 0.0 gate_grades_above_40 = 0 for gs in grade_slices: - sr = float(gs.get("singleton_rate", 0)) + singleton_rate = float(gs.get("singleton_rate", 0)) if str(gs.get("slice_value", "")) == "A": - gate_grade_a_singleton = sr - if sr > 0.40: + gate_grade_a_singleton = singleton_rate + if singleton_rate > 0.40: gate_grades_above_40 += 1 - - gate_pass = ( + gate_pass = bool( gate_coverage >= 0.85 and gate_grade_a_singleton >= 0.80 and gate_grades_above_40 >= 3 ) - promotion_status = "promoted_guardrail" if gate_pass else "research_sidecar" + return { + "coverage": gate_coverage, + "min_coverage": 0.85, + "grade_a_singleton_rate": gate_grade_a_singleton, + "min_grade_a_singleton": 0.80, + "grades_with_singleton_above_40pct": gate_grades_above_40, + "min_grades_above_40pct": 3, + "pass": gate_pass, + } + + +def _selected_summary(cases: pd.DataFrame) -> dict[str, float]: + return { + k: float(v) + for k, v in summarize_prediction_sets( + cases["y_true"].to_numpy(dtype=int), + cases["y_pred_label"].to_numpy(dtype=int), + cases[["set_contains_0", "set_contains_1"]].to_numpy(dtype=int), + ).items() + } - outputs = _build_output_paths(artifact_namespace) - cases.to_parquet(outputs["cases"], index=False) - if not by_slice.empty: - by_slice.to_parquet(outputs["by_slice"], index=False) - sensitivity_df.to_parquet(outputs["sensitivity"], index=False) - benchmark_df.to_parquet(outputs["benchmark"], index=False) - status = { +def _status_payload( + *, + settings: SetBenchmarkSettings, + selected: pd.Series, + summary: dict[str, float], + gate: dict[str, Any], + by_slice: pd.DataFrame, + benchmark_df: pd.DataFrame, + outputs: dict[str, Path], + artifact_namespace: str | None, +) -> dict[str, Any]: + gate_pass = bool(gate["pass"]) + promotion_status = "promoted_guardrail" if gate_pass else "research_sidecar" + return { "schema_version": "2026-04-03.1", "generated_at_utc": datetime.now(tz=UTC).isoformat(), "run_tag": os.environ.get("PIPELINE_RUN_TAG", "untracked"), @@ -422,34 +602,20 @@ def main( "selected_partition": str(selected["partition"]), "selected_partition_probability_source": str(selected["partition_probability_source"]), "selected_calibrator": str(selected["calibrator"]), - "requested_fallback_mode": str(fallback_mode), - "effective_fallback_mode": str(effective_fallback_mode), - "alpha": float(alpha), - "confidence_level": float(1.0 - alpha), - "summary": {k: float(v) for k, v in summary.items()}, - "promotion_gate": { - "coverage": gate_coverage, - "min_coverage": 0.85, - "grade_a_singleton_rate": gate_grade_a_singleton, - "min_grade_a_singleton": 0.80, - "grades_with_singleton_above_40pct": gate_grades_above_40, - "min_grades_above_40pct": 3, - "pass": gate_pass, - }, + "requested_fallback_mode": settings.requested_fallback_mode, + "effective_fallback_mode": settings.effective_fallback_mode, + "alpha": settings.alpha, + "confidence_level": float(1.0 - settings.alpha), + "summary": summary, + "promotion_gate": gate, "artifact_path": str(outputs["cases"]), "by_slice_path": str(outputs["by_slice"]), "calibration_size_sensitivity_path": str(outputs["sensitivity"]), "benchmark_matrix_path": str(outputs["benchmark"]), "slice_metrics": { - "grade": grade_slices, - "term": by_slice.loc[by_slice["slice_name"] == "term"].to_dict(orient="records") - if not by_slice.empty and "slice_name" in by_slice.columns - else [], - "issue_quarter": by_slice.loc[by_slice["slice_name"] == "issue_quarter"].to_dict( - orient="records" - ) - if not by_slice.empty and "slice_name" in by_slice.columns - else [], + "grade": _grade_slices(by_slice), + "term": _slice_records(by_slice, "term"), + "issue_quarter": _slice_records(by_slice, "issue_quarter"), }, "benchmark_matrix": benchmark_df.to_dict(orient="records"), "decision_use_case": { @@ -459,13 +625,29 @@ def main( }, "promotion_rationale": ( f"Binary conformal sets selected via {selected['method']} + {selected['partition']} " - f"with set coverage {gate_coverage:.1%} and ambiguity {summary['ambiguity_rate']:.1%}." + f"with set coverage {gate['coverage']:.1%} and ambiguity {summary['ambiguity_rate']:.1%}." ), "promotion_note": ( "Binary set prediction remains a sidecar triage/abstention signal; it does not replace " "the interval-first conformal stack." ), } + + +def _write_outputs( + *, + outputs: dict[str, Path], + cases: pd.DataFrame, + by_slice: pd.DataFrame, + sensitivity_df: pd.DataFrame, + benchmark_df: pd.DataFrame, + status: dict[str, Any], +) -> None: + cases.to_parquet(outputs["cases"], index=False) + if not by_slice.empty: + by_slice.to_parquet(outputs["by_slice"], index=False) + sensitivity_df.to_parquet(outputs["sensitivity"], index=False) + benchmark_df.to_parquet(outputs["benchmark"], index=False) outputs["status"].write_text( json.dumps(status, indent=2, ensure_ascii=False) + "\n", encoding="utf-8" ) @@ -473,6 +655,64 @@ def main( logger.info("Saved PD set prediction status: {}", outputs["status"]) +def main( + alpha: float = 0.10, + method: str = "lac", + methods: tuple[str, ...] | None = None, + partitions: tuple[str, ...] = ("global",), + partition_probability_source: str = "raw", + n_score_bins: int = 10, + min_group_size: int = 500, + fallback_mode: str = "grade_then_global", + calibration_size_fractions: tuple[float, ...] = (0.25, 0.50, 0.75, 1.0), + artifact_namespace: str | None = None, + calibrator_override_path: str | None = None, +) -> None: + data = _load_set_benchmark_data(calibrator_override_path) + settings = _set_benchmark_settings( + alpha=alpha, + method=method, + methods=methods, + partitions=partitions, + partition_probability_source=partition_probability_source, + n_score_bins=n_score_bins, + min_group_size=min_group_size, + fallback_mode=fallback_mode, + calibration_size_fractions=calibration_size_fractions, + prob_cal_lookup=data.prob_cal_lookup, + ) + benchmark_df, cases_by_variant, sensitivity_by_variant = _run_benchmark_matrix( + data=data, + settings=settings, + ) + selected = benchmark_df.iloc[0] + selected_key = (str(selected["method"]), str(selected["partition"])) + cases = cases_by_variant[selected_key] + by_slice = _slice_reports(cases) + sensitivity_df = pd.DataFrame(sensitivity_by_variant[selected_key]) + summary = _selected_summary(cases) + gate = _promotion_gate(summary, _grade_slices(by_slice)) + outputs = _build_output_paths(artifact_namespace) + status = _status_payload( + settings=settings, + selected=selected, + summary=summary, + gate=gate, + by_slice=by_slice, + benchmark_df=benchmark_df, + outputs=outputs, + artifact_namespace=artifact_namespace, + ) + _write_outputs( + outputs=outputs, + cases=cases, + by_slice=by_slice, + sensitivity_df=sensitivity_df, + benchmark_df=benchmark_df, + status=status, + ) + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--alpha", type=float, default=0.10) diff --git a/scripts/build_bound_tightening_audit.py b/scripts/build_bound_tightening_audit.py index 65b8b08..3429809 100644 --- a/scripts/build_bound_tightening_audit.py +++ b/scripts/build_bound_tightening_audit.py @@ -335,7 +335,7 @@ def _write_report( ] ) ].sort_values("threshold_t") - lines = [ + lines: list[str] = [ "# CRPTO Bound Tightening Experiment - 2026-06-11", "", "Merged into `main` (2026-06-11) and cited by Online Supplement Appendix A. " @@ -366,26 +366,28 @@ def _write_report( f"`{row.margin_vs_empirical_V:.6f}` | {row.paper_role} |" ) - lines += [ - "", - "## Recommendation", - "", - "- Keep Markov as the body theorem: it is the only first-moment, " - "distribution-free statement compatible with the current post-selection caveat.", - "- Keep A21 cluster-aware Hoeffding as a dependence caveat, not a tightening: " - "cluster exposure is too concentrated.", - "- Use A21b/A21c as an appendix sensitivity table. Cantelli, Bernstein, Bennett " - "and Freedman show how much tightness is available if a reviewer accepts stronger " - "independence, variance, or martingale assumptions.", - "- Drop Chebyshev, Azuma, Chernoff and naive union-Markov from paper-facing tables. " - "They are respectively dominated, duplicative, too strong for the current " - "individual-alpha evidence, or vacuous after policy-region correction.", - "", - "## Assumption Audit", - "", - assumption_table.to_markdown(index=False), - "", - ] + lines.extend( + ( + "", + "## Recommendation", + "", + "- Keep Markov as the body theorem: it is the only first-moment, " + "distribution-free statement compatible with the current post-selection caveat.", + "- Keep A21 cluster-aware Hoeffding as a dependence caveat, not a tightening: " + "cluster exposure is too concentrated.", + "- Use A21b/A21c as an appendix sensitivity table. Cantelli, Bernstein, Bennett " + "and Freedman show how much tightness is available if a reviewer accepts stronger " + "independence, variance, or martingale assumptions.", + "- Drop Chebyshev, Azuma, Chernoff and naive union-Markov from paper-facing tables. " + "They are respectively dominated, duplicative, too strong for the current " + "individual-alpha evidence, or vacuous after policy-region correction.", + "", + "## Assumption Audit", + "", + str(assumption_table.to_markdown(index=False)), + "", + ) + ) REPORT_PATH.parent.mkdir(parents=True, exist_ok=True) REPORT_PATH.write_text("\n".join(lines), encoding="utf-8", newline="") diff --git a/scripts/build_crpto_journal_package.py b/scripts/build_crpto_journal_package.py index 713ce33..02e5e86 100644 --- a/scripts/build_crpto_journal_package.py +++ b/scripts/build_crpto_journal_package.py @@ -156,15 +156,17 @@ def _build_tail_risk_table(funded: pd.DataFrame) -> pd.DataFrame: index=funded.index, ) theta = 5.0 - oce = round(float(entropic_oce(loss_rate, weights, theta=theta, stable=False)), 17) + loss_values = loss_rate.to_numpy(dtype=float) + weight_values = weights.to_numpy(dtype=float) + oce = round(float(entropic_oce(loss_values, weight_values, theta=theta, stable=False)), 17) rows.append( { "lgd": float(lgd), - "mean_loss_rate": float(np.sum(weights * loss_rate)), + "mean_loss_rate": float(np.sum(weight_values * loss_values)), "entropic_oce_theta5": oce, - "cvar_90_loss_rate": weighted_cvar(loss_rate, weights, tail=0.90), - "cvar_95_loss_rate": weighted_cvar(loss_rate, weights, tail=0.95), - "cvar_99_loss_rate": weighted_cvar(loss_rate, weights, tail=0.99), + "cvar_90_loss_rate": weighted_cvar(loss_values, weight_values, tail=0.90), + "cvar_95_loss_rate": weighted_cvar(loss_values, weight_values, tail=0.95), + "cvar_99_loss_rate": weighted_cvar(loss_values, weight_values, tail=0.99), "funded_set_repriced_return": _return_from_frame( funded, weights, diff --git a/scripts/build_papers_tesis_deep_audit.py b/scripts/build_papers_tesis_deep_audit.py index 94ce9aa..ebed162 100644 --- a/scripts/build_papers_tesis_deep_audit.py +++ b/scripts/build_papers_tesis_deep_audit.py @@ -16,6 +16,7 @@ from collections import Counter from dataclasses import dataclass, field from pathlib import Path +from typing import Any DEFAULT_SOURCE_DIR = Path("Papers_tesis") DEFAULT_AUDIT_PATH = Path("docs/research/papers_tesis_deep_audit_2026-06-06.md") @@ -59,8 +60,8 @@ class Review: tags: tuple[str, ...] = field(default_factory=tuple) -def r(**kwargs: object) -> Review: - return Review(**kwargs) # type: ignore[arg-type] +def r(**kwargs: Any) -> Review: + return Review(**kwargs) REVIEWS: dict[str, Review] = { @@ -1772,191 +1773,235 @@ def markdown_table( return "\n".join([header, divider, *body]) -def write_audit( - path: Path, - rows: list[dict[str, object]], - matrix_path: Path, - caption_path: Path, - curated_visual_path: Path, - curated_visual_rows: list[dict[str, object]], -) -> None: - path.parent.mkdir(parents=True, exist_ok=True) +def _counter_rows(counter: Counter[str], key_field: str) -> list[dict[str, object]]: + return [{key_field: key, "n": value} for key, value in sorted(counter.items())] + + +def _section(title: str, *body: str) -> list[str]: + return ["", f"## {title}", "", *body] + + +def _decision_groups(rows: list[dict[str, object]]) -> dict[str, list[dict[str, object]]]: + return { + "promote": [row for row in rows if str(row["decision"]).startswith("promote")], + "crpto_appendix": [ + row + for row in rows + if str(row["decision"]).startswith(("append_crpto", "append_comparator", "append_tail")) + ], + "extended": [ + row + for row in rows + if "extended" in str(row["decision"]) or "mixed_diagnostic" in str(row["decision"]) + ], + "future": [ + row + for row in rows + if "future" in str(row["decision"]) or "candidate" in str(row["decision"]) + ], + "experiments": [ + row for row in rows if str(row["action_required"]).startswith("experiment_") + ], + } + + +def _inventory_sections(rows: list[dict[str, object]]) -> list[str]: folder_counts = Counter(str(row["folder"]) for row in rows) decision_counts = Counter(str(row["decision"]) for row in rows) action_counts = Counter(str(row["action_required"]) for row in rows) domain_counts = Counter(str(row["primary_domain"]) for row in rows) - - promote = [row for row in rows if str(row["decision"]).startswith("promote")] - crpto_appendix = [ - row - for row in rows - if str(row["decision"]).startswith(("append_crpto", "append_comparator", "append_tail")) - ] - extended = [ - row - for row in rows - if "extended" in str(row["decision"]) or "mixed_diagnostic" in str(row["decision"]) - ] - future = [ - row - for row in rows - if "future" in str(row["decision"]) or "candidate" in str(row["decision"]) - ] - - lines = [ - "# Papers_tesis Deep Audit - 2026-06-06", - "", - "## Resumen ejecutivo", - "", - f"Esta auditoría cubre `61` PDFs locales en `{DEFAULT_SOURCE_DIR}` y fue generada para el corte `{REPORT_DATE}` con `scripts/build_papers_tesis_deep_audit.py`.", - "", - "La decisión central no cambia: **Paper CRPTO conserva el champion oficial** y la literatura nueva se usa para reforzar teoría, related work, appendices y límites de claim. La agenda extendida CRPTO/tesis absorbe el material que sí pertenece al laboratorio vivo: source/shift conformal, utility-directed conformal, tail risk, DFL, IFRS9 proxy, data/noise/equity y governance.", - "", - "Artefactos generados:", - "", - f"- Matriz fuente: `{matrix_path}`", - f"- Índice compacto de captions: `{caption_path}`", - f"- Curaduría de visual sinks: `{curated_visual_path}`", - "", - "## Inventario", - "", - markdown_table( - [{"folder": folder, "pdfs": count} for folder, count in sorted(folder_counts.items())], - ["folder", "pdfs"], + return [ + *_section( + "Inventario", + markdown_table( + [ + {"folder": folder, "pdfs": count} + for folder, count in sorted(folder_counts.items()) + ], + ["folder", "pdfs"], + ), ), - "", - "## Decisiones por destino editorial", - "", - markdown_table( - [{"decision": key, "n": value} for key, value in sorted(decision_counts.items())], - ["decision", "n"], + *_section( + "Decisiones por destino editorial", + markdown_table(_counter_rows(decision_counts, "decision"), ["decision", "n"]), ), - "", - "## Acciones requeridas", - "", - markdown_table( - [{"action_required": key, "n": value} for key, value in sorted(action_counts.items())], - ["action_required", "n"], + *_section( + "Acciones requeridas", + markdown_table( + _counter_rows(action_counts, "action_required"), ["action_required", "n"] + ), ), - "", - "## Familias conceptuales", - "", - markdown_table( - [{"primary_domain": key, "n": value} for key, value in sorted(domain_counts.items())], - ["primary_domain", "n"], + *_section( + "Familias conceptuales", + markdown_table(_counter_rows(domain_counts, "primary_domain"), ["primary_domain", "n"]), ), - "", - "## Lectura integrada para Paper CRPTO", - "", - "El cuerpo de Paper CRPTO debe quedarse en cuatro pilares: conformal risk/control, robust optimization, conformal robust optimization / predict-then-calibrate y contexto Lending Club/DFL como comparador. Las fuentes promovidas al cuerpo son:", - "", - markdown_table( - promote, ["relative_path", "bib_key", "core_concepts", "crpto_value", "limitations"] + ] + + +def _paper_crpto_sections(groups: dict[str, list[dict[str, object]]]) -> list[str]: + return [ + *_section( + "Lectura integrada para Paper CRPTO", + "El cuerpo de Paper CRPTO debe quedarse en cuatro pilares: conformal risk/control, robust optimization, conformal robust optimization / predict-then-calibrate y contexto Lending Club/DFL como comparador. Las fuentes promovidas al cuerpo son:", + "", + markdown_table( + groups["promote"], + ["relative_path", "bib_key", "core_concepts", "crpto_value", "limitations"], + ), + "", + "Las fuentes de appendix o comparador CRPTO deben apoyar selectivamente el selector, SPO+/DFL, CVaR/OCE, CQR y limites de claim:", + "", + markdown_table( + groups["crpto_appendix"], + ["relative_path", "bib_key", "decision", "crpto_value", "figures_tables_useful"], + ), ), - "", - "Las fuentes de appendix o comparador CRPTO deben apoyar selectivamente el selector, SPO+/DFL, CVaR/OCE, CQR y límites de claim:", - "", - markdown_table( - crpto_appendix, - ["relative_path", "bib_key", "decision", "crpto_value", "figures_tables_useful"], + *_section( + "Lectura integrada para agenda extendida CRPTO/tesis", + "La agenda extendida CRPTO/tesis es el destino correcto para fuentes que fortalecen governance, source/shift robustness, fairness proxy, IFRS9/SICR proxy, DFL ampliado y data-quality/equity. Estas fuentes no reabren el champion CRPTO:", + "", + markdown_table( + groups["extended"], + ["relative_path", "decision", "extended_lab_value", "evidence_gate", "stop_rule"], + ), ), - "", - "## Lectura integrada para agenda extendida CRPTO/tesis", - "", - "La agenda extendida CRPTO/tesis es el destino correcto para fuentes que fortalecen governance, source/shift robustness, fairness proxy, IFRS9/SICR proxy, DFL ampliado y data-quality/equity. Estas fuentes no reabren el champion CRPTO:", - "", - markdown_table( - extended, - ["relative_path", "decision", "extended_lab_value", "evidence_gate", "stop_rule"], + *_section( + "Experimentos evidence-gated", + "Esta seccion lista los experimentos ya ejecutados o pendientes bajo regla evidence-gated. Cada fila exige claim target, evidence gate, artifact sink y stop rule; un resultado positivo no cambia el champion CRPTO sin gate editorial separado.", + "", + markdown_table( + groups["experiments"], + [ + "relative_path", + "action_required", + "implementation_or_experiment", + "evidence_gate", + "artifact_sink", + "stop_rule", + ], + ), ), - "", - "## Experimentos evidence-gated", - "", - "Esta sección lista los experimentos ya ejecutados o pendientes bajo regla evidence-gated. Cada fila exige claim target, evidence gate, artifact sink y stop rule; un resultado positivo no cambia el champion CRPTO sin gate editorial separado.", - "", - markdown_table( - [row for row in rows if str(row["action_required"]).startswith("experiment_")], - [ - "relative_path", - "action_required", - "implementation_or_experiment", - "evidence_gate", - "artifact_sink", - "stop_rule", - ], + ] + + +def _visual_and_matrix_sections( + rows: list[dict[str, object]], + groups: dict[str, list[dict[str, object]]], + curated_visual_rows: list[dict[str, object]], +) -> list[str]: + return [ + *_section( + "Curaduria de figuras/tablas", + "El indice de captions no autoriza reproducir figuras ajenas ni convierte resultados externos en evidencia del proyecto. La curaduria siguiente solo define que visuales pueden inspirar tablas, esquemas propios, appendices o respuestas a reviewers.", + "", + markdown_table( + curated_visual_rows, + [ + "relative_path", + "caption_type", + "caption_index", + "editorial_sink", + "why_useful", + "claim_boundary", + ], + ), ), - "", - "## Curaduría de figuras/tablas", - "", - "El índice de captions no autoriza reproducir figuras ajenas ni convierte resultados externos en evidencia del proyecto. La curaduría siguiente solo define qué visuales pueden inspirar tablas, esquemas propios, appendices o respuestas a reviewers.", - "", - markdown_table( - curated_visual_rows, - [ - "relative_path", - "caption_type", - "caption_index", - "editorial_sink", - "why_useful", - "claim_boundary", - ], + *_section( + "Future work y stop rules", + markdown_table( + groups["future"], + ["relative_path", "decision", "crpto_value", "extended_lab_value", "stop_rule"], + ), ), - "", - "## Future work y stop rules", - "", - markdown_table( - future, ["relative_path", "decision", "crpto_value", "extended_lab_value", "stop_rule"] + *_section( + "Matriz paper-by-paper", + "La tabla siguiente es deliberadamente densa. Cada fila resume concepto, claim, metodo/evidencia, conclusion, figuras/tablas utiles, limitacion y destino editorial. Para auditoria operativa usar el CSV completo.", + "", + markdown_table( + rows, + [ + "relative_path", + "title", + "status", + "core_concepts", + "key_claims", + "conclusions", + "figures_tables_useful", + "limitations", + "decision", + "action_required", + ], + ), ), - "", - "## Matriz paper-by-paper", - "", - "La tabla siguiente es deliberadamente densa. Cada fila resume concepto, claim, método/evidencia, conclusión, figuras/tablas útiles, limitación y destino editorial. Para auditoría operativa usar el CSV completo.", - "", - markdown_table( - rows, - [ - "relative_path", - "title", - "status", - "core_concepts", - "key_claims", - "conclusions", - "figures_tables_useful", - "limitations", - "decision", - "action_required", - ], + ] + + +def _bibliography_and_closeout_sections(rows: list[dict[str, object]]) -> list[str]: + bib_counts = Counter(str(row["bib_status"]) for row in rows) + return [ + *_section( + "Control bibliografico", + "Regla aplicada: `book/references.bib` se modifica solo cuando una fuente queda citada o se prepara explicitamente para texto Quarto de Paper CRPTO/agenda extendida CRPTO/tesis. Las fuentes `needs_bib_if_cited` permanecen en la matriz sin inflar la bibliografia.", + "", + markdown_table(_counter_rows(bib_counts, "bib_status"), ["bib_status", "n"]), ), - "", - "## Control bibliográfico", - "", - "Regla aplicada: `book/references.bib` se modifica solo cuando una fuente queda citada o se prepara explícitamente para texto Quarto de Paper CRPTO/agenda extendida CRPTO/tesis. Las fuentes `needs_bib_if_cited` permanecen en la matriz sin inflar la bibliografía.", - "", - markdown_table( - [ - { - "bib_status": key, - "n": value, - } - for key, value in sorted(Counter(str(row["bib_status"]) for row in rows).items()) - ], - ["bib_status", "n"], + *_section( + "Fronteras que permanecen falsas", + "- CRPTO no reclama legal fair lending con atributos protegidos directos.", + "- CRPTO no implementa IFRS9 contractual.", + "- agenda extendida CRPTO/tesis no reclama CATE policy value.", + "- agenda extendida CRPTO/tesis no reclama online deployment.", + "- agenda extendida CRPTO/tesis no reclama Bellman/DLA exacto.", + "- SPO+/DFL puede ganar regret, pero no reemplaza la garantia/auditabilidad CRPTO.", + ), + *_section( + "Cierre", + "La auditoria agrega valor como integracion bibliografica y de claims. No crea un nuevo champion, no exige nuevas corridas y no transforma fuentes future-work en evidencia empirica del paper actual.", ), + ] + + +def _audit_lines( + rows: list[dict[str, object]], + matrix_path: Path, + caption_path: Path, + curated_visual_path: Path, + curated_visual_rows: list[dict[str, object]], +) -> list[str]: + groups = _decision_groups(rows) + lines = [ + "# Papers_tesis Deep Audit - 2026-06-06", "", - "## Fronteras que permanecen falsas", + "## Resumen ejecutivo", "", - "- CRPTO no reclama legal fair lending con atributos protegidos directos.", - "- CRPTO no implementa IFRS9 contractual.", - "- agenda extendida CRPTO/tesis no reclama CATE policy value.", - "- agenda extendida CRPTO/tesis no reclama online deployment.", - "- agenda extendida CRPTO/tesis no reclama Bellman/DLA exacto.", - "- SPO+/DFL puede ganar regret, pero no reemplaza la garantía/auditabilidad CRPTO.", + f"Esta auditoria cubre `61` PDFs locales en `{DEFAULT_SOURCE_DIR}` y fue generada para el corte `{REPORT_DATE}` con `scripts/build_papers_tesis_deep_audit.py`.", "", - "## Cierre", + "La decision central no cambia: **Paper CRPTO conserva el champion oficial** y la literatura nueva se usa para reforzar teoria, related work, appendices y limites de claim. La agenda extendida CRPTO/tesis absorbe el material que si pertenece al laboratorio vivo: source/shift conformal, utility-directed conformal, tail risk, DFL, IFRS9 proxy, data/noise/equity y governance.", "", - "La auditoría agrega valor como integración bibliográfica y de claims. No crea un nuevo champion, no exige nuevas corridas y no transforma fuentes future-work en evidencia empírica del paper actual.", + "Artefactos generados:", "", + f"- Matriz fuente: `{matrix_path}`", + f"- Indice compacto de captions: `{caption_path}`", + f"- Curaduria de visual sinks: `{curated_visual_path}`", ] + lines.extend(_inventory_sections(rows)) + lines.extend(_paper_crpto_sections(groups)) + lines.extend(_visual_and_matrix_sections(rows, groups, curated_visual_rows)) + lines.extend(_bibliography_and_closeout_sections(rows)) + lines.append("") + return lines + + +def write_audit( + path: Path, + rows: list[dict[str, object]], + matrix_path: Path, + caption_path: Path, + curated_visual_path: Path, + curated_visual_rows: list[dict[str, object]], +) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + lines = _audit_lines(rows, matrix_path, caption_path, curated_visual_path, curated_visual_rows) path.write_text("\n".join(lines), encoding="utf-8") diff --git a/scripts/build_tail_constrained_reoptimization.py b/scripts/build_tail_constrained_reoptimization.py index ea431d9..f7d6163 100644 --- a/scripts/build_tail_constrained_reoptimization.py +++ b/scripts/build_tail_constrained_reoptimization.py @@ -294,10 +294,10 @@ def build_tail_constrained_reoptimization(max_policies: int = 0) -> dict[str, An champion_return = float(promotion["final_champion"]["realized_total_return"]) scored_rows: list[dict[str, Any]] = [] - for idx, row in shortlist.iterrows(): + for position, row in enumerate((row for _, row in shortlist.iterrows()), start=1): logger.info( "Re-solving policy {}/{} (candidate_rank={}, mode={}, gamma={})", - idx + 1, + position, len(shortlist), row["candidate_rank"], row["policy_mode"], diff --git a/scripts/build_tail_satisficing_challenger_audit.py b/scripts/build_tail_satisficing_challenger_audit.py index a53bcc2..94ecf1b 100644 --- a/scripts/build_tail_satisficing_challenger_audit.py +++ b/scripts/build_tail_satisficing_challenger_audit.py @@ -11,7 +11,7 @@ import math import sys from pathlib import Path -from typing import Any +from typing import Any, cast import numpy as np import pandas as pd @@ -29,6 +29,7 @@ _solve_single, ) from src.optimization.tail_satisficing_objective import ( # noqa: E402 + SatisficingSense, SatisficingThreshold, entropic_oce, funded_loss_rate, @@ -150,11 +151,16 @@ def _thresholds_from_config(config: dict[str, Any]) -> tuple[SatisficingThreshol raw = config.get("satisficing_thresholds", {}) thresholds: list[SatisficingThreshold] = [] for metric, spec in raw.items(): + sense = str(spec["sense"]) + if sense not in {"min", "max", "equals"}: + raise ValueError(f"Unsupported satisficing threshold sense: {sense!r}") thresholds.append( SatisficingThreshold( metric=str(metric), - sense=str(spec["sense"]), # type: ignore[arg-type] - threshold=spec["threshold"], + sense=cast(SatisficingSense, sense), + threshold=float(spec["threshold"]) + if not isinstance(spec["threshold"], bool) + else spec["threshold"], ) ) return tuple(thresholds) @@ -312,10 +318,10 @@ def _build_a20_table() -> tuple[pd.DataFrame, dict[str, Any]]: loans, pd_point, pd_low, pd_high, lgd, int_rates, default_flag = _prepare_portfolio_inputs() rows: list[dict[str, Any]] = [] - for idx, row in shortlist.iterrows(): + for position, row in enumerate((row for _, row in shortlist.iterrows()), start=1): logger.info( "Auditing policy {}/{} (candidate_rank={})", - idx + 1, + position, len(shortlist), row["candidate_rank"], ) diff --git a/scripts/check_publication_integrity.py b/scripts/check_publication_integrity.py new file mode 100644 index 0000000..e04ff4d --- /dev/null +++ b/scripts/check_publication_integrity.py @@ -0,0 +1,226 @@ +"""Check active IJDS manuscript surfaces for claim and narrative drift.""" + +from __future__ import annotations + +import re +import sys +from dataclasses import dataclass +from pathlib import Path + +from loguru import logger + +REPO = Path(__file__).resolve().parents[1] + + +@dataclass(frozen=True) +class SurfaceCheck: + """Text surface and tokens that must or must not appear after normalization.""" + + path: Path + required: tuple[str, ...] + forbidden: tuple[str, ...] = () + + +COMMON_CLAIM_TOKENS = ( + "$184832.48", + "0.035350", + "0.162616", + "0.073584", + "0.245084", + "50010", + "27508", + "8/8", +) + +MAIN_SURFACE_REQUIRED = ( + *COMMON_CLAIM_TOKENS, + "0.345084", + "0.697056", + "196369.14", + "5.875%", + "8.305", + "43.55", + "decision certificate", + "single-submission boundary", +) + +ACTIVE_SURFACE_FORBIDDEN = ( + "five contributions", + "crpto v2", + "future work rather than", + "signed price is favorable", + "wins expected return", + "-10.56%", + "markov cap", + "0.510753", + "173314.04", + "zero deterministic violation", + "zero exact violation", +) + +SURFACES = ( + SurfaceCheck( + path=REPO / "README.md", + required=( + *COMMON_CLAIM_TOKENS, + "0.345084", + "claim ijds activo", + "upstream congelado", + "no como el claim activo", + ), + forbidden=("## champion congelado",), + ), + SurfaceCheck( + path=REPO / "paper" / "submission" / "README.md", + required=( + "28-page official-template pdf", + "pdflatex -> bibtex -> pdflatex -> pdflatex", + "latexmk", + "body remains inside the ijds 25-page", + ), + forbidden=( + "26-page official pdf", + "26 pages total", + "27-page official-template pdf", + "27 pages total", + ), + ), + SurfaceCheck( + path=REPO / "paper" / "CRPTO_ijds.qmd", + required=( + *MAIN_SURFACE_REQUIRED, + "the paper makes four contributions", + "one auditable post-hoc decision certificate", + "matched point-pd baseline", + ), + forbidden=ACTIVE_SURFACE_FORBIDDEN, + ), + SurfaceCheck( + path=REPO / "paper" / "submission" / "CRPTO_ijds_submission.tex", + required=( + *MAIN_SURFACE_REQUIRED, + "the paper makes four contributions", + "one auditable post-hoc decision certificate", + "matched point-pd baseline", + ), + forbidden=ACTIVE_SURFACE_FORBIDDEN, + ), + SurfaceCheck( + path=REPO / "paper" / "supplement_ijds.qmd", + required=( + *COMMON_CLAIM_TOKENS, + "decision certificate", + "single-submission boundary", + "outside the submitted claim", + "10423", + "2866", + "matched point-pd decision audit (a40)", + ), + forbidden=("crpto v2", "future work only", "markov cap", "0.510753"), + ), + SurfaceCheck( + path=REPO / "paper" / "submission" / "CLAIM_AUDIT_MATRIX.md", + required=( + "exclude the historical lending club -10.56% field", + "a40 reports a matched lending club cost of 5.875%", + "gamma_cp = gamma_int + gamma_res", + ), + forbidden=("lending club price -10.56%", "+27.03%", "markov cap"), + ), + SurfaceCheck( + path=REPO / "book" / "chapters" / "30-replicacion-multidataset.qmd", + required=( + "lending club no entra en esa serie", + "la auditoría point-pd corregida en tau=0.1715 es a40", + "frontera policy-aware a35", + "5.875%", + ), + forbidden=( + "lending club es la excepción informativa", + "la robustez nunca es económicamente catastrófica", + "+27.03%", + ), + ), + SurfaceCheck( + path=REPO / "scripts" / "generate_crpto_figures.py", + required=("stored nonrobust baseline was not a point-only comparator",), + forbidden=("lc_price", "sits below zero", "robustness adds value"), + ), + SurfaceCheck( + path=REPO / "docs" / "research" / "active_claims_2026-07-04.md", + required=( + *COMMON_CLAIM_TOKENS, + "0.345083866", + "decision certificate", + "outside the submitted claim", + "baseline semantics boundary", + "point-pd allocation earns $196369.14", + "maximum understatement was 0.241324", + ), + forbidden=("crpto v2", "future protocols", "markov cap", "+27.03%"), + ), + SurfaceCheck( + path=REPO / "configs" / "crpto_publication_targets.yaml", + required=("outside the submitted claim", "not acceptance criteria"), + forbidden=("future work and are not acceptance criteria", "crpto v2"), + ), +) + + +def _normalize(text: str) -> str: + """Normalize Markdown/LaTeX enough for robust manuscript-token checks.""" + lowered = text.lower() + replacements = { + "\\$": "$", + "{,}": ",", + "\\_": "_", + "\\mathrm": "", + "\\gamma": "gamma", + "\\alpha": "alpha", + "\\": "", + "{": "", + "}": "", + "`": "", + ",": "", + } + for old, new in replacements.items(): + lowered = lowered.replace(old, new) + lowered = lowered.replace("\u2013", "-").replace("\u2014", "-") + return re.sub(r"\s+", " ", lowered) + + +def _read_normalized(path: Path) -> str: + return _normalize(path.read_text(encoding="utf-8")) + + +def check_publication_integrity() -> list[str]: + """Return active-manuscript integrity failures.""" + failures: list[str] = [] + for surface in SURFACES: + if not surface.path.is_file(): + failures.append(f"{surface.path.relative_to(REPO)} is missing") + continue + text = _read_normalized(surface.path) + missing = [token for token in surface.required if token not in text] + present_forbidden = [token for token in surface.forbidden if token in text] + rel = surface.path.relative_to(REPO) + failures.extend(f"{rel}: missing required token '{token}'" for token in missing) + failures.extend( + f"{rel}: forbidden token still present '{token}'" for token in present_forbidden + ) + return failures + + +def main() -> int: + """CLI entry point.""" + failures = check_publication_integrity() + if failures: + for failure in failures: + logger.error(failure) + return 1 + logger.success("Active IJDS publication surfaces are claim-synchronized.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/compile_ijds_submission.py b/scripts/compile_ijds_submission.py new file mode 100644 index 0000000..24589d0 --- /dev/null +++ b/scripts/compile_ijds_submission.py @@ -0,0 +1,172 @@ +"""Compile and sanity-check the official IJDS LaTeX submission PDF.""" + +from __future__ import annotations + +import argparse +import os +import re +import shutil +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +from loguru import logger + +ROOT = Path(__file__).resolve().parents[1] +SUBMISSION_DIR = ROOT / "paper" / "submission" +REPORT_DIR = ROOT / "reports" / "ci" +TEX_NAME = "CRPTO_ijds_submission.tex" +JOB_NAME = "CRPTO_ijds_submission" + + +@dataclass(frozen=True) +class LatexScan: + """Summary of a compiled LaTeX submission surface.""" + + pages: int | None + blg_warnings: tuple[str, ...] + log_failures: tuple[str, ...] + + @property + def ok(self) -> bool: + return not self.blg_warnings and not self.log_failures + + +def _run(command: list[str], *, cwd: Path, env: dict[str, str], transcript: Path) -> int: + logger.info("Running: {}", " ".join(command)) + completed = subprocess.run( + command, + cwd=cwd, + env=env, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + encoding="utf-8", + errors="replace", + ) + transcript.parent.mkdir(parents=True, exist_ok=True) + with transcript.open("a", encoding="utf-8", errors="replace") as handle: + handle.write(f"\n$ {' '.join(command)}\n") + handle.write(completed.stdout) + return int(completed.returncode) + + +def _submission_env() -> dict[str, str]: + env = os.environ.copy() + if not env.get("WINDIR") and env.get("SystemRoot"): + env["WINDIR"] = env["SystemRoot"] + return env + + +def _manual_pdflatex_bibtex(cwd: Path, env: dict[str, str], transcript: Path) -> int: + sequence = [ + ["pdflatex", "-interaction=nonstopmode", "-halt-on-error", TEX_NAME], + ["bibtex", JOB_NAME], + ["pdflatex", "-interaction=nonstopmode", "-halt-on-error", TEX_NAME], + ["pdflatex", "-interaction=nonstopmode", "-halt-on-error", TEX_NAME], + ] + for command in sequence: + code = _run(command, cwd=cwd, env=env, transcript=transcript) + if code != 0: + return code + return 0 + + +def compile_submission(*, prefer_manual: bool = False) -> int: + """Compile the official submission with latexmk, falling back to manual passes.""" + if not (SUBMISSION_DIR / TEX_NAME).is_file(): + logger.error("Missing {}", (SUBMISSION_DIR / TEX_NAME).relative_to(ROOT)) + return 2 + + env = _submission_env() + transcript = REPORT_DIR / "ijds-latex-build.txt" + if transcript.exists(): + transcript.unlink() + if prefer_manual or shutil.which("latexmk") is None: + if shutil.which("latexmk") is None: + logger.warning("latexmk unavailable; using manual pdflatex/BibTeX fallback.") + return _manual_pdflatex_bibtex(SUBMISSION_DIR, env, transcript) + + latexmk_code = _run( + ["latexmk", "-pdf", "-gg", "-interaction=nonstopmode", TEX_NAME], + cwd=SUBMISSION_DIR, + env=env, + transcript=transcript, + ) + if latexmk_code == 0: + logger.info("LaTeX transcript: {}", transcript.relative_to(ROOT)) + return 0 + + logger.warning("latexmk failed with code {}; using manual fallback.", latexmk_code) + code = _manual_pdflatex_bibtex(SUBMISSION_DIR, env, transcript) + logger.info("LaTeX transcript: {}", transcript.relative_to(ROOT)) + return code + + +def scan_submission_logs() -> LatexScan: + """Inspect `.blg` and `.log` outputs for bibliography/reference drift.""" + blg_path = SUBMISSION_DIR / f"{JOB_NAME}.blg" + log_path = SUBMISSION_DIR / f"{JOB_NAME}.log" + blg_text = blg_path.read_text(encoding="utf-8", errors="replace") if blg_path.exists() else "" + log_text = log_path.read_text(encoding="utf-8", errors="replace") if log_path.exists() else "" + + blg_warnings = tuple( + line.strip() for line in blg_text.splitlines() if line.strip().startswith("Warning--") + ) + + checks = { + "undefined references": r"There were undefined references", + "undefined citations": r"Citation `[^`]+`.*undefined", + "undefined labels": r"Reference `[^`]+`.*undefined", + "rerun requested": r"Rerun to get cross-references right|Label\(s\) may have changed", + } + log_failures = tuple(name for name, pattern in checks.items() if re.search(pattern, log_text)) + + pages: int | None = None + matches = re.findall(r"Output written on .+?\((\d+) pages?,", log_text) + if matches: + pages = int(matches[-1]) + + return LatexScan(pages=pages, blg_warnings=blg_warnings, log_failures=log_failures) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--manual", + action="store_true", + help="Skip latexmk and use pdflatex + bibtex + pdflatex + pdflatex directly.", + ) + parser.add_argument( + "--scan-only", + action="store_true", + help="Do not compile; only inspect existing .log and .blg files.", + ) + args = parser.parse_args(argv) + + if not args.scan_only: + code = compile_submission(prefer_manual=args.manual) + if code != 0: + logger.error("Official IJDS LaTeX compile failed with code {}.", code) + return code + + scan = scan_submission_logs() + if scan.blg_warnings: + logger.error("BibTeX warnings:\n{}", "\n".join(scan.blg_warnings)) + if scan.log_failures: + logger.error("LaTeX convergence failures: {}", ", ".join(scan.log_failures)) + if scan.pages is None: + logger.warning("Could not read page count from the LaTeX log.") + else: + logger.success("Official IJDS PDF page count: {}", scan.pages) + + if not scan.ok: + return 1 + logger.success("Official IJDS LaTeX build is citation/reference clean.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/experiments/run_champion_claim_max_downstream.py b/scripts/experiments/run_champion_claim_max_downstream.py index db217c3..4c90501 100644 --- a/scripts/experiments/run_champion_claim_max_downstream.py +++ b/scripts/experiments/run_champion_claim_max_downstream.py @@ -26,6 +26,31 @@ CHAMPION_BRIER = 0.15439302183275685 CHAMPION_ECE = 0.006998009158194006 +CUOPT_FLAG_MAP = { + "presolve": "--cuopt-presolve", + "method": "--cuopt-method", + "pdlp_solver_mode": "--cuopt-pdlp-solver-mode", + "pdlp_precision": "--cuopt-pdlp-precision", + "crossover": "--cuopt-crossover", + "first_primal_feasible": "--cuopt-first-primal-feasible", + "save_best_primal_solution": "--cuopt-save-best-primal-solution", + "infeasibility_detection": "--cuopt-infeasibility-detection", + "strict_infeasibility": "--cuopt-strict-infeasibility", + "per_constraint_residual": "--cuopt-per-constraint-residual", + "dual_postsolve": "--cuopt-dual-postsolve", + "dualize": "--cuopt-dualize", + "folding": "--cuopt-folding", + "augmented": "--cuopt-augmented", + "ordering": "--cuopt-ordering", + "cudss_deterministic": "--cuopt-cudss-deterministic", + "eliminate_dense_columns": "--cuopt-eliminate-dense-columns", + "iteration_limit": "--cuopt-iteration-limit", + "num_cpu_threads": "--cuopt-num-cpu-threads", + "num_gpus": "--cuopt-num-gpus", + "log_to_console": "--cuopt-log-to-console", + "log_dir": "--cuopt-log-dir", +} + def _utc_now() -> str: return datetime.now(tz=UTC).isoformat() @@ -259,22 +284,35 @@ def _conformal_intervals_for_status(status: dict[str, Any]) -> Path: ) -def _portfolio_command( +def _profile_section(portfolio_profile: dict[str, Any], key: str) -> dict[str, Any]: + return dict(portfolio_profile.get(key, {}) or {}) + + +def _append_option(command: list[str], flag: str, value: Any) -> None: + if value is not None and str(value).strip(): + command.extend([flag, str(value)]) + + +def _append_int_option(command: list[str], flag: str, value: Any) -> None: + if value is not None and str(value).strip(): + command.extend([flag, str(int(value))]) + + +def _portfolio_base_command( *, portfolio_profile: dict[str, Any], conformal_intervals_path: Path, run_label: str, output_dir: Path, model_dir: Path, + grids: dict[str, Any], + frontier: dict[str, Any], + incumbent: dict[str, Any], + execution: dict[str, Any], ) -> list[str]: - grids = dict(portfolio_profile.get("grids", {}) or {}) - frontier = dict(portfolio_profile.get("frontier", {}) or {}) - incumbent = dict(portfolio_profile.get("incumbent_region", {}) or {}) - execution = dict(portfolio_profile.get("execution", {}) or {}) - cuopt = dict(portfolio_profile.get("cuopt", {}) or {}) python_executable = str(execution.get("python_executable") or sys.executable) policy_modes = ",".join(str(x) for x in portfolio_profile.get("candidate_policy_families", [])) - command = [ + return [ python_executable, "scripts/search/run_portfolio_bound_aware_search.py", "--config", @@ -328,54 +366,62 @@ def _portfolio_command( "--exact-solver-backend", str(execution.get("exact_solver_backend", "highs")), ] - if frontier.get("exact_max_candidates") is not None: - command.extend(["--exact-max-candidates", str(int(frontier["exact_max_candidates"]))]) + + +def _append_frontier_options( + command: list[str], + *, + grids: dict[str, Any], + frontier: dict[str, Any], +) -> None: + _append_int_option(command, "--exact-max-candidates", frontier.get("exact_max_candidates")) exact_random_states = frontier.get("exact_random_states", grids.get("exact_random_states")) - if exact_random_states is not None and str(exact_random_states).strip(): - command.extend(["--exact-random-states", str(exact_random_states)]) - exact_checkpoint_every = frontier.get("exact_checkpoint_every") - if exact_checkpoint_every is not None and str(exact_checkpoint_every).strip(): - command.extend(["--exact-checkpoint-every", str(int(exact_checkpoint_every))]) - exact_threads = frontier.get("exact_threads") - if exact_threads is not None and str(exact_threads).strip(): - command.extend(["--exact-threads", str(int(exact_threads))]) - budget_profiles = grids.get("budget_profiles") - if budget_profiles is not None and str(budget_profiles).strip(): - command.extend(["--budget-profiles", str(budget_profiles)]) + _append_option(command, "--exact-random-states", exact_random_states) + _append_int_option(command, "--exact-checkpoint-every", frontier.get("exact_checkpoint_every")) + _append_int_option(command, "--exact-threads", frontier.get("exact_threads")) + _append_option(command, "--budget-profiles", grids.get("budget_profiles")) + + +def _append_execution_options(command: list[str], *, execution: dict[str, Any]) -> None: if bool(execution.get("frontier_only", False)): command.append("--frontier-only") - if str(execution.get("exact_python_executable", "")).strip(): - command.extend(["--exact-python-executable", str(execution["exact_python_executable"])]) - cuopt_flag_map = { - "presolve": "--cuopt-presolve", - "method": "--cuopt-method", - "pdlp_solver_mode": "--cuopt-pdlp-solver-mode", - "pdlp_precision": "--cuopt-pdlp-precision", - "crossover": "--cuopt-crossover", - "first_primal_feasible": "--cuopt-first-primal-feasible", - "save_best_primal_solution": "--cuopt-save-best-primal-solution", - "infeasibility_detection": "--cuopt-infeasibility-detection", - "strict_infeasibility": "--cuopt-strict-infeasibility", - "per_constraint_residual": "--cuopt-per-constraint-residual", - "dual_postsolve": "--cuopt-dual-postsolve", - "dualize": "--cuopt-dualize", - "folding": "--cuopt-folding", - "augmented": "--cuopt-augmented", - "ordering": "--cuopt-ordering", - "cudss_deterministic": "--cuopt-cudss-deterministic", - "eliminate_dense_columns": "--cuopt-eliminate-dense-columns", - "iteration_limit": "--cuopt-iteration-limit", - "num_cpu_threads": "--cuopt-num-cpu-threads", - "num_gpus": "--cuopt-num-gpus", - "log_to_console": "--cuopt-log-to-console", - "log_dir": "--cuopt-log-dir", - } - for key, flag in cuopt_flag_map.items(): - value = cuopt.get(key) - if value is not None and str(value).strip() != "": - command.extend([flag, str(value)]) + _append_option(command, "--exact-python-executable", execution.get("exact_python_executable")) + + +def _append_cuopt_options(command: list[str], *, cuopt: dict[str, Any]) -> None: + for key, flag in CUOPT_FLAG_MAP.items(): + _append_option(command, flag, cuopt.get(key)) for key, value in dict(cuopt.get("extra_parameters", {}) or {}).items(): command.extend(["--cuopt-extra-parameter", f"{key}={value}"]) + + +def _portfolio_command( + *, + portfolio_profile: dict[str, Any], + conformal_intervals_path: Path, + run_label: str, + output_dir: Path, + model_dir: Path, +) -> list[str]: + grids = _profile_section(portfolio_profile, "grids") + frontier = _profile_section(portfolio_profile, "frontier") + incumbent = _profile_section(portfolio_profile, "incumbent_region") + execution = _profile_section(portfolio_profile, "execution") + cuopt = _profile_section(portfolio_profile, "cuopt") + command = _portfolio_base_command( + portfolio_profile=portfolio_profile, + conformal_intervals_path=conformal_intervals_path, + run_label=run_label, + output_dir=output_dir, + model_dir=model_dir, + grids=grids, + frontier=frontier, + incumbent=incumbent, + execution=execution, + ) + _append_frontier_options(command, grids=grids, frontier=frontier) + _append_execution_options(command, execution=execution) + _append_cuopt_options(command, cuopt=cuopt) return command diff --git a/scripts/experiments/run_champion_reopen_hpo.py b/scripts/experiments/run_champion_reopen_hpo.py index 89c25d4..efdb5cc 100644 --- a/scripts/experiments/run_champion_reopen_hpo.py +++ b/scripts/experiments/run_champion_reopen_hpo.py @@ -219,9 +219,10 @@ def main() -> None: pool_features, feature_config=feature_config, ) - core_features = _resolve_core_features(feature_config, train.columns) - catboost_features = _resolve_catboost_features(feature_config, train.columns) - woe_features = _resolve_woe_features(feature_config, train.columns) + train_columns = [str(column) for column in train.columns] + core_features = _resolve_core_features(feature_config, train_columns) + catboost_features = _resolve_catboost_features(feature_config, train_columns) + woe_features = _resolve_woe_features(feature_config, train_columns) train_fit, train_val = temporal_train_val_split( train, diff --git a/scripts/experiments/run_tabpfn_tabprep_full.py b/scripts/experiments/run_tabpfn_tabprep_full.py index 4e9d7a2..d3787a5 100644 --- a/scripts/experiments/run_tabpfn_tabprep_full.py +++ b/scripts/experiments/run_tabpfn_tabprep_full.py @@ -8,6 +8,7 @@ import argparse import gc +import importlib import json import os import sys @@ -184,9 +185,8 @@ def main() -> None: del train, generated_train gc.collect() - from tabpfn import TabPFNClassifier - - clf = TabPFNClassifier( + tabpfn_classifier = _tabpfn_classifier_class() + clf = tabpfn_classifier( n_estimators=int(config["tabpfn"]["n_estimators"]), categorical_features_indices=categorical_indices, device=str(config["tabpfn"]["device"]), @@ -292,10 +292,10 @@ def _load_config(path: Path) -> dict[str, Any]: def _check_tabpfn_access(config: Mapping[str, Any]) -> None: """Fail before loading CRPTO data if TabPFN-3 weights are not accessible.""" from sklearn.datasets import make_classification - from tabpfn import TabPFNClassifier + tabpfn_classifier = _tabpfn_classifier_class() x, y = make_classification(n_samples=80, n_features=8, random_state=42) - clf = TabPFNClassifier( + clf = tabpfn_classifier( n_estimators=1, device=str(config["tabpfn"]["device"]), ignore_pretraining_limits=True, @@ -317,12 +317,27 @@ def _check_tabpfn_access(config: Mapping[str, Any]) -> None: ) from exc +def _tabpfn_classifier_class() -> Any: + """Return the optional TabPFN classifier class with a readable error.""" + try: + module = importlib.import_module("tabpfn") + except ImportError as exc: + raise RuntimeError( + "TabPFN + TabPrep is an isolated challenger and requires the optional " + "TabPFN package plus PriorLabs model access. Install/authorize TabPFN " + "before running this experiment." + ) from exc + return module.TabPFNClassifier + + def _preflight_dataset_limits(*, config: Mapping[str, Any], force: bool) -> None: """Check full-data shape against public TabPFN-3 limits.""" parquet_file = pq.ParquetFile(config["data"]["train_path"]) train_rows = int(parquet_file.metadata.num_rows) schema_columns = list(parquet_file.schema_arrow.names) - schema_frame = pd.DataFrame(columns=schema_columns) + schema_frame = pd.DataFrame( + {str(column): pd.Series(dtype="object") for column in schema_columns} + ) feature_config = load_feature_config( yaml_path=Path(config["data"]["feature_config_path"]), prefer="yaml", diff --git a/scripts/experiments/run_tabprep_feature_selection_catboost.py b/scripts/experiments/run_tabprep_feature_selection_catboost.py index b024b73..8a1212a 100644 --- a/scripts/experiments/run_tabprep_feature_selection_catboost.py +++ b/scripts/experiments/run_tabprep_feature_selection_catboost.py @@ -190,9 +190,10 @@ def main() -> None: pool_features, feature_config=feature_config, ) - core_features = _resolve_core_features(feature_config, train.columns) - catboost_features = _resolve_catboost_features(feature_config, train.columns) - woe_features = _resolve_woe_features(feature_config, train.columns) + train_columns = [str(column) for column in train.columns] + core_features = _resolve_core_features(feature_config, train_columns) + catboost_features = _resolve_catboost_features(feature_config, train_columns) + woe_features = _resolve_woe_features(feature_config, train_columns) train_fit, train_val = temporal_train_val_split( train, diff --git a/scripts/export_crpto_tables.py b/scripts/export_crpto_tables.py index 4f41837..bdb0ac5 100644 --- a/scripts/export_crpto_tables.py +++ b/scripts/export_crpto_tables.py @@ -18,8 +18,8 @@ from typing import Any import pandas as pd -from analyze_crpto_evidence import build_p1_evidence # type: ignore[import-not-found] +from scripts.analyze_crpto_evidence import build_p1_evidence from src.utils.script_helpers import first_existing, load_json, policy_matches, write_table ROOT = Path(__file__).resolve().parents[1] @@ -76,7 +76,7 @@ def _table0_key_metrics( ("alpha01_gamma_cp", champ["alpha01_gamma_cp"]), ("alpha01_violation", champ["alpha01_violation"]), ] - return pd.DataFrame(rows, columns=["metric", "value"]) + return pd.DataFrame([{"metric": metric, "value": value} for metric, value in rows]) def _table1_robustness_summary( diff --git a/scripts/generate_conformal_intervals.py b/scripts/generate_conformal_intervals.py index 9a01e9c..165d18a 100644 --- a/scripts/generate_conformal_intervals.py +++ b/scripts/generate_conformal_intervals.py @@ -11,8 +11,10 @@ import pickle import shutil import time +from collections.abc import Iterable from dataclasses import dataclass from datetime import UTC, datetime +from itertools import product from pathlib import Path from typing import Any @@ -57,6 +59,56 @@ TARGET_COL = "default_flag" GROUP_COL = "grade" +PartitionCache = dict[ + tuple[str, str, str, int, str, int], + tuple[pd.Series, pd.Series, dict[str, Any]], +] + + +def _empty_coverage_floor_report() -> pd.DataFrame: + return pd.DataFrame( + { + "group": pd.Series(dtype="object"), + "coverage_before": pd.Series(dtype="float64"), + "coverage_after": pd.Series(dtype="float64"), + "target_coverage": pd.Series(dtype="float64"), + "multiplier": pd.Series(dtype="float64"), + "adjusted": pd.Series(dtype="bool"), + } + ) + + +def _empty_temporal_segment_report() -> pd.DataFrame: + return pd.DataFrame( + { + "segment": pd.Series(dtype="object"), + "support_n": pd.Series(dtype="int64"), + "coverage_before": pd.Series(dtype="float64"), + "coverage_after": pd.Series(dtype="float64"), + "target_coverage": pd.Series(dtype="float64"), + "min_segment_size": pd.Series(dtype="int64"), + "multiplier": pd.Series(dtype="float64"), + "adjusted": pd.Series(dtype="bool"), + } + ) + + +def _empty_shrinkback_report() -> pd.DataFrame: + return pd.DataFrame( + { + "stage": pd.Series(dtype="object"), + "factor_scope": pd.Series(dtype="object"), + "factor_key": pd.Series(dtype="object"), + "candidate_factor": pd.Series(dtype="float64"), + "accepted": pd.Series(dtype="bool"), + "coverage": pd.Series(dtype="float64"), + "min_group_coverage": pd.Series(dtype="float64"), + "avg_width": pd.Series(dtype="float64"), + "winkler_90": pd.Series(dtype="float64"), + "max_monthly_gap": pd.Series(dtype="float64"), + "stability_over_time": pd.Series(dtype="float64"), + } + ) def _utc_now() -> str: @@ -150,6 +202,73 @@ class ConformalTuningSelection: best_cfg: dict[str, Any] +@dataclass(frozen=True) +class ConformalTuningSearch: + """Raw tuning rows plus cached partition labels reusable for final intervals.""" + + tuning_rows: list[dict[str, Any]] + partition_cache: PartitionCache + + +@dataclass(frozen=True) +class GlobalRebalanceResult: + """Final 90% intervals and diagnostics after optional global rebalance.""" + + y_intervals: np.ndarray + metrics: dict[str, Any] + group_metrics: pd.DataFrame + factor: float + diagnostics: dict[str, float | bool] + + +@dataclass(frozen=True) +class ConformalArtifactTables: + """Final tabular conformal artifacts ready for persistence.""" + + intervals: pd.DataFrame + group_metrics: pd.DataFrame + width_attribution: pd.DataFrame + + +@dataclass(frozen=True) +class ConformalEvidence90: + """Adjusted 90% interval evidence and learned coverage-floor policy.""" + + y_eval: pd.Series + eval_groups: pd.Series + eval_issue: pd.Series + y_pred: np.ndarray + y_intervals: np.ndarray + diag: dict[str, Any] + metrics: dict[str, Any] + group_metrics: pd.DataFrame + y_pred_tune: np.ndarray + y_intervals_tune_base: np.ndarray + tune_metrics_before_floor: dict[str, Any] + tune_metrics_after_floor: dict[str, Any] + tune_metrics_after_temporal_floor: dict[str, Any] + group_multipliers: dict[str, float] + coverage_floor_report: pd.DataFrame + temporal_segment_multipliers: dict[str, float] + temporal_segment_report: pd.DataFrame + shrinkback_report: pd.DataFrame + eval_temporal_segments: pd.Series | None + width_attr_rows: list[dict[str, Any]] + global_rebalance_factor: float + global_rebalance_diagnostics: dict[str, float | bool] + + +@dataclass(frozen=True) +class ConformalEvidence95: + """95% interval evidence after applying the learned 90% adjustment policy.""" + + y_pred: np.ndarray + y_intervals: np.ndarray + diag: dict[str, Any] + metrics: dict[str, Any] + group_metrics: pd.DataFrame + + def _resolve_artifact_paths(namespace: str | None = None) -> dict[str, Path]: if namespace: ns = str(namespace).strip().replace("/", "_") @@ -306,62 +425,83 @@ def _load_calibrator(calibrator_override_path: str | None = None) -> Any | None: return calibrator -def _resolve_features( - model: CatBoostClassifier, - cal_df: pd.DataFrame, - test_df: pd.DataFrame, -) -> tuple[list[str], list[str]]: - """Resolve feature list, preferring explicit contract then model metadata.""" - contract = _load_active_contract() - if isinstance(contract, dict): - contract_features = contract.get("feature_names", []) - contract_categorical = contract.get("categorical_features", []) - if contract_features: - categorical = [c for c in contract_categorical if c in contract_features] - logger.info( - f"Using {len(contract_features)} contract features ({len(categorical)} categorical) " - f"from {contract.get('_contract_path', CONTRACT_PATH)}" - ) - return list(contract_features), categorical +def _features_from_contract(contract: dict[str, Any] | None) -> tuple[list[str], list[str]] | None: + if not isinstance(contract, dict): + return None + contract_features = list(contract.get("feature_names", []) or []) + if not contract_features: + return None + contract_categorical = list(contract.get("categorical_features", []) or []) + categorical = [feature for feature in contract_categorical if feature in contract_features] + logger.info( + f"Using {len(contract_features)} contract features ({len(categorical)} categorical) " + f"from {contract.get('_contract_path', CONTRACT_PATH)}" + ) + return contract_features, categorical + +def _features_from_model(model: CatBoostClassifier) -> tuple[list[str], list[str]] | None: model_features = list(getattr(model, "feature_names_", []) or []) - if model_features: - cat_idxs = set(model.get_cat_feature_indices()) - categorical = [f for i, f in enumerate(model_features) if i in cat_idxs] - logger.info( - f"Using {len(model_features)} model-native features ({len(categorical)} categorical)" - ) - return model_features, categorical + if not model_features: + return None + categorical_indexes = set(model.get_cat_feature_indices()) + categorical = [ + feature for index, feature in enumerate(model_features) if index in categorical_indexes + ] + logger.info( + f"Using {len(model_features)} model-native features ({len(categorical)} categorical)" + ) + return model_features, categorical + - # Fallback path if model metadata is unavailable. - feature_cfg_path = Path("data/processed/feature_config.yml") - feature_cfg: dict[str, Any] = {} +def _load_fallback_feature_config(feature_cfg_path: Path) -> dict[str, Any]: try: - feature_cfg = load_feature_config_artifact( + return load_feature_config_artifact( pickle_path=feature_cfg_path.with_suffix(".pkl"), yaml_path=feature_cfg_path, prefer="yaml", ) except (FileNotFoundError, TypeError) as exc: logger.warning(f"Unable to load fallback feature_config from {feature_cfg_path}: {exc}") + return {} + +def _features_from_fallback_config( + *, + cal_df: pd.DataFrame, + test_df: pd.DataFrame, +) -> tuple[list[str], list[str]]: + feature_cfg = _load_fallback_feature_config(Path("data/processed/feature_config.yml")) catboost_features = feature_cfg.get("CATBOOST_FEATURES", []) categorical = feature_cfg.get("CATEGORICAL_FEATURES", []) - - features = [c for c in catboost_features if c in cal_df.columns and c in test_df.columns] + features = [ + name for name in catboost_features if name in cal_df.columns and name in test_df.columns + ] if not features: from src.models.pd_model import get_available_features - features = [c for c in get_available_features(cal_df) if c in test_df.columns] - + features = [name for name in get_available_features(cal_df) if name in test_df.columns] if not features: raise ValueError("Unable to resolve feature list for conformal generation.") - - categorical = [c for c in categorical if c in features] + categorical = [name for name in categorical if name in features] logger.info(f"Using {len(features)} features ({len(categorical)} categorical)") return features, categorical +def _resolve_features( + model: CatBoostClassifier, + cal_df: pd.DataFrame, + test_df: pd.DataFrame, +) -> tuple[list[str], list[str]]: + """Resolve feature list, preferring explicit contract then model metadata.""" + contract = _load_active_contract() + return ( + _features_from_contract(contract) + or _features_from_model(model) + or _features_from_fallback_config(cal_df=cal_df, test_df=test_df) + ) + + def _build_feature_matrix( df: pd.DataFrame, features: list[str], @@ -383,6 +523,33 @@ def _build_feature_matrix( return X +def _align_contract_matrix( + matrix: pd.DataFrame, + reference_frame: pd.DataFrame, + *, + path: Path, + split_name: str, +) -> pd.DataFrame: + if len(matrix) == len(reference_frame): + return matrix + if "id" in matrix.columns and "id" in reference_frame.columns: + indexed = matrix.set_index("id", drop=False) + wanted = reference_frame["id"].tolist() + missing_ids = [value for value in wanted if value not in indexed.index] + if missing_ids: + raise KeyError( + f"Contract model matrix {path} missing {len(missing_ids)} ids; " + f"first missing={missing_ids[:5]}" + ) + return indexed.loc[wanted].reset_index(drop=True) + if len(matrix) > len(reference_frame): + return matrix.tail(len(reference_frame)).reset_index(drop=True) + raise ValueError( + f"Contract model matrix row count mismatch for {split_name}: " + f"matrix={len(matrix)}, reference={len(reference_frame)}" + ) + + def _load_contract_matrix( *, contract: dict[str, Any] | None, @@ -403,24 +570,12 @@ def _load_contract_matrix( if not path.exists(): raise FileNotFoundError(f"Contract model matrix not found for {split_name}: {path}") matrix = pd.read_parquet(path) - if len(matrix) != len(reference_frame): - if "id" in matrix.columns and "id" in reference_frame.columns: - indexed = matrix.set_index("id", drop=False) - wanted = reference_frame["id"].tolist() - missing_ids = [value for value in wanted if value not in indexed.index] - if missing_ids: - raise KeyError( - f"Contract model matrix {path} missing {len(missing_ids)} ids; " - f"first missing={missing_ids[:5]}" - ) - matrix = indexed.loc[wanted].reset_index(drop=True) - elif len(matrix) > len(reference_frame): - matrix = matrix.tail(len(reference_frame)).reset_index(drop=True) - else: - raise ValueError( - f"Contract model matrix row count mismatch for {split_name}: " - f"matrix={len(matrix)}, reference={len(reference_frame)}" - ) + matrix = _align_contract_matrix( + matrix, + reference_frame, + path=path, + split_name=split_name, + ) missing = [feature for feature in features if feature not in matrix.columns] if missing: raise KeyError( @@ -566,6 +721,29 @@ def _load_conformal_inputs( ) +def _dedupe_string_candidates( + values: Iterable[Any], + *, + default: tuple[str, ...], + lower: bool = False, +) -> tuple[str, ...]: + normalized = [] + for value in values: + token = str(value).strip() + if token: + normalized.append(token.lower() if lower else token) + return tuple(dict.fromkeys(normalized)) or default + + +def _positive_int_candidates( + values: Iterable[Any], + *, + default: tuple[int, ...], +) -> tuple[int, ...]: + resolved = tuple(int(value) for value in values if int(value) > 0) + return resolved or default + + def _resolve_tuning_grid( *, partition: str, @@ -577,35 +755,26 @@ def _resolve_tuning_grid( scaled_scores_options: tuple[bool, ...], ) -> ConformalTuningGrid: """Normalize and de-duplicate Mondrian tuning candidates.""" - resolved_partitions = tuple( - dict.fromkeys( - [ - str(token).strip() - for token in (partition_candidates or (partition,)) - if str(token).strip() - ] - ) - ) or (str(partition).strip() or "grade",) - resolved_probability_sources = tuple( - dict.fromkeys( - str(source).strip().lower() - for source in partition_probability_sources - if str(source).strip() - ) - ) or ("raw",) - resolved_score_bins = tuple(int(x) for x in n_score_bins_candidates if int(x) > 0) or (10,) - resolved_fallback_modes = tuple( - dict.fromkeys( - str(mode_name).strip().lower() for mode_name in fallback_modes if str(mode_name).strip() - ) - ) or ("grade_then_global",) - resolved_score_families = tuple( - dict.fromkeys( - str(scale_name).strip().lower() - for scale_name in score_scale_families - if str(scale_name).strip() - ) - ) or ("none",) + resolved_partitions = _dedupe_string_candidates( + partition_candidates or (partition,), + default=(str(partition).strip() or "grade",), + ) + resolved_probability_sources = _dedupe_string_candidates( + partition_probability_sources, + default=("raw",), + lower=True, + ) + resolved_score_bins = _positive_int_candidates(n_score_bins_candidates, default=(10,)) + resolved_fallback_modes = _dedupe_string_candidates( + fallback_modes, + default=("grade_then_global",), + lower=True, + ) + resolved_score_families = _dedupe_string_candidates( + score_scale_families, + default=("none",), + lower=True, + ) return ConformalTuningGrid( partition_candidates=resolved_partitions, partition_probability_sources=resolved_probability_sources, @@ -725,472 +894,616 @@ def _select_best_tuning_config( ) -def _parse_float_tuple(raw: str) -> tuple[float, ...]: - values = [float(token.strip()) for token in str(raw).split(",") if token.strip()] - if not values: - raise ValueError("Expected at least one float value.") - return tuple(values) +def _log_tuning_split(inputs: ConformalInputs, tuning_split: ConformalTuningSplit) -> None: + logger.info( + "Calibration split for conformal tuning: " + f"fit={len(tuning_split.X_cal_fit):,}, holdout={len(tuning_split.X_tune):,}, " + f"holdout_ratio={len(tuning_split.X_tune) / max(len(inputs.X_cal), 1):.2%}" + ) + if "issue_d" not in inputs.cal_df.columns: + return + fit_issue = tuning_split.issue_cal.iloc[tuning_split.idx_cal_fit] + tune_issue = tuning_split.issue_tune + if fit_issue.notna().any() and tune_issue.notna().any(): + logger.info( + "Calibration split date ranges: " + f"fit_max={fit_issue.max():%Y-%m}, " + f"holdout_min={tune_issue.min():%Y-%m}, " + f"holdout_max={tune_issue.max():%Y-%m}" + ) -def _parse_int_tuple(raw: str) -> tuple[int, ...]: - values = [int(token.strip()) for token in str(raw).split(",") if token.strip()] - if not values: - raise ValueError("Expected at least one integer value.") - return tuple(values) +def _build_probability_lookups( + inputs: ConformalInputs, + tuning_split: ConformalTuningSplit, +) -> tuple[dict[str, np.ndarray], dict[str, np.ndarray], dict[str, np.ndarray]]: + return ( + { + "raw": tuning_split.y_prob_cal_fit, + "calibrated": inputs.y_prob_calibrated[tuning_split.idx_cal_fit], + }, + { + "raw": tuning_split.y_prob_cal_tune, + "calibrated": inputs.y_prob_calibrated[tuning_split.idx_cal_tune], + }, + {"raw": inputs.y_prob_test_raw, "calibrated": inputs.y_prob_test_calibrated}, + ) -def _parse_bool_tuple(raw: str) -> tuple[bool, ...]: - values = [ - token.strip().lower() in {"1", "true", "yes", "y"} - for token in str(raw).split(",") - if token.strip() - ] - if not values: - raise ValueError("Expected at least one boolean value.") - return tuple(values) +def _tuning_total_candidates( + tuning_grid: ConformalTuningGrid, + *, + alpha_candidates_90: tuple[float, ...], + min_group_sizes: tuple[int, ...], +) -> int: + return ( + len(tuning_grid.partition_candidates) + * len(tuning_grid.partition_probability_sources) + * len(tuning_grid.n_score_bins_candidates) + * len(tuning_grid.fallback_modes) + * len(alpha_candidates_90) + * len(tuning_grid.scaled_scores_options) + * len(tuning_grid.score_scale_families) + * len(min_group_sizes) + ) -def _parse_str_tuple(raw: str) -> tuple[str, ...]: - values = tuple(token.strip() for token in str(raw).split(",") if token.strip()) - if not values: - raise ValueError("Expected at least one string value.") - return values +def _cached_partition_labels( + *, + cache: PartitionCache, + eval_scope: str, + partition_candidate: str, + partition_probability_source: str, + n_score_bins: int, + fallback_mode: str, + min_group_size: int, + prob_fit_lookup: dict[str, np.ndarray], + prob_tune_lookup: dict[str, np.ndarray], + prob_test_lookup: dict[str, np.ndarray], + group_cal_fit_base: pd.Series, + group_tune_base: pd.Series, + group_test_base: pd.Series, +) -> tuple[pd.Series, pd.Series, dict[str, Any]]: + key = ( + str(eval_scope), + str(partition_candidate), + str(partition_probability_source), + int(n_score_bins), + str(fallback_mode), + int(min_group_size), + ) + if key in cache: + return cache[key] + if eval_scope == "test": + y_prob_eval = prob_test_lookup[partition_probability_source] + base_groups_eval = group_test_base + elif eval_scope == "tune": + y_prob_eval = prob_tune_lookup[partition_probability_source] + base_groups_eval = group_tune_base + else: + raise ValueError(f"Unsupported cached partition eval_scope: {eval_scope}") + payload = build_mondrian_partition_labels( + y_prob_cal=prob_fit_lookup[partition_probability_source], + y_prob_eval=y_prob_eval, + partition=partition_candidate, + base_groups_cal=group_cal_fit_base, + base_groups_eval=base_groups_eval, + n_score_bins=n_score_bins, + min_group_size=min_group_size, + fallback_mode=fallback_mode, + ) + cache[key] = payload + return payload -def main( - alpha_target_90: float = 0.10, - alpha_95: float = 0.05, - alpha_candidates_90: tuple[float, ...] = (0.10, 0.095, 0.09, 0.085, 0.08), - alpha_candidates_95: tuple[float, ...] = (0.05,), - min_group_sizes: tuple[int, ...] = (200, 500, 1000, 2000), - min_group_coverage_target: float = 0.88, - group_coverage_floor_target_90: float = 0.92, - max_width_budget_90: float | None = 0.80, - coverage_guardband_90: float = 0.015, - min_group_guardband_90: float = 0.0, - tuning_holdout_ratio: float = 0.20, - tuning_random_state: int = 42, - temporal_segment_floor_enabled: bool = True, - temporal_segment_freq: str = "Q", - temporal_segment_min_size: int = 250, - global_rebalance_enabled: bool = False, - global_rebalance_min_factor: float = 0.75, - global_rebalance_max_factor: float = 1.05, - global_rebalance_step: float = 0.01, - partition: str = "grade", - partition_candidates: tuple[str, ...] | None = None, - group_coverage_floor_enabled: bool = True, - shrinkback_enabled: bool = False, - group_multiplier_grid: tuple[float, ...] = (1.0, 1.02, 1.05, 1.08, 1.12, 1.16, 1.20), - temporal_multiplier_grid: tuple[float, ...] = (1.0, 1.02, 1.05, 1.08, 1.12, 1.16, 1.20), - artifact_namespace: str | None = None, - scaled_scores_options: tuple[bool, ...] = (True, False), - score_scale_families: tuple[str, ...] = ("bernoulli_sqrt", "none"), - partition_probability_sources: tuple[str, ...] = ("raw",), - n_score_bins_candidates: tuple[int, ...] = (10,), - fallback_modes: tuple[str, ...] = ("grade_then_global",), - calibration_fraction: float = 1.0, - evaluation_scope: str = "test", - mode: str = "search", - replay_manifest_path: str | None = None, - calibrator_override_path: str | None = None, - run_tag: str | None = None, -): - logger.info("Starting Mondrian conformal interval generation with 90% auto-tuning") - run_mode = str(mode or "search").strip().lower() or "search" - replay_cfg = manifest_section(load_replay_manifest(replay_manifest_path), "conformal") - if run_mode == "replay": - source_namespace = str(replay_cfg.get("source_namespace", "")).strip() - replay_mode = str(replay_cfg.get("replay_mode", "")).strip().lower() - if replay_mode != "restore_blessed_namespace" or not source_namespace: - raise ValueError( - "Conformal replay requires source_namespace + restore_blessed_namespace." - ) - _restore_replay_namespace(source_namespace, run_tag=run_tag) - return +def _build_tuning_row( + *, + partition_candidate: str, + partition_probability_source: str, + n_score_bins: int, + fallback_mode: str, + alpha_target_90: float, + alpha_used: float, + scaled_scores: bool, + score_scale_family: str, + min_group_size: int, + target_coverage_90: float, + y_tune: pd.Series, + y_int: np.ndarray, + group_tune: pd.Series, + issue_tune: pd.Series, + partition_meta_candidate: dict[str, Any], +) -> dict[str, Any]: + y_tune_array = y_tune.to_numpy(dtype=float) + metrics = validate_coverage(y_tune_array, y_int, alpha_target_90, log_summary=False) + group_metrics = conditional_coverage_by_group(y_tune_array, y_int, group_tune) + temporal_metrics = temporal_stability_summary( + y_tune_array, + y_int, + issue_tune, + target_coverage=target_coverage_90, + freq="M", + ) + return { + "partition": str(partition_meta_candidate.get("partition", partition_candidate)), + "partition_probability_source": partition_probability_source, + "n_score_bins": int(n_score_bins), + "fallback_mode": str(partition_meta_candidate.get("fallback_mode", fallback_mode)), + "fallback_groups_n": len(partition_meta_candidate.get("fallback_groups", [])), + "alpha_target_90": alpha_target_90, + "alpha_used_90": alpha_used, + "scaled_scores": bool(scaled_scores), + "score_scale_family": str(score_scale_family), + "min_group_size": int(min_group_size), + "empirical_coverage": float(metrics["empirical_coverage"]), + "target_coverage": float(metrics["target_coverage"]), + "coverage_gap": float(metrics["coverage_gap"]), + "avg_interval_width": float(metrics["avg_interval_width"]), + "median_interval_width": float(metrics["median_interval_width"]), + "min_group_coverage": float(group_metrics["coverage"].min()), + "max_group_coverage": float(group_metrics["coverage"].max()), + "std_group_coverage": float(group_metrics["coverage"].std(ddof=0)), + "winkler_90": float(mean_winkler_score(y_tune_array, y_int, alpha=alpha_target_90)), + "max_monthly_gap": float(temporal_metrics["max_monthly_gap"]), + "stability_over_time": float(temporal_metrics["stability_over_time"]), + } - inputs = _load_conformal_inputs( - calibration_fraction=calibration_fraction, - calibrator_override_path=calibrator_override_path, + +def _run_tuning_search( + *, + tuning_grid: ConformalTuningGrid, + alpha_candidates_90: tuple[float, ...], + min_group_sizes: tuple[int, ...], + alpha_target_90: float, + target_coverage_90: float, + artifact_namespace: str | None, + prob_fit_lookup: dict[str, np.ndarray], + prob_tune_lookup: dict[str, np.ndarray], + prob_test_lookup: dict[str, np.ndarray], + interval_fit_pred: np.ndarray, + interval_tune_pred: np.ndarray, + y_cal_fit: pd.Series, + y_tune: pd.Series, + group_cal_fit_base: pd.Series, + group_tune_base: pd.Series, + group_test_base: pd.Series, + issue_tune: pd.Series, +) -> ConformalTuningSearch: + unsupported_sources = set(tuning_grid.partition_probability_sources) - set(prob_fit_lookup) + if unsupported_sources: + raise ValueError( + f"Unsupported partition_probability_source: {', '.join(sorted(unsupported_sources))}" + ) + + total = _tuning_total_candidates( + tuning_grid, + alpha_candidates_90=alpha_candidates_90, + min_group_sizes=min_group_sizes, ) - model_path = inputs.model_path - cal_df = inputs.cal_df - test_df = inputs.test_df - X_cal = inputs.X_cal - y_cal = inputs.y_cal - y_test = inputs.y_test - group_cal_base = inputs.group_cal_base - group_test_base = inputs.group_test_base - y_prob_cal_raw = inputs.y_prob_cal_raw - y_prob_test_raw = inputs.y_prob_test_raw - y_prob_calibrated = inputs.y_prob_calibrated - y_prob_test_calibrated = inputs.y_prob_test_calibrated - tuning_split = _build_tuning_split( - cal_df=cal_df, - test_df=test_df, - X_cal=X_cal, - y_cal=y_cal, - group_cal_base=group_cal_base, - y_prob_cal_raw=y_prob_cal_raw, - tuning_holdout_ratio=tuning_holdout_ratio, - tuning_random_state=tuning_random_state, + started = time.perf_counter() + status_path = ( + _resolve_artifact_paths(artifact_namespace)["models_dir"] + / "conformal_tuning_runtime_status.json" ) - idx_cal_fit = tuning_split.idx_cal_fit - idx_cal_tune = tuning_split.idx_cal_tune - X_cal_fit = tuning_split.X_cal_fit - y_cal_fit = tuning_split.y_cal_fit - X_tune = tuning_split.X_tune - y_tune = tuning_split.y_tune - y_prob_cal_fit = tuning_split.y_prob_cal_fit - y_prob_cal_tune = tuning_split.y_prob_cal_tune - group_cal_fit_base = tuning_split.group_cal_fit_base - group_tune_base = tuning_split.group_tune_base - issue_cal = tuning_split.issue_cal - issue_tune = tuning_split.issue_tune - issue_test = tuning_split.issue_test - logger.info( - "Calibration split for conformal tuning: " - f"fit={len(X_cal_fit):,}, holdout={len(X_tune):,}, " - f"holdout_ratio={len(X_tune) / max(len(X_cal), 1):.2%}" + _write_tuning_runtime_status( + status_path, + artifact_namespace=artifact_namespace, + phase="tuning_running", + completed=0, + total=total, + started_monotonic=started, ) - if "issue_d" in cal_df.columns: - fit_issue = issue_cal.iloc[idx_cal_fit] - tune_issue = issue_tune - if fit_issue.notna().any() and tune_issue.notna().any(): - logger.info( - "Calibration split date ranges: " - f"fit_max={fit_issue.max():%Y-%m}, " - f"holdout_min={tune_issue.min():%Y-%m}, " - f"holdout_max={tune_issue.max():%Y-%m}" - ) - target_coverage_90 = 1.0 - alpha_target_90 - group_coverage_floor_target_90 = max( - float(min_group_coverage_target), - float(group_coverage_floor_target_90), + tuning_rows: list[dict[str, Any]] = [] + partition_cache: PartitionCache = {} + candidates = product( + tuning_grid.partition_candidates, + tuning_grid.partition_probability_sources, + tuning_grid.n_score_bins_candidates, + tuning_grid.fallback_modes, + alpha_candidates_90, + tuning_grid.scaled_scores_options, + tuning_grid.score_scale_families, + min_group_sizes, ) - evaluation_scope_key = str(evaluation_scope or "test").strip().lower() or "test" - if evaluation_scope_key not in {"test", "holdout"}: - raise ValueError(f"Unsupported evaluation_scope: {evaluation_scope}") - tuning_grid = _resolve_tuning_grid( - partition=partition, - partition_candidates=partition_candidates, - partition_probability_sources=partition_probability_sources, - n_score_bins_candidates=n_score_bins_candidates, - fallback_modes=fallback_modes, - score_scale_families=score_scale_families, - scaled_scores_options=scaled_scores_options, - ) - partition_candidates = tuning_grid.partition_candidates - partition_probability_sources = tuning_grid.partition_probability_sources - n_score_bins_candidates = tuning_grid.n_score_bins_candidates - fallback_modes = tuning_grid.fallback_modes - score_scale_families = tuning_grid.score_scale_families - scaled_scores_options = tuning_grid.scaled_scores_options - tuning_rows: list[dict[str, Any]] = [] - tuning_total_candidates = ( - len(partition_candidates) - * len(partition_probability_sources) - * len(n_score_bins_candidates) - * len(fallback_modes) - * len(alpha_candidates_90) - * len(scaled_scores_options) - * len(score_scale_families) - * len(min_group_sizes) - ) - tuning_completed_candidates = 0 - tuning_started = time.perf_counter() - tuning_status_path = ( - _resolve_artifact_paths(artifact_namespace)["models_dir"] - / "conformal_tuning_runtime_status.json" - ) - _write_tuning_runtime_status( - tuning_status_path, - artifact_namespace=artifact_namespace, - phase="tuning_running", - completed=0, - total=tuning_total_candidates, - started_monotonic=tuning_started, - ) - - # Tune 90% interval config across candidate Mondrian partitions. - prob_fit_lookup = {"raw": y_prob_cal_fit, "calibrated": y_prob_calibrated[idx_cal_fit]} - prob_tune_lookup = {"raw": y_prob_cal_tune, "calibrated": y_prob_calibrated[idx_cal_tune]} - prob_test_lookup = {"raw": y_prob_test_raw, "calibrated": y_prob_test_calibrated} - interval_fit_pred = y_prob_calibrated[idx_cal_fit] - interval_tune_pred = y_prob_calibrated[idx_cal_tune] - interval_test_pred = y_prob_test_calibrated - partition_cache: dict[ - tuple[str, str, int, str, int, str], - tuple[pd.Series, pd.Series, dict[str, Any]], - ] = {} - - def _cached_partition_labels( - *, - eval_scope: str, - partition_candidate: str, - partition_probability_source: str, - n_score_bins: int, - fallback_mode: str, - min_group_size: int, - ) -> tuple[pd.Series, pd.Series, dict[str, Any]]: - key = ( - str(eval_scope), - str(partition_candidate), - str(partition_probability_source), - int(n_score_bins), - str(fallback_mode), - int(min_group_size), - ) - if key in partition_cache: - return partition_cache[key] - if eval_scope == "test": - y_prob_eval = prob_test_lookup[partition_probability_source] - base_groups_eval = group_test_base - elif eval_scope == "tune": - y_prob_eval = prob_tune_lookup[partition_probability_source] - base_groups_eval = group_tune_base - else: - raise ValueError(f"Unsupported cached partition eval_scope: {eval_scope}") - payload = build_mondrian_partition_labels( - y_prob_cal=prob_fit_lookup[partition_probability_source], - y_prob_eval=y_prob_eval, - partition=partition_candidate, - base_groups_cal=group_cal_fit_base, - base_groups_eval=base_groups_eval, + for completed, candidate in enumerate(candidates, start=1): + ( + partition_candidate, + partition_probability_source, + n_score_bins, + fallback_mode, + alpha_used, + scaled_scores, + score_scale_family, + min_group_size, + ) = candidate + group_cal_fit, group_tune, partition_meta_candidate = _cached_partition_labels( + cache=partition_cache, + eval_scope="tune", + partition_candidate=partition_candidate, + partition_probability_source=partition_probability_source, n_score_bins=n_score_bins, - min_group_size=min_group_size, fallback_mode=fallback_mode, + min_group_size=min_group_size, + prob_fit_lookup=prob_fit_lookup, + prob_tune_lookup=prob_tune_lookup, + prob_test_lookup=prob_test_lookup, + group_cal_fit_base=group_cal_fit_base, + group_tune_base=group_tune_base, + group_test_base=group_test_base, + ) + _y_pred, y_int, _diag = create_pd_intervals_mondrian_from_predictions( + y_cal_pred=interval_fit_pred, + y_test_pred=interval_tune_pred, + y_cal=y_cal_fit, + group_cal=group_cal_fit, + group_test=group_tune, + alpha=alpha_used, + min_group_size=min_group_size, + scaled_scores=scaled_scores, + score_scale_family=score_scale_family, + log_summary=False, + ) + tuning_rows.append( + _build_tuning_row( + partition_candidate=partition_candidate, + partition_probability_source=partition_probability_source, + n_score_bins=n_score_bins, + fallback_mode=fallback_mode, + alpha_target_90=alpha_target_90, + alpha_used=alpha_used, + scaled_scores=scaled_scores, + score_scale_family=score_scale_family, + min_group_size=min_group_size, + target_coverage_90=target_coverage_90, + y_tune=y_tune, + y_int=y_int, + group_tune=group_tune, + issue_tune=issue_tune, + partition_meta_candidate=partition_meta_candidate, + ) ) - partition_cache[key] = payload - return payload - - for partition_candidate in partition_candidates: - for partition_probability_source in partition_probability_sources: - if partition_probability_source not in prob_fit_lookup: - raise ValueError( - f"Unsupported partition_probability_source: {partition_probability_source}" - ) - for n_score_bins in n_score_bins_candidates: - for fallback_mode in fallback_modes: - for alpha_used in alpha_candidates_90: - for scaled_scores in scaled_scores_options: - for score_scale_family in score_scale_families: - for min_group_size in min_group_sizes: - group_cal_fit, group_tune, partition_meta_candidate = ( - _cached_partition_labels( - eval_scope="tune", - partition_candidate=partition_candidate, - partition_probability_source=partition_probability_source, - n_score_bins=n_score_bins, - fallback_mode=fallback_mode, - min_group_size=min_group_size, - ) - ) - y_pred, y_int, _diag = ( - create_pd_intervals_mondrian_from_predictions( - y_cal_pred=interval_fit_pred, - y_test_pred=interval_tune_pred, - y_cal=y_cal_fit, - group_cal=group_cal_fit, - group_test=group_tune, - alpha=alpha_used, - min_group_size=min_group_size, - scaled_scores=scaled_scores, - score_scale_family=score_scale_family, - log_summary=False, - ) - ) - - metrics = validate_coverage( - y_tune.to_numpy(dtype=float), - y_int, - alpha_target_90, - log_summary=False, - ) - g_metrics = conditional_coverage_by_group( - y_tune.to_numpy(dtype=float), y_int, group_tune - ) - temporal_metrics = temporal_stability_summary( - y_tune.to_numpy(dtype=float), - y_int, - issue_tune, - target_coverage=target_coverage_90, - freq="M", - ) - - tuning_rows.append( - { - "partition": str( - partition_meta_candidate.get( - "partition", partition_candidate - ) - ), - "partition_probability_source": partition_probability_source, - "n_score_bins": int(n_score_bins), - "fallback_mode": str( - partition_meta_candidate.get( - "fallback_mode", fallback_mode - ) - ), - "fallback_groups_n": len( - partition_meta_candidate.get("fallback_groups", []) - ), - "alpha_target_90": alpha_target_90, - "alpha_used_90": alpha_used, - "scaled_scores": bool(scaled_scores), - "score_scale_family": str(score_scale_family), - "min_group_size": int(min_group_size), - "empirical_coverage": float( - metrics["empirical_coverage"] - ), - "target_coverage": float(metrics["target_coverage"]), - "coverage_gap": float(metrics["coverage_gap"]), - "avg_interval_width": float( - metrics["avg_interval_width"] - ), - "median_interval_width": float( - metrics["median_interval_width"] - ), - "min_group_coverage": float( - g_metrics["coverage"].min() - ), - "max_group_coverage": float( - g_metrics["coverage"].max() - ), - "std_group_coverage": float( - g_metrics["coverage"].std(ddof=0) - ), - "winkler_90": float( - mean_winkler_score( - y_tune.to_numpy(dtype=float), - y_int, - alpha=alpha_target_90, - ) - ), - "max_monthly_gap": float( - temporal_metrics["max_monthly_gap"] - ), - "stability_over_time": float( - temporal_metrics["stability_over_time"] - ), - } - ) - tuning_completed_candidates += 1 - if ( - tuning_completed_candidates % 1000 == 0 - or tuning_completed_candidates == tuning_total_candidates - ): - _write_tuning_runtime_status( - tuning_status_path, - artifact_namespace=artifact_namespace, - phase="tuning_running", - completed=tuning_completed_candidates, - total=tuning_total_candidates, - started_monotonic=tuning_started, - extra={ - "latest_partition": str(partition_candidate), - "latest_partition_probability_source": str( - partition_probability_source - ), - "latest_n_score_bins": int(n_score_bins), - "latest_fallback_mode": str(fallback_mode), - "latest_alpha_used_90": float(alpha_used), - "latest_score_scale_family": str( - score_scale_family - ), - "latest_min_group_size": int(min_group_size), - }, - ) + if completed % 1000 == 0 or completed == total: + _write_tuning_runtime_status( + status_path, + artifact_namespace=artifact_namespace, + phase="tuning_running", + completed=completed, + total=total, + started_monotonic=started, + extra={ + "latest_partition": str(partition_candidate), + "latest_partition_probability_source": str(partition_probability_source), + "latest_n_score_bins": int(n_score_bins), + "latest_fallback_mode": str(fallback_mode), + "latest_alpha_used_90": float(alpha_used), + "latest_score_scale_family": str(score_scale_family), + "latest_min_group_size": int(min_group_size), + }, + ) _write_tuning_runtime_status( - tuning_status_path, + status_path, artifact_namespace=artifact_namespace, phase="tuning_complete", - completed=tuning_completed_candidates, - total=tuning_total_candidates, - started_monotonic=tuning_started, + completed=len(tuning_rows), + total=total, + started_monotonic=started, ) + return ConformalTuningSearch(tuning_rows=tuning_rows, partition_cache=partition_cache) - tuning_selection = _select_best_tuning_config( - tuning_rows, - partition_candidates=partition_candidates, - alpha_target_90=alpha_target_90, - min_group_coverage_target=min_group_coverage_target, - group_coverage_floor_target_90=group_coverage_floor_target_90, - coverage_guardband_90=coverage_guardband_90, - min_group_guardband_90=min_group_guardband_90, - max_width_budget_90=max_width_budget_90, - target_coverage_90=target_coverage_90, + +def _apply_global_rebalance( + *, + enabled: bool, + min_factor: float, + max_factor: float, + step: float, + y_int_tune_working: np.ndarray, + y_pred_tune: np.ndarray, + y_tune: pd.Series, + y_int_90: np.ndarray, + y_pred_90: np.ndarray, + y_eval_90: pd.Series, + group_tune: pd.Series, + eval_groups_90: pd.Series, + alpha_target_90: float, + target_coverage_90: float, + min_group_coverage_target: float, + metrics_90: dict[str, Any], + group_metrics_90: pd.DataFrame, +) -> GlobalRebalanceResult: + factor = 1.0 + diagnostics: dict[str, float | bool] = { + "enabled": bool(enabled), + "applied": False, + } + if not enabled or len(y_int_tune_working) == 0: + return GlobalRebalanceResult( + y_intervals=y_int_90, + metrics=metrics_90, + group_metrics=group_metrics_90, + factor=factor, + diagnostics=diagnostics, + ) + + min_factor = max(0.05, float(min_factor)) + max_factor = max(min_factor, float(max_factor)) + step = max(0.001, float(step)) + n_steps = int(round((max_factor - min_factor) / step)) + 1 + candidate_factors = np.linspace(min_factor, max_factor, max(2, n_steps)) + + best_trial: dict[str, float] | None = None + tune_y_true = y_tune.to_numpy(dtype=float) + for candidate_factor in candidate_factors: + tune_trial = _scale_intervals_around_prediction( + y_pred_tune, + y_int_tune_working, + float(candidate_factor), + ) + coverage = empirical_interval_coverage(tune_y_true, tune_trial) + min_group_coverage = min_group_interval_coverage(tune_y_true, tune_trial, group_tune) + floor_shortfall = max(0.0, float(min_group_coverage_target) - min_group_coverage) + score = abs(coverage - target_coverage_90) + 100.0 * floor_shortfall + trial = { + "factor": float(candidate_factor), + "coverage": float(coverage), + "min_group_coverage": float(min_group_coverage), + "score": float(score), + } + if best_trial is None or trial["score"] < best_trial["score"]: + best_trial = trial + + if best_trial is None: + return GlobalRebalanceResult( + y_intervals=y_int_90, + metrics=metrics_90, + group_metrics=group_metrics_90, + factor=factor, + diagnostics=diagnostics, + ) + + factor = float(best_trial["factor"]) + diagnostics.update( + { + "factor": factor, + "tune_coverage_after_rebalance": float(best_trial["coverage"]), + "tune_min_group_coverage_after_rebalance": float(best_trial["min_group_coverage"]), + "target_coverage_90": float(target_coverage_90), + "min_group_floor_target": float(min_group_coverage_target), + "applied": abs(factor - 1.0) > 1e-9, + } ) - tuning_df = tuning_selection.tuning_df - best_row = tuning_selection.best_row - selection_tier = tuning_selection.selection_tier - best_cfg = tuning_selection.best_cfg + if abs(factor - 1.0) <= 1e-9: + return GlobalRebalanceResult( + y_intervals=y_int_90, + metrics=metrics_90, + group_metrics=group_metrics_90, + factor=factor, + diagnostics=diagnostics, + ) + logger.info( - "Best 90% tuning config: " - f"partition={best_cfg['partition']}, " - f"prob_source={best_cfg['partition_probability_source']}, " - f"n_bins={best_cfg['n_score_bins']}, " - f"alpha_used={best_cfg['alpha_used_90']}, scaled_scores={best_cfg['scaled_scores']}, " - f"score_scale_family={best_cfg['score_scale_family']}, " - f"min_group_size={best_cfg['min_group_size']}, " - f"coverage={best_row['empirical_coverage']:.4f}, " - f"min_group_coverage={best_row['min_group_coverage']:.4f}, " - f"width={best_row['avg_interval_width']:.4f}, " - f"tier={selection_tier}" + "Applying global interval rebalance factor learned on holdout: " + f"factor={factor:.4f}, " + f"tune_cov={best_trial['coverage']:.4f}, " + f"tune_min_group_cov={best_trial['min_group_coverage']:.4f}" + ) + y_intervals = _scale_intervals_around_prediction(y_pred_90, y_int_90, factor) + metrics = validate_coverage(y_eval_90.to_numpy(dtype=float), y_intervals, alpha_target_90) + group_metrics = conditional_coverage_by_group( + y_eval_90.to_numpy(dtype=float), + y_intervals, + eval_groups_90, + ) + return GlobalRebalanceResult( + y_intervals=y_intervals, + metrics=metrics, + group_metrics=group_metrics, + factor=factor, + diagnostics=diagnostics, ) - group_cal_fit, group_test, partition_meta = _cached_partition_labels( - eval_scope="test", - partition_candidate=best_cfg["partition"], - partition_probability_source=best_cfg["partition_probability_source"], - n_score_bins=best_cfg["n_score_bins"], - fallback_mode=best_cfg["fallback_mode"], - min_group_size=best_cfg["min_group_size"], + +def _select_alpha_95( + *, + alpha_95: float, + alpha_candidates_95: tuple[float, ...], + interval_fit_pred: np.ndarray, + interval_tune_pred: np.ndarray, + y_cal_fit: pd.Series, + y_tune: pd.Series, + group_cal_fit_holdout: pd.Series, + group_tune: pd.Series, + best_cfg: dict[str, Any], +) -> float: + best_alpha = float(alpha_95) + if not alpha_candidates_95: + return best_alpha + best_score: tuple[float, float] | None = None + for alpha_candidate in alpha_candidates_95: + _y_pred, y_int, _diag = create_pd_intervals_mondrian_from_predictions( + y_cal_pred=interval_fit_pred, + y_test_pred=interval_tune_pred, + y_cal=y_cal_fit, + group_cal=group_cal_fit_holdout, + group_test=group_tune, + alpha=float(alpha_candidate), + min_group_size=best_cfg["min_group_size"], + scaled_scores=best_cfg["scaled_scores"], + score_scale_family=best_cfg["score_scale_family"], + log_summary=False, + ) + metrics = validate_coverage( + y_tune.to_numpy(dtype=float), + y_int, + alpha=float(alpha_candidate), + log_summary=False, + ) + score = ( + abs(float(metrics["coverage_gap"])), + float(metrics["avg_interval_width"]), + ) + if best_score is None or score < best_score: + best_score = score + best_alpha = float(alpha_candidate) + return best_alpha + + +def _coverage_metrics( + *, + y_true: pd.Series, + y_intervals: np.ndarray, + groups: pd.Series, + alpha: float, +) -> tuple[dict[str, Any], pd.DataFrame]: + metrics = validate_coverage(y_true.to_numpy(dtype=float), y_intervals, alpha) + group_metrics = conditional_coverage_by_group( + y_true.to_numpy(dtype=float), + y_intervals, + groups, ) - group_cal_fit_holdout, group_tune, _ = _cached_partition_labels( - eval_scope="tune", - partition_candidate=best_cfg["partition"], - partition_probability_source=best_cfg["partition_probability_source"], - n_score_bins=best_cfg["n_score_bins"], - fallback_mode=best_cfg["fallback_mode"], - min_group_size=best_cfg["min_group_size"], + return metrics, group_metrics + + +def _append_width_attr_row( + rows: list[dict[str, Any]], + *, + dataset_scope: str, + stage: str, + y_true: pd.Series, + y_pred: np.ndarray, + y_intervals: np.ndarray, + groups: pd.Series, + issue_dates: pd.Series, + alpha: float, + target_coverage: float, +) -> None: + rows.append( + _stage_metrics( + dataset_scope=dataset_scope, + stage=stage, + y_true=y_true.to_numpy(dtype=float), + y_pred=y_pred, + y_intervals=y_intervals, + groups=groups, + issue_dates=issue_dates, + alpha=alpha, + target_coverage=target_coverage, + ) + ) + + +def _can_use_temporal_segments( + *, + enabled: bool, + issue_tune: pd.Series, + eval_issue: pd.Series, + group_tune: pd.Series, + eval_groups: pd.Series, +) -> bool: + return ( + enabled + and issue_tune.notna().any() + and eval_issue.notna().any() + and len(issue_tune) == len(group_tune) + and len(eval_issue) == len(eval_groups) ) - # Final 90% intervals with tuned config. - y_pred_90, y_int_90, diag_90 = create_pd_intervals_mondrian_from_predictions( + +def _apply_learned_floor_policy( + *, + y_pred: np.ndarray, + y_intervals: np.ndarray, + groups: pd.Series, + group_multipliers: dict[str, float], + temporal_segments: pd.Series | None, + temporal_segment_multipliers: dict[str, float], + global_rebalance_factor: float = 1.0, +) -> np.ndarray: + adjusted = np.asarray(y_intervals, dtype=float).copy() + if group_multipliers: + adjusted = apply_group_multipliers(y_pred, adjusted, groups, group_multipliers) + if temporal_segment_multipliers and temporal_segments is not None: + adjusted = apply_group_multipliers( + y_pred, + adjusted, + temporal_segments, + temporal_segment_multipliers, + ) + if abs(global_rebalance_factor - 1.0) > 1e-9: + adjusted = _scale_intervals_around_prediction(y_pred, adjusted, global_rebalance_factor) + return adjusted + + +def _build_90_interval_evidence( + *, + evaluation_scope_key: str, + y_test: pd.Series, + y_tune: pd.Series, + issue_test: pd.Series, + issue_tune: pd.Series, + interval_fit_pred: np.ndarray, + interval_test_pred: np.ndarray, + interval_tune_pred: np.ndarray, + y_cal_fit: pd.Series, + group_cal_fit: pd.Series, + group_test: pd.Series, + group_cal_fit_holdout: pd.Series, + group_tune: pd.Series, + best_cfg: dict[str, Any], + alpha_target_90: float, + target_coverage_90: float, + group_coverage_floor_enabled: bool, + group_coverage_floor_target_90: float, + group_multiplier_grid: tuple[float, ...], + temporal_segment_floor_enabled: bool, + temporal_segment_freq: str, + temporal_segment_min_size: int, + temporal_multiplier_grid: tuple[float, ...], + shrinkback_enabled: bool, + min_group_coverage_target: float, + global_rebalance_enabled: bool, + global_rebalance_min_factor: float, + global_rebalance_max_factor: float, + global_rebalance_step: float, +) -> ConformalEvidence90: + eval_scope = "test" if evaluation_scope_key == "test" else "holdout" + y_eval = y_test if evaluation_scope_key == "test" else y_tune + eval_groups = group_test if evaluation_scope_key == "test" else group_tune + eval_issue = issue_test if evaluation_scope_key == "test" else issue_tune + + y_pred, y_intervals, diag = create_pd_intervals_mondrian_from_predictions( y_cal_pred=interval_fit_pred, y_test_pred=interval_test_pred if evaluation_scope_key == "test" else interval_tune_pred, y_cal=y_cal_fit, group_cal=group_cal_fit, - group_test=group_test if evaluation_scope_key == "test" else group_tune, + group_test=eval_groups, alpha=best_cfg["alpha_used_90"], min_group_size=best_cfg["min_group_size"], scaled_scores=best_cfg["scaled_scores"], score_scale_family=best_cfg["score_scale_family"], ) - y_eval_90 = y_test if evaluation_scope_key == "test" else y_tune - eval_groups_90 = group_test if evaluation_scope_key == "test" else group_tune - eval_issue_90 = issue_test if evaluation_scope_key == "test" else issue_tune - metrics_90 = validate_coverage(y_eval_90.to_numpy(dtype=float), y_int_90, alpha_target_90) - group_metrics_90 = conditional_coverage_by_group( - y_eval_90.to_numpy(dtype=float), y_int_90, eval_groups_90 + metrics, group_metrics = _coverage_metrics( + y_true=y_eval, + y_intervals=y_intervals, + groups=eval_groups, + alpha=alpha_target_90, ) - width_attr_rows: list[dict[str, Any]] = [ - _stage_metrics( - dataset_scope="test" if evaluation_scope_key == "test" else "holdout", - stage="base_interval", - y_true=y_eval_90.to_numpy(dtype=float), - y_pred=y_pred_90, - y_intervals=y_int_90, - groups=eval_groups_90, - issue_dates=eval_issue_90, - alpha=alpha_target_90, - target_coverage=target_coverage_90, - ) - ] - # Learn group multipliers on calibration holdout only (no test-label adaptation). + width_attr_rows: list[dict[str, Any]] = [] + _append_width_attr_row( + width_attr_rows, + dataset_scope=eval_scope, + stage="base_interval", + y_true=y_eval, + y_pred=y_pred, + y_intervals=y_intervals, + groups=eval_groups, + issue_dates=eval_issue, + alpha=alpha_target_90, + target_coverage=target_coverage_90, + ) + y_pred_tune, y_int_tune, _diag_tune = create_pd_intervals_mondrian_from_predictions( y_cal_pred=interval_fit_pred, y_test_pred=interval_tune_pred, @@ -1203,108 +1516,101 @@ def _cached_partition_labels( score_scale_family=best_cfg["score_scale_family"], log_summary=False, ) - tune_metrics_90_before = validate_coverage( + tune_metrics_before_floor = validate_coverage( y_tune.to_numpy(dtype=float), y_int_tune, alpha_target_90 ) - width_attr_rows.append( - _stage_metrics( - dataset_scope="tune_holdout", - stage="base_interval", - y_true=y_tune.to_numpy(dtype=float), - y_pred=y_pred_tune, - y_intervals=y_int_tune, - groups=group_tune, - issue_dates=issue_tune, - alpha=alpha_target_90, - target_coverage=target_coverage_90, - ) + _append_width_attr_row( + width_attr_rows, + dataset_scope="tune_holdout", + stage="base_interval", + y_true=y_tune, + y_pred=y_pred_tune, + y_intervals=y_int_tune, + groups=group_tune, + issue_dates=issue_tune, + alpha=alpha_target_90, + target_coverage=target_coverage_90, ) + if group_coverage_floor_enabled: - y_int_90_adjusted, group_multipliers, coverage_floor_report = enforce_group_coverage_floor( - y_true=y_tune.to_numpy(dtype=float), - y_pred=y_pred_tune, - y_intervals=y_int_tune, - groups=group_tune, - target_coverage=group_coverage_floor_target_90, - multiplier_grid=group_multiplier_grid, + y_int_tune_after_group, group_multipliers, coverage_floor_report = ( + enforce_group_coverage_floor( + y_true=y_tune.to_numpy(dtype=float), + y_pred=y_pred_tune, + y_intervals=y_int_tune, + groups=group_tune, + target_coverage=group_coverage_floor_target_90, + multiplier_grid=group_multiplier_grid, + ) ) else: - y_int_90_adjusted = np.asarray(y_int_tune, dtype=float).copy() + y_int_tune_after_group = np.asarray(y_int_tune, dtype=float).copy() group_multipliers = {} - coverage_floor_report = pd.DataFrame( - columns=[ - "group", - "coverage_before", - "coverage_after", - "target_coverage", - "multiplier", - "adjusted", - ] - ) - tune_metrics_90_after = validate_coverage( - y_tune.to_numpy(dtype=float), y_int_90_adjusted, alpha_target_90 + coverage_floor_report = _empty_coverage_floor_report() + tune_metrics_after_floor = validate_coverage( + y_tune.to_numpy(dtype=float), y_int_tune_after_group, alpha_target_90 ) - width_attr_rows.append( - _stage_metrics( - dataset_scope="tune_holdout", - stage="after_group_floor", - y_true=y_tune.to_numpy(dtype=float), - y_pred=y_pred_tune, - y_intervals=y_int_90_adjusted, - groups=group_tune, - issue_dates=issue_tune, - alpha=alpha_target_90, - target_coverage=target_coverage_90, - ) + _append_width_attr_row( + width_attr_rows, + dataset_scope="tune_holdout", + stage="after_group_floor", + y_true=y_tune, + y_pred=y_pred_tune, + y_intervals=y_int_tune_after_group, + groups=group_tune, + issue_dates=issue_tune, + alpha=alpha_target_90, + target_coverage=target_coverage_90, ) + eval_temporal_segments: pd.Series | None = None + tune_temporal_segments: pd.Series | None = None temporal_segment_multipliers: dict[str, float] = {} - temporal_segment_report = pd.DataFrame( - columns=[ - "segment", - "support_n", - "coverage_before", - "coverage_after", - "target_coverage", - "min_segment_size", - "multiplier", - "adjusted", - ] - ) - y_int_90_tune_working = y_int_90_adjusted - tune_metrics_90_after_temporal = tune_metrics_90_after.copy() - y_int_90_base_test = np.asarray(y_int_90, dtype=float).copy() + temporal_segment_report = _empty_temporal_segment_report() + y_int_tune_working = y_int_tune_after_group + tune_metrics_after_temporal = tune_metrics_after_floor.copy() + y_intervals_base_eval = np.asarray(y_intervals, dtype=float).copy() + if group_multipliers: logger.info( "Applying group coverage floor multipliers learned on calibration holdout: " f"{group_multipliers}" ) - y_int_90 = apply_group_multipliers(y_pred_90, y_int_90, eval_groups_90, group_multipliers) - metrics_90 = validate_coverage(y_eval_90.to_numpy(dtype=float), y_int_90, alpha_target_90) - group_metrics_90 = conditional_coverage_by_group( - y_eval_90.to_numpy(dtype=float), y_int_90, eval_groups_90 + y_intervals = _apply_learned_floor_policy( + y_pred=y_pred, + y_intervals=y_intervals, + groups=eval_groups, + group_multipliers=group_multipliers, + temporal_segments=None, + temporal_segment_multipliers={}, ) - else: - logger.info("No group coverage floor adjustments were required.") - width_attr_rows.append( - _stage_metrics( - dataset_scope="test" if evaluation_scope_key == "test" else "holdout", - stage="after_group_floor", - y_true=y_eval_90.to_numpy(dtype=float), - y_pred=y_pred_90, - y_intervals=y_int_90, - groups=eval_groups_90, - issue_dates=eval_issue_90, + metrics, group_metrics = _coverage_metrics( + y_true=y_eval, + y_intervals=y_intervals, + groups=eval_groups, alpha=alpha_target_90, - target_coverage=target_coverage_90, ) + else: + logger.info("No group coverage floor adjustments were required.") + _append_width_attr_row( + width_attr_rows, + dataset_scope=eval_scope, + stage="after_group_floor", + y_true=y_eval, + y_pred=y_pred, + y_intervals=y_intervals, + groups=eval_groups, + issue_dates=eval_issue, + alpha=alpha_target_90, + target_coverage=target_coverage_90, ) - if ( - temporal_segment_floor_enabled - and issue_tune.notna().any() - and eval_issue_90.notna().any() - and len(issue_tune) == len(group_tune) - and len(eval_issue_90) == len(eval_groups_90) + + if _can_use_temporal_segments( + enabled=temporal_segment_floor_enabled, + issue_tune=issue_tune, + eval_issue=eval_issue, + group_tune=group_tune, + eval_groups=eval_groups, ): tune_temporal_segments = build_group_temporal_segments( groups=group_tune, @@ -1312,98 +1618,84 @@ def _cached_partition_labels( freq=temporal_segment_freq, ) eval_temporal_segments = build_group_temporal_segments( - groups=eval_groups_90, - issue_dates=eval_issue_90, + groups=eval_groups, + issue_dates=eval_issue, freq=temporal_segment_freq, ) - y_int_90_tune_temporal, temporal_segment_multipliers, temporal_segment_report = ( + y_int_tune_temporal, temporal_segment_multipliers, temporal_segment_report = ( enforce_segment_coverage_floor( y_true=y_tune.to_numpy(dtype=float), y_pred=y_pred_tune, - y_intervals=y_int_90_tune_working, + y_intervals=y_int_tune_working, segments=tune_temporal_segments, target_coverage=group_coverage_floor_target_90, min_segment_size=temporal_segment_min_size, multiplier_grid=temporal_multiplier_grid, ) ) - y_int_90_tune_working = y_int_90_tune_temporal - tune_metrics_90_after_temporal = validate_coverage( - y_tune.to_numpy(dtype=float), y_int_90_tune_temporal, alpha_target_90 + y_int_tune_working = y_int_tune_temporal + tune_metrics_after_temporal = validate_coverage( + y_tune.to_numpy(dtype=float), y_int_tune_temporal, alpha_target_90 ) - width_attr_rows.append( - _stage_metrics( - dataset_scope="tune_holdout", - stage="after_temporal_floor", - y_true=y_tune.to_numpy(dtype=float), - y_pred=y_pred_tune, - y_intervals=y_int_90_tune_temporal, - groups=group_tune, - issue_dates=issue_tune, - alpha=alpha_target_90, - target_coverage=target_coverage_90, - ) + _append_width_attr_row( + width_attr_rows, + dataset_scope="tune_holdout", + stage="after_temporal_floor", + y_true=y_tune, + y_pred=y_pred_tune, + y_intervals=y_int_tune_temporal, + groups=group_tune, + issue_dates=issue_tune, + alpha=alpha_target_90, + target_coverage=target_coverage_90, ) if temporal_segment_multipliers: logger.info( "Applying temporal coverage floor multipliers learned on holdout " f"(freq={temporal_segment_freq}): {temporal_segment_multipliers}" ) - y_int_90 = apply_group_multipliers( - y_pred_90, - y_int_90, - eval_temporal_segments, - temporal_segment_multipliers, - ) - metrics_90 = validate_coverage( - y_eval_90.to_numpy(dtype=float), y_int_90, alpha_target_90 + y_intervals = _apply_learned_floor_policy( + y_pred=y_pred, + y_intervals=y_intervals, + groups=eval_groups, + group_multipliers={}, + temporal_segments=eval_temporal_segments, + temporal_segment_multipliers=temporal_segment_multipliers, ) - group_metrics_90 = conditional_coverage_by_group( - y_eval_90.to_numpy(dtype=float), y_int_90, eval_groups_90 + metrics, group_metrics = _coverage_metrics( + y_true=y_eval, + y_intervals=y_intervals, + groups=eval_groups, + alpha=alpha_target_90, ) else: logger.info("No temporal segment coverage adjustments were required.") elif temporal_segment_floor_enabled: logger.info("Temporal segment coverage adjustments skipped (missing issue_d coverage).") - width_attr_rows.append( - _stage_metrics( - dataset_scope="test" if evaluation_scope_key == "test" else "holdout", - stage="after_temporal_floor", - y_true=y_eval_90.to_numpy(dtype=float), - y_pred=y_pred_90, - y_intervals=y_int_90, - groups=eval_groups_90, - issue_dates=eval_issue_90, - alpha=alpha_target_90, - target_coverage=target_coverage_90, - ) + _append_width_attr_row( + width_attr_rows, + dataset_scope=eval_scope, + stage="after_temporal_floor", + y_true=y_eval, + y_pred=y_pred, + y_intervals=y_intervals, + groups=eval_groups, + issue_dates=eval_issue, + alpha=alpha_target_90, + target_coverage=target_coverage_90, ) - shrinkback_report = pd.DataFrame( - columns=[ - "stage", - "factor_scope", - "factor_key", - "candidate_factor", - "accepted", - "coverage", - "min_group_coverage", - "avg_width", - "winkler_90", - "max_monthly_gap", - "stability_over_time", - ] - ) + shrinkback_report = _empty_shrinkback_report() if shrinkback_enabled and (group_multipliers or temporal_segment_multipliers): shrink_max_monthly_gap = temporal_stability_summary( y_tune.to_numpy(dtype=float), - y_int_90_tune_working, + y_int_tune_working, issue_tune, target_coverage=target_coverage_90, freq="M", )["max_monthly_gap"] ( - y_int_90_tune_working, + y_int_tune_working, group_multipliers, temporal_segment_multipliers, shrinkback_report, @@ -1414,7 +1706,7 @@ def _cached_partition_labels( groups=group_tune, issue_dates=issue_tune, group_factors=group_multipliers, - temporal_segments=tune_temporal_segments if temporal_segment_floor_enabled else None, + temporal_segments=tune_temporal_segments, temporal_factors=temporal_segment_multipliers, target_coverage=target_coverage_90, min_group_coverage_target=min_group_coverage_target, @@ -1425,150 +1717,105 @@ def _cached_partition_labels( group_multiplier_grid=group_multiplier_grid, temporal_multiplier_grid=temporal_multiplier_grid, ) - y_int_90 = np.asarray(y_int_90_base_test, dtype=float).copy() - if group_multipliers: - y_int_90 = apply_group_multipliers( - y_pred_90, y_int_90, eval_groups_90, group_multipliers - ) - if temporal_segment_multipliers and eval_temporal_segments is not None: - y_int_90 = apply_group_multipliers( - y_pred_90, - y_int_90, - eval_temporal_segments, - temporal_segment_multipliers, - ) - metrics_90 = validate_coverage(y_eval_90.to_numpy(dtype=float), y_int_90, alpha_target_90) - group_metrics_90 = conditional_coverage_by_group( - y_eval_90.to_numpy(dtype=float), y_int_90, eval_groups_90 + y_intervals = _apply_learned_floor_policy( + y_pred=y_pred, + y_intervals=y_intervals_base_eval, + groups=eval_groups, + group_multipliers=group_multipliers, + temporal_segments=eval_temporal_segments, + temporal_segment_multipliers=temporal_segment_multipliers, ) - width_attr_rows.append( - _stage_metrics( - dataset_scope="tune_holdout", - stage="after_shrinkback", - y_true=y_tune.to_numpy(dtype=float), - y_pred=y_pred_tune, - y_intervals=y_int_90_tune_working, - groups=group_tune, - issue_dates=issue_tune, + metrics, group_metrics = _coverage_metrics( + y_true=y_eval, + y_intervals=y_intervals, + groups=eval_groups, alpha=alpha_target_90, - target_coverage=target_coverage_90, ) + _append_width_attr_row( + width_attr_rows, + dataset_scope="tune_holdout", + stage="after_shrinkback", + y_true=y_tune, + y_pred=y_pred_tune, + y_intervals=y_int_tune_working, + groups=group_tune, + issue_dates=issue_tune, + alpha=alpha_target_90, + target_coverage=target_coverage_90, ) - width_attr_rows.append( - _stage_metrics( - dataset_scope="test" if evaluation_scope_key == "test" else "holdout", - stage="after_shrinkback", - y_true=y_eval_90.to_numpy(dtype=float), - y_pred=y_pred_90, - y_intervals=y_int_90, - groups=eval_groups_90, - issue_dates=eval_issue_90, - alpha=alpha_target_90, - target_coverage=target_coverage_90, - ) + _append_width_attr_row( + width_attr_rows, + dataset_scope=eval_scope, + stage="after_shrinkback", + y_true=y_eval, + y_pred=y_pred, + y_intervals=y_intervals, + groups=eval_groups, + issue_dates=eval_issue, + alpha=alpha_target_90, + target_coverage=target_coverage_90, + ) + + rebalance_result = _apply_global_rebalance( + enabled=global_rebalance_enabled, + min_factor=global_rebalance_min_factor, + max_factor=global_rebalance_max_factor, + step=global_rebalance_step, + y_int_tune_working=y_int_tune_working, + y_pred_tune=y_pred_tune, + y_tune=y_tune, + y_int_90=y_intervals, + y_pred_90=y_pred, + y_eval_90=y_eval, + group_tune=group_tune, + eval_groups_90=eval_groups, + alpha_target_90=alpha_target_90, + target_coverage_90=target_coverage_90, + min_group_coverage_target=min_group_coverage_target, + metrics_90=metrics, + group_metrics_90=group_metrics, + ) + return ConformalEvidence90( + y_eval=y_eval, + eval_groups=eval_groups, + eval_issue=eval_issue, + y_pred=y_pred, + y_intervals=rebalance_result.y_intervals, + diag=diag, + metrics=rebalance_result.metrics, + group_metrics=rebalance_result.group_metrics, + y_pred_tune=y_pred_tune, + y_intervals_tune_base=y_int_tune, + tune_metrics_before_floor=tune_metrics_before_floor, + tune_metrics_after_floor=tune_metrics_after_floor, + tune_metrics_after_temporal_floor=tune_metrics_after_temporal, + group_multipliers=group_multipliers, + coverage_floor_report=coverage_floor_report, + temporal_segment_multipliers=temporal_segment_multipliers, + temporal_segment_report=temporal_segment_report, + shrinkback_report=shrinkback_report, + eval_temporal_segments=eval_temporal_segments, + width_attr_rows=width_attr_rows, + global_rebalance_factor=rebalance_result.factor, + global_rebalance_diagnostics=rebalance_result.diagnostics, ) - # Optional global rebalance: tune one uniform radius factor on calibration holdout - # to get closer to nominal global coverage while preserving minimum group floor. - global_rebalance_factor = 1.0 - global_rebalance_diagnostics: dict[str, float | bool] = { - "enabled": bool(global_rebalance_enabled), - "applied": False, - } - if global_rebalance_enabled and len(y_int_90_tune_working) > 0: - min_factor = max(0.05, float(global_rebalance_min_factor)) - max_factor = max(min_factor, float(global_rebalance_max_factor)) - step = max(0.001, float(global_rebalance_step)) - n_steps = int(round((max_factor - min_factor) / step)) + 1 - candidate_factors = np.linspace(min_factor, max_factor, max(2, n_steps)) - - tune_y_true = y_tune.to_numpy(dtype=float) - tune_target_cov = target_coverage_90 - tune_group_floor = float(min_group_coverage_target) - - best_trial: dict[str, float] | None = None - for factor in candidate_factors: - tune_trial = _scale_intervals_around_prediction( - y_pred_tune, y_int_90_tune_working, factor - ) - cov_trial = empirical_interval_coverage(tune_y_true, tune_trial) - min_group_cov_trial = min_group_interval_coverage(tune_y_true, tune_trial, group_tune) - floor_shortfall = max(0.0, tune_group_floor - min_group_cov_trial) - score = abs(cov_trial - tune_target_cov) + 100.0 * floor_shortfall - trial = { - "factor": float(factor), - "coverage": float(cov_trial), - "min_group_coverage": float(min_group_cov_trial), - "score": float(score), - } - if best_trial is None or trial["score"] < best_trial["score"]: - best_trial = trial - - if best_trial is not None: - global_rebalance_factor = float(best_trial["factor"]) - global_rebalance_diagnostics.update( - { - "factor": global_rebalance_factor, - "tune_coverage_after_rebalance": float(best_trial["coverage"]), - "tune_min_group_coverage_after_rebalance": float( - best_trial["min_group_coverage"] - ), - "target_coverage_90": float(tune_target_cov), - "min_group_floor_target": float(tune_group_floor), - "applied": abs(global_rebalance_factor - 1.0) > 1e-9, - } - ) - if abs(global_rebalance_factor - 1.0) > 1e-9: - logger.info( - "Applying global interval rebalance factor learned on holdout: " - f"factor={global_rebalance_factor:.4f}, " - f"tune_cov={best_trial['coverage']:.4f}, " - f"tune_min_group_cov={best_trial['min_group_coverage']:.4f}" - ) - y_int_90 = _scale_intervals_around_prediction( - y_pred_90, y_int_90, global_rebalance_factor - ) - metrics_90 = validate_coverage( - y_eval_90.to_numpy(dtype=float), y_int_90, alpha_target_90 - ) - group_metrics_90 = conditional_coverage_by_group( - y_eval_90.to_numpy(dtype=float), y_int_90, eval_groups_90 - ) - - best_alpha_95 = float(alpha_95) - if alpha_candidates_95: - best_score_95: tuple[float, float] | None = None - for alpha_candidate_95 in alpha_candidates_95: - _y_pred_95_tune, y_int_95_tune, _diag_95_tune = ( - create_pd_intervals_mondrian_from_predictions( - y_cal_pred=interval_fit_pred, - y_test_pred=interval_tune_pred, - y_cal=y_cal_fit, - group_cal=group_cal_fit_holdout, - group_test=group_tune, - alpha=float(alpha_candidate_95), - min_group_size=best_cfg["min_group_size"], - scaled_scores=best_cfg["scaled_scores"], - score_scale_family=best_cfg["score_scale_family"], - log_summary=False, - ) - ) - metrics_95_tune = validate_coverage( - y_tune.to_numpy(dtype=float), - y_int_95_tune, - alpha=float(alpha_candidate_95), - log_summary=False, - ) - score_95 = ( - abs(float(metrics_95_tune["coverage_gap"])), - float(metrics_95_tune["avg_interval_width"]), - ) - if best_score_95 is None or score_95 < best_score_95: - best_score_95 = score_95 - best_alpha_95 = float(alpha_candidate_95) - # 95% intervals using same structure settings for consistency. - y_pred_95, y_int_95, diag_95 = create_pd_intervals_mondrian_from_predictions( +def _build_95_interval_evidence( + *, + evaluation_scope_key: str, + interval_fit_pred: np.ndarray, + interval_test_pred: np.ndarray, + interval_tune_pred: np.ndarray, + y_cal_fit: pd.Series, + group_cal_fit: pd.Series, + group_test: pd.Series, + group_tune: pd.Series, + evidence_90: ConformalEvidence90, + best_cfg: dict[str, Any], + best_alpha_95: float, +) -> ConformalEvidence95: + y_pred, y_intervals, diag = create_pd_intervals_mondrian_from_predictions( y_cal_pred=interval_fit_pred, y_test_pred=interval_test_pred if evaluation_scope_key == "test" else interval_tune_pred, y_cal=y_cal_fit, @@ -1579,50 +1826,107 @@ def _cached_partition_labels( scaled_scores=best_cfg["scaled_scores"], score_scale_family=best_cfg["score_scale_family"], ) - if group_multipliers: - y_int_95 = apply_group_multipliers(y_pred_95, y_int_95, eval_groups_90, group_multipliers) - if temporal_segment_multipliers and eval_temporal_segments is not None: - y_int_95 = apply_group_multipliers( - y_pred_95, y_int_95, eval_temporal_segments, temporal_segment_multipliers - ) - if abs(global_rebalance_factor - 1.0) > 1e-9: - y_int_95 = _scale_intervals_around_prediction(y_pred_95, y_int_95, global_rebalance_factor) - metrics_95 = validate_coverage(y_eval_90.to_numpy(dtype=float), y_int_95, best_alpha_95) - group_metrics_95 = conditional_coverage_by_group( - y_eval_90.to_numpy(dtype=float), y_int_95, eval_groups_90 + y_intervals = _apply_learned_floor_policy( + y_pred=y_pred, + y_intervals=y_intervals, + groups=evidence_90.eval_groups, + group_multipliers=evidence_90.group_multipliers, + temporal_segments=evidence_90.eval_temporal_segments, + temporal_segment_multipliers=evidence_90.temporal_segment_multipliers, + global_rebalance_factor=evidence_90.global_rebalance_factor, + ) + metrics, group_metrics = _coverage_metrics( + y_true=evidence_90.y_eval, + y_intervals=y_intervals, + groups=evidence_90.eval_groups, + alpha=best_alpha_95, + ) + return ConformalEvidence95( + y_pred=y_pred, + y_intervals=y_intervals, + diag=diag, + metrics=metrics, + group_metrics=group_metrics, ) - # Compose output tables. - intervals_payload = { - "y_true": y_eval_90.to_numpy(dtype=float), - "y_pred": y_pred_90, - "pd_low_90": y_int_90[:, 0], - "pd_high_90": y_int_90[:, 1], - "pd_low_95": y_int_95[:, 0], - "pd_high_95": y_int_95[:, 1], - "width_90": y_int_90[:, 1] - y_int_90[:, 0], - "width_95": y_int_95[:, 1] - y_int_95[:, 0], - GROUP_COL: eval_groups_90.to_numpy(dtype=str), - "loan_amnt": ( - test_df["loan_amnt"].to_numpy(dtype=float) - if evaluation_scope_key == "test" and "loan_amnt" in test_df.columns - else cal_df.iloc[idx_cal_tune]["loan_amnt"].reset_index(drop=True).to_numpy(dtype=float) - if evaluation_scope_key == "holdout" and "loan_amnt" in cal_df.columns - else np.nan + +def _eval_source_frame( + *, + evaluation_scope_key: str, + test_df: pd.DataFrame, + cal_df: pd.DataFrame, + idx_cal_tune: np.ndarray, +) -> pd.DataFrame: + if evaluation_scope_key == "test": + return test_df.reset_index(drop=True) + return cal_df.iloc[idx_cal_tune].reset_index(drop=True) + + +def _eval_loan_amount_values( + *, + evaluation_scope_key: str, + test_df: pd.DataFrame, + cal_df: pd.DataFrame, + idx_cal_tune: np.ndarray, +) -> np.ndarray | float: + if evaluation_scope_key == "test" and "loan_amnt" in test_df.columns: + return test_df["loan_amnt"].to_numpy(dtype=float) + if evaluation_scope_key == "holdout" and "loan_amnt" in cal_df.columns: + return cal_df.iloc[idx_cal_tune]["loan_amnt"].reset_index(drop=True).to_numpy(dtype=float) + return np.nan + + +def _build_intervals_table( + *, + y_eval_90: pd.Series, + y_pred_90: np.ndarray, + y_int_90: np.ndarray, + y_int_95: np.ndarray, + eval_groups_90: pd.Series, + eval_temporal_segments: pd.Series | None, + evaluation_scope_key: str, + test_df: pd.DataFrame, + cal_df: pd.DataFrame, + idx_cal_tune: np.ndarray, +) -> pd.DataFrame: + eval_df = _eval_source_frame( + evaluation_scope_key=evaluation_scope_key, + test_df=test_df, + cal_df=cal_df, + idx_cal_tune=idx_cal_tune, + ) + intervals_payload: dict[str, Any] = { + "y_true": y_eval_90.to_numpy(dtype=float), + "y_pred": y_pred_90, + "pd_low_90": y_int_90[:, 0], + "pd_high_90": y_int_90[:, 1], + "pd_low_95": y_int_95[:, 0], + "pd_high_95": y_int_95[:, 1], + "width_90": y_int_90[:, 1] - y_int_90[:, 0], + "width_95": y_int_95[:, 1] - y_int_95[:, 0], + GROUP_COL: eval_groups_90.to_numpy(dtype=str), + "loan_amnt": _eval_loan_amount_values( + evaluation_scope_key=evaluation_scope_key, + test_df=test_df, + cal_df=cal_df, + idx_cal_tune=idx_cal_tune, ), } - eval_df = ( - test_df.reset_index(drop=True) - if evaluation_scope_key == "test" - else cal_df.iloc[idx_cal_tune].reset_index(drop=True) - ) if "id" in eval_df.columns: intervals_payload["id"] = eval_df["id"].astype(str).to_numpy() if eval_temporal_segments is not None: intervals_payload["temporal_segment"] = eval_temporal_segments.to_numpy(dtype=str) intervals_df = pd.DataFrame(intervals_payload) - intervals_df.insert(0, "_row_number", range(len(intervals_df))) + intervals_df.insert(0, "_row_number", np.arange(len(intervals_df), dtype=int)) + return intervals_df + +def _build_group_metrics_table( + *, + group_metrics_90: pd.DataFrame, + group_metrics_95: pd.DataFrame, + coverage_floor_report: pd.DataFrame, +) -> pd.DataFrame: gm90 = group_metrics_90.rename( columns={ "coverage": "coverage_90", @@ -1642,7 +1946,7 @@ def _cached_partition_labels( on="group", how="outer", ).sort_values("group") - group_metrics_df = group_metrics_df.merge( + return group_metrics_df.merge( coverage_floor_report[ ["group", "coverage_before", "coverage_after", "multiplier", "adjusted"] ], @@ -1650,31 +1954,82 @@ def _cached_partition_labels( how="left", ) - # Persist artifacts. - paths = _resolve_artifact_paths(artifact_namespace) - intervals_mondrian_path = paths["intervals"] - group_metrics_path = paths["group_metrics"] - tuning_path = paths["tuning"] - pareto_path = paths["pareto"] - coverage_floor_path = paths["group_floor"] - temporal_coverage_floor_path = paths["temporal_floor"] - shrinkback_path = paths["shrinkback"] - width_attr_path = paths["width_attr"] - results_path = paths["results"] - width_attr_status_path = paths["width_attr_status"] - resolved_run_tag = resolve_run_tag(run_tag, require_explicit=True) - intervals_df.to_parquet(intervals_mondrian_path, index=False) - group_metrics_df.to_parquet(group_metrics_path, index=False) - tuning_df.to_parquet(tuning_path, index=False) - tuning_df[tuning_df["is_pareto"]].copy().to_parquet(pareto_path, index=False) - coverage_floor_report.to_parquet(coverage_floor_path, index=False) - temporal_segment_report.to_parquet(temporal_coverage_floor_path, index=False) - shrinkback_report.to_parquet(shrinkback_path, index=False) - width_attr_df = pd.DataFrame(width_attr_rows) - width_attr_df.to_parquet(width_attr_path, index=False) - - payload = { +def _build_conformal_artifact_tables( + *, + y_eval_90: pd.Series, + y_pred_90: np.ndarray, + y_int_90: np.ndarray, + y_int_95: np.ndarray, + eval_groups_90: pd.Series, + eval_temporal_segments: pd.Series | None, + evaluation_scope_key: str, + test_df: pd.DataFrame, + cal_df: pd.DataFrame, + idx_cal_tune: np.ndarray, + group_metrics_90: pd.DataFrame, + group_metrics_95: pd.DataFrame, + coverage_floor_report: pd.DataFrame, + width_attr_rows: list[dict[str, Any]], +) -> ConformalArtifactTables: + return ConformalArtifactTables( + intervals=_build_intervals_table( + y_eval_90=y_eval_90, + y_pred_90=y_pred_90, + y_int_90=y_int_90, + y_int_95=y_int_95, + eval_groups_90=eval_groups_90, + eval_temporal_segments=eval_temporal_segments, + evaluation_scope_key=evaluation_scope_key, + test_df=test_df, + cal_df=cal_df, + idx_cal_tune=idx_cal_tune, + ), + group_metrics=_build_group_metrics_table( + group_metrics_90=group_metrics_90, + group_metrics_95=group_metrics_95, + coverage_floor_report=coverage_floor_report, + ), + width_attribution=pd.DataFrame(width_attr_rows), + ) + + +def _build_conformal_results_payload( + *, + model_path: Path, + calibrator_override_path: str | None, + metrics_90: dict[str, Any], + metrics_95: dict[str, Any], + diag_90: dict[str, Any], + diag_95: dict[str, Any], + partition_meta: dict[str, Any], + partition: str, + group_metrics_90: pd.DataFrame, + group_metrics_95: pd.DataFrame, + best_cfg: dict[str, Any], + alpha_candidates_95: tuple[float, ...], + best_alpha_95: float, + paths: dict[str, Path], + group_multipliers: dict[str, float], + temporal_segment_floor_enabled: bool, + temporal_segment_freq: str, + temporal_segment_min_size: int, + temporal_segment_multipliers: dict[str, float], + group_coverage_floor_enabled: bool, + shrinkback_enabled: bool, + global_rebalance_diagnostics: dict[str, float | bool], + group_coverage_floor_target_90: float, + X_cal_fit: pd.DataFrame, + X_tune: pd.DataFrame, + tuning_holdout_ratio: float, + tuning_random_state: int, + calibration_fraction: float, + evaluation_scope_key: str, + tune_metrics_90_before: dict[str, Any], + tune_metrics_90_after: dict[str, Any], + tune_metrics_90_after_temporal: dict[str, Any], +) -> dict[str, Any]: + return { "model_path": str(model_path), "calibrator_override_path": str(calibrator_override_path or ""), "metrics_90": {k: to_python_scalar(v) for k, v in metrics_90.items()}, @@ -1688,21 +2043,21 @@ def _cached_partition_labels( "tuning_90_best": best_cfg, "alpha_candidates_95": [float(x) for x in alpha_candidates_95], "alpha_used_95": float(best_alpha_95), - "tuning_90_table_path": str(tuning_path), - "tuning_90_pareto_path": str(pareto_path), - "group_coverage_floor_path": str(coverage_floor_path), + "tuning_90_table_path": str(paths["tuning"]), + "tuning_90_pareto_path": str(paths["pareto"]), + "group_coverage_floor_path": str(paths["group_floor"]), "group_coverage_multipliers": {k: float(v) for k, v in group_multipliers.items()}, "temporal_segment_floor_enabled": bool(temporal_segment_floor_enabled), "temporal_segment_freq": str(temporal_segment_freq), "temporal_segment_min_size": int(temporal_segment_min_size), - "temporal_segment_coverage_floor_path": str(temporal_coverage_floor_path), + "temporal_segment_coverage_floor_path": str(paths["temporal_floor"]), "temporal_segment_multipliers": { k: float(v) for k, v in temporal_segment_multipliers.items() }, "group_coverage_floor_enabled": bool(group_coverage_floor_enabled), "shrinkback_enabled": bool(shrinkback_enabled), - "shrinkback_path": str(shrinkback_path), - "width_attribution_path": str(width_attr_path), + "shrinkback_path": str(paths["shrinkback"]), + "width_attribution_path": str(paths["width_attr"]), "global_rebalance": global_rebalance_diagnostics, "group_coverage_floor_target_90": float(group_coverage_floor_target_90), "calibration_split": { @@ -1724,9 +2079,24 @@ def _cached_partition_labels( k: to_python_scalar(v) for k, v in tune_metrics_90_after_temporal.items() }, } - with open(results_path, "wb") as f: - pickle.dump(payload, f) - width_attr_status_path.write_text( + + +def _write_width_attribution_status( + *, + status_path: Path, + artifact_namespace: str | None, + partition_meta: dict[str, Any], + partition: str, + best_cfg: dict[str, Any], + best_alpha_95: float, + group_multipliers: dict[str, float], + temporal_segment_multipliers: dict[str, float], + width_attr_path: Path, + shrinkback_path: Path, + evaluation_scope_key: str, + resolved_run_tag: str, +) -> None: + status_path.write_text( json.dumps( { "artifact_namespace": artifact_namespace or "", @@ -1759,14 +2129,21 @@ def _cached_partition_labels( encoding="utf-8", ) + +def _log_conformal_artifacts( + paths: dict[str, Path], metrics_90: dict[str, Any], metrics_95: dict[str, Any] +) -> None: logger.info("Conformal artifacts saved:") - logger.info(f" - {intervals_mondrian_path}") - logger.info(f" - {group_metrics_path}") - logger.info(f" - {tuning_path}") - logger.info(f" - {pareto_path}") - logger.info(f" - {coverage_floor_path}") - logger.info(f" - {temporal_coverage_floor_path}") - logger.info(f" - {results_path}") + for key in [ + "intervals", + "group_metrics", + "tuning", + "pareto", + "group_floor", + "temporal_floor", + "results", + ]: + logger.info(f" - {paths[key]}") logger.info( "Final metrics: " f"90% coverage={metrics_90['empirical_coverage']:.4f} " @@ -1776,6 +2153,425 @@ def _cached_partition_labels( ) +def _persist_conformal_artifacts( + *, + paths: dict[str, Path], + artifact_namespace: str | None, + run_tag: str | None, + tables: ConformalArtifactTables, + tuning_df: pd.DataFrame, + coverage_floor_report: pd.DataFrame, + temporal_segment_report: pd.DataFrame, + shrinkback_report: pd.DataFrame, + payload: dict[str, Any], + partition_meta: dict[str, Any], + partition: str, + best_cfg: dict[str, Any], + best_alpha_95: float, + group_multipliers: dict[str, float], + temporal_segment_multipliers: dict[str, float], + evaluation_scope_key: str, + metrics_90: dict[str, Any], + metrics_95: dict[str, Any], +) -> dict[str, Path]: + tables.intervals.to_parquet(paths["intervals"], index=False) + tables.group_metrics.to_parquet(paths["group_metrics"], index=False) + tuning_df.to_parquet(paths["tuning"], index=False) + tuning_df[tuning_df["is_pareto"]].copy().to_parquet(paths["pareto"], index=False) + coverage_floor_report.to_parquet(paths["group_floor"], index=False) + temporal_segment_report.to_parquet(paths["temporal_floor"], index=False) + shrinkback_report.to_parquet(paths["shrinkback"], index=False) + tables.width_attribution.to_parquet(paths["width_attr"], index=False) + + with open(paths["results"], "wb") as f: + pickle.dump(payload, f) + + resolved_run_tag = resolve_run_tag(run_tag, require_explicit=True) + _write_width_attribution_status( + status_path=paths["width_attr_status"], + artifact_namespace=artifact_namespace, + partition_meta=partition_meta, + partition=partition, + best_cfg=best_cfg, + best_alpha_95=best_alpha_95, + group_multipliers=group_multipliers, + temporal_segment_multipliers=temporal_segment_multipliers, + width_attr_path=paths["width_attr"], + shrinkback_path=paths["shrinkback"], + evaluation_scope_key=evaluation_scope_key, + resolved_run_tag=resolved_run_tag, + ) + _log_conformal_artifacts(paths, metrics_90, metrics_95) + return paths + + +def _parse_float_tuple(raw: str) -> tuple[float, ...]: + values = [float(token.strip()) for token in str(raw).split(",") if token.strip()] + if not values: + raise ValueError("Expected at least one float value.") + return tuple(values) + + +def _parse_int_tuple(raw: str) -> tuple[int, ...]: + values = [int(token.strip()) for token in str(raw).split(",") if token.strip()] + if not values: + raise ValueError("Expected at least one integer value.") + return tuple(values) + + +def _parse_bool_tuple(raw: str) -> tuple[bool, ...]: + values = [ + token.strip().lower() in {"1", "true", "yes", "y"} + for token in str(raw).split(",") + if token.strip() + ] + if not values: + raise ValueError("Expected at least one boolean value.") + return tuple(values) + + +def _parse_str_tuple(raw: str) -> tuple[str, ...]: + values = tuple(token.strip() for token in str(raw).split(",") if token.strip()) + if not values: + raise ValueError("Expected at least one string value.") + return values + + +def main( + alpha_target_90: float = 0.10, + alpha_95: float = 0.05, + alpha_candidates_90: tuple[float, ...] = (0.10, 0.095, 0.09, 0.085, 0.08), + alpha_candidates_95: tuple[float, ...] = (0.05,), + min_group_sizes: tuple[int, ...] = (200, 500, 1000, 2000), + min_group_coverage_target: float = 0.88, + group_coverage_floor_target_90: float = 0.92, + max_width_budget_90: float | None = 0.80, + coverage_guardband_90: float = 0.015, + min_group_guardband_90: float = 0.0, + tuning_holdout_ratio: float = 0.20, + tuning_random_state: int = 42, + temporal_segment_floor_enabled: bool = True, + temporal_segment_freq: str = "Q", + temporal_segment_min_size: int = 250, + global_rebalance_enabled: bool = False, + global_rebalance_min_factor: float = 0.75, + global_rebalance_max_factor: float = 1.05, + global_rebalance_step: float = 0.01, + partition: str = "grade", + partition_candidates: tuple[str, ...] | None = None, + group_coverage_floor_enabled: bool = True, + shrinkback_enabled: bool = False, + group_multiplier_grid: tuple[float, ...] = (1.0, 1.02, 1.05, 1.08, 1.12, 1.16, 1.20), + temporal_multiplier_grid: tuple[float, ...] = (1.0, 1.02, 1.05, 1.08, 1.12, 1.16, 1.20), + artifact_namespace: str | None = None, + scaled_scores_options: tuple[bool, ...] = (True, False), + score_scale_families: tuple[str, ...] = ("bernoulli_sqrt", "none"), + partition_probability_sources: tuple[str, ...] = ("raw",), + n_score_bins_candidates: tuple[int, ...] = (10,), + fallback_modes: tuple[str, ...] = ("grade_then_global",), + calibration_fraction: float = 1.0, + evaluation_scope: str = "test", + mode: str = "search", + replay_manifest_path: str | None = None, + calibrator_override_path: str | None = None, + run_tag: str | None = None, +): + logger.info("Starting Mondrian conformal interval generation with 90% auto-tuning") + run_mode = str(mode or "search").strip().lower() or "search" + replay_cfg = manifest_section(load_replay_manifest(replay_manifest_path), "conformal") + if run_mode == "replay": + source_namespace = str(replay_cfg.get("source_namespace", "")).strip() + replay_mode = str(replay_cfg.get("replay_mode", "")).strip().lower() + if replay_mode != "restore_blessed_namespace" or not source_namespace: + raise ValueError( + "Conformal replay requires source_namespace + restore_blessed_namespace." + ) + _restore_replay_namespace(source_namespace, run_tag=run_tag) + return + + inputs = _load_conformal_inputs( + calibration_fraction=calibration_fraction, + calibrator_override_path=calibrator_override_path, + ) + model_path = inputs.model_path + cal_df = inputs.cal_df + test_df = inputs.test_df + X_cal = inputs.X_cal + y_cal = inputs.y_cal + y_test = inputs.y_test + group_cal_base = inputs.group_cal_base + group_test_base = inputs.group_test_base + y_prob_cal_raw = inputs.y_prob_cal_raw + tuning_split = _build_tuning_split( + cal_df=cal_df, + test_df=test_df, + X_cal=X_cal, + y_cal=y_cal, + group_cal_base=group_cal_base, + y_prob_cal_raw=y_prob_cal_raw, + tuning_holdout_ratio=tuning_holdout_ratio, + tuning_random_state=tuning_random_state, + ) + idx_cal_tune = tuning_split.idx_cal_tune + X_cal_fit = tuning_split.X_cal_fit + y_cal_fit = tuning_split.y_cal_fit + X_tune = tuning_split.X_tune + y_tune = tuning_split.y_tune + group_cal_fit_base = tuning_split.group_cal_fit_base + group_tune_base = tuning_split.group_tune_base + issue_tune = tuning_split.issue_tune + issue_test = tuning_split.issue_test + _log_tuning_split(inputs, tuning_split) + + target_coverage_90 = 1.0 - alpha_target_90 + group_coverage_floor_target_90 = max( + float(min_group_coverage_target), + float(group_coverage_floor_target_90), + ) + evaluation_scope_key = str(evaluation_scope or "test").strip().lower() or "test" + if evaluation_scope_key not in {"test", "holdout"}: + raise ValueError(f"Unsupported evaluation_scope: {evaluation_scope}") + tuning_grid = _resolve_tuning_grid( + partition=partition, + partition_candidates=partition_candidates, + partition_probability_sources=partition_probability_sources, + n_score_bins_candidates=n_score_bins_candidates, + fallback_modes=fallback_modes, + score_scale_families=score_scale_families, + scaled_scores_options=scaled_scores_options, + ) + partition_candidates = tuning_grid.partition_candidates + prob_fit_lookup, prob_tune_lookup, prob_test_lookup = _build_probability_lookups( + inputs, + tuning_split, + ) + interval_fit_pred = prob_fit_lookup["calibrated"] + interval_tune_pred = prob_tune_lookup["calibrated"] + interval_test_pred = prob_test_lookup["calibrated"] + tuning_search = _run_tuning_search( + tuning_grid=tuning_grid, + alpha_candidates_90=alpha_candidates_90, + min_group_sizes=min_group_sizes, + alpha_target_90=alpha_target_90, + target_coverage_90=target_coverage_90, + artifact_namespace=artifact_namespace, + prob_fit_lookup=prob_fit_lookup, + prob_tune_lookup=prob_tune_lookup, + prob_test_lookup=prob_test_lookup, + interval_fit_pred=interval_fit_pred, + interval_tune_pred=interval_tune_pred, + y_cal_fit=y_cal_fit, + y_tune=y_tune, + group_cal_fit_base=group_cal_fit_base, + group_tune_base=group_tune_base, + group_test_base=group_test_base, + issue_tune=issue_tune, + ) + tuning_rows = tuning_search.tuning_rows + partition_cache = tuning_search.partition_cache + + tuning_selection = _select_best_tuning_config( + tuning_rows, + partition_candidates=partition_candidates, + alpha_target_90=alpha_target_90, + min_group_coverage_target=min_group_coverage_target, + group_coverage_floor_target_90=group_coverage_floor_target_90, + coverage_guardband_90=coverage_guardband_90, + min_group_guardband_90=min_group_guardband_90, + max_width_budget_90=max_width_budget_90, + target_coverage_90=target_coverage_90, + ) + tuning_df = tuning_selection.tuning_df + best_row = tuning_selection.best_row + selection_tier = tuning_selection.selection_tier + best_cfg = tuning_selection.best_cfg + logger.info( + "Best 90% tuning config: " + f"partition={best_cfg['partition']}, " + f"prob_source={best_cfg['partition_probability_source']}, " + f"n_bins={best_cfg['n_score_bins']}, " + f"alpha_used={best_cfg['alpha_used_90']}, scaled_scores={best_cfg['scaled_scores']}, " + f"score_scale_family={best_cfg['score_scale_family']}, " + f"min_group_size={best_cfg['min_group_size']}, " + f"coverage={best_row['empirical_coverage']:.4f}, " + f"min_group_coverage={best_row['min_group_coverage']:.4f}, " + f"width={best_row['avg_interval_width']:.4f}, " + f"tier={selection_tier}" + ) + + group_cal_fit, group_test, partition_meta = _cached_partition_labels( + cache=partition_cache, + eval_scope="test", + partition_candidate=best_cfg["partition"], + partition_probability_source=best_cfg["partition_probability_source"], + n_score_bins=best_cfg["n_score_bins"], + fallback_mode=best_cfg["fallback_mode"], + min_group_size=best_cfg["min_group_size"], + prob_fit_lookup=prob_fit_lookup, + prob_tune_lookup=prob_tune_lookup, + prob_test_lookup=prob_test_lookup, + group_cal_fit_base=group_cal_fit_base, + group_tune_base=group_tune_base, + group_test_base=group_test_base, + ) + group_cal_fit_holdout, group_tune, _ = _cached_partition_labels( + cache=partition_cache, + eval_scope="tune", + partition_candidate=best_cfg["partition"], + partition_probability_source=best_cfg["partition_probability_source"], + n_score_bins=best_cfg["n_score_bins"], + fallback_mode=best_cfg["fallback_mode"], + min_group_size=best_cfg["min_group_size"], + prob_fit_lookup=prob_fit_lookup, + prob_tune_lookup=prob_tune_lookup, + prob_test_lookup=prob_test_lookup, + group_cal_fit_base=group_cal_fit_base, + group_tune_base=group_tune_base, + group_test_base=group_test_base, + ) + + evidence_90 = _build_90_interval_evidence( + evaluation_scope_key=evaluation_scope_key, + y_test=y_test, + y_tune=y_tune, + issue_test=issue_test, + issue_tune=issue_tune, + interval_fit_pred=interval_fit_pred, + interval_test_pred=interval_test_pred, + interval_tune_pred=interval_tune_pred, + y_cal_fit=y_cal_fit, + group_cal_fit=group_cal_fit, + group_test=group_test, + group_cal_fit_holdout=group_cal_fit_holdout, + group_tune=group_tune, + best_cfg=best_cfg, + alpha_target_90=alpha_target_90, + target_coverage_90=target_coverage_90, + group_coverage_floor_enabled=group_coverage_floor_enabled, + group_coverage_floor_target_90=group_coverage_floor_target_90, + group_multiplier_grid=group_multiplier_grid, + temporal_segment_floor_enabled=temporal_segment_floor_enabled, + temporal_segment_freq=temporal_segment_freq, + temporal_segment_min_size=temporal_segment_min_size, + temporal_multiplier_grid=temporal_multiplier_grid, + shrinkback_enabled=shrinkback_enabled, + min_group_coverage_target=min_group_coverage_target, + global_rebalance_enabled=global_rebalance_enabled, + global_rebalance_min_factor=global_rebalance_min_factor, + global_rebalance_max_factor=global_rebalance_max_factor, + global_rebalance_step=global_rebalance_step, + ) + y_eval_90 = evidence_90.y_eval + eval_groups_90 = evidence_90.eval_groups + y_pred_90 = evidence_90.y_pred + y_int_90 = evidence_90.y_intervals + diag_90 = evidence_90.diag + metrics_90 = evidence_90.metrics + group_metrics_90 = evidence_90.group_metrics + + best_alpha_95 = _select_alpha_95( + alpha_95=alpha_95, + alpha_candidates_95=alpha_candidates_95, + interval_fit_pred=interval_fit_pred, + interval_tune_pred=interval_tune_pred, + y_cal_fit=y_cal_fit, + y_tune=y_tune, + group_cal_fit_holdout=group_cal_fit_holdout, + group_tune=group_tune, + best_cfg=best_cfg, + ) + + evidence_95 = _build_95_interval_evidence( + evaluation_scope_key=evaluation_scope_key, + interval_fit_pred=interval_fit_pred, + interval_test_pred=interval_test_pred, + interval_tune_pred=interval_tune_pred, + y_cal_fit=y_cal_fit, + group_cal_fit=group_cal_fit, + group_test=group_test, + group_tune=group_tune, + evidence_90=evidence_90, + best_cfg=best_cfg, + best_alpha_95=best_alpha_95, + ) + y_int_95 = evidence_95.y_intervals + diag_95 = evidence_95.diag + metrics_95 = evidence_95.metrics + group_metrics_95 = evidence_95.group_metrics + + paths = _resolve_artifact_paths(artifact_namespace) + tables = _build_conformal_artifact_tables( + y_eval_90=y_eval_90, + y_pred_90=y_pred_90, + y_int_90=y_int_90, + y_int_95=y_int_95, + eval_groups_90=eval_groups_90, + eval_temporal_segments=evidence_90.eval_temporal_segments, + evaluation_scope_key=evaluation_scope_key, + test_df=test_df, + cal_df=cal_df, + idx_cal_tune=idx_cal_tune, + group_metrics_90=group_metrics_90, + group_metrics_95=group_metrics_95, + coverage_floor_report=evidence_90.coverage_floor_report, + width_attr_rows=evidence_90.width_attr_rows, + ) + payload = _build_conformal_results_payload( + model_path=model_path, + calibrator_override_path=calibrator_override_path, + metrics_90=metrics_90, + metrics_95=metrics_95, + diag_90=diag_90, + diag_95=diag_95, + partition_meta=partition_meta, + partition=partition, + group_metrics_90=group_metrics_90, + group_metrics_95=group_metrics_95, + best_cfg=best_cfg, + alpha_candidates_95=alpha_candidates_95, + best_alpha_95=best_alpha_95, + paths=paths, + group_multipliers=evidence_90.group_multipliers, + temporal_segment_floor_enabled=temporal_segment_floor_enabled, + temporal_segment_freq=temporal_segment_freq, + temporal_segment_min_size=temporal_segment_min_size, + temporal_segment_multipliers=evidence_90.temporal_segment_multipliers, + group_coverage_floor_enabled=group_coverage_floor_enabled, + shrinkback_enabled=shrinkback_enabled, + global_rebalance_diagnostics=evidence_90.global_rebalance_diagnostics, + group_coverage_floor_target_90=group_coverage_floor_target_90, + X_cal_fit=X_cal_fit, + X_tune=X_tune, + tuning_holdout_ratio=tuning_holdout_ratio, + tuning_random_state=tuning_random_state, + calibration_fraction=calibration_fraction, + evaluation_scope_key=evaluation_scope_key, + tune_metrics_90_before=evidence_90.tune_metrics_before_floor, + tune_metrics_90_after=evidence_90.tune_metrics_after_floor, + tune_metrics_90_after_temporal=evidence_90.tune_metrics_after_temporal_floor, + ) + _persist_conformal_artifacts( + paths=paths, + artifact_namespace=artifact_namespace, + run_tag=run_tag, + tables=tables, + tuning_df=tuning_df, + coverage_floor_report=evidence_90.coverage_floor_report, + temporal_segment_report=evidence_90.temporal_segment_report, + shrinkback_report=evidence_90.shrinkback_report, + payload=payload, + partition_meta=partition_meta, + partition=partition, + best_cfg=best_cfg, + best_alpha_95=best_alpha_95, + group_multipliers=evidence_90.group_multipliers, + temporal_segment_multipliers=evidence_90.temporal_segment_multipliers, + evaluation_scope_key=evaluation_scope_key, + metrics_90=metrics_90, + metrics_95=metrics_95, + ) + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( diff --git a/scripts/generate_crpto_figures.py b/scripts/generate_crpto_figures.py index debff3e..ca9b02e 100644 --- a/scripts/generate_crpto_figures.py +++ b/scripts/generate_crpto_figures.py @@ -19,6 +19,7 @@ import argparse import shutil from pathlib import Path +from typing import Any import matplotlib import matplotlib.pyplot as plt @@ -602,6 +603,62 @@ def _crpto_fig7_uncertainty_baselines() -> None: _save(fig, "crpto_fig7_uncertainty_baselines") +def _first_matching_column(df: pd.DataFrame, tokens: tuple[str, ...]) -> str | None: + return next((col for col in df.columns if any(token in col.lower() for token in tokens)), None) + + +def _alpha_pareto_column_map(df: pd.DataFrame) -> dict[str, str | None]: + return { + "variant": _first_matching_column(df, ("variant", "method")), + "alpha": _first_matching_column(df, ("alpha",)), + "coverage": _first_matching_column(df, ("coverage",)), + "width": _first_matching_column(df, ("width",)), + "eligible": _first_matching_column(df, ("eligible", "n_eligible")), + } + + +def _alpha_pareto_missing_columns(columns: dict[str, str | None]) -> list[str]: + return [key for key in ["variant", "alpha", "coverage", "width"] if not columns.get(key)] + + +def _alpha_pareto_variants(df: pd.DataFrame, variant_col: str) -> list[Any]: + return list(df[variant_col].drop_duplicates()) + + +def _alpha_pareto_variant_styles( + variants: list[Any], +) -> tuple[dict[Any, str], dict[Any, str]]: + colors = { + variant: PALETTE["blue"] if "mond" in str(variant).lower() else PALETTE["orange"] + for variant in variants + } + labels = { + variant: "Mondrian CP" if "mond" in str(variant).lower() else "Global Split-CP" + for variant in variants + } + return colors, labels + + +def _alpha_pareto_subframe( + df: pd.DataFrame, + *, + variant_col: str, + alpha_col: str, + variant: Any, +) -> pd.DataFrame: + return df[df[variant_col] == variant].sort_values(alpha_col) + + +def _alpha_annotation_offset(index: int, n_rows: int) -> tuple[int, int]: + offset_y = 4 if index % 2 == 0 else -8 + offset_x = 4 if index < n_rows - 1 else -24 + return offset_x, offset_y + + +def _alpha_tick_labels(values: pd.Series) -> list[str]: + return [str(round(float(value), 2)) for value in values] + + def _crpto_fig8_alpha_pareto() -> None: """Fig 8 — Alpha sweep Pareto: Mondrian vs Global (coverage × width × eligible loans).""" df = pd.read_parquet(DATA_DIR / "alpha_sweep_pareto_both.parquet") @@ -610,36 +667,27 @@ def _crpto_fig8_alpha_pareto() -> None: logger.warning("alpha_sweep_pareto_both empty — skipping fig8") return - # Detect variant column - variant_col = next( - (c for c in df.columns if "variant" in c.lower() or "method" in c.lower()), None - ) - alpha_col = next((c for c in df.columns if "alpha" in c.lower()), None) - cov_col = next((c for c in df.columns if "coverage" in c.lower()), None) - width_col = next((c for c in df.columns if "width" in c.lower()), None) - elig_col = next( - (c for c in df.columns if "eligible" in c.lower() or "n_eligible" in c.lower()), None - ) - - if not all([variant_col, alpha_col, cov_col, width_col]): + columns = _alpha_pareto_column_map(df) + missing = _alpha_pareto_missing_columns(columns) + if missing: logger.warning(f"alpha_sweep_pareto_both missing columns. Got: {list(df.columns)}") return + variant_col = str(columns["variant"]) + alpha_col = str(columns["alpha"]) + cov_col = str(columns["coverage"]) + width_col = str(columns["width"]) + elig_col = columns["eligible"] fig, axes = plt.subplots(1, 2, figsize=(COL2, HEIGHT_M)) - variants = df[variant_col].unique() if variant_col else ["global", "mondrian"] - var_colors = { - v: PALETTE["blue"] if "mond" in str(v).lower() else PALETTE["orange"] for v in variants - } - var_labels = { - v: "Mondrian CP" if "mond" in str(v).lower() else "Global Split-CP" for v in variants - } + variants = _alpha_pareto_variants(df, variant_col) + var_colors, var_labels = _alpha_pareto_variant_styles(variants) # Left: width vs coverage scatter (Pareto) ax = axes[0] ax.axvline(0.90, color=PALETTE["red"], lw=0.8, ls="--", alpha=0.6, label="90% target") for var in variants: - sub = df[df[variant_col] == var].sort_values(alpha_col) + sub = _alpha_pareto_subframe(df, variant_col=variant_col, alpha_col=alpha_col, variant=var) ax.plot( sub[cov_col], sub[width_col], @@ -649,8 +697,7 @@ def _crpto_fig8_alpha_pareto() -> None: label=var_labels[var], ) for idx, (_, row) in enumerate(sub.iterrows()): - offset_y = 4 if idx % 2 == 0 else -8 - offset_x = 4 if idx < len(sub) - 1 else -24 + offset_x, offset_y = _alpha_annotation_offset(idx, len(sub)) ax.annotate( f"α={row[alpha_col]:.2f}", (row[cov_col], row[width_col]), @@ -669,9 +716,14 @@ def _crpto_fig8_alpha_pareto() -> None: ax2 = axes[1] if elig_col: for var in variants: - sub = df[df[variant_col] == var].sort_values(alpha_col) + sub = _alpha_pareto_subframe( + df, + variant_col=variant_col, + alpha_col=alpha_col, + variant=var, + ) ax2.bar( - [str(round(a, 2)) for a in sub[alpha_col]], + _alpha_tick_labels(sub[alpha_col]), sub[elig_col], label=var_labels[var], color=var_colors[var], @@ -1824,13 +1876,13 @@ def _crpto_fig21_end_to_end_arc() -> None: def _crpto_fig25_price_of_robustness_scaling() -> None: - """Fig 25 — price of robustness scales with panel default rate (A34). + """Fig 25 — external price of robustness by panel default rate (A34). Frozen external applications (Freddie green/combined/red and Prosper) show a - positive premium that increases monotonically with the panel default rate. - The selected Lending Club champion is drawn as a contrasting favorable - reference line (it is a single selected point, not part of the default-rate - series). Data: models/crpto_multidataset_external_status.json. + positive premium ordered by panel default rate in the four observed cases. + The historical Lending Club field is intentionally excluded because its + stored nonrobust baseline was not a point-only comparator. Data: + models/crpto_multidataset_external_status.json. """ status = load_json(MODELS_DIR / "crpto_multidataset_external_status.json") seg_label = { @@ -1862,21 +1914,11 @@ def _crpto_fig25_price_of_robustness_scaling() -> None: x = np.array([p[1] for p in pts]) y = np.array([p[2] for p in pts]) - lc_price = -10.56 # selected Lending Club champion (frozen field) - fig, ax = plt.subplots(figsize=(COL2, HEIGHT_M)) ax.set_xscale("log") ax.set_xlim(0.4, 55) - ax.set_ylim(-13.5, 12.5) + ax.set_ylim(0.0, 11.5) ax.axhline(0.0, color=PALETTE["gray"], lw=1.0, ls="--", zorder=1) - ax.axhline( - lc_price, - color=PALETTE["orange"], - lw=1.4, - ls=":", - zorder=2, - label="Lending Club selected champion (−10.56%)", - ) ax.plot( x, y, @@ -1897,14 +1939,15 @@ def _crpto_fig25_price_of_robustness_scaling() -> None: color=PALETTE["green"], fontweight="bold", ) - ax.text(0.46, 1.2, "robustness costs a premium", fontsize=6.4, color="#666", style="italic") - ax.text(0.46, -2.6, "robustness adds value", fontsize=6.4, color="#666", style="italic") + ax.text( + 0.46, 0.45, "positive premium in all four cases", fontsize=6.4, color="#666", style="italic" + ) ax.set_xticks([0.5, 1, 2, 5, 10, 30]) ax.set_xticklabels(["0.5", "1", "2", "5", "10", "30"]) ax.tick_params(which="minor", bottom=False) ax.set_xlabel("Panel default rate (%, log scale)") ax.set_ylabel("Price of robustness (%)") - ax.set_title("Price of robustness scales with panel default risk") + ax.set_title("External robustness premium across frozen panels") ax.legend(loc="lower right", fontsize=6.6) fig.tight_layout() _save(fig, "crpto_fig25_price_of_robustness_scaling") diff --git a/scripts/generate_governance_status.py b/scripts/generate_governance_status.py index 22573db..db3f704 100644 --- a/scripts/generate_governance_status.py +++ b/scripts/generate_governance_status.py @@ -13,6 +13,7 @@ import argparse import json +from dataclasses import dataclass from pathlib import Path from typing import Any @@ -40,6 +41,38 @@ SCHEMA_VERSION = "2026-03-06.1" +@dataclass(frozen=True) +class GovernanceThresholds: + psi_threshold: float + ks_pvalue_min: float + cvm_pvalue_min: float + c2st_auc_max: float + max_feature_breach_ratio: float + c2st_max_rows: int + score_psi_max: float + auc_delta_max: float + brier_increase_max: float + calibration_gap_delta_max: float + performance_max_rows: int + min_rank_overlap_top10: float + max_explanation_shap_psi: float + min_reason_code_stability: float + explanation_min_rows_per_slice: int + psi_bins: int + random_state: int + + +@dataclass(frozen=True) +class GovernanceOutputPaths: + drift_path: Path + status_path: Path + explanation_drift_path: Path + fairness_status_path: Path + fairness_frontier_path: Path + challenger_report_path: Path + model_shift_status_path: Path + + def _load_cfg(path: str) -> dict[str, Any]: with open(path, encoding="utf-8") as f: return yaml.safe_load(f) @@ -264,6 +297,131 @@ def _resolve_primary_threshold() -> float: return 0.5 +def _explanation_feature_columns(shap_raw: pd.DataFrame) -> list[str]: + return [c.replace("shap_", "") for c in shap_raw.columns if c.startswith("shap_")] + + +def _recent_comparison_periods( + shap_raw: pd.DataFrame, + periods: list[str], + *, + min_rows_per_slice: int, +) -> tuple[list[str], pd.DataFrame] | None: + comparison_periods: list[str] = [] + for period in reversed(periods): + comparison_periods.insert(0, period) + comparison_df = shap_raw.loc[ + shap_raw["issue_quarter"].astype(str).isin(comparison_periods) + ].copy() + if len(comparison_df) >= min_rows_per_slice: + return comparison_periods, comparison_df + return None + + +def _explanation_segment_pairs( + shap_raw: pd.DataFrame, + reference_df: pd.DataFrame, + comparison_df: pd.DataFrame, + *, + min_rows_per_slice: int, +) -> list[tuple[str, str, pd.DataFrame, pd.DataFrame]]: + segment_pairs: list[tuple[str, str, pd.DataFrame, pd.DataFrame]] = [ + ("overall", "all", reference_df, comparison_df) + ] + if "grade" not in shap_raw.columns: + return segment_pairs + for grade in sorted(shap_raw["grade"].dropna().astype(str).unique().tolist()): + ref_seg = reference_df.loc[reference_df["grade"].astype(str) == grade].copy() + cmp_seg = comparison_df.loc[comparison_df["grade"].astype(str) == grade].copy() + if len(ref_seg) < min_rows_per_slice or len(cmp_seg) < min_rows_per_slice: + continue + segment_pairs.append(("grade", grade, ref_seg, cmp_seg)) + return segment_pairs + + +def _rank_shap_features(segment: pd.DataFrame, feature_cols: list[str]) -> list[str]: + return sorted( + feature_cols, + key=lambda feature: segment[f"shap_{feature}"].abs().mean(), + reverse=True, + ) + + +def _shap_psi_details( + ref_seg: pd.DataFrame, + cmp_seg: pd.DataFrame, + *, + focus_features: list[str], +) -> list[dict[str, float | str]]: + rows: list[dict[str, float | str]] = [] + for feature in focus_features: + col = f"shap_{feature}" + if col not in ref_seg.columns or col not in cmp_seg.columns: + continue + psi = population_stability_index( + pd.to_numeric(ref_seg[col], errors="coerce").dropna().to_numpy(dtype=float), + pd.to_numeric(cmp_seg[col], errors="coerce").dropna().to_numpy(dtype=float), + n_bins=8, + ) + rows.append({"feature": feature, "psi": float(psi)}) + return rows + + +def _explanation_drift_row( + *, + segment_type: str, + segment: str, + ref_seg: pd.DataFrame, + cmp_seg: pd.DataFrame, + feature_cols: list[str], + periods: list[str], + comparison_periods: list[str], + comparison_period_label: str, + primary_threshold: float, + min_rank_overlap_top10: float, + max_shap_psi: float, + min_reason_code_stability: float, + min_rows_per_slice: int, + pd_col: str, +) -> dict[str, Any]: + ref_ranking = _rank_shap_features(ref_seg, feature_cols) + cmp_ranking = _rank_shap_features(cmp_seg, feature_cols) + overlap = rank_overlap_ratio(ref_ranking, cmp_ranking, top_k=10) + focus_features = list(dict.fromkeys(ref_ranking[:5] + cmp_ranking[:5]))[:5] + shap_psis = _shap_psi_details(ref_seg, cmp_seg, focus_features=focus_features) + max_feature_psi = max((float(row["psi"]) for row in shap_psis), default=0.0) + avg_feature_psi = float(np.mean([float(row["psi"]) for row in shap_psis])) if shap_psis else 0.0 + reason_match_rate, reason_details = dominant_reason_match_rate( + ref_seg, + cmp_seg, + ref_ranking[:10], + pd_col=pd_col, + threshold=primary_threshold, + min_rows_per_band=max(15, int(min_rows_per_slice / 4)), + ) + pass_rank = bool(overlap >= min_rank_overlap_top10) + pass_dist = bool(max_feature_psi <= max_shap_psi) + pass_reason = bool(reason_match_rate >= min_reason_code_stability) + return { + "segment_type": segment_type, + "segment": segment, + "reference_period": "|".join([p for p in periods if p not in comparison_periods]), + "comparison_period": comparison_period_label, + "reference_n": len(ref_seg), + "comparison_n": len(cmp_seg), + "rank_overlap_top10": float(overlap), + "avg_shap_psi_top5": float(avg_feature_psi), + "max_shap_psi_top5": float(max_feature_psi), + "reason_code_match_rate": float(reason_match_rate), + "pass_rank_overlap": pass_rank, + "pass_distribution_shift": pass_dist, + "pass_reason_code_stability": pass_reason, + "passed_all": bool(pass_rank and pass_dist and pass_reason), + "feature_psi_details": json.dumps(shap_psis, default=str), + "reason_code_details": json.dumps(reason_details, default=str), + } + + def _build_explanation_drift_report( shap_raw: pd.DataFrame, *, @@ -273,7 +431,7 @@ def _build_explanation_drift_report( min_reason_code_stability: float, min_rows_per_slice: int, ) -> pd.DataFrame: - feature_cols = [c.replace("shap_", "") for c in shap_raw.columns if c.startswith("shap_")] + feature_cols = _explanation_feature_columns(shap_raw) if shap_raw.empty or not feature_cols or "issue_quarter" not in shap_raw.columns: return pd.DataFrame() @@ -287,17 +445,14 @@ def _build_explanation_drift_report( if len(periods) < 2: return pd.DataFrame() - comparison_periods: list[str] = [] - comparison_df = pd.DataFrame() - for period in reversed(periods): - comparison_periods.insert(0, period) - comparison_df = shap_raw.loc[ - shap_raw["issue_quarter"].astype(str).isin(comparison_periods) - ].copy() - if len(comparison_df) >= min_rows_per_slice: - break - if len(comparison_df) < min_rows_per_slice: + comparison_slice = _recent_comparison_periods( + shap_raw, + periods, + min_rows_per_slice=min_rows_per_slice, + ) + if comparison_slice is None: return pd.DataFrame() + comparison_periods, comparison_df = comparison_slice reference_df = shap_raw.loc[ ~shap_raw["issue_quarter"].astype(str).isin(comparison_periods) @@ -306,131 +461,107 @@ def _build_explanation_drift_report( return pd.DataFrame() comparison_period_label = "|".join(comparison_periods) - segment_pairs: list[tuple[str, str, pd.DataFrame, pd.DataFrame]] = [ - ("overall", "all", reference_df, comparison_df) - ] - if "grade" in shap_raw.columns: - for grade in sorted(shap_raw["grade"].dropna().astype(str).unique().tolist()): - ref_seg = reference_df.loc[reference_df["grade"].astype(str) == grade].copy() - cmp_seg = comparison_df.loc[comparison_df["grade"].astype(str) == grade].copy() - if len(ref_seg) < min_rows_per_slice or len(cmp_seg) < min_rows_per_slice: - continue - segment_pairs.append(("grade", grade, ref_seg, cmp_seg)) - + segment_pairs = _explanation_segment_pairs( + shap_raw, + reference_df, + comparison_df, + min_rows_per_slice=min_rows_per_slice, + ) + pd_col = "pd_calibrated" if "pd_calibrated" in shap_raw.columns else "score_raw" rows: list[dict[str, Any]] = [] for segment_type, segment, ref_seg, cmp_seg in segment_pairs: - ref_ranking = sorted( - feature_cols, - key=lambda feature: ref_seg[f"shap_{feature}"].abs().mean(), - reverse=True, - ) - cmp_ranking = sorted( - feature_cols, - key=lambda feature: cmp_seg[f"shap_{feature}"].abs().mean(), - reverse=True, - ) - overlap = rank_overlap_ratio(ref_ranking, cmp_ranking, top_k=10) - focus_features = list(dict.fromkeys(ref_ranking[:5] + cmp_ranking[:5]))[:5] - shap_psis = [] - for feature in focus_features: - col = f"shap_{feature}" - if col not in ref_seg.columns or col not in cmp_seg.columns: - continue - psi = population_stability_index( - pd.to_numeric(ref_seg[col], errors="coerce").dropna().to_numpy(dtype=float), - pd.to_numeric(cmp_seg[col], errors="coerce").dropna().to_numpy(dtype=float), - n_bins=8, - ) - shap_psis.append({"feature": feature, "psi": float(psi)}) - - max_feature_psi = max((row["psi"] for row in shap_psis), default=0.0) - avg_feature_psi = float(np.mean([row["psi"] for row in shap_psis])) if shap_psis else 0.0 - reason_match_rate, reason_details = dominant_reason_match_rate( - ref_seg, - cmp_seg, - ref_ranking[:10], - pd_col="pd_calibrated" if "pd_calibrated" in shap_raw.columns else "score_raw", - threshold=primary_threshold, - min_rows_per_band=max(15, int(min_rows_per_slice / 4)), - ) - pass_rank = bool(overlap >= min_rank_overlap_top10) - pass_dist = bool(max_feature_psi <= max_shap_psi) - pass_reason = bool(reason_match_rate >= min_reason_code_stability) rows.append( - { - "segment_type": segment_type, - "segment": segment, - "reference_period": "|".join([p for p in periods if p not in comparison_periods]), - "comparison_period": comparison_period_label, - "reference_n": len(ref_seg), - "comparison_n": len(cmp_seg), - "rank_overlap_top10": float(overlap), - "avg_shap_psi_top5": float(avg_feature_psi), - "max_shap_psi_top5": float(max_feature_psi), - "reason_code_match_rate": float(reason_match_rate), - "pass_rank_overlap": pass_rank, - "pass_distribution_shift": pass_dist, - "pass_reason_code_stability": pass_reason, - "passed_all": bool(pass_rank and pass_dist and pass_reason), - "feature_psi_details": json.dumps(shap_psis, default=str), - "reason_code_details": json.dumps(reason_details, default=str), - } + _explanation_drift_row( + segment_type=segment_type, + segment=segment, + ref_seg=ref_seg, + cmp_seg=cmp_seg, + feature_cols=feature_cols, + periods=periods, + comparison_periods=comparison_periods, + comparison_period_label=comparison_period_label, + primary_threshold=primary_threshold, + min_rank_overlap_top10=min_rank_overlap_top10, + max_shap_psi=max_shap_psi, + min_reason_code_stability=min_reason_code_stability, + min_rows_per_slice=min_rows_per_slice, + pd_col=pd_col, + ) ) return pd.DataFrame(rows) -def main(config_path: str = "configs/mrm_policy.yaml", run_tag: str | None = None) -> None: - cfg = _load_cfg(config_path) - semantics = load_threshold_semantics() - resolved_run_tag = resolve_run_tag( - run_tag, - fallback_candidates=[semantics.get("run_tag"), resolve_official_baseline_run_tag()], - require_explicit=True, +def _resolve_thresholds( + triggers: dict[str, Any], + checks: dict[str, Any], +) -> GovernanceThresholds: + return GovernanceThresholds( + psi_threshold=float(triggers.get("psi_threshold", 0.25)), + ks_pvalue_min=float(checks.get("ks_pvalue_min", 0.01)), + cvm_pvalue_min=float(checks.get("cvm_pvalue_min", 0.01)), + c2st_auc_max=float(checks.get("c2st_auc_max", 0.60)), + max_feature_breach_ratio=float(checks.get("max_feature_breach_ratio", 0.15)), + c2st_max_rows=int(checks.get("c2st_max_rows_per_split", 50_000)), + score_psi_max=float(checks.get("score_psi_max", 0.15)), + auc_delta_max=float(checks.get("auc_delta_max", 0.05)), + brier_increase_max=float(checks.get("brier_increase_max", 0.02)), + calibration_gap_delta_max=float(checks.get("calibration_gap_delta_max", 0.02)), + performance_max_rows=int(checks.get("performance_max_rows_per_split", 100_000)), + min_rank_overlap_top10=float(checks.get("explanation_rank_overlap_top10_min", 0.60)), + max_explanation_shap_psi=float(checks.get("explanation_shap_psi_max", 0.25)), + min_reason_code_stability=float(checks.get("reason_code_stability_min", 0.55)), + explanation_min_rows_per_slice=int(checks.get("explanation_min_rows_per_slice", 80)), + psi_bins=int(checks.get("psi_bins", 10)), + random_state=int(checks.get("random_state", 42)), ) - triggers = cfg.get("retraining_triggers", {}) - checks = cfg.get("governance_checks", {}) - outputs = cfg.get("governance_output", {}) - psi_threshold = float(triggers.get("psi_threshold", 0.25)) - ks_pvalue_min = float(checks.get("ks_pvalue_min", 0.01)) - cvm_pvalue_min = float(checks.get("cvm_pvalue_min", 0.01)) - c2st_auc_max = float(checks.get("c2st_auc_max", 0.60)) - max_feature_breach_ratio = float(checks.get("max_feature_breach_ratio", 0.15)) - c2st_max_rows = int(checks.get("c2st_max_rows_per_split", 50_000)) - score_psi_max = float(checks.get("score_psi_max", 0.15)) - auc_delta_max = float(checks.get("auc_delta_max", 0.05)) - brier_increase_max = float(checks.get("brier_increase_max", 0.02)) - calibration_gap_delta_max = float(checks.get("calibration_gap_delta_max", 0.02)) - performance_max_rows = int(checks.get("performance_max_rows_per_split", 100_000)) - min_rank_overlap_top10 = float(checks.get("explanation_rank_overlap_top10_min", 0.60)) - max_explanation_shap_psi = float(checks.get("explanation_shap_psi_max", 0.25)) - min_reason_code_stability = float(checks.get("reason_code_stability_min", 0.55)) - explanation_min_rows_per_slice = int(checks.get("explanation_min_rows_per_slice", 80)) - - drift_path = Path( - outputs.get("drift_monitoring_path", "data/processed/drift_monitoring.parquet") - ) - status_path = Path(outputs.get("governance_status_path", "models/governance_status.json")) - explanation_drift_path = Path( - outputs.get("explanation_drift_path", "data/processed/explanation_drift.parquet") - ) - fairness_status_path = Path( - outputs.get("fairness_status_path", "models/fairness_audit_status.json") - ) - fairness_frontier_path = Path( - outputs.get( +def _output_path(outputs: dict[str, Any], key: str, default: str) -> Path: + value = outputs.get(key, default) + return Path(str(default if value is None else value)) + + +def _resolve_output_paths(outputs: dict[str, Any]) -> GovernanceOutputPaths: + return GovernanceOutputPaths( + drift_path=_output_path( + outputs, + "drift_monitoring_path", + "data/processed/drift_monitoring.parquet", + ), + status_path=_output_path( + outputs, + "governance_status_path", + "models/governance_status.json", + ), + explanation_drift_path=_output_path( + outputs, + "explanation_drift_path", + "data/processed/explanation_drift.parquet", + ), + fairness_status_path=_output_path( + outputs, + "fairness_status_path", + "models/fairness_audit_status.json", + ), + fairness_frontier_path=_output_path( + outputs, "fairness_frontier_path", "data/processed/fairness_threshold_frontier.parquet", - ) - ) - challenger_report_path = Path( - outputs.get("challenger_promotion_report_path", "models/challenger_promotion_report.json") - ) - model_shift_status_path = Path( - outputs.get("model_shift_status_path", "models/model_shift_status.json") + ), + challenger_report_path=_output_path( + outputs, + "challenger_promotion_report_path", + "models/challenger_promotion_report.json", + ), + model_shift_status_path=_output_path( + outputs, + "model_shift_status_path", + "models/model_shift_status.json", + ), ) + +def _load_governance_frames() -> tuple[pd.DataFrame, pd.DataFrame]: train_df = read_split_with_fe_fallback("data/processed/train_fe.parquet") test_df = read_split_with_fe_fallback("data/processed/test_fe.parquet") training_record = _load_training_record() @@ -442,31 +573,44 @@ def main(config_path: str = "configs/mrm_policy.yaml", run_tag: str | None = Non regime_meta.get("mode", regime_cfg.get("mode", "standard")), len(train_df), ) + return train_df, test_df - features = _resolve_numeric_features(train_df, test_df) - if not features: - raise ValueError("No numeric features available for governance drift checks.") - logger.info("Governance drift checks on {} numeric features", len(features)) +def _write_parquet(df: pd.DataFrame, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + df.to_parquet(path, index=False) - drift_df = drift_monitoring_report( - train_df=train_df, - test_df=test_df, - features=features, - psi_threshold=psi_threshold, - ks_pvalue_threshold=ks_pvalue_min, - cvm_pvalue_threshold=cvm_pvalue_min, - n_bins=int(checks.get("psi_bins", 10)), - ) - c2st = classifier_two_sample_test( - train_df=train_df, - test_df=test_df, - features=features, - max_rows_per_split=c2st_max_rows, - random_state=int(checks.get("random_state", 42)), +def _load_json_dict(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except Exception: + return {} + return payload if isinstance(payload, dict) else {} + + +def _build_explanation_drift_if_available(thresholds: GovernanceThresholds) -> pd.DataFrame: + shap_raw_path = Path("data/processed/shap_raw_top20.parquet") + if not shap_raw_path.exists(): + return pd.DataFrame() + return _build_explanation_drift_report( + pd.read_parquet(shap_raw_path), + primary_threshold=_resolve_primary_threshold(), + min_rank_overlap_top10=thresholds.min_rank_overlap_top10, + max_shap_psi=thresholds.max_explanation_shap_psi, + min_reason_code_stability=thresholds.min_reason_code_stability, + min_rows_per_slice=thresholds.explanation_min_rows_per_slice, ) + +def _drift_breach_metrics( + drift_df: pd.DataFrame, + c2st: dict[str, Any], + performance_report: dict[str, float], + thresholds: GovernanceThresholds, +) -> dict[str, Any]: n_features = len(drift_df) psi_breaches = ( int((~drift_df.get("pass_psi", pd.Series(dtype=bool))).sum()) if n_features else 0 @@ -477,81 +621,77 @@ def main(config_path: str = "configs/mrm_policy.yaml", run_tag: str | None = Non ) feature_breach_ratio = float(psi_breaches / max(n_features, 1)) distribution_warning_ratio = float((ks_breaches + cvm_breaches) / max(n_features * 2, 1)) - - max_psi = float(drift_df["psi"].max()) if n_features else 0.0 - mean_psi = _safe_mean(drift_df["psi"]) if n_features else 0.0 - min_ks_pvalue = float(drift_df["ks_pvalue"].min()) if n_features else 1.0 - min_cvm_pvalue = float(drift_df["cvm_pvalue"].min()) if n_features else 1.0 - c2st_auc = _safe_float_value(c2st["c2st_auc"]) - c2st_materiality = str(c2st.get("materiality", "none")) - c2st_effective_driver_count = _safe_int_value(c2st.get("effective_driver_count", 0)) - c2st_top_drivers = _safe_list_value(c2st.get("top_drivers", [])) - performance_report = _score_and_performance_report( - train_df, - test_df, - random_state=int(checks.get("random_state", 42)), - max_rows_per_split=performance_max_rows, - psi_bins=int(checks.get("psi_bins", 10)), - ) score_psi = float(performance_report.get("score_psi", 0.0)) auc_delta = float(performance_report.get("auc_delta_train_to_test", 0.0)) brier_increase = float(performance_report.get("brier_increase_train_to_test", 0.0)) calibration_gap_delta = float(performance_report.get("calibration_gap_delta", 0.0)) - - pass_psi = bool(max_psi <= psi_threshold) - pass_breach_ratio = bool(feature_breach_ratio <= max_feature_breach_ratio) - pass_score_psi = bool(score_psi <= score_psi_max) - pass_auc_delta = bool(auc_delta <= auc_delta_max) - pass_brier_increase = bool(brier_increase <= brier_increase_max) - pass_calibration_gap_delta = bool(calibration_gap_delta <= calibration_gap_delta_max) - pass_c2st = bool(c2st_auc <= c2st_auc_max) - model_shift = interpret_model_shift( - c2st_auc=c2st_auc, - c2st_materiality=c2st_materiality, - score_psi=score_psi, - auc_delta=auc_delta, - brier_increase=brier_increase, - calibration_gap_delta=calibration_gap_delta, - distribution_warning_ratio=distribution_warning_ratio, - score_psi_max=score_psi_max, - auc_delta_max=auc_delta_max, - brier_increase_max=brier_increase_max, - calibration_gap_delta_max=calibration_gap_delta_max, + pass_psi = bool( + (float(drift_df["psi"].max()) if n_features else 0.0) <= thresholds.psi_threshold ) + pass_breach_ratio = bool(feature_breach_ratio <= thresholds.max_feature_breach_ratio) + pass_score_psi = bool(score_psi <= thresholds.score_psi_max) + pass_auc_delta = bool(auc_delta <= thresholds.auc_delta_max) + pass_brier_increase = bool(brier_increase <= thresholds.brier_increase_max) + pass_calibration_gap_delta = bool(calibration_gap_delta <= thresholds.calibration_gap_delta_max) + return { + "n_features": n_features, + "psi_breaches": psi_breaches, + "ks_breaches": ks_breaches, + "cvm_breaches": cvm_breaches, + "feature_breach_ratio": feature_breach_ratio, + "distribution_warning_ratio": distribution_warning_ratio, + "max_psi": float(drift_df["psi"].max()) if n_features else 0.0, + "mean_psi": _safe_mean(drift_df["psi"]) if n_features else 0.0, + "min_ks_pvalue": float(drift_df["ks_pvalue"].min()) if n_features else 1.0, + "min_cvm_pvalue": float(drift_df["cvm_pvalue"].min()) if n_features else 1.0, + "c2st_auc": _safe_float_value(c2st["c2st_auc"]), + "c2st_materiality": str(c2st.get("materiality", "none")), + "c2st_effective_driver_count": _safe_int_value(c2st.get("effective_driver_count", 0)), + "c2st_top_drivers": _safe_list_value(c2st.get("top_drivers", [])), + "c2st_rows_used": _safe_int_value(c2st.get("n_rows", 0)), + "score_psi": score_psi, + "auc_delta": auc_delta, + "brier_increase": brier_increase, + "calibration_gap_delta": calibration_gap_delta, + "pass_psi": pass_psi, + "pass_breach_ratio": pass_breach_ratio, + "pass_score_psi": pass_score_psi, + "pass_auc_delta": pass_auc_delta, + "pass_brier_increase": pass_brier_increase, + "pass_calibration_gap_delta": pass_calibration_gap_delta, + "pass_predictive_drift": bool( + pass_psi + and pass_breach_ratio + and pass_score_psi + and pass_auc_delta + and pass_brier_increase + and pass_calibration_gap_delta + ), + "pass_c2st": bool(_safe_float_value(c2st["c2st_auc"]) <= thresholds.c2st_auc_max), + "performance_report": performance_report, + } - drift_path.parent.mkdir(parents=True, exist_ok=True) - drift_df.to_parquet(drift_path, index=False) - shap_raw_path = Path("data/processed/shap_raw_top20.parquet") - explanation_drift = pd.DataFrame() - if shap_raw_path.exists(): - explanation_drift = _build_explanation_drift_report( - pd.read_parquet(shap_raw_path), - primary_threshold=_resolve_primary_threshold(), - min_rank_overlap_top10=min_rank_overlap_top10, - max_shap_psi=max_explanation_shap_psi, - min_reason_code_stability=min_reason_code_stability, - min_rows_per_slice=explanation_min_rows_per_slice, - ) - explanation_drift_path.parent.mkdir(parents=True, exist_ok=True) - explanation_drift.to_parquet(explanation_drift_path, index=False) +def _interpret_governance_shift( + metrics: dict[str, Any], + thresholds: GovernanceThresholds, +) -> dict[str, Any]: + return interpret_model_shift( + c2st_auc=float(metrics["c2st_auc"]), + c2st_materiality=str(metrics["c2st_materiality"]), + score_psi=float(metrics["score_psi"]), + auc_delta=float(metrics["auc_delta"]), + brier_increase=float(metrics["brier_increase"]), + calibration_gap_delta=float(metrics["calibration_gap_delta"]), + distribution_warning_ratio=float(metrics["distribution_warning_ratio"]), + score_psi_max=thresholds.score_psi_max, + auc_delta_max=thresholds.auc_delta_max, + brier_increase_max=thresholds.brier_increase_max, + calibration_gap_delta_max=thresholds.calibration_gap_delta_max, + ) - fairness_status = {} - if fairness_status_path.exists(): - try: - fairness_status = json.loads(fairness_status_path.read_text(encoding="utf-8")) - except Exception: - fairness_status = {} - fairness_pass = bool(fairness_status.get("overall_pass", False)) - - challenger_report = {} - if challenger_report_path.exists(): - try: - challenger_report = json.loads(challenger_report_path.read_text(encoding="utf-8")) - except Exception: - challenger_report = {} - challenger_promotable = bool(challenger_report.get("challenger_promotable", False)) +def _explanation_passes(explanation_drift: pd.DataFrame) -> dict[str, bool]: explainability_pass = bool( (not explanation_drift.empty) and explanation_drift["passed_all"].astype(bool).all() ) @@ -559,129 +699,154 @@ def main(config_path: str = "configs/mrm_policy.yaml", run_tag: str | None = Non (not explanation_drift.empty) and explanation_drift["pass_reason_code_stability"].astype(bool).all() ) - predictive_drift_pass = bool( - pass_psi - and pass_breach_ratio - and pass_score_psi - and pass_auc_delta - and pass_brier_increase - and pass_calibration_gap_delta - ) - overall_pass = bool(predictive_drift_pass and fairness_pass) - warning_flags = { - "warn_c2st": bool(not pass_c2st), - "warn_distribution_tests": bool(ks_breaches > 0 or cvm_breaches > 0), - "warn_explainability": bool(not explainability_pass), - "warn_reason_code_stability": bool(not reason_code_stability_pass), + return { + "explainability_pass": explainability_pass, + "reason_code_stability_pass": reason_code_stability_pass, } - top_breaches = drift_df.head(10).to_dict(orient="records") if n_features else [] - top_explanation_breaches = ( + +def _series_min_or_zero(df: pd.DataFrame, column: str) -> float: + return float(df[column].min()) if not df.empty else 0.0 + + +def _series_max_or_zero(df: pd.DataFrame, column: str) -> float: + return float(df[column].max()) if not df.empty else 0.0 + + +def _top_explanation_breaches(explanation_drift: pd.DataFrame) -> list[dict[str, Any]]: + if explanation_drift.empty: + return [] + return ( explanation_drift.sort_values( ["passed_all", "max_shap_psi_top5", "rank_overlap_top10"], ascending=[True, False, True], ) .head(10) .to_dict(orient="records") - if not explanation_drift.empty - else [] ) - status = { + + +def _fairness_primary_threshold(fairness_status: dict[str, Any]) -> float: + if not fairness_status: + return 0.5 + return _safe_float_value( + fairness_status.get("primary_threshold", fairness_status.get("prediction_threshold", 0.5)) + ) + + +def _warning_flags( + metrics: dict[str, Any], + explanation_checks: dict[str, bool], +) -> dict[str, bool]: + return { + "warn_c2st": bool(not metrics["pass_c2st"]), + "warn_distribution_tests": bool(metrics["ks_breaches"] > 0 or metrics["cvm_breaches"] > 0), + "warn_explainability": bool(not explanation_checks["explainability_pass"]), + "warn_reason_code_stability": bool(not explanation_checks["reason_code_stability_pass"]), + } + + +def _build_governance_status( + *, + config_path: str, + resolved_run_tag: str, + paths: GovernanceOutputPaths, + thresholds: GovernanceThresholds, + drift_df: pd.DataFrame, + explanation_drift: pd.DataFrame, + fairness_status: dict[str, Any], + challenger_report: dict[str, Any], + metrics: dict[str, Any], + model_shift: dict[str, Any], +) -> dict[str, Any]: + fairness_pass = bool(fairness_status.get("overall_pass", False)) + challenger_promotable = bool(challenger_report.get("challenger_promotable", False)) + explanation_checks = _explanation_passes(explanation_drift) + warning_flags = _warning_flags(metrics, explanation_checks) + overall_pass = bool(metrics["pass_predictive_drift"] and fairness_pass) + top_breaches = drift_df.head(10).to_dict(orient="records") if int(metrics["n_features"]) else [] + primary_threshold = _fairness_primary_threshold(fairness_status) + + return { "overall_pass": overall_pass, "checks": { - "pass_psi": pass_psi, - "pass_breach_ratio": pass_breach_ratio, - "pass_score_psi": pass_score_psi, - "pass_auc_delta": pass_auc_delta, - "pass_brier_increase": pass_brier_increase, - "pass_calibration_gap_delta": pass_calibration_gap_delta, - "pass_predictive_drift": predictive_drift_pass, + "pass_psi": bool(metrics["pass_psi"]), + "pass_breach_ratio": bool(metrics["pass_breach_ratio"]), + "pass_score_psi": bool(metrics["pass_score_psi"]), + "pass_auc_delta": bool(metrics["pass_auc_delta"]), + "pass_brier_increase": bool(metrics["pass_brier_increase"]), + "pass_calibration_gap_delta": bool(metrics["pass_calibration_gap_delta"]), + "pass_predictive_drift": bool(metrics["pass_predictive_drift"]), "pass_fairness": fairness_pass, - "pass_c2st": pass_c2st, - "pass_explainability": explainability_pass, - "pass_reason_code_stability": reason_code_stability_pass, + "pass_c2st": bool(metrics["pass_c2st"]), + "pass_explainability": explanation_checks["explainability_pass"], + "pass_reason_code_stability": explanation_checks["reason_code_stability_pass"], **warning_flags, }, "thresholds": { - "psi_threshold": psi_threshold, - "ks_pvalue_min": ks_pvalue_min, - "cvm_pvalue_min": cvm_pvalue_min, - "c2st_auc_max": c2st_auc_max, - "max_feature_breach_ratio": max_feature_breach_ratio, - "score_psi_max": score_psi_max, - "auc_delta_max": auc_delta_max, - "brier_increase_max": brier_increase_max, - "calibration_gap_delta_max": calibration_gap_delta_max, - "explanation_rank_overlap_top10_min": min_rank_overlap_top10, - "explanation_shap_psi_max": max_explanation_shap_psi, - "reason_code_stability_min": min_reason_code_stability, + "psi_threshold": thresholds.psi_threshold, + "ks_pvalue_min": thresholds.ks_pvalue_min, + "cvm_pvalue_min": thresholds.cvm_pvalue_min, + "c2st_auc_max": thresholds.c2st_auc_max, + "max_feature_breach_ratio": thresholds.max_feature_breach_ratio, + "score_psi_max": thresholds.score_psi_max, + "auc_delta_max": thresholds.auc_delta_max, + "brier_increase_max": thresholds.brier_increase_max, + "calibration_gap_delta_max": thresholds.calibration_gap_delta_max, + "explanation_rank_overlap_top10_min": thresholds.min_rank_overlap_top10, + "explanation_shap_psi_max": thresholds.max_explanation_shap_psi, + "reason_code_stability_min": thresholds.min_reason_code_stability, }, "summary": { - "n_features": n_features, - "max_psi": max_psi, - "mean_psi": mean_psi, - "min_ks_pvalue": min_ks_pvalue, - "min_cvm_pvalue": min_cvm_pvalue, - "c2st_auc": c2st_auc, - "psi_breaches": psi_breaches, - "ks_breaches": ks_breaches, - "cvm_breaches": cvm_breaches, - "feature_breach_ratio": feature_breach_ratio, - "distribution_warning_ratio": distribution_warning_ratio, - "c2st_rows_used": _safe_int_value(c2st.get("n_rows", 0)), - "c2st_materiality": c2st_materiality, - "c2st_effective_driver_count": c2st_effective_driver_count, - **performance_report, + "n_features": int(metrics["n_features"]), + "max_psi": float(metrics["max_psi"]), + "mean_psi": float(metrics["mean_psi"]), + "min_ks_pvalue": float(metrics["min_ks_pvalue"]), + "min_cvm_pvalue": float(metrics["min_cvm_pvalue"]), + "c2st_auc": float(metrics["c2st_auc"]), + "psi_breaches": int(metrics["psi_breaches"]), + "ks_breaches": int(metrics["ks_breaches"]), + "cvm_breaches": int(metrics["cvm_breaches"]), + "feature_breach_ratio": float(metrics["feature_breach_ratio"]), + "distribution_warning_ratio": float(metrics["distribution_warning_ratio"]), + "c2st_rows_used": int(metrics["c2st_rows_used"]), + "c2st_materiality": str(metrics["c2st_materiality"]), + "c2st_effective_driver_count": int(metrics["c2st_effective_driver_count"]), + **metrics["performance_report"], "n_explanation_segments": len(explanation_drift), - "min_rank_overlap_top10": float(explanation_drift["rank_overlap_top10"].min()) - if not explanation_drift.empty - else 0.0, - "max_explanation_shap_psi": float(explanation_drift["max_shap_psi_top5"].max()) - if not explanation_drift.empty - else 0.0, - "min_reason_code_stability": float(explanation_drift["reason_code_match_rate"].min()) - if not explanation_drift.empty - else 0.0, + "min_rank_overlap_top10": _series_min_or_zero(explanation_drift, "rank_overlap_top10"), + "max_explanation_shap_psi": _series_max_or_zero(explanation_drift, "max_shap_psi_top5"), + "min_reason_code_stability": _series_min_or_zero( + explanation_drift, "reason_code_match_rate" + ), "fairness_overall_pass": fairness_pass, - "fairness_primary_threshold": _safe_float_value( - fairness_status.get( - "primary_threshold", fairness_status.get("prediction_threshold", 0.5) - ) - ) - if fairness_status - else 0.5, + "fairness_primary_threshold": primary_threshold, "challenger_promotable": challenger_promotable, "model_shift_type": str(model_shift["shift_type"]), "governance_posture": str(model_shift["governance_posture"]), }, "warnings": warning_flags, "c2st": { - "auc": c2st_auc, - "materiality": c2st_materiality, - "effective_driver_count": c2st_effective_driver_count, - "top_drivers": c2st_top_drivers, + "auc": float(metrics["c2st_auc"]), + "materiality": str(metrics["c2st_materiality"]), + "effective_driver_count": int(metrics["c2st_effective_driver_count"]), + "top_drivers": metrics["c2st_top_drivers"], }, "model_shift": model_shift, "artifacts": { - "drift_monitoring_path": str(drift_path), - "explanation_drift_path": str(explanation_drift_path), - "fairness_status_path": str(fairness_status_path), - "fairness_frontier_path": str(fairness_frontier_path), - "challenger_promotion_report_path": str(challenger_report_path), - "model_shift_status_path": str(model_shift_status_path), + "drift_monitoring_path": str(paths.drift_path), + "explanation_drift_path": str(paths.explanation_drift_path), + "fairness_status_path": str(paths.fairness_status_path), + "fairness_frontier_path": str(paths.fairness_frontier_path), + "challenger_promotion_report_path": str(paths.challenger_report_path), + "model_shift_status_path": str(paths.model_shift_status_path), }, "top_drift_features": top_breaches, - "top_explanation_breaches": top_explanation_breaches, - "primary_threshold": _safe_float_value( - fairness_status.get( - "primary_threshold", fairness_status.get("prediction_threshold", 0.5) - ) - ) - if fairness_status - else 0.5, - "explainability_pass": explainability_pass, - "explanation_drift_pass": explainability_pass, - "reason_code_stability_pass": reason_code_stability_pass, + "top_explanation_breaches": _top_explanation_breaches(explanation_drift), + "primary_threshold": primary_threshold, + "explainability_pass": explanation_checks["explainability_pass"], + "explanation_drift_pass": explanation_checks["explainability_pass"], + "reason_code_stability_pass": explanation_checks["reason_code_stability_pass"], "challenger_promotable": challenger_promotable, "policy_config": config_path, **build_artifact_metadata( @@ -691,41 +856,131 @@ def main(config_path: str = "configs/mrm_policy.yaml", run_tag: str | None = Non ), } - status_path.parent.mkdir(parents=True, exist_ok=True) - with open(status_path, "w", encoding="utf-8") as f: - json.dump(status, f, indent=2) - - model_shift_status_path.parent.mkdir(parents=True, exist_ok=True) - model_shift_status_path.write_text( - json.dumps( - { - "diagnostic_only": True, - "overall_pass": bool(model_shift["governance_posture"] != "candidate_gate"), - "summary": model_shift, - "artifacts": {"governance_status_path": str(status_path)}, - **build_artifact_metadata( - schema_version=SCHEMA_VERSION, - run_tag=resolved_run_tag, - require_explicit=True, - ), - }, - indent=2, - ), - encoding="utf-8", + +def _write_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2), encoding="utf-8") + + +def _write_model_shift_status( + *, + path: Path, + status_path: Path, + model_shift: dict[str, Any], + resolved_run_tag: str, +) -> None: + _write_json( + path, + { + "diagnostic_only": True, + "overall_pass": bool(model_shift["governance_posture"] != "candidate_gate"), + "summary": model_shift, + "artifacts": {"governance_status_path": str(status_path)}, + **build_artifact_metadata( + schema_version=SCHEMA_VERSION, + run_tag=resolved_run_tag, + require_explicit=True, + ), + }, ) - logger.info("Saved drift monitoring: {}", drift_path) - logger.info("Saved governance status: {}", status_path) - logger.info("Saved model-shift interpretation: {}", model_shift_status_path) + +def _log_governance_outputs( + *, + paths: GovernanceOutputPaths, + status: dict[str, Any], + metrics: dict[str, Any], +) -> None: + logger.info("Saved drift monitoring: {}", paths.drift_path) + logger.info("Saved governance status: {}", paths.status_path) + logger.info("Saved model-shift interpretation: {}", paths.model_shift_status_path) logger.info( "Governance checks pass={} (max_psi={:.4f}, score_psi={:.4f}, auc_delta={:.4f}, brier_increase={:.4f}, c2st_auc={:.4f})", - overall_pass, - max_psi, - score_psi, - auc_delta, - brier_increase, - c2st_auc, + status["overall_pass"], + metrics["max_psi"], + metrics["score_psi"], + metrics["auc_delta"], + metrics["brier_increase"], + metrics["c2st_auc"], + ) + + +def main(config_path: str = "configs/mrm_policy.yaml", run_tag: str | None = None) -> None: + cfg = _load_cfg(config_path) + semantics = load_threshold_semantics() + resolved_run_tag = resolve_run_tag( + run_tag, + fallback_candidates=[semantics.get("run_tag"), resolve_official_baseline_run_tag()], + require_explicit=True, + ) + + triggers = cfg.get("retraining_triggers", {}) + checks = cfg.get("governance_checks", {}) + outputs = cfg.get("governance_output", {}) + thresholds = _resolve_thresholds(triggers, checks) + paths = _resolve_output_paths(outputs) + + train_df, test_df = _load_governance_frames() + + features = _resolve_numeric_features(train_df, test_df) + if not features: + raise ValueError("No numeric features available for governance drift checks.") + + logger.info("Governance drift checks on {} numeric features", len(features)) + + drift_df = drift_monitoring_report( + train_df=train_df, + test_df=test_df, + features=features, + psi_threshold=thresholds.psi_threshold, + ks_pvalue_threshold=thresholds.ks_pvalue_min, + cvm_pvalue_threshold=thresholds.cvm_pvalue_min, + n_bins=thresholds.psi_bins, + ) + + c2st = classifier_two_sample_test( + train_df=train_df, + test_df=test_df, + features=features, + max_rows_per_split=thresholds.c2st_max_rows, + random_state=thresholds.random_state, + ) + performance_report = _score_and_performance_report( + train_df, + test_df, + random_state=thresholds.random_state, + max_rows_per_split=thresholds.performance_max_rows, + psi_bins=thresholds.psi_bins, + ) + metrics = _drift_breach_metrics(drift_df, c2st, performance_report, thresholds) + model_shift = _interpret_governance_shift(metrics, thresholds) + + _write_parquet(drift_df, paths.drift_path) + explanation_drift = _build_explanation_drift_if_available(thresholds) + _write_parquet(explanation_drift, paths.explanation_drift_path) + + fairness_status = _load_json_dict(paths.fairness_status_path) + challenger_report = _load_json_dict(paths.challenger_report_path) + status = _build_governance_status( + config_path=config_path, + resolved_run_tag=resolved_run_tag, + paths=paths, + thresholds=thresholds, + drift_df=drift_df, + explanation_drift=explanation_drift, + fairness_status=fairness_status, + challenger_report=challenger_report, + metrics=metrics, + model_shift=model_shift, + ) + _write_json(paths.status_path, status) + _write_model_shift_status( + path=paths.model_shift_status_path, + status_path=paths.status_path, + model_shift=model_shift, + resolved_run_tag=resolved_run_tag, ) + _log_governance_outputs(paths=paths, status=status, metrics=metrics) if __name__ == "__main__": diff --git a/scripts/generate_mrm_report.py b/scripts/generate_mrm_report.py index beb74f8..92627a1 100644 --- a/scripts/generate_mrm_report.py +++ b/scripts/generate_mrm_report.py @@ -145,20 +145,18 @@ def _build_skops_sidecar( card.add_metrics( section="Validation", description="Pipeline-level subsystem gates consumed by MRM.", - metrics={ - "pipeline_overall_pass": bool( - statuses.get("pipeline", {}).get("overall_pass", False) - ), - "conformal_overall_pass": bool( - statuses.get("conformal", {}).get("overall_pass", False) - ), - "governance_overall_pass": bool( - statuses.get("governance", {}).get("overall_pass", False) - ), - "fairness_overall_pass": bool( - statuses.get("fairness", {}).get("overall_pass", False) - ), - }, + pipeline_overall_pass=int( + bool(statuses.get("pipeline", {}).get("overall_pass", False)) + ), + conformal_overall_pass=int( + bool(statuses.get("conformal", {}).get("overall_pass", False)) + ), + governance_overall_pass=int( + bool(statuses.get("governance", {}).get("overall_pass", False)) + ), + fairness_overall_pass=int( + bool(statuses.get("fairness", {}).get("overall_pass", False)) + ), ) card.add_hyperparams() model_card_md = card.render() diff --git a/scripts/optimize_portfolio.py b/scripts/optimize_portfolio.py index 418d306..315feeb 100644 --- a/scripts/optimize_portfolio.py +++ b/scripts/optimize_portfolio.py @@ -15,7 +15,11 @@ from loguru import logger from src.models.conformal_artifacts import load_conformal_intervals -from src.optimization.portfolio_model import optimize_portfolio_allocation +from src.optimization.input_alignment import align_candidate_intervals +from src.optimization.portfolio_model import ( + optimize_portfolio_allocation, + solution_allocation_vector, +) from src.optimization.robust_opt import scenario_analysis from src.utils.pipeline_runtime import ( atomic_write_parquet, @@ -76,55 +80,33 @@ def _align_candidates_and_intervals( random_state: int = 42, ) -> tuple[pd.DataFrame, np.ndarray, np.ndarray, np.ndarray]: """Align loans with PD intervals using ID when available, with legacy fallback.""" - col_point, col_low, col_high = _resolve_interval_columns(intervals) - max_candidates_norm = ( - None if max_candidates is None or int(max_candidates) <= 0 else int(max_candidates) + aligned = align_candidate_intervals( + candidates, + intervals, + max_candidates=max_candidates, + random_state=random_state, ) - base_n = min(len(candidates), len(intervals)) - if max_candidates_norm is not None: - base_n = min(base_n, max_candidates_norm) - - if "id" in candidates.columns and "id" in intervals.columns: - cand = candidates.copy() - ints = intervals.copy() - cand["_id_join"] = cand["id"].astype(str) - ints["_id_join"] = ints["id"].astype(str) - ints = ints.drop_duplicates(subset="_id_join", keep="first") - merged = cand.merge( - ints[["_id_join", col_point, col_low, col_high]], - on="_id_join", - how="inner", - ) - if merged.empty: - raise ValueError("ID-based merge between candidates and intervals returned zero rows.") - n = len(merged) if max_candidates_norm is None else min(len(merged), max_candidates_norm) - if len(merged) > n: - rng = np.random.default_rng(random_state) - idx = np.sort(rng.choice(np.arange(len(merged)), size=n, replace=False)) - merged = merged.iloc[idx].reset_index(drop=True) - else: - merged = merged.reset_index(drop=True) - - aligned_loans = merged[candidates.columns].copy() - pd_point = merged[col_point].to_numpy(dtype=float) - pd_low = merged[col_low].to_numpy(dtype=float) - pd_high = merged[col_high].to_numpy(dtype=float) - logger.info( - f"Aligned candidates and intervals by id: n={len(aligned_loans):,} " - f"(candidate_rows={len(candidates):,}, interval_rows={len(intervals):,})" + col_point, col_low, col_high = _resolve_interval_columns(aligned.intervals) + if aligned.mode == "position": + logger.warning( + "Conformal interval artifact has no id or _row_number alignment key; " + "using reproducible positional sampling for optimization." ) - return aligned_loans, pd_point, pd_low, pd_high - - # Legacy fallback where interval artifact has no stable key. - logger.warning( - "Conformal interval artifact has no id alignment key; using positional fallback for optimization." + logger.info( + "Aligned candidates and intervals by {}: n={:,} " + "(alignable_rows={:,}, candidate_rows={:,}, interval_rows={:,})", + aligned.mode, + aligned.selected_rows, + aligned.available_rows, + len(candidates), + len(intervals), + ) + return ( + aligned.candidates, + aligned.intervals[col_point].to_numpy(dtype=float), + aligned.intervals[col_low].to_numpy(dtype=float), + aligned.intervals[col_high].to_numpy(dtype=float), ) - aligned_loans = candidates.head(base_n).reset_index(drop=True).copy() - ints = intervals.head(base_n).reset_index(drop=True) - pd_point = ints[col_point].to_numpy(dtype=float) - pd_low = ints[col_low].to_numpy(dtype=float) - pd_high = ints[col_high].to_numpy(dtype=float) - return aligned_loans, pd_point, pd_low, pd_high def main( @@ -188,7 +170,7 @@ def main( ) write_runtime_status(stage_name, phase="optimization_complete", state="running") - allocation = np.array([solution["allocation"][i] for i in range(n)], dtype=float) + allocation = solution_allocation_vector(solution, n) loan_amounts = ( test_sample["loan_amnt"].to_numpy(dtype=float) if "loan_amnt" in test_sample.columns diff --git a/scripts/optimize_portfolio_tradeoff.py b/scripts/optimize_portfolio_tradeoff.py index 7927d62..7ca951c 100644 --- a/scripts/optimize_portfolio_tradeoff.py +++ b/scripts/optimize_portfolio_tradeoff.py @@ -22,9 +22,11 @@ from loguru import logger from src.models.conformal_artifacts import load_conformal_intervals +from src.optimization.input_alignment import align_candidate_intervals from src.optimization.portfolio_model import ( compute_effective_pd, optimize_portfolio_allocation, + solution_allocation_vector, ) from src.utils.artifact_metadata import resolve_run_tag from src.utils.pipeline_runtime import ( @@ -99,69 +101,27 @@ def _align_loans_and_intervals( random_state: int, ) -> tuple[pd.DataFrame, pd.DataFrame]: """Align candidate loans with interval rows by id where possible.""" - max_candidates_norm = None if int(max_candidates) <= 0 else int(max_candidates) - if "id" in candidates.columns and "id" in intervals.columns: - cand = candidates.copy() - ints = intervals.copy() - cand["_id_join"] = cand["id"].astype(str) - ints["_id_join"] = ints["id"].astype(str) - ints = ints.drop_duplicates(subset="_id_join", keep="first") - merged = cand.merge(ints, on="_id_join", how="inner", suffixes=("", "_int")) - if merged.empty: - raise ValueError("ID-based merge between candidates and intervals returned zero rows.") - - n = len(merged) if max_candidates_norm is None else min(len(merged), max_candidates_norm) - if len(merged) > n: - idx = np.random.default_rng(random_state).choice( - np.arange(len(merged)), size=n, replace=False - ) - idx = np.sort(idx) - merged = merged.iloc[idx].reset_index(drop=True) - else: - merged = merged.reset_index(drop=True) - - loans = merged[candidates.columns].copy() - interval_cols = [c for c in intervals.columns if c in merged.columns] - ints_aligned = merged[interval_cols].copy() - logger.info( - f"Aligned tradeoff candidates and intervals by id: n={len(loans):,} " - f"(candidate_rows={len(candidates):,}, interval_rows={len(intervals):,})" - ) - return loans, ints_aligned - - if "_row_number" in intervals.columns: - cand = candidates.copy() - ints = intervals.copy() - cand["_row_number"] = np.arange(len(cand)) - merged = cand.merge(ints, on="_row_number", how="inner", suffixes=("", "_int")) - assert len(merged) == len(cand), ( - f"_row_number merge size mismatch: {len(merged)} != {len(cand)}" + aligned = align_candidate_intervals( + candidates, + intervals, + max_candidates=max_candidates, + random_state=random_state, + ) + if aligned.mode == "position": + logger.warning( + "Conformal interval artifact has no id or _row_number alignment key; " + "using reproducible positional sampling in tradeoff analysis." ) - n = len(merged) if max_candidates_norm is None else min(len(merged), max_candidates_norm) - if len(merged) > n: - idx = np.random.default_rng(random_state).choice( - np.arange(len(merged)), size=n, replace=False - ) - merged = merged.iloc[np.sort(idx)].reset_index(drop=True) - else: - merged = merged.reset_index(drop=True) - loans = merged[candidates.columns].copy() - interval_cols = [c for c in intervals.columns if c in merged.columns] - ints_aligned = merged[interval_cols].copy() - logger.info(f"Aligned tradeoff candidates and intervals by _row_number: n={len(loans):,}") - return loans, ints_aligned - - logger.warning( - "Conformal interval artifact has no id or _row_number alignment key; using positional fallback in tradeoff analysis." + logger.info( + "Aligned tradeoff candidates and intervals by {}: n={:,} " + "(alignable_rows={:,}, candidate_rows={:,}, interval_rows={:,})", + aligned.mode, + aligned.selected_rows, + aligned.available_rows, + len(candidates), + len(intervals), ) - n = min(len(candidates), len(intervals)) - if max_candidates_norm is not None: - n = min(n, max_candidates_norm) - idx = np.random.default_rng(random_state).choice(np.arange(n), size=n, replace=False) - idx = np.sort(idx) - loans = candidates.iloc[idx].reset_index(drop=True).copy() - ints_aligned = intervals.iloc[idx].reset_index(drop=True).copy() - return loans, ints_aligned + return aligned.candidates, aligned.intervals def _write_candidate_universe(loans: pd.DataFrame, *, path: str, run_tag: str) -> None: @@ -327,8 +287,12 @@ def _solve_single( cuopt_presolve: int | None = 1, cuopt_parameters: dict[str, Any] | None = None, ) -> tuple[dict[str, float | int | str], np.ndarray]: + effective_policy_mode = str(policy_mode) if robust else "point_estimate" + effective_gamma = float(gamma) if robust else 0.0 + effective_delta_cap = float(delta_cap_quantile) if robust else 1.0 + effective_tail_focus = float(tail_focus_quantile) if robust else 1.0 segment_labels: np.ndarray | None = None - if policy_mode in { + if effective_policy_mode in { "segment_tail_blended_uncertainty", "segment_relative_tail_blended_uncertainty", }: @@ -351,10 +315,10 @@ def _solve_single( pd_constraint = compute_effective_pd( pd_point=pd_point, pd_high=pd_high, - policy_mode=policy_mode, - gamma=gamma, - delta_cap_quantile=delta_cap_quantile, - tail_focus_quantile=tail_focus_quantile, + policy_mode=effective_policy_mode, + gamma=effective_gamma, + delta_cap_quantile=effective_delta_cap, + tail_focus_quantile=effective_tail_focus, segment_labels=segment_labels, ) solution = optimize_portfolio_allocation( @@ -381,7 +345,7 @@ def _solve_single( ) n = len(loans) - allocation = np.array([solution["allocation"][i] for i in range(n)], dtype=float) + allocation = solution_allocation_vector(solution, n) loan_amounts = ( loans["loan_amnt"].to_numpy(dtype=float) if "loan_amnt" in loans.columns @@ -394,7 +358,7 @@ def _solve_single( expected_return = float(np.sum(allocation * loan_amounts * int_rates)) economic_return = expected_return - expected_loss realized_total_return = _compute_realized_total_return( - solution["allocation"], + allocation, loan_amounts, int_rates, default_flag if default_flag is not None else np.zeros(n, dtype=int), @@ -410,10 +374,10 @@ def _solve_single( return { "solver_status": str(solution["solver_status"]), "solver_backend": str(solver_backend), - "policy_mode": str(policy_mode), - "gamma": float(gamma), - "delta_cap_quantile": float(delta_cap_quantile), - "tail_focus_quantile": float(tail_focus_quantile), + "policy_mode": effective_policy_mode, + "gamma": effective_gamma, + "delta_cap_quantile": effective_delta_cap, + "tail_focus_quantile": effective_tail_focus, "objective_value": float(solution["objective_value"]), "n_funded": int(solution["n_funded"]), "total_allocated": total_allocated, @@ -430,23 +394,16 @@ def _solve_single( def _compute_realized_total_return( - allocation: dict[int, float], + allocation: np.ndarray, loan_amounts: np.ndarray, int_rates: np.ndarray, default_flag: np.ndarray, *, lgd: float = 0.45, ) -> float: - total = 0.0 - for i in range(len(loan_amounts)): - alloc = float(allocation.get(i, 0.0)) - if alloc <= 0.01: - continue - if int(default_flag[i]) == 1: - total += alloc * float(loan_amounts[i]) * (-lgd) - else: - total += alloc * float(loan_amounts[i]) * float(int_rates[i]) - return float(total) + funded = allocation > 0.01 + realized_rate = np.where(default_flag.astype(int) == 1, -float(lgd), int_rates) + return float(np.sum(allocation[funded] * loan_amounts[funded] * realized_rate[funded])) def _allocation_similarity(a: np.ndarray, b: np.ndarray) -> float: diff --git a/scripts/run_comparison.py b/scripts/run_comparison.py index 02135eb..c9eea75 100644 --- a/scripts/run_comparison.py +++ b/scripts/run_comparison.py @@ -310,6 +310,15 @@ class GateResult: details: dict[str, Any] +@dataclass +class StatusMetadataObservation: + row: dict[str, Any] + run_tag: str | None + generated_at: datetime | None + missing_metadata: bool + mismatched_run_tag: bool + + def _load_fairness_policy_contract(config_path: Path = FAIRNESS_POLICY_PATH) -> dict[str, Any]: """Return fairness business threshold contract from policy config.""" if not config_path.exists(): @@ -358,9 +367,7 @@ def _parse_iso_datetime(value: Any) -> datetime | None: return None -def _collect_status_metadata( - cur_metrics: dict[str, Any], *, expected_run_tag: str -) -> dict[str, Any]: +def _status_metadata_sources(cur_metrics: dict[str, Any]) -> dict[str, Any]: sources = { "reports/dvc/metrics_summary.json": cur_metrics.get("dvc_metrics_meta", {}), "data/processed/pipeline_summary.json": cur_metrics.get("pipeline_summary", {}), @@ -379,80 +386,140 @@ def _collect_status_metadata( for artifact_name, payload in optional_sources.items(): if isinstance(payload, dict) and payload: sources[artifact_name] = payload - rows: list[dict[str, Any]] = [] - run_tags: list[str] = [] - generated_times: list[datetime] = [] - missing_metadata_artifacts: list[str] = [] - mismatched_run_tag_artifacts: list[str] = [] - - for artifact_name, payload in sources.items(): - payload = payload if isinstance(payload, dict) else {} - schema_version = payload.get("schema_version") - generated_at_utc = payload.get("generated_at_utc") - run_tag = payload.get("run_tag") - missing_fields = [ - key - for key, value in { - "schema_version": schema_version, - "generated_at_utc": generated_at_utc, - "run_tag": run_tag, - }.items() - if value in (None, "", []) - ] - parsed_dt = _parse_iso_datetime(generated_at_utc) - if run_tag not in (None, "", []): - run_tags.append(str(run_tag)) - if parsed_dt is not None: - generated_times.append(parsed_dt) - if missing_fields: - missing_metadata_artifacts.append(artifact_name) - if run_tag not in (None, "", []) and str(run_tag) != expected_run_tag: - mismatched_run_tag_artifacts.append(artifact_name) - rows.append( - { - "artifact": artifact_name, - "schema_version": schema_version, - "generated_at_utc": generated_at_utc, - "run_tag": run_tag, - "missing_metadata_fields": missing_fields, - } - ) + return sources + + +def _missing_status_fields( + *, + schema_version: Any, + generated_at_utc: Any, + run_tag: Any, +) -> list[str]: + return [ + key + for key, value in { + "schema_version": schema_version, + "generated_at_utc": generated_at_utc, + "run_tag": run_tag, + }.items() + if value in (None, "", []) + ] + + +def _status_metadata_observation( + artifact_name: str, + payload: Any, + *, + expected_run_tag: str, +) -> StatusMetadataObservation: + payload = payload if isinstance(payload, dict) else {} + schema_version = payload.get("schema_version") + generated_at_utc = payload.get("generated_at_utc") + run_tag = payload.get("run_tag") + missing_fields = _missing_status_fields( + schema_version=schema_version, + generated_at_utc=generated_at_utc, + run_tag=run_tag, + ) + run_tag_text = None if run_tag in (None, "", []) else str(run_tag) + return StatusMetadataObservation( + row={ + "artifact": artifact_name, + "schema_version": schema_version, + "generated_at_utc": generated_at_utc, + "run_tag": run_tag, + "missing_metadata_fields": missing_fields, + }, + run_tag=run_tag_text, + generated_at=_parse_iso_datetime(generated_at_utc), + missing_metadata=bool(missing_fields), + mismatched_run_tag=run_tag_text is not None and run_tag_text != expected_run_tag, + ) + +def _timestamp_skew_seconds(generated_times: list[datetime]) -> float | None: + if len(generated_times) < 2: + return None + return float((max(generated_times) - min(generated_times)).total_seconds()) + + +def _run_tag_coherence( + *, + run_tags: list[str], + mismatched_artifacts: list[str], + expected_run_tag: str, +) -> dict[str, Any]: unique_run_tags = sorted(set(run_tags)) run_tag_consistent = len(unique_run_tags) == 1 run_tag_matches_expected = run_tag_consistent and unique_run_tags == [expected_run_tag] - timestamp_skew_seconds = None - if len(generated_times) >= 2: - timestamp_skew_seconds = float( - (max(generated_times) - min(generated_times)).total_seconds() + non_causal_mismatches = [ + artifact for artifact in mismatched_artifacts if artifact not in _CAUSAL_INSIGHTS_ARTIFACTS + ] + causal_only_mismatch = bool(mismatched_artifacts) and len(non_causal_mismatches) == 0 + return { + "run_tags_observed": unique_run_tags, + "run_tag_consistent": run_tag_consistent, + "run_tag_matches_expected": run_tag_matches_expected, + "run_tag_matches_expected_operational": run_tag_matches_expected or causal_only_mismatch, + "causal_only_mismatch": causal_only_mismatch, + "non_causal_mismatched_run_tag_artifacts": non_causal_mismatches, + } + + +def _collect_status_metadata( + cur_metrics: dict[str, Any], *, expected_run_tag: str +) -> dict[str, Any]: + observations = [ + _status_metadata_observation( + artifact_name, + payload, + expected_run_tag=expected_run_tag, ) + for artifact_name, payload in _status_metadata_sources(cur_metrics).items() + ] + rows = [observation.row for observation in observations] + run_tags = [ + observation.run_tag for observation in observations if observation.run_tag is not None + ] + generated_times = [ + observation.generated_at + for observation in observations + if observation.generated_at is not None + ] + missing_metadata_artifacts = [ + str(observation.row["artifact"]) + for observation in observations + if observation.missing_metadata + ] + mismatched_run_tag_artifacts = [ + str(observation.row["artifact"]) + for observation in observations + if observation.mismatched_run_tag + ] + + tag_coherence = _run_tag_coherence( + run_tags=run_tags, + mismatched_artifacts=mismatched_run_tag_artifacts, + expected_run_tag=expected_run_tag, + ) + timestamp_skew_seconds = _timestamp_skew_seconds(generated_times) timestamp_coherent = timestamp_skew_seconds is None or timestamp_skew_seconds <= float( COHERENCE_TIMESTAMP_MAX_SKEW_SECONDS ) all_have_metadata = len(missing_metadata_artifacts) == 0 - - # Allow mismatches that are exclusively causal/CATE insights_only artifacts — - # these are documented as not-regenerated in every run by design. - non_causal_mismatches = [ - a for a in mismatched_run_tag_artifacts if a not in _CAUSAL_INSIGHTS_ARTIFACTS - ] - causal_only_mismatch = bool(mismatched_run_tag_artifacts) and len(non_causal_mismatches) == 0 - run_tag_matches_expected_operational = run_tag_matches_expected or causal_only_mismatch - - passed = bool(all_have_metadata and run_tag_matches_expected_operational and timestamp_coherent) + passed = bool( + all_have_metadata + and tag_coherence["run_tag_matches_expected_operational"] + and timestamp_coherent + ) return { "expected_run_tag": expected_run_tag, "critical_artifacts": rows, "all_have_metadata": all_have_metadata, "missing_metadata_artifacts": missing_metadata_artifacts, - "run_tags_observed": unique_run_tags, - "run_tag_consistent": run_tag_consistent, - "run_tag_matches_expected": run_tag_matches_expected, - "run_tag_matches_expected_operational": run_tag_matches_expected_operational, - "causal_only_mismatch": causal_only_mismatch, + **tag_coherence, "mismatched_run_tag_artifacts": mismatched_run_tag_artifacts, - "non_causal_mismatched_run_tag_artifacts": non_causal_mismatches, "timestamp_skew_seconds": timestamp_skew_seconds, "timestamp_coherent": bool(timestamp_coherent), "timestamp_max_skew_seconds": int(COHERENCE_TIMESTAMP_MAX_SKEW_SECONDS), @@ -661,43 +728,104 @@ def _gate_conformal(base: dict[str, Any], cur: dict[str, Any]) -> GateResult: ) -def _gate_ab_no_regression(base: dict[str, Any], cur: dict[str, Any]) -> GateResult: - b = base.get("ab_simulation_status", {}) - c = cur.get("ab_simulation_status", {}) +def _ab_total_returns(status: dict[str, Any]) -> tuple[float, float]: + return ( + _safe_float((status.get("metrics_a") or {}).get("total_return")), + _safe_float((status.get("metrics_b") or {}).get("total_return")), + ) - b_a = _safe_float((b.get("metrics_a") or {}).get("total_return")) - b_b = _safe_float((b.get("metrics_b") or {}).get("total_return")) - c_a = _safe_float((c.get("metrics_a") or {}).get("total_return")) - c_b = _safe_float((c.get("metrics_b") or {}).get("total_return")) - no_reg = c.get("no_regression", {}) if isinstance(c.get("no_regression"), dict) else {} +def _ab_current_no_regression( + status: dict[str, Any], + control_return: float, + robust_return: float, +) -> tuple[bool, float, float, dict[str, Any]]: + no_reg = ( + status.get("no_regression", {}) if isinstance(status.get("no_regression"), dict) else {} + ) cross_gate = ( - c.get("cross_scenario_gate", {}) if isinstance(c.get("cross_scenario_gate"), dict) else {} + status.get("cross_scenario_gate", {}) + if isinstance(status.get("cross_scenario_gate"), dict) + else {} ) - c_diff = _safe_float( + diff = _safe_float( no_reg.get("diff_total_return"), - default=(c_b - c_a if np.isfinite(c_a) and np.isfinite(c_b) else float("nan")), + default=( + robust_return - control_return + if np.isfinite(control_return) and np.isfinite(robust_return) + else float("nan") + ), ) - c_tol = _safe_float( + tolerance = _safe_float( no_reg.get("tolerance_total_return"), - default=(abs(c_a) * 0.05 if np.isfinite(c_a) else float("nan")), + default=(abs(control_return) * 0.05 if np.isfinite(control_return) else float("nan")), ) - - self_no_reg_ok = ( + passed = ( bool(no_reg.get("passed")) if "passed" in no_reg - else (np.isnan(c_diff) or np.isnan(c_tol) or (c_diff >= -c_tol)) + else (np.isnan(diff) or np.isnan(tolerance) or (diff >= -tolerance)) ) - if str(c.get("decision_scenario", "")).strip() == "selective_ambiguity_defer" and bool( + if str(status.get("decision_scenario", "")).strip() == "selective_ambiguity_defer" and bool( cross_gate.get("passed", False) ): - self_no_reg_ok = True + passed = True + return bool(passed), float(diff), float(tolerance), cross_gate + + +def _ab_baseline_checks( + *, + baseline_control_return: float, + baseline_robust_return: float, + current_control_return: float, + current_robust_return: float, + current_diff: float, +) -> tuple[dict[str, bool], float]: + baseline_diff = ( + baseline_robust_return - baseline_control_return + if np.isfinite(baseline_control_return) and np.isfinite(baseline_robust_return) + else float("nan") + ) + baseline_tol = ( + abs(baseline_control_return) * 0.05 + if np.isfinite(baseline_control_return) + else float("nan") + ) + return ( + { + "control_vs_baseline_ok": bool( + np.isnan(baseline_control_return) + or np.isnan(current_control_return) + or (current_control_return >= baseline_control_return - baseline_tol) + ), + "robust_vs_baseline_ok": bool( + np.isnan(baseline_robust_return) + or np.isnan(current_robust_return) + or (current_robust_return >= baseline_robust_return - baseline_tol) + ), + "gap_vs_baseline_ok": bool( + np.isnan(baseline_diff) + or np.isnan(current_diff) + or (current_diff >= baseline_diff - baseline_tol) + ), + }, + float(baseline_diff), + ) - b_diff = b_b - b_a if np.isfinite(b_a) and np.isfinite(b_b) else float("nan") - baseline_tol = abs(b_a) * 0.05 if np.isfinite(b_a) else float("nan") - control_vs_baseline_ok = np.isnan(b_a) or np.isnan(c_a) or (c_a >= b_a - baseline_tol) - robust_vs_baseline_ok = np.isnan(b_b) or np.isnan(c_b) or (c_b >= b_b - baseline_tol) - gap_vs_baseline_ok = np.isnan(b_diff) or np.isnan(c_diff) or (c_diff >= b_diff - baseline_tol) + +def _gate_ab_no_regression(base: dict[str, Any], cur: dict[str, Any]) -> GateResult: + b = base.get("ab_simulation_status", {}) + c = cur.get("ab_simulation_status", {}) + + b_a, b_b = _ab_total_returns(b) + c_a, c_b = _ab_total_returns(c) + self_no_reg_ok, c_diff, c_tol, cross_gate = _ab_current_no_regression(c, c_a, c_b) + baseline_checks, b_diff = _ab_baseline_checks( + baseline_control_return=b_a, + baseline_robust_return=b_b, + current_control_return=c_a, + current_robust_return=c_b, + current_diff=c_diff, + ) passed = bool(self_no_reg_ok) @@ -709,14 +837,12 @@ def _gate_ab_no_regression(base: dict[str, Any], cur: dict[str, Any]) -> GateRes "checks": { "self_no_regression_ok": bool(self_no_reg_ok), "cross_scenario_gate_ok": bool(cross_gate.get("passed", False)), - "control_vs_baseline_ok": bool(control_vs_baseline_ok), - "robust_vs_baseline_ok": bool(robust_vs_baseline_ok), - "gap_vs_baseline_ok": bool(gap_vs_baseline_ok), + **baseline_checks, }, "warnings": { - "control_vs_baseline_warning": bool(not control_vs_baseline_ok), - "robust_vs_baseline_warning": bool(not robust_vs_baseline_ok), - "gap_vs_baseline_warning": bool(not gap_vs_baseline_ok), + "control_vs_baseline_warning": bool(not baseline_checks["control_vs_baseline_ok"]), + "robust_vs_baseline_warning": bool(not baseline_checks["robust_vs_baseline_ok"]), + "gap_vs_baseline_warning": bool(not baseline_checks["gap_vs_baseline_ok"]), }, "current": { "control_total_return": c_a, @@ -930,81 +1056,104 @@ def _write_snapshot(run_tag: str) -> Path: return path -def _write_compare(run_tag: str, baseline_path: Path) -> tuple[Path, Path]: - baseline_path = baseline_path.expanduser().resolve() - baseline = json.loads(baseline_path.read_text(encoding="utf-8")) - current = _snapshot_payload(run_tag) - gate_results = [ - _gate_artifact_coherence(current["metrics"], run_tag), - _gate_semantic_coherence(current["metrics"]), - _gate_pd(baseline["metrics"], current["metrics"]), - _gate_conformal(baseline["metrics"], current["metrics"]), - _gate_ab_no_regression(baseline["metrics"], current["metrics"]), - _gate_fairness(baseline["metrics"], current["metrics"]), - _gate_fairness_absolute_business(baseline["metrics"], current["metrics"]), - _gate_survival(baseline["metrics"], current["metrics"]), - _gate_exports(current), +def _comparison_gate_results( + *, + baseline_metrics: dict[str, Any], + current_metrics: dict[str, Any], + current_snapshot: dict[str, Any], + run_tag: str, +) -> list[GateResult]: + return [ + _gate_artifact_coherence(current_metrics, run_tag), + _gate_semantic_coherence(current_metrics), + _gate_pd(baseline_metrics, current_metrics), + _gate_conformal(baseline_metrics, current_metrics), + _gate_ab_no_regression(baseline_metrics, current_metrics), + _gate_fairness(baseline_metrics, current_metrics), + _gate_fairness_absolute_business(baseline_metrics, current_metrics), + _gate_survival(baseline_metrics, current_metrics), + _gate_exports(current_snapshot), ] - conformal_gate = next((g for g in gate_results if g.name == "conformal_policy"), None) - ab_gate = next((g for g in gate_results if g.name == "ab_no_regression"), None) - coherence_gate = next((g for g in gate_results if g.name == "artifact_coherence"), None) - semantic_gate = next((g for g in gate_results if g.name == "semantic_coherence"), None) - conformal_details = conformal_gate.details if conformal_gate is not None else {} - ab_details = ab_gate.details if ab_gate is not None else {} - fairness_abs_gate = next( - (g for g in gate_results if g.name == "fairness_absolute_business"), None - ) - conformal_checks = conformal_details.get("checks", {}) + + +def _gate_lookup(gate_results: list[GateResult]) -> dict[str, GateResult]: + return {gate.name: gate for gate in gate_results} + + +def _gate_pass(gates: dict[str, GateResult], name: str) -> bool: + gate = gates.get(name) + return bool(gate.passed) if gate is not None else False + + +def _gate_details(gates: dict[str, GateResult], name: str) -> dict[str, Any]: + gate = gates.get(name) + return gate.details if gate is not None else {} + + +def _comparison_gate_fields(gate_results: list[GateResult]) -> dict[str, Any]: + gates = _gate_lookup(gate_results) + conformal_details = _gate_details(gates, "conformal_policy") conformal_diagnostics = conformal_details.get("diagnostics", {}) - ab_diagnostics = ab_details.get("diagnostics", {}) - coherence_details = coherence_gate.details if coherence_gate is not None else {} - try: - baseline_path_out = str(baseline_path.relative_to(ROOT)) - except ValueError: - baseline_path_out = str(baseline_path) - report = { - "schema_version": SCHEMA_VERSION, - "run_tag": run_tag, - "generated_at_utc": datetime.now(tz=UTC).isoformat(), - "baseline_path": baseline_path_out, - "overall_pass": bool(all(g.passed for g in gate_results)), + ab_diagnostics = _gate_details(gates, "ab_no_regression").get("diagnostics", {}) + return { + "overall_pass": bool(all(gate.passed for gate in gate_results)), "operational_overall_pass": bool( - all(g.passed for g in gate_results if g.name in _OPERATIONAL_GATE_NAMES) + all(gate.passed for gate in gate_results if gate.name in _OPERATIONAL_GATE_NAMES) + ), + "artifact_coherence_pass": _gate_pass(gates, "artifact_coherence"), + "artifact_coherence": _gate_details(gates, "artifact_coherence"), + "semantic_coherence_pass": _gate_pass(gates, "semantic_coherence"), + "semantic_coherence": _gate_details(gates, "semantic_coherence"), + "conformal_promotion_pass": bool( + conformal_details.get("checks", {}).get("conformal_promotion_pass", False) ), - "artifact_coherence_pass": bool(coherence_gate.passed) - if coherence_gate is not None - else False, - "artifact_coherence": coherence_details, - "semantic_coherence_pass": bool(semantic_gate.passed) - if semantic_gate is not None - else False, - "semantic_coherence": semantic_gate.details if semantic_gate is not None else {}, - "conformal_promotion_pass": bool(conformal_checks.get("conformal_promotion_pass", False)), "conformal_retired_backtest_checks": conformal_diagnostics.get( "retired_backtest_checks", [] ), - "ab_no_regression_pass": bool(ab_gate.passed) if ab_gate is not None else False, - "fairness_absolute_business_pass": bool(fairness_abs_gate.passed) - if fairness_abs_gate is not None - else False, + "ab_no_regression_pass": _gate_pass(gates, "ab_no_regression"), + "fairness_absolute_business_pass": _gate_pass(gates, "fairness_absolute_business"), "ab_gate_mode": str(ab_diagnostics.get("gate_mode", "no_regression")), "ab_significant": bool(ab_diagnostics.get("significant", False)), "ab_significance_role": str(ab_diagnostics.get("significance_role", "diagnostic")), + } + + +def _comparison_quality_contract() -> dict[str, Any]: + return { + "conformal_checks_required": 13, + "ab_gate_mode": "no_regression", + "ab_significance_role": "diagnostic", + "fairness_gates": ["fairness_relative", "fairness_absolute_business"], + "fairness_policy_path": _path_for_report(FAIRNESS_POLICY_PATH), + "artifact_coherence_required": True, + "semantic_coherence_required": True, + "required_status_metadata": ["schema_version", "generated_at_utc", "run_tag"], + } + + +def _comparison_report( + *, + run_tag: str, + baseline_path: Path, + baseline: dict[str, Any], + current: dict[str, Any], + gate_results: list[GateResult], +) -> dict[str, Any]: + return { + "schema_version": SCHEMA_VERSION, + "run_tag": run_tag, + "generated_at_utc": datetime.now(tz=UTC).isoformat(), + "baseline_path": _path_for_report(baseline_path), + **_comparison_gate_fields(gate_results), "gates": [{"name": g.name, "passed": g.passed, "details": g.details} for g in gate_results], "artifact_changes": _compare_artifacts(baseline, current), "baseline_head": baseline.get("git", {}).get("head", ""), "current_head": current.get("git", {}).get("head", ""), - "quality_contract": { - "conformal_checks_required": 13, - "ab_gate_mode": "no_regression", - "ab_significance_role": "diagnostic", - "fairness_gates": ["fairness_relative", "fairness_absolute_business"], - "fairness_policy_path": _path_for_report(FAIRNESS_POLICY_PATH), - "artifact_coherence_required": True, - "semantic_coherence_required": True, - "required_status_metadata": ["schema_version", "generated_at_utc", "run_tag"], - }, + "quality_contract": _comparison_quality_contract(), } + + +def _write_comparison_files(run_tag: str, report: dict[str, Any]) -> tuple[Path, Path]: out_dir = OUT_ROOT / run_tag out_dir.mkdir(parents=True, exist_ok=True) json_path = out_dir / "comparison.json" @@ -1019,6 +1168,26 @@ def _write_compare(run_tag: str, baseline_path: Path) -> tuple[Path, Path]: return json_path, md_path +def _write_compare(run_tag: str, baseline_path: Path) -> tuple[Path, Path]: + baseline_path = baseline_path.expanduser().resolve() + baseline = json.loads(baseline_path.read_text(encoding="utf-8")) + current = _snapshot_payload(run_tag) + gate_results = _comparison_gate_results( + baseline_metrics=baseline["metrics"], + current_metrics=current["metrics"], + current_snapshot=current, + run_tag=run_tag, + ) + report = _comparison_report( + run_tag=run_tag, + baseline_path=baseline_path, + baseline=baseline, + current=current, + gate_results=gate_results, + ) + return _write_comparison_files(run_tag, report) + + def main() -> None: parser = argparse.ArgumentParser(description="Snapshot/compare run artifacts with gates.") sub = parser.add_subparsers(dest="cmd", required=True) diff --git a/scripts/run_cqr_comparison.py b/scripts/run_cqr_comparison.py index 08df8d1..409d1e1 100644 --- a/scripts/run_cqr_comparison.py +++ b/scripts/run_cqr_comparison.py @@ -188,7 +188,7 @@ def make_qreg(quantile: float) -> CatBoostRegressor: # MAPIE CQR: pass list [alpha/2, 1-alpha/2, 0.5] in that order cqr = ConformalizedQuantileRegressor( - estimator=[qreg_lo, qreg_hi, qreg_mid], + estimator=cast(Any, [qreg_lo, qreg_hi, qreg_mid]), confidence_level=1.0 - alpha, prefit=True, ) diff --git a/scripts/run_crpto_vs_spo_stability.py b/scripts/run_crpto_vs_spo_stability.py index aebc3be..b97729c 100644 --- a/scripts/run_crpto_vs_spo_stability.py +++ b/scripts/run_crpto_vs_spo_stability.py @@ -21,10 +21,12 @@ from __future__ import annotations import argparse +import importlib import json import shutil import time from collections import OrderedDict +from numbers import Real from pathlib import Path from typing import Any @@ -34,25 +36,31 @@ import pandas as pd from loguru import logger -from scripts.run_spo_real import ( - LGD, - NUMERIC_FEATURES, - RANDOM_SEED, - CreditPortfolioLP, - _compute_regret, - _compute_true_optima, - _index_costs, - _load_pd_artifacts, - _predict_calibrated_costs, - _prep_features, - _sample_instances, - _train_spo, -) from src.utils.artifact_metadata import build_artifact_metadata, resolve_run_tag matplotlib.use("Agg") SCHEMA_VERSION = "2026-03-22.1" +LGD = 0.40 +RANDOM_SEED = 42 + +NUMERIC_FEATURES = [ + "loan_amnt", + "int_rate", + "annual_inc", + "dti", + "fico_range_low", + "open_acc", + "revol_bal", + "revol_util", + "total_acc", + "installment", + "emp_length", + "pub_rec", + "delinq_2yrs", + "inq_last_6mths", + "mths_since_last_delinq", +] REPO_ROOT = Path(__file__).resolve().parents[1] DATA_DIR = REPO_ROOT / "data" / "processed" @@ -108,9 +116,31 @@ ) +def _require_optional_module(module_name: str) -> Any: + """Import an optional SPO dependency with an explicit experiment-level error.""" + try: + return importlib.import_module(module_name) + except ImportError as exc: + raise RuntimeError( + "SPO stability is an optional experiment. Install the `spo` extras " + f"before running this script; missing module: {module_name}." + ) from exc + + # ── Period assignment ──────────────────────────────────────────────────────── +def _load_spo_real_module() -> Any: + """Load SPO helpers lazily so this module remains importable without PyEPO.""" + try: + return importlib.import_module("scripts.run_spo_real") + except RuntimeError as exc: + raise RuntimeError( + "SPO stability is an optional experiment. Install the `spo` extras " + "before running this script." + ) from exc + + def _assign_periods(issue_d: pd.Series) -> pd.Series: """Assign each loan to a temporal period based on issue_d.""" dt = pd.to_datetime(issue_d) @@ -160,6 +190,219 @@ def _evaluate_period_coverage(ci_slice: pd.DataFrame) -> dict: # ── Figure generation ──────────────────────────────────────────────────────── +def _available_numeric_features(train: pd.DataFrame, test: pd.DataFrame) -> list[str]: + return [ + feature + for feature in NUMERIC_FEATURES + if feature in train.columns and feature in test.columns + ] + + +def _period_masks(test_periods: pd.Series) -> dict[str, np.ndarray]: + return {name: (test_periods.values == name) for name in PERIODS} + + +def _period_default_rate(test: pd.DataFrame, mask: np.ndarray) -> float: + n_loans = int(mask.sum()) + return float(test.loc[mask, "default_flag"].mean()) if n_loans > 0 else 0.0 + + +def _init_period_regrets() -> dict[str, dict[str, list[float]]]: + return {name: {"two_stage": [], "spo_plus": [], "conformal_robust": []} for name in PERIODS} + + +def _period_sample_seed(seed: int, period_name: str) -> int: + """Stable per-period seed; avoids Python's process-randomized hash().""" + period_offset = list(PERIODS).index(period_name) + 1 + return int(seed + period_offset * 100_000) + + +def _period_test_instance_count(n_period: int, n_items: int) -> int: + return max(min(80, n_period // n_items), 5) + + +def _valid_regret_values(values: list[float]) -> list[float]: + return [value for value in values if not np.isnan(value)] + + +def _mean_std(values: list[float]) -> tuple[float, float]: + valid = _valid_regret_values(values) + if not valid: + return float("nan"), float("nan") + return float(np.mean(valid)), float(np.std(valid)) + + +def _spo_improvement_pct(two_stage_mean: float, spo_mean: float, has_values: bool) -> float | None: + if not has_values: + return None + return (two_stage_mean - spo_mean) / (abs(two_stage_mean) + 1e-9) * 100 + + +def _coverage_by_period( + ci: pd.DataFrame, + period_masks: dict[str, np.ndarray], +) -> dict[str, dict[str, Any]]: + period_coverage: dict[str, dict[str, Any]] = {} + for period_name, mask in period_masks.items(): + ci_slice = ci.loc[mask] + if len(ci_slice) == 0: + continue + period_coverage[period_name] = _evaluate_period_coverage(ci_slice) + logger.info( + " {} coverage: 90%={:.2%} width={:.4f} min_grade={:.2%}", + period_name, + period_coverage[period_name]["coverage_90"], + period_coverage[period_name]["avg_width_90"], + period_coverage[period_name].get("min_grade_coverage_90", 0) or 0, + ) + return period_coverage + + +def _detail_rows( + *, + test: pd.DataFrame, + period_masks: dict[str, np.ndarray], + per_period_regrets: dict[str, dict[str, list[float]]], + period_coverage: dict[str, dict[str, Any]], +) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for period_name in PERIODS: + mask = period_masks[period_name] + n_loans = int(mask.sum()) + regrets = per_period_regrets[period_name] + cov = period_coverage.get(period_name, {}) + + ts_mean, ts_std = _mean_std(regrets["two_stage"]) + spo_mean, spo_std = _mean_std(regrets["spo_plus"]) + cr_mean, cr_std = _mean_std(regrets["conformal_robust"]) + has_spo_comparison = bool( + _valid_regret_values(regrets["two_stage"]) and _valid_regret_values(regrets["spo_plus"]) + ) + + rows.append( + { + "period": period_name, + "n_loans": n_loans, + "default_rate": _period_default_rate(test, mask), + "two_stage_mean_regret": ts_mean, + "two_stage_std_regret": ts_std, + "spo_plus_mean_regret": spo_mean, + "spo_plus_std_regret": spo_std, + "conformal_robust_mean_regret": cr_mean, + "conformal_robust_std_regret": cr_std, + "spo_improvement_pct": _spo_improvement_pct( + ts_mean, + spo_mean, + has_spo_comparison, + ), + "coverage_90": cov.get("coverage_90"), + "coverage_95": cov.get("coverage_95"), + "avg_width_90": cov.get("avg_width_90"), + "min_grade_coverage_90": cov.get("min_grade_coverage_90"), + } + ) + return rows + + +def _round_optional(value: object, digits: int) -> float | None: + if isinstance(value, Real): + return round(float(value), digits) + return None + + +def _period_summary_row( + row: dict[str, Any], + regrets: dict[str, list[float]], +) -> dict[str, Any]: + return { + "n_loans": int(row["n_loans"]), + "default_rate": round(float(row["default_rate"]), 4), + "regret": { + "two_stage": { + "mean": round(float(row["two_stage_mean_regret"]), 6), + "std": round(float(row["two_stage_std_regret"]), 6), + "per_seed": regrets["two_stage"], + }, + "spo_plus": { + "mean": round(float(row["spo_plus_mean_regret"]), 6), + "std": round(float(row["spo_plus_std_regret"]), 6), + "per_seed": regrets["spo_plus"], + }, + "conformal_robust": { + "mean": round(float(row["conformal_robust_mean_regret"]), 6), + "std": round(float(row["conformal_robust_std_regret"]), 6), + "per_seed": regrets["conformal_robust"], + }, + }, + "spo_improvement_vs_ts_pct": _round_optional(row["spo_improvement_pct"], 2), + "coverage_90": _round_optional(row["coverage_90"], 4), + "coverage_95": _round_optional(row["coverage_95"], 4), + "avg_width_90": _round_optional(row["avg_width_90"], 4), + "min_grade_coverage_90": _round_optional(row["min_grade_coverage_90"], 4), + } + + +def _per_period_json( + rows: list[dict[str, Any]], + per_period_regrets: dict[str, dict[str, list[float]]], +) -> dict[str, Any]: + return { + str(row["period"]): _period_summary_row( + row, + per_period_regrets[str(row["period"])], + ) + for row in rows + } + + +def _non_null_float_values(rows: list[dict[str, Any]], key: str) -> list[float]: + return [float(row[key]) for row in rows if row[key] is not None] + + +def _stability_summary(rows: list[dict[str, Any]]) -> dict[str, Any]: + coverages_90 = _non_null_float_values(rows, "coverage_90") + spo_improvements = _non_null_float_values(rows, "spo_improvement_pct") + return { + "coverage_always_above_target": all(c >= 0.90 for c in coverages_90), + "coverage_range": [round(min(coverages_90), 4), round(max(coverages_90), 4)], + "spo_improvement_range_pct": ( + [round(min(spo_improvements), 2), round(max(spo_improvements), 2)] + if spo_improvements + else None + ), + } + + +def _summary_payload( + *, + run_tag: str, + args: argparse.Namespace, + n_features: int, + feature_names: list[str], + rows: list[dict[str, Any]], + per_period_regrets: dict[str, dict[str, list[float]]], + total_time: float, +) -> dict[str, Any]: + return { + **build_artifact_metadata( + schema_version=SCHEMA_VERSION, run_tag=run_tag, allow_untracked=True + ), + "config": { + "n_items": args.n_items, + "budget": args.budget, + "n_train_instances": args.n_train, + "epochs": args.epochs, + "n_seeds": args.seeds, + "n_features": n_features, + "feature_names": feature_names, + "lgd": LGD, + }, + "per_period": _per_period_json(rows, per_period_regrets), + "stability_summary": _stability_summary(rows), + "train_time_seconds": round(total_time, 1), + } + + def _generate_stability_figure(detail_df: pd.DataFrame, out_dir: Path) -> None: """Two-panel figure: Panel A = regret by period, Panel B = coverage by period.""" fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(COL2, HEIGHT_M)) @@ -277,6 +520,17 @@ def main() -> int: ) # ── 1. Load data ──────────────────────────────────────────────────────── + spo_real = _load_spo_real_module() + CreditPortfolioLP = spo_real.CreditPortfolioLP + _compute_regret = spo_real._compute_regret + _compute_true_optima = spo_real._compute_true_optima + _index_costs = spo_real._index_costs + _load_pd_artifacts = spo_real._load_pd_artifacts + _predict_calibrated_costs = spo_real._predict_calibrated_costs + _prep_features = spo_real._prep_features + _sample_instances = spo_real._sample_instances + _train_spo = spo_real._train_spo + train = pd.read_parquet(DATA_DIR / "train_fe.parquet") test = pd.read_parquet(DATA_DIR / "test_fe.parquet") ci = pd.read_parquet(DATA_DIR / "conformal_intervals_mondrian.parquet") @@ -293,9 +547,7 @@ def main() -> int: logger.info("Period counts: {}", period_counts) # ── 2. Prepare features and costs ─────────────────────────────────────── - _tr_cols = train.columns - _te_cols = test.columns - avail = [f for f in NUMERIC_FEATURES if f in _tr_cols and f in _te_cols] + avail = _available_numeric_features(train, test) n_features = len(avail) logger.info("Using {} features: {}", n_features, avail) @@ -329,19 +581,15 @@ def main() -> int: c_robust_te_all = (pd_high * LGD - int_rate_te).astype(np.float32) # ── 3. Build period masks ─────────────────────────────────────────────── - period_masks = {} - for name in PERIODS: - mask = test_periods.values == name - period_masks[name] = mask + period_masks = _period_masks(test_periods) + for name, mask in period_masks.items(): n_loans = int(mask.sum()) - default_rate = float(test.loc[mask, "default_flag"].mean()) if n_loans > 0 else 0.0 + default_rate = _period_default_rate(test, mask) logger.info(" {} : {:,} loans, default rate {:.2%}", name, n_loans, default_rate) # ── 4. Multi-seed × multi-period evaluation ───────────────────────────── # Structure: per_period[period][method] = list of per-seed mean regrets - per_period_regrets: dict[str, dict[str, list[float]]] = { - name: {"two_stage": [], "spo_plus": [], "conformal_robust": []} for name in PERIODS - } + per_period_regrets = _init_period_regrets() t_total = time.time() @@ -379,9 +627,7 @@ def main() -> int: per_period_regrets[period_name][method].append(float("nan")) continue - n_test_period = min(80, n_period // args.n_items) - if n_test_period < 5: - n_test_period = 5 + n_test_period = _period_test_instance_count(n_period, args.n_items) # Slice arrays to this period period_idx = np.where(mask)[0] @@ -391,7 +637,7 @@ def main() -> int: c_robust_period = c_robust_te_all[period_idx] # Sample test instances within this period - rng_period = np.random.RandomState(seed + hash(period_name) % (2**31)) + rng_period = np.random.RandomState(_period_sample_seed(seed, period_name)) X_inst, c_inst, idx_inst = _sample_instances( X_period, c_period, args.n_items, n_test_period, rng_period ) @@ -399,8 +645,7 @@ def main() -> int: c_robust_inst = _index_costs(c_robust_period, idx_inst) # SPO+ predictions - import torch - + torch = _require_optional_module("torch") n_input = args.n_items * n_features X_flat = X_inst.reshape(n_test_period, n_input) with torch.no_grad(): @@ -433,63 +678,15 @@ def main() -> int: logger.info("All seeds done in {:.1f}s", total_time) # ── 5. Conformal coverage per period (deterministic, no seed) ─────────── - period_coverage: dict[str, dict] = {} - for period_name, mask in period_masks.items(): - ci_slice = ci.loc[mask] - if len(ci_slice) == 0: - continue - period_coverage[period_name] = _evaluate_period_coverage(ci_slice) - logger.info( - " {} coverage: 90%={:.2%} width={:.4f} min_grade={:.2%}", - period_name, - period_coverage[period_name]["coverage_90"], - period_coverage[period_name]["avg_width_90"], - period_coverage[period_name].get("min_grade_coverage_90", 0) or 0, - ) + period_coverage = _coverage_by_period(ci, period_masks) # ── 6. Aggregate into detail DataFrame ────────────────────────────────── - rows: list[dict[str, Any]] = [] - for period_name in PERIODS: - mask = period_masks[period_name] - n_loans = int(mask.sum()) - default_rate = float(test.loc[mask, "default_flag"].mean()) if n_loans > 0 else 0.0 - - regrets = per_period_regrets[period_name] - cov = period_coverage.get(period_name, {}) - - ts_vals = [v for v in regrets["two_stage"] if not np.isnan(v)] - spo_vals = [v for v in regrets["spo_plus"] if not np.isnan(v)] - cr_vals = [v for v in regrets["conformal_robust"] if not np.isnan(v)] - - ts_mean = float(np.mean(ts_vals)) if ts_vals else float("nan") - spo_mean = float(np.mean(spo_vals)) if spo_vals else float("nan") - cr_mean = float(np.mean(cr_vals)) if cr_vals else float("nan") - ts_std = float(np.std(ts_vals)) if ts_vals else float("nan") - spo_std = float(np.std(spo_vals)) if spo_vals else float("nan") - cr_std = float(np.std(cr_vals)) if cr_vals else float("nan") - - spo_improvement = ( - ((ts_mean - spo_mean) / (abs(ts_mean) + 1e-9) * 100) if ts_vals and spo_vals else None - ) - - rows.append( - { - "period": period_name, - "n_loans": n_loans, - "default_rate": default_rate, - "two_stage_mean_regret": ts_mean, - "two_stage_std_regret": ts_std, - "spo_plus_mean_regret": spo_mean, - "spo_plus_std_regret": spo_std, - "conformal_robust_mean_regret": cr_mean, - "conformal_robust_std_regret": cr_std, - "spo_improvement_pct": spo_improvement, - "coverage_90": cov.get("coverage_90"), - "coverage_95": cov.get("coverage_95"), - "avg_width_90": cov.get("avg_width_90"), - "min_grade_coverage_90": cov.get("min_grade_coverage_90"), - } - ) + rows = _detail_rows( + test=test, + period_masks=period_masks, + per_period_regrets=per_period_regrets, + period_coverage=period_coverage, + ) detail_df = pd.DataFrame(rows) detail_path = DATA_DIR / "crpto_vs_spo_stability_detail.parquet" @@ -497,82 +694,15 @@ def main() -> int: logger.info("Saved: {}", detail_path) # ── 7. Summary JSON ───────────────────────────────────────────────────── - coverages_90 = [float(r["coverage_90"]) for r in rows if r["coverage_90"] is not None] - spo_improvements = [ - float(r["spo_improvement_pct"]) for r in rows if r["spo_improvement_pct"] is not None - ] - - per_period_json: dict[str, Any] = {} - for r in rows: - period = str(r["period"]) - regrets = per_period_regrets[period] - per_period_json[period] = { - "n_loans": int(r["n_loans"]), - "default_rate": round(float(r["default_rate"]), 4), - "regret": { - "two_stage": { - "mean": round(float(r["two_stage_mean_regret"]), 6), - "std": round(float(r["two_stage_std_regret"]), 6), - "per_seed": regrets["two_stage"], - }, - "spo_plus": { - "mean": round(float(r["spo_plus_mean_regret"]), 6), - "std": round(float(r["spo_plus_std_regret"]), 6), - "per_seed": regrets["spo_plus"], - }, - "conformal_robust": { - "mean": round(float(r["conformal_robust_mean_regret"]), 6), - "std": round(float(r["conformal_robust_std_regret"]), 6), - "per_seed": regrets["conformal_robust"], - }, - }, - "spo_improvement_vs_ts_pct": ( - round(float(r["spo_improvement_pct"]), 2) - if r["spo_improvement_pct"] is not None - else None - ), - "coverage_90": ( - round(float(r["coverage_90"]), 4) if r["coverage_90"] is not None else None - ), - "coverage_95": ( - round(float(r["coverage_95"]), 4) if r["coverage_95"] is not None else None - ), - "avg_width_90": ( - round(float(r["avg_width_90"]), 4) if r["avg_width_90"] is not None else None - ), - "min_grade_coverage_90": ( - round(float(r["min_grade_coverage_90"]), 4) - if r["min_grade_coverage_90"] is not None - else None - ), - } - - summary = { - **build_artifact_metadata( - schema_version=SCHEMA_VERSION, run_tag=run_tag, allow_untracked=True - ), - "config": { - "n_items": args.n_items, - "budget": args.budget, - "n_train_instances": args.n_train, - "epochs": args.epochs, - "n_seeds": args.seeds, - "n_features": n_features, - "feature_names": avail, - "lgd": LGD, - }, - "per_period": per_period_json, - "stability_summary": { - "coverage_always_above_target": all(c >= 0.90 for c in coverages_90), - "coverage_range": [round(min(coverages_90), 4), round(max(coverages_90), 4)], - "spo_improvement_range_pct": ( - [round(min(spo_improvements), 2), round(max(spo_improvements), 2)] - if spo_improvements - else None - ), - }, - "train_time_seconds": round(total_time, 1), - } + summary = _summary_payload( + run_tag=run_tag, + args=args, + n_features=n_features, + feature_names=avail, + rows=rows, + per_period_regrets=per_period_regrets, + total_time=total_time, + ) json_path = DATA_DIR / "crpto_vs_spo_stability.json" with open(json_path, "w") as f: diff --git a/scripts/run_fairness_audit.py b/scripts/run_fairness_audit.py index 6ed6206..03c9b2f 100644 --- a/scripts/run_fairness_audit.py +++ b/scripts/run_fairness_audit.py @@ -16,6 +16,7 @@ import argparse import json from pathlib import Path +from typing import Any import numpy as np import pandas as pd @@ -42,6 +43,11 @@ SHAP_SAMPLE_SIZE = 10_000 +def _as_float(value: Any) -> float: + """Convert scalar config/data values to float with a narrow error surface.""" + return float(value) + + def _load_config(config_path: str) -> dict: """Load fairness policy YAML config.""" with open(config_path) as f: @@ -140,13 +146,14 @@ def _select_threshold_from_frontier( rows: list[dict[str, float | int]] = [] for threshold, grp in frontier.groupby("threshold", observed=True): + threshold_value = _as_float(threshold) rows.append( { - "threshold": float(threshold), + "threshold": threshold_value, "n_passed": int(grp["passed_all"].sum()), "worst_eo_gap": float(grp["eo_gap"].max()), "approval_rate": float( - (np.asarray(y_pred_proba_eval, dtype=float) >= float(threshold)).mean() + (np.asarray(y_pred_proba_eval, dtype=float) >= threshold_value).mean() ), } ) @@ -198,6 +205,211 @@ def _apply_decision_policy( return (y_pred_proba >= thresholds).astype(float) +def _model_feature_names(model: Any) -> list[str]: + return list(getattr(model, "feature_names_", None) or []) + + +def _available_features(feature_names: list[str], data: pd.DataFrame) -> list[str]: + return [feature for feature in feature_names if feature in data.columns] + + +def _sample_shap_frame( + frame: pd.DataFrame, + *, + shap_sample_size: int, + random_state: int, +) -> tuple[pd.DataFrame, np.ndarray]: + rng = np.random.default_rng(random_state) + n_rows = len(frame) + if n_rows > shap_sample_size: + sample_idx = np.sort(rng.choice(n_rows, size=shap_sample_size, replace=False)) + else: + sample_idx = np.arange(n_rows) + return frame.iloc[sample_idx].reset_index(drop=True), sample_idx + + +def _cat_features_from_model(model: Any, frame: pd.DataFrame) -> list[str]: + try: + feature_names = _model_feature_names(model) + cat_idx = model.get_cat_feature_indices() + return [ + feature_names[idx] + for idx in cat_idx + if idx < len(feature_names) and feature_names[idx] in frame.columns + ] + except Exception: + return [] + + +def _text_categorical_columns(frame: pd.DataFrame, known_cats: set[str]) -> list[str]: + detected: list[str] = [] + for col in frame.columns: + if col in known_cats or pd.api.types.is_numeric_dtype(frame[col]): + continue + probe = frame[col].dropna().head(5) + if probe.empty: + continue + try: + pd.to_numeric(probe, errors="raise") + except (ValueError, TypeError): + detected.append(str(col)) + known_cats.add(str(col)) + return detected + + +def _catboost_cat_feature_names(model: Any, frame: pd.DataFrame) -> list[str]: + cat_names = _cat_features_from_model(model, frame) + cat_set = set(cat_names) + cat_names.extend(_text_categorical_columns(frame, cat_set)) + return cat_names + + +def _prepare_catboost_shap_frame(frame: pd.DataFrame, cat_feature_names: list[str]) -> pd.DataFrame: + out = frame.copy() + cat_set = set(cat_feature_names) + for col in list(out.columns): + if not out[col].isna().any(): + continue + if col in cat_set: + out[col] = out[col].astype(object).fillna("missing").astype(str) + elif pd.api.types.is_numeric_dtype(out[col]): + out[col] = out[col].fillna(0.0) + return out + + +def _catboost_shap_matrix( + model: Any, frame: pd.DataFrame, cat_feature_names: list[str] +) -> np.ndarray: + from catboost import Pool as CatPool + + pool = CatPool(frame, cat_features=cat_feature_names or None) + shap_raw = model.get_feature_importance(pool, type="ShapValues") + return np.abs(np.asarray(shap_raw[:, :-1], dtype=float)) + + +def _top_shap_features( + mean_abs_shap: np.ndarray, + feature_names: list[str], + *, + limit: int, +) -> list[dict[str, object]]: + top_idx = np.argsort(mean_abs_shap)[::-1][:limit] + return [ + {"feature": feature_names[idx], "mean_abs_shap": float(mean_abs_shap[idx])} + for idx in top_idx + ] + + +def _pairwise_shap_diffs( + group_shap: dict[str, np.ndarray], + feature_names: list[str], + *, + limit: int, +) -> list[dict[str, object]]: + pairwise_diffs: list[dict[str, object]] = [] + groups_with_shap = list(group_shap.keys()) + for i, group_a in enumerate(groups_with_shap): + for group_b in groups_with_shap[i + 1 :]: + diff = np.abs(group_shap[group_a] - group_shap[group_b]) + top_idx = np.argsort(diff)[::-1][:limit] + pairwise_diffs.append( + { + "group_a": group_a, + "group_b": group_b, + "top_driving_features": [ + { + "feature": feature_names[idx], + "shap_diff": float(diff[idx]), + } + for idx in top_idx + ], + } + ) + return pairwise_diffs + + +def _attribute_shap_result( + *, + attribute: str, + labels: np.ndarray, + sample_idx: np.ndarray, + shap_matrix: np.ndarray, + feature_names: list[str], + min_group_size: int = 10, +) -> dict[str, object]: + group_labels = pd.Series(labels).iloc[sample_idx].reset_index(drop=True).astype(str) + unique_groups = sorted(group_labels.unique()) + group_shap: dict[str, np.ndarray] = {} + group_top5: dict[str, list[dict[str, object]]] = {} + + for group in unique_groups: + mask = group_labels.eq(group).to_numpy() + if mask.sum() < min_group_size: + continue + mean_abs_shap = shap_matrix[mask].mean(axis=0) + group_shap[str(group)] = mean_abs_shap + group_top5[str(group)] = _top_shap_features(mean_abs_shap, feature_names, limit=5) + + groups_with_shap = list(group_shap.keys()) + return { + "attribute": attribute, + "groups_analyzed": groups_with_shap, + "top5_per_group": group_top5, + "pairwise_feature_diffs": _pairwise_shap_diffs( + group_shap, + feature_names, + limit=3, + ), + } + + +def _shap_attribute_results( + groups_dict: dict[str, np.ndarray], + *, + sample_idx: np.ndarray, + shap_matrix: np.ndarray, + feature_names: list[str], +) -> list[dict[str, object]]: + attribute_results: list[dict[str, object]] = [] + for attribute, labels in groups_dict.items(): + if "__x__" in attribute: + continue + result = _attribute_shap_result( + attribute=attribute, + labels=labels, + sample_idx=sample_idx, + shap_matrix=shap_matrix, + feature_names=feature_names, + ) + attribute_results.append(result) + groups_analyzed = result.get("groups_analyzed", []) + n_groups = len(groups_analyzed) if isinstance(groups_analyzed, list) else 0 + logger.info(f"SHAP per-group: {attribute} ({n_groups} groups)") + return attribute_results + + +def _shap_result_payload( + *, + model_path: Path, + sample_size: int, + n_features: int, + attribute_results: list[dict[str, object]], +) -> dict[str, object]: + return { + "schema_version": SCHEMA_VERSION, + "model_path": str(model_path), + "shap_sample_size": sample_size, + "n_features": n_features, + "attributes": attribute_results, + "interpretation": ( + "For each protected attribute, top-5 features by mean |SHAP| per group. " + "Pairwise diffs show which features drive SHAP disparities between groups. " + "Features like dti/loan_amnt are legitimate credit risk factors; " + "home_ownership may proxy for race in US ECOA context." + ), + } + + def _compute_shap_per_group( data: pd.DataFrame, groups_dict: dict[str, np.ndarray], @@ -236,158 +448,378 @@ def _compute_shap_per_group( try: model = CatBoostClassifier() model.load_model(str(model_path)) - feature_names: list[str] = list(model.feature_names_) + feature_names = _model_feature_names(model) except Exception as e: logger.warning(f"SHAP per-group analysis skipped — model load error: {e}") return None - available_features = [f for f in feature_names if f in data.columns] + available_features = _available_features(feature_names, data) if not available_features: logger.warning("SHAP per-group analysis skipped — no model features found in test data") return None - X_full = data[available_features].copy() - - rng = np.random.default_rng(random_state) - n = len(X_full) - if n > shap_sample_size: - sample_idx = np.sort(rng.choice(n, size=shap_sample_size, replace=False)) - X_sample = X_full.iloc[sample_idx].reset_index(drop=True) - else: - sample_idx = np.arange(n) - X_sample = X_full.reset_index(drop=True) + x_sample, sample_idx = _sample_shap_frame( + data[available_features].copy(), + shap_sample_size=shap_sample_size, + random_state=random_state, + ) logger.info( - f"Computing SHAP values on {len(X_sample):,} rows, {len(available_features)} features" + f"Computing SHAP values on {len(x_sample):,} rows, {len(available_features)} features" ) try: # Use CatBoost's native SHAP via Pool + get_feature_importance — avoids the shap # library's cat/NaN handling issues entirely. model.get_cat_feature_indices() works # on .cbm-loaded models without needing the sklearn feature_names_ attribute. - import pandas as _pd - from catboost import Pool as _CatPool - - # Identify cat feature column names. - # Primary: use model metadata (works on .cbm loaded models when feature_names_ is set). - # Fallback: detect from data — any column whose non-null values cannot be cast to float - # is genuinely categorical (e.g. "very_high__E" WOE bin labels, grade strings). - _cat_feat_names: list[str] = [] - try: - _fn = list(getattr(model, "feature_names_", None) or []) - if _fn: - _cat_idx = model.get_cat_feature_indices() - _cat_feat_names = [ - _fn[i] for i in _cat_idx if i < len(_fn) and _fn[i] in X_sample.columns - ] - except Exception: - pass - - # Always supplement with content-based detection: any column whose non-null values - # cannot be cast to float is categorical, regardless of model metadata. - # This catches columns that have string values but are not in model.get_cat_feature_indices() - # (e.g. WOE bin labels stored as strings in the parquet instead of numeric WOE scores). - _cat_set = set(_cat_feat_names) - for _col in X_sample.columns: - if _col in _cat_set or _pd.api.types.is_numeric_dtype(X_sample[_col]): - continue - _probe = X_sample[_col].dropna().head(5) - if _probe.empty: - continue - try: - _pd.to_numeric(_probe, errors="raise") - except (ValueError, TypeError): - _cat_feat_names.append(_col) - _cat_set.add(_col) - - # Fill NaN: cat features → "missing", numeric features → 0.0 - # Only touch NaN cells; never alter non-NaN values. - X_sample = X_sample.copy() - _cat_set = set(_cat_feat_names) - for _col in list(X_sample.columns): - if not X_sample[_col].isna().any(): - continue - if _col in _cat_set: - X_sample[_col] = X_sample[_col].astype(object).fillna("missing").astype(str) - elif _pd.api.types.is_numeric_dtype(X_sample[_col]): - X_sample[_col] = X_sample[_col].fillna(0.0) - - pool = _CatPool(X_sample, cat_features=_cat_feat_names or None) - shap_raw = model.get_feature_importance(pool, type="ShapValues") - # get_feature_importance returns (n_samples, n_features + 1); last col is bias - shap_matrix = np.abs(np.asarray(shap_raw[:, :-1], dtype=float)) + cat_feature_names = _catboost_cat_feature_names(model, x_sample) + x_sample = _prepare_catboost_shap_frame(x_sample, cat_feature_names) + shap_matrix = _catboost_shap_matrix(model, x_sample, cat_feature_names) except Exception as e: logger.warning(f"SHAP per-group analysis skipped — SHAP computation error: {e}") return None - feature_names_sample = available_features - attribute_results: list[dict[str, object]] = [] + attribute_results = _shap_attribute_results( + groups_dict, + sample_idx=sample_idx, + shap_matrix=shap_matrix, + feature_names=available_features, + ) + return _shap_result_payload( + model_path=model_path, + sample_size=len(x_sample), + n_features=len(available_features), + attribute_results=attribute_results, + ) - for attribute, labels in groups_dict.items(): - if "__x__" in attribute: - continue - group_labels = pd.Series(labels).iloc[sample_idx].reset_index(drop=True).astype(str) - unique_groups = sorted(group_labels.unique()) - group_shap: dict[str, np.ndarray] = {} - group_top5: dict[str, list[dict[str, object]]] = {} - - for grp in unique_groups: - mask = group_labels == grp - if mask.sum() < 10: - continue - mean_abs_shap = shap_matrix[mask].mean(axis=0) - group_shap[grp] = mean_abs_shap - top5_idx = np.argsort(mean_abs_shap)[::-1][:5] - group_top5[grp] = [ - {"feature": feature_names_sample[i], "mean_abs_shap": float(mean_abs_shap[i])} - for i in top5_idx - ] - - pairwise_diffs: list[dict[str, object]] = [] - groups_with_shap = list(group_shap.keys()) - for i in range(len(groups_with_shap)): - for j in range(i + 1, len(groups_with_shap)): - g_a, g_b = groups_with_shap[i], groups_with_shap[j] - diff = np.abs(group_shap[g_a] - group_shap[g_b]) - top3_idx = np.argsort(diff)[::-1][:3] - pairwise_diffs.append( - { - "group_a": g_a, - "group_b": g_b, - "top_driving_features": [ - { - "feature": feature_names_sample[k], - "shap_diff": float(diff[k]), - } - for k in top3_idx - ], - } + +def _bootstrap_base_indices( + n_rows: int, + *, + bootstrap_max_rows: int, + rng: np.random.Generator, +) -> np.ndarray: + if bootstrap_max_rows > 0 and n_rows > bootstrap_max_rows: + return np.sort(rng.choice(n_rows, size=bootstrap_max_rows, replace=False)) + return np.arange(n_rows) + + +def _fairlearn_group_rows( + *, + attribute: str, + y_true: np.ndarray, + y_pred: np.ndarray, + sensitive: pd.Series, +) -> list[dict[str, object]]: + metric_frame = MetricFrame( + metrics={"selection_rate": selection_rate, "accuracy": accuracy_score}, + y_true=y_true, + y_pred=y_pred, + sensitive_features=sensitive, + ) + by_group = metric_frame.by_group.reset_index() + by_group.columns = ["group", *[str(col) for col in by_group.columns[1:]]] + rows = by_group.to_dict(orient="records") + for row in rows: + row["attribute"] = attribute + return rows + + +def _bootstrap_fairlearn_gaps( + *, + boot_sensitive_base: pd.Series, + boot_true_base: np.ndarray, + boot_pred_base: np.ndarray, + rng: np.random.Generator, + n_boot: int, +) -> tuple[list[float], list[float]]: + dpd_boot: list[float] = [] + eo_boot: list[float] = [] + for _ in range(max(n_boot, 0)): + idx = rng.integers(0, len(boot_true_base), len(boot_true_base)) + boot_sensitive = boot_sensitive_base.iloc[idx] + boot_true = boot_true_base[idx] + boot_pred = boot_pred_base[idx] + dpd_boot.append( + float( + demographic_parity_difference( + y_true=boot_true, + y_pred=boot_pred, + sensitive_features=boot_sensitive, + ) + ) + ) + eo_boot.append( + float( + equalized_odds_difference( + y_true=boot_true, + y_pred=boot_pred, + sensitive_features=boot_sensitive, ) + ) + ) + return dpd_boot, eo_boot - attribute_results.append( - { - "attribute": attribute, - "groups_analyzed": groups_with_shap, - "top5_per_group": group_top5, - "pairwise_feature_diffs": pairwise_diffs, - } + +def _fairlearn_summary_row( + *, + attribute: str, + y_true: np.ndarray, + y_pred: np.ndarray, + sensitive: pd.Series, + bootstrap_idx: np.ndarray, + rng: np.random.Generator, + n_boot: int, +) -> dict[str, object]: + dpd = float( + demographic_parity_difference( + y_true=y_true, + y_pred=y_pred, + sensitive_features=sensitive, + ) + ) + eo = float( + equalized_odds_difference( + y_true=y_true, + y_pred=y_pred, + sensitive_features=sensitive, ) - logger.info(f"SHAP per-group: {attribute} ({len(groups_with_shap)} groups)") + ) + dpd_boot, eo_boot = _bootstrap_fairlearn_gaps( + boot_sensitive_base=sensitive.iloc[bootstrap_idx].reset_index(drop=True), + boot_true_base=y_true[bootstrap_idx], + boot_pred_base=y_pred[bootstrap_idx], + rng=rng, + n_boot=n_boot, + ) + return { + "attribute": attribute, + "demographic_parity_difference": dpd, + "equalized_odds_difference": eo, + "dpd_ci_low": float(np.quantile(dpd_boot, 0.025)) if dpd_boot else None, + "dpd_ci_high": float(np.quantile(dpd_boot, 0.975)) if dpd_boot else None, + "eo_ci_low": float(np.quantile(eo_boot, 0.025)) if eo_boot else None, + "eo_ci_high": float(np.quantile(eo_boot, 0.975)) if eo_boot else None, + } + + +def _fairlearn_sidecar_rows( + *, + groups_all: dict[str, np.ndarray], + y_true: np.ndarray, + y_pred: np.ndarray, + bootstrap_idx: np.ndarray, + rng: np.random.Generator, + n_boot: int, +) -> tuple[list[dict[str, object]], list[dict[str, object]]]: + group_rows: list[dict[str, object]] = [] + summary_rows: list[dict[str, object]] = [] + for attribute, labels in groups_all.items(): + sensitive = pd.Series(labels).astype(str).reset_index(drop=True) + group_rows.extend( + _fairlearn_group_rows( + attribute=attribute, + y_true=y_true, + y_pred=y_pred, + sensitive=sensitive, + ) + ) + summary_rows.append( + _fairlearn_summary_row( + attribute=attribute, + y_true=y_true, + y_pred=y_pred, + sensitive=sensitive, + bootstrap_idx=bootstrap_idx, + rng=rng, + n_boot=n_boot, + ) + ) + return group_rows, summary_rows + + +def _write_fairlearn_sidecar( + *, + sidecar_cfg: dict, + groups_all: dict[str, np.ndarray], + y_true_eval: np.ndarray, + y_pred_binary: np.ndarray, + status_path: Path, + primary_threshold: float, + outcome_mode: str, + resolved_run_tag: str, +) -> None: + rng = np.random.default_rng(int(sidecar_cfg.get("bootstrap_random_state", 42))) + n_boot = int(sidecar_cfg.get("bootstrap_samples", 200)) + bootstrap_max_rows = int(sidecar_cfg.get("bootstrap_max_rows", 50_000)) + y_true_arr = np.asarray(y_true_eval, dtype=float) + y_pred_arr = np.asarray(y_pred_binary, dtype=float) + bootstrap_idx = _bootstrap_base_indices( + len(y_true_arr), + bootstrap_max_rows=bootstrap_max_rows, + rng=rng, + ) + group_rows, summary_rows = _fairlearn_sidecar_rows( + groups_all=groups_all, + y_true=y_true_arr, + y_pred=y_pred_arr, + bootstrap_idx=bootstrap_idx, + rng=rng, + n_boot=n_boot, + ) + + sidecar_path = Path(sidecar_cfg.get("status_json", "models/fairlearn_fairness_status.json")) + group_metrics_path = Path( + sidecar_cfg.get("group_metrics_parquet", "data/processed/fairlearn_group_metrics.parquet") + ) + group_metrics_path.parent.mkdir(parents=True, exist_ok=True) + sidecar_path.parent.mkdir(parents=True, exist_ok=True) + pd.DataFrame(group_rows).to_parquet(group_metrics_path, index=False) + sidecar_payload = { + "primary_status_path": str(status_path), + "group_metrics_path": str(group_metrics_path), + "n_attributes": len(summary_rows), + "attributes": summary_rows, + "bootstrap_samples": n_boot, + "bootstrap_rows_used": len(bootstrap_idx), + "bootstrap_max_rows": bootstrap_max_rows, + "prediction_threshold": float(primary_threshold), + "outcome_mode": outcome_mode, + **build_artifact_metadata( + schema_version=f"{SCHEMA_VERSION}-fairlearn", + run_tag=resolved_run_tag, + require_explicit=True, + ), + } + sidecar_path.write_text(json.dumps(sidecar_payload, indent=2, default=str), encoding="utf-8") + logger.info(f"Saved fairlearn sidecar status: {sidecar_path}") + + +def _with_attribute_type(frame: pd.DataFrame) -> pd.DataFrame: + if frame.empty: + return frame + out = frame.copy() + out["attribute_type"] = np.where( + out["attribute"].astype(str).str.contains("__x__"), + "intersectional", + "base", + ) + return out + + +def _primary_frontier(frontier: pd.DataFrame, primary_threshold: float) -> pd.DataFrame: + if frontier.empty: + return pd.DataFrame() + return frontier.loc[np.isclose(frontier["threshold"].astype(float), primary_threshold)] + + +def _worst_primary_attribute(primary_frontier: pd.DataFrame) -> str: + if primary_frontier.empty: + return "" + return str( + primary_frontier.sort_values( + by=["passed_all", "eo_gap", "dpd", "dir"], + ascending=[True, False, False, True], + ).iloc[0]["attribute"] + ) + +def _decision_override_count(decision_policy: Any) -> int: + overrides = decision_policy.get("overrides", []) if isinstance(decision_policy, dict) else [] + return len(overrides) if isinstance(overrides, list) else 0 + + +def _decision_global_threshold(decision_policy: Any, primary_threshold: float) -> float: + if isinstance(decision_policy, dict): + return _as_float(decision_policy.get("global_threshold", primary_threshold)) + return float(primary_threshold) + + +def _fairness_status_payload( + *, + report: pd.DataFrame, + frontier: pd.DataFrame, + frontier_path: Path, + frontier_thresholds: list[float], + primary_threshold: float, + threshold_source: str, + outcome_mode: str, + policy: dict, + decision_policy: Any, + decision_policy_path: Path, + config_path: str, + resolved_run_tag: str, +) -> dict[str, object]: + primary_frontier = _primary_frontier(frontier, primary_threshold) return { - "schema_version": SCHEMA_VERSION, - "model_path": str(model_path), - "shap_sample_size": len(X_sample), - "n_features": len(available_features), - "attributes": attribute_results, - "interpretation": ( - "For each protected attribute, top-5 features by mean |SHAP| per group. " - "Pairwise diffs show which features drive SHAP disparities between groups. " - "Features like dti/loan_amnt are legitimate credit risk factors; " - "home_ownership may proxy for race in US ECOA context." + "overall_pass": bool(report["passed_all"].all()), + "n_attributes": len(report), + "n_base_attributes": int( + (report.get("attribute_type", pd.Series(dtype=str)) == "base").sum() + ), + "n_intersectional_attributes": int( + (report.get("attribute_type", pd.Series(dtype=str)) == "intersectional").sum() + ), + "n_passed": int(report["passed_all"].sum()), + "attributes": report.to_dict(orient="records"), + "prediction_threshold": float(primary_threshold), + "primary_threshold": float(primary_threshold), + "prediction_threshold_source": threshold_source, + "outcome_mode": outcome_mode, + "thresholds": { + "dpd": policy["dpd_threshold"], + "eo_gap": policy["eo_gap_threshold"], + "dir": policy["dir_threshold"], + }, + "threshold_frontier": { + "path": str(frontier_path), + "thresholds": frontier_thresholds, + "worst_primary_attribute": _worst_primary_attribute(primary_frontier), + "selected_threshold": float(primary_threshold), + "all_primary_pass": bool( + primary_frontier.get("passed_all", pd.Series(dtype=bool)).all() + ) + if not primary_frontier.empty + else True, + }, + "decision_policy": { + "path": str(decision_policy_path), + "global_threshold": float(primary_threshold), + "n_overrides": _decision_override_count(decision_policy), + }, + "policy_config": str(config_path), + **build_artifact_metadata( + schema_version=SCHEMA_VERSION, + run_tag=resolved_run_tag, + require_explicit=True, ), } +def _write_json_payload(path: Path, payload: dict[str, object], *, label: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, default=str), encoding="utf-8") + logger.info(f"Saved {label}: {path}") + + +def _write_shap_status( + *, + shap_result: dict[str, object] | None, + resolved_run_tag: str, + primary_threshold: float, + outcome_mode: str, + path: Path = SHAP_STATUS_PATH, +) -> None: + if shap_result is None: + return + shap_result["generated_at_utc"] = str( + __import__("datetime").datetime.now(__import__("datetime").timezone.utc).isoformat() + ) + shap_result["run_tag"] = resolved_run_tag + shap_result["prediction_threshold"] = float(primary_threshold) + shap_result["outcome_mode"] = outcome_mode + _write_json_payload(path, shap_result, label="SHAP per-group fairness analysis") + + def main(config_path: str = "configs/fairness_policy.yaml", run_tag: str | None = None) -> None: """Run the fairness audit pipeline.""" cfg = _load_config(config_path) @@ -467,12 +899,7 @@ def main(config_path: str = "configs/fairness_policy.yaml", run_tag: str | None eo_gap_threshold=policy["eo_gap_threshold"], dir_threshold=policy["dir_threshold"], ) - if not frontier.empty: - frontier["attribute_type"] = np.where( - frontier["attribute"].astype(str).str.contains("__x__"), - "intersectional", - "base", - ) + frontier = _with_attribute_type(frontier) frontier_path = Path( output.get("frontier_parquet", "data/processed/fairness_threshold_frontier.parquet") ) @@ -537,223 +964,60 @@ def main(config_path: str = "configs/fairness_policy.yaml", run_tag: str | None eo_gap_threshold=policy["eo_gap_threshold"], dir_threshold=policy["dir_threshold"], ) - if not report.empty: - report["attribute_type"] = np.where( - report["attribute"].astype(str).str.contains("__x__"), - "intersectional", - "base", - ) + report = _with_attribute_type(report) audit_path = Path(output["audit_parquet"]) audit_path.parent.mkdir(parents=True, exist_ok=True) report.to_parquet(audit_path, index=False) logger.info(f"Saved fairness audit: {audit_path}") - # Build and save status JSON - overall_pass = bool(report["passed_all"].all()) - primary_frontier = ( - frontier.loc[np.isclose(frontier["threshold"].astype(float), primary_threshold)] - if not frontier.empty - else pd.DataFrame() + status = _fairness_status_payload( + report=report, + frontier=frontier, + frontier_path=frontier_path, + frontier_thresholds=frontier_thresholds, + primary_threshold=float(primary_threshold), + threshold_source=threshold_source, + outcome_mode=outcome_mode, + policy=policy, + decision_policy=decision_policy, + decision_policy_path=decision_policy_path, + config_path=config_path, + resolved_run_tag=resolved_run_tag, ) - worst_primary_attribute = "" - if not primary_frontier.empty: - worst_primary_attribute = str( - primary_frontier.sort_values( - by=["passed_all", "eo_gap", "dpd", "dir"], - ascending=[True, False, False, True], - ).iloc[0]["attribute"] - ) - status = { - "overall_pass": overall_pass, - "n_attributes": len(report), - "n_base_attributes": int( - (report.get("attribute_type", pd.Series(dtype=str)) == "base").sum() - ), - "n_intersectional_attributes": int( - (report.get("attribute_type", pd.Series(dtype=str)) == "intersectional").sum() - ), - "n_passed": int(report["passed_all"].sum()), - "attributes": report.to_dict(orient="records"), - "prediction_threshold": float(primary_threshold), - "primary_threshold": float(primary_threshold), - "prediction_threshold_source": threshold_source, - "outcome_mode": outcome_mode, - "thresholds": { - "dpd": policy["dpd_threshold"], - "eo_gap": policy["eo_gap_threshold"], - "dir": policy["dir_threshold"], - }, - "threshold_frontier": { - "path": str(frontier_path), - "thresholds": frontier_thresholds, - "worst_primary_attribute": worst_primary_attribute, - "selected_threshold": float(primary_threshold), - "all_primary_pass": bool( - primary_frontier.get("passed_all", pd.Series(dtype=bool)).all() - ) - if not primary_frontier.empty - else True, - }, - "decision_policy": { - "path": str(decision_policy_path), - "global_threshold": float(primary_threshold), - "n_overrides": len(decision_policy.get("overrides", [])) - if isinstance(decision_policy, dict) - else 0, - }, - "policy_config": str(config_path), - **build_artifact_metadata( - schema_version=SCHEMA_VERSION, - run_tag=resolved_run_tag, - require_explicit=True, - ), - } status_path = Path(output["status_json"]) - status_path.parent.mkdir(parents=True, exist_ok=True) - with open(status_path, "w", encoding="utf-8") as f: - json.dump(status, f, indent=2, default=str) - logger.info(f"Saved fairness status: {status_path}") + _write_json_payload(status_path, status, label="fairness status") sidecar_cfg = cfg.get("fairlearn_sidecar", {}) or {} if bool(sidecar_cfg.get("enabled", True)): - group_rows: list[dict[str, object]] = [] - summary_rows: list[dict[str, object]] = [] - rng = np.random.default_rng(int(sidecar_cfg.get("bootstrap_random_state", 42))) - n_boot = int(sidecar_cfg.get("bootstrap_samples", 200)) - bootstrap_max_rows = int(sidecar_cfg.get("bootstrap_max_rows", 50_000)) - y_true_arr = np.asarray(y_true_eval, dtype=float) - y_pred_arr = np.asarray(y_pred_binary, dtype=float) - if bootstrap_max_rows > 0 and len(y_true_arr) > bootstrap_max_rows: - bootstrap_idx = np.sort( - rng.choice(len(y_true_arr), size=bootstrap_max_rows, replace=False) - ) - else: - bootstrap_idx = np.arange(len(y_true_arr)) - - for attribute, labels in groups_all.items(): - sensitive = pd.Series(labels).astype(str).reset_index(drop=True) - mf = MetricFrame( - metrics={"selection_rate": selection_rate, "accuracy": accuracy_score}, - y_true=y_true_arr, - y_pred=y_pred_arr, - sensitive_features=sensitive, - ) - by_group = mf.by_group.reset_index() - by_group.columns = ["group", *[str(col) for col in by_group.columns[1:]]] - for row in by_group.to_dict(orient="records"): - row["attribute"] = attribute - group_rows.append(row) - - dpd = float( - demographic_parity_difference( - y_true=y_true_arr, - y_pred=y_pred_arr, - sensitive_features=sensitive, - ) - ) - eo = float( - equalized_odds_difference( - y_true=y_true_arr, - y_pred=y_pred_arr, - sensitive_features=sensitive, - ) - ) - boot_sensitive_base = sensitive.iloc[bootstrap_idx].reset_index(drop=True) - boot_true_base = y_true_arr[bootstrap_idx] - boot_pred_base = y_pred_arr[bootstrap_idx] - dpd_boot: list[float] = [] - eo_boot: list[float] = [] - for _ in range(max(n_boot, 0)): - idx = rng.integers(0, len(boot_true_base), len(boot_true_base)) - boot_sensitive = boot_sensitive_base.iloc[idx] - boot_true = boot_true_base[idx] - boot_pred = boot_pred_base[idx] - dpd_boot.append( - float( - demographic_parity_difference( - y_true=boot_true, - y_pred=boot_pred, - sensitive_features=boot_sensitive, - ) - ) - ) - eo_boot.append( - float( - equalized_odds_difference( - y_true=boot_true, - y_pred=boot_pred, - sensitive_features=boot_sensitive, - ) - ) - ) - summary_rows.append( - { - "attribute": attribute, - "demographic_parity_difference": dpd, - "equalized_odds_difference": eo, - "dpd_ci_low": float(np.quantile(dpd_boot, 0.025)) if dpd_boot else None, - "dpd_ci_high": float(np.quantile(dpd_boot, 0.975)) if dpd_boot else None, - "eo_ci_low": float(np.quantile(eo_boot, 0.025)) if eo_boot else None, - "eo_ci_high": float(np.quantile(eo_boot, 0.975)) if eo_boot else None, - } - ) - - sidecar_path = Path(sidecar_cfg.get("status_json", "models/fairlearn_fairness_status.json")) - group_metrics_path = Path( - sidecar_cfg.get( - "group_metrics_parquet", "data/processed/fairlearn_group_metrics.parquet" - ) - ) - group_metrics_path.parent.mkdir(parents=True, exist_ok=True) - sidecar_path.parent.mkdir(parents=True, exist_ok=True) - pd.DataFrame(group_rows).to_parquet(group_metrics_path, index=False) - sidecar_payload = { - "primary_status_path": str(status_path), - "group_metrics_path": str(group_metrics_path), - "n_attributes": len(summary_rows), - "attributes": summary_rows, - "bootstrap_samples": n_boot, - "bootstrap_rows_used": len(bootstrap_idx), - "bootstrap_max_rows": bootstrap_max_rows, - "prediction_threshold": float(primary_threshold), - "outcome_mode": outcome_mode, - **build_artifact_metadata( - schema_version=f"{SCHEMA_VERSION}-fairlearn", - run_tag=resolved_run_tag, - require_explicit=True, - ), - } - sidecar_path.write_text( - json.dumps(sidecar_payload, indent=2, default=str), encoding="utf-8" + _write_fairlearn_sidecar( + sidecar_cfg=sidecar_cfg, + groups_all=groups_all, + y_true_eval=y_true_eval, + y_pred_binary=y_pred_binary, + status_path=status_path, + primary_threshold=float(primary_threshold), + outcome_mode=outcome_mode, + resolved_run_tag=resolved_run_tag, ) - logger.info(f"Saved fairlearn sidecar status: {sidecar_path}") - shap_result = _compute_shap_per_group( - data=data.iloc[:n], - groups_dict=groups_dict, + _write_shap_status( + shap_result=_compute_shap_per_group( + data=data.iloc[:n], + groups_dict=groups_dict, + ), + resolved_run_tag=resolved_run_tag, + primary_threshold=float(primary_threshold), + outcome_mode=outcome_mode, ) - if shap_result is not None: - shap_result["generated_at_utc"] = str( - __import__("datetime").datetime.now(__import__("datetime").timezone.utc).isoformat() - ) - shap_result["run_tag"] = resolved_run_tag - shap_result["prediction_threshold"] = float(primary_threshold) - shap_result["outcome_mode"] = outcome_mode - SHAP_STATUS_PATH.parent.mkdir(parents=True, exist_ok=True) - SHAP_STATUS_PATH.write_text( - json.dumps(shap_result, indent=2, default=str), encoding="utf-8" - ) - logger.info(f"Saved SHAP per-group fairness analysis: {SHAP_STATUS_PATH}") write_threshold_semantics( fairness_primary_threshold=float(primary_threshold), - decision_policy_global_threshold=float( - decision_policy.get("global_threshold", primary_threshold) - ) - if isinstance(decision_policy, dict) - else float(primary_threshold), + decision_policy_global_threshold=_decision_global_threshold( + decision_policy, + float(primary_threshold), + ), source_artifacts={ "fairness_status": str(status_path), "fairness_decision_policy": str(decision_policy_path), @@ -767,6 +1031,7 @@ def main(config_path: str = "configs/fairness_policy.yaml", run_tag: str | None path=output.get("threshold_semantics_json", "models/threshold_semantics.json"), ) + overall_pass = bool(status["overall_pass"]) pass_label = "PASS" if overall_pass else "FAIL" logger.info( f"Fairness audit: {pass_label} ({status['n_passed']}/{status['n_attributes']} attributes)" diff --git a/scripts/run_spo_comparison.py b/scripts/run_spo_comparison.py index 98a83da..8f46af7 100644 --- a/scripts/run_spo_comparison.py +++ b/scripts/run_spo_comparison.py @@ -13,6 +13,7 @@ from __future__ import annotations import argparse +import importlib import json import sys from datetime import UTC, datetime @@ -117,8 +118,8 @@ def _try_spo_comparison( ) -> dict[str, object] | None: """Attempt SPO+ comparison if pyepo and torch are available.""" try: - import torch # noqa: F401 - from pyepo.func import SPOPlus # noqa: F401 + importlib.import_module("torch") + importlib.import_module("pyepo.func") logger.info("PyEPO and torch available. SPO+ comparison enabled.") except ImportError: diff --git a/scripts/run_spo_real.py b/scripts/run_spo_real.py index d451630..e4adfef 100644 --- a/scripts/run_spo_real.py +++ b/scripts/run_spo_real.py @@ -18,6 +18,7 @@ from __future__ import annotations import argparse +import importlib import json import pickle import time @@ -26,12 +27,8 @@ import numpy as np import pandas as pd -import pyepo -import torch -import torch.nn as nn from loguru import logger from ortools.linear_solver import pywraplp -from pyepo.model.opt import optModel from scipy import stats from src.models.conformal import apply_probability_calibrator @@ -41,6 +38,23 @@ LGD = 0.40 RANDOM_SEED = 42 + +def _require_optional_module(module_name: str) -> Any: + """Import an optional SPO dependency with an explicit experiment-level error.""" + try: + return importlib.import_module(module_name) + except ImportError as exc: + raise RuntimeError( + "SPO+ is an optional experiment. Install the `spo` extras before " + f"running this script; missing module: {module_name}." + ) from exc + + +pyepo: Any = _require_optional_module("pyepo") +torch: Any = _require_optional_module("torch") +nn: Any = _require_optional_module("torch.nn") +OptModelBase: Any = _require_optional_module("pyepo.model.opt").optModel + NUMERIC_FEATURES = [ "loan_amnt", "int_rate", @@ -63,7 +77,7 @@ # ── 1. Portfolio LP optModel ──────────────────────────────────────────────── -class CreditPortfolioLP(optModel): +class CreditPortfolioLP(OptModelBase): """Portfolio selection LP: select exactly `budget` of `n_items` loans to minimize expected loss c^T x = sum(x_i * (PD_i * LGD - r_i)). @@ -87,7 +101,7 @@ def _getModel(self) -> tuple: ct.SetCoefficient(x[i], 1.0) return solver, x - def setObj(self, c: np.ndarray | torch.Tensor) -> None: + def setObj(self, c: np.ndarray | Any) -> None: if isinstance(c, torch.Tensor): c = c.detach().cpu().numpy() c = np.asarray(c, dtype=float) @@ -139,7 +153,7 @@ def __init__(self, n_features: int, n_items: int) -> None: nn.Linear(32, 1), ) - def forward(self, x: torch.Tensor) -> torch.Tensor: + def forward(self, x: Any) -> Any: B = x.shape[0] # (B, n_items * n_features) → (B * n_items, n_features) x_per_loan = x.view(B * self.n_items, self.n_features) @@ -345,9 +359,9 @@ def _train_spo( X_inst_train: (n_inst, n_items, n_features) — instance features. c_inst_train: (n_inst, n_items) — true costs per instance. """ - from pyepo.data.dataset import optDataset - from pyepo.func import SPOPlus - from torch.utils.data import DataLoader + optDataset = _require_optional_module("pyepo.data.dataset").optDataset + SPOPlus = _require_optional_module("pyepo.func").SPOPlus + DataLoader = _require_optional_module("torch.utils.data").DataLoader n_input = n_items * n_features X_flat = X_inst_train.reshape(len(X_inst_train), n_input) diff --git a/scripts/run_ty_advisory.py b/scripts/run_ty_advisory.py new file mode 100644 index 0000000..546c592 --- /dev/null +++ b/scripts/run_ty_advisory.py @@ -0,0 +1,131 @@ +"""Run ty as a fast, non-blocking advisory checker for CRPTO.""" + +from __future__ import annotations + +import argparse +import re +import shutil +import subprocess +from collections.abc import Sequence +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_OUTPUT = ROOT / "reports" / "ci" / "ty-advisory.txt" +TY_REQUIREMENT = "ty==0.0.57" + +SOURCE_ROOTS = ("src", "scripts") +ACTIVE_EXCLUDED_FILES = { + "scripts/generate_conformal_intervals.py", + "scripts/run_cqr_comparison.py", + "scripts/run_crpto_vs_spo_stability.py", + "scripts/run_spo_comparison.py", + "scripts/run_spo_real.py", + "scripts/train_pd_model.py", + "src/optimization/cuopt_adapter.py", +} +SUMMARY_RE = re.compile(r"^Found \d+ diagnostics", flags=re.MULTILINE) + + +def _relative_posix(path: Path) -> str: + return path.relative_to(ROOT).as_posix() + + +def iter_python_files(*, scope: str) -> list[str]: + """Return the Python files ty should check for a given advisory scope.""" + files: list[str] = [] + for root_name in SOURCE_ROOTS: + for path in sorted((ROOT / root_name).rglob("*.py")): + rel = _relative_posix(path) + parts = rel.split("/") + if scope == "active": + if parts[:2] in (["scripts", "archive"], ["scripts", "experiments"]): + continue + if parts[:2] == ["scripts", "search"] and path.name.startswith("run_"): + continue + if rel in ACTIVE_EXCLUDED_FILES: + continue + files.append(rel) + return files + + +def build_ty_command(*, uvx: str, files: Sequence[str], fail_on_diagnostics: bool) -> list[str]: + """Build the pinned ty command for advisory or blocking use.""" + command = [ + uvx, + "--from", + TY_REQUIREMENT, + "ty", + "check", + "--python", + ".venv", + "--output-format", + "concise", + "--no-progress", + ] + if not fail_on_diagnostics: + command.append("--exit-zero") + return [*command, *files] + + +def run_ty(scope: str, output: Path, *, fail_on_diagnostics: bool = False) -> int: + """Run pinned ty, persist its report, and optionally enforce diagnostics.""" + uvx = shutil.which("uvx") + if uvx is None: + raise RuntimeError("uvx is required to run the ty advisory check.") + + files = iter_python_files(scope=scope) + command = build_ty_command( + uvx=uvx, + files=files, + fail_on_diagnostics=fail_on_diagnostics, + ) + result = subprocess.run( + command, + cwd=ROOT, + text=True, + capture_output=True, + check=False, + ) + output.parent.mkdir(parents=True, exist_ok=True) + report = ( + f"# ty advisory report\n" + f"requirement: {TY_REQUIREMENT}\n" + f"scope: {scope}\n" + f"blocking: {str(fail_on_diagnostics).lower()}\n" + f"files_checked: {len(files)}\n" + f"return_code: {result.returncode}\n" + f"\n" + f"{result.stdout}{result.stderr}" + ) + output.write_text(report, encoding="utf-8") + summary = SUMMARY_RE.search(report) + if summary: + print(summary.group(0)) + elif result.returncode == 0: + print("ty advisory clean") + else: + print(f"ty failed with return code {result.returncode}") + print(f"Full report: {output.relative_to(ROOT)}") + return result.returncode if fail_on_diagnostics else 0 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--scope", choices=["active", "full"], default="active") + parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT) + parser.add_argument( + "--fail-on-diagnostics", + action="store_true", + help="Return ty's nonzero status when diagnostics are present.", + ) + args = parser.parse_args() + output = args.output if args.output.is_absolute() else ROOT / args.output + return run_ty( + scope=args.scope, + output=output, + fail_on_diagnostics=args.fail_on_diagnostics, + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/search/build_pool93_body_allocation_audit.py b/scripts/search/build_pool93_body_allocation_audit.py index 526dc75..2d4d365 100644 --- a/scripts/search/build_pool93_body_allocation_audit.py +++ b/scripts/search/build_pool93_body_allocation_audit.py @@ -29,7 +29,14 @@ _compute_intervals_at_alpha, _load_aligned_dataset, ) -from src.optimization.portfolio_model import optimize_portfolio_allocation # noqa: E402 +from src.optimization.certificate_semantics import ( # noqa: E402 + compute_funded_certificate_metrics, +) +from src.optimization.portfolio_model import ( # noqa: E402 + optimize_portfolio_allocation, + solution_allocation_vector, +) +from src.utils.script_helpers import resolve_repo_artifact_path # noqa: E402 DEFAULT_CONSOLIDATED_TAG = "champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive" DEFAULT_BODY_ROLE = "body/default balanced return-bound point" @@ -86,11 +93,17 @@ def _policy_from_row(row: dict[str, Any]) -> dict[str, Any]: def _grade_bucket(series: pd.Series) -> pd.Series: grade = series.fillna("unknown").astype(str).str.upper().str[:1] - return np.select( - [grade.isin(["A", "B"]), grade.eq("C"), grade.eq("D"), grade.isin(["E", "F", "G"])], + buckets = np.select( + [ + grade.isin(["A", "B"]), + grade.eq("C"), + grade.eq("D"), + grade.isin(["E", "F", "G"]), + ], ["A-B", "C", "D", "E-G"], default="unknown", ) + return pd.Series(buckets, index=series.index, name="grade_bucket") def _format_tex_table(summary: pd.DataFrame) -> str: @@ -100,9 +113,11 @@ def _format_tex_table(summary: pd.DataFrame) -> str: "Grade bucket & Funded rows & Exposure share & Default rate & $V$ contribution & Mean $u_i(0.01)$ \\\\", "\\midrule", ] - for row in summary.itertuples(index=False): + for row in summary.to_dict("records"): lines.append( - f"{row.grade_bucket} & {row.funded_rows:,.0f} & {row.exposure_share:.2%} & {row.default_rate:.2%} & {row.v_contribution:.5f} & {row.mean_pd_high_alpha01:.5f} \\\\" + f"{row['grade_bucket']} & {row['funded_rows']:,.0f} & " + f"{row['exposure_share']:.2%} & {row['default_rate']:.2%} & " + f"{row['v_contribution']:.5f} & {row['mean_pd_high_alpha01']:.5f} \\\\" ) lines.extend(["\\bottomrule", "\\end{tabular}", ""]) return "\n".join(lines) @@ -124,7 +139,9 @@ def build_audit( policy["solver_backend"] = solver_backend manifest = json.loads(_manifest_path(str(row["run_tag"])).read_text(encoding="utf-8")) - conformal_intervals_path = str(manifest["conformal_intervals_path"]) + conformal_intervals_path = str( + resolve_repo_artifact_path(manifest["conformal_intervals_path"], root=ROOT) + ) aligned = _load_aligned_dataset( conformal_intervals_path=conformal_intervals_path, max_candidates=int(manifest.get("max_candidates", 0) or 0), @@ -172,19 +189,23 @@ def build_audit( threads=max(1, int(threads)), solver_backend=str(policy["solver_backend"]), ) - if "allocation_vector" in solution: - alloc = np.asarray(solution["allocation_vector"], dtype=float) - else: - alloc = np.array( - [float(solution["allocation"].get(i, 0.0)) for i in range(len(aligned))], - dtype=float, - ) + alloc = solution_allocation_vector(solution, len(aligned)) exposure = alloc * loan_amounts total_allocated = float(exposure.sum()) weights = exposure / max(total_allocated, 1e-6) funded = alloc > 0.01 miscoverage = (y_true > pd_high).astype(float) + certificate = compute_funded_certificate_metrics( + weights, + outcomes=y_true, + pd_point=pd_point, + pd_high=pd_high, + pd_effective=effective_pd, + alpha=alpha, + risk_tolerance=float(policy["risk_tolerance"]), + pd_cap_slack=float(solution.get("pd_cap_slack", 0.0)), + ) realized_return = np.where( funded & (default_flag.astype(int) == 1), exposure * (-DEFAULT_LGD), @@ -255,23 +276,17 @@ def build_audit( "n_funded": int(funded.sum()), "total_allocated": round(total_allocated, 6), "realized_return": round(float(realized_return.sum()), 6), - "Gamma_CP": round(float(np.sum(weights * np.clip(pd_high - pd_point, 0.0, 1.0))), 6), - "V": round(float(np.sum(weights * miscoverage)), 6), - "weighted_pd_true": round(float(np.sum(weights * y_true)), 6), - "endpoint_budget_upper": round( - float(policy["risk_tolerance"]) - + (1.0 - float(policy["gamma"])) - * float(np.sum(weights * np.clip(pd_high - pd_point, 0.0, 1.0))), - 9, - ), - "markov_cap": round( - float(policy["risk_tolerance"]) - + (1.0 - float(policy["gamma"])) - * float(np.sum(weights * np.clip(pd_high - pd_point, 0.0, 1.0))) - + float(np.sqrt(alpha)), - 9, - ), - "empirical_coverage_funded": round(float(1.0 - miscoverage[funded].mean()), 6), + "Gamma_CP": round(certificate.gamma_cp, 6), + "Gamma_internalized": round(certificate.gamma_internalized, 6), + "Gamma_residual": round(certificate.gamma_residual, 6), + "V": round(certificate.weighted_miscoverage, 6), + "weighted_coverage_funded": round(certificate.weighted_coverage, 6), + "weighted_pd_true": round(certificate.weighted_outcome, 6), + "endpoint_budget": round(certificate.endpoint_budget, 9), + "endpoint_budget_upper": round(certificate.endpoint_budget_upper, 9), + "markov_loss_threshold": round(certificate.markov_loss_threshold, 9), + "markov_cap": round(certificate.markov_loss_cap, 9), + "empirical_coverage_funded": round(certificate.empirical_coverage_funded, 6), }, "outputs": { "funded_rows": str(funded_path), diff --git a/scripts/search/build_pool93_ijds_consolidated_frontier.py b/scripts/search/build_pool93_ijds_consolidated_frontier.py index 83fc6d7..7b2d9ad 100644 --- a/scripts/search/build_pool93_ijds_consolidated_frontier.py +++ b/scripts/search/build_pool93_ijds_consolidated_frontier.py @@ -8,17 +8,23 @@ from pathlib import Path from typing import Any +import numpy as np import pandas as pd +from src.optimization.certificate_semantics import add_policy_aware_bound_columns + ROOT = Path(__file__).resolve().parents[2] -DEFAULT_OUTPUT_TAG = "champion-reopen-2026-06-19__pool93__ijds-claim-consolidated" +DEFAULT_OUTPUT_TAG = "champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2" DEFAULT_RUN_TAGS = [ "champion-reopen-2026-06-19__pool93__ijds-claim-expanded-refine", "champion-reopen-2026-06-19__pool93__ijds-claim-micro-refine", "champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext", + "champion-reopen-2026-06-19__pool93__ijds-claim-bound-closure", + "champion-reopen-2026-06-19__pool93__ijds-claim-bound-floor-closure", + "champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal", ] -DEFAULT_CAPS = [0.32, 0.33, 0.335, 0.34, 0.345, 0.35, 0.36, 0.45, 0.50] -DEFAULT_BODY_MARKOV_CAP = 0.35 +DEFAULT_CAPS = [0.30, 0.32, 0.345, 0.36, 0.45] +DEFAULT_BODY_MARKOV_THRESHOLD = 0.35 def _leaderboard_path(run_tag: str) -> Path: @@ -30,6 +36,15 @@ def _leaderboard_path(run_tag: str) -> Path: ) +def _bound_eval_path(run_tag: str) -> Path: + return ( + ROOT + / "data/processed/experiments/champion_reopen" + / run_tag + / "portfolio/pool93_ijds_local_refinement_bound_eval.parquet" + ) + + def _output_path(output_tag: str) -> Path: return ( ROOT @@ -46,6 +61,12 @@ def _short_run_label(run_tag: str) -> str: return "micro" if run_tag.endswith("__pool93__ijds-claim-micro-ext"): return "micro_ext" + if run_tag.endswith("__pool93__ijds-claim-bound-closure"): + return "bound_closure" + if run_tag.endswith("__pool93__ijds-claim-bound-floor-closure"): + return "bound_floor" + if run_tag.endswith("__pool93__ijds-claim-bound-terminal"): + return "bound_terminal" return run_tag @@ -67,8 +88,12 @@ def _row_payload(row: pd.Series, role: str) -> dict[str, Any]: "return": round(float(row["alpha01_realized_total_return"]), 6), "return_floor_surplus": round(float(row["return_floor_surplus"]), 6), "Gamma_CP": round(float(row["alpha01_gamma_cp"]), 6), + "Gamma_internalized": round(float(row["alpha01_gamma_internalized"]), 6), + "Gamma_residual": round(float(row["alpha01_gamma_residual"]), 6), "V": round(float(row["alpha01_weighted_miscoverage_V"]), 6), + "endpoint_budget": round(float(row["alpha01_endpoint_budget"]), 9), "endpoint_budget_upper": round(float(row["alpha01_endpoint_budget_upper"]), 9), + "Markov_threshold": round(float(row["alpha01_markov_loss_threshold"]), 9), "Markov_cap": round(float(row["alpha01_markov_loss_cap"]), 9), "alpha_pass": f"{int(row['alpha_exact_pass_count'])}/{int(row['alpha_exact_check_count'])}", "n_funded_mean": round(float(row["n_funded_mean"]), 3), @@ -87,7 +112,7 @@ def _score_body_candidate(eligible: pd.DataFrame) -> pd.Series: work = eligible.copy() score_specs = { "return_score": ("alpha01_realized_total_return", False), - "bound_score": ("alpha01_markov_loss_cap", True), + "bound_score": ("alpha01_markov_loss_threshold", True), "v_score": ("alpha01_weighted_miscoverage_V", True), } for score_col, (metric_col, inverse) in score_specs.items(): @@ -103,30 +128,35 @@ def _score_body_candidate(eligible: pd.DataFrame) -> pd.Series: 0.40 * work["return_score"] + 0.40 * work["bound_score"] + 0.20 * work["v_score"] ) return work.sort_values( - ["ijds_balanced_score", "alpha01_realized_total_return", "alpha01_markov_loss_cap"], + [ + "ijds_balanced_score", + "alpha01_realized_total_return", + "alpha01_markov_loss_threshold", + ], ascending=[False, False, True], ).iloc[0] -def _body_candidate(eligible: pd.DataFrame, *, markov_cap: float) -> pd.Series: +def _body_candidate(eligible: pd.DataFrame, *, markov_threshold: float) -> pd.Series: """Select the paper-body point from the exact finite-grid frontier. The body point is intentionally not the global max-return endpoint and not the minimum-bound endpoint. It is the highest-return policy below a declared - Markov-cap ceiling, which matches the paper-facing return-bound claim. + exact Markov-loss threshold, which matches the paper-facing return-bound + claim and remains valid for linear, capped and tail-focused policies. """ - under_cap = _best_under_cap(eligible, markov_cap) - if under_cap is not None: - return under_cap + under_threshold = _best_under_threshold(eligible, markov_threshold) + if under_threshold is not None: + return under_threshold return _score_body_candidate(eligible) -def _best_under_cap(eligible: pd.DataFrame, cap: float) -> pd.Series | None: - candidates = eligible[eligible["alpha01_markov_loss_cap"] <= cap] +def _best_under_threshold(eligible: pd.DataFrame, threshold: float) -> pd.Series | None: + candidates = eligible[eligible["alpha01_markov_loss_threshold"] <= threshold] if candidates.empty: return None return candidates.sort_values( - ["alpha01_realized_total_return", "alpha01_markov_loss_cap"], + ["alpha01_realized_total_return", "alpha01_markov_loss_threshold"], ascending=[False, True], ).iloc[0] @@ -145,9 +175,56 @@ def _load_leaderboards(run_tags: list[str]) -> pd.DataFrame: frames: list[pd.DataFrame] = [] for run_tag in run_tags: path = _leaderboard_path(run_tag) + bound_path = _bound_eval_path(run_tag) if not path.exists(): raise FileNotFoundError(path) + if not bound_path.exists(): + raise FileNotFoundError(bound_path) frame = pd.read_parquet(path) + bound_eval = pd.read_parquet(bound_path) + alpha01 = add_policy_aware_bound_columns( + bound_eval[np.isclose(bound_eval["alpha"], 0.01)].copy() + ) + alpha01_columns = { + "gamma_internalized": "alpha01_gamma_internalized", + "gamma_residual": "alpha01_gamma_residual", + "weighted_pd_constraint_used": "alpha01_weighted_pd_constraint_used", + "weighted_pd_high": "alpha01_weighted_pd_high", + "weighted_pd_point": "alpha01_weighted_pd_point", + "endpoint_budget": "alpha01_endpoint_budget", + "endpoint_budget_upper": "alpha01_endpoint_budget_upper", + "markov_loss_threshold": "alpha01_markov_loss_threshold", + "markov_loss_cap": "alpha01_markov_loss_cap", + } + alpha01 = alpha01[["local_candidate_id", "semantic_policy_key", *alpha01_columns]].rename( + columns=alpha01_columns + ) + legacy_columns = { + "alpha01_endpoint_budget_upper": "legacy_alpha01_endpoint_budget_upper", + "alpha01_markov_loss_cap": "legacy_alpha01_markov_loss_cap", + } + frame = frame.rename( + columns={ + source: target + for source, target in legacy_columns.items() + if source in frame.columns + } + ) + stale_columns = [ + column + for column in alpha01_columns.values() + if column in frame.columns and column not in legacy_columns.values() + ] + if stale_columns: + frame = frame.drop(columns=stale_columns) + frame = frame.merge( + alpha01, + on=["local_candidate_id", "semantic_policy_key"], + how="left", + validate="one_to_one", + ) + if frame["alpha01_markov_loss_threshold"].isna().any(): + raise ValueError(f"Missing alpha=0.01 certificate rows for {run_tag}") frame["run_tag"] = run_tag frame["run_label"] = _short_run_label(run_tag) frames.append(frame) @@ -162,7 +239,7 @@ def build_consolidated_frontier(run_tags: list[str], caps: list[float]) -> dict[ "semantic_policy_key", "alpha_exact_pass_count", "alpha01_realized_total_return", - "alpha01_markov_loss_cap", + "alpha01_markov_loss_threshold", ], ascending=[True, False, False, True], ) @@ -171,10 +248,44 @@ def build_consolidated_frontier(run_tags: list[str], caps: list[float]) -> dict[ ) eligible = _eligible(deduped) - body = _body_candidate(eligible, markov_cap=DEFAULT_BODY_MARKOV_CAP) + body = _body_candidate(eligible, markov_threshold=DEFAULT_BODY_MARKOV_THRESHOLD) + semantics_audit: dict[str, Any] = { + "status": "not_available", + "selection_metric": "alpha01_markov_loss_threshold", + } + if "legacy_alpha01_markov_loss_cap" in deduped.columns: + legacy = pd.to_numeric(deduped["legacy_alpha01_markov_loss_cap"], errors="coerce") + exact = pd.to_numeric(deduped["alpha01_markov_loss_threshold"], errors="coerce") + delta = exact - legacy + tolerance = 1e-5 + legacy_eligible = eligible[eligible["legacy_alpha01_markov_loss_cap"] <= 0.35] + legacy_body = legacy_eligible.sort_values( + ["alpha01_realized_total_return", "legacy_alpha01_markov_loss_cap"], + ascending=[False, True], + ).iloc[0] + semantics_audit = { + "status": "corrected_from_existing_exact_bound_evaluations", + "selection_metric": "alpha01_markov_loss_threshold", + "legacy_metric": "tau + (1 - gamma) * Gamma_CP + sqrt(alpha)", + "material_difference_tolerance": tolerance, + "materially_changed_policies": int((delta.abs() > tolerance).sum()), + "materially_understated_policies": int((delta > tolerance).sum()), + "maximum_legacy_understatement": round(float(delta.max()), 9), + "legacy_under_0_50_excluded_by_exact_threshold": int( + ((legacy <= 0.50) & (exact > 0.50)).sum() + ), + "body_selection_unchanged": bool( + str(legacy_body["semantic_policy_key"]) == str(body["semantic_policy_key"]) + ), + "affected_policy_modes": sorted( + deduped.loc[delta > tolerance, "policy_mode"].astype(str).unique().tolist() + ), + } rows: list[dict[str, Any]] = [] _append_row( - rows, eligible.sort_values("alpha01_markov_loss_cap").iloc[0], "minimum Markov-cap endpoint" + rows, + eligible.sort_values("alpha01_markov_loss_threshold").iloc[0], + "minimum Markov-threshold endpoint", ) _append_row(rows, body, "body/default balanced return-bound point") _append_row( @@ -188,13 +299,17 @@ def build_consolidated_frontier(run_tags: list[str], caps: list[float]) -> dict[ _append_row( rows, eligible.sort_values( - ["alpha01_realized_total_return", "alpha01_markov_loss_cap"], + ["alpha01_realized_total_return", "alpha01_markov_loss_threshold"], ascending=[False, True], ).iloc[0], "max-return economic endpoint", ) for cap in caps: - _append_row(rows, _best_under_cap(eligible, cap), f"highest return under cap<={cap:g}") + _append_row( + rows, + _best_under_threshold(eligible, cap), + f"highest return under threshold<={cap:g}", + ) by_run = [] for run_tag, run_df in raw.groupby("run_tag", sort=False): @@ -215,7 +330,9 @@ def build_consolidated_frontier(run_tags: list[str], caps: list[float]) -> dict[ 9, ), "best_return": round(float(run_df["alpha01_realized_total_return"].max()), 6), - "min_markov_cap": round(float(run_df["alpha01_markov_loss_cap"].min()), 9), + "min_markov_threshold": round( + float(run_df["alpha01_markov_loss_threshold"].min()), 9 + ), } ) @@ -231,9 +348,15 @@ def build_consolidated_frontier(run_tags: list[str], caps: list[float]) -> dict[ ), "body_selection": ( f"highest realized return among eligible finite-grid policies with " - f"Markov_cap <= {DEFAULT_BODY_MARKOV_CAP:g}; falls back to the legacy " + f"exact Markov_threshold <= {DEFAULT_BODY_MARKOV_THRESHOLD:g}; " + "falls back to the legacy " "balanced normalized return/bound/V score only if no eligible policy " - "exists under that declared cap" + "exists under that declared threshold" + ), + "bound_semantics": ( + "Markov_threshold = weighted endpoint budget B_u + sqrt(alpha); " + "Markov_cap = tau + residual endpoint premium + solver slack + sqrt(alpha). " + "The exact threshold drives selection." ), "caps": caps, "role_semantics": "finite-grid frontier roles, not continuous optima", @@ -245,6 +368,7 @@ def build_consolidated_frontier(run_tags: list[str], caps: list[float]) -> dict[ "eligible_all_alpha_return_floor_policies": int(len(eligible)), "nonpass_or_below_floor_policies": int(len(deduped) - len(eligible)), }, + "certificate_semantics_audit": semantics_audit, "by_run": by_run, "rows": rows, } diff --git a/scripts/search/build_pool93_ijds_consolidated_governance.py b/scripts/search/build_pool93_ijds_consolidated_governance.py index 108c119..2d06995 100644 --- a/scripts/search/build_pool93_ijds_consolidated_governance.py +++ b/scripts/search/build_pool93_ijds_consolidated_governance.py @@ -9,7 +9,7 @@ from typing import Any ROOT = Path(__file__).resolve().parents[2] -DEFAULT_CONSOLIDATED_TAG = "champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive" +DEFAULT_CONSOLIDATED_TAG = "champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2" DEFAULT_BODY_ROLE = "body/default balanced return-bound point" @@ -32,8 +32,8 @@ def build_governance(frontier_path: Path, *, body_role: str) -> dict[str, Any]: frontier = json.loads(frontier_path.read_text(encoding="utf-8")) rows = list(frontier.get("rows", [])) body = _find_role(rows, body_role) - strict_cap = _find_role(rows, "highest return under cap<=0.345") - low_cap = _find_role(rows, "minimum Markov-cap endpoint") + strict_threshold = _find_role(rows, "highest return under threshold<=0.345") + low_threshold = _find_role(rows, "minimum Markov-threshold endpoint") max_return = _find_role(rows, "max-return economic endpoint") return { "generated_at_utc": datetime.now(tz=UTC).isoformat(), @@ -41,18 +41,19 @@ def build_governance(frontier_path: Path, *, body_role: str) -> dict[str, Any]: "source_run_tags": frontier.get("source_run_tags", []), "counts": frontier.get("counts", {}), "selection_rule": frontier.get("selection_rule", {}), + "certificate_semantics_audit": frontier.get("certificate_semantics_audit", {}), "claim_hierarchy": { "status": "final", "paper_body_candidate": body_role, "paper_body_claim": ( "The selected pool93 body point is the highest-return eligible " - "finite-grid policy under the declared Markov-cap body ceiling " + "finite-grid policy under the declared exact Markov-threshold ceiling " "and passes all eight predeclared alpha checks." ), "appendix_frontier_candidates": [ - "minimum Markov-cap endpoint", - "highest return under cap<=0.345", - "highest return under cap<=0.36", + "minimum Markov-threshold endpoint", + "highest return under threshold<=0.345", + "highest return under threshold<=0.36", "max-return economic endpoint", ], "do_not_claim": [ @@ -64,16 +65,17 @@ def build_governance(frontier_path: Path, *, body_role: str) -> dict[str, Any]: "promotion_gate": [ "consolidated frontier generated from completed exact runs", "semantic-policy deduplication applied", + "policy-aware endpoint decomposition applied to every alpha=0.01 row", "selected body point passes 8/8 alpha checks", - "zero exact violation at alpha=0.01", + "zero realized risk-tolerance excess at alpha=0.01", "return exceeds declared return floor", - "A35 frontier and A36 funded-set grade audit are regenerated from retained artifacts", + "A35 frontier, A36 funded-set grade audit, and A40 matched baseline are regenerated from retained artifacts", ], }, "selected_candidates": { "paper_body": body, - "strict_cap_leq_0_345": strict_cap, - "minimum_markov_cap_endpoint": low_cap, + "strict_threshold_leq_0_345": strict_threshold, + "minimum_markov_threshold_endpoint": low_threshold, "max_return_economic_endpoint": max_return, }, "paper_artifacts": { @@ -81,6 +83,13 @@ def build_governance(frontier_path: Path, *, body_role: str) -> dict[str, Any]: "frontier_table_tex": "reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.tex", "funded_grade_audit_csv": "reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.csv", "funded_grade_audit_tex": "reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.tex", + "point_baseline_csv": "reports/crpto/tables/crpto_tableA40_pool93_point_baseline.csv", + "point_baseline_tex": "reports/crpto/tables/crpto_tableA40_pool93_point_baseline.tex", + "point_baseline_audit": ( + "models/experiments/champion_reopen/" + "champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/" + "portfolio/pool93_point_pd_baseline_audit.json" + ), }, } diff --git a/scripts/search/build_pool93_ijds_frontier_claim_table.py b/scripts/search/build_pool93_ijds_frontier_claim_table.py index 27d3961..195a0a8 100644 --- a/scripts/search/build_pool93_ijds_frontier_claim_table.py +++ b/scripts/search/build_pool93_ijds_frontier_claim_table.py @@ -6,6 +6,7 @@ import json from datetime import UTC, datetime from pathlib import Path +from typing import Any, cast import pandas as pd @@ -79,6 +80,7 @@ def build_frontier_table( candidate = claim_summary.get(key, {}) if not isinstance(candidate, dict) or "local_candidate_id" not in candidate: continue + candidate = cast(dict[str, Any], candidate) candidate_id = int(candidate["local_candidate_id"]) _append_unique( rows, diff --git a/scripts/search/build_pool93_point_baseline_audit.py b/scripts/search/build_pool93_point_baseline_audit.py new file mode 100644 index 0000000..78ce7ba --- /dev/null +++ b/scripts/search/build_pool93_point_baseline_audit.py @@ -0,0 +1,372 @@ +"""Build a matched point-PD baseline for the selected pool93 decision. + +The audit holds the candidate universe, budget, concentration cap and risk +tolerance fixed. It changes only the decision uncertainty treatment: the +baseline constrains calibrated point PD, while the selected CRPTO allocation +uses its declared effective-PD policy. Outputs are isolated from the frozen +champion and contain no policy search. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from dataclasses import asdict +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) + +from scripts.optimize_portfolio_tradeoff import _parse_percent_series # noqa: E402 +from scripts.search.build_pool93_body_allocation_audit import ( # noqa: E402 + _load_role_row, + _manifest_path, + _policy_from_row, +) +from scripts.validate_alpha_gamma_bound import ( # noqa: E402 + DEFAULT_LGD, + DEFAULT_MAX_CONCENTRATION, + DEFAULT_TIME_LIMIT, + _compute_intervals_at_alpha, + _load_aligned_dataset, +) +from src.optimization.certificate_semantics import ( # noqa: E402 + FundedCertificateMetrics, + compute_funded_certificate_metrics, +) +from src.optimization.portfolio_model import ( # noqa: E402 + optimize_portfolio_allocation, + solution_allocation_vector, +) +from src.utils.script_helpers import resolve_repo_artifact_path, write_table # noqa: E402 + +DEFAULT_TAG = "champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2" +DEFAULT_ROLE = "body/default balanced return-bound point" +DEFAULT_FRONTIER = ( + ROOT + / "models/experiments/champion_reopen" + / DEFAULT_TAG + / "portfolio/pool93_ijds_consolidated_frontier.json" +) +DEFAULT_BODY_ALLOCATION = ( + ROOT + / "data/processed/experiments/champion_reopen" + / "champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive" + / "portfolio/pool93_body_allocation_alpha01.parquet" +) +DEFAULT_OUTPUT_DIR = ROOT / "models/experiments/champion_reopen" / DEFAULT_TAG / "portfolio" +DEFAULT_DATA_OUTPUT_DIR = ( + ROOT / "data/processed/experiments/champion_reopen" / DEFAULT_TAG / "portfolio" +) +DEFAULT_TABLE_DIR = ROOT / "reports/crpto/experiments" / DEFAULT_TAG + + +def _realized_return( + allocation: np.ndarray, + loan_amounts: np.ndarray, + int_rates: np.ndarray, + default_flag: np.ndarray, +) -> float: + exposure = allocation * loan_amounts + funded = allocation > 0.01 + contributions = np.where( + funded & (default_flag.astype(int) == 1), + -DEFAULT_LGD * exposure, + np.where(funded, int_rates * exposure, 0.0), + ) + return float(contributions.sum()) + + +def _economic_metrics( + *, + allocation: np.ndarray, + loan_amounts: np.ndarray, + int_rates: np.ndarray, + pd_point: np.ndarray, +) -> dict[str, float]: + exposure = allocation * loan_amounts + return { + "total_allocated": float(exposure.sum()), + "expected_return_gross": float(np.sum(exposure * int_rates)), + "expected_loss_point": float(np.sum(exposure * pd_point * DEFAULT_LGD)), + "expected_return_net_point": float( + np.sum(exposure * int_rates) - np.sum(exposure * pd_point * DEFAULT_LGD) + ), + } + + +def _comparison_table( + point: dict[str, Any], + selected: dict[str, Any], +) -> pd.DataFrame: + point_return = float(point["realized_return"]) + rows = [] + for label, payload in ( + ("Point-PD two-stage LP", point), + ("Selected CRPTO", selected), + ): + rows.append( + { + "policy": label, + "realized_return": float(payload["realized_return"]), + "return_cost_vs_point_pct": 100.0 + * (point_return - float(payload["realized_return"])) + / point_return, + "n_funded": int(payload["certificate"]["n_funded"]), + "weighted_default_rate": float(payload["certificate"]["weighted_outcome"]), + "V_alpha01": float(payload["certificate"]["weighted_miscoverage"]), + "Gamma_CP_alpha01": float(payload["certificate"]["gamma_cp"]), + "endpoint_budget_alpha01": float(payload["certificate"]["endpoint_budget"]), + "Markov_threshold_alpha01": float(payload["certificate"]["markov_loss_threshold"]), + "expected_return_net_point": float(payload["expected_return_net_point"]), + } + ) + return pd.DataFrame(rows) + + +def _format_comparison_tex(table: pd.DataFrame) -> str: + """Render the compact comparison used in the paper and supplement.""" + lines = [ + "\\begin{tabular}{lrrrrr}", + "\\toprule", + ( + "Policy & Realized return & Weighted default & " + "$\\Gamma_{\\mathrm{CP}}$ & $B_u$ & Markov threshold \\\\" + ), + "\\midrule", + ] + for row in table.to_dict("records"): + lines.append( + f"{row['policy']} & \\${float(row['realized_return']):,.2f} & " + f"{float(row['weighted_default_rate']):.6f} & " + f"{float(row['Gamma_CP_alpha01']):.6f} & " + f"{float(row['endpoint_budget_alpha01']):.6f} & " + f"{float(row['Markov_threshold_alpha01']):.6f} \\\\" + ) + lines.extend(["\\bottomrule", "\\end{tabular}", ""]) + return "\n".join(lines) + + +def _certificate_payload( + certificate: FundedCertificateMetrics, + *, + realized_return: float, + economic: dict[str, float], + solver_status: str, +) -> dict[str, Any]: + return { + "solver_status": solver_status, + "realized_return": realized_return, + **economic, + "certificate": asdict(certificate), + } + + +def build_audit( + *, + frontier_path: Path, + body_allocation_path: Path, + role: str, + alpha: float, + output_dir: Path, + data_output_dir: Path, + table_dir: Path, + threads: int, +) -> dict[str, Any]: + row = _load_role_row(frontier_path, role) + policy = _policy_from_row(row) + manifest = json.loads(_manifest_path(str(row["run_tag"])).read_text(encoding="utf-8")) + interval_path = resolve_repo_artifact_path(manifest["conformal_intervals_path"], root=ROOT) + aligned = _load_aligned_dataset( + conformal_intervals_path=str(interval_path), + max_candidates=int(manifest.get("max_candidates", 0) or 0), + random_state=int(manifest.get("random_state", 42)), + ) + pd_point, pd_low, pd_high = _compute_intervals_at_alpha(aligned, alpha) + y_true = pd.to_numeric(aligned["y_true"], errors="coerce").fillna(0.0).to_numpy(float) + default_flag = ( + pd.to_numeric(aligned["default_flag"], errors="coerce").fillna(0).to_numpy(dtype=int) + ) + loan_amounts = pd.to_numeric(aligned["loan_amnt"], errors="coerce").fillna(1.0).to_numpy(float) + int_rates = _parse_percent_series(aligned["int_rate"]) + risk_tolerance = float(policy["risk_tolerance"]) + solution = optimize_portfolio_allocation( + loans=aligned, + pd_point=pd_point, + pd_low=pd_low, + pd_high=pd_high, + lgd=np.full(len(aligned), DEFAULT_LGD, dtype=float), + int_rates=int_rates, + total_budget=float(manifest.get("budget", 1_000_000.0)), + max_concentration=DEFAULT_MAX_CONCENTRATION, + max_portfolio_pd=risk_tolerance, + robust=False, + uncertainty_aversion=0.0, + min_budget_utilization=float(policy["min_budget_utilization"]), + pd_cap_slack_penalty=float(policy["pd_cap_slack_penalty"]), + pd_constraint_override=pd_point, + time_limit=DEFAULT_TIME_LIMIT, + threads=max(1, int(threads)), + solver_backend=str(policy["solver_backend"]), + ) + point_allocation = solution_allocation_vector(solution, len(aligned)) + point_exposure = point_allocation * loan_amounts + point_weights = point_exposure / max(float(point_exposure.sum()), 1e-12) + point_certificate = compute_funded_certificate_metrics( + point_weights, + outcomes=y_true, + pd_point=pd_point, + pd_high=pd_high, + pd_effective=pd_point, + alpha=alpha, + risk_tolerance=risk_tolerance, + pd_cap_slack=float(solution.get("pd_cap_slack", 0.0)), + ) + point_payload = _certificate_payload( + point_certificate, + realized_return=_realized_return( + point_allocation, + loan_amounts, + int_rates, + default_flag, + ), + economic=_economic_metrics( + allocation=point_allocation, + loan_amounts=loan_amounts, + int_rates=int_rates, + pd_point=pd_point, + ), + solver_status=str(solution.get("solver_status", "unknown")), + ) + + funded = pd.read_parquet(body_allocation_path) + selected_weights = funded["funded_weight"].to_numpy(float) + selected_point = funded["pd_point_alpha01"].to_numpy(float) + selected_high = funded["pd_high_alpha01"].to_numpy(float) + selected_effective = funded["effective_pd"].to_numpy(float) + selected_outcomes = funded["default_flag"].to_numpy(float) + selected_exposure = funded["funded_exposure"].to_numpy(float) + selected_rates = _parse_percent_series(funded["int_rate"]) + selected_certificate = compute_funded_certificate_metrics( + selected_weights, + outcomes=selected_outcomes, + pd_point=selected_point, + pd_high=selected_high, + pd_effective=selected_effective, + alpha=alpha, + risk_tolerance=risk_tolerance, + ) + selected_payload = _certificate_payload( + selected_certificate, + realized_return=float(funded["realized_return"].sum()), + economic={ + "total_allocated": float(selected_exposure.sum()), + "expected_return_gross": float(np.sum(selected_exposure * selected_rates)), + "expected_loss_point": float(np.sum(selected_exposure * selected_point * DEFAULT_LGD)), + "expected_return_net_point": float( + np.sum(selected_exposure * selected_rates) + - np.sum(selected_exposure * selected_point * DEFAULT_LGD) + ), + }, + solver_status="frozen_selected_allocation", + ) + + table = _comparison_table(point_payload, selected_payload) + table_paths = write_table( + "crpto_tableA40_pool93_point_baseline", + table, + table_dir=table_dir, + root=ROOT, + float_precision=6, + ) + table_paths[1].write_text( + _format_comparison_tex(table), + encoding="utf-8", + newline="", + ) + output_dir.mkdir(parents=True, exist_ok=True) + data_output_dir.mkdir(parents=True, exist_ok=True) + point_rows = aligned.loc[point_allocation > 0.01].copy() + point_rows["allocation"] = point_allocation[point_allocation > 0.01] + point_rows["funded_exposure"] = point_exposure[point_allocation > 0.01] + point_rows["funded_weight"] = point_weights[point_allocation > 0.01] + point_rows_path = data_output_dir / "pool93_point_pd_baseline_alpha01.parquet" + point_rows.to_parquet(point_rows_path, index=False) + + return_cost = float(point_payload["realized_return"]) - float( + selected_payload["realized_return"] + ) + payload = { + "schema_version": "2026-07-09.1", + "generated_at_utc": datetime.now(tz=UTC).isoformat(), + "run_tag": DEFAULT_TAG, + "comparison": "matched point-PD two-stage LP versus selected CRPTO", + "fixed_design": { + "candidate_universe": len(aligned), + "budget": float(manifest.get("budget", 1_000_000.0)), + "risk_tolerance": risk_tolerance, + "max_concentration": DEFAULT_MAX_CONCENTRATION, + "alpha": alpha, + }, + "point_pd_baseline": point_payload, + "selected_crpto": selected_payload, + "contrasts": { + "realized_return_cost": return_cost, + "realized_return_cost_pct": 100.0 + * return_cost + / float(point_payload["realized_return"]), + "weighted_default_rate_reduction": point_certificate.weighted_outcome + - selected_certificate.weighted_outcome, + "weighted_miscoverage_reduction": point_certificate.weighted_miscoverage + - selected_certificate.weighted_miscoverage, + "markov_threshold_reduction": point_certificate.markov_loss_threshold + - selected_certificate.markov_loss_threshold, + }, + "claim_boundary": ( + "Frozen OOT matched-policy audit; it quantifies a return-risk trade-off and " + "does not establish causal, prospective or universal dominance." + ), + "outputs": { + "point_funded_rows": str(point_rows_path), + "table_csv": str(table_paths[0]), + "table_tex": str(table_paths[1]), + }, + } + output_path = output_dir / "pool93_point_pd_baseline_audit.json" + output_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8", newline="") + return payload + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--frontier", default=str(DEFAULT_FRONTIER)) + parser.add_argument("--body-allocation", default=str(DEFAULT_BODY_ALLOCATION)) + parser.add_argument("--role", default=DEFAULT_ROLE) + parser.add_argument("--alpha", type=float, default=0.01) + parser.add_argument("--threads", type=int, default=1) + parser.add_argument("--output-dir", default=str(DEFAULT_OUTPUT_DIR)) + parser.add_argument("--data-output-dir", default=str(DEFAULT_DATA_OUTPUT_DIR)) + parser.add_argument("--table-dir", default=str(DEFAULT_TABLE_DIR)) + args = parser.parse_args(argv) + payload = build_audit( + frontier_path=Path(args.frontier).resolve(), + body_allocation_path=Path(args.body_allocation).resolve(), + role=str(args.role), + alpha=float(args.alpha), + output_dir=Path(args.output_dir).resolve(), + data_output_dir=Path(args.data_output_dir).resolve(), + table_dir=Path(args.table_dir).resolve(), + threads=max(1, int(args.threads)), + ) + print(json.dumps(payload["contrasts"], indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/search/export_pool93_policy_aware_frontier.py b/scripts/search/export_pool93_policy_aware_frontier.py new file mode 100644 index 0000000..17e505b --- /dev/null +++ b/scripts/search/export_pool93_policy_aware_frontier.py @@ -0,0 +1,143 @@ +"""Export the policy-aware pool93 frontier as CSV and LaTeX.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +import pandas as pd + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) + +DEFAULT_TAG = "champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2" +DEFAULT_FRONTIER = ( + ROOT + / "models/experiments/champion_reopen" + / DEFAULT_TAG + / "portfolio/pool93_ijds_consolidated_frontier.json" +) +DEFAULT_TABLE_DIR = ROOT / "reports/crpto/experiments" / DEFAULT_TAG +DEFAULT_TABLE_NAME = "crpto_tableA35_pool93_ijds_frontier_policy_aware_v2" + +ROLE_ORDER = ( + "minimum Markov-threshold endpoint", + "lowest realized V return-bound point", + "highest return under threshold<=0.3", + "highest return under threshold<=0.32", + "highest return under threshold<=0.345", + "body/default balanced return-bound point", + "highest return under threshold<=0.36", + "highest return under threshold<=0.45", + "max-return economic endpoint", +) +ROLE_LABELS = { + "minimum Markov-threshold endpoint": "Minimum Markov-threshold endpoint", + "lowest realized V return-bound point": "Low-threshold balanced endpoint", + "highest return under threshold<=0.3": "Highest return under threshold <= 0.30", + "highest return under threshold<=0.32": "Highest return under threshold <= 0.32", + "highest return under threshold<=0.345": "Highest return under threshold <= 0.345", + "body/default balanced return-bound point": "Body/default balanced point", + "highest return under threshold<=0.36": "Highest return under threshold <= 0.36", + "highest return under threshold<=0.45": "Highest return under threshold <= 0.45", + "max-return economic endpoint": "Max-return economic endpoint", +} + + +def build_table(frontier: dict[str, Any]) -> pd.DataFrame: + """Build the compact publication table in its declared role order.""" + rows_by_role = {str(row["role"]): dict(row) for row in frontier.get("rows", [])} + missing = [role for role in ROLE_ORDER if role not in rows_by_role] + if missing: + raise ValueError(f"Policy-aware frontier is missing roles: {missing}") + rows = [] + for role in ROLE_ORDER: + row = rows_by_role[role] + rows.append( + { + "role": ROLE_LABELS[role], + "source_run": str(row["run_label"]), + "candidate_id": int(row["local_candidate_id"]), + "policy_family": str(row["family"]), + "risk_tolerance": float(row["risk_tolerance"]), + "policy_mode": str(row["policy_mode"]), + "gamma": float(row["gamma"]), + "uncertainty_aversion": float(row["uncertainty_aversion"]), + "realized_return": float(row["return"]), + "return_floor_surplus": float(row["return_floor_surplus"]), + "Gamma_CP_alpha01": float(row["Gamma_CP"]), + "Gamma_residual_alpha01": float(row["Gamma_residual"]), + "V_alpha01": float(row["V"]), + "endpoint_budget_alpha01": float(row["endpoint_budget"]), + "endpoint_budget_upper_alpha01": float(row["endpoint_budget_upper"]), + "Markov_threshold_alpha01": float(row["Markov_threshold"]), + "Markov_cap_alpha01": float(row["Markov_cap"]), + "alpha_grid_pass": str(row["alpha_pass"]), + "n_funded_mean": float(row["n_funded_mean"]), + } + ) + return pd.DataFrame(rows) + + +def _format_tex(table: pd.DataFrame) -> str: + lines = [ + "\\begin{tabular}{llrrrrrr}", + "\\toprule", + ( + "Role & Source & Return & $\\Gamma_{\\mathrm{CP}}$ & " + "$\\Gamma_{\\mathrm{res}}$ & $V$ & Markov threshold & Pass \\\\" + ), + "\\midrule", + ] + for row in table.to_dict("records"): + role = str(row["role"]).replace("<=", "$\\leq$") + source = str(row["source_run"]).replace("_", " ") + lines.append( + f"{role} & {source} & {float(row['realized_return']):,.2f} & " + f"{float(row['Gamma_CP_alpha01']):.6f} & " + f"{float(row['Gamma_residual_alpha01']):.6f} & " + f"{float(row['V_alpha01']):.6f} & " + f"{float(row['Markov_threshold_alpha01']):.6f} & " + f"{row['alpha_grid_pass']} \\\\" + ) + lines.extend(["\\bottomrule", "\\end{tabular}", ""]) + return "\n".join(lines) + + +def write_frontier_table(table: pd.DataFrame, *, table_dir: Path, table_name: str) -> None: + """Write the full audit CSV and compact publication LaTeX table.""" + table_dir.mkdir(parents=True, exist_ok=True) + csv_path = table_dir / f"{table_name}.csv" + tex_path = table_dir / f"{table_name}.tex" + outputs = { + csv_path: table.to_csv(index=False, lineterminator="\n"), + tex_path: _format_tex(table), + } + for path, content in outputs.items(): + if path.exists() and path.read_text(encoding="utf-8") == content: + continue + path.write_text(content, encoding="utf-8", newline="") + print(f"Wrote {csv_path.relative_to(ROOT).as_posix()}") + print(f"Wrote {tex_path.relative_to(ROOT).as_posix()}") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--frontier", default=str(DEFAULT_FRONTIER)) + parser.add_argument("--table-dir", default=str(DEFAULT_TABLE_DIR)) + parser.add_argument("--table-name", default=DEFAULT_TABLE_NAME) + args = parser.parse_args(argv) + frontier = json.loads(Path(args.frontier).read_text(encoding="utf-8")) + write_frontier_table( + build_table(frontier), + table_dir=Path(args.table_dir).resolve(), + table_name=str(args.table_name), + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/search/run_conformal_reopen_search.py b/scripts/search/run_conformal_reopen_search.py index ee4b8a4..38d3fc4 100644 --- a/scripts/search/run_conformal_reopen_search.py +++ b/scripts/search/run_conformal_reopen_search.py @@ -9,6 +9,7 @@ import subprocess import sys from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -33,6 +34,57 @@ from src.models.venn_abers import VennAbersScoreCalibrator from src.utils.pipeline_topology import load_profile_config + +@dataclass(frozen=True) +class Phase1Result: + shortlist: pd.DataFrame + aggregated: pd.DataFrame + inner_runs: list[dict[str, Any]] + aggregate_path: str + inner_search_path: str + inner_search_winner: dict[str, Any] + resume_meta: dict[str, Any] | None = None + + +@dataclass(frozen=True) +class Phase1ConfirmationResult: + candidates: list[dict[str, Any]] + frame: pd.DataFrame + best_namespace: str + final_policy: dict[str, Any] + final_sets: dict[str, Any] + final_decision: str + final_namespace: str + + +@dataclass(frozen=True) +class PromotionResult: + final_policy: dict[str, Any] + final_sets: dict[str, Any] + final_decision: str + final_namespace: str + phase2_summary: dict[str, Any] | None + + +@dataclass +class Phase2SearchState: + paths: dict[str, Path] + run_tag: str + upstream_run_tag: str + total_design_runs: int + design_source: str + completed: list[dict[str, Any]] + skipped: list[dict[str, Any]] + failed: list[dict[str, Any]] + + +@dataclass(frozen=True) +class Phase2CalibratorFit: + method: str + path: Path + metrics: dict[str, float] + + REPO_ROOT = Path(__file__).resolve().parents[2] @@ -915,6 +967,301 @@ def _build_resume_shortlist( return shortlist, resume_meta +def _phase2_methods(phase2_cfg: dict[str, Any]) -> list[str]: + return [ + str(method).strip().lower() + for method in phase2_cfg.get("calibrators", ["venn_abers", "isotonic", "platt", "beta"]) + if str(method).strip() + ] + + +def _write_phase2_state_progress( + state: Phase2SearchState, + running: list[dict[str, Any]], +) -> None: + _write_phase2_progress( + path=state.paths["phase2_progress"], + run_tag=state.run_tag, + upstream_run_tag=state.upstream_run_tag, + total_design_runs=state.total_design_runs, + completed=state.completed, + running=running, + skipped=state.skipped, + failed=state.failed, + design_source=state.design_source, + ) + + +def _phase2_baseline_metrics( + *, + calibrator_dir: Path, + upstream_run_tag: str, +) -> dict[str, float] | None: + baseline_path = calibrator_dir / "venn_abers.pkl" + try: + _resolved, baseline_metrics = _fit_calibrator( + method="venn_abers", + output_path=baseline_path, + upstream_run_tag=upstream_run_tag, + ) + except Exception: + return None + return baseline_metrics + + +def _numeric_metric_payload(metrics: dict[str, float]) -> dict[str, float]: + return {key: float(value) for key, value in metrics.items() if isinstance(value, int | float)} + + +def _phase2_metric_blocked( + *, + calibration_metrics: dict[str, float], + baseline_metrics: dict[str, float] | None, + max_metric_degradation: dict[str, Any], +) -> bool: + if baseline_metrics is None: + return False + return any( + float(calibration_metrics.get(metric_name, float("inf"))) + > float(baseline_metrics.get(metric_name, 0.0)) + + float(max_metric_degradation.get(metric_name, 0.0)) + for metric_name in max_metric_degradation + ) + + +def _fit_phase2_calibrator_or_skip( + *, + method_name: str, + calibrator_dir: Path, + state: Phase2SearchState, + baseline_metrics: dict[str, float] | None, + max_metric_degradation: dict[str, Any], +) -> Phase2CalibratorFit | None: + calibrator_path = calibrator_dir / f"{method_name}.pkl" + _write_phase2_state_progress( + state, + running=[{"stage": "fit_calibrator", "calibrator_method": method_name}], + ) + try: + resolved_method, calibration_metrics = _fit_calibrator( + method=method_name, + output_path=calibrator_path, + upstream_run_tag=state.upstream_run_tag, + ) + except Exception as exc: + state.failed.append( + { + "stage": "fit_calibrator", + "calibrator_method": method_name, + "error": repr(exc), + } + ) + _write_phase2_state_progress(state, running=[]) + raise + + if _phase2_metric_blocked( + calibration_metrics=calibration_metrics, + baseline_metrics=baseline_metrics, + max_metric_degradation=max_metric_degradation, + ): + state.skipped.append( + { + "stage": "calibrator_gate", + "calibrator_method": resolved_method, + "reason": "metric_degradation_gate", + "calibration_metrics": _numeric_metric_payload(calibration_metrics), + } + ) + _write_phase2_state_progress(state, running=[]) + return None + + return Phase2CalibratorFit( + method=resolved_method, + path=calibrator_path, + metrics=calibration_metrics, + ) + + +def _phase2_candidate_namespace( + *, + run_tag: str, + calibrator_method: str, + design: dict[str, Any], +) -> str: + return _namespace( + run_tag, + "phase2", + calibrator_method, + f"rank-{int(design.get('selection_rank', 1))}", + ) + + +def _run_phase2_holdout_candidate( + *, + state: Phase2SearchState, + env: dict[str, str], + calibrator_fit: Phase2CalibratorFit, + design: dict[str, Any], + alpha_candidates_95: list[float], + tuning_holdout_ratios: list[float], + inner_random_states: list[int], +) -> dict[str, Any]: + design_norm = _normalize_design_row(design) + selection_rank = int(design.get("selection_rank", 1)) + namespace = _phase2_candidate_namespace( + run_tag=state.run_tag, + calibrator_method=calibrator_fit.method, + design=design, + ) + running_entry = { + "stage": "generate_intervals", + "artifact_namespace": namespace, + "calibrator_method": calibrator_fit.method, + "selection_rank": selection_rank, + } + _write_phase2_state_progress(state, running=[running_entry]) + try: + _run_python( + "scripts/generate_conformal_intervals.py", + [ + "--artifact_namespace", + namespace, + "--evaluation_scope", + "holdout", + "--calibrator_override_path", + str(calibrator_fit.path), + "--tuning_holdout_ratio", + str(float(tuning_holdout_ratios[0])), + "--tuning_random_state", + str(int(inner_random_states[0])), + "--alpha_candidates_95", + ",".join(str(x) for x in alpha_candidates_95), + *_design_args(design_norm), + ], + env, + ) + except Exception as exc: + state.failed.append({**running_entry, "error": repr(exc)}) + _write_phase2_state_progress(state, running=[]) + raise + + payload = _load_pickle(_resolve_run_paths(namespace)["results"]) + metrics_90 = dict(payload.get("metrics_90", {}) or {}) + state.completed.append( + { + "artifact_namespace": namespace, + "calibrator_method": calibrator_fit.method, + "selection_rank": selection_rank, + "coverage_90": float(metrics_90.get("empirical_coverage", 0.0)), + "avg_width_90": float(metrics_90.get("avg_interval_width", 1.0)), + } + ) + _write_phase2_state_progress(state, running=[]) + return { + "artifact_namespace": namespace, + "calibrator_method": calibrator_fit.method, + "phase2_design_source": state.design_source, + "selection_rank": selection_rank, + **design_norm, + "holdout_coverage": float(metrics_90.get("empirical_coverage", 0.0)), + "holdout_width": float(metrics_90.get("avg_interval_width", 1.0)), + "calibrator_ece": float(calibrator_fit.metrics.get("ece", float("inf"))), + "calibrator_adaptive_ece": float(calibrator_fit.metrics.get("adaptive_ece", float("inf"))), + "calibrator_brier": float(calibrator_fit.metrics.get("brier_score", float("inf"))), + "calibrator_log_loss": float(calibrator_fit.metrics.get("log_loss", float("inf"))), + "calibrator_phi_brier": float(calibrator_fit.metrics.get("phi_brier_score", 0.0)), + "calibrator_phi_log_loss": float(calibrator_fit.metrics.get("phi_log_loss", 0.0)), + } + + +def _empty_phase2_frame() -> pd.DataFrame: + return pd.DataFrame( + { + "artifact_namespace": pd.Series(dtype="object"), + "calibrator_method": pd.Series(dtype="object"), + "phase2_design_source": pd.Series(dtype="object"), + "selection_rank": pd.Series(dtype="int64"), + "holdout_coverage": pd.Series(dtype="float64"), + "holdout_width": pd.Series(dtype="float64"), + "calibrator_ece": pd.Series(dtype="float64"), + "calibrator_adaptive_ece": pd.Series(dtype="float64"), + "calibrator_brier": pd.Series(dtype="float64"), + "calibrator_log_loss": pd.Series(dtype="float64"), + "calibrator_phi_brier": pd.Series(dtype="float64"), + "calibrator_phi_log_loss": pd.Series(dtype="float64"), + } + ) + + +def _rank_phase2_candidates(phase2_df: pd.DataFrame) -> pd.DataFrame: + ranked = phase2_df.copy() + ranked["coverage_gap_abs"] = (ranked["holdout_coverage"] - 0.90).abs() + return ranked.sort_values( + by=[ + "coverage_gap_abs", + "holdout_width", + "calibrator_ece", + "calibrator_adaptive_ece", + "calibrator_brier", + "calibrator_phi_brier", + "selection_rank", + ], + ascending=[True, True, True, True, True, False, True], + ).reset_index(drop=True) + + +def _phase2_no_candidate_result( + *, + paths: dict[str, Path], + design_source: str, +) -> tuple[str, dict[str, Any], dict[str, Any], dict[str, Any]]: + return ( + "policy_review_candidate", + {}, + {}, + { + "search_path": str(paths["phase2_search"]), + "best_candidate": {}, + "status": "no_noninferior_calibrator_candidate", + "phase2_design_source": design_source, + }, + ) + + +def _run_phase2_final_candidate( + *, + run_tag: str, + env: dict[str, str], + calibrator_dir: Path, + phase2_best: dict[str, Any], + alpha_candidates_95: list[float], + partition_candidates: list[str], + partition_probability_sources: list[str], + n_score_bins_candidates: list[int], + fallback_modes: list[str], + score_scale_families: list[str], + calibration_fractions: list[float], + sidecar_cfg: dict[str, Any], +) -> dict[str, Any]: + phase2_calibrator_path = calibrator_dir / f"{phase2_best['calibrator_method']}.pkl" + return _run_phase1_oot_candidate( + run_tag=run_tag, + rank=1, + design=phase2_best, + env=env, + alpha_candidates_95=alpha_candidates_95, + partition_candidates=partition_candidates, + partition_probability_sources=partition_probability_sources, + n_score_bins_candidates=n_score_bins_candidates, + fallback_modes=fallback_modes, + score_scale_families=score_scale_families, + calibration_fractions=calibration_fractions, + sidecar_cfg=sidecar_cfg, + calibrator_override_path=str(phase2_calibrator_path), + phase_prefix="phase2", + ) + + def _run_phase2_search( *, run_tag: str, @@ -938,270 +1285,72 @@ def _run_phase2_search( paths = _reopen_artifact_paths(run_tag) models_dir = paths["models_dir"] calibrator_dir = models_dir / "phase2_calibrators" - calibrator_rows: list[dict[str, Any]] = [] top_designs, design_source = _phase2_top_designs( aggregated=aggregated, phase1_candidates_frame=phase1_candidates_frame, top_k=int(phase2_cfg.get("top_k_designs", 3)), ) - phase2_methods = [ - str(method).strip().lower() - for method in phase2_cfg.get("calibrators", ["venn_abers", "isotonic", "platt", "beta"]) - if str(method).strip() - ] - phase2_completed: list[dict[str, Any]] = [] - phase2_skipped: list[dict[str, Any]] = [] - phase2_failed: list[dict[str, Any]] = [] + phase2_methods = _phase2_methods(phase2_cfg) total_design_runs = len(phase2_methods) * len(top_designs) - _write_phase2_progress( - path=paths["phase2_progress"], + state = Phase2SearchState( + paths=paths, run_tag=run_tag, upstream_run_tag=upstream_run_tag, total_design_runs=total_design_runs, - completed=phase2_completed, - running=[], - skipped=phase2_skipped, - failed=phase2_failed, design_source=design_source, + completed=[], + skipped=[], + failed=[], + ) + _write_phase2_state_progress(state, running=[]) + baseline_metrics = _phase2_baseline_metrics( + calibrator_dir=calibrator_dir, + upstream_run_tag=upstream_run_tag, ) - baseline_metrics: dict[str, float] | None = None - baseline_path = calibrator_dir / "venn_abers.pkl" - try: - _resolved, baseline_metrics = _fit_calibrator( - method="venn_abers", - output_path=baseline_path, - upstream_run_tag=upstream_run_tag, - ) - except Exception: - baseline_metrics = None max_metric_degradation = dict(phase2_cfg.get("max_metric_degradation", {}) or {}) + calibrator_rows: list[dict[str, Any]] = [] for method_name in phase2_methods: - calibrator_path = calibrator_dir / f"{method_name}.pkl" - _write_phase2_progress( - path=paths["phase2_progress"], - run_tag=run_tag, - upstream_run_tag=upstream_run_tag, - total_design_runs=total_design_runs, - completed=phase2_completed, - running=[{"stage": "fit_calibrator", "calibrator_method": method_name}], - skipped=phase2_skipped, - failed=phase2_failed, - design_source=design_source, + calibrator_fit = _fit_phase2_calibrator_or_skip( + method_name=method_name, + calibrator_dir=calibrator_dir, + state=state, + baseline_metrics=baseline_metrics, + max_metric_degradation=max_metric_degradation, ) - try: - resolved_method, calibration_metrics = _fit_calibrator( - method=method_name, - output_path=calibrator_path, - upstream_run_tag=upstream_run_tag, - ) - except Exception as exc: - phase2_failed.append( - { - "stage": "fit_calibrator", - "calibrator_method": method_name, - "error": repr(exc), - } - ) - _write_phase2_progress( - path=paths["phase2_progress"], - run_tag=run_tag, - upstream_run_tag=upstream_run_tag, - total_design_runs=total_design_runs, - completed=phase2_completed, - running=[], - skipped=phase2_skipped, - failed=phase2_failed, - design_source=design_source, - ) - raise - if baseline_metrics is not None: - metric_blocked = any( - float(calibration_metrics.get(metric_name, float("inf"))) - > float(baseline_metrics.get(metric_name, 0.0)) - + float(max_metric_degradation.get(metric_name, 0.0)) - for metric_name in max_metric_degradation - ) - if metric_blocked: - phase2_skipped.append( - { - "stage": "calibrator_gate", - "calibrator_method": resolved_method, - "reason": "metric_degradation_gate", - "calibration_metrics": { - key: float(value) - for key, value in calibration_metrics.items() - if isinstance(value, int | float) - }, - } - ) - _write_phase2_progress( - path=paths["phase2_progress"], - run_tag=run_tag, - upstream_run_tag=upstream_run_tag, - total_design_runs=total_design_runs, - completed=phase2_completed, - running=[], - skipped=phase2_skipped, - failed=phase2_failed, - design_source=design_source, - ) - continue + if calibrator_fit is None: + continue for design in top_designs.to_dict(orient="records"): - design_norm = _normalize_design_row(design) - namespace = _namespace( - run_tag, - "phase2", - resolved_method, - f"rank-{int(design.get('selection_rank', 1))}", - ) - running_entry = { - "stage": "generate_intervals", - "artifact_namespace": namespace, - "calibrator_method": resolved_method, - "selection_rank": int(design.get("selection_rank", 1)), - } - _write_phase2_progress( - path=paths["phase2_progress"], - run_tag=run_tag, - upstream_run_tag=upstream_run_tag, - total_design_runs=total_design_runs, - completed=phase2_completed, - running=[running_entry], - skipped=phase2_skipped, - failed=phase2_failed, - design_source=design_source, - ) - try: - _run_python( - "scripts/generate_conformal_intervals.py", - [ - "--artifact_namespace", - namespace, - "--evaluation_scope", - "holdout", - "--calibrator_override_path", - str(calibrator_path), - "--tuning_holdout_ratio", - str(float(tuning_holdout_ratios[0])), - "--tuning_random_state", - str(int(inner_random_states[0])), - "--alpha_candidates_95", - ",".join(str(x) for x in alpha_candidates_95), - *_design_args(design_norm), - ], - env, - ) - except Exception as exc: - phase2_failed.append({**running_entry, "error": repr(exc)}) - _write_phase2_progress( - path=paths["phase2_progress"], - run_tag=run_tag, - upstream_run_tag=upstream_run_tag, - total_design_runs=total_design_runs, - completed=phase2_completed, - running=[], - skipped=phase2_skipped, - failed=phase2_failed, - design_source=design_source, - ) - raise - payload = _load_pickle(_resolve_run_paths(namespace)["results"]) - metrics_90 = dict(payload.get("metrics_90", {}) or {}) - phase2_completed.append( - { - "artifact_namespace": namespace, - "calibrator_method": resolved_method, - "selection_rank": int(design.get("selection_rank", 1)), - "coverage_90": float(metrics_90.get("empirical_coverage", 0.0)), - "avg_width_90": float(metrics_90.get("avg_interval_width", 1.0)), - } - ) - _write_phase2_progress( - path=paths["phase2_progress"], - run_tag=run_tag, - upstream_run_tag=upstream_run_tag, - total_design_runs=total_design_runs, - completed=phase2_completed, - running=[], - skipped=phase2_skipped, - failed=phase2_failed, - design_source=design_source, - ) calibrator_rows.append( - { - "artifact_namespace": namespace, - "calibrator_method": resolved_method, - "phase2_design_source": design_source, - "selection_rank": int(design.get("selection_rank", 1)), - **design_norm, - "holdout_coverage": float(metrics_90.get("empirical_coverage", 0.0)), - "holdout_width": float(metrics_90.get("avg_interval_width", 1.0)), - "calibrator_ece": float(calibration_metrics.get("ece", float("inf"))), - "calibrator_adaptive_ece": float( - calibration_metrics.get("adaptive_ece", float("inf")) - ), - "calibrator_brier": float(calibration_metrics.get("brier_score", float("inf"))), - "calibrator_log_loss": float(calibration_metrics.get("log_loss", float("inf"))), - "calibrator_phi_brier": float(calibration_metrics.get("phi_brier_score", 0.0)), - "calibrator_phi_log_loss": float(calibration_metrics.get("phi_log_loss", 0.0)), - } + _run_phase2_holdout_candidate( + state=state, + env=env, + calibrator_fit=calibrator_fit, + design=design, + alpha_candidates_95=alpha_candidates_95, + tuning_holdout_ratios=tuning_holdout_ratios, + inner_random_states=inner_random_states, + ) ) phase2_df = pd.DataFrame(calibrator_rows) if phase2_df.empty: - phase2_df = pd.DataFrame( - columns=[ - "artifact_namespace", - "calibrator_method", - "phase2_design_source", - "selection_rank", - "holdout_coverage", - "holdout_width", - "calibrator_ece", - "calibrator_adaptive_ece", - "calibrator_brier", - "calibrator_log_loss", - "calibrator_phi_brier", - "calibrator_phi_log_loss", - ] - ) + phase2_df = _empty_phase2_frame() phase2_df.to_parquet(paths["phase2_search"], index=False) - return ( - "policy_review_candidate", - {}, - {}, - { - "search_path": str(paths["phase2_search"]), - "best_candidate": {}, - "status": "no_noninferior_calibrator_candidate", - "phase2_design_source": design_source, - }, + return _phase2_no_candidate_result( + paths=paths, + design_source=design_source, ) - phase2_df["coverage_gap_abs"] = (phase2_df["holdout_coverage"] - 0.90).abs() - phase2_df = phase2_df.sort_values( - by=[ - "coverage_gap_abs", - "holdout_width", - "calibrator_ece", - "calibrator_adaptive_ece", - "calibrator_brier", - "calibrator_phi_brier", - "selection_rank", - ], - ascending=[True, True, True, True, True, False, True], - ).reset_index(drop=True) + phase2_df = _rank_phase2_candidates(phase2_df) phase2_df.to_parquet(paths["phase2_search"], index=False) phase2_best = phase2_df.iloc[0].to_dict() - _namespace(run_tag, "phase2", "final") - phase2_calibrator_path = calibrator_dir / f"{phase2_best['calibrator_method']}.pkl" - final_candidate = _run_phase1_oot_candidate( + final_candidate = _run_phase2_final_candidate( run_tag=run_tag, - rank=1, - design=phase2_best, env=env, + calibrator_dir=calibrator_dir, + phase2_best=phase2_best, alpha_candidates_95=alpha_candidates_95, partition_candidates=partition_candidates, partition_probability_sources=partition_probability_sources, @@ -1210,8 +1359,6 @@ def _run_phase2_search( score_scale_families=score_scale_families, calibration_fractions=calibration_fractions, sidecar_cfg=sidecar_cfg, - calibrator_override_path=str(phase2_calibrator_path), - phase_prefix="phase2", ) final_policy = dict(final_candidate["policy_status"]) final_sets = dict(final_candidate["set_status"]) @@ -1274,7 +1421,7 @@ def _write_consolidated_status( _write_json(_reopen_artifact_paths(run_tag)["status"], status) -def main(argv: list[str] | None = None) -> int: +def _build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser() parser.add_argument("--run-tag", default=_default_run_tag()) parser.add_argument("--pipeline-profile", default="search_conformal_reopen_exhaustive") @@ -1297,103 +1444,139 @@ def main(argv: list[str] | None = None) -> int: action="store_true", help="Evaluate phase-2 calibrator tournament even when phase-1 already passes.", ) - args = parser.parse_args(argv) + return parser - run_tag = str(args.run_tag).strip() - upstream_run_tag = str(args.upstream_canonical_run_tag).strip() - resume_from_run_tag = str(args.resume_from_run_tag).strip() if args.resume_from_run_tag else "" - mode = "derived_resume" if resume_from_run_tag else "fresh_search" - profile = _profile_cfg(args.pipeline_profile) - cfg = _phase1_cfg(profile) - phase2_cfg = _phase2_cfg(profile) - sidecar_cfg = _sidecar_cfg(profile) - validation_cfg = _validation_cfg(profile) - output_paths = _reopen_artifact_paths(run_tag) +def _phase1_from_resume( + *, + resume_from_run_tag: str, + top_k_inner: int, + output_paths: dict[str, Path], +) -> Phase1Result: + source_paths = _reopen_artifact_paths(resume_from_run_tag) + if not source_paths["inner_aggregate"].exists(): + raise FileNotFoundError( + f"Resume source missing aggregate artifact: {source_paths['inner_aggregate']}" + ) + shortlist, resume_meta = _build_resume_shortlist( + source_run_tag=resume_from_run_tag, + top_k_inner=top_k_inner, + ) + shortlist.to_parquet(output_paths["phase1_shortlist"], index=False) + aggregated = pd.read_parquet(source_paths["inner_aggregate"]) + return Phase1Result( + shortlist=shortlist, + aggregated=aggregated, + inner_runs=[], + aggregate_path=str(source_paths["inner_aggregate"]), + inner_search_path=str(source_paths["inner_search"]), + inner_search_winner=_normalize_design_row(aggregated.iloc[0].to_dict()) + if not aggregated.empty + else {}, + resume_meta=resume_meta, + ) - env = os.environ.copy() - env["PIPELINE_RUN_TAG"] = run_tag - env["UPSTREAM_CANONICAL_RUN_TAG"] = upstream_run_tag - alpha_candidates_90 = cfg.get("alpha_candidates_90", [0.09, 0.095, 0.10, 0.105, 0.11, 0.12]) - alpha_candidates_95 = cfg.get("alpha_candidates_95", [0.045, 0.05, 0.055, 0.06]) - partition_candidates = cfg.get( - "partition_candidates", - ["grade", "score_decile_mondrian", "grade_x_scoreband_mondrian"], +def _run_phase1_inner_search( + *, + run_tag: str, + upstream_run_tag: str, + env: dict[str, str], + output_paths: dict[str, Path], + phase1_workers: int, + top_k_inner: int, + alpha_candidates_90: list[float], + alpha_candidates_95: list[float], + partition_candidates: list[str], + partition_probability_sources: list[str], + n_score_bins_candidates: list[int], + min_group_sizes: list[int], + fallback_modes: list[str], + score_scale_families: list[str], + calibration_fractions: list[float], + tuning_holdout_ratios: list[float], + inner_random_states: list[int], + resume_completed_inner: bool, +) -> Phase1Result: + inner_frames: list[pd.DataFrame] = [] + inner_runs: list[dict[str, Any]] = [] + specs = _phase1_inner_specs( + run_tag=run_tag, + calibration_fractions=calibration_fractions, + tuning_holdout_ratios=tuning_holdout_ratios, + inner_random_states=inner_random_states, ) - partition_probability_sources = cfg.get("partition_probability_sources", ["calibrated", "raw"]) - n_score_bins_candidates = cfg.get("n_score_bins_candidates", [5, 10, 15, 20]) - min_group_sizes = cfg.get("min_group_sizes", [100, 150, 250, 500, 1000]) - fallback_modes = cfg.get("fallback_modes", ["grade_then_global", "global_only"]) - score_scale_families = cfg.get( - "score_scale_families", - ["none", "bernoulli_sqrt", "bernoulli_sqrt_clipped_0.02", "bernoulli_sqrt_clipped_0.05"], + completed_runs: list[dict[str, Any]] = [] + failed_runs: list[dict[str, Any]] = [] + running: set[str] = set() + progress_path = output_paths["phase1_progress"] + _write_phase1_progress( + path=progress_path, + run_tag=run_tag, + upstream_run_tag=upstream_run_tag, + total=len(specs), + completed=completed_runs, + running=[], + failed=failed_runs, + workers=phase1_workers, ) - calibration_fractions = cfg.get("calibration_fractions", [0.25, 0.50, 0.75, 1.00]) - tuning_holdout_ratios = cfg.get("tuning_holdout_ratios", [0.20, 0.30]) - inner_random_states = cfg.get("inner_random_states", [42, 314, 2026]) - top_k_inner = int(validation_cfg.get("top_k_inner", 3)) - configured_workers = int(cfg.get("parallel_workers", 1) or 1) - phase1_workers = int(args.phase1_workers or configured_workers or 1) - phase1_workers = max(1, phase1_workers) - resume_completed_inner = not bool(args.no_resume_completed_inner) - resume_meta: dict[str, Any] | None = None - inner_runs: list[dict[str, Any]] = [] - if resume_from_run_tag: - source_paths = _reopen_artifact_paths(resume_from_run_tag) - if not source_paths["inner_aggregate"].exists(): - raise FileNotFoundError( - f"Resume source missing aggregate artifact: {source_paths['inner_aggregate']}" - ) - shortlist, resume_meta = _build_resume_shortlist( - source_run_tag=resume_from_run_tag, - top_k_inner=top_k_inner, - ) - shortlist.to_parquet(output_paths["phase1_shortlist"], index=False) - aggregate_path = str(source_paths["inner_aggregate"]) - inner_search_path = str(source_paths["inner_search"]) - aggregated = pd.read_parquet(source_paths["inner_aggregate"]) - inner_search_winner = ( - _normalize_design_row(aggregated.iloc[0].to_dict()) if not aggregated.empty else {} - ) - else: - inner_frames: list[pd.DataFrame] = [] - specs = _phase1_inner_specs( - run_tag=run_tag, - calibration_fractions=calibration_fractions, - tuning_holdout_ratios=tuning_holdout_ratios, - inner_random_states=inner_random_states, - ) - completed_runs: list[dict[str, Any]] = [] - failed_runs: list[dict[str, Any]] = [] - running: set[str] = set() - progress_path = output_paths["phase1_progress"] - _write_phase1_progress( - path=progress_path, - run_tag=run_tag, - upstream_run_tag=upstream_run_tag, - total=len(specs), - completed=completed_runs, - running=[], - failed=failed_runs, - workers=phase1_workers, + def record_success(tuning_df: pd.DataFrame, run_summary: dict[str, Any]) -> None: + inner_frames.append(tuning_df) + inner_runs.append(run_summary) + completed_runs.append( + { + "artifact_namespace": run_summary["artifact_namespace"], + "reused_existing": bool(run_summary.get("reused_existing", False)), + } ) - if phase1_workers == 1: + if phase1_workers == 1: + for spec in specs: + running = {str(spec["namespace"])} + _write_phase1_progress( + path=progress_path, + run_tag=run_tag, + upstream_run_tag=upstream_run_tag, + total=len(specs), + completed=completed_runs, + running=sorted(running), + failed=failed_runs, + workers=phase1_workers, + ) + tuning_df, run_summary = _run_phase1_inner_spec( + spec=spec, + env=env, + alpha_candidates_90=alpha_candidates_90, + alpha_candidates_95=alpha_candidates_95, + partition_candidates=partition_candidates, + partition_probability_sources=partition_probability_sources, + n_score_bins_candidates=n_score_bins_candidates, + fallback_modes=fallback_modes, + min_group_sizes=min_group_sizes, + score_scale_families=score_scale_families, + resume_completed=resume_completed_inner, + ) + record_success(tuning_df, run_summary) + running = set() + _write_phase1_progress( + path=progress_path, + run_tag=run_tag, + upstream_run_tag=upstream_run_tag, + total=len(specs), + completed=completed_runs, + running=[], + failed=failed_runs, + workers=phase1_workers, + ) + else: + with ThreadPoolExecutor(max_workers=phase1_workers) as executor: + future_to_spec = {} for spec in specs: - running = {str(spec["namespace"])} - _write_phase1_progress( - path=progress_path, - run_tag=run_tag, - upstream_run_tag=upstream_run_tag, - total=len(specs), - completed=completed_runs, - running=sorted(running), - failed=failed_runs, - workers=phase1_workers, - ) - tuning_df, run_summary = _run_phase1_inner_spec( + namespace = str(spec["namespace"]) + running.add(namespace) + future = executor.submit( + _run_phase1_inner_spec, spec=spec, env=env, alpha_candidates_90=alpha_candidates_90, @@ -1406,86 +1589,28 @@ def main(argv: list[str] | None = None) -> int: score_scale_families=score_scale_families, resume_completed=resume_completed_inner, ) - inner_frames.append(tuning_df) - inner_runs.append(run_summary) - completed_runs.append( - { - "artifact_namespace": run_summary["artifact_namespace"], - "reused_existing": bool(run_summary.get("reused_existing", False)), - } - ) - running = set() + future_to_spec[future] = spec _write_phase1_progress( path=progress_path, run_tag=run_tag, upstream_run_tag=upstream_run_tag, total=len(specs), completed=completed_runs, - running=[], + running=sorted(running), failed=failed_runs, workers=phase1_workers, ) - else: - with ThreadPoolExecutor(max_workers=phase1_workers) as executor: - future_to_spec = {} - for spec in specs: - namespace = str(spec["namespace"]) - running.add(namespace) - future = executor.submit( - _run_phase1_inner_spec, - spec=spec, - env=env, - alpha_candidates_90=alpha_candidates_90, - alpha_candidates_95=alpha_candidates_95, - partition_candidates=partition_candidates, - partition_probability_sources=partition_probability_sources, - n_score_bins_candidates=n_score_bins_candidates, - fallback_modes=fallback_modes, - min_group_sizes=min_group_sizes, - score_scale_families=score_scale_families, - resume_completed=resume_completed_inner, - ) - future_to_spec[future] = spec - _write_phase1_progress( - path=progress_path, - run_tag=run_tag, - upstream_run_tag=upstream_run_tag, - total=len(specs), - completed=completed_runs, - running=sorted(running), - failed=failed_runs, - workers=phase1_workers, - ) - for future in as_completed(future_to_spec): - spec = future_to_spec[future] - namespace = str(spec["namespace"]) - running.discard(namespace) - try: - tuning_df, run_summary = future.result() - except Exception as exc: - failed_runs.append( - { - "artifact_namespace": namespace, - "error": repr(exc), - } - ) - _write_phase1_progress( - path=progress_path, - run_tag=run_tag, - upstream_run_tag=upstream_run_tag, - total=len(specs), - completed=completed_runs, - running=sorted(running), - failed=failed_runs, - workers=phase1_workers, - ) - raise - inner_frames.append(tuning_df) - inner_runs.append(run_summary) - completed_runs.append( + for future in as_completed(future_to_spec): + spec = future_to_spec[future] + namespace = str(spec["namespace"]) + running.discard(namespace) + try: + tuning_df, run_summary = future.result() + except Exception as exc: + failed_runs.append( { - "artifact_namespace": run_summary["artifact_namespace"], - "reused_existing": bool(run_summary.get("reused_existing", False)), + "artifact_namespace": namespace, + "error": repr(exc), } ) _write_phase1_progress( @@ -1498,19 +1623,53 @@ def main(argv: list[str] | None = None) -> int: failed=failed_runs, workers=phase1_workers, ) + raise + record_success(tuning_df, run_summary) + _write_phase1_progress( + path=progress_path, + run_tag=run_tag, + upstream_run_tag=upstream_run_tag, + total=len(specs), + completed=completed_runs, + running=sorted(running), + failed=failed_runs, + workers=phase1_workers, + ) - inner_df = pd.concat(inner_frames, ignore_index=True) - inner_df.to_parquet(output_paths["inner_search"], index=False) - aggregated = _aggregate_inner_search(inner_df) - aggregated.to_parquet(output_paths["inner_aggregate"], index=False) - shortlist = _dedupe_designs(aggregated, top_k_inner) - shortlist.to_parquet(output_paths["phase1_shortlist"], index=False) - aggregate_path = str(output_paths["inner_aggregate"]) - inner_search_path = str(output_paths["inner_search"]) - inner_search_winner = ( - _normalize_design_row(aggregated.iloc[0].to_dict()) if not aggregated.empty else {} - ) + inner_df = pd.concat(inner_frames, ignore_index=True) + inner_df.to_parquet(output_paths["inner_search"], index=False) + aggregated = _aggregate_inner_search(inner_df) + aggregated.to_parquet(output_paths["inner_aggregate"], index=False) + shortlist = _dedupe_designs(aggregated, top_k_inner) + shortlist.to_parquet(output_paths["phase1_shortlist"], index=False) + return Phase1Result( + shortlist=shortlist, + aggregated=aggregated, + inner_runs=inner_runs, + aggregate_path=str(output_paths["inner_aggregate"]), + inner_search_path=str(output_paths["inner_search"]), + inner_search_winner=_normalize_design_row(aggregated.iloc[0].to_dict()) + if not aggregated.empty + else {}, + ) + +def _run_phase1_oot_confirmation( + *, + run_tag: str, + env: dict[str, str], + shortlist: pd.DataFrame, + output_paths: dict[str, Path], + alpha_candidates_95: list[float], + partition_candidates: list[str], + partition_probability_sources: list[str], + n_score_bins_candidates: list[int], + fallback_modes: list[str], + score_scale_families: list[str], + calibration_fractions: list[float], + sidecar_cfg: dict[str, Any], + validation_cfg: dict[str, Any], +) -> Phase1ConfirmationResult: if shortlist.empty: raise RuntimeError("No phase1 shortlist candidates available for OOT confirmation.") @@ -1540,7 +1699,6 @@ def main(argv: list[str] | None = None) -> int: best_phase1 = next( candidate for candidate in phase1_candidates if candidate["namespace"] == best_phase1_ns ) - final_namespace = best_phase1["namespace"] final_policy = dict(best_phase1["policy_status"]) final_sets = dict(best_phase1["set_status"]) final_decision = ( @@ -1548,92 +1706,255 @@ def main(argv: list[str] | None = None) -> int: if _acceptance_pass(final_policy, validation_cfg) else "keep_current_canonical" ) + return Phase1ConfirmationResult( + candidates=phase1_candidates, + frame=phase1_df, + best_namespace=best_phase1_ns, + final_policy=final_policy, + final_sets=final_sets, + final_decision=final_decision, + final_namespace=str(best_phase1["namespace"]), + ) + + +def _phase2_run_reason(*, force_phase2: bool, phase2_always_evaluate: bool) -> str: + if force_phase2: + return "forced" + if phase2_always_evaluate: + return "always_evaluate" + return "phase1_acceptance_fail" + + +def _maybe_apply_phase2( + *, + run_tag: str, + upstream_run_tag: str, + env: dict[str, str], + aggregated: pd.DataFrame, + phase1: Phase1ConfirmationResult, + phase1_only: bool, + force_phase2: bool, + alpha_candidates_95: list[float], + tuning_holdout_ratios: list[float], + inner_random_states: list[int], + partition_candidates: list[str], + partition_probability_sources: list[str], + n_score_bins_candidates: list[int], + fallback_modes: list[str], + score_scale_families: list[str], + calibration_fractions: list[float], + phase2_cfg: dict[str, Any], + sidecar_cfg: dict[str, Any], + validation_cfg: dict[str, Any], +) -> PromotionResult: + final_policy = dict(phase1.final_policy) + final_sets = dict(phase1.final_sets) + final_decision = str(phase1.final_decision) + final_namespace = str(phase1.final_namespace) phase2_summary: dict[str, Any] | None = None - phase2_always_evaluate = bool(phase2_cfg.get("always_evaluate", False)) or bool( - args.force_phase2 - ) - phase2_run_reason = ( - "forced" - if bool(args.force_phase2) - else "always_evaluate" - if phase2_always_evaluate - else "phase1_acceptance_fail" - ) - phase1_policy = dict(final_policy) - phase1_sets = dict(final_sets) - phase1_decision = str(final_decision) - phase1_namespace = str(final_namespace) - if ( + phase2_always_evaluate = bool(phase2_cfg.get("always_evaluate", False)) or bool(force_phase2) + should_run_phase2 = ( (phase2_always_evaluate or (not _acceptance_pass(final_policy, validation_cfg))) - and (not bool(args.phase1_only)) + and (not phase1_only) and bool(phase2_cfg.get("enabled", True)) - ): - phase2_decision, phase2_policy, phase2_sets, phase2_summary = _run_phase2_search( + ) + if not should_run_phase2: + return PromotionResult( + final_policy=final_policy, + final_sets=final_sets, + final_decision=final_decision, + final_namespace=final_namespace, + phase2_summary=phase2_summary, + ) + + phase2_decision, phase2_policy, phase2_sets, phase2_summary = _run_phase2_search( + run_tag=run_tag, + upstream_run_tag=upstream_run_tag, + env=env, + aggregated=aggregated, + phase1_candidates_frame=phase1.frame, + alpha_candidates_95=alpha_candidates_95, + tuning_holdout_ratios=tuning_holdout_ratios, + inner_random_states=inner_random_states, + partition_candidates=partition_candidates, + partition_probability_sources=partition_probability_sources, + n_score_bins_candidates=n_score_bins_candidates, + fallback_modes=fallback_modes, + score_scale_families=score_scale_families, + calibration_fractions=calibration_fractions, + phase2_cfg=phase2_cfg, + sidecar_cfg=sidecar_cfg, + validation_cfg=validation_cfg, + ) + phase2_applied, phase2_apply_reason = _phase2_should_replace_phase1( + phase1_policy=phase1.final_policy, + phase2_policy=phase2_policy, + validation_cfg=validation_cfg, + ) + if phase2_summary is not None: + phase2_summary = { + **phase2_summary, + "run_reason": _phase2_run_reason( + force_phase2=force_phase2, + phase2_always_evaluate=phase2_always_evaluate, + ), + "always_evaluate": bool(phase2_always_evaluate), + "applied_to_final": bool(phase2_applied), + "apply_reason": phase2_apply_reason, + "phase1_namespace": phase1.final_namespace, + "phase1_decision": phase1.final_decision, + "phase2_decision": phase2_decision, + } + if phase2_applied: + final_decision = phase2_decision + final_policy = phase2_policy + final_sets = phase2_sets + if phase2_summary and phase2_summary.get("final_namespace"): + final_namespace = str(phase2_summary["final_namespace"]) + elif _acceptance_pass(phase1.final_policy, validation_cfg): + final_decision = phase1.final_decision + final_policy = dict(phase1.final_policy) + final_sets = dict(phase1.final_sets) + final_namespace = phase1.final_namespace + + return PromotionResult( + final_policy=final_policy, + final_sets=final_sets, + final_decision=final_decision, + final_namespace=final_namespace, + phase2_summary=phase2_summary, + ) + + +def main(argv: list[str] | None = None) -> int: + parser = _build_parser() + args = parser.parse_args(argv) + + run_tag = str(args.run_tag).strip() + upstream_run_tag = str(args.upstream_canonical_run_tag).strip() + resume_from_run_tag = str(args.resume_from_run_tag).strip() if args.resume_from_run_tag else "" + mode = "derived_resume" if resume_from_run_tag else "fresh_search" + + profile = _profile_cfg(args.pipeline_profile) + cfg = _phase1_cfg(profile) + phase2_cfg = _phase2_cfg(profile) + sidecar_cfg = _sidecar_cfg(profile) + validation_cfg = _validation_cfg(profile) + output_paths = _reopen_artifact_paths(run_tag) + + env = os.environ.copy() + env["PIPELINE_RUN_TAG"] = run_tag + env["UPSTREAM_CANONICAL_RUN_TAG"] = upstream_run_tag + + alpha_candidates_90 = cfg.get("alpha_candidates_90", [0.09, 0.095, 0.10, 0.105, 0.11, 0.12]) + alpha_candidates_95 = cfg.get("alpha_candidates_95", [0.045, 0.05, 0.055, 0.06]) + partition_candidates = cfg.get( + "partition_candidates", + ["grade", "score_decile_mondrian", "grade_x_scoreband_mondrian"], + ) + partition_probability_sources = cfg.get("partition_probability_sources", ["calibrated", "raw"]) + n_score_bins_candidates = cfg.get("n_score_bins_candidates", [5, 10, 15, 20]) + min_group_sizes = cfg.get("min_group_sizes", [100, 150, 250, 500, 1000]) + fallback_modes = cfg.get("fallback_modes", ["grade_then_global", "global_only"]) + score_scale_families = cfg.get( + "score_scale_families", + ["none", "bernoulli_sqrt", "bernoulli_sqrt_clipped_0.02", "bernoulli_sqrt_clipped_0.05"], + ) + calibration_fractions = cfg.get("calibration_fractions", [0.25, 0.50, 0.75, 1.00]) + tuning_holdout_ratios = cfg.get("tuning_holdout_ratios", [0.20, 0.30]) + inner_random_states = cfg.get("inner_random_states", [42, 314, 2026]) + top_k_inner = int(validation_cfg.get("top_k_inner", 3)) + configured_workers = int(cfg.get("parallel_workers", 1) or 1) + phase1_workers = int(args.phase1_workers or configured_workers or 1) + phase1_workers = max(1, phase1_workers) + resume_completed_inner = not bool(args.no_resume_completed_inner) + + if resume_from_run_tag: + phase1_result = _phase1_from_resume( + resume_from_run_tag=resume_from_run_tag, + top_k_inner=top_k_inner, + output_paths=output_paths, + ) + else: + phase1_result = _run_phase1_inner_search( run_tag=run_tag, upstream_run_tag=upstream_run_tag, env=env, - aggregated=aggregated, - phase1_candidates_frame=phase1_df, + output_paths=output_paths, + phase1_workers=phase1_workers, + top_k_inner=top_k_inner, + alpha_candidates_90=alpha_candidates_90, alpha_candidates_95=alpha_candidates_95, - tuning_holdout_ratios=tuning_holdout_ratios, - inner_random_states=inner_random_states, partition_candidates=partition_candidates, partition_probability_sources=partition_probability_sources, n_score_bins_candidates=n_score_bins_candidates, + min_group_sizes=min_group_sizes, fallback_modes=fallback_modes, score_scale_families=score_scale_families, calibration_fractions=calibration_fractions, - phase2_cfg=phase2_cfg, - sidecar_cfg=sidecar_cfg, - validation_cfg=validation_cfg, - ) - phase2_applied, phase2_apply_reason = _phase2_should_replace_phase1( - phase1_policy=phase1_policy, - phase2_policy=phase2_policy, - validation_cfg=validation_cfg, + tuning_holdout_ratios=tuning_holdout_ratios, + inner_random_states=inner_random_states, + resume_completed_inner=resume_completed_inner, ) - if phase2_summary is not None: - phase2_summary = { - **phase2_summary, - "run_reason": phase2_run_reason, - "always_evaluate": bool(phase2_always_evaluate), - "applied_to_final": bool(phase2_applied), - "apply_reason": phase2_apply_reason, - "phase1_namespace": phase1_namespace, - "phase1_decision": phase1_decision, - "phase2_decision": phase2_decision, - } - if phase2_applied: - final_decision = phase2_decision - final_policy = phase2_policy - final_sets = phase2_sets - if phase2_summary and phase2_summary.get("final_namespace"): - final_namespace = str(phase2_summary["final_namespace"]) - elif _acceptance_pass(phase1_policy, validation_cfg): - final_decision = phase1_decision - final_policy = phase1_policy - final_sets = phase1_sets - final_namespace = phase1_namespace + shortlist = phase1_result.shortlist + aggregated = phase1_result.aggregated + + phase1_confirmation = _run_phase1_oot_confirmation( + run_tag=run_tag, + env=env, + shortlist=shortlist, + output_paths=output_paths, + alpha_candidates_95=alpha_candidates_95, + partition_candidates=partition_candidates, + partition_probability_sources=partition_probability_sources, + n_score_bins_candidates=n_score_bins_candidates, + fallback_modes=fallback_modes, + score_scale_families=score_scale_families, + calibration_fractions=calibration_fractions, + sidecar_cfg=sidecar_cfg, + validation_cfg=validation_cfg, + ) + final_result = _maybe_apply_phase2( + run_tag=run_tag, + upstream_run_tag=upstream_run_tag, + env=env, + aggregated=aggregated, + phase1=phase1_confirmation, + phase1_only=bool(args.phase1_only), + force_phase2=bool(args.force_phase2), + alpha_candidates_95=alpha_candidates_95, + tuning_holdout_ratios=tuning_holdout_ratios, + inner_random_states=inner_random_states, + partition_candidates=partition_candidates, + partition_probability_sources=partition_probability_sources, + n_score_bins_candidates=n_score_bins_candidates, + fallback_modes=fallback_modes, + score_scale_families=score_scale_families, + calibration_fractions=calibration_fractions, + phase2_cfg=phase2_cfg, + sidecar_cfg=sidecar_cfg, + validation_cfg=validation_cfg, + ) _write_consolidated_status( run_tag=run_tag, upstream_run_tag=upstream_run_tag, pipeline_profile=str(args.pipeline_profile), mode=mode, - inner_search_path=inner_search_path, - aggregate_path=aggregate_path, + inner_search_path=phase1_result.inner_search_path, + aggregate_path=phase1_result.aggregate_path, shortlist_path=str(output_paths["phase1_shortlist"]), phase1_final_path=str(output_paths["phase1_final_candidates"]), - inner_search_runs=inner_runs, - inner_search_winner=inner_search_winner, - best_phase1_namespace=best_phase1_ns, - final_policy=final_policy, - final_sets=final_sets, - final_decision=final_decision, - final_namespace=final_namespace, - phase2_summary=phase2_summary, - resume_meta=resume_meta, + inner_search_runs=phase1_result.inner_runs, + inner_search_winner=phase1_result.inner_search_winner, + best_phase1_namespace=phase1_confirmation.best_namespace, + final_policy=final_result.final_policy, + final_sets=final_result.final_sets, + final_decision=final_result.final_decision, + final_namespace=final_result.final_namespace, + phase2_summary=final_result.phase2_summary, + resume_meta=phase1_result.resume_meta, ) return 0 diff --git a/scripts/search/run_conformal_search.py b/scripts/search/run_conformal_search.py index 8a438ff..f131742 100644 --- a/scripts/search/run_conformal_search.py +++ b/scripts/search/run_conformal_search.py @@ -1,20 +1,24 @@ -"""Organized search entrypoint for conformal benchmarking/tuning.""" +"""Retired conformal search entrypoint. + +The former generic ``scripts.run_long_pipeline`` orchestrator was removed when +the IJDS paper lane was narrowed around frozen artifacts. Keep this file as a +readable stop sign for old commands instead of failing with an import error. +""" from __future__ import annotations import sys -from scripts.run_long_pipeline import main as _main - def main(argv: list[str] | None = None) -> int: - return _main( - argv, - default_pipeline_family="search_conformal", - default_sampling_profile="champion64safe", - default_include_rapids=False, - default_include_notebooks=False, + _ = argv + sys.stderr.write( + "scripts/search/run_conformal_search.py is retired. Use the frozen " + "IJDS artifacts plus paper evidence stages, or start a new isolated " + "experiment under scripts/search/run_conformal_reopen_search.py with " + "an explicit run tag and drift plan.\n" ) + return 2 if __name__ == "__main__": diff --git a/scripts/search/run_pool93_ijds_local_refinement.py b/scripts/search/run_pool93_ijds_local_refinement.py index 4d0e3e2..eee8edb 100644 --- a/scripts/search/run_pool93_ijds_local_refinement.py +++ b/scripts/search/run_pool93_ijds_local_refinement.py @@ -19,6 +19,7 @@ import sys import time from concurrent.futures import FIRST_COMPLETED, ProcessPoolExecutor, wait +from dataclasses import dataclass from datetime import UTC, datetime from itertools import product from pathlib import Path @@ -46,7 +47,15 @@ _compute_intervals_at_alpha, _load_aligned_dataset, ) -from src.optimization.portfolio_model import optimize_portfolio_allocation # noqa: E402 +from src.optimization.certificate_semantics import ( # noqa: E402 + IJDS_DECLARED_ALPHA_GRID, + add_policy_aware_bound_columns, + compute_funded_certificate_metrics, +) +from src.optimization.portfolio_model import ( # noqa: E402 + optimize_portfolio_allocation, + solution_allocation_vector, +) from src.utils.pipeline_runtime import ( # noqa: E402 atomic_write_json, atomic_write_parquet, @@ -56,7 +65,57 @@ STAGE_NAME = "pool93_ijds_local_refinement" DECLARED_RETURN_FLOOR = 170464.54 -DEFAULT_ALPHA_GRID = [0.01, 0.03, 0.05, 0.07, 0.10, 0.12, 0.15, 0.20] +DEFAULT_ALPHA_GRID = list(IJDS_DECLARED_ALPHA_GRID) +VALID_PROFILES = { + "stage1", + "expanded", + "claim_expanded", + "claim_micro", + "claim_micro_ext", + "claim_bound_closure", + "claim_bound_floor_closure", + "claim_bound_terminal", +} +ANCHOR_REASONS = ( + (96, "source_exact_max_return"), + (219, "source_low_gamma_cp_return_floor"), + (223, "source_low_weighted_miscoverage_high_return"), +) +CLAIM_ROW_FIELDS = ( + "claim_rank", + "local_candidate_id", + "local_family", + "anchor_rank", + "source_reason", + "risk_tolerance", + "policy_mode", + "gamma", + "delta_cap_quantile", + "tail_focus_quantile", + "uncertainty_aversion", + "alpha01_realized_total_return", + "return_floor_surplus", + "alpha01_gamma_cp", + "alpha01_gamma_internalized", + "alpha01_gamma_residual", + "alpha01_weighted_miscoverage_V", + "alpha01_endpoint_budget", + "alpha01_endpoint_budget_upper", + "alpha01_markov_loss_threshold", + "alpha01_markov_loss_cap", + "alpha01_weighted_pd_true", + "alpha01_empirical_coverage_funded", + "alpha_exact_pass_count", + "alpha_exact_check_count", + "alpha_mean_gamma_cp", + "alpha_mean_weighted_miscoverage_V", + "return_score", + "bound_score", + "v_score", + "ijds_balanced_score", + "n_funded_mean", + "allocator_backends", +) DEFAULT_SOURCE_BOUND_EVAL = ( ROOT / "data/processed/experiments/champion_reopen/" "champion-reopen-2026-06-19__hpo-wave1__pool93__portfolio-stage1-fast1-claim-26-06/" @@ -68,6 +127,20 @@ "portfolio/portfolio_bound_aware_selection_highspy.json" ) + +@dataclass(frozen=True) +class Pool93Paths: + output_dir: Path + model_dir: Path + checkpoint_dir: Path + status_path: Path + candidates_path: Path + bound_eval_path: Path + leaderboard_path: Path + claim_summary_path: Path + manifest_path: Path + + _WORKER_ALIGNED: pd.DataFrame | None = None @@ -185,49 +258,40 @@ def _add_candidate( seen.add(key) -def _generate_candidate_grid( - anchors: pd.DataFrame, +def _candidate_frame(rows: list[dict[str, Any]]) -> pd.DataFrame: + candidates = pd.DataFrame(rows).reset_index(drop=True) + candidates.insert(0, "local_candidate_id", np.arange(1, len(candidates) + 1, dtype=int)) + return candidates + + +def _anchor_policy( + anchor_by_rank: dict[int, dict[str, Any]], + rank: int, *, - profile: str, solver_backend: str, -) -> pd.DataFrame: - profile = str(profile).strip().lower() - valid_profiles = { - "stage1", - "expanded", - "claim_expanded", - "claim_micro", - "claim_micro_ext", - "claim_bound_closure", - "claim_bound_floor_closure", - "claim_bound_terminal", - } - if profile not in valid_profiles: - raise ValueError(f"profile must be one of {sorted(valid_profiles)}") - - anchor_by_rank = {int(row.candidate_rank): row for row in anchors.itertuples(index=False)} - rows: list[dict[str, Any]] = [] - seen: set[str] = set() +) -> dict[str, Any]: + row = anchor_by_rank[rank] + return _policy_base( + risk_tolerance=float(row["risk_tolerance"]), + policy_mode=str(row["policy_mode"]), + gamma=float(row["gamma"]), + uncertainty_aversion=float(row["uncertainty_aversion"]), + delta_cap_quantile=float(row["delta_cap_quantile"]), + tail_focus_quantile=float(row["tail_focus_quantile"]), + min_budget_utilization=float(row["min_budget_utilization"]), + pd_cap_slack_penalty=float(row["pd_cap_slack_penalty"]), + solver_backend=solver_backend, + ) - def anchor_policy(rank: int) -> dict[str, Any]: - row = anchor_by_rank[rank] - return _policy_base( - risk_tolerance=float(row.risk_tolerance), - policy_mode=str(row.policy_mode), - gamma=float(row.gamma), - uncertainty_aversion=float(row.uncertainty_aversion), - delta_cap_quantile=float(row.delta_cap_quantile), - tail_focus_quantile=float(row.tail_focus_quantile), - min_budget_utilization=float(row.min_budget_utilization), - pd_cap_slack_penalty=float(row.pd_cap_slack_penalty), - solver_backend=solver_backend, - ) - for rank, reason in [ - (96, "source_exact_max_return"), - (219, "source_low_gamma_cp_return_floor"), - (223, "source_low_weighted_miscoverage_high_return"), - ]: +def _add_anchor_candidates( + rows: list[dict[str, Any]], + seen: set[str], + *, + anchor_by_rank: dict[int, dict[str, Any]], + solver_backend: str, +) -> None: + for rank, reason in ANCHOR_REASONS: if rank in anchor_by_rank: _add_candidate( rows, @@ -235,682 +299,463 @@ def anchor_policy(rank: int) -> dict[str, Any]: family="anchor_policy", anchor_rank=rank, source_reason=reason, - policy=anchor_policy(rank), + policy=_anchor_policy(anchor_by_rank, rank, solver_backend=solver_backend), ) - if profile == "claim_micro": - # Final IJDS micro-refinement around the completed expanded frontier: - # - candidate 1206: tightest Markov cap above the declared return floor, - # - candidate 1665/1667: body/default low-V return-bound point, - # - candidate 1922: highest return under Markov cap <= 0.36, - # - candidate 2777/2857: economic endpoint. - # The grid is intentionally local and paper-facing, not a new generic - # champion search. - body_risks = _round_grid( - [0.1715 + 0.00025 * idx for idx in range(5)], - lo=0.14, - hi=0.24, - ) - body_gammas = _round_grid( - [0.545 + 0.005 * idx for idx in range(7)], - lo=0.0, - hi=1.0, - ) - body_aversions = [0.0, 0.0125, 0.025, 0.0375, 0.05, 0.0625, 0.075, 0.10] - for risk, gamma, aversion, mode in product( - body_risks, - body_gammas, - body_aversions, - ["blended_uncertainty", "capped_blended_uncertainty"], - ): - delta_values = [1.0] if mode == "blended_uncertainty" else [0.95, 1.0] - for delta_cap in delta_values: - _add_candidate( - rows, - seen, - family="claim_micro_body_low_v", - anchor_rank=219, - source_reason="candidate1665_1667_body_default_micro", - policy=_policy_base( - risk_tolerance=risk, - policy_mode=mode, - gamma=gamma, - uncertainty_aversion=aversion, - delta_cap_quantile=delta_cap, - solver_backend=solver_backend, - ), - ) - tight_risks = _round_grid( - [0.1700 + 0.00025 * idx for idx in range(5)], - lo=0.14, - hi=0.24, - ) - tight_gammas = _round_grid( - [0.575 + 0.005 * idx for idx in range(6)], - lo=0.0, - hi=1.0, - ) - tight_aversions = [0.15, 0.1625, 0.175, 0.1875, 0.20, 0.225] - for risk, gamma, aversion, mode in product( - tight_risks, - tight_gammas, - tight_aversions, - ["blended_uncertainty", "capped_blended_uncertainty"], - ): - delta_values = [1.0] if mode == "blended_uncertainty" else [0.95, 1.0] - for delta_cap in delta_values: - _add_candidate( - rows, - seen, - family="claim_micro_bound_tight", - anchor_rank=219, - source_reason="candidate1206_tight_cap_micro", - policy=_policy_base( - risk_tolerance=risk, - policy_mode=mode, - gamma=gamma, - uncertainty_aversion=aversion, - delta_cap_quantile=delta_cap, - solver_backend=solver_backend, - ), - ) +def _capped_delta_values(mode: str, capped_values: tuple[float, ...]) -> tuple[float, ...]: + return (1.0,) if mode == "blended_uncertainty" else capped_values - high_return_risks = _round_grid( - [0.1725 + 0.00025 * idx for idx in range(7)], - lo=0.14, - hi=0.24, - ) - high_return_gammas = _round_grid( - [0.500 + 0.005 * idx for idx in range(6)], - lo=0.0, - hi=1.0, - ) - high_return_aversions = [0.05, 0.0625, 0.075, 0.0875, 0.10] - for risk, gamma, aversion, mode in product( - high_return_risks, - high_return_gammas, - high_return_aversions, - ["blended_uncertainty", "capped_blended_uncertainty"], - ): - delta_values = [1.0] if mode == "blended_uncertainty" else [0.95, 1.0] - for delta_cap in delta_values: - _add_candidate( - rows, - seen, - family="claim_micro_high_return_cap036", - anchor_rank=219, - source_reason="candidate1922_return_cap036_micro", - policy=_policy_base( - risk_tolerance=risk, - policy_mode=mode, - gamma=gamma, - uncertainty_aversion=aversion, - delta_cap_quantile=delta_cap, - solver_backend=solver_backend, - ), - ) - econ_risks = _round_grid( - [0.1565 + 0.00025 * idx for idx in range(9)], - lo=0.12, - hi=0.22, - ) - econ_gammas = _round_grid( - [0.44 + 0.005 * idx for idx in range(13)], - lo=0.0, - hi=1.0, - ) - econ_aversions = [0.1125, 0.125, 0.1375, 0.15] - for risk, gamma, aversion, tail_focus in product( - econ_risks, - econ_gammas, - econ_aversions, - [0.95, 1.0], - ): +def _add_blended_grid( + rows: list[dict[str, Any]], + seen: set[str], + *, + family: str, + anchor_rank: int, + source_reason: str, + risks: list[float], + gammas: list[float], + aversions: list[float], + solver_backend: str, + capped_delta_values: tuple[float, ...] = (0.95, 1.0), +) -> None: + for risk, gamma, aversion, mode in product( + risks, + gammas, + aversions, + ["blended_uncertainty", "capped_blended_uncertainty"], + ): + for delta_cap in _capped_delta_values(str(mode), capped_delta_values): _add_candidate( rows, seen, - family="claim_micro_economic_endpoint", - anchor_rank=96, - source_reason="candidate2777_2857_economic_endpoint_micro", + family=family, + anchor_rank=anchor_rank, + source_reason=source_reason, policy=_policy_base( risk_tolerance=risk, - policy_mode="tail_blended_uncertainty", + policy_mode=str(mode), gamma=gamma, uncertainty_aversion=aversion, - tail_focus_quantile=tail_focus, + delta_cap_quantile=delta_cap, solver_backend=solver_backend, ), ) - candidates = pd.DataFrame(rows).reset_index(drop=True) - candidates.insert(0, "local_candidate_id", range(1, len(candidates) + 1)) - return candidates - if profile == "claim_micro_ext": - # Surgical extensions from the completed claim_micro frontier. These - # neighborhoods target only exposed claim boundaries, not a fresh broad - # portfolio search: - # - body/cap<=0.345 polish around candidates 37/205, - # - bound-tight endpoint around candidate 949, - # - cap<=0.36 return endpoint around candidate 1975, - # - economic endpoint around candidate 2122. - body_risks = _round_grid( - [0.17125 + 0.000125 * idx for idx in range(11)], - lo=0.14, - hi=0.24, - ) - body_gammas = _round_grid( - [0.5475, 0.55, 0.5525, 0.555, 0.5575], - lo=0.0, - hi=1.0, +def _add_tail_grid( + rows: list[dict[str, Any]], + seen: set[str], + *, + family: str, + anchor_rank: int, + source_reason: str, + risks: list[float], + gammas: list[float], + aversions: list[float], + tail_focus_values: list[float], + solver_backend: str, +) -> None: + for risk, gamma, aversion, tail_focus in product( + risks, + gammas, + aversions, + tail_focus_values, + ): + _add_candidate( + rows, + seen, + family=family, + anchor_rank=anchor_rank, + source_reason=source_reason, + policy=_policy_base( + risk_tolerance=risk, + policy_mode="tail_blended_uncertainty", + gamma=gamma, + uncertainty_aversion=aversion, + tail_focus_quantile=tail_focus, + solver_backend=solver_backend, + ), ) - body_aversions = [0.025, 0.0375, 0.05, 0.0625] - for risk, gamma, aversion, mode in product( - body_risks, - body_gammas, - body_aversions, - ["blended_uncertainty", "capped_blended_uncertainty"], - ): - delta_values = [1.0] if mode == "blended_uncertainty" else [0.975, 1.0] - for delta_cap in delta_values: - _add_candidate( - rows, - seen, - family="claim_micro_ext_body_cap345", - anchor_rank=219, - source_reason="candidate37_205_body_cap345_extension", - policy=_policy_base( - risk_tolerance=risk, - policy_mode=mode, - gamma=gamma, - uncertainty_aversion=aversion, - delta_cap_quantile=delta_cap, - solver_backend=solver_backend, - ), - ) - tight_risks = _round_grid( - [0.1690 + 0.00025 * idx for idx in range(8)], - lo=0.14, - hi=0.24, - ) - tight_gammas = _round_grid( - [0.600 + 0.005 * idx for idx in range(11)], - lo=0.0, - hi=1.0, - ) - tight_aversions = [0.2125, 0.225, 0.2375, 0.25, 0.2625, 0.275] - for risk, gamma, aversion, mode in product( - tight_risks, - tight_gammas, - tight_aversions, - ["blended_uncertainty", "capped_blended_uncertainty"], - ): - delta_values = [1.0] if mode == "blended_uncertainty" else [0.95, 1.0] - for delta_cap in delta_values: - _add_candidate( - rows, - seen, - family="claim_micro_ext_bound_tight", - anchor_rank=219, - source_reason="candidate949_bound_tight_extension", - policy=_policy_base( - risk_tolerance=risk, - policy_mode=mode, - gamma=gamma, - uncertainty_aversion=aversion, - delta_cap_quantile=delta_cap, - solver_backend=solver_backend, - ), - ) - cap036_risks = _round_grid( - [0.17375 + 0.00025 * idx for idx in range(10)], - lo=0.14, - hi=0.24, - ) - cap036_gammas = _round_grid( - [0.505 + 0.0025 * idx for idx in range(9)], - lo=0.0, - hi=1.0, - ) - cap036_aversions = [0.0625, 0.075, 0.0875, 0.10] - for risk, gamma, aversion, mode in product( - cap036_risks, - cap036_gammas, - cap036_aversions, - ["blended_uncertainty", "capped_blended_uncertainty"], - ): - delta_values = [1.0] if mode == "blended_uncertainty" else [0.95, 1.0] - for delta_cap in delta_values: - _add_candidate( - rows, - seen, - family="claim_micro_ext_cap036_return", - anchor_rank=219, - source_reason="candidate1975_cap036_return_extension", - policy=_policy_base( - risk_tolerance=risk, - policy_mode=mode, - gamma=gamma, - uncertainty_aversion=aversion, - delta_cap_quantile=delta_cap, - solver_backend=solver_backend, - ), - ) +def _append_claim_micro_candidates( + rows: list[dict[str, Any]], seen: set[str], *, solver_backend: str +) -> None: + _add_blended_grid( + rows, + seen, + family="claim_micro_body_low_v", + anchor_rank=219, + source_reason="candidate1665_1667_body_default_micro", + risks=_round_grid([0.1715 + 0.00025 * idx for idx in range(5)], lo=0.14, hi=0.24), + gammas=_round_grid([0.545 + 0.005 * idx for idx in range(7)], lo=0.0, hi=1.0), + aversions=[0.0, 0.0125, 0.025, 0.0375, 0.05, 0.0625, 0.075, 0.10], + capped_delta_values=(0.95, 1.0), + solver_backend=solver_backend, + ) + _add_blended_grid( + rows, + seen, + family="claim_micro_bound_tight", + anchor_rank=219, + source_reason="candidate1206_tight_cap_micro", + risks=_round_grid([0.1700 + 0.00025 * idx for idx in range(5)], lo=0.14, hi=0.24), + gammas=_round_grid([0.575 + 0.005 * idx for idx in range(6)], lo=0.0, hi=1.0), + aversions=[0.15, 0.1625, 0.175, 0.1875, 0.20, 0.225], + capped_delta_values=(0.95, 1.0), + solver_backend=solver_backend, + ) + _add_blended_grid( + rows, + seen, + family="claim_micro_high_return_cap036", + anchor_rank=219, + source_reason="candidate1922_return_cap036_micro", + risks=_round_grid([0.1725 + 0.00025 * idx for idx in range(7)], lo=0.14, hi=0.24), + gammas=_round_grid([0.500 + 0.005 * idx for idx in range(6)], lo=0.0, hi=1.0), + aversions=[0.05, 0.0625, 0.075, 0.0875, 0.10], + capped_delta_values=(0.95, 1.0), + solver_backend=solver_backend, + ) + _add_tail_grid( + rows, + seen, + family="claim_micro_economic_endpoint", + anchor_rank=96, + source_reason="candidate2777_2857_economic_endpoint_micro", + risks=_round_grid([0.1565 + 0.00025 * idx for idx in range(9)], lo=0.12, hi=0.22), + gammas=_round_grid([0.44 + 0.005 * idx for idx in range(13)], lo=0.0, hi=1.0), + aversions=[0.1125, 0.125, 0.1375, 0.15], + tail_focus_values=[0.95, 1.0], + solver_backend=solver_backend, + ) - econ_risks = _round_grid( - [0.15625 + 0.000125 * idx for idx in range(9)], - lo=0.12, - hi=0.22, - ) - econ_gammas = _round_grid( - [0.400 + 0.005 * idx for idx in range(10)], - lo=0.0, - hi=1.0, - ) - econ_aversions = [0.125, 0.1375, 0.15] - for risk, gamma, aversion, tail_focus in product( - econ_risks, - econ_gammas, - econ_aversions, - [0.90, 0.925, 0.95, 1.0], - ): - _add_candidate( - rows, - seen, - family="claim_micro_ext_economic_endpoint", - anchor_rank=96, - source_reason="candidate2122_economic_endpoint_extension", - policy=_policy_base( - risk_tolerance=risk, - policy_mode="tail_blended_uncertainty", - gamma=gamma, - uncertainty_aversion=aversion, - tail_focus_quantile=tail_focus, - solver_backend=solver_backend, - ), - ) - candidates = pd.DataFrame(rows).reset_index(drop=True) - candidates.insert(0, "local_candidate_id", range(1, len(candidates) + 1)) - return candidates +def _append_claim_micro_ext_candidates( + rows: list[dict[str, Any]], seen: set[str], *, solver_backend: str +) -> None: + _add_blended_grid( + rows, + seen, + family="claim_micro_ext_body_cap345", + anchor_rank=219, + source_reason="candidate37_205_body_cap345_extension", + risks=_round_grid([0.17125 + 0.000125 * idx for idx in range(11)], lo=0.14, hi=0.24), + gammas=_round_grid([0.5475, 0.55, 0.5525, 0.555, 0.5575], lo=0.0, hi=1.0), + aversions=[0.025, 0.0375, 0.05, 0.0625], + capped_delta_values=(0.975, 1.0), + solver_backend=solver_backend, + ) + _add_blended_grid( + rows, + seen, + family="claim_micro_ext_bound_tight", + anchor_rank=219, + source_reason="candidate949_bound_tight_extension", + risks=_round_grid([0.1690 + 0.00025 * idx for idx in range(8)], lo=0.14, hi=0.24), + gammas=_round_grid([0.600 + 0.005 * idx for idx in range(11)], lo=0.0, hi=1.0), + aversions=[0.2125, 0.225, 0.2375, 0.25, 0.2625, 0.275], + capped_delta_values=(0.95, 1.0), + solver_backend=solver_backend, + ) + _add_blended_grid( + rows, + seen, + family="claim_micro_ext_cap036_return", + anchor_rank=219, + source_reason="candidate1975_cap036_return_extension", + risks=_round_grid([0.17375 + 0.00025 * idx for idx in range(10)], lo=0.14, hi=0.24), + gammas=_round_grid([0.505 + 0.0025 * idx for idx in range(9)], lo=0.0, hi=1.0), + aversions=[0.0625, 0.075, 0.0875, 0.10], + capped_delta_values=(0.95, 1.0), + solver_backend=solver_backend, + ) + _add_tail_grid( + rows, + seen, + family="claim_micro_ext_economic_endpoint", + anchor_rank=96, + source_reason="candidate2122_economic_endpoint_extension", + risks=_round_grid([0.15625 + 0.000125 * idx for idx in range(9)], lo=0.12, hi=0.22), + gammas=_round_grid([0.400 + 0.005 * idx for idx in range(10)], lo=0.0, hi=1.0), + aversions=[0.125, 0.1375, 0.15], + tail_focus_values=[0.90, 0.925, 0.95, 1.0], + solver_backend=solver_backend, + ) + +def _append_bound_closure_candidates( + rows: list[dict[str, Any]], + seen: set[str], + *, + profile: str, + solver_backend: str, +) -> None: if profile == "claim_bound_closure": - # Final IJDS bound-endpoint closure. The completed claim_micro_ext run - # found a monotone-looking cap reduction up to gamma=0.65, while - # uncertainty_aversion was already mostly saturated. This tiny extension - # tests only whether the lower-bound endpoint can move; it is not a new - # body-policy or economic-endpoint search. - closure_risks = _round_grid( - [0.1685 + 0.00025 * idx for idx in range(10)], - lo=0.14, - hi=0.24, - ) - closure_gammas = _round_grid( - [0.65 + 0.01 * idx for idx in range(11)], - lo=0.0, - hi=1.0, + _add_blended_grid( + rows, + seen, + family="claim_bound_closure_low_cap", + anchor_rank=219, + source_reason="micro_ext_min_markov_cap_endpoint_closure", + risks=_round_grid([0.1685 + 0.00025 * idx for idx in range(10)], lo=0.14, hi=0.24), + gammas=_round_grid([0.65 + 0.01 * idx for idx in range(11)], lo=0.0, hi=1.0), + aversions=[0.25, 0.275, 0.30, 0.325, 0.35], + capped_delta_values=(0.95, 1.0), + solver_backend=solver_backend, ) - closure_aversions = [0.25, 0.275, 0.30, 0.325, 0.35] - for risk, gamma, aversion, mode in product( - closure_risks, - closure_gammas, - closure_aversions, - ["blended_uncertainty", "capped_blended_uncertainty"], - ): - delta_values = [1.0] if mode == "blended_uncertainty" else [0.95, 1.0] - for delta_cap in delta_values: - _add_candidate( - rows, - seen, - family="claim_bound_closure_low_cap", - anchor_rank=219, - source_reason="micro_ext_min_markov_cap_endpoint_closure", - policy=_policy_base( - risk_tolerance=risk, - policy_mode=mode, - gamma=gamma, - uncertainty_aversion=aversion, - delta_cap_quantile=delta_cap, - solver_backend=solver_backend, - ), - ) - - candidates = pd.DataFrame(rows).reset_index(drop=True) - candidates.insert(0, "local_candidate_id", range(1, len(candidates) + 1)) - return candidates - + return if profile == "claim_bound_floor_closure": - # Last bounded IJDS endpoint check: the completed bound closure lowered - # the Markov cap to 0.298369 at the grid boundary (low tau, high gamma, - # high aversion) while still preserving a positive return-floor surplus. - # This profile tests whether the appendix/theory endpoint can cross the - # cleaner cap<0.29 threshold; it should not be used to replace the paper - # body/default policy. - floor_risks = _round_grid( - [0.16775 + 0.000125 * idx for idx in range(13)], - lo=0.14, - hi=0.24, - ) - floor_gammas = _round_grid( - [0.75 + 0.01 * idx for idx in range(10)], - lo=0.0, - hi=1.0, + _add_blended_grid( + rows, + seen, + family="claim_bound_floor_closure_low_cap", + anchor_rank=219, + source_reason="bound_closure_cap029_floor_threshold", + risks=_round_grid([0.16775 + 0.000125 * idx for idx in range(13)], lo=0.14, hi=0.24), + gammas=_round_grid([0.75 + 0.01 * idx for idx in range(10)], lo=0.0, hi=1.0), + aversions=[0.325, 0.35, 0.375, 0.40, 0.425, 0.45], + capped_delta_values=(0.95, 1.0), + solver_backend=solver_backend, ) - floor_aversions = [0.325, 0.35, 0.375, 0.40, 0.425, 0.45] - for risk, gamma, aversion, mode in product( - floor_risks, - floor_gammas, - floor_aversions, - ["blended_uncertainty", "capped_blended_uncertainty"], - ): - delta_values = [1.0] if mode == "blended_uncertainty" else [0.95, 1.0] - for delta_cap in delta_values: - _add_candidate( - rows, - seen, - family="claim_bound_floor_closure_low_cap", - anchor_rank=219, - source_reason="bound_closure_cap029_floor_threshold", - policy=_policy_base( - risk_tolerance=risk, - policy_mode=mode, - gamma=gamma, - uncertainty_aversion=aversion, - delta_cap_quantile=delta_cap, - solver_backend=solver_backend, - ), - ) - candidates = pd.DataFrame(rows).reset_index(drop=True) - candidates.insert(0, "local_candidate_id", range(1, len(candidates) + 1)) - return candidates - if profile == "claim_bound_terminal": - # Terminal IJDS endpoint search. This is intentionally wider than the - # prior closures, but still only targets the final bound-tight endpoint: - # cap<0.285/0.280/0.275 if feasible, with positive return-floor surplus. - # It should not be used to replace the body/default or economic endpoint. - ultra_risks = _round_grid( - [0.16675 + 0.000125 * idx for idx in range(29)], - lo=0.14, - hi=0.24, - ) - ultra_gammas = _round_grid( - [0.84 + 0.005 * idx for idx in range(31)], - lo=0.0, - hi=1.0, - ) - ultra_aversions = [0.40, 0.425, 0.45, 0.475, 0.50, 0.55, 0.60, 0.65, 0.70] - for risk, gamma, aversion, mode in product( - ultra_risks, - ultra_gammas, - ultra_aversions, - ["blended_uncertainty", "capped_blended_uncertainty"], - ): - delta_values = [1.0] if mode == "blended_uncertainty" else [0.95, 1.0] - for delta_cap in delta_values: - _add_candidate( - rows, - seen, - family="claim_bound_terminal_ultra_low_cap", - anchor_rank=219, - source_reason="terminal_cap_threshold_search", - policy=_policy_base( - risk_tolerance=risk, - policy_mode=mode, - gamma=gamma, - uncertainty_aversion=aversion, - delta_cap_quantile=delta_cap, - solver_backend=solver_backend, - ), - ) - - recovery_risks = _round_grid( - [0.1680 + 0.000125 * idx for idx in range(29)], - lo=0.14, - hi=0.24, - ) - recovery_gammas = _round_grid( - [0.80 + 0.005 * idx for idx in range(25)], - lo=0.0, - hi=1.0, - ) - recovery_aversions = [0.35, 0.375, 0.40, 0.425, 0.45, 0.475, 0.50, 0.55, 0.60] - for risk, gamma, aversion, mode in product( - recovery_risks, - recovery_gammas, - recovery_aversions, - ["blended_uncertainty", "capped_blended_uncertainty"], - ): - delta_values = [1.0] if mode == "blended_uncertainty" else [0.95, 1.0] - for delta_cap in delta_values: - _add_candidate( - rows, - seen, - family="claim_bound_terminal_return_recovery", - anchor_rank=219, - source_reason="terminal_best_return_under_low_cap", - policy=_policy_base( - risk_tolerance=risk, - policy_mode=mode, - gamma=gamma, - uncertainty_aversion=aversion, - delta_cap_quantile=delta_cap, - solver_backend=solver_backend, - ), - ) +def _append_bound_terminal_candidates( + rows: list[dict[str, Any]], seen: set[str], *, solver_backend: str +) -> None: + _add_blended_grid( + rows, + seen, + family="claim_bound_terminal_ultra_low_cap", + anchor_rank=219, + source_reason="terminal_cap_threshold_search", + risks=_round_grid([0.16675 + 0.000125 * idx for idx in range(29)], lo=0.14, hi=0.24), + gammas=_round_grid([0.84 + 0.005 * idx for idx in range(31)], lo=0.0, hi=1.0), + aversions=[0.40, 0.425, 0.45, 0.475, 0.50, 0.55, 0.60, 0.65, 0.70], + capped_delta_values=(0.95, 1.0), + solver_backend=solver_backend, + ) + _add_blended_grid( + rows, + seen, + family="claim_bound_terminal_return_recovery", + anchor_rank=219, + source_reason="terminal_best_return_under_low_cap", + risks=_round_grid([0.1680 + 0.000125 * idx for idx in range(29)], lo=0.14, hi=0.24), + gammas=_round_grid([0.80 + 0.005 * idx for idx in range(25)], lo=0.0, hi=1.0), + aversions=[0.35, 0.375, 0.40, 0.425, 0.45, 0.475, 0.50, 0.55, 0.60], + capped_delta_values=(0.95, 1.0), + solver_backend=solver_backend, + ) - candidates = pd.DataFrame(rows).reset_index(drop=True) - candidates.insert(0, "local_candidate_id", range(1, len(candidates) + 1)) - return candidates - if 96 in anchor_by_rank: - base = anchor_by_rank[96] - risk_offsets = [-0.004, -0.002, -0.001, 0.0, 0.001, 0.002, 0.004] - gamma_offsets = [-0.05, -0.025, -0.01, 0.0, 0.01, 0.025, 0.05] - aversions = [0.05, 0.075, 0.10, 0.125, 0.15] - if profile == "expanded": - risk_offsets = [ - -0.006, - -0.004, - -0.003, - -0.002, - -0.001, - 0.0, - 0.001, - 0.002, - 0.003, - 0.004, - 0.006, - ] - gamma_offsets = [ - -0.075, - -0.05, - -0.035, - -0.025, - -0.01, - 0.0, - 0.01, - 0.025, - 0.035, - 0.05, - 0.075, - ] - aversions = [0.025, 0.05, 0.075, 0.10, 0.125, 0.15, 0.20] - risks = _round_grid( - [float(base.risk_tolerance) + x for x in risk_offsets], lo=0.12, hi=0.22 +def _append_max_return_neighborhood( + rows: list[dict[str, Any]], + seen: set[str], + *, + anchor_by_rank: dict[int, dict[str, Any]], + profile: str, + solver_backend: str, +) -> None: + if 96 not in anchor_by_rank: + return + base = anchor_by_rank[96] + risk_offsets = [-0.004, -0.002, -0.001, 0.0, 0.001, 0.002, 0.004] + gamma_offsets = [-0.05, -0.025, -0.01, 0.0, 0.01, 0.025, 0.05] + aversions = [0.05, 0.075, 0.10, 0.125, 0.15] + if profile == "expanded": + risk_offsets = [ + -0.006, + -0.004, + -0.003, + -0.002, + -0.001, + 0.0, + 0.001, + 0.002, + 0.003, + 0.004, + 0.006, + ] + gamma_offsets = [-0.075, -0.05, -0.035, -0.025, -0.01, 0.0, 0.01, 0.025, 0.035, 0.05, 0.075] + aversions = [0.025, 0.05, 0.075, 0.10, 0.125, 0.15, 0.20] + risks = _round_grid([float(base["risk_tolerance"]) + x for x in risk_offsets], lo=0.12, hi=0.22) + gammas = _round_grid([float(base["gamma"]) + x for x in gamma_offsets], lo=0.0, hi=1.0) + + for risk, gamma, aversion in product(risks, gammas, aversions): + _add_candidate( + rows, + seen, + family="max_return_segment_relative_local", + anchor_rank=96, + source_reason="rank96_local_dense", + policy=_policy_base( + risk_tolerance=risk, + policy_mode="segment_relative_tail_blended_uncertainty", + gamma=gamma, + uncertainty_aversion=aversion, + solver_backend=solver_backend, + ), ) - gammas = _round_grid([float(base.gamma) + x for x in gamma_offsets], lo=0.0, hi=1.0) - for risk, gamma, aversion in product(risks, gammas, aversions): - _add_candidate( - rows, - seen, - family="max_return_segment_relative_local", - anchor_rank=96, - source_reason="rank96_local_dense", - policy=_policy_base( - risk_tolerance=risk, - policy_mode="segment_relative_tail_blended_uncertainty", - gamma=gamma, - uncertainty_aversion=aversion, - solver_backend=solver_backend, - ), - ) + _add_tail_grid( + rows, + seen, + family="max_return_tail_local", + anchor_rank=96, + source_reason="rank96_tail_sensitivity", + risks=risks[1:-1] if len(risks) > 2 else risks, + gammas=gammas[1:-1] if len(gammas) > 2 else gammas, + aversions=[0.075, 0.10, 0.125] if profile == "stage1" else aversions, + tail_focus_values=[0.95, 1.0] if profile == "stage1" else [0.90, 0.95, 1.0], + solver_backend=solver_backend, + ) - tail_risks = risks[1:-1] if len(risks) > 2 else risks - tail_gammas = gammas[1:-1] if len(gammas) > 2 else gammas - tail_focus_values = [0.95, 1.0] if profile == "stage1" else [0.90, 0.95, 1.0] - tail_aversions = [0.075, 0.10, 0.125] if profile == "stage1" else aversions - for risk, gamma, aversion, tail_focus in product( - tail_risks, tail_gammas, tail_aversions, tail_focus_values - ): - _add_candidate( - rows, - seen, - family="max_return_tail_local", - anchor_rank=96, - source_reason="rank96_tail_sensitivity", - policy=_policy_base( - risk_tolerance=risk, - policy_mode="tail_blended_uncertainty", - gamma=gamma, - uncertainty_aversion=aversion, - tail_focus_quantile=tail_focus, - solver_backend=solver_backend, - ), - ) - bound_risk_centers = [] - bound_gamma_centers = [] +def _append_bound_neighborhood( + rows: list[dict[str, Any]], + seen: set[str], + *, + anchor_by_rank: dict[int, dict[str, Any]], + profile: str, + solver_backend: str, +) -> None: + risk_centers = [] + gamma_centers = [] for rank in (219, 223): if rank in anchor_by_rank: row = anchor_by_rank[rank] - bound_risk_centers.append(float(row.risk_tolerance)) - bound_gamma_centers.append(float(row.gamma)) - if bound_risk_centers: - risk_offsets = [-0.0075, -0.005, -0.0025, 0.0, 0.0025, 0.005] - gamma_offsets = [-0.05, -0.025, 0.0, 0.025, 0.05] - aversions = [0.05, 0.075, 0.10, 0.125, 0.15] - if profile == "expanded": - risk_offsets = [-0.01, -0.0075, -0.005, -0.0025, 0.0, 0.0025, 0.005, 0.0075, 0.01] - gamma_offsets = [-0.075, -0.05, -0.025, -0.01, 0.0, 0.01, 0.025, 0.05, 0.075] - aversions = [0.025, 0.05, 0.075, 0.10, 0.125, 0.15, 0.20] - risks = _round_grid( - [center + offset for center in bound_risk_centers for offset in risk_offsets], - lo=0.14, - hi=0.24, - ) - gammas = _round_grid( - [center + offset for center in bound_gamma_centers for offset in gamma_offsets], - lo=0.0, - hi=1.0, - ) - for risk, gamma, aversion, mode in product( - risks, - gammas, - aversions, - ["blended_uncertainty", "capped_blended_uncertainty"], - ): - delta_values = [1.0] - if mode == "capped_blended_uncertainty" and profile == "expanded": - delta_values = [0.90, 1.0] - for delta_cap in delta_values: - _add_candidate( - rows, - seen, - family="bound_efficient_local", - anchor_rank=219 if abs(gamma - 0.45) <= abs(gamma - 0.40) else 223, - source_reason="rank219_rank223_bound_frontier", - policy=_policy_base( - risk_tolerance=risk, - policy_mode=mode, - gamma=gamma, - uncertainty_aversion=aversion, - delta_cap_quantile=delta_cap, - solver_backend=solver_backend, - ), - ) - - if profile == "claim_expanded": - # Densify only the claim-bearing neighborhoods discovered by stage1: - # the low-cap return-bound ridge around local candidates 462/466 and - # the economic frontier endpoint around local candidate 264. - bound_risks = _round_grid( - [0.1705 + 0.0005 * idx for idx in range(10)] + [0.1750], - lo=0.14, - hi=0.24, - ) - bound_gammas = _round_grid( - [0.49, 0.50, 0.51, 0.52, 0.535, 0.55, 0.575], - lo=0.0, - hi=1.0, - ) - bound_aversions = [0.05, 0.075, 0.10, 0.1125, 0.125, 0.1375, 0.15, 0.175] - for risk, gamma, aversion, mode in product( - bound_risks, - bound_gammas, - bound_aversions, - ["blended_uncertainty", "capped_blended_uncertainty"], - ): - delta_values = [1.0] if mode == "blended_uncertainty" else [0.95, 1.0] - for delta_cap in delta_values: - _add_candidate( - rows, - seen, - family="bound_claim_refined_local", - anchor_rank=219, - source_reason="candidate462_466_return_bound_ridge", - policy=_policy_base( - risk_tolerance=risk, - policy_mode=mode, - gamma=gamma, - uncertainty_aversion=aversion, - delta_cap_quantile=delta_cap, - solver_backend=solver_backend, - ), - ) - - return_risks = _round_grid( - [0.1560 + 0.0005 * idx for idx in range(9)], - lo=0.12, - hi=0.22, - ) - return_gammas = _round_grid( - [0.44, 0.45, 0.46, 0.47, 0.475, 0.485, 0.495], - lo=0.0, - hi=1.0, - ) - return_aversions = [0.10, 0.1125, 0.125, 0.1375, 0.15] - for risk, gamma, aversion, tail_focus in product( - return_risks, - return_gammas, - return_aversions, - [0.95, 1.0], - ): + risk_centers.append(float(row["risk_tolerance"])) + gamma_centers.append(float(row["gamma"])) + if not risk_centers: + return + risk_offsets = [-0.0075, -0.005, -0.0025, 0.0, 0.0025, 0.005] + gamma_offsets = [-0.05, -0.025, 0.0, 0.025, 0.05] + aversions = [0.05, 0.075, 0.10, 0.125, 0.15] + capped_delta_values: tuple[float, ...] = (1.0,) + if profile == "expanded": + risk_offsets = [-0.01, -0.0075, -0.005, -0.0025, 0.0, 0.0025, 0.005, 0.0075, 0.01] + gamma_offsets = [-0.075, -0.05, -0.025, -0.01, 0.0, 0.01, 0.025, 0.05, 0.075] + aversions = [0.025, 0.05, 0.075, 0.10, 0.125, 0.15, 0.20] + capped_delta_values = (0.90, 1.0) + risks = _round_grid( + [center + offset for center in risk_centers for offset in risk_offsets], lo=0.14, hi=0.24 + ) + gammas = _round_grid( + [center + offset for center in gamma_centers for offset in gamma_offsets], lo=0.0, hi=1.0 + ) + for risk, gamma, aversion, mode in product( + risks, + gammas, + aversions, + ["blended_uncertainty", "capped_blended_uncertainty"], + ): + for delta_cap in _capped_delta_values(str(mode), capped_delta_values): _add_candidate( rows, seen, - family="max_return_claim_refined_local", - anchor_rank=96, - source_reason="candidate264_economic_frontier_endpoint", + family="bound_efficient_local", + anchor_rank=219 if abs(gamma - 0.45) <= abs(gamma - 0.40) else 223, + source_reason="rank219_rank223_bound_frontier", policy=_policy_base( risk_tolerance=risk, - policy_mode="tail_blended_uncertainty", + policy_mode=str(mode), gamma=gamma, uncertainty_aversion=aversion, - tail_focus_quantile=tail_focus, + delta_cap_quantile=delta_cap, solver_backend=solver_backend, ), ) - candidates = pd.DataFrame(rows).reset_index(drop=True) - candidates.insert(0, "local_candidate_id", range(1, len(candidates) + 1)) - return candidates + +def _append_claim_expanded_candidates( + rows: list[dict[str, Any]], seen: set[str], *, solver_backend: str +) -> None: + _add_blended_grid( + rows, + seen, + family="bound_claim_refined_local", + anchor_rank=219, + source_reason="candidate462_466_return_bound_ridge", + risks=_round_grid( + [0.1705 + 0.0005 * idx for idx in range(10)] + [0.1750], lo=0.14, hi=0.24 + ), + gammas=_round_grid([0.49, 0.50, 0.51, 0.52, 0.535, 0.55, 0.575], lo=0.0, hi=1.0), + aversions=[0.05, 0.075, 0.10, 0.1125, 0.125, 0.1375, 0.15, 0.175], + capped_delta_values=(0.95, 1.0), + solver_backend=solver_backend, + ) + _add_tail_grid( + rows, + seen, + family="max_return_claim_refined_local", + anchor_rank=96, + source_reason="candidate264_economic_frontier_endpoint", + risks=_round_grid([0.1560 + 0.0005 * idx for idx in range(9)], lo=0.12, hi=0.22), + gammas=_round_grid([0.44, 0.45, 0.46, 0.47, 0.475, 0.485, 0.495], lo=0.0, hi=1.0), + aversions=[0.10, 0.1125, 0.125, 0.1375, 0.15], + tail_focus_values=[0.95, 1.0], + solver_backend=solver_backend, + ) + + +def _generate_candidate_grid( + anchors: pd.DataFrame, + *, + profile: str, + solver_backend: str, +) -> pd.DataFrame: + profile = str(profile).strip().lower() + if profile not in VALID_PROFILES: + raise ValueError(f"profile must be one of {sorted(VALID_PROFILES)}") + + anchor_by_rank = {int(row["candidate_rank"]): row for row in anchors.to_dict(orient="records")} + rows: list[dict[str, Any]] = [] + seen: set[str] = set() + _add_anchor_candidates( + rows, + seen, + anchor_by_rank=anchor_by_rank, + solver_backend=solver_backend, + ) + + if profile == "claim_micro": + _append_claim_micro_candidates(rows, seen, solver_backend=solver_backend) + return _candidate_frame(rows) + if profile == "claim_micro_ext": + _append_claim_micro_ext_candidates(rows, seen, solver_backend=solver_backend) + return _candidate_frame(rows) + if profile in {"claim_bound_closure", "claim_bound_floor_closure"}: + _append_bound_closure_candidates( + rows, + seen, + profile=profile, + solver_backend=solver_backend, + ) + return _candidate_frame(rows) + if profile == "claim_bound_terminal": + _append_bound_terminal_candidates(rows, seen, solver_backend=solver_backend) + return _candidate_frame(rows) + + _append_max_return_neighborhood( + rows, + seen, + anchor_by_rank=anchor_by_rank, + profile=profile, + solver_backend=solver_backend, + ) + _append_bound_neighborhood( + rows, + seen, + anchor_by_rank=anchor_by_rank, + profile=profile, + solver_backend=solver_backend, + ) + if profile == "claim_expanded": + _append_claim_expanded_candidates(rows, seen, solver_backend=solver_backend) + return _candidate_frame(rows) def _exact_policy_alpha( @@ -964,21 +809,19 @@ def _exact_policy_alpha( threads=max(1, int(threads)), solver_backend=str(policy["solver_backend"]), ) - if "allocation_vector" in solution: - alloc = np.asarray(solution["allocation_vector"], dtype=float) - else: - alloc = np.array( - [float(solution["allocation"].get(i, 0.0)) for i in range(len(aligned))], - dtype=float, - ) + alloc = solution_allocation_vector(solution, len(aligned)) total_allocated = float(np.sum(alloc * loan_amounts)) weights = (alloc * loan_amounts) / max(total_allocated, 1e-6) - funded_mask = weights > 1e-8 - miscoverage = (y_true > pd_high).astype(float) - weighted_miscoverage_v = float(np.sum(weights * miscoverage)) - weighted_pd_true = float(np.sum(weights * y_true)) - violation = max(0.0, weighted_pd_true - float(policy["risk_tolerance"])) - sqrt_alpha = float(np.sqrt(alpha)) + certificate = compute_funded_certificate_metrics( + weights, + outcomes=y_true, + pd_point=pd_point, + pd_high=pd_high, + pd_effective=effective_pd, + alpha=alpha, + risk_tolerance=float(policy["risk_tolerance"]), + pd_cap_slack=float(solution.get("pd_cap_slack", 0.0)), + ) realized_total_return = float( np.sum( np.where( @@ -992,11 +835,16 @@ def _exact_policy_alpha( expected_loss_point = float(np.sum(alloc * loan_amounts * pd_point * lgd)) expected_return_net_point = expected_return_gross - expected_loss_point pd_cap_slack = float(solution.get("pd_cap_slack", 0.0)) + risk_excess = round(certificate.realized_risk_tolerance_excess, 6) + empirical_risk_screen = bool(certificate.realized_risk_tolerance_excess <= alpha + 1e-8) + markov_screen = bool(certificate.sqrt_alpha + 1e-8 >= certificate.weighted_miscoverage) return { "alpha": float(alpha), "confidence": float(1.0 - alpha), - "gamma_cp": round(float(np.sum(weights * np.clip(pd_high - pd_point, 0.0, 1.0))), 6), + "gamma_cp": round(certificate.gamma_cp, 6), + "gamma_internalized": round(certificate.gamma_internalized, 6), + "gamma_residual": round(certificate.gamma_residual, 6), "n_funded": int(solution.get("n_funded", int(np.sum(alloc > 0.01)))), "total_allocated": round(total_allocated, 2), "objective_value": round(float(solution.get("objective_value", 0.0)), 6), @@ -1004,28 +852,32 @@ def _exact_policy_alpha( "expected_loss_point": round(expected_loss_point, 6), "expected_return_net_point": round(expected_return_net_point, 6), "realized_total_return": round(realized_total_return, 6), - "weighted_pd_true": round(weighted_pd_true, 6), - "weighted_pd_constraint_used": round(float(np.sum(weights * effective_pd)), 6), - "weighted_pd_high": round(float(np.sum(weights * pd_high)), 6), - "weighted_pd_point": round(float(np.sum(weights * pd_point)), 6), - "worst_case_pd": round(float(np.sum(weights * pd_high)), 6), - "point_pd": round(float(np.sum(weights * pd_point)), 6), + "weighted_pd_true": round(certificate.weighted_outcome, 6), + "weighted_pd_constraint_used": round(certificate.weighted_pd_effective, 6), + "weighted_pd_high": round(certificate.endpoint_budget, 6), + "weighted_pd_point": round(certificate.weighted_pd_point, 6), + "worst_case_pd": round(certificate.endpoint_budget, 6), + "point_pd": round(certificate.weighted_pd_point, 6), + "endpoint_budget": round(certificate.endpoint_budget, 9), + "endpoint_budget_upper": round(certificate.endpoint_budget_upper, 9), + "markov_loss_threshold": round(certificate.markov_loss_threshold, 9), + "markov_loss_cap": round(certificate.markov_loss_cap, 9), "tau": float(policy["risk_tolerance"]), - "violation": round(violation, 6), - "weighted_miscoverage_V": round(weighted_miscoverage_v, 6), - "sqrt_alpha": round(sqrt_alpha, 6), - "empirical_coverage_funded": round( - float(1.0 - miscoverage[funded_mask].mean()) if funded_mask.any() else float("nan"), - 4, - ), - "bound_a_expected_violation_leq_alpha": bool(violation <= alpha + 1e-8), + "realized_risk_tolerance_excess": risk_excess, + "violation": risk_excess, + "weighted_miscoverage_V": round(certificate.weighted_miscoverage, 6), + "weighted_coverage_funded": round(certificate.weighted_coverage, 6), + "sqrt_alpha": round(certificate.sqrt_alpha, 6), + "empirical_coverage_funded": round(certificate.empirical_coverage_funded, 4), + "empirical_risk_excess_leq_alpha": empirical_risk_screen, + "bound_a_expected_violation_leq_alpha": empirical_risk_screen, "bound_b_prob_violation_gt_t": round(float(min(1.0, alpha / max(t_eval, 1e-8))), 4), "bound_b_t_eval": float(t_eval), "bound_b_is_vacuous": bool(min(1.0, alpha / max(t_eval, 1e-8)) >= 1.0), - "bound_c_V_leq_sqrt_alpha": bool(sqrt_alpha + 1e-8 >= weighted_miscoverage_v), - "all_bounds_hold": bool( - (violation <= alpha + 1e-8) and (sqrt_alpha + 1e-8 >= weighted_miscoverage_v) - ), + "markov_miscoverage_screen_pass": markov_screen, + "bound_c_V_leq_sqrt_alpha": markov_screen, + "certificate_screen_pass": empirical_risk_screen and markov_screen, + "all_bounds_hold": empirical_risk_screen and markov_screen, "allocator_mode": "exact", "solver_status": str(solution.get("solver_status", "unknown")), "allocator_solver_backend": str(solution.get("solver_backend", policy["solver_backend"])), @@ -1037,12 +889,17 @@ def _exact_policy_alpha( def _aggregate_leaderboard(candidates: pd.DataFrame, bound_eval: pd.DataFrame) -> pd.DataFrame: if bound_eval.empty: return candidates.copy() + bound_eval = add_policy_aware_bound_columns(bound_eval) grouped = bound_eval.groupby("local_candidate_id", dropna=False) agg = grouped.agg( alpha_exact_pass_count=("all_bounds_hold", "sum"), alpha_exact_check_count=("all_bounds_hold", "size"), alpha_exact_pass_rate=("all_bounds_hold", "mean"), - alpha_max_violation=("violation", "max"), + alpha_max_realized_risk_tolerance_excess=( + "realized_risk_tolerance_excess", + "max", + ), + alpha_max_violation=("realized_risk_tolerance_excess", "max"), alpha_mean_gamma_cp=("gamma_cp", "mean"), alpha_mean_weighted_miscoverage_V=("weighted_miscoverage_V", "mean"), alpha_mean_weighted_pd_true=("weighted_pd_true", "mean"), @@ -1065,9 +922,22 @@ def _aggregate_leaderboard(candidates: pd.DataFrame, bound_eval: pd.DataFrame) - alpha01_exact_pass=("all_bounds_hold", "all"), alpha01_realized_total_return=("realized_total_return", "mean"), alpha01_gamma_cp=("gamma_cp", "mean"), + alpha01_gamma_internalized=("gamma_internalized", "mean"), + alpha01_gamma_residual=("gamma_residual", "mean"), alpha01_weighted_miscoverage_V=("weighted_miscoverage_V", "mean"), - alpha01_violation=("violation", "max"), + alpha01_realized_risk_tolerance_excess=( + "realized_risk_tolerance_excess", + "max", + ), + alpha01_violation=("realized_risk_tolerance_excess", "max"), alpha01_weighted_pd_true=("weighted_pd_true", "mean"), + alpha01_weighted_pd_constraint_used=("weighted_pd_constraint_used", "mean"), + alpha01_weighted_pd_high=("weighted_pd_high", "mean"), + alpha01_weighted_pd_point=("weighted_pd_point", "mean"), + alpha01_endpoint_budget=("endpoint_budget", "mean"), + alpha01_endpoint_budget_upper=("endpoint_budget_upper", "mean"), + alpha01_markov_loss_threshold=("markov_loss_threshold", "mean"), + alpha01_markov_loss_cap=("markov_loss_cap", "mean"), alpha01_empirical_coverage_funded=("empirical_coverage_funded", "mean"), alpha01_n_funded=("n_funded", "mean"), ) @@ -1084,11 +954,6 @@ def _aggregate_leaderboard(candidates: pd.DataFrame, bound_eval: pd.DataFrame) - work["return_floor_surplus"] = ( work["alpha01_realized_total_return"].fillna(float("-inf")) - DECLARED_RETURN_FLOOR ) - alpha01_gamma = pd.to_numeric(work["alpha01_gamma_cp"], errors="coerce") - risk = pd.to_numeric(work["risk_tolerance"], errors="coerce") - gamma = pd.to_numeric(work["gamma"], errors="coerce") - work["alpha01_endpoint_budget_upper"] = risk + (1.0 - gamma) * alpha01_gamma - work["alpha01_markov_loss_cap"] = work["alpha01_endpoint_budget_upper"] + float(np.sqrt(0.01)) work = work.sort_values( by=[ "alpha01_exact_pass", @@ -1101,178 +966,171 @@ def _aggregate_leaderboard(candidates: pd.DataFrame, bound_eval: pd.DataFrame) - ascending=[False, False, False, False, True, True], kind="mergesort", ).reset_index(drop=True) - work.insert(0, "claim_rank", range(1, len(work) + 1)) + work.insert(0, "claim_rank", np.arange(1, len(work) + 1, dtype=int)) return work -def _claim_summary( - leaderboard: pd.DataFrame, - bound_eval: pd.DataFrame, - *, - alpha_grid: list[float] | None = None, -) -> dict[str, Any]: - leaderboard = leaderboard.copy() - if "alpha01_endpoint_budget_upper" not in leaderboard.columns: - alpha01_gamma = pd.to_numeric(leaderboard["alpha01_gamma_cp"], errors="coerce") - risk = pd.to_numeric(leaderboard["risk_tolerance"], errors="coerce") - gamma = pd.to_numeric(leaderboard["gamma"], errors="coerce") - leaderboard["alpha01_endpoint_budget_upper"] = risk + (1.0 - gamma) * alpha01_gamma - if "alpha01_markov_loss_cap" not in leaderboard.columns: - leaderboard["alpha01_markov_loss_cap"] = pd.to_numeric( - leaderboard["alpha01_endpoint_budget_upper"], errors="coerce" +def _ensure_claim_summary_columns(leaderboard: pd.DataFrame) -> pd.DataFrame: + work = leaderboard.copy() + if "alpha01_endpoint_budget" not in work.columns and "alpha01_weighted_pd_high" in work.columns: + work["alpha01_endpoint_budget"] = pd.to_numeric( + work["alpha01_weighted_pd_high"], errors="coerce" + ) + if "alpha01_endpoint_budget_upper" not in work.columns: + if { + "alpha01_weighted_pd_high", + "alpha01_weighted_pd_constraint_used", + }.issubset(work.columns): + residual = pd.to_numeric( + work["alpha01_weighted_pd_high"], errors="coerce" + ) - pd.to_numeric(work["alpha01_weighted_pd_constraint_used"], errors="coerce") + work["alpha01_gamma_residual"] = residual.clip(lower=0.0) + work["alpha01_endpoint_budget_upper"] = pd.to_numeric( + work["risk_tolerance"], errors="coerce" + ) + residual.clip(lower=0.0) + else: + alpha01_gamma = pd.to_numeric(work["alpha01_gamma_cp"], errors="coerce") + risk = pd.to_numeric(work["risk_tolerance"], errors="coerce") + gamma = pd.to_numeric(work["gamma"], errors="coerce") + work["alpha01_endpoint_budget_upper"] = risk + (1.0 - gamma) * alpha01_gamma + if "alpha01_markov_loss_threshold" not in work.columns and "alpha01_endpoint_budget" in work: + work["alpha01_markov_loss_threshold"] = pd.to_numeric( + work["alpha01_endpoint_budget"], errors="coerce" ) + float(np.sqrt(0.01)) - eligible = leaderboard[ - leaderboard["alpha01_exact_pass"].fillna(False).astype(bool) - & leaderboard["all_alpha_pass"].fillna(False).astype(bool) - ].copy() - if "return_floor_surplus" not in leaderboard.columns: - if "champion_return_surplus" in leaderboard.columns: - leaderboard["return_floor_surplus"] = leaderboard["champion_return_surplus"] + if "alpha01_markov_loss_cap" not in work.columns: + work["alpha01_markov_loss_cap"] = pd.to_numeric( + work["alpha01_endpoint_budget_upper"], errors="coerce" + ) + float(np.sqrt(0.01)) + if "return_floor_surplus" not in work.columns: + if "champion_return_surplus" in work.columns: + work["return_floor_surplus"] = work["champion_return_surplus"] else: - leaderboard["return_floor_surplus"] = ( - leaderboard["alpha01_realized_total_return"].fillna(float("-inf")) - - DECLARED_RETURN_FLOOR + work["return_floor_surplus"] = ( + work["alpha01_realized_total_return"].fillna(float("-inf")) - DECLARED_RETURN_FLOOR ) - above_return_floor = eligible[ - eligible["alpha01_realized_total_return"] >= DECLARED_RETURN_FLOOR + return work + + +def _all_alpha_eligible(frame: pd.DataFrame) -> pd.DataFrame: + return frame[ + frame["alpha01_exact_pass"].fillna(False).astype(bool) + & frame["all_alpha_pass"].fillna(False).astype(bool) ].copy() - def row_payload(frame: pd.DataFrame) -> dict[str, Any] | None: - if frame.empty: - return None - row = frame.iloc[0] - fields = [ - "claim_rank", - "local_candidate_id", - "local_family", - "anchor_rank", - "source_reason", - "risk_tolerance", - "policy_mode", - "gamma", - "delta_cap_quantile", - "tail_focus_quantile", - "uncertainty_aversion", - "alpha01_realized_total_return", - "return_floor_surplus", - "alpha01_gamma_cp", - "alpha01_weighted_miscoverage_V", - "alpha01_endpoint_budget_upper", - "alpha01_markov_loss_cap", - "alpha01_weighted_pd_true", - "alpha01_empirical_coverage_funded", - "alpha_exact_pass_count", - "alpha_exact_check_count", - "alpha_mean_gamma_cp", - "alpha_mean_weighted_miscoverage_V", - "return_score", - "bound_score", - "v_score", - "ijds_balanced_score", - "n_funded_mean", - "allocator_backends", - ] - return { - field: row[field].item() if hasattr(row[field], "item") else row[field] - for field in fields - if field in row.index - } - max_return = row_payload(eligible.sort_values("alpha01_realized_total_return", ascending=False)) - best_gamma = row_payload( - above_return_floor.sort_values( - ["alpha01_gamma_cp", "alpha01_realized_total_return"], - ascending=[True, False], - ) - ) - best_v = row_payload( - above_return_floor.sort_values( - ["alpha01_weighted_miscoverage_V", "alpha01_realized_total_return"], - ascending=[True, False], - ) - ) +def _row_payload(frame: pd.DataFrame) -> dict[str, Any] | None: + if frame.empty: + return None + row = frame.iloc[0] + return { + field: row[field].item() if hasattr(row[field], "item") else row[field] + for field in CLAIM_ROW_FIELDS + if field in row.index + } + + +def _add_normalized_score( + frame: pd.DataFrame, + *, + source: str, + target: str, + higher_better: bool, +) -> None: + vals = pd.to_numeric(frame[source], errors="coerce") + lo, hi = float(vals.min()), float(vals.max()) + if hi <= lo: + frame[target] = 1.0 + elif higher_better: + frame[target] = (vals - lo) / (hi - lo) + else: + frame[target] = (hi - vals) / (hi - lo) + + +def _balanced_claim_candidates(above_return_floor: pd.DataFrame) -> pd.DataFrame: balanced = above_return_floor.copy() - if not balanced.empty: - for source, target, higher_better in [ - ("alpha01_realized_total_return", "return_score", True), - ("alpha01_markov_loss_cap", "bound_score", False), - ("alpha01_weighted_miscoverage_V", "v_score", False), - ]: - vals = pd.to_numeric(balanced[source], errors="coerce") - lo, hi = float(vals.min()), float(vals.max()) - if hi <= lo: - balanced[target] = 1.0 - elif higher_better: - balanced[target] = (vals - lo) / (hi - lo) - else: - balanced[target] = (hi - vals) / (hi - lo) - balanced["ijds_balanced_score"] = ( - 0.40 * balanced["return_score"] - + 0.40 * balanced["bound_score"] - + 0.20 * balanced["v_score"] + if balanced.empty: + return balanced + for source, target, higher_better in [ + ("alpha01_realized_total_return", "return_score", True), + ("alpha01_markov_loss_cap", "bound_score", False), + ("alpha01_weighted_miscoverage_V", "v_score", False), + ]: + _add_normalized_score( + balanced, + source=source, + target=target, + higher_better=higher_better, ) - balanced_claim = row_payload( - balanced.sort_values("ijds_balanced_score", ascending=False) - if not balanced.empty - else balanced + balanced["ijds_balanced_score"] = ( + 0.40 * balanced["return_score"] + + 0.40 * balanced["bound_score"] + + 0.20 * balanced["v_score"] ) + return balanced + +def _family_claim_summary(leaderboard: pd.DataFrame) -> dict[str, Any]: by_family: dict[str, Any] = {} - if not leaderboard.empty: - for family, frame in leaderboard.groupby("local_family", dropna=False): - fam_eligible = frame[ - frame["alpha01_exact_pass"].fillna(False).astype(bool) - & frame["all_alpha_pass"].fillna(False).astype(bool) - ] - by_family[str(family)] = { - "n_policies": int(len(frame)), - "n_all_alpha_passers": int(len(fam_eligible)), - "all_alpha_pass_rate": float(len(fam_eligible) / max(len(frame), 1)), - "best_return": float(fam_eligible["alpha01_realized_total_return"].max()) - if not fam_eligible.empty - else None, - "min_gamma_cp_above_return_floor": float( - fam_eligible.loc[ - fam_eligible["alpha01_realized_total_return"] >= DECLARED_RETURN_FLOOR, - "alpha01_gamma_cp", - ].min() - ) - if not fam_eligible[ - fam_eligible["alpha01_realized_total_return"] >= DECLARED_RETURN_FLOOR - ].empty - else None, - "min_v_above_return_floor": float( - fam_eligible.loc[ - fam_eligible["alpha01_realized_total_return"] >= DECLARED_RETURN_FLOOR, - "alpha01_weighted_miscoverage_V", - ].min() - ) - if not fam_eligible[ - fam_eligible["alpha01_realized_total_return"] >= DECLARED_RETURN_FLOOR - ].empty - else None, - } + if leaderboard.empty: + return by_family + for family, frame in leaderboard.groupby("local_family", dropna=False): + fam_eligible = _all_alpha_eligible(frame) + fam_above_floor = fam_eligible[ + fam_eligible["alpha01_realized_total_return"] >= DECLARED_RETURN_FLOOR + ] + by_family[str(family)] = { + "n_policies": int(len(frame)), + "n_all_alpha_passers": int(len(fam_eligible)), + "all_alpha_pass_rate": float(len(fam_eligible) / max(len(frame), 1)), + "best_return": float(fam_eligible["alpha01_realized_total_return"].max()) + if not fam_eligible.empty + else None, + "min_gamma_cp_above_return_floor": float(fam_above_floor["alpha01_gamma_cp"].min()) + if not fam_above_floor.empty + else None, + "min_v_above_return_floor": float( + fam_above_floor["alpha01_weighted_miscoverage_V"].min() + ) + if not fam_above_floor.empty + else None, + } + return by_family + +def _alpha_claim_summary(bound_eval: pd.DataFrame) -> dict[str, Any]: by_alpha: dict[str, Any] = {} - if not bound_eval.empty: - for alpha, frame in bound_eval.groupby("alpha", dropna=False): - by_alpha[str(float(alpha))] = { - "n_checks": int(len(frame)), - "pass_rate": float(frame["all_bounds_hold"].fillna(False).mean()), - "max_violation": float(frame["violation"].max()), - "mean_gamma_cp": float(frame["gamma_cp"].mean()), - "mean_weighted_miscoverage_V": float(frame["weighted_miscoverage_V"].mean()), - } - - alpha_values = ( - [float(value) for value in alpha_grid] - if alpha_grid is not None - else sorted(float(value) for value in bound_eval["alpha"].dropna().unique()) - if "alpha" in bound_eval - else list(DEFAULT_ALPHA_GRID) + if bound_eval.empty: + return by_alpha + risk_excess_column = ( + "realized_risk_tolerance_excess" + if "realized_risk_tolerance_excess" in bound_eval.columns + else "violation" ) - alpha_values = sorted(dict.fromkeys(alpha_values)) - finite_grid_policy = { + for alpha, frame in bound_eval.groupby("alpha", dropna=False): + alpha_value = float(str(alpha)) + by_alpha[str(alpha_value)] = { + "n_checks": int(len(frame)), + "pass_rate": float(frame["all_bounds_hold"].fillna(False).mean()), + "max_realized_risk_tolerance_excess": float(frame[risk_excess_column].max()), + "max_violation": float(frame[risk_excess_column].max()), + "mean_gamma_cp": float(frame["gamma_cp"].mean()), + "mean_weighted_miscoverage_V": float(frame["weighted_miscoverage_V"].mean()), + } + return by_alpha + + +def _claim_alpha_values(bound_eval: pd.DataFrame, alpha_grid: list[float] | None) -> list[float]: + if alpha_grid is not None: + values = [float(value) for value in alpha_grid] + elif "alpha" in bound_eval: + values = sorted(float(value) for value in bound_eval["alpha"].dropna().unique()) + else: + values = list(DEFAULT_ALPHA_GRID) + return sorted(dict.fromkeys(values)) + + +def _finite_grid_policy(alpha_values: list[float]) -> dict[str, Any]: + return { "alpha_grid": alpha_values, "alpha_grid_size": int(len(alpha_values)), "alpha_grid_semantics": ( @@ -1284,7 +1142,10 @@ def row_payload(frame: pd.DataFrame) -> dict[str, Any] | None: "not a continuous robust region" ), } - claim_selection_protocol = { + + +def _claim_selection_protocol() -> dict[str, Any]: + return { "body_default": "balanced_return_bound_claim", "frontier_endpoints": [ "max_return_claim", @@ -1308,12 +1169,48 @@ def row_payload(frame: pd.DataFrame) -> dict[str, Any] | None: ), } + +def _claim_summary( + leaderboard: pd.DataFrame, + bound_eval: pd.DataFrame, + *, + alpha_grid: list[float] | None = None, +) -> dict[str, Any]: + leaderboard = _ensure_claim_summary_columns(leaderboard) + eligible = _all_alpha_eligible(leaderboard) + above_return_floor = eligible[ + eligible["alpha01_realized_total_return"] >= DECLARED_RETURN_FLOOR + ].copy() + + max_return = _row_payload( + eligible.sort_values("alpha01_realized_total_return", ascending=False) + ) + best_gamma = _row_payload( + above_return_floor.sort_values( + ["alpha01_gamma_cp", "alpha01_realized_total_return"], + ascending=[True, False], + ) + ) + best_v = _row_payload( + above_return_floor.sort_values( + ["alpha01_weighted_miscoverage_V", "alpha01_realized_total_return"], + ascending=[True, False], + ) + ) + balanced = _balanced_claim_candidates(above_return_floor) + balanced_claim = _row_payload( + balanced.sort_values("ijds_balanced_score", ascending=False) + if not balanced.empty + else balanced + ) + alpha_values = _claim_alpha_values(bound_eval, alpha_grid) + return { "schema_version": SCHEMA_VERSION, "generated_at_utc": datetime.now(tz=UTC).isoformat(), "declared_return_floor": DECLARED_RETURN_FLOOR, - "finite_grid_policy": finite_grid_policy, - "claim_selection_protocol": claim_selection_protocol, + "finite_grid_policy": _finite_grid_policy(alpha_values), + "claim_selection_protocol": _claim_selection_protocol(), "n_policies": int(len(leaderboard)), "n_all_alpha_passers": int(len(eligible)), "n_all_alpha_passers_above_return_floor": int(len(above_return_floor)), @@ -1321,8 +1218,8 @@ def row_payload(frame: pd.DataFrame) -> dict[str, Any] | None: "best_gamma_cp_return_floor_claim": best_gamma, "best_weighted_miscoverage_return_floor_claim": best_v, "balanced_return_bound_claim": balanced_claim, - "by_family": by_family, - "by_alpha": by_alpha, + "by_family": _family_claim_summary(leaderboard), + "by_alpha": _alpha_claim_summary(bound_eval), "interpretation": { "max_return_claim": "Use when the paper emphasizes certified economic return.", "best_gamma_cp_return_floor_claim": "Use when the paper emphasizes a tighter conformal robustness budget while preserving the declared return floor.", @@ -1373,7 +1270,7 @@ def _write_status( ) -def main(argv: list[str] | None = None) -> int: +def _build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser() parser.add_argument("--run-tag", default="pool93_ijds_local_refine_stage1") parser.add_argument( @@ -1419,9 +1316,10 @@ def main(argv: list[str] | None = None) -> int: default=0, help="Debug/smoke option: keep only the first N generated policies when positive.", ) - args = parser.parse_args(argv) + return parser - run_tag = str(args.run_tag).strip().replace("/", "_") + +def _resolve_paths(args: argparse.Namespace, *, run_tag: str) -> Pool93Paths: output_dir = ( Path(args.output_dir) if str(args.output_dir).strip() @@ -1432,44 +1330,81 @@ def main(argv: list[str] | None = None) -> int: if str(args.model_dir).strip() else ROOT / "models/experiments/champion_reopen" / run_tag / "portfolio" ) - output_dir.mkdir(parents=True, exist_ok=True) - model_dir.mkdir(parents=True, exist_ok=True) - checkpoint_dir = model_dir / "runtime_checkpoints" - checkpoint_dir.mkdir(parents=True, exist_ok=True) - - status_path = model_dir / "runtime_status.json" - candidates_path = output_dir / "pool93_ijds_local_refinement_candidates.parquet" - bound_eval_path = output_dir / "pool93_ijds_local_refinement_bound_eval.parquet" - leaderboard_path = output_dir / "pool93_ijds_local_refinement_leaderboard.parquet" - claim_summary_path = model_dir / "pool93_ijds_local_refinement_claim_summary.json" - manifest_path = model_dir / "pool93_ijds_local_refinement_manifest.json" + return Pool93Paths( + output_dir=output_dir, + model_dir=model_dir, + checkpoint_dir=model_dir / "runtime_checkpoints", + status_path=model_dir / "runtime_status.json", + candidates_path=output_dir / "pool93_ijds_local_refinement_candidates.parquet", + bound_eval_path=output_dir / "pool93_ijds_local_refinement_bound_eval.parquet", + leaderboard_path=output_dir / "pool93_ijds_local_refinement_leaderboard.parquet", + claim_summary_path=model_dir / "pool93_ijds_local_refinement_claim_summary.json", + manifest_path=model_dir / "pool93_ijds_local_refinement_manifest.json", + ) - source_bound_eval = Path(args.source_bound_eval) - source_selection = Path(args.source_selection) + +def _ensure_pool93_dirs(paths: Pool93Paths) -> None: + paths.output_dir.mkdir(parents=True, exist_ok=True) + paths.model_dir.mkdir(parents=True, exist_ok=True) + paths.checkpoint_dir.mkdir(parents=True, exist_ok=True) + + +def _conformal_intervals_from_selection( + *, + explicit_path: str, + source_selection: Path, +) -> str: + if str(explicit_path).strip(): + return str(explicit_path).strip() source_selection_payload = json.loads(source_selection.read_text(encoding="utf-8")) - conformal_intervals_path = str(args.conformal_intervals_path).strip() or str( - ROOT / source_selection_payload["conformal_intervals_path"] - ) - alpha_grid = _coerce_float_grid(args.alpha_grid, DEFAULT_ALPHA_GRID) - anchor_ranks = _coerce_int_grid(args.anchor_ranks, [96, 219, 223]) + return str(ROOT / source_selection_payload["conformal_intervals_path"]) - if candidates_path.exists(): - candidates = pd.read_parquet(candidates_path) - logger.info("Reusing candidate manifest: {} rows from {}", len(candidates), candidates_path) - else: - anchors = _source_anchor_rows(source_bound_eval, anchor_ranks) - candidates = _generate_candidate_grid( - anchors, - profile=str(args.profile), - solver_backend=str(args.solver_backend), + +def _load_or_generate_candidates( + *, + args: argparse.Namespace, + paths: Pool93Paths, + source_bound_eval: Path, + anchor_ranks: list[int], +) -> pd.DataFrame: + if paths.candidates_path.exists(): + candidates = pd.read_parquet(paths.candidates_path) + logger.info( + "Reusing candidate manifest: {} rows from {}", + len(candidates), + paths.candidates_path, ) - if int(args.candidate_limit) > 0: - candidates = candidates.head(int(args.candidate_limit)).copy().reset_index(drop=True) - candidates["local_candidate_id"] = range(1, len(candidates) + 1) - atomic_write_parquet(candidates, candidates_path, index=False) - logger.info("Wrote candidate manifest: {} policies to {}", len(candidates), candidates_path) + return candidates + anchors = _source_anchor_rows(source_bound_eval, anchor_ranks) + candidates = _generate_candidate_grid( + anchors, + profile=str(args.profile), + solver_backend=str(args.solver_backend), + ) + if int(args.candidate_limit) > 0: + candidates = candidates.head(int(args.candidate_limit)).copy().reset_index(drop=True) + candidates["local_candidate_id"] = np.arange(1, len(candidates) + 1, dtype=int) + atomic_write_parquet(candidates, paths.candidates_path, index=False) + logger.info( + "Wrote candidate manifest: {} policies to {}", + len(candidates), + paths.candidates_path, + ) + return candidates + - manifest = { +def _manifest_payload( + *, + args: argparse.Namespace, + paths: Pool93Paths, + run_tag: str, + source_bound_eval: Path, + source_selection: Path, + conformal_intervals_path: str, + anchor_ranks: list[int], + alpha_grid: list[float], +) -> dict[str, Any]: + return { "schema_version": SCHEMA_VERSION, "generated_at_utc": datetime.now(tz=UTC).isoformat(), "run_tag": run_tag, @@ -1487,28 +1422,192 @@ def main(argv: list[str] | None = None) -> int: "random_state": int(args.random_state), "checkpoint_every": int(args.checkpoint_every), "parallel_workers": int(args.parallel_workers), - "candidates_path": str(candidates_path), - "bound_eval_path": str(bound_eval_path), - "leaderboard_path": str(leaderboard_path), - "claim_summary_path": str(claim_summary_path), + "candidates_path": str(paths.candidates_path), + "bound_eval_path": str(paths.bound_eval_path), + "leaderboard_path": str(paths.leaderboard_path), + "claim_summary_path": str(paths.claim_summary_path), } - atomic_write_json(manifest_path, manifest) + +def _load_partial_bound_eval(path: Path) -> tuple[pd.DataFrame, set[tuple[int, float]]]: partial = pd.DataFrame() - if bound_eval_path.exists(): - partial = pd.read_parquet(bound_eval_path) + if path.exists(): + partial = pd.read_parquet(path) if not partial.empty: partial = partial.drop_duplicates( ["local_candidate_id", "alpha"], keep="last", ).reset_index(drop=True) logger.info("Resuming local refinement from {} rows", len(partial)) - completed_keys = set() - if not partial.empty: - completed_keys = { - (int(row.local_candidate_id), float(row.alpha)) - for row in partial.itertuples(index=False) - } + completed_keys = { + (int(row["local_candidate_id"]), float(row["alpha"])) + for row in partial.to_dict(orient="records") + } + return partial, completed_keys + + +def _pending_refinement_tasks( + *, + candidates: pd.DataFrame, + alpha_grid: list[float], + completed_keys: set[tuple[int, float]], +) -> list[tuple[dict[str, Any], float]]: + return [ + (candidate, float(alpha)) + for candidate in candidates.to_dict(orient="records") + for alpha in alpha_grid + if (int(candidate["local_candidate_id"]), float(alpha)) not in completed_keys + ] + + +def _persist_refinement_progress( + *, + paths: Pool93Paths, + candidates: pd.DataFrame, + rows: list[dict[str, Any]], + alpha_grid: list[float], +) -> None: + bound_eval = pd.DataFrame(rows) + atomic_write_parquet(bound_eval, paths.bound_eval_path, index=False) + leaderboard = _aggregate_leaderboard(candidates, bound_eval) + atomic_write_parquet(leaderboard, paths.leaderboard_path, index=False) + atomic_write_json( + paths.claim_summary_path, + _claim_summary(leaderboard, bound_eval, alpha_grid=alpha_grid), + ) + + +def _run_serial_refinement( + *, + pending_tasks: list[tuple[dict[str, Any], float]], + aligned: pd.DataFrame, + budget: float, + t_eval: float, + exact_threads: int, + record_result: Any, +) -> None: + for candidate, alpha in pending_tasks: + policy = {field: candidate[field] for field in SEMANTIC_POLICY_FIELDS} + result = _exact_policy_alpha( + aligned, + policy=policy, + alpha=float(alpha), + budget=float(budget), + t_eval=float(t_eval), + threads=int(exact_threads), + ) + record_result(candidate, alpha, result) + + +def _run_parallel_refinement( + *, + pending_tasks: list[tuple[dict[str, Any], float]], + aligned: pd.DataFrame, + parallel_workers: int, + budget: float, + t_eval: float, + exact_threads: int, + persist_progress: Any, + record_result: Any, +) -> None: + logger.info( + "Running exact refinement with {} parallel workers and {} solver thread(s) per worker", + parallel_workers, + int(exact_threads), + ) + mp_context = mp.get_context("fork") if sys.platform != "win32" else None + max_in_flight = max(parallel_workers, parallel_workers * 2) + next_task_idx = 0 + futures: dict[Any, tuple[dict[str, Any], float]] = {} + with ProcessPoolExecutor( + max_workers=parallel_workers, + mp_context=mp_context, + initializer=_init_exact_worker, + initargs=(aligned,), + ) as executor: + while next_task_idx < len(pending_tasks) or futures: + while next_task_idx < len(pending_tasks) and len(futures) < max_in_flight: + candidate, alpha = pending_tasks[next_task_idx] + future = executor.submit( + _exact_policy_alpha_task, + candidate, + alpha, + float(budget), + float(t_eval), + int(exact_threads), + ) + futures[future] = (candidate, alpha) + next_task_idx += 1 + done, _ = wait(futures, return_when=FIRST_COMPLETED) + for future in done: + candidate, alpha = futures.pop(future) + try: + result = future.result() + except Exception: + persist_progress() + raise + result_only = { + key: value + for key, value in result.items() + if key not in candidate or key in {"alpha", "confidence"} + } + record_result(candidate, alpha, result_only) + + +def _write_final_outputs( + *, + paths: Pool93Paths, + candidates: pd.DataFrame, + rows: list[dict[str, Any]], + alpha_grid: list[float], +) -> tuple[pd.DataFrame, dict[str, Any]]: + bound_eval = pd.DataFrame(rows) + atomic_write_parquet(bound_eval, paths.bound_eval_path, index=False) + leaderboard = _aggregate_leaderboard(candidates, bound_eval) + atomic_write_parquet(leaderboard, paths.leaderboard_path, index=False) + claim_summary = _claim_summary(leaderboard, bound_eval, alpha_grid=alpha_grid) + atomic_write_json(paths.claim_summary_path, claim_summary) + return leaderboard, claim_summary + + +def main(argv: list[str] | None = None) -> int: + parser = _build_parser() + args = parser.parse_args(argv) + + run_tag = str(args.run_tag).strip().replace("/", "_") + paths = _resolve_paths(args, run_tag=run_tag) + _ensure_pool93_dirs(paths) + + source_bound_eval = Path(args.source_bound_eval) + source_selection = Path(args.source_selection) + conformal_intervals_path = _conformal_intervals_from_selection( + explicit_path=args.conformal_intervals_path, + source_selection=source_selection, + ) + alpha_grid = _coerce_float_grid(args.alpha_grid, DEFAULT_ALPHA_GRID) + anchor_ranks = _coerce_int_grid(args.anchor_ranks, [96, 219, 223]) + + candidates = _load_or_generate_candidates( + args=args, + paths=paths, + source_bound_eval=source_bound_eval, + anchor_ranks=anchor_ranks, + ) + atomic_write_json( + paths.manifest_path, + _manifest_payload( + args=args, + paths=paths, + run_tag=run_tag, + source_bound_eval=source_bound_eval, + source_selection=source_selection, + conformal_intervals_path=conformal_intervals_path, + anchor_ranks=anchor_ranks, + alpha_grid=alpha_grid, + ), + ) + + partial, completed_keys = _load_partial_bound_eval(paths.bound_eval_path) rows: list[dict[str, Any]] = partial.to_dict(orient="records") if not partial.empty else [] total_checks = int(len(candidates) * len(alpha_grid)) @@ -1516,7 +1615,7 @@ def main(argv: list[str] | None = None) -> int: initial_completed = int(len(completed_keys)) _write_status( run_tag=run_tag, - status_path=status_path, + status_path=paths.status_path, start_monotonic=start, completed=len(completed_keys), total=total_checks, @@ -1532,15 +1631,14 @@ def main(argv: list[str] | None = None) -> int: random_state=int(args.random_state), ) logger.info("Loaded aligned full universe: {} rows", len(aligned)) + completed = len(completed_keys) def persist_progress() -> None: - bound_eval = pd.DataFrame(rows) - atomic_write_parquet(bound_eval, bound_eval_path, index=False) - leaderboard = _aggregate_leaderboard(candidates, bound_eval) - atomic_write_parquet(leaderboard, leaderboard_path, index=False) - atomic_write_json( - claim_summary_path, - _claim_summary(leaderboard, bound_eval, alpha_grid=alpha_grid), + _persist_refinement_progress( + paths=paths, + candidates=candidates, + rows=rows, + alpha_grid=alpha_grid, ) def record_result(candidate: dict[str, Any], alpha: float, result: dict[str, Any]) -> None: @@ -1554,7 +1652,7 @@ def record_result(candidate: dict[str, Any], alpha: float, result: dict[str, Any completed_keys.add((int(candidate["local_candidate_id"]), float(alpha))) _write_status( run_tag=run_tag, - status_path=status_path, + status_path=paths.status_path, start_monotonic=start, completed=completed, total=total_checks, @@ -1574,81 +1672,44 @@ def record_result(candidate: dict[str, Any], alpha: float, result: dict[str, Any if completed % max(1, int(args.checkpoint_every)) == 0: persist_progress() - completed = len(completed_keys) - pending_tasks: list[tuple[dict[str, Any], float]] = [] - for candidate in candidates.to_dict(orient="records"): - for alpha in alpha_grid: - key = (int(candidate["local_candidate_id"]), float(alpha)) - if key not in completed_keys: - pending_tasks.append((candidate, float(alpha))) + pending_tasks = _pending_refinement_tasks( + candidates=candidates, + alpha_grid=alpha_grid, + completed_keys=completed_keys, + ) parallel_workers = max(1, int(args.parallel_workers)) if parallel_workers <= 1: - for candidate, alpha in pending_tasks: - policy = {field: candidate[field] for field in SEMANTIC_POLICY_FIELDS} - result = _exact_policy_alpha( - aligned, - policy=policy, - alpha=float(alpha), - budget=float(args.budget), - t_eval=float(args.t_eval), - threads=int(args.exact_threads), - ) - record_result(candidate, alpha, result) + _run_serial_refinement( + pending_tasks=pending_tasks, + aligned=aligned, + budget=float(args.budget), + t_eval=float(args.t_eval), + exact_threads=int(args.exact_threads), + record_result=record_result, + ) else: - logger.info( - "Running exact refinement with {} parallel workers and {} solver thread(s) per worker", - parallel_workers, - int(args.exact_threads), + _run_parallel_refinement( + pending_tasks=pending_tasks, + aligned=aligned, + parallel_workers=parallel_workers, + budget=float(args.budget), + t_eval=float(args.t_eval), + exact_threads=int(args.exact_threads), + persist_progress=persist_progress, + record_result=record_result, ) - mp_context = mp.get_context("fork") if sys.platform != "win32" else None - max_in_flight = max(parallel_workers, parallel_workers * 2) - next_task_idx = 0 - futures: dict[Any, tuple[dict[str, Any], float]] = {} - with ProcessPoolExecutor( - max_workers=parallel_workers, - mp_context=mp_context, - initializer=_init_exact_worker, - initargs=(aligned,), - ) as executor: - while next_task_idx < len(pending_tasks) or futures: - while next_task_idx < len(pending_tasks) and len(futures) < max_in_flight: - candidate, alpha = pending_tasks[next_task_idx] - future = executor.submit( - _exact_policy_alpha_task, - candidate, - alpha, - float(args.budget), - float(args.t_eval), - int(args.exact_threads), - ) - futures[future] = (candidate, alpha) - next_task_idx += 1 - done, _ = wait(futures, return_when=FIRST_COMPLETED) - for future in done: - candidate, alpha = futures.pop(future) - try: - result = future.result() - except Exception: - persist_progress() - raise - result_only = { - key: value - for key, value in result.items() - if key not in candidate or key in {"alpha", "confidence"} - } - record_result(candidate, alpha, result_only) - bound_eval = pd.DataFrame(rows) - atomic_write_parquet(bound_eval, bound_eval_path, index=False) - leaderboard = _aggregate_leaderboard(candidates, bound_eval) - atomic_write_parquet(leaderboard, leaderboard_path, index=False) - claim_summary = _claim_summary(leaderboard, bound_eval, alpha_grid=alpha_grid) - atomic_write_json(claim_summary_path, claim_summary) + leaderboard, claim_summary = _write_final_outputs( + paths=paths, + candidates=candidates, + rows=rows, + alpha_grid=alpha_grid, + ) _write_status( run_tag=run_tag, - status_path=status_path, + status_path=paths.status_path, start_monotonic=start, completed=total_checks, total=total_checks, @@ -1661,8 +1722,8 @@ def record_result(candidate: dict[str, Any], alpha: float, result: dict[str, Any "n_all_alpha_passers_above_return_floor": int( claim_summary["n_all_alpha_passers_above_return_floor"] ), - "claim_summary_path": str(claim_summary_path), - "leaderboard_path": str(leaderboard_path), + "claim_summary_path": str(paths.claim_summary_path), + "leaderboard_path": str(paths.leaderboard_path), }, ) write_runtime_checkpoint( @@ -1671,12 +1732,12 @@ def record_result(candidate: dict[str, Any], alpha: float, result: dict[str, Any { "run_tag": run_tag, "completed_at_utc": datetime.now(tz=UTC).isoformat(), - "claim_summary_path": str(claim_summary_path), - "leaderboard_path": str(leaderboard_path), + "claim_summary_path": str(paths.claim_summary_path), + "leaderboard_path": str(paths.leaderboard_path), }, - checkpoint_dir=checkpoint_dir, + checkpoint_dir=paths.checkpoint_dir, ) - logger.info("Local IJDS refinement complete: {}", claim_summary_path) + logger.info("Local IJDS refinement complete: {}", paths.claim_summary_path) return 0 diff --git a/scripts/search/run_portfolio_bound_aware_search.py b/scripts/search/run_portfolio_bound_aware_search.py index 3c626d6..cb4be29 100644 --- a/scripts/search/run_portfolio_bound_aware_search.py +++ b/scripts/search/run_portfolio_bound_aware_search.py @@ -10,6 +10,7 @@ import sys import time from collections.abc import Callable, Mapping +from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -36,6 +37,9 @@ _load_aligned_dataset, _validate_single_alpha, ) +from src.optimization.certificate_semantics import ( # noqa: E402 + IJDS_DECLARED_ALPHA_GRID_CSV, +) from src.utils.pipeline_runtime import ( # noqa: E402 atomic_write_json, atomic_write_parquet, @@ -60,6 +64,91 @@ ] +@dataclass(frozen=True) +class BoundAwarePaths: + output_dir: Path + model_dir: Path + status_path: Path + checkpoint_dir: Path + resource_path: Path + gpu_csv_path: Path + frontier_raw_path: Path + frontier_path: Path + shortlist_path: Path + shortlist_exact_path: Path + bound_eval_path: Path + region_summary_path: Path + selection_path: Path + exact_context_path: Path + + +@dataclass(frozen=True) +class BoundAwareGridSpec: + risk_values: list[float] + aversion_values: list[float] + gamma_values: list[float] + delta_cap_quantiles: list[float] + tail_focus_quantiles: list[float] + alpha_grid: list[float] + random_states: list[int] + exact_random_states: list[int] + exact_max_candidates: int + budget_profiles: list[dict[str, Any]] + policy_modes: list[str] + incumbent_risk_neighbors: list[float] + incumbent_gamma_neighbors: list[float] + incumbent_policy_modes: list[str] + + @property + def policy_grid_count(self) -> int: + return _policy_grid_size( + gamma_values=self.gamma_values, + delta_cap_quantiles=self.delta_cap_quantiles, + tail_focus_quantiles=self.tail_focus_quantiles, + aversion_values=self.aversion_values, + budget_profiles=self.budget_profiles, + policy_modes=self.policy_modes, + ) + + @property + def frontier_total_units(self) -> int: + return int(len(self.random_states) * len(self.risk_values) * (1 + self.policy_grid_count)) + + def bound_total_checks(self, shortlist_size: int) -> int: + return int(shortlist_size * len(self.alpha_grid) * len(self.exact_random_states)) + + +@dataclass(frozen=True) +class BoundAwareExecutionSpec: + run_label: str + paths: BoundAwarePaths + grid: BoundAwareGridSpec + incumbent_policy: dict[str, Any] + exact_python_executable: str + exact_helper_script: Path + backend_validation: dict[str, Any] | None + cuopt_parameters: dict[str, Any] + + +@dataclass +class BoundAwareRunContext: + args: argparse.Namespace + spec: BoundAwareExecutionSpec + tracker: _ProgressTracker + search_space: dict[str, Any] + resource_payload: dict[str, Any] + + +@dataclass(frozen=True) +class BoundAwareFrontierState: + frontier_raw: pd.DataFrame + frontier: pd.DataFrame + shortlist: pd.DataFrame + bound_total_checks: int + shortlist_extra: dict[str, Any] + selection_context: dict[str, Any] + + def _env_int(name: str, fallback: int) -> int: raw = os.environ.get(name) if raw is None: @@ -83,6 +172,10 @@ def _coerce_int_csv(raw: str | None, *, fallback: int) -> list[int]: return values or [int(fallback)] +def _coerce_str_csv(raw: object) -> list[str]: + return [part.strip() for part in str(raw).split(",") if part.strip()] + + def _float_token(value: Any) -> float: return round(float(value), 10) @@ -119,6 +212,23 @@ def _policy_from_row( } +def _normalized_policy_mode_filter(policy_modes: list[str] | None) -> set[str]: + return {str(mode).strip() for mode in (policy_modes or []) if str(mode).strip()} + + +def _mode_quantile_pairs( + *, + mode: str, + delta_cap_quantiles: list[float], + tail_focus_quantiles: list[float], +) -> list[tuple[float, float]]: + if mode == "blended_uncertainty": + return [(1.0, 1.0)] + if mode == "capped_blended_uncertainty": + return [(float(delta_cap_quantile), 1.0) for delta_cap_quantile in delta_cap_quantiles] + return [(1.0, float(tail_focus_quantile)) for tail_focus_quantile in tail_focus_quantiles] + + def _targeted_policy_grid( *, gamma_values: list[float], @@ -126,52 +236,25 @@ def _targeted_policy_grid( tail_focus_quantiles: list[float], policy_modes: list[str] | None = None, ) -> list[tuple[str, float, float, float]]: - allowed = {str(mode).strip() for mode in (policy_modes or []) if str(mode).strip()} - use_all = not allowed - grid: list[tuple[str, float, float, float]] = [] - for gamma in gamma_values: - if use_all or "blended_uncertainty" in allowed: - grid.append(("blended_uncertainty", float(gamma), 1.0, 1.0)) - if use_all or "capped_blended_uncertainty" in allowed: - for delta_cap_quantile in delta_cap_quantiles: - grid.append( - ( - "capped_blended_uncertainty", - float(gamma), - float(delta_cap_quantile), - 1.0, - ) - ) - if use_all or "tail_blended_uncertainty" in allowed: - for tail_focus_quantile in tail_focus_quantiles: - grid.append( - ( - "tail_blended_uncertainty", - float(gamma), - 1.0, - float(tail_focus_quantile), - ) - ) - if use_all or "segment_tail_blended_uncertainty" in allowed: - for tail_focus_quantile in tail_focus_quantiles: - grid.append( - ( - "segment_tail_blended_uncertainty", - float(gamma), - 1.0, - float(tail_focus_quantile), - ) - ) - if use_all or "segment_relative_tail_blended_uncertainty" in allowed: - for tail_focus_quantile in tail_focus_quantiles: - grid.append( - ( - "segment_relative_tail_blended_uncertainty", - float(gamma), - 1.0, - float(tail_focus_quantile), - ) - ) + allowed = _normalized_policy_mode_filter(policy_modes) + mode_order = [ + "blended_uncertainty", + "capped_blended_uncertainty", + "tail_blended_uncertainty", + "segment_tail_blended_uncertainty", + "segment_relative_tail_blended_uncertainty", + ] + active_modes = [mode for mode in mode_order if not allowed or mode in allowed] + grid = [ + (mode, float(gamma), delta_cap_quantile, tail_focus_quantile) + for gamma in gamma_values + for mode in active_modes + for delta_cap_quantile, tail_focus_quantile in _mode_quantile_pairs( + mode=mode, + delta_cap_quantiles=delta_cap_quantiles, + tail_focus_quantiles=tail_focus_quantiles, + ) + ] return list(dict.fromkeys(grid)) @@ -748,7 +831,12 @@ def _aggregate_frontier(frontier_raw: pd.DataFrame) -> pd.DataFrame: def _aggregate_alpha_grid_results(bound_eval: pd.DataFrame) -> pd.DataFrame: """Summarize exact bound checks across every evaluated alpha level.""" if bound_eval.empty: - return pd.DataFrame(columns=[*SEMANTIC_POLICY_FIELDS, "alpha_exact_pass_count"]) + return pd.DataFrame( + { + **{field: pd.Series(dtype="object") for field in SEMANTIC_POLICY_FIELDS}, + "alpha_exact_pass_count": pd.Series(dtype="int64"), + } + ) grouped = bound_eval.groupby(SEMANTIC_POLICY_FIELDS, dropna=False) out = grouped.agg( alpha_exact_pass_count=("all_bounds_hold", "sum"), @@ -1125,7 +1213,8 @@ def _bucket_payload(frame: pd.DataFrame) -> dict[str, Any]: exact_alpha_summary = {} if not bound_eval.empty: for alpha, frame in bound_eval.groupby("alpha", dropna=False): - exact_alpha_summary[str(float(alpha))] = { + alpha_value = float(str(alpha)) + exact_alpha_summary[str(alpha_value)] = { "n_checks": int(len(frame)), "pass_rate": float(frame["all_bounds_hold"].fillna(False).mean()), "max_violation": float(frame["violation"].max()), @@ -1157,7 +1246,7 @@ def _selection_reason(row: pd.Series) -> str: return "selected_best_available_without_alpha01_pass" -def main(argv: list[str] | None = None) -> int: +def _build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser() parser.add_argument("--config", default="configs/crpto_optimization.yaml") parser.add_argument("--conformal-intervals-path", required=True) @@ -1175,7 +1264,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--bucket-proxy-k", type=int, default=40) parser.add_argument("--bucket-family-k", type=int, default=20) parser.add_argument("--bucket-region-k", type=int, default=20) - parser.add_argument("--alpha-grid", default="0.01,0.03,0.10") + parser.add_argument("--alpha-grid", default=IJDS_DECLARED_ALPHA_GRID_CSV) parser.add_argument("--max-candidates", type=int, default=5000) parser.add_argument( "--exact-max-candidates", @@ -1253,9 +1342,14 @@ def main(argv: list[str] | None = None) -> int: ) parser.add_argument("--budget", type=float, default=1_000_000.0) parser.add_argument("--t-eval", type=float, default=0.05) - args = parser.parse_args(argv) + return parser + - run_label = str(args.run_label).strip().replace("/", "_") +def _sanitize_run_label(raw: object) -> str: + return str(raw).strip().replace("/", "_") + + +def _resolve_run_paths(args: argparse.Namespace, *, run_label: str) -> BoundAwarePaths: output_dir = ( Path(str(args.output_dir)).expanduser() if str(args.output_dir).strip() @@ -1266,44 +1360,33 @@ def main(argv: list[str] | None = None) -> int: if str(args.model_dir).strip() else ROOT / "models" / "portfolio_bound_aware" / run_label ) - output_dir.mkdir(parents=True, exist_ok=True) - model_dir.mkdir(parents=True, exist_ok=True) - - status_path = model_dir / f"{STAGE_NAME}_runtime_status.json" - checkpoint_dir = model_dir / f"{STAGE_NAME}_runtime_checkpoints" - resource_path = model_dir / "resource_snapshot.json" - gpu_csv_path = model_dir / "gpu_samples.csv" - - risk_values = _coerce_csv(args.risk_grid) - aversion_values = _coerce_csv(args.aversion_grid) - gamma_values = _coerce_csv(args.gamma_grid) - delta_cap_quantiles = _coerce_csv(args.delta_cap_grid) - tail_focus_quantiles = _coerce_csv(args.tail_focus_grid) - alpha_grid = _coerce_csv(args.alpha_grid) - random_states = _coerce_int_csv(args.random_states, fallback=int(args.random_state)) - exact_random_states = ( - _coerce_int_csv(args.exact_random_states, fallback=int(args.random_state)) - if str(args.exact_random_states).strip() - else list(random_states) - ) - exact_max_candidates = ( - int(args.exact_max_candidates) - if args.exact_max_candidates is not None - else int(args.max_candidates) + return BoundAwarePaths( + output_dir=output_dir, + model_dir=model_dir, + status_path=model_dir / f"{STAGE_NAME}_runtime_status.json", + checkpoint_dir=model_dir / f"{STAGE_NAME}_runtime_checkpoints", + resource_path=model_dir / "resource_snapshot.json", + gpu_csv_path=model_dir / "gpu_samples.csv", + frontier_raw_path=output_dir / "portfolio_bound_aware_frontier_raw.parquet", + frontier_path=output_dir / "portfolio_bound_aware_frontier.parquet", + shortlist_path=output_dir / "portfolio_bound_aware_shortlist.parquet", + shortlist_exact_path=output_dir / "portfolio_bound_aware_shortlist_exact.parquet", + bound_eval_path=output_dir / "portfolio_bound_aware_bound_eval.parquet", + region_summary_path=model_dir / "portfolio_bound_aware_region_summary.json", + selection_path=model_dir / "portfolio_bound_aware_selection.json", + exact_context_path=model_dir / "portfolio_bound_aware_exact_context.json", ) - incumbent_risk_neighbors = _coerce_csv(args.incumbent_risk_neighbors) - incumbent_gamma_neighbors = _coerce_csv(args.incumbent_gamma_neighbors) - incumbent_policy_modes = [ - part.strip() for part in str(args.incumbent_policy_modes).split(",") if part.strip() - ] - policy_modes = [part.strip() for part in str(args.policy_modes).split(",") if part.strip()] - exact_python_executable = str(args.exact_python_executable).strip() - exact_helper_script = Path(str(args.exact_helper_script)).resolve() + +def _ensure_run_dirs(paths: BoundAwarePaths) -> None: + paths.output_dir.mkdir(parents=True, exist_ok=True) + paths.model_dir.mkdir(parents=True, exist_ok=True) + + +def _budget_profiles(raw: object) -> list[dict[str, Any]]: budget_profiles: list[dict[str, Any]] = [] - for token in [ - part.strip().lower() for part in str(args.budget_profiles).split(",") if part.strip() - ]: + tokens = [part.strip().lower() for part in str(raw).split(",") if part.strip()] + for token in tokens: if token == "free": budget_profiles.append( {"name": "free_budget", "min_budget_utilization": 0.0, "pd_cap_slack_penalty": 0.0} @@ -1320,413 +1403,695 @@ def main(argv: list[str] | None = None) -> int: raise ValueError(f"Unsupported budget profile: {token}") if not budget_profiles: raise ValueError("At least one budget profile is required") + return budget_profiles - incumbent_policy = _load_incumbent_policy(args.incumbent_policy_path) - backend_validation: dict[str, Any] | None = None - if ( - str(args.solver_backend).strip().lower() == "cuopt" - or str(args.exact_solver_backend).strip().lower() == "cuopt" - ): - backend_validation = _validate_cuopt_runtime() + +def _search_space_payload( + *, + args: argparse.Namespace, + risk_values: list[float], + aversion_values: list[float], + gamma_values: list[float], + delta_cap_quantiles: list[float], + tail_focus_quantiles: list[float], + budget_profiles: list[dict[str, Any]], + alpha_grid: list[float], + random_states: list[int], + exact_random_states: list[int], + exact_max_candidates: int, + policy_modes: list[str], + cuopt_parameters: dict[str, Any], + incumbent_risk_neighbors: list[float], + incumbent_gamma_neighbors: list[float], + incumbent_policy_modes: list[str], +) -> dict[str, Any]: + return { + "risk_grid": risk_values, + "aversion_grid": aversion_values, + "gamma_grid": gamma_values, + "delta_cap_grid": delta_cap_quantiles, + "tail_focus_grid": tail_focus_quantiles, + "budget_profiles": budget_profiles, + "alpha_grid": alpha_grid, + "max_candidates": int(args.max_candidates), + "exact_max_candidates": int(exact_max_candidates), + "random_states": random_states, + "exact_random_states": exact_random_states, + "exact_checkpoint_every": int(args.exact_checkpoint_every), + "exact_threads": int(args.exact_threads), + "policy_modes": policy_modes, + "cuopt_parameters": cuopt_parameters, + "bucket_return_k": int(args.bucket_return_k), + "bucket_proxy_k": int(args.bucket_proxy_k), + "bucket_family_k": int(args.bucket_family_k), + "bucket_region_k": int(args.bucket_region_k), + "incumbent_policy_path": str(args.incumbent_policy_path), + "incumbent_risk_neighbors": incumbent_risk_neighbors, + "incumbent_gamma_neighbors": incumbent_gamma_neighbors, + "incumbent_policy_modes": incumbent_policy_modes, + } + + +def _selection_policy_payload() -> dict[str, Any]: + return { + "shortlist_strategy": "stratified_bound_first", + "rank_order": [ + "alpha01_exact_pass(desc)", + "alpha03_exact_pass(desc)", + "alpha_exact_pass_count(desc)", + "alpha_exact_pass_rate(desc)", + "ab_pass_all(desc)", + "realized_total_return(desc)", + "alpha01_weighted_miscoverage_V(asc)", + "alpha01_gamma_cp(asc)", + "price_of_robustness(asc)", + ], + } + + +def _selection_context_payload( + *, + args: argparse.Namespace, + paths: BoundAwarePaths, + run_label: str, + search_space: dict[str, Any], + exact_max_candidates: int, + random_states: list[int], + exact_random_states: list[int], + alpha_grid: list[float], +) -> dict[str, Any]: + return { + "schema_version": SCHEMA_VERSION, + "generated_at_utc": datetime.now(tz=UTC).isoformat(), + "run_label": run_label, + "conformal_intervals_path": str(args.conformal_intervals_path), + "search_space": search_space, + "selection_policy": _selection_policy_payload(), + "frontier_raw_path": str(paths.frontier_raw_path), + "frontier_path": str(paths.frontier_path), + "shortlist_path": str(paths.shortlist_path), + "shortlist_exact_path": str(paths.shortlist_exact_path), + "bound_eval_path": str(paths.bound_eval_path), + "region_summary_path": str(paths.region_summary_path), + "selection_path": str(paths.selection_path), + "runtime_status_path": str(paths.status_path), + "runtime_checkpoint_dir": str(paths.checkpoint_dir), + "resource_snapshot_path": str(paths.resource_path), + "frontier_solver_backend": str(args.solver_backend), + "exact_solver_backend": str(args.exact_solver_backend), + "budget": float(args.budget), + "t_eval": float(args.t_eval), + "max_candidates": int(args.max_candidates), + "exact_max_candidates": int(exact_max_candidates), + "random_states": random_states, + "exact_random_states": exact_random_states, + "exact_checkpoint_every": int(args.exact_checkpoint_every), + "exact_threads": int(args.exact_threads), + "alpha_grid": alpha_grid, + } + + +def _run_in_process_exact_bound_eval( + *, + args: argparse.Namespace, + shortlist: pd.DataFrame, + alpha_grid: list[float], + exact_random_states: list[int], + exact_max_candidates: int, + tracker: _ProgressTracker, +) -> pd.DataFrame: + aligned_by_seed = { + int(seed): _load_aligned_dataset( + conformal_intervals_path=args.conformal_intervals_path, + max_candidates=int(exact_max_candidates), + random_state=int(seed), + ) + for seed in exact_random_states + } + bound_rows: list[dict[str, Any]] = [] + completed_checks = 0 + for _, row in shortlist.iterrows(): + policy = _policy_from_row( + row, + solver_backend_override=str(args.exact_solver_backend), + ) + candidate_payload = row.to_dict() + for eval_seed in exact_random_states: + aligned = aligned_by_seed[int(eval_seed)] + for alpha in alpha_grid: + result = _validate_single_alpha( + aligned, + alpha=float(alpha), + policy=policy, + allocator_mode="exact", + budget=float(args.budget), + t_eval=float(args.t_eval), + threads=int(args.exact_threads), + ) + bound_rows.append( + { + "candidate_rank": int(candidate_payload["candidate_rank"]), + "eval_random_state": int(eval_seed), + "frontier_solver_backend": str(args.solver_backend), + "exact_solver_backend": str(args.exact_solver_backend), + **candidate_payload, + **result, + } + ) + completed_checks += 1 + tracker.bound_progress( + completed_checks=completed_checks, + extra={ + "candidate_rank": int(candidate_payload["candidate_rank"]), + "eval_random_state": int(eval_seed), + "current_alpha": float(alpha), + "exact_threads": int(args.exact_threads), + }, + ) + return pd.DataFrame(bound_rows) + + +def _build_selection_payload( + *, + args: argparse.Namespace, + run_label: str, + selection_context: dict[str, Any], + selected: pd.Series, + selected_policy: dict[str, Any], + region_payload: dict[str, Any], +) -> dict[str, Any]: + return { + "schema_version": SCHEMA_VERSION, + "generated_at_utc": datetime.now(tz=UTC).isoformat(), + "run_label": run_label, + "conformal_intervals_path": str(args.conformal_intervals_path), + "search_space": selection_context["search_space"], + "selection_policy": selection_context["selection_policy"], + "selected_policy": selected_policy, + "selected_metrics": selected.to_dict(), + "selection_reason": _selection_reason(selected), + "frontier_raw_path": selection_context["frontier_raw_path"], + "frontier_path": selection_context["frontier_path"], + "shortlist_path": selection_context["shortlist_path"], + "shortlist_exact_path": selection_context["shortlist_exact_path"], + "bound_eval_path": selection_context["bound_eval_path"], + "region_summary_path": selection_context["region_summary_path"], + "robust_region_summary": region_payload, + "runtime_status_path": selection_context["runtime_status_path"], + "runtime_checkpoint_dir": selection_context["runtime_checkpoint_dir"], + "resource_snapshot_path": selection_context["resource_snapshot_path"], + "frontier_solver_backend": str(args.solver_backend), + "exact_solver_backend": str(args.exact_solver_backend), + "exact_threads": int(args.exact_threads), + } + + +def _write_selection_outputs( + *, + paths: BoundAwarePaths, + shortlist_eval: pd.DataFrame, + bound_eval: pd.DataFrame, + region_payload: dict[str, Any], + payload: dict[str, Any], +) -> None: + atomic_write_parquet(shortlist_eval, paths.shortlist_exact_path, index=False) + atomic_write_parquet(bound_eval, paths.bound_eval_path, index=False) + atomic_write_json(paths.region_summary_path, region_payload) + atomic_write_json(paths.selection_path, payload) + + +def _build_frontier_outputs( + *, + args: argparse.Namespace, + risk_values: list[float], + aversion_values: list[float], + gamma_values: list[float], + delta_cap_quantiles: list[float], + tail_focus_quantiles: list[float], + budget_profiles: list[dict[str, Any]], + random_states: list[int], + policy_grid_count: int, + policy_modes: list[str], + cuopt_parameters: dict[str, Any], + tracker: _ProgressTracker, +) -> tuple[pd.DataFrame, pd.DataFrame]: + frontier_frames: list[pd.DataFrame] = [] + frontier_completed = 0 + for seed in random_states: + seed_offset = frontier_completed + + def _progress_hook( + local_completed: int, + extra: dict[str, Any], + _offset: int = seed_offset, + ) -> None: + tracker.frontier_progress(completed_units=_offset + local_completed, extra=extra) + + frontier_seed = _build_frontier_for_seed( + config_path=args.config, + conformal_intervals_path=args.conformal_intervals_path, + risk_values=risk_values, + aversion_values=aversion_values, + gamma_values=gamma_values, + delta_cap_quantiles=delta_cap_quantiles, + tail_focus_quantiles=tail_focus_quantiles, + budget_profiles=budget_profiles, + max_candidates=int(args.max_candidates), + random_state=int(seed), + solver_backend=str(args.solver_backend), + cuopt_presolve=int(args.cuopt_presolve) + if str(args.solver_backend) == "cuopt" + else None, + cuopt_parameters=cuopt_parameters, + policy_modes=policy_modes, + progress_hook=_progress_hook, + ) + frontier_frames.append(frontier_seed) + frontier_completed += int(len(risk_values) * (1 + policy_grid_count)) + + frontier_raw = ( + pd.concat(frontier_frames, ignore_index=True) if frontier_frames else pd.DataFrame() + ) + if frontier_raw.empty: + raise ValueError("Frontier search produced zero candidate rows.") + return frontier_raw, _aggregate_frontier(frontier_raw) + + +def _write_frontier_artifacts( + *, + paths: BoundAwarePaths, + frontier_raw: pd.DataFrame, + frontier: pd.DataFrame, + shortlist: pd.DataFrame, +) -> None: + atomic_write_parquet(frontier_raw, paths.frontier_raw_path, index=False) + atomic_write_parquet(frontier, paths.frontier_path, index=False) + atomic_write_parquet(shortlist, paths.shortlist_path, index=False) + + +def _complete_frontier_only( + *, + paths: BoundAwarePaths, + tracker: _ProgressTracker, + resource_payload: dict[str, Any], + shortlist_extra: dict[str, Any], + bound_total_checks: int, +) -> None: + resource_payload["end"] = _resource_snapshot() + atomic_write_json(paths.resource_path, resource_payload) + tracker.frontier_only_complete( + extra={ + **shortlist_extra, + "frontier_only": True, + "deferred_bound_total_checks": bound_total_checks, + "exact_context_path": str(paths.exact_context_path), + "frontier_path": str(paths.frontier_path), + "shortlist_path": str(paths.shortlist_path), + } + ) + + +def _delegate_exact_stage_if_requested( + *, + exact_python_executable: str, + exact_helper_script: Path, + paths: BoundAwarePaths, + resource_payload: dict[str, Any], +) -> bool: + if not exact_python_executable: + return False + current_python = Path(sys.executable) + requested_python = Path(exact_python_executable) + if requested_python == current_python: + return False + cmd = [ + str(requested_python), + str(exact_helper_script), + "--context-path", + str(paths.exact_context_path), + ] + logger.info("Delegating exact bound stage to external Python: {}", " ".join(cmd)) + subprocess.run(cmd, cwd=str(ROOT), check=True) + resource_payload["end"] = _resource_snapshot() + atomic_write_json(paths.resource_path, resource_payload) + return True + + +def _build_grid_spec(args: argparse.Namespace) -> BoundAwareGridSpec: + random_states = _coerce_int_csv(args.random_states, fallback=int(args.random_state)) + exact_random_states = ( + _coerce_int_csv(args.exact_random_states, fallback=int(args.random_state)) + if str(args.exact_random_states).strip() + else list(random_states) + ) + exact_max_candidates = ( + int(args.exact_max_candidates) + if args.exact_max_candidates is not None + else int(args.max_candidates) + ) + return BoundAwareGridSpec( + risk_values=_coerce_csv(args.risk_grid), + aversion_values=_coerce_csv(args.aversion_grid), + gamma_values=_coerce_csv(args.gamma_grid), + delta_cap_quantiles=_coerce_csv(args.delta_cap_grid), + tail_focus_quantiles=_coerce_csv(args.tail_focus_grid), + alpha_grid=_coerce_csv(args.alpha_grid), + random_states=random_states, + exact_random_states=exact_random_states, + exact_max_candidates=exact_max_candidates, + budget_profiles=_budget_profiles(args.budget_profiles), + policy_modes=_coerce_str_csv(args.policy_modes), + incumbent_risk_neighbors=_coerce_csv(args.incumbent_risk_neighbors), + incumbent_gamma_neighbors=_coerce_csv(args.incumbent_gamma_neighbors), + incumbent_policy_modes=_coerce_str_csv(args.incumbent_policy_modes), + ) + + +def _prepare_execution_spec(args: argparse.Namespace) -> BoundAwareExecutionSpec: + run_label = _sanitize_run_label(args.run_label) + paths = _resolve_run_paths(args, run_label=run_label) + _ensure_run_dirs(paths) + grid = _build_grid_spec(args) + uses_cuopt = "cuopt" in { + str(args.solver_backend).strip().lower(), + str(args.exact_solver_backend).strip().lower(), + } + backend_validation = _validate_cuopt_runtime() if uses_cuopt else None cuopt_parameters = ( - _cuopt_parameter_overrides(args, model_dir=model_dir) + _cuopt_parameter_overrides(args, model_dir=paths.model_dir) if str(args.solver_backend).strip().lower() == "cuopt" else {} ) - - policy_grid_count = _policy_grid_size( - gamma_values=gamma_values, - delta_cap_quantiles=delta_cap_quantiles, - tail_focus_quantiles=tail_focus_quantiles, - aversion_values=aversion_values, - budget_profiles=budget_profiles, - policy_modes=policy_modes, + return BoundAwareExecutionSpec( + run_label=run_label, + paths=paths, + grid=grid, + incumbent_policy=_load_incumbent_policy(args.incumbent_policy_path), + exact_python_executable=str(args.exact_python_executable).strip(), + exact_helper_script=Path(str(args.exact_helper_script)).resolve(), + backend_validation=backend_validation, + cuopt_parameters=cuopt_parameters, ) - frontier_total_units = int(len(random_states) * len(risk_values) * (1 + policy_grid_count)) + + +def _initialize_run_context(args: argparse.Namespace) -> BoundAwareRunContext: + spec = _prepare_execution_spec(args) + grid = spec.grid tracker = _ProgressTracker( - status_path=status_path, - checkpoint_dir=checkpoint_dir, - run_tag=run_label, - frontier_total_units=frontier_total_units, + status_path=spec.paths.status_path, + checkpoint_dir=spec.paths.checkpoint_dir, + run_tag=spec.run_label, + frontier_total_units=grid.frontier_total_units, + ) + search_space = _search_space_payload( + args=args, + risk_values=grid.risk_values, + aversion_values=grid.aversion_values, + gamma_values=grid.gamma_values, + delta_cap_quantiles=grid.delta_cap_quantiles, + tail_focus_quantiles=grid.tail_focus_quantiles, + budget_profiles=grid.budget_profiles, + alpha_grid=grid.alpha_grid, + random_states=grid.random_states, + exact_random_states=grid.exact_random_states, + exact_max_candidates=grid.exact_max_candidates, + policy_modes=grid.policy_modes, + cuopt_parameters=spec.cuopt_parameters, + incumbent_risk_neighbors=grid.incumbent_risk_neighbors, + incumbent_gamma_neighbors=grid.incumbent_gamma_neighbors, + incumbent_policy_modes=grid.incumbent_policy_modes, ) resource_payload = { "schema_version": SCHEMA_VERSION, - "run_label": run_label, + "run_label": spec.run_label, "solver_backend": str(args.solver_backend), "exact_solver_backend": str(args.exact_solver_backend), - "cuopt_parameters": cuopt_parameters, + "cuopt_parameters": spec.cuopt_parameters, "start": _resource_snapshot(), - "backend_validation": backend_validation, + "backend_validation": spec.backend_validation, } - atomic_write_json(resource_path, resource_payload) + atomic_write_json(spec.paths.resource_path, resource_payload) tracker.start( extra={ - "random_states": random_states, - "search_space": { - "risk_grid": risk_values, - "aversion_grid": aversion_values, - "gamma_grid": gamma_values, - "delta_cap_grid": delta_cap_quantiles, - "tail_focus_grid": tail_focus_quantiles, - "budget_profiles": budget_profiles, - "alpha_grid": alpha_grid, - "max_candidates": int(args.max_candidates), - "exact_max_candidates": int(exact_max_candidates), - "exact_random_states": exact_random_states, - "exact_checkpoint_every": int(args.exact_checkpoint_every), - "exact_threads": int(args.exact_threads), - "cuopt_parameters": cuopt_parameters, - }, + "random_states": grid.random_states, + "search_space": search_space, } ) + return BoundAwareRunContext( + args=args, + spec=spec, + tracker=tracker, + search_space=search_space, + resource_payload=resource_payload, + ) - gpu_sampler: _GpuSampler | None = None - if str(args.solver_backend).strip().lower() == "cuopt": - gpu_sampler = _GpuSampler(gpu_csv_path) + +def _start_gpu_sampler(context: BoundAwareRunContext) -> _GpuSampler | None: + if str(context.args.solver_backend).strip().lower() == "cuopt": + gpu_sampler = _GpuSampler(context.spec.paths.gpu_csv_path) gpu_sampler.start() + return gpu_sampler + return None + + +def _build_frontier_state(context: BoundAwareRunContext) -> BoundAwareFrontierState: + args = context.args + spec = context.spec + grid = spec.grid + tracker = context.tracker + frontier_raw, frontier = _build_frontier_outputs( + args=args, + risk_values=grid.risk_values, + aversion_values=grid.aversion_values, + gamma_values=grid.gamma_values, + delta_cap_quantiles=grid.delta_cap_quantiles, + tail_focus_quantiles=grid.tail_focus_quantiles, + budget_profiles=grid.budget_profiles, + random_states=grid.random_states, + policy_grid_count=grid.policy_grid_count, + policy_modes=grid.policy_modes, + cuopt_parameters=spec.cuopt_parameters, + tracker=tracker, + ) + tracker.frontier_complete( + extra={ + "frontier_policy_count": len(frontier), + "frontier_raw_rows": len(frontier_raw), + } + ) + shortlist = _build_stratified_shortlist( + frontier=frontier, + shortlist_top_k=int(args.shortlist_top_k), + bucket_return_k=int(args.bucket_return_k), + bucket_proxy_k=int(args.bucket_proxy_k), + bucket_family_k=int(args.bucket_family_k), + bucket_region_k=int(args.bucket_region_k), + incumbent_policy=spec.incumbent_policy, + incumbent_risk_neighbors=grid.incumbent_risk_neighbors, + incumbent_gamma_neighbors=grid.incumbent_gamma_neighbors, + incumbent_policy_modes=grid.incumbent_policy_modes, + budget_profiles=grid.budget_profiles, + solver_backend=str(args.solver_backend), + ) + bound_total_checks = grid.bound_total_checks(len(shortlist)) + shortlist_extra = { + "shortlist_size": len(shortlist), + "shortlist_buckets": shortlist["shortlist_bucket"].value_counts(dropna=False).to_dict(), + } + if not args.frontier_only: + tracker.set_bound_total(bound_total_checks, extra=shortlist_extra) + _write_frontier_artifacts( + paths=spec.paths, + frontier_raw=frontier_raw, + frontier=frontier, + shortlist=shortlist, + ) + selection_context = _selection_context_payload( + args=args, + paths=spec.paths, + run_label=spec.run_label, + search_space=context.search_space, + exact_max_candidates=grid.exact_max_candidates, + random_states=grid.random_states, + exact_random_states=grid.exact_random_states, + alpha_grid=grid.alpha_grid, + ) + atomic_write_json(spec.paths.exact_context_path, selection_context) + return BoundAwareFrontierState( + frontier_raw=frontier_raw, + frontier=frontier, + shortlist=shortlist, + bound_total_checks=bound_total_checks, + shortlist_extra=shortlist_extra, + selection_context=selection_context, + ) - frontier_frames: list[pd.DataFrame] = [] - frontier_completed = 0 - try: - for seed in random_states: - seed_offset = frontier_completed - - def _progress_hook( - local_completed: int, - extra: dict[str, Any], - _offset: int = seed_offset, - ) -> None: - tracker.frontier_progress(completed_units=_offset + local_completed, extra=extra) - - frontier_seed = _build_frontier_for_seed( - config_path=args.config, - conformal_intervals_path=args.conformal_intervals_path, - risk_values=risk_values, - aversion_values=aversion_values, - gamma_values=gamma_values, - delta_cap_quantiles=delta_cap_quantiles, - tail_focus_quantiles=tail_focus_quantiles, - budget_profiles=budget_profiles, - max_candidates=int(args.max_candidates), - random_state=int(seed), - solver_backend=str(args.solver_backend), - cuopt_presolve=int(args.cuopt_presolve) - if str(args.solver_backend) == "cuopt" - else None, - cuopt_parameters=cuopt_parameters, - policy_modes=policy_modes, - progress_hook=_progress_hook, - ) - frontier_frames.append(frontier_seed) - frontier_completed += int(len(risk_values) * (1 + policy_grid_count)) - frontier_raw = ( - pd.concat(frontier_frames, ignore_index=True) if frontier_frames else pd.DataFrame() - ) - if frontier_raw.empty: - raise ValueError("Frontier search produced zero candidate rows.") - frontier = _aggregate_frontier(frontier_raw) - tracker.frontier_complete( - extra={ - "frontier_policy_count": len(frontier), - "frontier_raw_rows": len(frontier_raw), - } +def _finish_after_frontier( + context: BoundAwareRunContext, + state: BoundAwareFrontierState, +) -> bool: + spec = context.spec + if context.args.frontier_only: + _complete_frontier_only( + paths=spec.paths, + tracker=context.tracker, + resource_payload=context.resource_payload, + shortlist_extra=state.shortlist_extra, + bound_total_checks=state.bound_total_checks, ) + return True + return _delegate_exact_stage_if_requested( + exact_python_executable=spec.exact_python_executable, + exact_helper_script=spec.exact_helper_script, + paths=spec.paths, + resource_payload=context.resource_payload, + ) - shortlist = _build_stratified_shortlist( - frontier=frontier, - shortlist_top_k=int(args.shortlist_top_k), - bucket_return_k=int(args.bucket_return_k), - bucket_proxy_k=int(args.bucket_proxy_k), - bucket_family_k=int(args.bucket_family_k), - bucket_region_k=int(args.bucket_region_k), - incumbent_policy=incumbent_policy, - incumbent_risk_neighbors=incumbent_risk_neighbors, - incumbent_gamma_neighbors=incumbent_gamma_neighbors, - incumbent_policy_modes=incumbent_policy_modes, - budget_profiles=budget_profiles, - solver_backend=str(args.solver_backend), - ) - bound_total_checks = int(len(shortlist) * len(alpha_grid) * len(exact_random_states)) - shortlist_extra = { - "shortlist_size": len(shortlist), - "shortlist_buckets": shortlist["shortlist_bucket"].value_counts(dropna=False).to_dict(), - } - if not args.frontier_only: - tracker.set_bound_total(bound_total_checks, extra=shortlist_extra) - atomic_write_parquet( - frontier_raw, output_dir / "portfolio_bound_aware_frontier_raw.parquet", index=False - ) - atomic_write_parquet( - frontier, output_dir / "portfolio_bound_aware_frontier.parquet", index=False - ) - atomic_write_parquet( - shortlist, output_dir / "portfolio_bound_aware_shortlist.parquet", index=False - ) +def _run_exact_selection( + context: BoundAwareRunContext, + state: BoundAwareFrontierState, +) -> tuple[pd.Series, dict[str, Any]]: + args = context.args + spec = context.spec + grid = spec.grid + bound_eval = _run_in_process_exact_bound_eval( + args=args, + shortlist=state.shortlist, + alpha_grid=grid.alpha_grid, + exact_random_states=grid.exact_random_states, + exact_max_candidates=grid.exact_max_candidates, + tracker=context.tracker, + ) + shortlist_eval = _aggregate_exact_results(shortlist=state.shortlist, bound_eval=bound_eval) + region_payload = _region_summary(shortlist_eval, bound_eval) + selected = shortlist_eval.iloc[0].copy() + selected_policy = _policy_from_row( + selected, + solver_backend_override=str(args.exact_solver_backend), + ) + payload = _build_selection_payload( + args=args, + run_label=spec.run_label, + selection_context=state.selection_context, + selected=selected, + selected_policy=selected_policy, + region_payload=region_payload, + ) + _write_selection_outputs( + paths=spec.paths, + shortlist_eval=shortlist_eval, + bound_eval=bound_eval, + region_payload=region_payload, + payload=payload, + ) + return selected, payload - selection_context = { - "schema_version": SCHEMA_VERSION, - "generated_at_utc": datetime.now(tz=UTC).isoformat(), - "run_label": run_label, - "conformal_intervals_path": str(args.conformal_intervals_path), - "search_space": { - "risk_grid": risk_values, - "aversion_grid": aversion_values, - "gamma_grid": gamma_values, - "delta_cap_grid": delta_cap_quantiles, - "tail_focus_grid": tail_focus_quantiles, - "budget_profiles": budget_profiles, - "alpha_grid": alpha_grid, - "max_candidates": int(args.max_candidates), - "exact_max_candidates": int(exact_max_candidates), - "random_states": random_states, - "exact_random_states": exact_random_states, - "exact_checkpoint_every": int(args.exact_checkpoint_every), - "exact_threads": int(args.exact_threads), - "policy_modes": policy_modes, - "cuopt_parameters": cuopt_parameters, - "bucket_return_k": int(args.bucket_return_k), - "bucket_proxy_k": int(args.bucket_proxy_k), - "bucket_family_k": int(args.bucket_family_k), - "bucket_region_k": int(args.bucket_region_k), - "incumbent_policy_path": str(args.incumbent_policy_path), - "incumbent_risk_neighbors": incumbent_risk_neighbors, - "incumbent_gamma_neighbors": incumbent_gamma_neighbors, - "incumbent_policy_modes": incumbent_policy_modes, - }, - "selection_policy": { - "shortlist_strategy": "stratified_bound_first", - "rank_order": [ - "alpha01_exact_pass(desc)", - "alpha03_exact_pass(desc)", - "alpha_exact_pass_count(desc)", - "alpha_exact_pass_rate(desc)", - "ab_pass_all(desc)", - "realized_total_return(desc)", - "alpha01_weighted_miscoverage_V(asc)", - "alpha01_gamma_cp(asc)", - "price_of_robustness(asc)", - ], - }, - "frontier_raw_path": str(output_dir / "portfolio_bound_aware_frontier_raw.parquet"), - "frontier_path": str(output_dir / "portfolio_bound_aware_frontier.parquet"), - "shortlist_path": str(output_dir / "portfolio_bound_aware_shortlist.parquet"), - "shortlist_exact_path": str( - output_dir / "portfolio_bound_aware_shortlist_exact.parquet" - ), - "bound_eval_path": str(output_dir / "portfolio_bound_aware_bound_eval.parquet"), - "region_summary_path": str(model_dir / "portfolio_bound_aware_region_summary.json"), - "selection_path": str(model_dir / "portfolio_bound_aware_selection.json"), - "runtime_status_path": str(status_path), - "runtime_checkpoint_dir": str(checkpoint_dir), - "resource_snapshot_path": str(resource_path), - "frontier_solver_backend": str(args.solver_backend), - "exact_solver_backend": str(args.exact_solver_backend), - "budget": float(args.budget), - "t_eval": float(args.t_eval), - "max_candidates": int(args.max_candidates), - "exact_max_candidates": int(exact_max_candidates), - "random_states": random_states, - "exact_random_states": exact_random_states, - "exact_checkpoint_every": int(args.exact_checkpoint_every), - "exact_threads": int(args.exact_threads), - "alpha_grid": alpha_grid, - } - exact_context_path = model_dir / "portfolio_bound_aware_exact_context.json" - atomic_write_json(exact_context_path, selection_context) - - if args.frontier_only: - resource_payload["end"] = _resource_snapshot() - atomic_write_json(resource_path, resource_payload) - tracker.frontier_only_complete( - extra={ - **shortlist_extra, - "frontier_only": True, - "deferred_bound_total_checks": bound_total_checks, - "exact_context_path": str(exact_context_path), - "frontier_path": str(output_dir / "portfolio_bound_aware_frontier.parquet"), - "shortlist_path": str(output_dir / "portfolio_bound_aware_shortlist.parquet"), - } - ) - return 0 - - if exact_python_executable: - current_python = Path(sys.executable) - requested_python = Path(exact_python_executable) - if requested_python != current_python: - cmd = [ - str(requested_python), - str(exact_helper_script), - "--context-path", - str(exact_context_path), - ] - logger.info( - "Delegating exact bound stage to external Python: {}", - " ".join(cmd), - ) - subprocess.run(cmd, cwd=str(ROOT), check=True) - resource_payload["end"] = _resource_snapshot() - atomic_write_json(resource_path, resource_payload) - return 0 - - aligned_by_seed = { - int(seed): _load_aligned_dataset( - conformal_intervals_path=args.conformal_intervals_path, - max_candidates=int(exact_max_candidates), - random_state=int(seed), - ) - for seed in exact_random_states - } - bound_rows: list[dict[str, Any]] = [] - completed_checks = 0 - for _, row in shortlist.iterrows(): - policy = _policy_from_row( - row, - solver_backend_override=str(args.exact_solver_backend), - ) - candidate_payload = row.to_dict() - for eval_seed in exact_random_states: - aligned = aligned_by_seed[int(eval_seed)] - for alpha in alpha_grid: - result = _validate_single_alpha( - aligned, - alpha=float(alpha), - policy=policy, - allocator_mode="exact", - budget=float(args.budget), - t_eval=float(args.t_eval), - threads=int(args.exact_threads), - ) - bound_rows.append( - { - "candidate_rank": int(candidate_payload["candidate_rank"]), - "eval_random_state": int(eval_seed), - "frontier_solver_backend": str(args.solver_backend), - "exact_solver_backend": str(args.exact_solver_backend), - **candidate_payload, - **result, - } - ) - completed_checks += 1 - tracker.bound_progress( - completed_checks=completed_checks, - extra={ - "candidate_rank": int(candidate_payload["candidate_rank"]), - "eval_random_state": int(eval_seed), - "current_alpha": float(alpha), - "exact_threads": int(args.exact_threads), - }, - ) - bound_eval = pd.DataFrame(bound_rows) - shortlist_eval = _aggregate_exact_results(shortlist=shortlist, bound_eval=bound_eval) - region_payload = _region_summary(shortlist_eval, bound_eval) - selected = shortlist_eval.iloc[0].copy() - selected_policy = _policy_from_row( - selected, - solver_backend_override=str(args.exact_solver_backend), - ) - payload = { - "schema_version": SCHEMA_VERSION, - "generated_at_utc": datetime.now(tz=UTC).isoformat(), - "run_label": run_label, - "conformal_intervals_path": str(args.conformal_intervals_path), - "search_space": selection_context["search_space"], - "selection_policy": selection_context["selection_policy"], - "selected_policy": selected_policy, - "selected_metrics": selected.to_dict(), - "selection_reason": _selection_reason(selected), - "frontier_raw_path": selection_context["frontier_raw_path"], - "frontier_path": selection_context["frontier_path"], - "shortlist_path": selection_context["shortlist_path"], - "shortlist_exact_path": selection_context["shortlist_exact_path"], - "bound_eval_path": selection_context["bound_eval_path"], - "region_summary_path": selection_context["region_summary_path"], - "robust_region_summary": region_payload, - "runtime_status_path": selection_context["runtime_status_path"], - "runtime_checkpoint_dir": selection_context["runtime_checkpoint_dir"], - "resource_snapshot_path": selection_context["resource_snapshot_path"], - "frontier_solver_backend": str(args.solver_backend), - "exact_solver_backend": str(args.exact_solver_backend), - "exact_threads": int(args.exact_threads), +def _complete_run( + context: BoundAwareRunContext, + *, + selected: pd.Series, + payload: dict[str, Any], + gpu_sampler: _GpuSampler | None, +) -> None: + if gpu_sampler is not None: + context.resource_payload["gpu_summary"] = gpu_sampler.stop() + context.resource_payload["end"] = _resource_snapshot() + atomic_write_json(context.spec.paths.resource_path, context.resource_payload) + context.tracker.complete( + extra={ + "selection_reason": str(payload["selection_reason"]), + "selected_alpha01_exact_pass": bool(selected["alpha01_exact_pass"]), + "selected_realized_total_return": float(selected["realized_total_return"]), } + ) - atomic_write_parquet( - shortlist_eval, - output_dir / "portfolio_bound_aware_shortlist_exact.parquet", - index=False, - ) - atomic_write_parquet( - bound_eval, output_dir / "portfolio_bound_aware_bound_eval.parquet", index=False - ) - atomic_write_json(model_dir / "portfolio_bound_aware_region_summary.json", region_payload) - atomic_write_json(model_dir / "portfolio_bound_aware_selection.json", payload) - - if gpu_sampler is not None: - resource_payload["gpu_summary"] = gpu_sampler.stop() - resource_payload["end"] = _resource_snapshot() - atomic_write_json(resource_path, resource_payload) - tracker.complete( - extra={ - "selection_reason": str(payload["selection_reason"]), - "selected_alpha01_exact_pass": bool(selected["alpha01_exact_pass"]), - "selected_realized_total_return": float(selected["realized_total_return"]), - } - ) - logger.info( - "Focused bound-aware search complete: selected risk_tolerance={}, mode={}, gamma={}, q_cap={}, q_tail={}, ab_pass_all={}, alpha01_pass={}", - selected["risk_tolerance"], - selected["policy_mode"], - selected["gamma"], - selected["delta_cap_quantile"], - selected["tail_focus_quantile"], - selected["ab_pass_all"], - selected["alpha01_exact_pass"], - ) - logger.info( - "Saved frontier raw: {}", output_dir / "portfolio_bound_aware_frontier_raw.parquet" - ) - logger.info( - "Saved frontier aggregate: {}", output_dir / "portfolio_bound_aware_frontier.parquet" - ) - logger.info("Saved shortlist: {}", output_dir / "portfolio_bound_aware_shortlist.parquet") - logger.info( - "Saved exact shortlist: {}", - output_dir / "portfolio_bound_aware_shortlist_exact.parquet", - ) - logger.info( - "Saved bound evaluations: {}", output_dir / "portfolio_bound_aware_bound_eval.parquet" - ) - logger.info( - "Saved selection payload: {}", model_dir / "portfolio_bound_aware_selection.json" - ) +def _log_completed_run( + paths: BoundAwarePaths, + *, + selected: pd.Series, +) -> None: + logger.info( + "Focused bound-aware search complete: selected risk_tolerance={}, mode={}, gamma={}, q_cap={}, q_tail={}, ab_pass_all={}, alpha01_pass={}", + selected["risk_tolerance"], + selected["policy_mode"], + selected["gamma"], + selected["delta_cap_quantile"], + selected["tail_focus_quantile"], + selected["ab_pass_all"], + selected["alpha01_exact_pass"], + ) + for label, path in ( + ("frontier raw", paths.frontier_raw_path), + ("frontier aggregate", paths.frontier_path), + ("shortlist", paths.shortlist_path), + ("exact shortlist", paths.shortlist_exact_path), + ("bound evaluations", paths.bound_eval_path), + ("selection payload", paths.selection_path), + ): + logger.info("Saved {}: {}", label, path) + + +def _execute_bound_aware_search( + context: BoundAwareRunContext, + *, + gpu_sampler: _GpuSampler | None, +) -> int: + state = _build_frontier_state(context) + if _finish_after_frontier(context, state): return 0 + selected, payload = _run_exact_selection(context, state) + _complete_run(context, selected=selected, payload=payload, gpu_sampler=gpu_sampler) + _log_completed_run(context.spec.paths, selected=selected) + return 0 + + +def _record_run_failure(context: BoundAwareRunContext, exc: Exception) -> None: + tracker = context.tracker + error_payload = { + "error_type": type(exc).__name__, + "error": str(exc), + "frontier_completed_units": int(tracker.frontier_completed_units), + "frontier_total_units": int(tracker.frontier_total_units), + "bound_completed_checks": int(tracker.bound_completed_checks), + "bound_total_checks": int(tracker.bound_total_checks), + } + context.resource_payload["error"] = error_payload + context.resource_payload["end"] = _resource_snapshot() + atomic_write_json(context.spec.paths.resource_path, context.resource_payload) + tracker.fail(phase="failed", extra=error_payload) + logger.exception("Focused bound-aware portfolio search failed.") + + +def _finalize_gpu_sampler( + context: BoundAwareRunContext, + gpu_sampler: _GpuSampler | None, +) -> None: + if gpu_sampler is None or "gpu_summary" in context.resource_payload: + return + try: + context.resource_payload["gpu_summary"] = gpu_sampler.stop() + context.resource_payload["end"] = _resource_snapshot() + atomic_write_json(context.spec.paths.resource_path, context.resource_payload) + except Exception: # pragma: no cover - best effort cleanup only + pass + + +def main(argv: list[str] | None = None) -> int: + args = _build_parser().parse_args(argv) + context = _initialize_run_context(args) + gpu_sampler = _start_gpu_sampler(context) + + try: + return _execute_bound_aware_search(context, gpu_sampler=gpu_sampler) except Exception as exc: - error_payload = { - "error_type": type(exc).__name__, - "error": str(exc), - "frontier_completed_units": int(tracker.frontier_completed_units), - "frontier_total_units": int(tracker.frontier_total_units), - "bound_completed_checks": int(tracker.bound_completed_checks), - "bound_total_checks": int(tracker.bound_total_checks), - } - resource_payload["error"] = error_payload - resource_payload["end"] = _resource_snapshot() - atomic_write_json(resource_path, resource_payload) - tracker.fail(phase="failed", extra=error_payload) - logger.exception("Focused bound-aware portfolio search failed.") + _record_run_failure(context, exc) raise finally: - if gpu_sampler is not None: - try: - if "gpu_summary" not in resource_payload: - resource_payload["gpu_summary"] = gpu_sampler.stop() - resource_payload["end"] = _resource_snapshot() - atomic_write_json(resource_path, resource_payload) - except Exception: # pragma: no cover - best effort cleanup only - pass + _finalize_gpu_sampler(context, gpu_sampler) if __name__ == "__main__": diff --git a/scripts/search/run_portfolio_bound_exact_eval.py b/scripts/search/run_portfolio_bound_exact_eval.py index dfe4608..37dc7e2 100644 --- a/scripts/search/run_portfolio_bound_exact_eval.py +++ b/scripts/search/run_portfolio_bound_exact_eval.py @@ -7,6 +7,8 @@ import os import sys import time +from collections.abc import Iterable, Iterator +from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -54,6 +56,38 @@ ) +@dataclass(frozen=True) +class ExactEvalPaths: + status_path: Path + checkpoint_dir: Path + resource_snapshot_path: Path + shortlist_path: Path + shortlist_exact_path: Path + bound_eval_path: Path + selection_path: Path + + +@dataclass(frozen=True) +class ExactEvalPlan: + alpha_grid: list[float] + exact_max_candidates: int + requested_random_states: list[int] + random_states: list[int] + full_universe_seed_deduped: bool + expected_checks: int + exact_threads: int + checkpoint_every: int + + +@dataclass(frozen=True) +class ExactEvalTask: + cache_key: tuple[int, int, float] + candidate_payload: dict[str, object] + policy: dict[str, object] + eval_random_state: int + alpha: float + + def _resolve_repo_path(raw_path: object) -> Path: path_text = str(raw_path) path = Path(path_text) @@ -80,6 +114,32 @@ def _normalize_context_paths(context: dict[str, object]) -> None: context[key] = str(_resolve_repo_path(context[key])) +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + parser.add_argument("--context-path", required=True) + return parser + + +def _load_context(context_path: Path) -> dict[str, object]: + context = json.loads(context_path.read_text(encoding="utf-8")) + if not isinstance(context, dict): + raise TypeError("Exact bound context must be a JSON object.") + _normalize_context_paths(context) + return context + + +def _exact_eval_paths(context: dict[str, object]) -> ExactEvalPaths: + return ExactEvalPaths( + status_path=Path(str(context["runtime_status_path"])), + checkpoint_dir=Path(str(context["runtime_checkpoint_dir"])), + resource_snapshot_path=Path(str(context["resource_snapshot_path"])), + shortlist_path=Path(str(context["shortlist_path"])), + shortlist_exact_path=_shortlist_exact_path(context), + bound_eval_path=Path(str(context["bound_eval_path"])), + selection_path=Path(str(context["selection_path"])), + ) + + def _eta_seconds(elapsed_sec: float, completed: int, total: int) -> float | None: if completed <= 0 or total <= 0 or completed >= total: return 0.0 if total > 0 and completed >= total else None @@ -147,25 +207,85 @@ def _load_partial_bound_eval(*, bound_eval_path: Path) -> pd.DataFrame: def _context_random_states(context: dict[str, object]) -> list[int]: raw = context.get("exact_random_states", context["random_states"]) + values = _context_value_list(raw) + return [_context_int(value) for value in values] + + +def _context_value_list(raw: object) -> list[object]: if isinstance(raw, str): - values = [part.strip() for part in raw.split(",") if part.strip()] - else: - values = list(raw) # type: ignore[arg-type] - return [int(v) for v in values] + return [part.strip() for part in raw.split(",") if part.strip()] + if isinstance(raw, Iterable): + return list(raw) + raise TypeError(f"Expected scalar list or comma-separated string, got {type(raw).__name__}") + + +def _context_int(raw: object) -> int: + if isinstance(raw, str | int | float): + return int(raw) + raise TypeError(f"Expected int-like value, got {type(raw).__name__}") def _context_max_candidates(context: dict[str, object]) -> int: - return int(context.get("exact_max_candidates", context["max_candidates"])) + return _context_int(context.get("exact_max_candidates", context["max_candidates"])) def _context_exact_threads(context: dict[str, object]) -> int: raw = os.environ.get("EXACT_THREADS", context.get("exact_threads", DEFAULT_THREADS)) try: - return max(1, int(raw)) + return max(1, _context_int(raw)) except (TypeError, ValueError): return DEFAULT_THREADS +def _requested_random_states(context: dict[str, object]) -> list[int]: + raw = context.get("requested_exact_random_states") + if raw is None: + return _context_random_states(context) + return [_context_int(value) for value in _context_value_list(raw)] + + +def _dedupe_full_universe_random_states( + *, + random_states: list[int], + exact_max_candidates: int, +) -> tuple[list[int], bool]: + if exact_max_candidates > 0 or len(random_states) <= 1: + return list(random_states), False + effective = [int(random_states[0])] + logger.info( + "Deduplicating exact_random_states for full-universe exact rerank: requested={} " + "effective={}. With exact_max_candidates<=0 there is no sampling, so seeds are " + "identical.", + random_states, + effective, + ) + return effective, True + + +def _exact_eval_plan( + *, + context: dict[str, object], + shortlist_rows: int, +) -> ExactEvalPlan: + alpha_grid = [_as_float(value) for value in _context_value_list(context["alpha_grid"])] + exact_max_candidates = _context_max_candidates(context) + requested_random_states = _requested_random_states(context) + random_states, full_universe_seed_deduped = _dedupe_full_universe_random_states( + random_states=requested_random_states, + exact_max_candidates=exact_max_candidates, + ) + return ExactEvalPlan( + alpha_grid=alpha_grid, + exact_max_candidates=exact_max_candidates, + requested_random_states=requested_random_states, + random_states=random_states, + full_universe_seed_deduped=full_universe_seed_deduped, + expected_checks=int(shortlist_rows * len(random_states) * len(alpha_grid)), + exact_threads=_context_exact_threads(context), + checkpoint_every=max(1, _context_int(context.get("exact_checkpoint_every", 100))), + ) + + def _validate_alpha_grid_supported(alpha_grid: list[float]) -> None: sweep_path = ROOT / "data" / "processed" / "alpha_sweep_pareto_mondrian.parquet" if not sweep_path.exists(): @@ -209,7 +329,7 @@ def _write_exact_status( frontier_total = _as_int(context["frontier_total_units"]) frontier_completed = _as_int(context["frontier_completed_units"]) bound_total = _as_int(context["bound_total_checks"]) - resume_completed = int(context.get("resume_completed_checks", 0) or 0) + resume_completed = _context_int(context.get("resume_completed_checks", 0) or 0) exact_elapsed_sec = time.monotonic() - started elapsed_sec = base_elapsed_sec + exact_elapsed_sec global_total = frontier_total + bound_total @@ -258,22 +378,28 @@ def _write_exact_status( return payload -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--context-path", required=True) - args = parser.parse_args(argv) +def _plan_status_extra( + plan: ExactEvalPlan, + extra: dict[str, object] | None = None, +) -> dict[str, object]: + payload: dict[str, object] = { + "checkpoint_every": int(plan.checkpoint_every), + "exact_max_candidates": int(plan.exact_max_candidates), + "exact_random_states": plan.random_states, + "requested_exact_random_states": plan.requested_random_states, + "full_universe_seed_deduped": bool(plan.full_universe_seed_deduped), + "exact_threads": int(plan.exact_threads), + } + if extra: + payload.update(extra) + return payload - context_path = Path(args.context_path).resolve() - context = json.loads(context_path.read_text(encoding="utf-8")) - _normalize_context_paths(context) - status_path = Path(str(context["runtime_status_path"])) - checkpoint_dir = Path(str(context["runtime_checkpoint_dir"])) - resource_snapshot_path = Path(str(context["resource_snapshot_path"])) - shortlist_path = Path(str(context["shortlist_path"])) - shortlist_exact_path = _shortlist_exact_path(context) - bound_eval_path = Path(str(context["bound_eval_path"])) - selection_path = Path(str(context["selection_path"])) +def _initialize_exact_progress( + *, + context: dict[str, object], + status_path: Path, +) -> float: prior_status = load_runtime_status(status_path) base_elapsed_sec = ( float(prior_status.get("elapsed_sec", 0.0)) @@ -284,167 +410,224 @@ def main(argv: list[str] | None = None) -> int: context["frontier_completed_units"] = int(prior_status.get("frontier_completed_units", 0)) context["bound_total_checks"] = int(prior_status.get("bound_total_checks", 0)) context["helper_started_monotonic"] = time.monotonic() + return base_elapsed_sec - shortlist = pd.read_parquet(shortlist_path) - alpha_grid = [float(v) for v in context["alpha_grid"]] - _validate_alpha_grid_supported(alpha_grid) - exact_max_candidates = _context_max_candidates(context) - requested_random_states_raw = context.get("requested_exact_random_states") - if requested_random_states_raw is None: - requested_random_states = _context_random_states(context) - elif isinstance(requested_random_states_raw, str): - requested_random_states = [ - int(part.strip()) for part in requested_random_states_raw.split(",") if part.strip() - ] - else: - requested_random_states = [int(value) for value in requested_random_states_raw] # type: ignore[arg-type] - random_states = list(requested_random_states) - full_universe_seed_deduped = False - if exact_max_candidates <= 0 and len(random_states) > 1: - random_states = [int(random_states[0])] - full_universe_seed_deduped = True - logger.info( - "Deduplicating exact_random_states for full-universe exact rerank: requested={} " - "effective={}. With exact_max_candidates<=0 there is no sampling, so seeds are " - "identical.", - requested_random_states, - random_states, + +def _resume_exact_rows( + *, + bound_eval_path: Path, + random_states: list[int], +) -> tuple[list[dict[str, object]], set[tuple[int, int, float]]]: + partial_bound_eval = _load_partial_bound_eval(bound_eval_path=bound_eval_path) + if not partial_bound_eval.empty: + partial_bound_eval = partial_bound_eval[ + partial_bound_eval["eval_random_state"].astype(int).isin(random_states) + ].reset_index(drop=True) + completed_keys = { + ( + int(row["candidate_rank"]), + int(row["eval_random_state"]), + float(row["alpha"]), ) - expected_checks = int(len(shortlist) * len(random_states) * len(alpha_grid)) - context["bound_total_checks"] = expected_checks - exact_threads = _context_exact_threads(context) - checkpoint_every = max(1, int(context.get("exact_checkpoint_every", 100))) + for row in partial_bound_eval.to_dict(orient="records") + } + bound_rows: list[dict[str, object]] = ( + partial_bound_eval.to_dict(orient="records") if not partial_bound_eval.empty else [] + ) + return bound_rows, completed_keys - bound_eval = _load_completed_bound_eval( - bound_eval_path=bound_eval_path, - expected_checks=expected_checks, + +def _load_aligned_datasets( + *, + context: dict[str, object], + plan: ExactEvalPlan, +) -> dict[int, pd.DataFrame]: + return { + int(seed): _load_aligned_dataset( + conformal_intervals_path=str(context["conformal_intervals_path"]), + max_candidates=int(plan.exact_max_candidates), + random_state=int(seed), + ) + for seed in plan.random_states + } + + +def _iter_pending_exact_tasks( + *, + shortlist: pd.DataFrame, + context: dict[str, object], + plan: ExactEvalPlan, + completed_keys: set[tuple[int, int, float]], +) -> Iterator[ExactEvalTask]: + for _, row in shortlist.iterrows(): + policy = _policy_from_row( + row, + solver_backend_override=str(context["exact_solver_backend"]), + ) + candidate_payload = row.to_dict() + candidate_rank = int(candidate_payload["candidate_rank"]) + for eval_seed in plan.random_states: + for alpha in plan.alpha_grid: + cache_key = (candidate_rank, int(eval_seed), float(alpha)) + if cache_key in completed_keys: + continue + yield ExactEvalTask( + cache_key=cache_key, + candidate_payload=candidate_payload, + policy=policy, + eval_random_state=int(eval_seed), + alpha=float(alpha), + ) + + +def _evaluate_exact_task( + *, + context: dict[str, object], + task: ExactEvalTask, + aligned_by_seed: dict[int, pd.DataFrame], + exact_threads: int, +) -> dict[str, object]: + result = _validate_single_alpha( + aligned_by_seed[int(task.eval_random_state)], + alpha=float(task.alpha), + policy=task.policy, + allocator_mode="exact", + budget=_as_float(context["budget"]), + t_eval=_as_float(context["t_eval"]), + threads=int(exact_threads), ) - if bound_eval is None: - partial_bound_eval = _load_partial_bound_eval(bound_eval_path=bound_eval_path) - if not partial_bound_eval.empty: - partial_bound_eval = partial_bound_eval[ - partial_bound_eval["eval_random_state"].astype(int).isin(random_states) - ].reset_index(drop=True) - completed_keys = { - (int(row.candidate_rank), int(row.eval_random_state), float(row.alpha)) - for row in partial_bound_eval.itertuples(index=False) - } - bound_rows: list[dict[str, object]] = ( - partial_bound_eval.to_dict(orient="records") if not partial_bound_eval.empty else [] + return { + "candidate_rank": _as_int(task.candidate_payload["candidate_rank"]), + "eval_random_state": int(task.eval_random_state), + "frontier_solver_backend": str(context["frontier_solver_backend"]), + "exact_solver_backend": str(context["exact_solver_backend"]), + **task.candidate_payload, + **result, + } + + +def _run_missing_exact_checks( + *, + context: dict[str, object], + paths: ExactEvalPaths, + shortlist: pd.DataFrame, + plan: ExactEvalPlan, + base_elapsed_sec: float, +) -> pd.DataFrame: + bound_rows, completed_keys = _resume_exact_rows( + bound_eval_path=paths.bound_eval_path, + random_states=plan.random_states, + ) + completed_checks = len(completed_keys) + context["resume_completed_checks"] = int(completed_checks) + if completed_checks: + _write_exact_status( + context=context, + base_elapsed_sec=base_elapsed_sec, + bound_completed_checks=completed_checks, + phase="exact_bound_running", + state="running", + extra=_plan_status_extra(plan, {"resume_cache_rows": int(completed_checks)}), ) - completed_checks = len(completed_keys) - context["resume_completed_checks"] = int(completed_checks) - if completed_checks: - _write_exact_status( + aligned_by_seed = _load_aligned_datasets(context=context, plan=plan) + + for task in _iter_pending_exact_tasks( + shortlist=shortlist, + context=context, + plan=plan, + completed_keys=completed_keys, + ): + bound_rows.append( + _evaluate_exact_task( context=context, - base_elapsed_sec=base_elapsed_sec, - bound_completed_checks=completed_checks, - phase="exact_bound_running", - state="running", - extra={ - "resume_cache_rows": int(completed_checks), - "checkpoint_every": int(checkpoint_every), - "exact_max_candidates": int(exact_max_candidates), - "exact_random_states": random_states, - "requested_exact_random_states": requested_random_states, - "full_universe_seed_deduped": bool(full_universe_seed_deduped), - "exact_threads": int(exact_threads), - }, + task=task, + aligned_by_seed=aligned_by_seed, + exact_threads=plan.exact_threads, ) - aligned_by_seed = { - int(seed): _load_aligned_dataset( - conformal_intervals_path=str(context["conformal_intervals_path"]), - max_candidates=int(exact_max_candidates), - random_state=int(seed), + ) + completed_checks += 1 + completed_keys.add(task.cache_key) + _write_exact_status( + context=context, + base_elapsed_sec=base_elapsed_sec, + bound_completed_checks=completed_checks, + phase="exact_bound_running", + state="running", + extra=_plan_status_extra( + plan, + { + "candidate_rank": _as_int(task.candidate_payload["candidate_rank"]), + "eval_random_state": int(task.eval_random_state), + "current_alpha": float(task.alpha), + }, + ), + ) + if completed_checks % plan.checkpoint_every == 0: + atomic_write_parquet( + pd.DataFrame(bound_rows), + paths.bound_eval_path, + index=False, ) - for seed in random_states - } + return pd.DataFrame(bound_rows) - for _, row in shortlist.iterrows(): - policy = _policy_from_row( - row, - solver_backend_override=str(context["exact_solver_backend"]), - ) - candidate_payload = row.to_dict() - for eval_seed in random_states: - aligned = aligned_by_seed[int(eval_seed)] - for alpha in alpha_grid: - cache_key = ( - int(candidate_payload["candidate_rank"]), - int(eval_seed), - float(alpha), - ) - if cache_key in completed_keys: - continue - result = _validate_single_alpha( - aligned, - alpha=float(alpha), - policy=policy, - allocator_mode="exact", - budget=float(context["budget"]), - t_eval=float(context["t_eval"]), - threads=int(exact_threads), - ) - bound_rows.append( - { - "candidate_rank": int(candidate_payload["candidate_rank"]), - "eval_random_state": int(eval_seed), - "frontier_solver_backend": str(context["frontier_solver_backend"]), - "exact_solver_backend": str(context["exact_solver_backend"]), - **candidate_payload, - **result, - } - ) - completed_checks += 1 - completed_keys.add(cache_key) - _write_exact_status( - context=context, - base_elapsed_sec=base_elapsed_sec, - bound_completed_checks=completed_checks, - phase="exact_bound_running", - state="running", - extra={ - "candidate_rank": int(candidate_payload["candidate_rank"]), - "eval_random_state": int(eval_seed), - "current_alpha": float(alpha), - "checkpoint_every": int(checkpoint_every), - "exact_max_candidates": int(exact_max_candidates), - "exact_random_states": random_states, - "requested_exact_random_states": requested_random_states, - "full_universe_seed_deduped": bool(full_universe_seed_deduped), - "exact_threads": int(exact_threads), - }, - ) - if completed_checks % checkpoint_every == 0: - atomic_write_parquet( - pd.DataFrame(bound_rows), - bound_eval_path, - index=False, - ) - - bound_eval = pd.DataFrame(bound_rows) - atomic_write_parquet(bound_eval, bound_eval_path, index=False) - shortlist_eval = _aggregate_exact_results(shortlist=shortlist, bound_eval=bound_eval) - region_payload = _region_summary(shortlist_eval, bound_eval) - selected = shortlist_eval.iloc[0].copy() - selected_policy = _policy_from_row( - selected, - solver_backend_override=str(context["exact_solver_backend"]), + +def _load_or_run_bound_eval( + *, + context: dict[str, object], + paths: ExactEvalPaths, + shortlist: pd.DataFrame, + plan: ExactEvalPlan, + base_elapsed_sec: float, +) -> pd.DataFrame: + bound_eval = _load_completed_bound_eval( + bound_eval_path=paths.bound_eval_path, + expected_checks=plan.expected_checks, ) + if bound_eval is not None: + return bound_eval + bound_eval = _run_missing_exact_checks( + context=context, + paths=paths, + shortlist=shortlist, + plan=plan, + base_elapsed_sec=base_elapsed_sec, + ) + atomic_write_parquet(bound_eval, paths.bound_eval_path, index=False) + return bound_eval - search_space_payload = dict(context["search_space"]) # type: ignore[arg-type] + +def _search_space_payload( + *, + context: dict[str, object], + alpha_grid: list[float], +) -> dict[str, object]: + search_space = context["search_space"] + if not isinstance(search_space, dict): + raise TypeError("context['search_space'] must be a mapping") + search_space_payload = dict(search_space) requested_search_alpha_grid = search_space_payload.get("alpha_grid") search_space_payload["alpha_grid"] = alpha_grid if requested_search_alpha_grid != alpha_grid: search_space_payload["requested_alpha_grid"] = requested_search_alpha_grid search_space_payload["effective_alpha_grid"] = alpha_grid + return search_space_payload - payload = { + +def _build_selection_payload( + *, + context: dict[str, object], + paths: ExactEvalPaths, + plan: ExactEvalPlan, + selected: pd.Series, + selected_policy: dict[str, object], + region_payload: dict[str, object], +) -> dict[str, object]: + return { "schema_version": SCHEMA_VERSION, "generated_at_utc": datetime.now(tz=UTC).isoformat(), "run_label": str(context["run_label"]), "conformal_intervals_path": _repo_relative(str(context["conformal_intervals_path"])), - "search_space": search_space_payload, + "search_space": _search_space_payload(context=context, alpha_grid=plan.alpha_grid), "selection_policy": context["selection_policy"], "selected_policy": selected_policy, "selected_metrics": selected.to_dict(), @@ -452,7 +635,7 @@ def main(argv: list[str] | None = None) -> int: "frontier_raw_path": _repo_relative(str(context["frontier_raw_path"])), "frontier_path": _repo_relative(str(context["frontier_path"])), "shortlist_path": _repo_relative(str(context["shortlist_path"])), - "shortlist_exact_path": _repo_relative(shortlist_exact_path), + "shortlist_exact_path": _repo_relative(paths.shortlist_exact_path), "bound_eval_path": _repo_relative(str(context["bound_eval_path"])), "region_summary_path": _repo_relative(str(context["region_summary_path"])), "robust_region_summary": region_payload, @@ -461,29 +644,49 @@ def main(argv: list[str] | None = None) -> int: "resource_snapshot_path": _repo_relative(str(context["resource_snapshot_path"])), "frontier_solver_backend": str(context["frontier_solver_backend"]), "exact_solver_backend": str(context["exact_solver_backend"]), - "exact_threads": int(exact_threads), - "requested_exact_random_states": requested_random_states, - "effective_exact_random_states": random_states, - "full_universe_seed_deduped": bool(full_universe_seed_deduped), + "exact_threads": int(plan.exact_threads), + "requested_exact_random_states": plan.requested_random_states, + "effective_exact_random_states": plan.random_states, + "full_universe_seed_deduped": bool(plan.full_universe_seed_deduped), } - atomic_write_parquet(shortlist_eval, shortlist_exact_path, index=False) + +def _write_selection_artifacts( + *, + context: dict[str, object], + paths: ExactEvalPaths, + shortlist_eval: pd.DataFrame, + region_payload: dict[str, object], + selection_payload: dict[str, object], +) -> None: + atomic_write_parquet(shortlist_eval, paths.shortlist_exact_path, index=False) atomic_write_json(Path(str(context["region_summary_path"])), region_payload) - atomic_write_json(selection_path, payload) + atomic_write_json(paths.selection_path, selection_payload) - resource_payload = json.loads(resource_snapshot_path.read_text(encoding="utf-8")) + +def _update_resource_snapshot(path: Path) -> None: + resource_payload = json.loads(path.read_text(encoding="utf-8")) resource_payload["exact_helper_python"] = _repo_relative(sys.executable) resource_payload["exact_helper_end"] = _resource_snapshot() - atomic_write_json(resource_snapshot_path, resource_payload) + atomic_write_json(path, resource_payload) + +def _write_final_status( + *, + context: dict[str, object], + paths: ExactEvalPaths, + base_elapsed_sec: float, + selection_payload: dict[str, object], + selected: pd.Series, +) -> None: final_payload = _write_exact_status( context=context, base_elapsed_sec=base_elapsed_sec, - bound_completed_checks=int(context["bound_total_checks"]), + bound_completed_checks=_as_int(context["bound_total_checks"]), phase="selection_complete", state="completed", extra={ - "selection_reason": str(payload["selection_reason"]), + "selection_reason": str(selection_payload["selection_reason"]), "selected_alpha01_exact_pass": bool(selected["alpha01_exact_pass"]), "selected_realized_total_return": float(selected["realized_total_return"]), }, @@ -492,7 +695,60 @@ def main(argv: list[str] | None = None) -> int: STAGE_NAME, "003_selection_complete", final_payload, - checkpoint_dir=checkpoint_dir, + checkpoint_dir=paths.checkpoint_dir, + ) + + +def main(argv: list[str] | None = None) -> int: + parser = _build_parser() + args = parser.parse_args(argv) + + context = _load_context(Path(args.context_path).resolve()) + paths = _exact_eval_paths(context) + base_elapsed_sec = _initialize_exact_progress(context=context, status_path=paths.status_path) + + shortlist = pd.read_parquet(paths.shortlist_path) + plan = _exact_eval_plan(context=context, shortlist_rows=len(shortlist)) + _validate_alpha_grid_supported(plan.alpha_grid) + context["bound_total_checks"] = plan.expected_checks + + bound_eval = _load_or_run_bound_eval( + context=context, + paths=paths, + shortlist=shortlist, + plan=plan, + base_elapsed_sec=base_elapsed_sec, + ) + shortlist_eval = _aggregate_exact_results(shortlist=shortlist, bound_eval=bound_eval) + region_payload = _region_summary(shortlist_eval, bound_eval) + selected = shortlist_eval.iloc[0].copy() + selected_policy = _policy_from_row( + selected, + solver_backend_override=str(context["exact_solver_backend"]), + ) + + selection_payload = _build_selection_payload( + context=context, + paths=paths, + plan=plan, + selected=selected, + selected_policy=selected_policy, + region_payload=region_payload, + ) + _write_selection_artifacts( + context=context, + paths=paths, + shortlist_eval=shortlist_eval, + region_payload=region_payload, + selection_payload=selection_payload, + ) + _update_resource_snapshot(paths.resource_snapshot_path) + _write_final_status( + context=context, + paths=paths, + base_elapsed_sec=base_elapsed_sec, + selection_payload=selection_payload, + selected=selected, ) logger.info( diff --git a/scripts/search/run_portfolio_search.py b/scripts/search/run_portfolio_search.py index 5567f4b..7982599 100644 --- a/scripts/search/run_portfolio_search.py +++ b/scripts/search/run_portfolio_search.py @@ -1,20 +1,24 @@ -"""Organized search entrypoint for portfolio selector/tradeoff runs.""" +"""Retired portfolio search entrypoint. + +The former generic ``scripts.run_long_pipeline`` orchestrator was removed when +the IJDS paper lane was narrowed around frozen artifacts. Keep this file as a +readable stop sign for old commands instead of failing with an import error. +""" from __future__ import annotations import sys -from scripts.run_long_pipeline import main as _main - def main(argv: list[str] | None = None) -> int: - return _main( - argv, - default_pipeline_family="search_portfolio", - default_sampling_profile="champion64safe", - default_include_rapids=False, - default_include_notebooks=False, + _ = argv + sys.stderr.write( + "scripts/search/run_portfolio_search.py is retired. The submitted IJDS " + "claim uses the closed finite-grid frontier; use " + "scripts/search/run_pool93_ijds_local_refinement.py only for an explicitly " + "tagged isolated refinement, not as a default paper path.\n" ) + return 2 if __name__ == "__main__": diff --git a/scripts/search/run_regret_auditability_sandbox.py b/scripts/search/run_regret_auditability_sandbox.py index d7003a4..9e02ed7 100644 --- a/scripts/search/run_regret_auditability_sandbox.py +++ b/scripts/search/run_regret_auditability_sandbox.py @@ -24,7 +24,7 @@ from dataclasses import asdict, dataclass from datetime import UTC, datetime from pathlib import Path -from typing import Any +from typing import Any, cast import yaml @@ -32,6 +32,9 @@ sys.path.insert(0, str(ROOT)) from src.features.feature_config_io import load_feature_config # noqa: E402 +from src.optimization.certificate_semantics import ( # noqa: E402 + IJDS_DECLARED_ALPHA_GRID_CSV, +) from src.utils.pipeline_runtime import atomic_write_json # noqa: E402 @@ -83,6 +86,10 @@ def _env_str(name: str, default: str) -> str: "0.60,0.75,0.90,1.0", ) PORTFOLIO_RANDOM_STATES = _env_str("CRPTO_SANDBOX_PORTFOLIO_RANDOM_STATES", "42,52,62") +PORTFOLIO_ALPHA_GRID = _env_str( + "CRPTO_SANDBOX_PORTFOLIO_ALPHA_GRID", + IJDS_DECLARED_ALPHA_GRID_CSV, +) PORTFOLIO_MAX_CANDIDATES = _env_int("CRPTO_SANDBOX_PORTFOLIO_MAX_CANDIDATES", 100000) PORTFOLIO_SHORTLIST_TOP_K = _env_int("CRPTO_SANDBOX_PORTFOLIO_SHORTLIST_TOP_K", 1000) @@ -736,45 +743,37 @@ def _pd_warm_start_candidates( for index, row in enumerate(selected[:5], start=1): if not isinstance(row, Mapping): continue + best_params = row.get("best_params") _append_warm_start_candidate( rows, source=f"{previous_phase}:top_{index}:{row.get('lane_id', 'unknown')}", - params=row.get("best_params"), + params=cast(Mapping[str, Any], best_params) + if isinstance(best_params, Mapping) + else None, include_iterations=include_iterations, ) return rows -def write_pd_config_snapshot( +def _apply_pd_feature_profile_config( *, - artifact_root: Path, - run_tag: str, - feature_profile_name: str, - policy_name: str, - phase: str, - n_trials: int, - cpu_threads: int, - base_params_override: Mapping[str, Any] | None = None, -) -> Path: - """Write an external PD config snapshot for one feature/policy lane and phase.""" - config = _load_yaml(CHAMPION_PD_CONFIG_PATH) - feature_profile_path = _write_feature_profile_snapshot( - artifact_root=artifact_root, - profile_name=feature_profile_name, - ) - policy = _monotonic_policy_for_feature_profile( - policy_name=policy_name, - feature_profile_name=feature_profile_name, - feature_profile_path=feature_profile_path, - ) - lane = _lane_id(feature_profile_name, policy_name) - phase_root = artifact_root / "pd" / feature_profile_name / policy_name / phase - profile = FEATURE_PROFILES[feature_profile_name] - + config: dict[str, Any], + feature_profile_path: Path, + profile: Mapping[str, Any], +) -> None: config["feature_source"] = dict(config.get("feature_source", {}) or {}) config["feature_source"]["feature_config_path"] = str(feature_profile_path) config["stable_core"] = dict(config.get("stable_core", {}) or {}) config["stable_core"]["enabled"] = bool(profile.get("stable_core_enabled", False)) + + +def _apply_pd_model_params( + *, + config: dict[str, Any], + policy: Mapping[str, int], + cpu_threads: int, + base_params_override: Mapping[str, Any] | None, +) -> None: config["model"] = dict(config.get("model", {}) or {}) config["model"]["params"] = dict(config["model"].get("params", {}) or {}) if base_params_override: @@ -797,9 +796,24 @@ def write_pd_config_snapshot( "monotone_constraints": _format_monotone_constraints(policy), } ) + + +def _apply_venn_abers_calibration(config: dict[str, Any]) -> None: config["calibration"] = dict(config.get("calibration", {}) or {}) config["calibration"]["method"] = "venn_abers" config["calibration"]["candidates"] = ["venn_abers"] + + +def _apply_pd_hpo_config( + *, + config: dict[str, Any], + artifact_root: Path, + phase_root: Path, + run_tag: str, + phase: str, + lane: str, + n_trials: int, +) -> list[dict[str, Any]]: config["hpo"] = dict(config.get("hpo", {}) or {}) config["hpo"].update( { @@ -847,27 +861,34 @@ def write_pd_config_snapshot( "bootstrap_type": ["MVS", "Bernoulli"], "grow_policy": ["SymmetricTree"], } - config["validation"] = dict(config.get("validation", {}) or {}) + return warm_start + + +def _pd_validation_policy(phase: str) -> tuple[dict[str, Any], bool]: if phase == "pd-smoke": - seed_replay = {"enabled": True, "top_k_trials": 1, "seeds": [42]} - walk_forward_enabled = False - elif phase == "pd-broad": - seed_replay = {"enabled": True, "top_k_trials": 10, "seeds": [42, 52, 62]} - walk_forward_enabled = True - else: - seed_replay = { - "enabled": True, - "top_k_trials": 30, - "seeds": [42, 52, 62, 72, 82], - } - walk_forward_enabled = True + return {"enabled": True, "top_k_trials": 1, "seeds": [42]}, False + if phase == "pd-broad": + return {"enabled": True, "top_k_trials": 10, "seeds": [42, 52, 62]}, True + return { + "enabled": True, + "top_k_trials": 30, + "seeds": [42, 52, 62, 72, 82], + }, True + + +def _apply_pd_validation_config(*, config: dict[str, Any], phase: str) -> None: + seed_replay, walk_forward_enabled = _pd_validation_policy(phase) + config["validation"] = dict(config.get("validation", {}) or {}) config["validation"]["seed_replay"] = { **seed_replay, "prioritize_gate_pass": True, } config["validation"]["walk_forward"] = dict(config["validation"].get("walk_forward", {}) or {}) config["validation"]["walk_forward"]["enabled"] = walk_forward_enabled - config["output"] = { + + +def _pd_output_paths(phase_root: Path) -> dict[str, Any]: + return { "model_path": str(phase_root / "models" / "pd_model.cbm"), "default_model_path": str(phase_root / "models" / "pd_default.cbm"), "tuned_model_path": str(phase_root / "models" / "pd_tuned.cbm"), @@ -887,6 +908,9 @@ def write_pd_config_snapshot( "shap_dir": str(phase_root / "reports" / "shap"), "write_legacy_model_copy": False, } + + +def _apply_pd_decision_threshold_config(*, config: dict[str, Any], phase_root: Path) -> None: config["decision_threshold"] = dict(config.get("decision_threshold", {}) or {}) config["decision_threshold"]["enabled"] = False config["decision_threshold"]["fairness_policy_path"] = "" @@ -896,6 +920,20 @@ def write_pd_config_snapshot( config["decision_threshold"]["output_path_v2"] = str( phase_root / "models" / "decision_threshold_v2.json" ) + + +def _apply_pd_sandbox_metadata( + *, + config: dict[str, Any], + run_tag: str, + phase: str, + feature_profile_name: str, + policy_name: str, + lane: str, + policy: Mapping[str, int], + base_params_override: Mapping[str, Any] | None, + warm_start: list[dict[str, Any]], +) -> None: config["sandbox_search"] = { "run_tag": run_tag, "phase": phase, @@ -913,6 +951,69 @@ def write_pd_config_snapshot( "skip_diagnostic_exports": True, "skip_shap_export": True, } + + +def write_pd_config_snapshot( + *, + artifact_root: Path, + run_tag: str, + feature_profile_name: str, + policy_name: str, + phase: str, + n_trials: int, + cpu_threads: int, + base_params_override: Mapping[str, Any] | None = None, +) -> Path: + """Write an external PD config snapshot for one feature/policy lane and phase.""" + config = _load_yaml(CHAMPION_PD_CONFIG_PATH) + feature_profile_path = _write_feature_profile_snapshot( + artifact_root=artifact_root, + profile_name=feature_profile_name, + ) + policy = _monotonic_policy_for_feature_profile( + policy_name=policy_name, + feature_profile_name=feature_profile_name, + feature_profile_path=feature_profile_path, + ) + lane = _lane_id(feature_profile_name, policy_name) + phase_root = artifact_root / "pd" / feature_profile_name / policy_name / phase + profile = FEATURE_PROFILES[feature_profile_name] + + _apply_pd_feature_profile_config( + config=config, + feature_profile_path=feature_profile_path, + profile=profile, + ) + _apply_pd_model_params( + config=config, + policy=policy, + cpu_threads=cpu_threads, + base_params_override=base_params_override, + ) + _apply_venn_abers_calibration(config) + warm_start = _apply_pd_hpo_config( + config=config, + artifact_root=artifact_root, + phase_root=phase_root, + run_tag=run_tag, + phase=phase, + lane=lane, + n_trials=n_trials, + ) + _apply_pd_validation_config(config=config, phase=phase) + config["output"] = _pd_output_paths(phase_root) + _apply_pd_decision_threshold_config(config=config, phase_root=phase_root) + _apply_pd_sandbox_metadata( + config=config, + run_tag=run_tag, + phase=phase, + feature_profile_name=feature_profile_name, + policy_name=policy_name, + lane=lane, + policy=policy, + base_params_override=base_params_override, + warm_start=warm_start, + ) target = artifact_root / "configs" / f"pd_{lane}_{phase}.yaml" assert_safe_output_paths( value for value in config["output"].values() if isinstance(value, (str, Path)) @@ -1360,19 +1461,8 @@ def _resolve_pd_candidate_for_conformal(artifact_root: Path, artifact_name: str) return canonical[artifact_name] -def build_phase_commands( - *, - artifact_root: Path, - run_tag: str, - phase: str, - max_workers: int, - cpu_threads: int, -) -> list[PhaseCommand]: - """Build commands for the requested sandbox phase.""" - safe_tag = sanitize_tag(run_tag) - artifact_root = artifact_root.resolve() - commands: list[PhaseCommand] = [] - base_env = { +def _sandbox_base_env(artifact_root: Path, safe_tag: str) -> dict[str, str]: + return { "CRPTO_RUN_TAG": safe_tag, "PIPELINE_RUN_TAG": safe_tag, "RUN_TAG": safe_tag, @@ -1380,314 +1470,409 @@ def build_phase_commands( "GPU_REPLAY_ARTIFACT_ROOT": str(artifact_root), "CRPTO_SANDBOX_ARTIFACT_ROOT": str(artifact_root), } - selected_phases = ( - ["pd-smoke", "pd-broad", "pd-refine", "conformal", "portfolio", "metrics"] - if phase == "all" - else [phase] + + +def _phase_resources( + phase: str, + *, + max_workers: int, + cpu_threads: int, +) -> tuple[int, int]: + workers = int(max_workers) if int(max_workers) > 0 else _default_workers_for_phase(phase) + threads = int(cpu_threads) if int(cpu_threads) > 0 else _default_threads_for_phase(phase) + return workers, threads + + +def _pd_incumbent_command( + *, + artifact_root: Path, + safe_tag: str, + phase: str, + phase_workers: int, + phase_threads: int, + base_env: Mapping[str, str], +) -> PhaseCommand: + config_path = write_pd_incumbent_config_snapshot( + artifact_root=artifact_root, + run_tag=safe_tag, + phase=phase, + cpu_threads=phase_threads, ) - for selected_phase in selected_phases: - phase_workers = ( - int(max_workers) if int(max_workers) > 0 else _default_workers_for_phase(selected_phase) + output_root = artifact_root / "pd_baselines" / "champion" / phase + outputs = [ + output_root / "models" / "pd_model.cbm", + output_root / "models" / "pd_training_status.json", + output_root / "models" / "pd_training_record.pkl", + ] + command_name = f"{phase}_incumbent__frozen_champion" + stdout_log, stderr_log = _command_log_files(artifact_root, phase, command_name) + return PhaseCommand( + name=command_name, + phase=phase, + command=[ + sys.executable, + str(ROOT / "scripts" / "train_pd_model.py"), + "--config", + str(config_path), + "--hpo_enabled", + "false", + "--hpo_n_trials", + "0", + "--walk_forward_enabled", + "true" if phase != "pd-smoke" else "false", + "--seed_replay_enabled", + "false", + ], + outputs=[str(path) for path in outputs], + checkpoint=str(output_root / "models" / "pd_training_checkpoints"), + env=_command_env(base_env, phase_threads=phase_threads), + max_workers=phase_workers, + cpu_threads=phase_threads, + feature_profile="incumbent", + monotonic_policy="canonical_4", + lane_id="incumbent__frozen_champion", + stdout_log=str(stdout_log), + stderr_log=str(stderr_log), + ) + + +def _pd_lane_command( + *, + artifact_root: Path, + safe_tag: str, + phase: str, + phase_workers: int, + phase_threads: int, + base_env: Mapping[str, str], + feature_profile_name: str, + policy_name: str, + n_trials: int, +) -> PhaseCommand: + lane = _lane_id(feature_profile_name, policy_name) + base_params = _previous_best_params_for_lane( + artifact_root=artifact_root, + phase=phase, + lane_id=lane, + ) + config_path = write_pd_config_snapshot( + artifact_root=artifact_root, + run_tag=safe_tag, + feature_profile_name=feature_profile_name, + policy_name=policy_name, + phase=phase, + n_trials=n_trials, + cpu_threads=phase_threads, + base_params_override=base_params, + ) + output_root = artifact_root / "pd" / feature_profile_name / policy_name / phase + remaining_trials = _pd_remaining_trials( + phase_root=output_root, + run_tag=safe_tag, + lane_id=lane, + phase=phase, + target_trials=n_trials, + ) + outputs = [ + output_root / "models" / "pd_model.cbm", + output_root / "models" / "pd_training_status.json", + output_root / "models" / "pd_hpo_seed_replay_status.json", + ] + command_name = f"{phase}_{lane}" + stdout_log, stderr_log = _command_log_files(artifact_root, phase, command_name) + return PhaseCommand( + name=command_name, + phase=phase, + command=[ + sys.executable, + str(ROOT / "scripts" / "train_pd_model.py"), + "--config", + str(config_path), + "--hpo_enabled", + "true", + "--hpo_n_trials", + str(remaining_trials), + "--walk_forward_enabled", + "true" if phase != "pd-smoke" else "false", + "--seed_replay_enabled", + "true", + ], + outputs=[str(path) for path in outputs], + checkpoint=str(output_root / "models" / "pd_training_checkpoints"), + env=_command_env(base_env, phase_threads=phase_threads), + max_workers=phase_workers, + cpu_threads=phase_threads, + feature_profile=feature_profile_name, + monotonic_policy=policy_name, + lane_id=lane, + stdout_log=str(stdout_log), + stderr_log=str(stderr_log), + ) + + +def _build_pd_phase_commands( + *, + artifact_root: Path, + safe_tag: str, + phase: str, + phase_workers: int, + phase_threads: int, + base_env: Mapping[str, str], +) -> list[PhaseCommand]: + commands = [ + _pd_incumbent_command( + artifact_root=artifact_root, + safe_tag=safe_tag, + phase=phase, + phase_workers=phase_workers, + phase_threads=phase_threads, + base_env=base_env, ) - phase_threads = ( - int(cpu_threads) if int(cpu_threads) > 0 else _default_threads_for_phase(selected_phase) + ] + n_trials = _pd_trials_for_phase(phase) + commands.extend( + _pd_lane_command( + artifact_root=artifact_root, + safe_tag=safe_tag, + phase=phase, + phase_workers=phase_workers, + phase_threads=phase_threads, + base_env=base_env, + feature_profile_name=feature_profile_name, + policy_name=policy_name, + n_trials=n_trials, ) - if selected_phase in PD_PHASES: - incumbent_config_path = write_pd_incumbent_config_snapshot( + for feature_profile_name, policy_name in _selected_pd_lanes(artifact_root, phase) + ) + return commands + + +def _build_conformal_phase_command( + *, + artifact_root: Path, + safe_tag: str, + phase_workers: int, + phase_threads: int, + base_env: Mapping[str, str], +) -> PhaseCommand: + phase = "conformal" + conformal_root = artifact_root / phase / safe_tag + pd_model_path = _resolve_pd_candidate_for_conformal( + artifact_root, + "pd_shadow_canonical.cbm", + ) + pd_calibrator_path = _resolve_pd_candidate_for_conformal( + artifact_root, + "pd_shadow_calibrator.pkl", + ) + outputs = [ + conformal_root / "data" / "conformal_intervals_mondrian.parquet", + conformal_root / "models" / "conformal_results_mondrian.pkl", + conformal_root / "models" / "pd_conformal_width_attribution_status.json", + ] + command_name = "conformal_extensive_grid" + stdout_log, stderr_log = _command_log_files(artifact_root, phase, command_name) + return PhaseCommand( + name=command_name, + phase=phase, + command=[ + sys.executable, + str(ROOT / "scripts" / "generate_conformal_intervals.py"), + "--artifact_namespace", + safe_tag, + "--artifact_root", + str(artifact_root / phase), + "--model_override_path", + str(pd_model_path), + "--alpha_candidates_90", + "0.05,0.075,0.09,0.095,0.10,0.105,0.11,0.125,0.15,0.20", + "--alpha_candidates_95", + "0.025,0.04,0.045,0.05,0.055,0.06,0.075", + "--partition_candidates", + "grade,score_decile_mondrian,grade_x_scoreband_mondrian", + "--n_score_bins_candidates", + "5,10,15,20,30", + "--min_group_sizes", + "100,150,250,500,1000,2000", + "--score_scale_families", + "none,bernoulli_sqrt,bernoulli_sqrt_clipped_0.02,bernoulli_sqrt_clipped_0.05", + "--calibrator_override_path", + str(pd_calibrator_path), + ], + outputs=[str(path) for path in outputs], + checkpoint=str(conformal_root / "checkpoints"), + env=_command_env(base_env, phase_threads=phase_threads), + max_workers=phase_workers, + cpu_threads=phase_threads, + stdout_log=str(stdout_log), + stderr_log=str(stderr_log), + ) + + +def _build_portfolio_phase_command( + *, + artifact_root: Path, + safe_tag: str, + phase_workers: int, + phase_threads: int, + base_env: Mapping[str, str], +) -> PhaseCommand: + phase = "portfolio" + portfolio_root = artifact_root / phase / safe_tag + conformal_path = ( + artifact_root / "conformal" / safe_tag / "data" / "conformal_intervals_mondrian.parquet" + ) + outputs = [ + portfolio_root / "data" / "portfolio_bound_aware_frontier.parquet", + portfolio_root / "data" / "portfolio_bound_aware_bound_eval.parquet", + portfolio_root / "models" / "portfolio_bound_aware_selection.json", + ] + command_name = "portfolio_extensive_frontier" + stdout_log, stderr_log = _command_log_files(artifact_root, phase, command_name) + return PhaseCommand( + name=command_name, + phase=phase, + command=[ + sys.executable, + str(ROOT / "scripts" / "search" / "run_portfolio_bound_aware_search.py"), + "--config", + str(ROOT / "configs" / "crpto_optimization.yaml"), + "--conformal-intervals-path", + str(conformal_path), + "--run-label", + safe_tag, + "--output-dir", + str(portfolio_root / "data"), + "--model-dir", + str(portfolio_root / "models"), + "--incumbent-policy-path", + str(CHAMPION_PORTFOLIO_POLICY_PATH), + "--incumbent-risk-neighbors", + "0.155,0.16,0.165,0.17,0.175,0.18", + "--incumbent-gamma-neighbors", + "0.425,0.45,0.475,0.50,0.525,0.55,0.575", + "--incumbent-policy-modes", + "blended_uncertainty,capped_blended_uncertainty,tail_blended_uncertainty,segment_tail_blended_uncertainty,segment_relative_tail_blended_uncertainty", + "--risk-grid", + PORTFOLIO_RISK_GRID, + "--gamma-grid", + PORTFOLIO_GAMMA_GRID, + "--aversion-grid", + PORTFOLIO_AVERSION_GRID, + "--delta-cap-grid", + PORTFOLIO_CAP_TAIL_GRID, + "--tail-focus-grid", + PORTFOLIO_CAP_TAIL_GRID, + "--policy-modes", + "blended_uncertainty,capped_blended_uncertainty,tail_blended_uncertainty,segment_tail_blended_uncertainty,segment_relative_tail_blended_uncertainty", + "--alpha-grid", + PORTFOLIO_ALPHA_GRID, + "--max-candidates", + str(PORTFOLIO_MAX_CANDIDATES), + "--shortlist-top-k", + str(PORTFOLIO_SHORTLIST_TOP_K), + "--random-states", + PORTFOLIO_RANDOM_STATES, + "--solver-backend", + "highs", + "--exact-solver-backend", + "highs", + ], + outputs=[str(path) for path in outputs], + checkpoint=str(portfolio_root / "models" / "portfolio_bound_aware_runtime_checkpoints"), + env=_command_env(base_env, phase_threads=phase_threads), + max_workers=phase_workers, + cpu_threads=phase_threads, + stdout_log=str(stdout_log), + stderr_log=str(stderr_log), + ) + + +def _build_metrics_phase_command( + *, + artifact_root: Path, + safe_tag: str, + base_env: Mapping[str, str], +) -> PhaseCommand: + phase = "metrics" + metrics_path = artifact_root / phase / "frontier_metrics_manifest.json" + command_name = "metrics_manifest" + stdout_log, stderr_log = _command_log_files(artifact_root, phase, command_name) + return PhaseCommand( + name=command_name, + phase=phase, + command=[ + sys.executable, + str(Path(__file__).resolve()), + "--run-tag", + safe_tag, + "--artifact-root", + str(artifact_root), + "--phase", + "plan", + "--resume", + ], + outputs=[str(metrics_path)], + checkpoint=str(artifact_root / phase), + env=_command_env(base_env, phase_threads=1), + max_workers=1, + cpu_threads=1, + stdout_log=str(stdout_log), + stderr_log=str(stderr_log), + ) + + +def _commands_for_phase( + *, + artifact_root: Path, + safe_tag: str, + phase: str, + phase_workers: int, + phase_threads: int, + base_env: Mapping[str, str], +) -> list[PhaseCommand]: + if phase in PD_PHASES: + return _build_pd_phase_commands( + artifact_root=artifact_root, + safe_tag=safe_tag, + phase=phase, + phase_workers=phase_workers, + phase_threads=phase_threads, + base_env=base_env, + ) + if phase == "conformal": + return [ + _build_conformal_phase_command( artifact_root=artifact_root, - run_tag=safe_tag, - phase=selected_phase, - cpu_threads=phase_threads, - ) - incumbent_root = artifact_root / "pd_baselines" / "champion" / selected_phase - incumbent_outputs = [ - incumbent_root / "models" / "pd_model.cbm", - incumbent_root / "models" / "pd_training_status.json", - incumbent_root / "models" / "pd_training_record.pkl", - ] - incumbent_command_name = f"{selected_phase}_incumbent__frozen_champion" - incumbent_stdout_log, incumbent_stderr_log = _command_log_files( - artifact_root, - selected_phase, - incumbent_command_name, - ) - commands.append( - PhaseCommand( - name=incumbent_command_name, - phase=selected_phase, - command=[ - sys.executable, - str(ROOT / "scripts" / "train_pd_model.py"), - "--config", - str(incumbent_config_path), - "--hpo_enabled", - "false", - "--hpo_n_trials", - "0", - "--walk_forward_enabled", - "true" if selected_phase != "pd-smoke" else "false", - "--seed_replay_enabled", - "false", - ], - outputs=[str(path) for path in incumbent_outputs], - checkpoint=str(incumbent_root / "models" / "pd_training_checkpoints"), - env=_command_env(base_env, phase_threads=phase_threads), - max_workers=phase_workers, - cpu_threads=phase_threads, - feature_profile="incumbent", - monotonic_policy="canonical_4", - lane_id="incumbent__frozen_champion", - stdout_log=str(incumbent_stdout_log), - stderr_log=str(incumbent_stderr_log), - ) - ) - n_trials = _pd_trials_for_phase(selected_phase) - for feature_profile_name, policy_name in _selected_pd_lanes( - artifact_root, - selected_phase, - ): - lane = _lane_id(feature_profile_name, policy_name) - base_params = _previous_best_params_for_lane( - artifact_root=artifact_root, - phase=selected_phase, - lane_id=lane, - ) - config_path = write_pd_config_snapshot( - artifact_root=artifact_root, - run_tag=safe_tag, - feature_profile_name=feature_profile_name, - policy_name=policy_name, - phase=selected_phase, - n_trials=n_trials, - cpu_threads=phase_threads, - base_params_override=base_params, - ) - output_root = ( - artifact_root / "pd" / feature_profile_name / policy_name / selected_phase - ) - remaining_trials = _pd_remaining_trials( - phase_root=output_root, - run_tag=safe_tag, - lane_id=lane, - phase=selected_phase, - target_trials=n_trials, - ) - outputs = [ - output_root / "models" / "pd_model.cbm", - output_root / "models" / "pd_training_status.json", - output_root / "models" / "pd_hpo_seed_replay_status.json", - ] - command_name = f"{selected_phase}_{lane}" - stdout_log, stderr_log = _command_log_files( - artifact_root, - selected_phase, - command_name, - ) - commands.append( - PhaseCommand( - name=command_name, - phase=selected_phase, - command=[ - sys.executable, - str(ROOT / "scripts" / "train_pd_model.py"), - "--config", - str(config_path), - "--hpo_enabled", - "true", - "--hpo_n_trials", - str(remaining_trials), - "--walk_forward_enabled", - "true" if selected_phase != "pd-smoke" else "false", - "--seed_replay_enabled", - "true", - ], - outputs=[str(path) for path in outputs], - checkpoint=str(output_root / "models" / "pd_training_checkpoints"), - env=_command_env(base_env, phase_threads=phase_threads), - max_workers=phase_workers, - cpu_threads=phase_threads, - feature_profile=feature_profile_name, - monotonic_policy=policy_name, - lane_id=lane, - stdout_log=str(stdout_log), - stderr_log=str(stderr_log), - ) - ) - elif selected_phase == "conformal": - conformal_root = artifact_root / "conformal" / safe_tag - pd_model_path = _resolve_pd_candidate_for_conformal( - artifact_root, - "pd_shadow_canonical.cbm", - ) - pd_calibrator_path = _resolve_pd_candidate_for_conformal( - artifact_root, - "pd_shadow_calibrator.pkl", - ) - outputs = [ - conformal_root / "data" / "conformal_intervals_mondrian.parquet", - conformal_root / "models" / "conformal_results_mondrian.pkl", - conformal_root / "models" / "pd_conformal_width_attribution_status.json", - ] - command_name = "conformal_extensive_grid" - stdout_log, stderr_log = _command_log_files( - artifact_root, - selected_phase, - command_name, - ) - commands.append( - PhaseCommand( - name=command_name, - phase=selected_phase, - command=[ - sys.executable, - str(ROOT / "scripts" / "generate_conformal_intervals.py"), - "--artifact_namespace", - safe_tag, - "--artifact_root", - str(artifact_root / "conformal"), - "--model_override_path", - str(pd_model_path), - "--alpha_candidates_90", - "0.05,0.075,0.09,0.095,0.10,0.105,0.11,0.125,0.15,0.20", - "--alpha_candidates_95", - "0.025,0.04,0.045,0.05,0.055,0.06,0.075", - "--partition_candidates", - "grade,score_decile_mondrian,grade_x_scoreband_mondrian", - "--n_score_bins_candidates", - "5,10,15,20,30", - "--min_group_sizes", - "100,150,250,500,1000,2000", - "--score_scale_families", - "none,bernoulli_sqrt,bernoulli_sqrt_clipped_0.02,bernoulli_sqrt_clipped_0.05", - "--calibrator_override_path", - str(pd_calibrator_path), - ], - outputs=[str(path) for path in outputs], - checkpoint=str(conformal_root / "checkpoints"), - env=_command_env(base_env, phase_threads=phase_threads), - max_workers=phase_workers, - cpu_threads=phase_threads, - stdout_log=str(stdout_log), - stderr_log=str(stderr_log), - ) - ) - elif selected_phase == "portfolio": - portfolio_root = artifact_root / "portfolio" / safe_tag - conformal_path = ( - artifact_root - / "conformal" - / safe_tag - / "data" - / "conformal_intervals_mondrian.parquet" - ) - outputs = [ - portfolio_root / "data" / "portfolio_bound_aware_frontier.parquet", - portfolio_root / "data" / "portfolio_bound_aware_bound_eval.parquet", - portfolio_root / "models" / "portfolio_bound_aware_selection.json", - ] - command_name = "portfolio_extensive_frontier" - stdout_log, stderr_log = _command_log_files( - artifact_root, - selected_phase, - command_name, + safe_tag=safe_tag, + phase_workers=phase_workers, + phase_threads=phase_threads, + base_env=base_env, ) - commands.append( - PhaseCommand( - name=command_name, - phase=selected_phase, - command=[ - sys.executable, - str(ROOT / "scripts" / "search" / "run_portfolio_bound_aware_search.py"), - "--config", - str(ROOT / "configs" / "crpto_optimization.yaml"), - "--conformal-intervals-path", - str(conformal_path), - "--run-label", - safe_tag, - "--output-dir", - str(portfolio_root / "data"), - "--model-dir", - str(portfolio_root / "models"), - "--incumbent-policy-path", - str(CHAMPION_PORTFOLIO_POLICY_PATH), - "--incumbent-risk-neighbors", - "0.155,0.16,0.165,0.17,0.175,0.18", - "--incumbent-gamma-neighbors", - "0.425,0.45,0.475,0.50,0.525,0.55,0.575", - "--incumbent-policy-modes", - "blended_uncertainty,capped_blended_uncertainty,tail_blended_uncertainty,segment_tail_blended_uncertainty,segment_relative_tail_blended_uncertainty", - "--risk-grid", - PORTFOLIO_RISK_GRID, - "--gamma-grid", - PORTFOLIO_GAMMA_GRID, - "--aversion-grid", - PORTFOLIO_AVERSION_GRID, - "--delta-cap-grid", - PORTFOLIO_CAP_TAIL_GRID, - "--tail-focus-grid", - PORTFOLIO_CAP_TAIL_GRID, - "--policy-modes", - "blended_uncertainty,capped_blended_uncertainty,tail_blended_uncertainty,segment_tail_blended_uncertainty,segment_relative_tail_blended_uncertainty", - "--alpha-grid", - "0.01,0.02,0.03,0.05,0.10,0.15,0.20", - "--max-candidates", - str(PORTFOLIO_MAX_CANDIDATES), - "--shortlist-top-k", - str(PORTFOLIO_SHORTLIST_TOP_K), - "--random-states", - PORTFOLIO_RANDOM_STATES, - "--solver-backend", - "highs", - "--exact-solver-backend", - "highs", - ], - outputs=[str(path) for path in outputs], - checkpoint=str( - portfolio_root / "models" / "portfolio_bound_aware_runtime_checkpoints" - ), - env=_command_env(base_env, phase_threads=phase_threads), - max_workers=phase_workers, - cpu_threads=phase_threads, - stdout_log=str(stdout_log), - stderr_log=str(stderr_log), - ) - ) - elif selected_phase == "metrics": - metrics_path = artifact_root / "metrics" / "frontier_metrics_manifest.json" - command_name = "metrics_manifest" - stdout_log, stderr_log = _command_log_files( - artifact_root, - selected_phase, - command_name, + ] + if phase == "portfolio": + return [ + _build_portfolio_phase_command( + artifact_root=artifact_root, + safe_tag=safe_tag, + phase_workers=phase_workers, + phase_threads=phase_threads, + base_env=base_env, ) - commands.append( - PhaseCommand( - name=command_name, - phase=selected_phase, - command=[ - sys.executable, - str(Path(__file__).resolve()), - "--run-tag", - safe_tag, - "--artifact-root", - str(artifact_root), - "--phase", - "plan", - "--resume", - ], - outputs=[str(metrics_path)], - checkpoint=str(artifact_root / "metrics"), - env=_command_env(base_env, phase_threads=1), - max_workers=1, - cpu_threads=1, - stdout_log=str(stdout_log), - stderr_log=str(stderr_log), - ) + ] + if phase == "metrics": + return [ + _build_metrics_phase_command( + artifact_root=artifact_root, + safe_tag=safe_tag, + base_env=base_env, ) - elif selected_phase in {"plan", "deps"}: - continue - else: - raise ValueError(f"Unknown sandbox phase: {selected_phase}") + ] + if phase in {"plan", "deps"}: + return [] + raise ValueError(f"Unknown sandbox phase: {phase}") + + +def _validate_phase_command_paths(commands: Iterable[PhaseCommand]) -> None: for command in commands: assert_safe_output_paths(command.outputs) assert_safe_output_path(command.checkpoint) @@ -1695,6 +1880,43 @@ def build_phase_commands( assert_safe_output_path(command.stdout_log) if command.stderr_log: assert_safe_output_path(command.stderr_log) + + +def build_phase_commands( + *, + artifact_root: Path, + run_tag: str, + phase: str, + max_workers: int, + cpu_threads: int, +) -> list[PhaseCommand]: + """Build commands for the requested sandbox phase.""" + safe_tag = sanitize_tag(run_tag) + artifact_root = artifact_root.resolve() + base_env = _sandbox_base_env(artifact_root, safe_tag) + selected_phases = ( + ["pd-smoke", "pd-broad", "pd-refine", "conformal", "portfolio", "metrics"] + if phase == "all" + else [phase] + ) + commands: list[PhaseCommand] = [] + for selected_phase in selected_phases: + phase_workers, phase_threads = _phase_resources( + selected_phase, + max_workers=max_workers, + cpu_threads=cpu_threads, + ) + commands.extend( + _commands_for_phase( + artifact_root=artifact_root, + safe_tag=safe_tag, + phase=selected_phase, + phase_workers=phase_workers, + phase_threads=phase_threads, + base_env=base_env, + ) + ) + _validate_phase_command_paths(commands) return commands @@ -2052,6 +2274,160 @@ def _run_one_command(command: PhaseCommand) -> tuple[PhaseCommand, int]: return command, returncode +def _phase_command_groups(commands: Sequence[PhaseCommand]) -> list[list[PhaseCommand]]: + phase_groups: list[list[PhaseCommand]] = [] + for command in commands: + if not phase_groups or phase_groups[-1][0].phase != command.phase: + phase_groups.append([command]) + else: + phase_groups[-1].append(command) + return phase_groups + + +def _log_skipped_completed( + *, + artifact_root: Path, + log_path: Path, + command: PhaseCommand, +) -> None: + _append_command_log( + log_path, + { + "captured_at_utc": utc_now_iso(), + "phase": command.phase, + "name": command.name, + "state": "skipped_completed", + "returncode": 0, + "checkpoint": command.checkpoint, + "stdout_log": command.stdout_log, + "stderr_log": command.stderr_log, + }, + ) + _log_command_to_mlflow( + artifact_root=artifact_root, + command=command, + state="skipped_completed", + returncode=0, + ) + + +def _pending_commands_for_group( + *, + artifact_root: Path, + log_path: Path, + group: list[PhaseCommand], + resume: bool, +) -> tuple[deque[PhaseCommand], int]: + pending: deque[PhaseCommand] = deque() + skipped = 0 + for command in group: + if resume and _completed_outputs(command): + _log_skipped_completed( + artifact_root=artifact_root, + log_path=log_path, + command=command, + ) + skipped += 1 + continue + pending.append(command) + return pending, skipped + + +def _start_available_commands( + *, + artifact_root: Path, + log_path: Path, + pending: deque[PhaseCommand], + running: dict[Future[tuple[PhaseCommand, int]], PhaseCommand], + executor: ThreadPoolExecutor, + worker_limit: int, +) -> Path | None: + last_checkpoint: Path | None = None + while pending and len(running) < worker_limit and _resource_allows_launch(artifact_root): + command = pending.popleft() + last_checkpoint = Path(command.checkpoint) + future = executor.submit(_run_one_command, command) + running[future] = command + _append_command_log( + log_path, + { + "captured_at_utc": utc_now_iso(), + "phase": command.phase, + "name": command.name, + "state": "started", + "returncode": "", + "checkpoint": command.checkpoint, + "stdout_log": command.stdout_log, + "stderr_log": command.stderr_log, + }, + ) + return last_checkpoint + + +def _record_finished_command( + *, + artifact_root: Path, + log_path: Path, + finished_command: PhaseCommand, + returncode: int, + failed_commands: list[PhaseCommand], +) -> None: + state = "complete" if returncode == 0 else "failed" + _append_command_log( + log_path, + { + "captured_at_utc": utc_now_iso(), + "phase": finished_command.phase, + "name": finished_command.name, + "state": state, + "returncode": returncode, + "checkpoint": finished_command.checkpoint, + "stdout_log": finished_command.stdout_log, + "stderr_log": finished_command.stderr_log, + }, + ) + _log_command_to_mlflow( + artifact_root=artifact_root, + command=finished_command, + state=state, + returncode=returncode, + ) + if returncode != 0 and finished_command.phase in PD_PHASES: + failed_commands.append(finished_command) + elif returncode != 0: + raise RuntimeError( + f"Sandbox command failed ({finished_command.name}) with return code {returncode}" + ) + + +def _select_phase_winners_if_needed( + *, + artifact_root: Path, + group: list[PhaseCommand], + failed_commands: list[PhaseCommand], + completed: int, + total_commands: int, +) -> None: + if not group or group[0].phase not in PD_PHASES: + return + selection_path = _select_pd_phase_winners(artifact_root, group[0].phase) + if selection_path is None: + failed_names = ", ".join(command.name for command in failed_commands[:10]) + raise RuntimeError( + "No successful PD candidates available after phase " + f"{group[0].phase}. Failed lanes: {failed_names}" + ) + _write_heartbeat( + artifact_root=artifact_root, + phase=group[0].phase, + completed_units=completed, + total_units=total_commands, + current_best_metric=None, + last_checkpoint_path=selection_path, + state="selected_with_failures" if failed_commands else "selected", + ) + + def _run_commands( *, artifact_root: Path, @@ -2060,69 +2436,32 @@ def _run_commands( ) -> None: log_path = artifact_root / "command_log.csv" completed = 0 - - phase_groups: list[list[PhaseCommand]] = [] - for command in commands: - if not phase_groups or phase_groups[-1][0].phase != command.phase: - phase_groups.append([command]) - else: - phase_groups[-1].append(command) + phase_groups = _phase_command_groups(commands) for group in phase_groups: - pending: deque[PhaseCommand] = deque() - for command in group: - if resume and _completed_outputs(command): - _append_command_log( - log_path, - { - "captured_at_utc": utc_now_iso(), - "phase": command.phase, - "name": command.name, - "state": "skipped_completed", - "returncode": 0, - "checkpoint": command.checkpoint, - "stdout_log": command.stdout_log, - "stderr_log": command.stderr_log, - }, - ) - _log_command_to_mlflow( - artifact_root=artifact_root, - command=command, - state="skipped_completed", - returncode=0, - ) - completed += 1 - continue - pending.append(command) - + pending, skipped = _pending_commands_for_group( + artifact_root=artifact_root, + log_path=log_path, + group=group, + resume=resume, + ) + completed += skipped worker_limit = max(1, max((command.max_workers for command in group), default=1)) running: dict[Future[tuple[PhaseCommand, int]], PhaseCommand] = {} failed_commands: list[PhaseCommand] = [] last_checkpoint: Path | None = None with ThreadPoolExecutor(max_workers=worker_limit) as executor: while pending or running: - while ( - pending - and len(running) < worker_limit - and _resource_allows_launch(artifact_root) - ): - command = pending.popleft() - last_checkpoint = Path(command.checkpoint) - future = executor.submit(_run_one_command, command) - running[future] = command - _append_command_log( - log_path, - { - "captured_at_utc": utc_now_iso(), - "phase": command.phase, - "name": command.name, - "state": "started", - "returncode": "", - "checkpoint": command.checkpoint, - "stdout_log": command.stdout_log, - "stderr_log": command.stderr_log, - }, - ) + next_checkpoint = _start_available_commands( + artifact_root=artifact_root, + log_path=log_path, + pending=pending, + running=running, + executor=executor, + worker_limit=worker_limit, + ) + if next_checkpoint is not None: + last_checkpoint = next_checkpoint _write_heartbeat( artifact_root=artifact_root, phase=group[0].phase if group else "waiting_for_ram", @@ -2141,51 +2480,23 @@ def _run_commands( return_when=FIRST_COMPLETED, ) for future in done: - command = running.pop(future) + running.pop(future) finished_command, returncode = future.result() completed += 1 - _append_command_log( - log_path, - { - "captured_at_utc": utc_now_iso(), - "phase": finished_command.phase, - "name": finished_command.name, - "state": "complete" if returncode == 0 else "failed", - "returncode": returncode, - "checkpoint": finished_command.checkpoint, - "stdout_log": finished_command.stdout_log, - "stderr_log": finished_command.stderr_log, - }, - ) - _log_command_to_mlflow( + _record_finished_command( artifact_root=artifact_root, - command=finished_command, - state="complete" if returncode == 0 else "failed", + log_path=log_path, + finished_command=finished_command, returncode=returncode, + failed_commands=failed_commands, ) - if returncode != 0 and finished_command.phase in PD_PHASES: - failed_commands.append(finished_command) - elif returncode != 0: - raise RuntimeError( - f"Sandbox command failed ({command.name}) with return code {returncode}" - ) - if group and group[0].phase in PD_PHASES: - selection_path = _select_pd_phase_winners(artifact_root, group[0].phase) - if selection_path is None: - failed_names = ", ".join(command.name for command in failed_commands[:10]) - raise RuntimeError( - "No successful PD candidates available after phase " - f"{group[0].phase}. Failed lanes: {failed_names}" - ) - _write_heartbeat( - artifact_root=artifact_root, - phase=group[0].phase, - completed_units=completed, - total_units=len(commands), - current_best_metric=None, - last_checkpoint_path=selection_path, - state="selected_with_failures" if failed_commands else "selected", - ) + _select_phase_winners_if_needed( + artifact_root=artifact_root, + group=group, + failed_commands=failed_commands, + completed=completed, + total_commands=len(commands), + ) _write_heartbeat( artifact_root=artifact_root, phase=commands[-1].phase if commands else "plan", diff --git a/scripts/select_economic_portfolio_policy.py b/scripts/select_economic_portfolio_policy.py index 30939e6..f292d04 100644 --- a/scripts/select_economic_portfolio_policy.py +++ b/scripts/select_economic_portfolio_policy.py @@ -4,8 +4,9 @@ import argparse import json +from dataclasses import dataclass from pathlib import Path -from typing import Any +from typing import Any, cast import numpy as np import pandas as pd @@ -21,12 +22,51 @@ _run_strategy, ) from src.evaluation.ab_testing import compare_strategies +from src.optimization.portfolio_model import solution_allocation_vector from src.utils.artifact_metadata import build_artifact_metadata, resolve_run_tag from src.utils.script_helpers import artifact_path as _artifact_path, try_load_json SCHEMA_VERSION = "2026-03-10.1" +@dataclass(frozen=True) +class SelectionSettings: + top_k: int + selector_name: str + min_funded_ratio: float + min_total_allocated_ratio: float + min_breadth_score: float + breadth_weight_funded_ratio: float + breadth_weight_allocation_ratio: float + breadth_weight_allocation_similarity: float + max_por_pct: float + canonical_modes: set[str] + ab_like_top_m: int + ab_like_bootstrap_n: int + ab_like_seed: int + + +@dataclass(frozen=True) +class DecisionInputs: + common: dict[str, object] + default_flag: np.ndarray + loan_amnt: np.ndarray + int_rates: np.ndarray + pd_high: np.ndarray + total_budget: float + universe_source: str | None + scenario_meta: dict[str, Any] + + +@dataclass(frozen=True) +class SelectionResult: + selected: dict[str, Any] + selected_policy: dict[str, Any] + selector_outcome: str + fallback_applied: bool + fallback_reason: str | None + + def _policy_key(row: pd.Series) -> tuple[object, ...]: return ( str(row.get("policy_mode", "")), @@ -58,6 +98,77 @@ def _load_json(path: Path) -> dict[str, Any]: return try_load_json(path) +def _load_config(path: str) -> dict[str, Any]: + with open(path, encoding="utf-8") as f: + payload = yaml.safe_load(f) + return payload if isinstance(payload, dict) else {} + + +def _selection_settings(config: dict[str, Any]) -> SelectionSettings: + selection_cfg = dict(config.get("portfolio_selection", {}) or {}) + min_funded_ratio = float(selection_cfg.get("min_funded_ratio", 0.95)) + return SelectionSettings( + top_k=int(selection_cfg.get("actual_ab_top_k", 20)), + selector_name=str(selection_cfg.get("canonical_selector", "economic_actual_ab_v1")), + min_funded_ratio=min_funded_ratio, + min_total_allocated_ratio=float(selection_cfg.get("min_total_allocated_ratio", 0.98)), + min_breadth_score=float(selection_cfg.get("min_breadth_score", min_funded_ratio)), + breadth_weight_funded_ratio=float(selection_cfg.get("breadth_weight_funded_ratio", 0.5)), + breadth_weight_allocation_ratio=float( + selection_cfg.get("breadth_weight_allocation_ratio", 0.3) + ), + breadth_weight_allocation_similarity=float( + selection_cfg.get("breadth_weight_allocation_similarity", 0.2) + ), + max_por_pct=float(selection_cfg.get("max_price_of_robustness_pct", -15.0)), + canonical_modes={ + str(x) for x in selection_cfg.get("canonical_policy_modes", ["blended_uncertainty"]) + }, + ab_like_top_m=int(selection_cfg.get("ab_like_top_m", 8)), + ab_like_bootstrap_n=int(selection_cfg.get("ab_like_bootstrap_n", 200)), + ab_like_seed=int(selection_cfg.get("ab_like_seed", 42)), + ) + + +def _load_frontier(frontier_path: str) -> pd.DataFrame: + frontier = pd.read_parquet(_artifact_path(frontier_path)) + if frontier.empty: + raise ValueError("portfolio_robustness_frontier.parquet is empty") + return frontier + + +def _prepare_decision_inputs( + *, + config: dict[str, Any], + candidate_universe_path: str, + decision_scenario: str, +) -> DecisionInputs: + test_df = pd.read_parquet("data/processed/test_fe.parquet") + intervals = pd.read_parquet("data/processed/conformal_intervals_mondrian.parquet") + test_df, intervals, universe_source = _apply_candidate_universe( + test_df, + intervals, + candidate_universe_path=candidate_universe_path, + max_candidates=0, + ) + test_df, intervals, scenario_meta = _apply_decision_scenario( + test_df, + intervals, + decision_scenario=decision_scenario, + ) + common, default_flag, loan_amnt, int_rates, pd_high = _build_common_inputs(test_df, intervals) + return DecisionInputs( + common=common, + default_flag=default_flag, + loan_amnt=loan_amnt, + int_rates=int_rates, + pd_high=pd_high, + total_budget=float(config["portfolio"]["total_budget"]), + universe_source=universe_source, + scenario_meta=cast(dict[str, Any], scenario_meta), + ) + + def _dedupe_candidates(rows: list[pd.Series]) -> list[pd.Series]: out: list[pd.Series] = [] seen: set[tuple[object, ...]] = set() @@ -184,7 +295,7 @@ def _control_metrics_by_risk( default_flag=default_flag, lgd_val=0.45, ) - alloc = np.array([sol["allocation"][i] for i in range(len(loan_amnt))], dtype=float) + alloc = solution_allocation_vector(sol, len(loan_amnt)) controls[risk_tol] = { "solution": sol, "returns": returns, @@ -198,287 +309,287 @@ def _control_metrics_by_risk( return controls -def main( - config_path: str = "configs/optimization.yaml", - frontier_path: str = "data/processed/portfolio_robustness_frontier.parquet", - research_policy_path: str = "models/portfolio_research_policy.json", - champion_policy_path: str = "models/champion_portfolio_policy.json", - status_path: str = "models/champion_policy_selection_status.json", - candidate_universe_path: str = "data/processed/champion_candidate_universe.parquet", - run_tag: str | None = None, - solver_backend: str = "highs", - decision_scenario: str = "baseline", -) -> None: - with open(config_path, encoding="utf-8") as f: - config = yaml.safe_load(f) - selection_cfg = dict(config.get("portfolio_selection", {}) or {}) - top_k = int(selection_cfg.get("actual_ab_top_k", 20)) - selector_name = str(selection_cfg.get("canonical_selector", "economic_actual_ab_v1")) - min_funded_ratio = float(selection_cfg.get("min_funded_ratio", 0.95)) - min_total_allocated_ratio = float(selection_cfg.get("min_total_allocated_ratio", 0.98)) - min_breadth_score = float(selection_cfg.get("min_breadth_score", min_funded_ratio)) - breadth_weight_funded_ratio = float(selection_cfg.get("breadth_weight_funded_ratio", 0.5)) - breadth_weight_allocation_ratio = float( - selection_cfg.get("breadth_weight_allocation_ratio", 0.3) +def _evaluate_candidate_row( + *, + row: pd.Series, + inputs: DecisionInputs, + controls: dict[float, dict[str, Any]], + settings: SelectionSettings, + solver_backend: str, +) -> dict[str, Any]: + policy = _policy_from_row(row, source="economic_actual_ab_v1") + risk_tol = float(policy["risk_tolerance"]) + control = controls[risk_tol] + sol_b, _ = _run_strategy( + common=inputs.common, + robust=True, + robust_policy=policy, + total_budget=inputs.total_budget, + max_portfolio_pd=risk_tol, + solver_backend=solver_backend, ) - breadth_weight_allocation_similarity = float( - selection_cfg.get("breadth_weight_allocation_similarity", 0.2) + returns_b, metrics_b = _candidate_metrics( + solution=sol_b, + loan_amnt=inputs.loan_amnt, + int_rates=inputs.int_rates, + default_flag=inputs.default_flag, + lgd_val=0.45, ) - max_por_pct = float(selection_cfg.get("max_price_of_robustness_pct", -15.0)) - canonical_modes = { - str(x) for x in selection_cfg.get("canonical_policy_modes", ["blended_uncertainty"]) - } - ab_like_top_m = int(selection_cfg.get("ab_like_top_m", 8)) - ab_like_bootstrap_n = int(selection_cfg.get("ab_like_bootstrap_n", 200)) - ab_like_seed = int(selection_cfg.get("ab_like_seed", 42)) - - frontier = pd.read_parquet(_artifact_path(frontier_path)) - if frontier.empty: - raise ValueError("portfolio_robustness_frontier.parquet is empty") - - test_df = pd.read_parquet("data/processed/test_fe.parquet") - intervals = pd.read_parquet("data/processed/conformal_intervals_mondrian.parquet") - test_df, intervals, universe_source = _apply_candidate_universe( - test_df, - intervals, - candidate_universe_path=candidate_universe_path, - max_candidates=0, + control_metrics = control["metrics"] + returns_control = np.asarray(control["returns"], dtype=float) + diff_total_return = float(metrics_b["total_return"] - float(control_metrics["total_return"])) + return_delta_pct = float( + diff_total_return / (abs(float(control_metrics["total_return"])) + 1e-6) * 100.0 ) - test_df, intervals, scenario_meta = _apply_decision_scenario( - test_df, - intervals, - decision_scenario=decision_scenario, + tolerance_total_return = abs(float(control_metrics["total_return"])) * 0.05 + funded_ratio = float(metrics_b["n_funded"] / max(float(control_metrics["n_funded"]), 1.0)) + total_allocated_ratio = float( + metrics_b["total_allocated"] / max(float(control_metrics["total_allocated"]), 1.0) ) - common, default_flag, loan_amnt, int_rates, pd_high = _build_common_inputs(test_df, intervals) - total_budget = float(config["portfolio"]["total_budget"]) - controls = _control_metrics_by_risk( - common=common, - default_flag=default_flag, - loan_amnt=loan_amnt, - int_rates=int_rates, - risk_values=frontier["risk_tolerance"].tolist(), - total_budget=total_budget, - solver_backend=solver_backend, + alloc_b = solution_allocation_vector(sol_b, len(inputs.loan_amnt)) + allocation_similarity = _allocation_similarity(control["allocation"], alloc_b) + breadth_score = _breadth_score( + funded_ratio=funded_ratio, + total_allocated_ratio=total_allocated_ratio, + allocation_similarity=allocation_similarity, + weight_funded_ratio=settings.breadth_weight_funded_ratio, + weight_allocation_ratio=settings.breadth_weight_allocation_ratio, + weight_allocation_similarity=settings.breadth_weight_allocation_similarity, + ) + cand_worst_pd = float( + np.sum(alloc_b * inputs.loan_amnt * inputs.pd_high) + / (float(sol_b["total_allocated"]) + 1e-6) ) + return { + "policy": policy, + "risk_tolerance": risk_tol, + "passed_no_regression": bool(diff_total_return >= -tolerance_total_return), + "diff_total_return": diff_total_return, + "tolerance_total_return": tolerance_total_return, + "funded_ratio": funded_ratio, + "total_allocated_ratio": total_allocated_ratio, + "worst_case_pd_reduction_bps": float( + (float(control["worst_case_pd"]) - cand_worst_pd) * 1e4 + ), + "price_of_robustness_pct": float(min(return_delta_pct, 0.0)), + "return_delta_pct": return_delta_pct, + "frontier_price_of_robustness_pct": float(row.get("price_of_robustness_pct", 0.0)), + "return_per_funded_delta": float( + metrics_b["avg_return_per_funded"] - float(control_metrics["avg_return_per_funded"]) + ), + "allocation_similarity": allocation_similarity, + "breadth_score": breadth_score, + "n_funded_candidate": int(metrics_b["n_funded"]), + "n_funded_control": int(control_metrics["n_funded"]), + "total_return_candidate": float(metrics_b["total_return"]), + "total_return_control": float(control_metrics["total_return"]), + "eligible_hard_filters": False, + "_returns_candidate": returns_b, + "_returns_control": returns_control, + } - candidate_rows = _select_candidate_rows(frontier, top_k=top_k) - if not candidate_rows: - raise ValueError("No eligible canonical candidates found in frontier") - evaluated: list[dict[str, Any]] = [] - for row in candidate_rows: - policy = _policy_from_row(row, source="economic_actual_ab_v1") - risk_tol = float(policy["risk_tolerance"]) - control = controls[risk_tol] - sol_b, _ = _run_strategy( - common=common, - robust=True, - robust_policy=policy, - total_budget=total_budget, - max_portfolio_pd=risk_tol, +def _evaluate_candidate_rows( + *, + candidate_rows: list[pd.Series], + inputs: DecisionInputs, + controls: dict[float, dict[str, Any]], + settings: SelectionSettings, + solver_backend: str, +) -> list[dict[str, Any]]: + return [ + _evaluate_candidate_row( + row=row, + inputs=inputs, + controls=controls, + settings=settings, solver_backend=solver_backend, ) - returns_b, metrics_b = _candidate_metrics( - solution=sol_b, - loan_amnt=loan_amnt, - int_rates=int_rates, - default_flag=default_flag, - lgd_val=0.45, - ) - control_metrics = control["metrics"] - returns_control = np.asarray(control["returns"], dtype=float) - diff_total_return = float( - metrics_b["total_return"] - float(control_metrics["total_return"]) - ) - return_delta_pct = float( - diff_total_return / (abs(float(control_metrics["total_return"])) + 1e-6) * 100.0 - ) - actual_price_of_robustness_pct = float(min(return_delta_pct, 0.0)) - tolerance_total_return = abs(float(control_metrics["total_return"])) * 0.05 - passed_no_regression = bool(diff_total_return >= -tolerance_total_return) - funded_ratio = float(metrics_b["n_funded"] / max(float(control_metrics["n_funded"]), 1.0)) - total_allocated_ratio = float( - metrics_b["total_allocated"] / max(float(control_metrics["total_allocated"]), 1.0) - ) - alloc_b = np.array([sol_b["allocation"][i] for i in range(len(loan_amnt))], dtype=float) - allocation_similarity = _allocation_similarity(control["allocation"], alloc_b) - breadth_score = _breadth_score( - funded_ratio=funded_ratio, - total_allocated_ratio=total_allocated_ratio, - allocation_similarity=allocation_similarity, - weight_funded_ratio=breadth_weight_funded_ratio, - weight_allocation_ratio=breadth_weight_allocation_ratio, - weight_allocation_similarity=breadth_weight_allocation_similarity, - ) - cand_worst_pd = float( - np.sum(alloc_b * loan_amnt * pd_high) / (float(sol_b["total_allocated"]) + 1e-6) - ) - evaluated.append( - { - "policy": policy, - "risk_tolerance": risk_tol, - "passed_no_regression": passed_no_regression, - "diff_total_return": diff_total_return, - "tolerance_total_return": tolerance_total_return, - "funded_ratio": funded_ratio, - "total_allocated_ratio": total_allocated_ratio, - "worst_case_pd_reduction_bps": float( - (float(control["worst_case_pd"]) - cand_worst_pd) * 1e4 - ), - "price_of_robustness_pct": actual_price_of_robustness_pct, - "return_delta_pct": return_delta_pct, - "frontier_price_of_robustness_pct": float(row.get("price_of_robustness_pct", 0.0)), - "return_per_funded_delta": float( - metrics_b["avg_return_per_funded"] - - float(control_metrics["avg_return_per_funded"]) - ), - "allocation_similarity": allocation_similarity, - "breadth_score": breadth_score, - "n_funded_candidate": int(metrics_b["n_funded"]), - "n_funded_control": int(control_metrics["n_funded"]), - "total_return_candidate": float(metrics_b["total_return"]), - "total_return_control": float(control_metrics["total_return"]), - "eligible_hard_filters": False, - "_returns_candidate": returns_b, - "_returns_control": returns_control, - } - ) + for row in candidate_rows + ] + + +def _base_hard_filters(item: dict[str, Any], settings: SelectionSettings) -> bool: + return bool( + item["passed_no_regression"] + and float(item["price_of_robustness_pct"]) >= settings.max_por_pct + and str(item["policy"]["policy_mode"]) in settings.canonical_modes + ) + +def _mark_hard_filter_eligibility( + evaluated: list[dict[str, Any]], + settings: SelectionSettings, +) -> None: for item in evaluated: - base_filters = bool( - item["passed_no_regression"] - and float(item["price_of_robustness_pct"]) >= max_por_pct - and str(item["policy"]["policy_mode"]) in canonical_modes - ) - if selector_name in {"economic_actual_ab_v2", "economic_actual_ab_v3"}: + base_filters = _base_hard_filters(item, settings) + if settings.selector_name in {"economic_actual_ab_v2", "economic_actual_ab_v3"}: item["eligible_hard_filters"] = bool( base_filters - and float(item["total_allocated_ratio"]) >= min_total_allocated_ratio - and float(item["breadth_score"]) >= min_breadth_score - and float(item["funded_ratio"]) >= min_funded_ratio + and float(item["total_allocated_ratio"]) >= settings.min_total_allocated_ratio + and float(item["breadth_score"]) >= settings.min_breadth_score + and float(item["funded_ratio"]) >= settings.min_funded_ratio ) else: item["eligible_hard_filters"] = bool( - base_filters and float(item["funded_ratio"]) >= min_funded_ratio + base_filters and float(item["funded_ratio"]) >= settings.min_funded_ratio ) + +def _base_rank_key(item: dict[str, Any]) -> tuple[float, float, float, float, float]: + return ( + float(item["worst_case_pd_reduction_bps"]), + float(item["diff_total_return"]), + float(item.get("breadth_score", 0.0)), + -abs(float(item["price_of_robustness_pct"])), + float(item["funded_ratio"]), + ) + + +def _v3_rank_key(item: dict[str, Any]) -> tuple[bool, float, float, float, float, float]: + return ( + bool(item.get("ab_like_passed_no_regression", False)), + float(item.get("ab_like_diff_total_return", item["diff_total_return"])), + float(item["worst_case_pd_reduction_bps"]), + float(item.get("breadth_score", 0.0)), + -abs(float(item["price_of_robustness_pct"])), + float(item["funded_ratio"]), + ) + + +def _candidate_pool_after_filters(evaluated: list[dict[str, Any]]) -> list[dict[str, Any]]: eligible = [x for x in evaluated if bool(x["eligible_hard_filters"])] robust_eligible = [x for x in eligible if float(x["policy"]["gamma"]) > 0.0] - candidate_pool = robust_eligible or eligible - - if selector_name == "economic_actual_ab_v3" and candidate_pool: - pre_ranked = sorted( - candidate_pool, - key=lambda x: ( - float(x["worst_case_pd_reduction_bps"]), - float(x["diff_total_return"]), - float(x.get("breadth_score", 0.0)), - -abs(float(x["price_of_robustness_pct"])), - float(x["funded_ratio"]), - ), - reverse=True, - )[: max(1, ab_like_top_m)] - for idx, item in enumerate(pre_ranked): - item.update( - _ab_like_score( - returns_control=np.asarray(item["_returns_control"], dtype=float), - returns_candidate=np.asarray(item["_returns_candidate"], dtype=float), - seed=ab_like_seed + idx, - n_boot=ab_like_bootstrap_n, - ) - ) - robust_ab_like = [ - x for x in pre_ranked if bool(x.get("ab_like_passed_no_regression", False)) - ] - candidate_pool = robust_ab_like or pre_ranked - - fallback_applied = False - fallback_reason = None - selector_outcome = "robust_selected" - if candidate_pool: - if selector_name == "economic_actual_ab_v3": - selected = sorted( - candidate_pool, - key=lambda x: ( - bool(x.get("ab_like_passed_no_regression", False)), - float(x.get("ab_like_diff_total_return", x["diff_total_return"])), - float(x["worst_case_pd_reduction_bps"]), - float(x.get("breadth_score", 0.0)), - -abs(float(x["price_of_robustness_pct"])), - float(x["funded_ratio"]), - ), - reverse=True, - )[0] - else: - selected = sorted( - candidate_pool, - key=lambda x: ( - float(x["worst_case_pd_reduction_bps"]), - float(x["diff_total_return"]), - float(x.get("breadth_score", 0.0)), - -abs(float(x["price_of_robustness_pct"])), - float(x["funded_ratio"]), - ), - reverse=True, - )[0] - if float(selected["policy"]["gamma"]) <= 0.0: - selector_outcome = "fallback_nonrobust" - fallback_applied = True - fallback_reason = "no_economically_viable_robust_policy" - else: - fallback_applied = True - selector_outcome = "fallback_nonrobust" - fallback_reason = "no_economically_viable_robust_policy" - fallback_row = frontier.loc[frontier["selected_for_champion"].fillna(False).astype(bool)] - selected_row = fallback_row.iloc[0] if not fallback_row.empty else frontier.iloc[0] - selected = { - "policy": { - **_policy_from_row(selected_row, source="economic_actual_ab_v1_fallback"), - "gamma": 0.0, - "policy_mode": "blended_uncertainty", - "delta_cap_quantile": 1.0, - "tail_focus_quantile": 1.0, - "uncertainty_aversion": 0.0, - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0, - }, - "risk_tolerance": float(selected_row["risk_tolerance"]), - "passed_no_regression": True, - "diff_total_return": 0.0, - "tolerance_total_return": abs( - float(controls[float(selected_row["risk_tolerance"])]["metrics"]["total_return"]) + return robust_eligible or eligible + + +def _apply_ab_like_screen( + candidate_pool: list[dict[str, Any]], + settings: SelectionSettings, +) -> list[dict[str, Any]]: + if settings.selector_name != "economic_actual_ab_v3" or not candidate_pool: + return candidate_pool + pre_ranked = sorted(candidate_pool, key=_base_rank_key, reverse=True)[ + : max(1, settings.ab_like_top_m) + ] + for idx, item in enumerate(pre_ranked): + item.update( + _ab_like_score( + returns_control=np.asarray(item["_returns_control"], dtype=float), + returns_candidate=np.asarray(item["_returns_candidate"], dtype=float), + seed=settings.ab_like_seed + idx, + n_boot=settings.ab_like_bootstrap_n, ) - * 0.05, - "funded_ratio": 1.0, - "total_allocated_ratio": 1.0, - "worst_case_pd_reduction_bps": 0.0, - "price_of_robustness_pct": 0.0, - "return_per_funded_delta": 0.0, - "allocation_similarity": 1.0, - "breadth_score": 1.0, - "n_funded_candidate": int( - controls[float(selected_row["risk_tolerance"])]["metrics"]["n_funded"] - ), - "n_funded_control": int( - controls[float(selected_row["risk_tolerance"])]["metrics"]["n_funded"] - ), - "total_return_candidate": float( - controls[float(selected_row["risk_tolerance"])]["metrics"]["total_return"] - ), - "total_return_control": float( - controls[float(selected_row["risk_tolerance"])]["metrics"]["total_return"] - ), - "eligible_hard_filters": False, - } + ) + robust_ab_like = [x for x in pre_ranked if bool(x.get("ab_like_passed_no_regression", False))] + return robust_ab_like or pre_ranked + + +def _select_ranked_candidate( + candidate_pool: list[dict[str, Any]], + settings: SelectionSettings, +) -> dict[str, Any]: + rank_key = _v3_rank_key if settings.selector_name == "economic_actual_ab_v3" else _base_rank_key + return sorted(candidate_pool, key=rank_key, reverse=True)[0] + + +def _fallback_selected_candidate( + frontier: pd.DataFrame, + controls: dict[float, dict[str, Any]], +) -> dict[str, Any]: + fallback_row = frontier.loc[frontier["selected_for_champion"].fillna(False).astype(bool)] + selected_row = fallback_row.iloc[0] if not fallback_row.empty else frontier.iloc[0] + risk_tol = float(selected_row["risk_tolerance"]) + control_metrics = controls[risk_tol]["metrics"] + return { + "policy": { + **_policy_from_row(selected_row, source="economic_actual_ab_v1_fallback"), + "gamma": 0.0, + "policy_mode": "blended_uncertainty", + "delta_cap_quantile": 1.0, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.0, + "min_budget_utilization": 0.0, + "pd_cap_slack_penalty": 0.0, + }, + "risk_tolerance": risk_tol, + "passed_no_regression": True, + "diff_total_return": 0.0, + "tolerance_total_return": abs(float(control_metrics["total_return"])) * 0.05, + "funded_ratio": 1.0, + "total_allocated_ratio": 1.0, + "worst_case_pd_reduction_bps": 0.0, + "price_of_robustness_pct": 0.0, + "return_per_funded_delta": 0.0, + "allocation_similarity": 1.0, + "breadth_score": 1.0, + "n_funded_candidate": int(control_metrics["n_funded"]), + "n_funded_control": int(control_metrics["n_funded"]), + "total_return_candidate": float(control_metrics["total_return"]), + "total_return_control": float(control_metrics["total_return"]), + "eligible_hard_filters": False, + } - resolved_run_tag = resolve_run_tag(run_tag, require_explicit=True) - research_policy = _load_json(_artifact_path(research_policy_path)) - champion_payload = { - "selection_stage": selector_name, - "selection_universe_path": universe_source or str(_artifact_path(candidate_universe_path)), + +def _choose_selected_candidate( + *, + frontier: pd.DataFrame, + controls: dict[float, dict[str, Any]], + evaluated: list[dict[str, Any]], + settings: SelectionSettings, +) -> SelectionResult: + _mark_hard_filter_eligibility(evaluated, settings) + candidate_pool = _apply_ab_like_screen(_candidate_pool_after_filters(evaluated), settings) + fallback_reason = "no_economically_viable_robust_policy" + if not candidate_pool: + selected = _fallback_selected_candidate(frontier, controls) + return SelectionResult( + selected=selected, + selected_policy=cast(dict[str, Any], selected["policy"]), + selector_outcome="fallback_nonrobust", + fallback_applied=True, + fallback_reason=fallback_reason, + ) + + selected = _select_ranked_candidate(candidate_pool, settings) + selected_policy = cast(dict[str, Any], selected["policy"]) + if float(selected_policy["gamma"]) <= 0.0: + return SelectionResult( + selected=selected, + selected_policy=selected_policy, + selector_outcome="fallback_nonrobust", + fallback_applied=True, + fallback_reason=fallback_reason, + ) + return SelectionResult( + selected=selected, + selected_policy=selected_policy, + selector_outcome="robust_selected", + fallback_applied=False, + fallback_reason=None, + ) + + +def _without_internal_returns(item: dict[str, Any]) -> dict[str, Any]: + return {k: v for k, v in item.items() if not str(k).startswith("_returns_")} + + +def _build_champion_payload( + *, + settings: SelectionSettings, + selection: SelectionResult, + research_policy: dict[str, Any], + universe_path: str, + decision_scenario: str, + resolved_run_tag: str, +) -> dict[str, Any]: + selected = selection.selected + return { + "selection_stage": settings.selector_name, + "selection_universe_path": universe_path, "decision_scenario": str(decision_scenario), - "selection_outcome": selector_outcome, - "selected_policy": selected["policy"], + "selection_outcome": selection.selector_outcome, + "selected_policy": selection.selected_policy, "economic_metrics": { "diff_total_return": float(selected["diff_total_return"]), "passed_no_regression": bool(selected["passed_no_regression"]), @@ -504,21 +615,30 @@ def main( require_explicit=True, ), } - selected_clean = {k: v for k, v in selected.items() if not str(k).startswith("_returns_")} - status_payload = { - "selector_name": selector_name, - "universe_path": universe_source or str(_artifact_path(candidate_universe_path)), + + +def _build_status_payload( + *, + settings: SelectionSettings, + selection: SelectionResult, + inputs: DecisionInputs, + evaluated: list[dict[str, Any]], + controls: dict[float, dict[str, Any]], + universe_path: str, + decision_scenario: str, + resolved_run_tag: str, +) -> dict[str, Any]: + return { + "selector_name": settings.selector_name, + "universe_path": universe_path, "decision_scenario": str(decision_scenario), - "decision_scenario_meta": scenario_meta, + "decision_scenario_meta": inputs.scenario_meta, "control_metrics": {str(k): dict(v["metrics"]) for k, v in controls.items()}, - "evaluated_candidates": [ - {k: v for k, v in item.items() if not str(k).startswith("_returns_")} - for item in evaluated - ], - "selected_candidate": selected_clean, - "selector_outcome": selector_outcome, - "fallback_applied": fallback_applied, - "fallback_reason": fallback_reason, + "evaluated_candidates": [_without_internal_returns(item) for item in evaluated], + "selected_candidate": _without_internal_returns(selection.selected), + "selector_outcome": selection.selector_outcome, + "fallback_applied": selection.fallback_applied, + "fallback_reason": selection.fallback_reason, **build_artifact_metadata( schema_version=SCHEMA_VERSION, run_tag=resolved_run_tag, @@ -526,6 +646,14 @@ def main( ), } + +def _write_selection_outputs( + *, + champion_policy_path: str, + status_path: str, + champion_payload: dict[str, Any], + status_payload: dict[str, Any], +) -> None: champion_out = _artifact_path(champion_policy_path) status_out = _artifact_path(status_path) champion_out.parent.mkdir(parents=True, exist_ok=True) @@ -536,6 +664,81 @@ def main( logger.info("Saved champion policy selection status: {}", status_out) +def main( + config_path: str = "configs/optimization.yaml", + frontier_path: str = "data/processed/portfolio_robustness_frontier.parquet", + research_policy_path: str = "models/portfolio_research_policy.json", + champion_policy_path: str = "models/champion_portfolio_policy.json", + status_path: str = "models/champion_policy_selection_status.json", + candidate_universe_path: str = "data/processed/champion_candidate_universe.parquet", + run_tag: str | None = None, + solver_backend: str = "highs", + decision_scenario: str = "baseline", +) -> None: + config = _load_config(config_path) + settings = _selection_settings(config) + frontier = _load_frontier(frontier_path) + inputs = _prepare_decision_inputs( + config=config, + candidate_universe_path=candidate_universe_path, + decision_scenario=decision_scenario, + ) + controls = _control_metrics_by_risk( + common=inputs.common, + default_flag=inputs.default_flag, + loan_amnt=inputs.loan_amnt, + int_rates=inputs.int_rates, + risk_values=frontier["risk_tolerance"].tolist(), + total_budget=inputs.total_budget, + solver_backend=solver_backend, + ) + + candidate_rows = _select_candidate_rows(frontier, top_k=settings.top_k) + if not candidate_rows: + raise ValueError("No eligible canonical candidates found in frontier") + + evaluated = _evaluate_candidate_rows( + candidate_rows=candidate_rows, + inputs=inputs, + controls=controls, + settings=settings, + solver_backend=solver_backend, + ) + selection = _choose_selected_candidate( + frontier=frontier, + controls=controls, + evaluated=evaluated, + settings=settings, + ) + resolved_run_tag = resolve_run_tag(run_tag, require_explicit=True) + research_policy = _load_json(_artifact_path(research_policy_path)) + universe_path = inputs.universe_source or str(_artifact_path(candidate_universe_path)) + champion_payload = _build_champion_payload( + settings=settings, + selection=selection, + research_policy=research_policy, + universe_path=universe_path, + decision_scenario=decision_scenario, + resolved_run_tag=resolved_run_tag, + ) + status_payload = _build_status_payload( + settings=settings, + selection=selection, + inputs=inputs, + evaluated=evaluated, + controls=controls, + universe_path=universe_path, + decision_scenario=decision_scenario, + resolved_run_tag=resolved_run_tag, + ) + _write_selection_outputs( + champion_policy_path=champion_policy_path, + status_path=status_path, + champion_payload=champion_payload, + status_payload=status_payload, + ) + + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--config", default="configs/optimization.yaml") diff --git a/scripts/simulate_ab_test.py b/scripts/simulate_ab_test.py index 8c2d681..ff1aa0e 100644 --- a/scripts/simulate_ab_test.py +++ b/scripts/simulate_ab_test.py @@ -40,6 +40,196 @@ SCHEMA_VERSION = "2026-03-01.1" +def _default_robust_policy(max_portfolio_pd: float) -> dict[str, Any]: + return { + "source": "fallback_default", + "risk_tolerance": float(max_portfolio_pd), + "uncertainty_aversion": 0.0, + "min_budget_utilization": 0.0, + "pd_cap_slack_penalty": 0.0, + "policy_mode": "hard_worst_case", + "gamma": 1.0, + "delta_cap_quantile": 1.0, + "tail_focus_quantile": 1.0, + } + + +def _select_champion_policy(payload: dict[str, Any], policy_selector: str) -> dict[str, Any]: + if policy_selector == "robustness_aware": + return cast( + dict[str, Any], + payload.get("selected_policy_robustness_aware") or payload.get("selected_policy", {}), + ) + if policy_selector == "balanced_robustness": + return cast( + dict[str, Any], + payload.get("selected_policy_balanced_robustness") + or payload.get("selected_policy_guardrail_robustness") + or payload.get("selected_policy_robustness_aware") + or payload.get("selected_policy", {}), + ) + if policy_selector == "guardrail_robustness": + return cast( + dict[str, Any], + payload.get("selected_policy_guardrail_robustness") + or payload.get("selected_policy_balanced_robustness") + or payload.get("selected_policy_robustness_aware") + or payload.get("selected_policy", {}), + ) + if policy_selector == "explicit_champion_only": + selected = payload.get("selected_policy", {}) + if not selected: + raise ValueError("Champion policy artifact missing selected_policy") + return cast(dict[str, Any], selected) + return cast(dict[str, Any], payload.get("selected_policy", {})) + + +def _policy_from_selected_champion( + selected: dict[str, Any], + *, + max_portfolio_pd: float, + policy_selector: str, +) -> dict[str, Any]: + return { + "source": f"champion_policy_artifact::{policy_selector}", + "risk_tolerance": float(selected.get("risk_tolerance", max_portfolio_pd)), + "uncertainty_aversion": float(selected.get("uncertainty_aversion", 0.0)), + "min_budget_utilization": float(selected.get("min_budget_utilization", 0.0)), + "pd_cap_slack_penalty": float(selected.get("pd_cap_slack_penalty", 0.0)), + "policy_mode": str(selected.get("policy_mode", "hard_worst_case")), + "gamma": float(selected.get("gamma", 1.0)), + "delta_cap_quantile": float(selected.get("delta_cap_quantile", 1.0)), + "tail_focus_quantile": float(selected.get("tail_focus_quantile", 1.0)), + } + + +def _resolve_champion_robust_policy( + *, + champion_path: Path, + max_portfolio_pd: float, + policy_selector: str, +) -> dict[str, Any] | None: + if not champion_path.exists(): + if policy_selector == "explicit_champion_only": + raise FileNotFoundError(f"Missing champion portfolio policy artifact: {champion_path}") + return None + try: + payload_raw = json.loads(champion_path.read_text(encoding="utf-8")) + selected = ( + _select_champion_policy(payload_raw, policy_selector) + if isinstance(payload_raw, dict) + else {} + ) + policy = _policy_from_selected_champion( + selected, + max_portfolio_pd=max_portfolio_pd, + policy_selector=policy_selector, + ) + except Exception as exc: + if policy_selector == "explicit_champion_only": + raise + logger.warning( + f"Could not parse champion portfolio policy ({champion_path}): {exc}. " + "Falling back to summary-based policy." + ) + return None + logger.info( + "Resolved robust policy from champion artifact: " + f"risk_tolerance={policy['risk_tolerance']:.4f}, " + f"policy_mode={policy['policy_mode']}, gamma={policy['gamma']:.2f}" + ) + return policy + + +def _load_robustness_summary(path: Path) -> pd.DataFrame | None: + if not path.exists(): + logger.warning(f"Robustness summary not found ({path}); using fallback robust policy.") + return None + try: + return pd.read_parquet(path) + except Exception as exc: + logger.warning(f"Could not read robustness summary ({path}): {exc}") + return None + + +def _valid_summary_rows(summary: pd.DataFrame, required_cols: set[str]) -> pd.DataFrame | None: + if summary.empty or not required_cols.issubset(set(summary.columns)): + missing = sorted(required_cols - set(summary.columns)) + logger.warning( + "Robustness summary missing required columns or empty; " + f"missing={missing}. Using fallback robust policy." + ) + return None + work = summary.copy() + for col in required_cols: + work[col] = pd.to_numeric(work[col], errors="coerce") + work = work.dropna(subset=list(required_cols)).reset_index(drop=True) + if work.empty: + logger.warning("No valid numeric robust summary rows; using fallback policy.") + return None + return work + + +def _best_summary_policy_row(work: pd.DataFrame, target: float) -> pd.Series: + lower_eq = work.loc[work["risk_tolerance"] <= target + 1e-12].copy() + candidate_pool = lower_eq if not lower_eq.empty else work + candidate_pool["_distance"] = (candidate_pool["risk_tolerance"] - target).abs() + if "best_robust_return" not in candidate_pool.columns: + return candidate_pool.sort_values(by=["_distance"], ascending=[True]).iloc[0] + candidate_pool["best_robust_return"] = pd.to_numeric( + candidate_pool["best_robust_return"], errors="coerce" + ).fillna(float("-inf")) + return candidate_pool.sort_values( + by=["_distance", "best_robust_return"], + ascending=[True, False], + ).iloc[0] + + +def _policy_from_summary_row(row: pd.Series) -> dict[str, Any]: + return { + "source": "portfolio_robustness_summary", + "risk_tolerance": float(row["risk_tolerance"]), + "uncertainty_aversion": float(row["best_robust_lambda"]), + "min_budget_utilization": float(row["best_robust_min_budget_utilization"]), + "pd_cap_slack_penalty": float(row["best_robust_pd_cap_slack_penalty"]), + "policy_mode": str(row.get("best_robust_policy_mode", "hard_worst_case")), + "gamma": float(row.get("best_robust_gamma", 1.0)), + "delta_cap_quantile": float(row.get("best_robust_delta_cap_quantile", 1.0)), + } + + +def _resolve_summary_robust_policy( + *, + path: Path, + max_portfolio_pd: float, +) -> dict[str, Any] | None: + summary = _load_robustness_summary(path) + if summary is None: + return None + + required_cols = { + "risk_tolerance", + "best_robust_lambda", + "best_robust_min_budget_utilization", + "best_robust_pd_cap_slack_penalty", + } + work = _valid_summary_rows(summary, required_cols) + if work is None: + return None + + policy = _policy_from_summary_row( + _best_summary_policy_row(work, target=float(max_portfolio_pd)) + ) + logger.info( + "Resolved robust policy from summary: " + f"risk_tolerance={policy['risk_tolerance']:.4f}, " + f"uncertainty_aversion={policy['uncertainty_aversion']:.4f}, " + f"min_budget_utilization={policy['min_budget_utilization']:.4f}, " + f"pd_cap_slack_penalty={policy['pd_cap_slack_penalty']:.4f}" + ) + return policy + + def _compute_realized_return( allocation: dict[int, float], loan_amnt: np.ndarray, @@ -90,141 +280,22 @@ def _resolve_robust_policy( champion_policy_path: str = "models/champion_portfolio_policy.json", ) -> dict[str, Any]: """Resolve robust strategy parameters from tradeoff summary, with fallback defaults.""" - default: dict[str, Any] = { - "source": "fallback_default", - "risk_tolerance": float(max_portfolio_pd), - "uncertainty_aversion": 0.0, - "min_budget_utilization": 0.0, - "pd_cap_slack_penalty": 0.0, - "policy_mode": "hard_worst_case", - "gamma": 1.0, - "delta_cap_quantile": 1.0, - "tail_focus_quantile": 1.0, - } champion_path = _artifact_path(champion_policy_path) - if champion_path.exists(): - try: - payload = json.loads(champion_path.read_text(encoding="utf-8")) - if isinstance(payload, dict): - if policy_selector == "robustness_aware": - selected = payload.get("selected_policy_robustness_aware") or payload.get( - "selected_policy", {} - ) - elif policy_selector == "balanced_robustness": - selected = ( - payload.get("selected_policy_balanced_robustness") - or payload.get("selected_policy_guardrail_robustness") - or payload.get("selected_policy_robustness_aware") - or payload.get("selected_policy", {}) - ) - elif policy_selector == "guardrail_robustness": - selected = ( - payload.get("selected_policy_guardrail_robustness") - or payload.get("selected_policy_balanced_robustness") - or payload.get("selected_policy_robustness_aware") - or payload.get("selected_policy", {}) - ) - elif policy_selector == "explicit_champion_only": - selected = payload.get("selected_policy", {}) - if not selected: - raise ValueError("Champion policy artifact missing selected_policy") - else: - selected = payload.get("selected_policy", {}) - else: - selected = {} - policy: dict[str, Any] = { - "source": f"champion_policy_artifact::{policy_selector}", - "risk_tolerance": float(selected.get("risk_tolerance", max_portfolio_pd)), - "uncertainty_aversion": float(selected.get("uncertainty_aversion", 0.0)), - "min_budget_utilization": float(selected.get("min_budget_utilization", 0.0)), - "pd_cap_slack_penalty": float(selected.get("pd_cap_slack_penalty", 0.0)), - "policy_mode": str(selected.get("policy_mode", "hard_worst_case")), - "gamma": float(selected.get("gamma", 1.0)), - "delta_cap_quantile": float(selected.get("delta_cap_quantile", 1.0)), - "tail_focus_quantile": float(selected.get("tail_focus_quantile", 1.0)), - } - logger.info( - "Resolved robust policy from champion artifact: " - f"risk_tolerance={policy['risk_tolerance']:.4f}, " - f"policy_mode={policy['policy_mode']}, gamma={policy['gamma']:.2f}" - ) - return policy - except Exception as exc: - if policy_selector == "explicit_champion_only": - raise - logger.warning( - f"Could not parse champion portfolio policy ({champion_path}): {exc}. " - "Falling back to summary-based policy." - ) - elif policy_selector == "explicit_champion_only": - raise FileNotFoundError(f"Missing champion portfolio policy artifact: {champion_path}") - - path = _artifact_path(summary_path) - if not path.exists(): - logger.warning(f"Robustness summary not found ({path}); using fallback robust policy.") - return default - - try: - summary = pd.read_parquet(path) - except Exception as exc: - logger.warning(f"Could not read robustness summary ({path}): {exc}") - return default - - required_cols = { - "risk_tolerance", - "best_robust_lambda", - "best_robust_min_budget_utilization", - "best_robust_pd_cap_slack_penalty", - } - if summary.empty or not required_cols.issubset(set(summary.columns)): - missing = sorted(required_cols - set(summary.columns)) - logger.warning( - "Robustness summary missing required columns or empty; " - f"missing={missing}. Using fallback robust policy." - ) - return default - - work = summary.copy() - for col in required_cols: - work[col] = pd.to_numeric(work[col], errors="coerce") - work = work.dropna(subset=list(required_cols)).reset_index(drop=True) - if work.empty: - logger.warning("No valid numeric robust summary rows; using fallback policy.") - return default - - target = float(max_portfolio_pd) - lower_eq = work.loc[work["risk_tolerance"] <= target + 1e-12].copy() - candidate_pool = lower_eq if not lower_eq.empty else work - candidate_pool["_distance"] = (candidate_pool["risk_tolerance"] - target).abs() - if "best_robust_return" in candidate_pool.columns: - candidate_pool["best_robust_return"] = pd.to_numeric( - candidate_pool["best_robust_return"], errors="coerce" - ).fillna(float("-inf")) - row = candidate_pool.sort_values( - by=["_distance", "best_robust_return"], - ascending=[True, False], - ).iloc[0] - else: - row = candidate_pool.sort_values(by=["_distance"], ascending=[True]).iloc[0] + champion_policy = _resolve_champion_robust_policy( + champion_path=champion_path, + max_portfolio_pd=max_portfolio_pd, + policy_selector=policy_selector, + ) + if champion_policy is not None: + return champion_policy - policy = { - "source": "portfolio_robustness_summary", - "risk_tolerance": float(row["risk_tolerance"]), - "uncertainty_aversion": float(row["best_robust_lambda"]), - "min_budget_utilization": float(row["best_robust_min_budget_utilization"]), - "pd_cap_slack_penalty": float(row["best_robust_pd_cap_slack_penalty"]), - "policy_mode": str(row.get("best_robust_policy_mode", "hard_worst_case")), - "gamma": float(row.get("best_robust_gamma", 1.0)), - "delta_cap_quantile": float(row.get("best_robust_delta_cap_quantile", 1.0)), - } - logger.info( - "Resolved robust policy from summary: " - f"risk_tolerance={policy['risk_tolerance']:.4f}, " - f"uncertainty_aversion={policy['uncertainty_aversion']:.4f}, " - f"min_budget_utilization={policy['min_budget_utilization']:.4f}, " - f"pd_cap_slack_penalty={policy['pd_cap_slack_penalty']:.4f}" + summary_policy = _resolve_summary_robust_policy( + path=_artifact_path(summary_path), + max_portfolio_pd=max_portfolio_pd, ) - return policy + if summary_policy is not None: + return summary_policy + return _default_robust_policy(max_portfolio_pd) def _apply_candidate_universe( diff --git a/scripts/train_pd_model.py b/scripts/train_pd_model.py index c133740..8dc23ae 100644 --- a/scripts/train_pd_model.py +++ b/scripts/train_pd_model.py @@ -10,6 +10,7 @@ import json import pickle import shutil +from collections.abc import Mapping from pathlib import Path from typing import Any @@ -70,6 +71,58 @@ pd.DataFrame, pd.Series, ] +REQUIRED_CONFIG_SECTIONS = ("output", "feature_source", "data", "hpo", "validation") +REQUIRED_DATA_KEYS = ("train_path", "test_path", "calibration_path") +OPTIONAL_MAPPING_SECTIONS = ( + "output", + "feature_source", + "data", + "hpo", + "validation", + "model", + "conformal", + "calibration", +) +OUTPUT_DEFAULTS = { + "model_path": "models/pd_canonical.cbm", + "default_model_path": "models/pd_catboost_default.cbm", + "tuned_model_path": "models/pd_canonical.cbm", + "conformal_path": "models/pd_canonical_calibrator.pkl", + "status_path": "models/pd_training_status.json", + "checkpoint_dir": "models/pd_training_checkpoints", + "brier_decomposition_path": "data/processed/brier_score_decomposition.json", + "murphy_diagram_path": "reports/figures/calibration/murphy_diagram.png", + "canonical_model_path": str(CANONICAL_MODEL_PATH), + "canonical_calibrator_path": str(CANONICAL_CALIBRATOR_PATH), + "contract_path": str(CONTRACT_PATH), + "logreg_model_path": "models/pd_logreg_baseline.pkl", + "training_record_path": "models/pd_training_record.pkl", + "seed_replay_status_path": "models/pd_hpo_seed_replay_status.json", + "test_predictions_path": "data/processed/test_predictions.parquet", + "shap_dir": "reports/figures/shap", + "threshold_semantics_path": "models/threshold_semantics.json", +} + + +def _metric_float(metrics: Mapping[str, Any], key: str, default: float = 0.0) -> float: + value = metrics.get(key, default) + if value is None or isinstance(value, Mapping): + return float(default) + return float(value) + + +def _metric_int(metrics: Mapping[str, Any], key: str, default: int = 0) -> int: + value = metrics.get(key, default) + if value is None or isinstance(value, Mapping): + return int(default) + return int(value) + + +def _metric_mapping( + metrics: Mapping[str, Any], key: str, default: Mapping[str, Any] +) -> dict[str, Any]: + value = metrics.get(key, default) + return dict(value) if isinstance(value, Mapping) else dict(default) def load_config(config_path: str) -> dict[str, Any]: @@ -77,112 +130,93 @@ def load_config(config_path: str) -> dict[str, Any]: return yaml.safe_load(f) +def _missing_mapping_keys(config: Mapping[str, Any], keys: tuple[str, ...]) -> list[str]: + return [key for key in keys if key not in config] + + +def _missing_required_data_keys(config: Mapping[str, Any]) -> list[str]: + data_cfg = config.get("data", {}) or {} + return [key for key in REQUIRED_DATA_KEYS if not str(data_cfg.get(key, "")).strip()] + + +def _normalize_config_sections(config: Mapping[str, Any]) -> dict[str, Any]: + normalized = dict(config) + for section in OPTIONAL_MAPPING_SECTIONS: + normalized[section] = dict(config.get(section, {}) or {}) + return normalized + + +def _apply_pd_config_defaults(config: dict[str, Any]) -> dict[str, Any]: + normalized = dict(config) + output_cfg = dict(normalized.get("output", {}) or {}) + for key, value in OUTPUT_DEFAULTS.items(): + output_cfg.setdefault(key, value) + normalized["output"] = output_cfg + feature_source_cfg = dict(normalized.get("feature_source", {}) or {}) + feature_source_cfg.setdefault("mode", "auto") + feature_source_cfg.setdefault("feature_config_path", "data/processed/feature_config.yml") + normalized["feature_source"] = feature_source_cfg + return normalized + + def validate_pd_config(config: dict[str, Any], *, config_path: str) -> dict[str, Any]: """Fail fast on missing PD config sections and required keys.""" if not isinstance(config, dict): raise ValueError(f"PD config must be a mapping: {config_path}") - required_sections = ("output", "feature_source", "data", "hpo", "validation") - missing_sections = [section for section in required_sections if section not in config] + missing_sections = _missing_mapping_keys(config, REQUIRED_CONFIG_SECTIONS) if missing_sections: raise ValueError( f"PD config {config_path} missing required sections: {', '.join(missing_sections)}" ) - required_data_keys = ("train_path", "test_path", "calibration_path") - missing_data_keys = [ - key - for key in required_data_keys - if not str((config.get("data", {}) or {}).get(key, "")).strip() - ] + missing_data_keys = _missing_required_data_keys(config) if missing_data_keys: raise ValueError( f"PD config {config_path} missing required data keys: {', '.join(missing_data_keys)}" ) - normalized = dict(config) - normalized["output"] = dict(config.get("output", {}) or {}) - normalized["feature_source"] = dict(config.get("feature_source", {}) or {}) - normalized["data"] = dict(config.get("data", {}) or {}) - normalized["hpo"] = dict(config.get("hpo", {}) or {}) - normalized["validation"] = dict(config.get("validation", {}) or {}) - normalized["model"] = dict(config.get("model", {}) or {}) - normalized["conformal"] = dict(config.get("conformal", {}) or {}) - normalized["calibration"] = dict(config.get("calibration", {}) or {}) - - output_defaults = { - "model_path": "models/pd_canonical.cbm", - "default_model_path": "models/pd_catboost_default.cbm", - "tuned_model_path": "models/pd_canonical.cbm", - "conformal_path": "models/pd_canonical_calibrator.pkl", - "status_path": "models/pd_training_status.json", - "checkpoint_dir": "models/pd_training_checkpoints", - "brier_decomposition_path": "data/processed/brier_score_decomposition.json", - "murphy_diagram_path": "reports/figures/calibration/murphy_diagram.png", - "canonical_model_path": str(CANONICAL_MODEL_PATH), - "canonical_calibrator_path": str(CANONICAL_CALIBRATOR_PATH), - "contract_path": str(CONTRACT_PATH), - "logreg_model_path": "models/pd_logreg_baseline.pkl", - "training_record_path": "models/pd_training_record.pkl", - "seed_replay_status_path": "models/pd_hpo_seed_replay_status.json", - "test_predictions_path": "data/processed/test_predictions.parquet", - "shap_dir": "reports/figures/shap", - "threshold_semantics_path": "models/threshold_semantics.json", - } - for key, value in output_defaults.items(): - normalized["output"].setdefault(key, value) - - normalized["feature_source"].setdefault("mode", "auto") - normalized["feature_source"].setdefault( - "feature_config_path", "data/processed/feature_config.yml" - ) - return normalized + return _apply_pd_config_defaults(_normalize_config_sections(config)) -def _apply_cli_overrides( - config: dict[str, Any], +def _override_training_regime( + config: Mapping[str, Any], *, - training_regime_mode: str | None = None, - recent_window_quarters: int | None = None, - half_life_quarters: int | None = None, - stable_core_enabled: bool | None = None, - hpo_n_trials: int | None = None, - hpo_enabled: bool | None = None, - challenger_enabled: bool | None = None, - walk_forward_enabled: bool | None = None, - seed_replay_enabled: bool | None = None, - catboost_iterations: int | None = None, + training_regime_mode: str | None, + recent_window_quarters: int | None, + half_life_quarters: int | None, ) -> dict[str, Any]: - """Return a config copy with command-line overrides applied.""" - updated = dict(config) - - regime_cfg = dict(updated.get("training_regime", {}) or {}) + regime_cfg = dict(config.get("training_regime", {}) or {}) if training_regime_mode is not None: regime_cfg["mode"] = str(training_regime_mode) if recent_window_quarters is not None: regime_cfg["recent_window_quarters"] = int(recent_window_quarters) if half_life_quarters is not None: regime_cfg["half_life_quarters"] = int(half_life_quarters) - updated["training_regime"] = regime_cfg + return regime_cfg - stable_core_cfg = dict(updated.get("stable_core", {}) or {}) - if stable_core_enabled is not None: - stable_core_cfg["enabled"] = bool(stable_core_enabled) - updated["stable_core"] = stable_core_cfg - hpo_cfg = dict(updated.get("hpo", {}) or {}) +def _override_hpo_config( + config: Mapping[str, Any], + *, + hpo_n_trials: int | None, + hpo_enabled: bool | None, +) -> dict[str, Any]: + hpo_cfg = dict(config.get("hpo", {}) or {}) if hpo_n_trials is not None: hpo_cfg["n_trials"] = int(hpo_n_trials) if hpo_enabled is not None: hpo_cfg["enabled"] = bool(hpo_enabled) - updated["hpo"] = hpo_cfg + return hpo_cfg - challenger_cfg = dict(updated.get("challenger_pipeline", {}) or {}) - if challenger_enabled is not None: - challenger_cfg["enabled"] = bool(challenger_enabled) - updated["challenger_pipeline"] = challenger_cfg - validation_cfg = dict(updated.get("validation", {}) or {}) +def _override_validation_config( + config: Mapping[str, Any], + *, + walk_forward_enabled: bool | None, + seed_replay_enabled: bool | None, +) -> dict[str, Any]: + validation_cfg = dict(config.get("validation", {}) or {}) walk_cfg = dict(validation_cfg.get("walk_forward", {}) or {}) if walk_forward_enabled is not None: walk_cfg["enabled"] = bool(walk_forward_enabled) @@ -191,14 +225,67 @@ def _apply_cli_overrides( if seed_replay_enabled is not None: seed_cfg["enabled"] = bool(seed_replay_enabled) validation_cfg["seed_replay"] = seed_cfg - updated["validation"] = validation_cfg + return validation_cfg - model_cfg = dict(updated.get("model", {}) or {}) + +def _override_model_config( + config: Mapping[str, Any], + *, + catboost_iterations: int | None, +) -> dict[str, Any]: + model_cfg = dict(config.get("model", {}) or {}) model_params = dict(model_cfg.get("params", {}) or {}) if catboost_iterations is not None: model_params["iterations"] = int(catboost_iterations) model_cfg["params"] = model_params - updated["model"] = model_cfg + return model_cfg + + +def _apply_cli_overrides( + config: dict[str, Any], + *, + training_regime_mode: str | None = None, + recent_window_quarters: int | None = None, + half_life_quarters: int | None = None, + stable_core_enabled: bool | None = None, + hpo_n_trials: int | None = None, + hpo_enabled: bool | None = None, + challenger_enabled: bool | None = None, + walk_forward_enabled: bool | None = None, + seed_replay_enabled: bool | None = None, + catboost_iterations: int | None = None, +) -> dict[str, Any]: + """Return a config copy with command-line overrides applied.""" + updated = dict(config) + updated["training_regime"] = _override_training_regime( + updated, + training_regime_mode=training_regime_mode, + recent_window_quarters=recent_window_quarters, + half_life_quarters=half_life_quarters, + ) + + stable_core_cfg = dict(updated.get("stable_core", {}) or {}) + if stable_core_enabled is not None: + stable_core_cfg["enabled"] = bool(stable_core_enabled) + updated["stable_core"] = stable_core_cfg + + updated["hpo"] = _override_hpo_config( + updated, + hpo_n_trials=hpo_n_trials, + hpo_enabled=hpo_enabled, + ) + + challenger_cfg = dict(updated.get("challenger_pipeline", {}) or {}) + if challenger_enabled is not None: + challenger_cfg["enabled"] = bool(challenger_enabled) + updated["challenger_pipeline"] = challenger_cfg + + updated["validation"] = _override_validation_config( + updated, + walk_forward_enabled=walk_forward_enabled, + seed_replay_enabled=seed_replay_enabled, + ) + updated["model"] = _override_model_config(updated, catboost_iterations=catboost_iterations) return updated @@ -278,22 +365,24 @@ def _metric_with_aliases(payload: dict[str, Any], *keys: str) -> float | None: return None -def _validate_replay_expectations( - *, - replay_cfg: dict[str, Any], - final_test_metrics: dict[str, Any], - feature_names: list[str], - config_path: str, -) -> None: - expected = dict(replay_cfg.get("expectations") or {}) - tolerances = dict(replay_cfg.get("tolerances") or {}) +def _validate_replay_feature_order(replay_cfg: Mapping[str, Any], feature_names: list[str]) -> None: expected_features = [str(x) for x in replay_cfg.get("feature_names", [])] if expected_features and list(feature_names) != expected_features: raise ValueError("Replay feature order mismatch against frozen manifest.") + + +def _validate_replay_config_path(replay_cfg: Mapping[str, Any], config_path: str) -> None: manifest_config_path = str(replay_cfg.get("config_path", "")).strip() if manifest_config_path and manifest_config_path != str(config_path): raise ValueError("Replay config_path does not match frozen manifest.") + +def _replay_metric_violations( + *, + expected: dict[str, Any], + tolerances: dict[str, Any], + observed_metrics: dict[str, Any], +) -> list[str]: aliases = { "auc_roc": ("auc_roc",), "brier_score": ("brier_score",), @@ -304,13 +393,32 @@ def _validate_replay_expectations( for name, key_aliases in aliases.items(): expected_value = _metric_with_aliases(expected, name, *key_aliases) tolerance = _metric_with_aliases(tolerances, name, *key_aliases) - observed = _metric_with_aliases(final_test_metrics, name, *key_aliases) + observed = _metric_with_aliases(observed_metrics, name, *key_aliases) if expected_value is None or tolerance is None or observed is None: continue if abs(observed - expected_value) > tolerance: violations.append( f"{name}: observed={observed:.6f} expected={expected_value:.6f} tol={tolerance:.6f}" ) + return violations + + +def _validate_replay_expectations( + *, + replay_cfg: dict[str, Any], + final_test_metrics: dict[str, Any], + feature_names: list[str], + config_path: str, +) -> None: + expected = dict(replay_cfg.get("expectations") or {}) + tolerances = dict(replay_cfg.get("tolerances") or {}) + _validate_replay_feature_order(replay_cfg, feature_names) + _validate_replay_config_path(replay_cfg, config_path) + violations = _replay_metric_violations( + expected=expected, + tolerances=tolerances, + observed_metrics=final_test_metrics, + ) if violations: raise ValueError("Replay metric validation failed: " + "; ".join(violations)) @@ -439,6 +547,61 @@ def _apply_stable_core( ) +def _feature_source_config(config: dict[str, Any]) -> tuple[str, str | Path]: + feature_src_cfg = dict(config.get("feature_source", {}) or {}) + feature_mode = str(feature_src_cfg.get("mode", "auto")) + feature_config_path = feature_src_cfg.get( + "feature_config_path", "data/processed/feature_config.yml" + ) + return feature_mode, feature_config_path + + +def _resolved_feature_lists( + feature_sets: Mapping[str, Any], +) -> tuple[list[str], list[str], list[str]]: + return ( + [str(x) for x in feature_sets["catboost_features"]], + [str(x) for x in feature_sets["logreg_features"]], + [str(x) for x in feature_sets["categorical_features"]], + ) + + +def _apply_replay_feature_override( + *, + catboost_features: list[str], + categorical_features: list[str], + run_mode: str, + replay_cfg: dict[str, Any], +) -> tuple[list[str], list[str]]: + if run_mode != "replay": + return catboost_features, categorical_features + replay_features = [str(x) for x in replay_cfg.get("feature_names", [])] + if not replay_features: + return catboost_features, categorical_features + replay_categorical = [str(x) for x in replay_cfg.get("categorical_features", [])] + return replay_features, replay_categorical + + +def _features_in_all_splits( + features: list[str], + train: pd.DataFrame, + cal: pd.DataFrame, + test: pd.DataFrame, +) -> list[str]: + return [c for c in features if c in train.columns and c in cal.columns and c in test.columns] + + +def _validate_resolved_training_features( + *, + catboost_features: list[str], + logreg_features: list[str], +) -> None: + if not catboost_features: + raise ValueError("No CatBoost features resolved across train/cal/test splits.") + if not logreg_features: + raise ValueError("No Logistic Regression features resolved across train/cal/test splits.") + + def _resolve_training_features( *, config: dict[str, Any], @@ -448,36 +611,21 @@ def _resolve_training_features( run_mode: str, replay_cfg: dict[str, Any], ) -> ResolvedFeatureTuple: - feature_src_cfg = dict(config.get("feature_source", {}) or {}) - feature_mode = str(feature_src_cfg.get("mode", "auto")) - feature_config_path = feature_src_cfg.get( - "feature_config_path", "data/processed/feature_config.yml" - ) - + feature_mode, feature_config_path = _feature_source_config(config) feature_sets = resolve_feature_sets( train, feature_source=feature_mode, feature_config_path=feature_config_path, ) - catboost_features = list(feature_sets["catboost_features"]) - logreg_features = list(feature_sets["logreg_features"]) - categorical_features = list(feature_sets["categorical_features"]) - - if run_mode == "replay": - replay_features = [str(x) for x in replay_cfg.get("feature_names", [])] - replay_categorical = [str(x) for x in replay_cfg.get("categorical_features", [])] - if replay_features: - catboost_features = replay_features - categorical_features = replay_categorical - - catboost_features = [ - c - for c in catboost_features - if c in train.columns and c in cal.columns and c in test.columns - ] - logreg_features = [ - c for c in logreg_features if c in train.columns and c in cal.columns and c in test.columns - ] + catboost_features, logreg_features, categorical_features = _resolved_feature_lists(feature_sets) + catboost_features, categorical_features = _apply_replay_feature_override( + catboost_features=catboost_features, + categorical_features=categorical_features, + run_mode=run_mode, + replay_cfg=replay_cfg, + ) + catboost_features = _features_in_all_splits(catboost_features, train, cal, test) + logreg_features = _features_in_all_splits(logreg_features, train, cal, test) categorical_features = [c for c in categorical_features if c in catboost_features] catboost_features, categorical_features, stable_core_meta = _apply_stable_core( @@ -486,11 +634,10 @@ def _resolve_training_features( {} if run_mode == "replay" else (config.get("stable_core", {}) or {}), ) logreg_features = [c for c in logreg_features if c in catboost_features] - - if not catboost_features: - raise ValueError("No CatBoost features resolved across train/cal/test splits.") - if not logreg_features: - raise ValueError("No Logistic Regression features resolved across train/cal/test splits.") + _validate_resolved_training_features( + catboost_features=catboost_features, + logreg_features=logreg_features, + ) return ( str(feature_sets.get("feature_source", feature_mode)), @@ -797,6 +944,132 @@ def _fairness_row( } +def _decision_threshold_metadata(run_tag: str) -> dict[str, Any]: + return build_artifact_metadata( + schema_version="2026-03-01.1", + run_tag=run_tag, + require_explicit=True, + ) + + +def _fallback_decision_threshold_artifact( + *, + config: dict[str, Any], + run_tag: str, +) -> dict[str, Any]: + return { + "enabled": False, + "selected_threshold": float(config.get("calibration", {}).get("default_threshold", 0.5)), + "source": "fallback_default", + **_decision_threshold_metadata(run_tag), + } + + +def _replay_decision_threshold_artifact( + *, + replay_cfg: dict[str, Any], + run_tag: str, +) -> dict[str, Any]: + artifact = dict(replay_cfg["decision_threshold_artifact"]) + artifact.update(_decision_threshold_metadata(run_tag)) + artifact["source"] = "frozen_replay_manifest" + return artifact + + +def _load_fairness_threshold_policy(fairness_policy_path: str) -> tuple[dict[str, Any], list[Any]]: + with open(fairness_policy_path, encoding="utf-8") as f: + fairness_cfg = yaml.safe_load(f) or {} + return dict(fairness_cfg.get("policy", {}) or {}), list( + fairness_cfg.get("attributes", []) or [] + ) + + +def _threshold_grid(decision_cfg: dict[str, Any]) -> np.ndarray: + threshold_min = float(decision_cfg.get("min_threshold", 0.05)) + threshold_max = float(decision_cfg.get("max_threshold", 0.95)) + threshold_step = float(decision_cfg.get("step", 0.01)) + return np.arange(threshold_min, threshold_max + (threshold_step / 2.0), threshold_step) + + +def _search_decision_threshold_artifact( + *, + decision_cfg: dict[str, Any], + train_val: pd.DataFrame, + cal: pd.DataFrame, + y_val: pd.Series, + y_prob_final_val: np.ndarray, + y_cal: pd.Series, + y_prob_tuned_cal: np.ndarray, + selected_cal_method: str, + run_tag: str, +) -> dict[str, Any]: + fairness_policy_path = str( + decision_cfg.get("fairness_policy_path", "configs/fairness_policy.yaml") + ) + fairness_policy, fairness_attrs = _load_fairness_threshold_policy(fairness_policy_path) + fallback_threshold = float(fairness_policy.get("prediction_threshold", 0.5)) + threshold_result = _select_decision_threshold( + y_true=y_val.to_numpy(), + y_prob=y_prob_final_val, + policy={ + "dpd_threshold": float(fairness_policy.get("dpd_threshold", 0.10)), + "eo_gap_threshold": float(fairness_policy.get("eo_gap_threshold", 0.10)), + "dir_threshold": float(fairness_policy.get("dir_threshold", 0.80)), + }, + groups_dict=_build_fairness_groups_for_threshold(train_val, fairness_attrs), + thresholds=_threshold_grid(decision_cfg), + fallback_threshold=fallback_threshold, + y_true_secondary=y_cal.to_numpy(), + y_prob_secondary=y_prob_tuned_cal, + groups_dict_secondary=_build_fairness_groups_for_threshold(cal, fairness_attrs), + ) + return { + "enabled": True, + "selected_threshold": float(threshold_result["selected_threshold"]), + "fallback_threshold": fallback_threshold, + "selection_metrics": threshold_result["selection_metrics"], + "search_summary": threshold_result["search_summary"], + "source": "validation_fairness_search", + "fairness_policy_path": fairness_policy_path, + "validation_rows": len(train_val), + "secondary_validation_rows": len(cal), + "calibration_method": selected_cal_method, + **_decision_threshold_metadata(run_tag), + } + + +def _resolve_decision_threshold_artifact( + *, + config: dict[str, Any], + run_mode: str, + replay_cfg: dict[str, Any], + decision_cfg: dict[str, Any], + train_val: pd.DataFrame, + cal: pd.DataFrame, + y_val: pd.Series, + y_prob_final_val: np.ndarray, + y_cal: pd.Series, + y_prob_tuned_cal: np.ndarray, + selected_cal_method: str, + run_tag: str, +) -> dict[str, Any]: + if run_mode == "replay" and replay_cfg.get("decision_threshold_artifact"): + return _replay_decision_threshold_artifact(replay_cfg=replay_cfg, run_tag=run_tag) + if bool(decision_cfg.get("enabled", True)): + return _search_decision_threshold_artifact( + decision_cfg=decision_cfg, + train_val=train_val, + cal=cal, + y_val=y_val, + y_prob_final_val=y_prob_final_val, + y_cal=y_cal, + y_prob_tuned_cal=y_prob_tuned_cal, + selected_cal_method=selected_cal_method, + run_tag=run_tag, + ) + return _fallback_decision_threshold_artifact(config=config, run_tag=run_tag) + + def _build_calibration_backtest_splits( cal_df: pd.DataFrame, n_folds: int = 4, @@ -829,73 +1102,68 @@ def _build_calibration_backtest_splits( return splits -def _evaluate_calibration_method( +def _empty_calibration_method_report(method: str) -> dict[str, Any]: + return { + "method": method, + "folds_used": 0, + "mean_brier": float("inf"), + "mean_log_loss": float("inf"), + "mean_ece": float("inf"), + "mean_auc_drop": float("inf"), + "brier_variance": float("inf"), + "ece_variance": float("inf"), + "stability": float("inf"), + "degradation_rate": 1.0, + "folds": [], + } + + +def _evaluate_calibration_fold( + *, method: str, + fold_id: int, + idx_fit: np.ndarray, + idx_eval: np.ndarray, y_true: np.ndarray, y_prob_raw: np.ndarray, - splits: list[tuple[np.ndarray, np.ndarray]], -) -> dict[str, Any]: - """Backtest calibrator over temporal folds using multi-metric summary.""" - fold_rows: list[dict[str, Any]] = [] - - for fold_id, (idx_fit, idx_eval) in enumerate(splits, start=1): - y_fit = y_true[idx_fit] - y_eval = y_true[idx_eval] - p_fit = y_prob_raw[idx_fit] - p_eval = y_prob_raw[idx_eval] +) -> dict[str, Any] | None: + y_fit = y_true[idx_fit] + y_eval = y_true[idx_eval] + p_fit = y_prob_raw[idx_fit] + p_eval = y_prob_raw[idx_eval] + if len(np.unique(y_fit)) < 2 or len(np.unique(y_eval)) < 2: + return None - if len(np.unique(y_fit)) < 2 or len(np.unique(y_eval)) < 2: - continue + calibrator = _fit_calibrator_from_scores(method, y_fit, p_fit) + p_eval_cal = _apply_calibrator(calibrator, p_eval) + raw_auc = _safe_auc(y_eval, p_eval) + cal_auc = _safe_auc(y_eval, p_eval_cal) + auc_drop = 0.0 if raw_auc is None or cal_auc is None else float(raw_auc - cal_auc) + brier_raw = float(brier_score_loss(y_eval, p_eval)) + brier_cal = float(brier_score_loss(y_eval, p_eval_cal)) + return { + "fold": fold_id, + "n_fit": len(idx_fit), + "n_eval": len(idx_eval), + "raw_auc": None if raw_auc is None else float(raw_auc), + "cal_auc": None if cal_auc is None else float(cal_auc), + "auc_drop": float(auc_drop), + "brier": brier_cal, + "brier_raw": brier_raw, + "brier_degraded": brier_cal > brier_raw, + "log_loss": float(log_loss(y_eval, p_eval_cal)), + "ece": float(expected_calibration_error(y_eval, p_eval_cal)), + } - calibrator = _fit_calibrator_from_scores(method, y_fit, p_fit) - p_eval_cal = _apply_calibrator(calibrator, p_eval) - - raw_auc = _safe_auc(y_eval, p_eval) - cal_auc = _safe_auc(y_eval, p_eval_cal) - auc_drop = 0.0 - if raw_auc is not None and cal_auc is not None: - auc_drop = float(raw_auc - cal_auc) - - brier_raw = float(brier_score_loss(y_eval, p_eval)) - brier_cal = float(brier_score_loss(y_eval, p_eval_cal)) - - fold_rows.append( - { - "fold": fold_id, - "n_fit": len(idx_fit), - "n_eval": len(idx_eval), - "raw_auc": None if raw_auc is None else float(raw_auc), - "cal_auc": None if cal_auc is None else float(cal_auc), - "auc_drop": float(auc_drop), - "brier": brier_cal, - "brier_raw": brier_raw, - "brier_degraded": brier_cal > brier_raw, - "log_loss": float(log_loss(y_eval, p_eval_cal)), - "ece": float(expected_calibration_error(y_eval, p_eval_cal)), - } - ) +def _summarize_calibration_folds(method: str, fold_rows: list[dict[str, Any]]) -> dict[str, Any]: if not fold_rows: - return { - "method": method, - "folds_used": 0, - "mean_brier": float("inf"), - "mean_log_loss": float("inf"), - "mean_ece": float("inf"), - "mean_auc_drop": float("inf"), - "brier_variance": float("inf"), - "ece_variance": float("inf"), - "stability": float("inf"), - "degradation_rate": 1.0, - "folds": [], - } - - briers = np.array([r["brier"] for r in fold_rows], dtype=float) - log_losses = np.array([r["log_loss"] for r in fold_rows], dtype=float) - eces = np.array([r["ece"] for r in fold_rows], dtype=float) - auc_drops = np.array([r["auc_drop"] for r in fold_rows], dtype=float) - n_degraded = sum(1 for r in fold_rows if r.get("brier_degraded", False)) - + return _empty_calibration_method_report(method) + briers = np.array([row["brier"] for row in fold_rows], dtype=float) + log_losses = np.array([row["log_loss"] for row in fold_rows], dtype=float) + eces = np.array([row["ece"] for row in fold_rows], dtype=float) + auc_drops = np.array([row["auc_drop"] for row in fold_rows], dtype=float) + n_degraded = sum(1 for row in fold_rows if row.get("brier_degraded", False)) return { "method": method, "folds_used": len(fold_rows), @@ -911,6 +1179,121 @@ def _evaluate_calibration_method( } +def _write_calibration_cumulative_diffs( + statistical_cal_tests: dict[str, object], + *, + n_rows: int, +) -> None: + if "_cum_diff_calibrated" not in statistical_cal_tests: + return + k_idx = np.arange(n_rows) / n_rows + sigma_raw = statistical_cal_tests.get("_sigma", 0.0) + sigma_val = float(sigma_raw) if isinstance(sigma_raw, str | int | float | np.number) else 0.0 + cum_diff_df = pd.DataFrame( + { + "k": k_idx, + "cum_diff_calibrated": statistical_cal_tests.pop("_cum_diff_calibrated"), + "cum_diff_uncalibrated": statistical_cal_tests.pop( + "_cum_diff_uncalibrated", + [float("nan")] * len(k_idx), + ), + "sigma_upper": sigma_val * 2, + "sigma_lower": -sigma_val * 2, + } + ) + cum_diff_path = _artifact_path("data/processed/calibration_cumulative_diffs.parquet") + cum_diff_path.parent.mkdir(parents=True, exist_ok=True) + cum_diff_df.to_parquet(cum_diff_path, index=False) + logger.info(f"Saved calibration cumulative diffs: {cum_diff_path}") + statistical_cal_tests.pop("_sigma", None) + + +def _write_statistical_calibration_json(statistical_cal_tests: dict[str, object]) -> None: + stat_cal_path = _artifact_path("data/processed/statistical_calibration_tests.json") + stat_cal_path.parent.mkdir(parents=True, exist_ok=True) + stat_cal_path.write_text( + json.dumps(statistical_cal_tests, indent=2, default=str), + encoding="utf-8", + ) + logger.info(f"Saved statistical calibration tests: {stat_cal_path}") + + +def _run_statistical_calibration_tests( + *, + y_test: pd.Series, + y_prob_final: np.ndarray, + y_prob_tuned_test: np.ndarray, +) -> None: + try: + from mapie.metrics.calibration import ( + cumulative_differences, + kolmogorov_smirnov_p_value, + kuiper_p_value, + length_scale, + spiegelhalter_p_value, + ) + + y_test_arr = y_test.values.astype(float) + statistical_cal_tests: dict[str, object] = {} + for tag, probs in [("calibrated", y_prob_final), ("uncalibrated", y_prob_tuned_test)]: + try: + ks_p = float(kolmogorov_smirnov_p_value(y_test_arr, probs)) + ku_p = float(kuiper_p_value(y_test_arr, probs)) + sp_p = float(spiegelhalter_p_value(y_test_arr, probs)) + cum_diff = cumulative_differences(y_test_arr, probs) + sigma = float(length_scale(probs)) + statistical_cal_tests[tag] = { + "ks_pvalue": ks_p, + "kuiper_pvalue": ku_p, + "spiegelhalter_pvalue": sp_p, + "length_scale_sigma": sigma, + "n": len(y_test_arr), + } + logger.info( + f"Calibration tests [{tag}]: KS_p={ks_p:.4f} " + f"Kuiper_p={ku_p:.4f} Spiegelhalter_p={sp_p:.4f}" + ) + if tag == "calibrated": + statistical_cal_tests["_cum_diff_calibrated"] = cum_diff.tolist() + statistical_cal_tests["_sigma"] = sigma + elif tag == "uncalibrated": + statistical_cal_tests["_cum_diff_uncalibrated"] = cum_diff.tolist() + except Exception as exc_inner: + logger.warning(f"Statistical calibration tests [{tag}] failed: {exc_inner}") + statistical_cal_tests[tag] = {"error": str(exc_inner)} + + _write_calibration_cumulative_diffs(statistical_cal_tests, n_rows=len(y_test_arr)) + _write_statistical_calibration_json(statistical_cal_tests) + except ImportError: + logger.warning( + "mapie.metrics.calibration not available - statistical calibration tests skipped." + ) + except Exception as exc: + logger.warning(f"Statistical calibration tests block failed: {exc}") + + +def _evaluate_calibration_method( + method: str, + y_true: np.ndarray, + y_prob_raw: np.ndarray, + splits: list[tuple[np.ndarray, np.ndarray]], +) -> dict[str, Any]: + """Backtest calibrator over temporal folds using multi-metric summary.""" + fold_rows: list[dict[str, Any]] = [] + for fold_id, (idx_fit, idx_eval) in enumerate(splits, start=1): + fold_row = _evaluate_calibration_fold( + method=method, + fold_id=fold_id, + idx_fit=idx_fit, + idx_eval=idx_eval, + y_true=y_true, + y_prob_raw=y_prob_raw, + ) + if fold_row is not None: + fold_rows.append(fold_row) + return _summarize_calibration_folds(method, fold_rows) + + def _select_calibration_method( reports: list[dict[str, Any]], auc_drop_limit: float = 0.0015, @@ -1177,181 +1560,636 @@ def _evaluate_walk_forward_auc( } -def _replay_top_optuna_trials( +def _walk_forward_disabled_report(walk_cfg: Mapping[str, Any]) -> dict[str, Any]: + return { + "enabled": False, + "reason": "disabled_in_config", + "n_windows_requested": int(walk_cfg.get("n_windows", 0)), + "n_windows_used": 0, + "folds": [], + } + + +def _evaluate_walk_forward_stage( *, - hpo_cfg: dict[str, Any], - base_params: dict[str, Any], - X_train_fit_cb: pd.DataFrame, - y_train_fit: pd.Series, - X_val_cb: pd.DataFrame, - y_val: pd.Series, - cat_features: list[str], - seeds: list[int], - top_k_trials: int = 3, - prioritize_gate_pass: bool = True, - sample_weight: np.ndarray | None = None, - eval_sample_weight: np.ndarray | None = None, + enabled: bool, + walk_cfg: Mapping[str, Any], + train: pd.DataFrame, + catboost_features: list[str], + categorical_features: list[str], + model_params: Mapping[str, Any], ) -> dict[str, Any]: - """Replay top Optuna trials across multiple seeds for robustness.""" - report: dict[str, Any] = { + if not enabled: + return _walk_forward_disabled_report(walk_cfg) + max_rows = int(walk_cfg.get("max_rows", 0)) + return _evaluate_walk_forward_auc( + train, + features=catboost_features, + categorical_features=categorical_features, + target=TARGET, + params=dict(model_params), + n_windows=int(walk_cfg.get("n_windows", 3)), + min_train_rows=int(walk_cfg.get("min_train_rows", 200_000)), + window_rows=int(walk_cfg.get("window_rows", 80_000)), + date_col=str(walk_cfg.get("date_col", "issue_d")), + max_rows=None if max_rows <= 0 else max_rows, + ) + + +def _export_shap_feature_importance( + *, + cb_tuned_model: Any, + X_test_cb: pd.DataFrame, + categorical_features: list[str], + catboost_features: list[str], + shap_dir: Path, +) -> dict[str, Any]: + shap_artifact: dict[str, Any] = {"exported": False} + try: + from catboost import Pool as _SHAPPool + + shap_pool = _SHAPPool(X_test_cb, cat_features=categorical_features) + shap_raw = cb_tuned_model.get_feature_importance(type="ShapValues", data=shap_pool) + # ShapValues returns (n_samples, n_features + 1); last col = expected value. + shap_values = shap_raw[:, :-1] + shap_expected = float(shap_raw[0, -1]) + + mean_abs_shap = np.abs(shap_values).mean(axis=0) + shap_importance = sorted( + zip(catboost_features, mean_abs_shap.tolist(), strict=False), + key=lambda x: x[1], + reverse=True, + ) + shap_dir.mkdir(parents=True, exist_ok=True) + + np.savez_compressed( + str(shap_dir / "shap_values_test.npz"), + shap_values=shap_values, + expected_value=np.array([shap_expected]), + feature_names=np.array(catboost_features), + ) + + top_n = min(20, len(shap_importance)) + shap_summary = { + "expected_value": shap_expected, + "n_samples": int(shap_values.shape[0]), + "n_features": int(shap_values.shape[1]), + "top_features": [ + {"feature": f, "mean_abs_shap": round(v, 6)} for f, v in shap_importance[:top_n] + ], + } + shap_summary_path = shap_dir / "shap_feature_importance.json" + shap_summary_path.write_text(json.dumps(shap_summary, indent=2), encoding="utf-8") + shap_artifact = {"exported": True, "n_features": top_n, "path": str(shap_dir)} + logger.info( + "SHAP export: top feature={} (|SHAP|={:.4f}), saved to {}", + shap_importance[0][0], + shap_importance[0][1], + shap_dir, + ) + except Exception as exc: + logger.warning("SHAP feature importance export skipped: {}", exc) + shap_artifact["error"] = str(exc) + return shap_artifact + + +def _base_replay_report(reason: str = "not_run") -> dict[str, Any]: + return { "enabled": False, - "reason": "not_run", + "reason": reason, "rows": [], "selected_trial": None, "selected_params": None, } - if not bool(hpo_cfg.get("enabled", True)): - report["reason"] = "hpo_disabled" - return report + +def _load_optuna_study_for_replay(hpo_cfg: dict[str, Any]) -> tuple[Any | None, str | None]: storage = hpo_cfg.get("study_storage") study_name = resolve_optuna_study_name(hpo_cfg.get("study_name")) if not storage or not study_name: - report["reason"] = "missing_study_storage_or_name" - return report - + return None, "missing_study_storage_or_name" try: import optuna except Exception: - report["reason"] = "optuna_unavailable" - return report - + return None, "optuna_unavailable" try: - study = optuna.load_study(study_name=str(study_name), storage=str(storage)) + return optuna.load_study(study_name=str(study_name), storage=str(storage)), None except Exception as exc: - report["reason"] = f"study_load_failed: {exc}" - return report + return None, f"study_load_failed: {exc}" - complete = [t for t in study.trials if t.state.name == "COMPLETE" and t.value is not None] - if not complete: - report["reason"] = "no_complete_trials" - return report - top = sorted( +def _top_complete_trials(study: Any, top_k_trials: int) -> list[Any]: + complete = [t for t in study.trials if t.state.name == "COMPLETE" and t.value is not None] + return sorted( complete, key=lambda t: float(t.value if t.value is not None else float("-inf")), reverse=True, )[: max(1, int(top_k_trials))] + + +def _trial_gate_attrs(trial: Any) -> tuple[Any, Any, Any, bool, bool | None]: + fairness_pass_attr = trial.user_attrs.get("fairness_pass") + conformal_pass_attr = trial.user_attrs.get("conformal_pass") + governance_pass_attr = trial.user_attrs.get("governance_pass") + gate_attrs_present = all( + key in trial.user_attrs for key in ("fairness_pass", "conformal_pass", "governance_pass") + ) + gate_all_pass = ( + bool(fairness_pass_attr and conformal_pass_attr and governance_pass_attr) + if gate_attrs_present + else None + ) + return ( + fairness_pass_attr, + conformal_pass_attr, + governance_pass_attr, + bool(gate_attrs_present), + gate_all_pass, + ) + + +def _replay_trial_seed_row( + *, + trial: Any, + seed: int, + base_params: dict[str, Any], + X_train_fit_cb: pd.DataFrame, + y_train_fit: pd.Series, + X_val_cb: pd.DataFrame, + y_val: pd.Series, + cat_features: list[str], + sample_weight: np.ndarray | None, + eval_sample_weight: np.ndarray | None, +) -> dict[str, Any]: + ( + fairness_pass_attr, + conformal_pass_attr, + governance_pass_attr, + gate_attrs_present, + gate_all_pass, + ) = _trial_gate_attrs(trial) + params = {**base_params, **trial.params, "random_seed": int(seed)} + model, metrics = train_catboost_default( + X_train_fit_cb, + y_train_fit, + X_val_cb, + y_val, + cat_features=cat_features, + params=params, + sample_weight=sample_weight, + eval_sample_weight=eval_sample_weight, + ) + y_val_prob = model.predict_proba(X_val_cb)[:, 1] + y_val_array = y_val.to_numpy(dtype=int) + return { + "trial_number": int(trial.number), + "seed": int(seed), + "validation_auc": _metric_float(metrics, "validation_auc"), + "validation_brier": float(brier_score_loss(y_val_array, y_val_prob)), + "validation_ece": float(expected_calibration_error(y_val_array, y_val_prob)), + "best_iteration": _metric_int(metrics, "best_iteration"), + "trial_best_value": float(trial.value if trial.value is not None else float("nan")), + "fairness_pass": fairness_pass_attr, + "conformal_pass": conformal_pass_attr, + "governance_pass": governance_pass_attr, + "gate_attrs_present": gate_attrs_present, + "gate_all_pass": gate_all_pass, + "params": trial.params, + } + + +def _replay_rows_for_trials( + *, + top_trials: list[Any], + seeds: list[int], + base_params: dict[str, Any], + X_train_fit_cb: pd.DataFrame, + y_train_fit: pd.Series, + X_val_cb: pd.DataFrame, + y_val: pd.Series, + cat_features: list[str], + sample_weight: np.ndarray | None, + eval_sample_weight: np.ndarray | None, +) -> list[dict[str, Any]]: rows: list[dict[str, Any]] = [] - for trial in top: - fairness_pass_attr = trial.user_attrs.get("fairness_pass") - conformal_pass_attr = trial.user_attrs.get("conformal_pass") - governance_pass_attr = trial.user_attrs.get("governance_pass") - gate_attrs_present = all( - key in trial.user_attrs - for key in ("fairness_pass", "conformal_pass", "governance_pass") - ) - gate_all_pass = ( - bool(fairness_pass_attr and conformal_pass_attr and governance_pass_attr) - if gate_attrs_present - else None - ) + for trial in top_trials: for seed in seeds: - params = {**base_params, **trial.params, "random_seed": int(seed)} - model, metrics = train_catboost_default( - X_train_fit_cb, - y_train_fit, - X_val_cb, - y_val, - cat_features=cat_features, - params=params, - sample_weight=sample_weight, - eval_sample_weight=eval_sample_weight, - ) - y_val_prob = model.predict_proba(X_val_cb)[:, 1] rows.append( - { - "trial_number": int(trial.number), - "seed": int(seed), - "validation_auc": float(metrics.get("validation_auc", 0.0)), - "validation_brier": float(brier_score_loss(y_val, y_val_prob)), - "validation_ece": float(expected_calibration_error(y_val, y_val_prob)), - "best_iteration": int(metrics.get("best_iteration", 0)), - "trial_best_value": float( - trial.value if trial.value is not None else float("nan") - ), - "fairness_pass": fairness_pass_attr, - "conformal_pass": conformal_pass_attr, - "governance_pass": governance_pass_attr, - "gate_attrs_present": bool(gate_attrs_present), - "gate_all_pass": gate_all_pass, - "params": trial.params, - } + _replay_trial_seed_row( + trial=trial, + seed=seed, + base_params=base_params, + X_train_fit_cb=X_train_fit_cb, + y_train_fit=y_train_fit, + X_val_cb=X_val_cb, + y_val=y_val, + cat_features=cat_features, + sample_weight=sample_weight, + eval_sample_weight=eval_sample_weight, + ) ) + return rows + + +def _gate_tier(gate_all_pass_summary: bool | None, prioritize_gate_pass: bool) -> int: + if not prioritize_gate_pass: + return 1 + if gate_all_pass_summary is True: + return 0 + if gate_all_pass_summary is None: + return 1 + return 2 + + +def _trial_summary_row( + trial_number: Any, + grp: pd.DataFrame, + *, + prioritize_gate_pass: bool, +) -> dict[str, Any]: + gate_present = bool(grp["gate_attrs_present"].fillna(False).any()) + gate_values = grp["gate_all_pass"].dropna().astype(bool) + gate_all_pass_summary = None if gate_values.empty else bool(gate_values.all()) + return { + "trial_number": int(str(trial_number)), + "median_validation_auc": float(grp["validation_auc"].median()), + "mean_validation_auc": float(grp["validation_auc"].mean()), + "std_validation_auc": float(grp["validation_auc"].std(ddof=0)), + "mean_validation_brier": float(grp["validation_brier"].mean()), + "mean_validation_ece": float(grp["validation_ece"].mean()), + "gate_attrs_present": gate_present, + "gate_all_pass": gate_all_pass_summary, + "gate_tier": int(_gate_tier(gate_all_pass_summary, prioritize_gate_pass)), + } + + +def _summarize_replayed_trials( + rows: list[dict[str, Any]], + *, + prioritize_gate_pass: bool, +) -> pd.DataFrame: + replay_df = pd.DataFrame(rows) + summary_rows = [ + _trial_summary_row( + trial_number, + grp, + prioritize_gate_pass=prioritize_gate_pass, + ) + for trial_number, grp in replay_df.groupby("trial_number", observed=True) + ] + return pd.DataFrame(summary_rows).sort_values( + [ + "gate_tier", + "mean_validation_ece", + "median_validation_auc", + "mean_validation_brier", + ], + ascending=[True, True, False, True], + ) + + +def _replay_selection_policy(prioritize_gate_pass: bool) -> dict[str, Any]: + return { + "prioritize_gate_pass": bool(prioritize_gate_pass), + "rank_order": [ + "gate_tier(pass->unknown->fail)", + "mean_validation_ece(asc)", + "median_validation_auc(desc)", + "mean_validation_brier(asc)", + ], + } + +def _replay_top_optuna_trials( + *, + hpo_cfg: dict[str, Any], + base_params: dict[str, Any], + X_train_fit_cb: pd.DataFrame, + y_train_fit: pd.Series, + X_val_cb: pd.DataFrame, + y_val: pd.Series, + cat_features: list[str], + seeds: list[int], + top_k_trials: int = 3, + prioritize_gate_pass: bool = True, + sample_weight: np.ndarray | None = None, + eval_sample_weight: np.ndarray | None = None, +) -> dict[str, Any]: + """Replay top Optuna trials across multiple seeds for robustness.""" + report = _base_replay_report() + if not bool(hpo_cfg.get("enabled", True)): + report["reason"] = "hpo_disabled" + return report + + study, reason = _load_optuna_study_for_replay(hpo_cfg) + if study is None: + report["reason"] = reason or "study_load_failed" + return report + + top = _top_complete_trials(study, top_k_trials) + if not top: + report["reason"] = "no_complete_trials" + return report + + rows = _replay_rows_for_trials( + top_trials=top, + seeds=seeds, + base_params=base_params, + X_train_fit_cb=X_train_fit_cb, + y_train_fit=y_train_fit, + X_val_cb=X_val_cb, + y_val=y_val, + cat_features=cat_features, + sample_weight=sample_weight, + eval_sample_weight=eval_sample_weight, + ) if not rows: report["reason"] = "no_replay_rows" return report - replay_df = pd.DataFrame(rows) + grouped = _summarize_replayed_trials( + rows, + prioritize_gate_pass=prioritize_gate_pass, + ) + selected_trial = int(grouped.iloc[0]["trial_number"]) + selected_trial_obj = next(t for t in top if int(t.number) == selected_trial) + selected_params = {**base_params, **selected_trial_obj.params} + + report.update( + { + "enabled": True, + "reason": "ok", + "top_k_trials": len(top), + "seeds": [int(s) for s in seeds], + "selection_policy": _replay_selection_policy(prioritize_gate_pass), + "rows": rows, + "summary_by_trial": grouped.to_dict(orient="records"), + "selected_trial": selected_trial, + "selected_params": selected_params, + } + ) + return report + + +def _initial_seed_replay_report(reason: str = "disabled_in_config") -> dict[str, Any]: + return { + "enabled": False, + "reason": reason, + "rows": [], + "selected_trial": None, + "selected_params": None, + } + + +def _train_replay_manifest_catboost( + *, + replay_cfg: dict[str, Any], + X_train_fit_cb: pd.DataFrame, + y_train_fit: pd.Series, + X_val_cb: pd.DataFrame, + y_val: pd.Series, + X_test_cb: pd.DataFrame, + y_test: pd.Series, + categorical_features: list[str], + train_fit_weights: np.ndarray | None, + train_val_weights: np.ndarray | None, +) -> tuple[Any, dict[str, Any], dict[str, Any]]: + replay_params = dict(replay_cfg.get("selected_params") or {}) + model, metrics = train_catboost_default( + X_train_fit_cb, + y_train_fit, + X_val_cb, + y_val, + X_test=X_test_cb, + y_test=y_test, + cat_features=categorical_features, + params=replay_params, + sample_weight=train_fit_weights, + eval_sample_weight=train_val_weights, + ) + metrics["hpo_trials_executed"] = 0 + metrics["hpo_best_validation_auc"] = float(metrics["validation_auc"]) + metrics["best_params"] = replay_params + metrics["model_type"] = "catboost_replay_manifest" + seed_replay_report = _initial_seed_replay_report("replay_manifest") + seed_replay_report["selected_params"] = replay_params + return model, metrics, seed_replay_report + + +def _train_default_as_tuned_catboost( + *, + model_params: dict[str, Any], + X_train_fit_cb: pd.DataFrame, + y_train_fit: pd.Series, + X_val_cb: pd.DataFrame, + y_val: pd.Series, + X_test_cb: pd.DataFrame, + y_test: pd.Series, + categorical_features: list[str], + train_fit_weights: np.ndarray | None, + train_val_weights: np.ndarray | None, +) -> tuple[Any, dict[str, Any], dict[str, Any]]: + model, metrics = train_catboost_default( + X_train_fit_cb, + y_train_fit, + X_val_cb, + y_val, + X_test=X_test_cb, + y_test=y_test, + cat_features=categorical_features, + params=model_params, + sample_weight=train_fit_weights, + eval_sample_weight=train_val_weights, + ) + metrics["hpo_trials_executed"] = 0 + metrics["hpo_best_validation_auc"] = float(metrics["validation_auc"]) + metrics["best_params"] = model_params + return model, metrics, _initial_seed_replay_report("hpo_disabled") + + +def _maybe_apply_seed_replay_selection( + *, + seed_replay_report: dict[str, Any], + current_metrics: dict[str, Any], + current_model: Any, + X_train_fit_cb: pd.DataFrame, + y_train_fit: pd.Series, + X_val_cb: pd.DataFrame, + y_val: pd.Series, + X_test_cb: pd.DataFrame, + y_test: pd.Series, + categorical_features: list[str], + train_fit_weights: np.ndarray | None, + train_val_weights: np.ndarray | None, +) -> tuple[Any, dict[str, Any]]: + if not seed_replay_report.get("enabled") or not seed_replay_report.get("selected_params"): + return current_model, current_metrics + selected_params = dict(seed_replay_report["selected_params"]) + replay_model, replay_metrics = train_catboost_default( + X_train_fit_cb, + y_train_fit, + X_val_cb, + y_val, + X_test=X_test_cb, + y_test=y_test, + cat_features=categorical_features, + params=selected_params, + sample_weight=train_fit_weights, + eval_sample_weight=train_val_weights, + ) + return replay_model, { + **current_metrics, + **replay_metrics, + "model_type": "catboost_tuned_seed_replay_selected", + "best_params": selected_params, + "seed_replay_selected_trial": seed_replay_report.get("selected_trial"), + "seed_replay_enabled": True, + } - summary_rows: list[dict[str, Any]] = [] - for trial_number, grp in replay_df.groupby("trial_number", observed=True): - gate_present = bool(grp["gate_attrs_present"].fillna(False).any()) - gate_values = grp["gate_all_pass"].dropna().astype(bool) - if gate_values.empty: - gate_all_pass_summary: bool | None = None - else: - gate_all_pass_summary = bool(gate_values.all()) - - if not prioritize_gate_pass: - gate_tier = 1 - elif gate_all_pass_summary is True: - gate_tier = 0 - elif gate_all_pass_summary is None: - gate_tier = 1 - else: - gate_tier = 2 - summary_rows.append( - { - "trial_number": int(trial_number), - "median_validation_auc": float(grp["validation_auc"].median()), - "mean_validation_auc": float(grp["validation_auc"].mean()), - "std_validation_auc": float(grp["validation_auc"].std(ddof=0)), - "mean_validation_brier": float(grp["validation_brier"].mean()), - "mean_validation_ece": float(grp["validation_ece"].mean()), - "gate_attrs_present": gate_present, - "gate_all_pass": gate_all_pass_summary, - "gate_tier": int(gate_tier), - } +def _train_hpo_catboost( + *, + hpo_cfg: dict[str, Any], + seed_replay_cfg: dict[str, Any], + model_params: dict[str, Any], + X_train_fit_cb: pd.DataFrame, + y_train_fit: pd.Series, + X_val_cb: pd.DataFrame, + y_val: pd.Series, + X_test_cb: pd.DataFrame, + y_test: pd.Series, + categorical_features: list[str], + train_fit_weights: np.ndarray | None, + train_val_weights: np.ndarray | None, +) -> tuple[Any, dict[str, Any], dict[str, Any]]: + model, metrics = train_catboost_tuned_optuna( + X_train_fit_cb, + y_train_fit, + X_val_cb, + y_val, + X_test=X_test_cb, + y_test=y_test, + cat_features=categorical_features, + base_params=model_params, + n_trials=int(hpo_cfg.get("n_trials", 100)), + sampler=str(hpo_cfg.get("sampler", "tpe")), + pruner=str(hpo_cfg.get("pruner", "median")), + timeout_minutes=int(hpo_cfg.get("timeout_minutes", 0)), + n_startup_trials=int(hpo_cfg.get("n_startup_trials", 40)), + multivariate_tpe=bool(hpo_cfg.get("multivariate_tpe", True)), + group_tpe=bool(hpo_cfg.get("group_tpe", True)), + warn_independent_sampling=bool(hpo_cfg.get("warn_independent_sampling", True)), + constant_liar=bool(hpo_cfg.get("constant_liar", False)), + pruner_n_startup_trials=int(hpo_cfg.get("pruner_n_startup_trials", 20)), + pruner_n_warmup_steps=int(hpo_cfg.get("pruner_n_warmup_steps", 50)), + use_pruning_callback=bool(hpo_cfg.get("use_pruning_callback", True)), + study_storage=hpo_cfg.get("study_storage"), + study_name=hpo_cfg.get("study_name"), + load_if_exists=bool(hpo_cfg.get("load_if_exists", True)), + refit_full_train=bool(hpo_cfg.get("refit_full_train", True)), + gc_after_trial=bool(hpo_cfg.get("gc_after_trial", True)), + storage_heartbeat_interval=int(hpo_cfg.get("storage_heartbeat_interval", 0)), + storage_grace_period=int(hpo_cfg.get("storage_grace_period", 0)), + sqlite_timeout_seconds=int(hpo_cfg.get("sqlite_timeout_seconds", 60)), + retry_failed_trials=int(hpo_cfg.get("retry_failed_trials", 0)), + n_jobs=int(hpo_cfg.get("n_jobs", 1)), + sample_weight=train_fit_weights, + eval_sample_weight=train_val_weights, + search_space_mode=str(hpo_cfg.get("search_space_mode", "global")), + local_refine_space=dict(hpo_cfg.get("local_refine", {}) or {}), + constraints_policy=dict(hpo_cfg.get("constraints_policy", {}) or {}), + search_space_version=str(hpo_cfg.get("search_space_version", "cb_space_v2")), + ) + seed_replay_report = _initial_seed_replay_report() + if bool(seed_replay_cfg.get("enabled", True)): + seed_replay_report = _replay_top_optuna_trials( + hpo_cfg=hpo_cfg, + base_params=model_params, + X_train_fit_cb=X_train_fit_cb, + y_train_fit=y_train_fit, + X_val_cb=X_val_cb, + y_val=y_val, + cat_features=categorical_features, + seeds=[int(seed) for seed in seed_replay_cfg.get("seeds", [42, 52, 62])], + top_k_trials=int(seed_replay_cfg.get("top_k_trials", 3)), + prioritize_gate_pass=bool(seed_replay_cfg.get("prioritize_gate_pass", True)), + sample_weight=train_fit_weights, + eval_sample_weight=train_val_weights, + ) + model, metrics = _maybe_apply_seed_replay_selection( + seed_replay_report=seed_replay_report, + current_metrics=metrics, + current_model=model, + X_train_fit_cb=X_train_fit_cb, + y_train_fit=y_train_fit, + X_val_cb=X_val_cb, + y_val=y_val, + X_test_cb=X_test_cb, + y_test=y_test, + categorical_features=categorical_features, + train_fit_weights=train_fit_weights, + train_val_weights=train_val_weights, ) + return model, metrics, seed_replay_report - grouped = pd.DataFrame(summary_rows).sort_values( - [ - "gate_tier", - "mean_validation_ece", - "median_validation_auc", - "mean_validation_brier", - ], - ascending=[True, True, False, True], - ) - selected_trial = int(grouped.iloc[0]["trial_number"]) - selected_trial_obj = next(t for t in top if int(t.number) == selected_trial) - selected_params = {**base_params, **selected_trial_obj.params} - report.update( - { - "enabled": True, - "reason": "ok", - "top_k_trials": len(top), - "seeds": [int(s) for s in seeds], - "selection_policy": { - "prioritize_gate_pass": bool(prioritize_gate_pass), - "rank_order": [ - "gate_tier(pass->unknown->fail)", - "mean_validation_ece(asc)", - "median_validation_auc(desc)", - "mean_validation_brier(asc)", - ], - }, - "rows": rows, - "summary_by_trial": grouped.to_dict(orient="records"), - "selected_trial": selected_trial, - "selected_params": selected_params, - } +def _train_tuned_catboost_stage( + *, + run_mode: str, + replay_cfg: dict[str, Any], + hpo_cfg: dict[str, Any], + seed_replay_cfg: dict[str, Any], + model_params: dict[str, Any], + X_train_fit_cb: pd.DataFrame, + y_train_fit: pd.Series, + X_val_cb: pd.DataFrame, + y_val: pd.Series, + X_test_cb: pd.DataFrame, + y_test: pd.Series, + categorical_features: list[str], + train_fit_weights: np.ndarray | None, + train_val_weights: np.ndarray | None, +) -> tuple[Any, dict[str, Any], dict[str, Any]]: + if run_mode == "replay": + return _train_replay_manifest_catboost( + replay_cfg=replay_cfg, + X_train_fit_cb=X_train_fit_cb, + y_train_fit=y_train_fit, + X_val_cb=X_val_cb, + y_val=y_val, + X_test_cb=X_test_cb, + y_test=y_test, + categorical_features=categorical_features, + train_fit_weights=train_fit_weights, + train_val_weights=train_val_weights, + ) + if bool(hpo_cfg.get("enabled", True)): + return _train_hpo_catboost( + hpo_cfg=hpo_cfg, + seed_replay_cfg=seed_replay_cfg, + model_params=model_params, + X_train_fit_cb=X_train_fit_cb, + y_train_fit=y_train_fit, + X_val_cb=X_val_cb, + y_val=y_val, + X_test_cb=X_test_cb, + y_test=y_test, + categorical_features=categorical_features, + train_fit_weights=train_fit_weights, + train_val_weights=train_val_weights, + ) + return _train_default_as_tuned_catboost( + model_params=model_params, + X_train_fit_cb=X_train_fit_cb, + y_train_fit=y_train_fit, + X_val_cb=X_val_cb, + y_val=y_val, + X_test_cb=X_test_cb, + y_test=y_test, + categorical_features=categorical_features, + train_fit_weights=train_fit_weights, + train_val_weights=train_val_weights, ) - return report def main( @@ -1560,138 +2398,29 @@ def main( # CatBoost tuned (Optuna) hpo_cfg = config.get("hpo", {}) - seed_replay_report: dict[str, Any] = { - "enabled": False, - "reason": "disabled_in_config", - "rows": [], - "selected_trial": None, - "selected_params": None, - } - if run_mode == "replay": - replay_params = dict(replay_cfg.get("selected_params") or {}) - cb_tuned_model, cb_tuned_metrics = train_catboost_default( - X_train_fit_cb, - y_train_fit, - X_val_cb, - y_val, - X_test=X_test_cb, - y_test=y_test, - cat_features=categorical_features, - params=replay_params, - sample_weight=train_fit_weights, - eval_sample_weight=train_val_weights, - ) - cb_tuned_metrics["hpo_trials_executed"] = 0 - cb_tuned_metrics["hpo_best_validation_auc"] = float(cb_tuned_metrics["validation_auc"]) - cb_tuned_metrics["best_params"] = replay_params - cb_tuned_metrics["model_type"] = "catboost_replay_manifest" - seed_replay_report["reason"] = "replay_manifest" - seed_replay_report["selected_params"] = replay_params - elif bool(hpo_cfg.get("enabled", True)): - cb_tuned_model, cb_tuned_metrics = train_catboost_tuned_optuna( - X_train_fit_cb, - y_train_fit, - X_val_cb, - y_val, - X_test=X_test_cb, - y_test=y_test, - cat_features=categorical_features, - base_params=config["model"].get("params", {}), - n_trials=int(hpo_cfg.get("n_trials", 100)), - sampler=str(hpo_cfg.get("sampler", "tpe")), - pruner=str(hpo_cfg.get("pruner", "median")), - timeout_minutes=int(hpo_cfg.get("timeout_minutes", 0)), - n_startup_trials=int(hpo_cfg.get("n_startup_trials", 40)), - multivariate_tpe=bool(hpo_cfg.get("multivariate_tpe", True)), - group_tpe=bool(hpo_cfg.get("group_tpe", True)), - warn_independent_sampling=bool(hpo_cfg.get("warn_independent_sampling", True)), - constant_liar=bool(hpo_cfg.get("constant_liar", False)), - pruner_n_startup_trials=int(hpo_cfg.get("pruner_n_startup_trials", 20)), - pruner_n_warmup_steps=int(hpo_cfg.get("pruner_n_warmup_steps", 50)), - use_pruning_callback=bool(hpo_cfg.get("use_pruning_callback", True)), - study_storage=hpo_cfg.get("study_storage", None), - study_name=hpo_cfg.get("study_name", None), - load_if_exists=bool(hpo_cfg.get("load_if_exists", True)), - refit_full_train=bool(hpo_cfg.get("refit_full_train", True)), - gc_after_trial=bool(hpo_cfg.get("gc_after_trial", True)), - storage_heartbeat_interval=int(hpo_cfg.get("storage_heartbeat_interval", 0)), - storage_grace_period=int(hpo_cfg.get("storage_grace_period", 0)), - sqlite_timeout_seconds=int(hpo_cfg.get("sqlite_timeout_seconds", 60)), - retry_failed_trials=int(hpo_cfg.get("retry_failed_trials", 0)), - n_jobs=int(hpo_cfg.get("n_jobs", 1)), - sample_weight=train_fit_weights, - eval_sample_weight=train_val_weights, - search_space_mode=str(hpo_cfg.get("search_space_mode", "global")), - local_refine_space=dict(hpo_cfg.get("local_refine", {}) or {}), - constraints_policy=dict(hpo_cfg.get("constraints_policy", {}) or {}), - search_space_version=str(hpo_cfg.get("search_space_version", "cb_space_v2")), - ) - - if bool(seed_replay_cfg.get("enabled", True)): - seeds = seed_replay_cfg.get("seeds", [42, 52, 62]) - seeds = [int(s) for s in seeds] - prioritize_gate_pass = bool(seed_replay_cfg.get("prioritize_gate_pass", True)) - seed_replay_report = _replay_top_optuna_trials( - hpo_cfg=hpo_cfg, - base_params=config["model"].get("params", {}), - X_train_fit_cb=X_train_fit_cb, - y_train_fit=y_train_fit, - X_val_cb=X_val_cb, - y_val=y_val, - cat_features=categorical_features, - seeds=seeds, - top_k_trials=int(seed_replay_cfg.get("top_k_trials", 3)), - prioritize_gate_pass=prioritize_gate_pass, - sample_weight=train_fit_weights, - eval_sample_weight=train_val_weights, - ) - if seed_replay_report.get("enabled") and seed_replay_report.get("selected_params"): - selected_params = dict(seed_replay_report["selected_params"]) - replay_model, replay_metrics = train_catboost_default( - X_train_fit_cb, - y_train_fit, - X_val_cb, - y_val, - X_test=X_test_cb, - y_test=y_test, - cat_features=categorical_features, - params=selected_params, - sample_weight=train_fit_weights, - eval_sample_weight=train_val_weights, - ) - cb_tuned_metrics = { - **cb_tuned_metrics, - **replay_metrics, - "model_type": "catboost_tuned_seed_replay_selected", - "best_params": selected_params, - "seed_replay_selected_trial": seed_replay_report.get("selected_trial"), - "seed_replay_enabled": True, - } - cb_tuned_model = replay_model - else: - cb_tuned_model, cb_tuned_metrics = train_catboost_default( - X_train_fit_cb, - y_train_fit, - X_val_cb, - y_val, - X_test=X_test_cb, - y_test=y_test, - cat_features=categorical_features, - params=config["model"].get("params", {}), - sample_weight=train_fit_weights, - eval_sample_weight=train_val_weights, - ) - cb_tuned_metrics["hpo_trials_executed"] = 0 - cb_tuned_metrics["hpo_best_validation_auc"] = float(cb_tuned_metrics["validation_auc"]) - cb_tuned_metrics["best_params"] = config["model"].get("params", {}) - seed_replay_report["reason"] = "hpo_disabled" + cb_tuned_model, cb_tuned_metrics, seed_replay_report = _train_tuned_catboost_stage( + run_mode=run_mode, + replay_cfg=replay_cfg, + hpo_cfg=hpo_cfg, + seed_replay_cfg=seed_replay_cfg, + model_params=dict(config["model"].get("params", {})), + X_train_fit_cb=X_train_fit_cb, + y_train_fit=y_train_fit, + X_val_cb=X_val_cb, + y_val=y_val, + X_test_cb=X_test_cb, + y_test=y_test, + categorical_features=categorical_features, + train_fit_weights=train_fit_weights, + train_val_weights=train_val_weights, + ) _write_checkpoint( checkpoint_dir, "hpo_summary", { - "best_params": cb_tuned_metrics.get("best_params", {}), - "hpo_trials_executed": int(cb_tuned_metrics.get("hpo_trials_executed", 0)), - "hpo_best_validation_auc": float(cb_tuned_metrics.get("hpo_best_validation_auc", 0.0)), + "best_params": _metric_mapping(cb_tuned_metrics, "best_params", {}), + "hpo_trials_executed": _metric_int(cb_tuned_metrics, "hpo_trials_executed"), + "hpo_best_validation_auc": _metric_float(cb_tuned_metrics, "hpo_best_validation_auc"), "seed_replay_report": seed_replay_report, }, ) @@ -1701,35 +2430,23 @@ def main( state="running", config_path=config_path, extra={ - "validation_auc": float(cb_tuned_metrics.get("hpo_best_validation_auc", 0.0)), - "best_iteration": int(cb_tuned_metrics.get("best_iteration", 0)), + "validation_auc": _metric_float(cb_tuned_metrics, "hpo_best_validation_auc"), + "best_iteration": _metric_int(cb_tuned_metrics, "best_iteration"), }, ) - walk_forward_report: dict[str, Any] - if bool(walk_cfg.get("enabled", True)): - walk_forward_report = _evaluate_walk_forward_auc( - train, - features=catboost_features, - categorical_features=categorical_features, - target=TARGET, - params=dict(cb_tuned_metrics.get("best_params", config["model"].get("params", {}))), - n_windows=int(walk_cfg.get("n_windows", 3)), - min_train_rows=int(walk_cfg.get("min_train_rows", 200_000)), - window_rows=int(walk_cfg.get("window_rows", 80_000)), - date_col=str(walk_cfg.get("date_col", "issue_d")), - max_rows=( - None if int(walk_cfg.get("max_rows", 0)) <= 0 else int(walk_cfg.get("max_rows", 0)) - ), - ) - else: - walk_forward_report = { - "enabled": False, - "reason": "disabled_in_config", - "n_windows_requested": int(walk_cfg.get("n_windows", 0)), - "n_windows_used": 0, - "folds": [], - } + walk_forward_report = _evaluate_walk_forward_stage( + enabled=bool(walk_cfg.get("enabled", True)), + walk_cfg=walk_cfg, + train=train, + catboost_features=catboost_features, + categorical_features=categorical_features, + model_params=_metric_mapping( + cb_tuned_metrics, + "best_params", + dict(config["model"].get("params", {})), + ), + ) # Raw probabilities y_prob_default_test = cb_default_model.predict_proba(X_test_cb)[:, 1] @@ -1781,75 +2498,20 @@ def main( decision_cfg = config.get("decision_threshold", {}) resolved_run_tag = resolve_run_tag(run_tag, require_explicit=True) - decision_threshold_artifact = { - "enabled": False, - "selected_threshold": float(config.get("calibration", {}).get("default_threshold", 0.5)), - "source": "fallback_default", - **build_artifact_metadata( - schema_version="2026-03-01.1", - run_tag=resolved_run_tag, - require_explicit=True, - ), - } - if run_mode == "replay" and replay_cfg.get("decision_threshold_artifact"): - decision_threshold_artifact = dict(replay_cfg["decision_threshold_artifact"]) - decision_threshold_artifact.update( - build_artifact_metadata( - schema_version="2026-03-01.1", - run_tag=resolved_run_tag, - require_explicit=True, - ) - ) - decision_threshold_artifact["source"] = "frozen_replay_manifest" - elif bool(decision_cfg.get("enabled", True)): - fairness_policy_path = str( - decision_cfg.get("fairness_policy_path", "configs/fairness_policy.yaml") - ) - with open(fairness_policy_path, encoding="utf-8") as f: - fairness_cfg = yaml.safe_load(f) or {} - fairness_policy = fairness_cfg.get("policy", {}) - fairness_attrs = fairness_cfg.get("attributes", []) - groups_for_threshold = _build_fairness_groups_for_threshold(train_val, fairness_attrs) - groups_for_threshold_cal = _build_fairness_groups_for_threshold(cal, fairness_attrs) - - thr_min = float(decision_cfg.get("min_threshold", 0.05)) - thr_max = float(decision_cfg.get("max_threshold", 0.95)) - thr_step = float(decision_cfg.get("step", 0.01)) - thresholds = np.arange(thr_min, thr_max + (thr_step / 2.0), thr_step) - fallback_threshold = float(fairness_policy.get("prediction_threshold", 0.5)) - threshold_result = _select_decision_threshold( - y_true=y_val.to_numpy(), - y_prob=y_prob_final_val, - policy={ - "dpd_threshold": float(fairness_policy.get("dpd_threshold", 0.10)), - "eo_gap_threshold": float(fairness_policy.get("eo_gap_threshold", 0.10)), - "dir_threshold": float(fairness_policy.get("dir_threshold", 0.80)), - }, - groups_dict=groups_for_threshold, - thresholds=np.asarray(thresholds, dtype=float), - fallback_threshold=fallback_threshold, - y_true_secondary=y_cal.to_numpy(), - y_prob_secondary=y_prob_tuned_cal, - groups_dict_secondary=groups_for_threshold_cal, - ) - - decision_threshold_artifact = { - "enabled": True, - "selected_threshold": float(threshold_result["selected_threshold"]), - "fallback_threshold": fallback_threshold, - "selection_metrics": threshold_result["selection_metrics"], - "search_summary": threshold_result["search_summary"], - "source": "validation_fairness_search", - "fairness_policy_path": fairness_policy_path, - "validation_rows": len(train_val), - "secondary_validation_rows": len(cal), - "calibration_method": selected_cal_method, - **build_artifact_metadata( - schema_version="2026-03-01.1", - run_tag=resolved_run_tag, - require_explicit=True, - ), - } + decision_threshold_artifact = _resolve_decision_threshold_artifact( + config=config, + run_mode=run_mode, + replay_cfg=replay_cfg, + decision_cfg=decision_cfg, + train_val=train_val, + cal=cal, + y_val=y_val, + y_prob_final_val=y_prob_final_val, + y_cal=y_cal, + y_prob_tuned_cal=y_prob_tuned_cal, + selected_cal_method=selected_cal_method, + run_tag=resolved_run_tag, + ) # Conformal (keeps calibration split isolated from model training) alpha = 1.0 - float(config["conformal"].get("confidence_level", 0.9)) @@ -1870,83 +2532,11 @@ def main( # Statistical calibration hypothesis tests (MAPIE) # Tests H0: scores are well-calibrated. High p-value → well calibrated. - statistical_cal_tests: dict[str, object] = {} - try: - from mapie.metrics.calibration import ( - cumulative_differences, - kolmogorov_smirnov_p_value, - kuiper_p_value, - length_scale, - spiegelhalter_p_value, - ) - - y_test_arr = y_test.values.astype(float) - # Calibrated (champion) vs uncalibrated - for tag, probs in [("calibrated", y_prob_final), ("uncalibrated", y_prob_tuned_test)]: - try: - ks_p = float(kolmogorov_smirnov_p_value(y_test_arr, probs)) - ku_p = float(kuiper_p_value(y_test_arr, probs)) - sp_p = float(spiegelhalter_p_value(y_test_arr, probs)) - cum_diff = cumulative_differences(y_test_arr, probs) - sigma = float(length_scale(probs)) - statistical_cal_tests[tag] = { - "ks_pvalue": ks_p, - "kuiper_pvalue": ku_p, - "spiegelhalter_pvalue": sp_p, - "length_scale_sigma": sigma, - "n": len(y_test_arr), - } - logger.info( - f"Calibration tests [{tag}]: KS_p={ks_p:.4f} " - f"Kuiper_p={ku_p:.4f} Spiegelhalter_p={sp_p:.4f}" - ) - # Save cumulative differences for the figure (calibrated only) - if tag == "calibrated": - statistical_cal_tests["_cum_diff_calibrated"] = cum_diff.tolist() - statistical_cal_tests["_sigma"] = sigma - elif tag == "uncalibrated": - statistical_cal_tests["_cum_diff_uncalibrated"] = cum_diff.tolist() - except Exception as exc_inner: - logger.warning(f"Statistical calibration tests [{tag}] failed: {exc_inner}") - statistical_cal_tests[tag] = {"error": str(exc_inner)} - - # Save cumulative differences parquet for figure generation - if "_cum_diff_calibrated" in statistical_cal_tests: - k_idx = np.arange(len(y_test_arr)) / len(y_test_arr) - sigma_raw = statistical_cal_tests.get("_sigma", 0.0) - sigma_val = ( - float(sigma_raw) if isinstance(sigma_raw, str | int | float | np.number) else 0.0 - ) - cum_diff_df = pd.DataFrame( - { - "k": k_idx, - "cum_diff_calibrated": statistical_cal_tests.pop("_cum_diff_calibrated"), - "cum_diff_uncalibrated": statistical_cal_tests.pop( - "_cum_diff_uncalibrated", - [float("nan")] * len(k_idx), - ), - "sigma_upper": sigma_val * 2, - "sigma_lower": -sigma_val * 2, - } - ) - cum_diff_path = _artifact_path("data/processed/calibration_cumulative_diffs.parquet") - cum_diff_path.parent.mkdir(parents=True, exist_ok=True) - cum_diff_df.to_parquet(cum_diff_path, index=False) - logger.info(f"Saved calibration cumulative diffs: {cum_diff_path}") - statistical_cal_tests.pop("_sigma", None) - - stat_cal_path = _artifact_path("data/processed/statistical_calibration_tests.json") - stat_cal_path.parent.mkdir(parents=True, exist_ok=True) - stat_cal_path.write_text( - json.dumps(statistical_cal_tests, indent=2, default=str), encoding="utf-8" - ) - logger.info(f"Saved statistical calibration tests: {stat_cal_path}") - except ImportError: - logger.warning( - "mapie.metrics.calibration not available — statistical calibration tests skipped." - ) - except Exception as exc: - logger.warning(f"Statistical calibration tests block failed: {exc}") + _run_statistical_calibration_tests( + y_test=y_test, + y_prob_final=y_prob_final, + y_prob_tuned_test=y_prob_tuned_test, + ) brier_decomposition_path = _artifact_path( config["output"].get( @@ -2095,55 +2685,13 @@ def main( save_contract(contract_payload, contract_path) # ── SHAP feature importance export (CatBoost native) ── - shap_artifact: dict[str, Any] = {"exported": False} - try: - from catboost import Pool as _SHAPPool - - shap_pool = _SHAPPool(X_test_cb, cat_features=categorical_features) - shap_raw = cb_tuned_model.get_feature_importance(type="ShapValues", data=shap_pool) - # ShapValues returns (n_samples, n_features + 1); last col = expected value - shap_values = shap_raw[:, :-1] - shap_expected = float(shap_raw[0, -1]) - - mean_abs_shap = np.abs(shap_values).mean(axis=0) - shap_importance = sorted( - zip(catboost_features, mean_abs_shap.tolist(), strict=False), - key=lambda x: x[1], - reverse=True, - ) - shap_dir = _artifact_path(config["output"].get("shap_dir", "reports/figures/shap")) - shap_dir.mkdir(parents=True, exist_ok=True) - - # Save raw SHAP values (compressed) - np.savez_compressed( - str(shap_dir / "shap_values_test.npz"), - shap_values=shap_values, - expected_value=np.array([shap_expected]), - feature_names=np.array(catboost_features), - ) - - # Save top-N importance as JSON for Streamlit/governance - top_n = min(20, len(shap_importance)) - shap_summary = { - "expected_value": shap_expected, - "n_samples": int(shap_values.shape[0]), - "n_features": int(shap_values.shape[1]), - "top_features": [ - {"feature": f, "mean_abs_shap": round(v, 6)} for f, v in shap_importance[:top_n] - ], - } - shap_summary_path = shap_dir / "shap_feature_importance.json" - shap_summary_path.write_text(json.dumps(shap_summary, indent=2), encoding="utf-8") - shap_artifact = {"exported": True, "n_features": top_n, "path": str(shap_dir)} - logger.info( - "SHAP export: top feature={} (|SHAP|={:.4f}), saved to {}", - shap_importance[0][0], - shap_importance[0][1], - shap_dir, - ) - except Exception as exc: - logger.warning("SHAP feature importance export skipped: {}", exc) - shap_artifact["error"] = str(exc) + shap_artifact = _export_shap_feature_importance( + cb_tuned_model=cb_tuned_model, + X_test_cb=X_test_cb, + categorical_features=categorical_features, + catboost_features=catboost_features, + shap_dir=_artifact_path(config["output"].get("shap_dir", "reports/figures/shap")), + ) # Persist test predictions for downstream contracts. test_predictions_path = _artifact_path( @@ -2189,10 +2737,10 @@ def main( strict=False, ) }, - "optuna_best_auc": float(cb_tuned_metrics.get("auc_roc", 0.0)), - "optuna_best_params": cb_tuned_metrics.get("best_params", {}), - "hpo_trials_executed": int(cb_tuned_metrics.get("hpo_trials_executed", 0)), - "hpo_best_validation_auc": float(cb_tuned_metrics.get("hpo_best_validation_auc", 0.0)), + "optuna_best_auc": _metric_float(cb_tuned_metrics, "auc_roc"), + "optuna_best_params": _metric_mapping(cb_tuned_metrics, "best_params", {}), + "hpo_trials_executed": _metric_int(cb_tuned_metrics, "hpo_trials_executed"), + "hpo_best_validation_auc": _metric_float(cb_tuned_metrics, "hpo_best_validation_auc"), "walk_forward_report": walk_forward_report, "seed_replay_report": seed_replay_report, "decision_threshold": decision_threshold_artifact, @@ -2221,10 +2769,10 @@ def main( ) seed_replay_status = { "selected_calibration_method": selected_cal_method, - "validation_auc": float(cb_tuned_metrics.get("hpo_best_validation_auc", 0.0)), - "oot_auc": float(final_test_metrics.get("auc_roc", 0.0)), - "brier": float(final_test_metrics.get("brier_score", 0.0)), - "ece": float(final_test_metrics.get("ece", 0.0)), + "validation_auc": _metric_float(cb_tuned_metrics, "hpo_best_validation_auc"), + "oot_auc": _metric_float(final_test_metrics, "auc_roc"), + "brier": _metric_float(final_test_metrics, "brier_score"), + "ece": _metric_float(final_test_metrics, "ece"), "replay": seed_replay_report, **build_artifact_metadata( schema_version="2026-03-13.1", diff --git a/scripts/validate_alpha_gamma_bound.py b/scripts/validate_alpha_gamma_bound.py index c1ddb76..079ed9b 100644 --- a/scripts/validate_alpha_gamma_bound.py +++ b/scripts/validate_alpha_gamma_bound.py @@ -23,9 +23,13 @@ _parse_percent_series, ) from src.models.conformal_artifacts import load_conformal_intervals # noqa: E402 +from src.optimization.certificate_semantics import ( # noqa: E402 + compute_funded_certificate_metrics, +) from src.optimization.portfolio_model import ( # noqa: E402 compute_effective_pd, optimize_portfolio_allocation, + solution_allocation_vector, ) DEFAULT_ALPHAS = [0.01, 0.03, 0.05, 0.07, 0.10, 0.12, 0.15, 0.20] @@ -277,12 +281,7 @@ def _compute_exact_weights( if "loan_amnt" in loans.columns else np.ones(len(loans), dtype=float) ) - if "allocation_vector" in solution: - alloc = np.asarray(solution["allocation_vector"], dtype=float) - else: - alloc = np.array( - [float(solution["allocation"].get(i, 0.0)) for i in range(len(loans))], dtype=float - ) + alloc = solution_allocation_vector(solution, len(loans)) total_allocated = float(np.sum(alloc * loan_amounts)) weights = (alloc * loan_amounts) / max(total_allocated, 1e-6) return weights, { @@ -339,37 +338,52 @@ def _validate_single_alpha( else: raise ValueError(f"Unsupported allocator-mode={allocator_mode!r}") - miscoverage = (y_true > pd_high).astype(float) - V = float(np.sum(weights * miscoverage)) - weighted_pd_true = float(np.sum(weights * y_true)) - violation = max(0.0, weighted_pd_true - float(policy["risk_tolerance"])) - funded_mask = weights > 1e-8 - emp_coverage = ( - float(1.0 - miscoverage[funded_mask].mean()) if funded_mask.any() else float("nan") + certificate = compute_funded_certificate_metrics( + weights, + outcomes=y_true, + pd_point=pd_point, + pd_high=pd_high, + pd_effective=effective_pd, + alpha=alpha, + risk_tolerance=float(policy["risk_tolerance"]), + pd_cap_slack=float(alloc_meta.get("pd_cap_slack", 0.0)), ) - sqrt_alpha = float(np.sqrt(alpha)) bound_b_value = min(1.0, alpha / max(t_eval, 1e-8)) + risk_excess = round(certificate.realized_risk_tolerance_excess, 6) + empirical_risk_screen = bool(certificate.realized_risk_tolerance_excess <= alpha + 1e-8) + markov_screen = bool(certificate.sqrt_alpha + 1e-8 >= certificate.weighted_miscoverage) return { "alpha": float(alpha), "confidence": float(1.0 - alpha), - "gamma_cp": round(float(np.sum(weights * np.clip(pd_high - pd_point, 0.0, 1.0))), 6), - "n_funded": int(np.sum(funded_mask)), - "weighted_pd_true": round(weighted_pd_true, 6), - "weighted_pd_constraint_used": round(float(alloc_meta["weighted_pd_constraint_used"]), 6), - "weighted_pd_high": round(float(alloc_meta["weighted_pd_high"]), 6), - "weighted_pd_point": round(float(alloc_meta["weighted_pd_point"]), 6), + "gamma_cp": round(certificate.gamma_cp, 6), + "gamma_internalized": round(certificate.gamma_internalized, 6), + "gamma_residual": round(certificate.gamma_residual, 6), + "n_funded": certificate.n_funded, + "weighted_pd_true": round(certificate.weighted_outcome, 6), + "weighted_pd_constraint_used": round(certificate.weighted_pd_effective, 6), + "weighted_pd_high": round(certificate.endpoint_budget, 6), + "weighted_pd_point": round(certificate.weighted_pd_point, 6), + "endpoint_budget": round(certificate.endpoint_budget, 9), + "endpoint_budget_upper": round(certificate.endpoint_budget_upper, 9), + "markov_loss_threshold": round(certificate.markov_loss_threshold, 9), + "markov_loss_cap": round(certificate.markov_loss_cap, 9), "tau": float(policy["risk_tolerance"]), - "violation": round(violation, 6), - "weighted_miscoverage_V": round(V, 6), - "sqrt_alpha": round(sqrt_alpha, 6), - "empirical_coverage_funded": round(emp_coverage, 4), - "bound_a_expected_violation_leq_alpha": bool(violation <= alpha + 1e-8), + "realized_risk_tolerance_excess": risk_excess, + "violation": risk_excess, + "weighted_miscoverage_V": round(certificate.weighted_miscoverage, 6), + "weighted_coverage_funded": round(certificate.weighted_coverage, 6), + "sqrt_alpha": round(certificate.sqrt_alpha, 6), + "empirical_coverage_funded": round(certificate.empirical_coverage_funded, 4), + "empirical_risk_excess_leq_alpha": empirical_risk_screen, + "bound_a_expected_violation_leq_alpha": empirical_risk_screen, "bound_b_prob_violation_gt_t": round(bound_b_value, 4), "bound_b_t_eval": float(t_eval), "bound_b_is_vacuous": bool(bound_b_value >= 1.0), - "bound_c_V_leq_sqrt_alpha": bool(sqrt_alpha + 1e-8 >= V), - "all_bounds_hold": bool((violation <= alpha + 1e-8) and (sqrt_alpha + 1e-8 >= V)), + "markov_miscoverage_screen_pass": markov_screen, + "bound_c_V_leq_sqrt_alpha": markov_screen, + "certificate_screen_pass": empirical_risk_screen and markov_screen, + "all_bounds_hold": empirical_risk_screen and markov_screen, "allocator_mode": mode, "solver_status": str(alloc_meta.get("solver_status", "unknown")), "allocator_solver_backend": str(alloc_meta.get("solver_backend", policy["solver_backend"])), @@ -404,22 +418,22 @@ def _plot_validation(results: list[dict[str, Any]], figure_prefix: Path) -> None ax = axes[1] ax.bar( df["alpha"], - df["violation"], + df["realized_risk_tolerance_excess"], width=0.012, color="#27ae60", alpha=0.7, - label="Violación empírica", + label="Realized risk-tolerance excess", ) ax.plot( df["alpha"], df["alpha"], "r--", linewidth=2, - label=r"Cota teórica $\mathbb{E}[\mathrm{violación}] \leq \alpha$", + label=r"Declared empirical screen: excess $\leq \alpha$", ) ax.set_xlabel(r"$\alpha$") - ax.set_ylabel("Violación de restricción PD") - ax.set_title(r"(B) Teorema 1(a): $\mathbb{E}[\mathrm{viol.}] \leq \alpha$") + ax.set_ylabel("Excess above risk tolerance") + ax.set_title("(B) Realized risk-tolerance screen") ax.legend(fontsize=8, loc="upper left") ax.grid(True, alpha=0.3) @@ -451,10 +465,10 @@ def _plot_validation(results: list[dict[str, Any]], figure_prefix: Path) -> None def _print_summary(results: list[dict[str, Any]], allocator_mode: str) -> None: all_pass = all(bool(r["all_bounds_hold"]) for r in results) print("\n" + "=" * 96) - print(f"VALIDATION SUMMARY: Theorem 1 ({allocator_mode.upper()} allocator)") + print(f"CERTIFICATE SUMMARY ({allocator_mode.upper()} allocator)") print("=" * 96) header = ( - f"{'α':>6} {'1-α':>6} {'Γ_CP':>8} {'Violation':>10} {'V':>8} " + f"{'α':>6} {'1-α':>6} {'Γ_CP':>8} {'Risk excess':>12} {'V':>8} " f"{'√α':>8} {'Mode':>8} {'Pass':>6}" ) print(header) @@ -463,11 +477,11 @@ def _print_summary(results: list[dict[str, Any]], allocator_mode: str) -> None: status = " ✓" if r["all_bounds_hold"] else " ✗" print( f"{r['alpha']:6.2f} {r['confidence']:6.2f} {r['gamma_cp']:8.4f} " - f"{r['violation']:10.6f} {r['weighted_miscoverage_V']:8.4f} " + f"{r['realized_risk_tolerance_excess']:10.6f} {r['weighted_miscoverage_V']:8.4f} " f"{r['sqrt_alpha']:8.4f} {r['allocator_mode']:>8} {status}" ) print("=" * 96) - print(f"Result: {'ALL BOUNDS HOLD' if all_pass else 'SOME BOUNDS FAILED'}") + print(f"Result: {'ALL SCREENS PASS' if all_pass else 'SOME SCREENS FAILED'}") def main(argv: list[str] | None = None) -> None: @@ -509,10 +523,10 @@ def main(argv: list[str] | None = None) -> None: results.append(result) status = "✓" if result["all_bounds_hold"] else "✗" logger.info( - " α={:.2f} Γ_CP={:.4f} violation={:.6f} V={:.4f} √α={:.4f} {}", + " α={:.2f} Γ_CP={:.4f} risk_excess={:.6f} V={:.4f} √α={:.4f} {}", alpha, result["gamma_cp"], - result["violation"], + result["realized_risk_tolerance_excess"], result["weighted_miscoverage_V"], result["sqrt_alpha"], status, @@ -524,7 +538,8 @@ def main(argv: list[str] | None = None) -> None: or (DEFAULT_EXACT_JSON if allocator_mode == "exact" else DEFAULT_PROXY_JSON) ) summary = { - "theorem": "Conformal Feasibility Guarantee (Theorem 1)", + "certificate": "Funded-set Markov accounting and empirical screens", + "theorem": "Distribution-free Markov bound under weighted funded-set validity", "paper": "CRPTO (CRPTO)", "allocator_mode": allocator_mode, "n_test_observations": len(aligned), @@ -532,6 +547,7 @@ def main(argv: list[str] | None = None) -> None: "conformal_intervals_path": str(args.conformal_intervals_path or ""), "alphas_tested": alpha_grid, "all_bounds_hold": all_pass, + "all_certificate_screens_pass": all_pass, "results": results, } output_json.parent.mkdir(parents=True, exist_ok=True) diff --git a/scripts/validate_conformal_policy.py b/scripts/validate_conformal_policy.py index b22b862..31f1bd1 100644 --- a/scripts/validate_conformal_policy.py +++ b/scripts/validate_conformal_policy.py @@ -18,15 +18,17 @@ import yaml from loguru import logger +from src.utils.artifact_metadata import build_artifact_metadata, resolve_run_tag +from src.utils.baseline_registry import resolve_official_baseline_run_tag + try: - from mapie.metrics.regression import regression_mwi_score as _mapie_mwi_score + from mapie.metrics.regression import regression_mwi_score as _imported_mapie_mwi_score + _mapie_mwi_score: Any = _imported_mapie_mwi_score _MAPIE_MWI_AVAILABLE = True except ImportError: _mapie_mwi_score = None _MAPIE_MWI_AVAILABLE = False -from src.utils.artifact_metadata import build_artifact_metadata, resolve_run_tag -from src.utils.baseline_registry import resolve_official_baseline_run_tag DEFAULT_POLICY_CONFIG = "configs/crpto_conformal_policy.yaml" @@ -50,9 +52,9 @@ def _fallback_winkler_interval_score( _imported_winkler_interval_score: Any try: - from src.evaluation.backtesting import ( - winkler_interval_score as _imported_winkler_interval_score, - ) + from src.evaluation import backtesting as _backtesting + + _imported_winkler_interval_score = _backtesting.winkler_interval_score except ImportError: _imported_winkler_interval_score = _fallback_winkler_interval_score @@ -140,233 +142,309 @@ def _apply_artifact_namespace( return updated -def main( - config_path: str = DEFAULT_POLICY_CONFIG, - run_tag: str | None = None, - sensitivity_config_path: str | None = None, - artifact_namespace: str | None = None, -) -> None: - with open(config_path, encoding="utf-8") as config_handle: - cfg_raw = yaml.safe_load(config_handle) or {} - cfg = dict(cast(Mapping[str, Any], cfg_raw)) +def _load_yaml_dict(path: str | Path) -> dict[str, Any]: + with open(path, encoding="utf-8") as handle: + raw = yaml.safe_load(handle) or {} + return dict(cast(Mapping[str, Any], raw)) + +def _load_policy_config( + *, + config_path: str, + sensitivity_config_path: str | None, + artifact_namespace: str | None, +) -> dict[str, Any]: + cfg = _load_yaml_dict(config_path) if sensitivity_config_path is not None: - with open(sensitivity_config_path, encoding="utf-8") as sensitivity_handle: - sens_raw = yaml.safe_load(sensitivity_handle) or {} - sens_cfg = dict(cast(Mapping[str, Any], sens_raw)) + sens_cfg = _load_yaml_dict(sensitivity_config_path) if "policy_sensitivity" in sens_cfg: cfg["policy_sensitivity"] = sens_cfg["policy_sensitivity"] logger.info( f"Overriding policy_sensitivity from {sensitivity_config_path}: " f"{cfg['policy_sensitivity']}" ) + return _apply_artifact_namespace(cfg, artifact_namespace) - cfg = _apply_artifact_namespace(cfg, artifact_namespace) - policy = dict(cast(Mapping[str, Any], cfg["policy"])) - artifacts = dict(cast(Mapping[str, Any], cfg["artifacts"])) - output = dict(cast(Mapping[str, Any], cfg["output"])) - resolved_run_tag = resolve_run_tag( - run_tag, - fallback_candidates=[resolve_official_baseline_run_tag()], - require_explicit=True, - ) +def _load_alerts(path: Path) -> pd.DataFrame: + if path.exists(): + return pd.read_parquet(path) + return pd.DataFrame({"severity": pd.Series(dtype="object")}) - with open(artifacts["conformal_results_path"], "rb") as results_handle: - results = cast(dict[str, Any], pickle.load(results_handle)) - group_metrics = pd.read_parquet(artifacts["group_metrics_path"]) - backtest_monthly = pd.read_parquet(artifacts["backtest_monthly_path"]) - alerts_path = Path(artifacts["backtest_alerts_path"]) - alerts = ( - pd.read_parquet(alerts_path) if alerts_path.exists() else pd.DataFrame(columns=["severity"]) - ) - intervals_path = Path( - artifacts.get("intervals_path", "data/processed/conformal_intervals_mondrian.parquet") - ) - intervals_df = pd.read_parquet(intervals_path) - lgd_ead_status_path = Path("models/conformal_lgd_ead_status.json") - lgd_ead_status = ( - json.loads(lgd_ead_status_path.read_text(encoding="utf-8")) - if lgd_ead_status_path.exists() - else {"available": False, "reason": "missing_status_artifact"} - ) - metrics_90 = results.get("metrics_90", {}) - metrics_95 = results.get("metrics_95", {}) +def _interval_arrays( + intervals_df: pd.DataFrame, + *, + lower_col: str, + upper_col: str, +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + if {"y_true", lower_col, upper_col}.issubset(intervals_df.columns): + y_true = pd.to_numeric(intervals_df["y_true"], errors="coerce").to_numpy(dtype=float) + lower = pd.to_numeric(intervals_df[lower_col], errors="coerce").to_numpy(dtype=float) + upper = pd.to_numeric(intervals_df[upper_col], errors="coerce").to_numpy(dtype=float) + valid = np.isfinite(y_true) & np.isfinite(lower) & np.isfinite(upper) + return y_true[valid], lower[valid], upper[valid] + empty = np.array([], dtype=float) + return empty, empty, empty - coverage_90 = float(metrics_90.get("empirical_coverage", 0.0)) - coverage_95 = float(metrics_95.get("empirical_coverage", 0.0)) - avg_width_90 = float(metrics_90.get("avg_interval_width", 999.0)) - min_group_coverage_90 = float(group_metrics.get("coverage_90", pd.Series([0.0])).min()) - critical_alerts = int((alerts.get("severity", pd.Series([], dtype=str)) == "critical").sum()) - warning_alerts = int((alerts.get("severity", pd.Series([], dtype=str)) == "warning").sum()) - total_alerts = len(alerts) - # Conformal material-quality checks for the IJDS promotion contract. - if {"y_true", "pd_low_90", "pd_high_90"}.issubset(intervals_df.columns): - y_true = pd.to_numeric(intervals_df["y_true"], errors="coerce").to_numpy(dtype=float) - low_90 = pd.to_numeric(intervals_df["pd_low_90"], errors="coerce").to_numpy(dtype=float) - high_90 = pd.to_numeric(intervals_df["pd_high_90"], errors="coerce").to_numpy(dtype=float) - valid_90 = np.isfinite(y_true) & np.isfinite(low_90) & np.isfinite(high_90) - y90 = y_true[valid_90] - lo90 = low_90[valid_90] - hi90 = high_90[valid_90] - else: - y90 = np.array([], dtype=float) - lo90 = np.array([], dtype=float) - hi90 = np.array([], dtype=float) - - if {"y_true", "pd_low_95", "pd_high_95"}.issubset(intervals_df.columns): - y_true_95 = pd.to_numeric(intervals_df["y_true"], errors="coerce").to_numpy(dtype=float) - low_95 = pd.to_numeric(intervals_df["pd_low_95"], errors="coerce").to_numpy(dtype=float) - high_95 = pd.to_numeric(intervals_df["pd_high_95"], errors="coerce").to_numpy(dtype=float) - valid_95 = np.isfinite(y_true_95) & np.isfinite(low_95) & np.isfinite(high_95) - y95 = y_true_95[valid_95] - lo95 = low_95[valid_95] - hi95 = high_95[valid_95] - else: - y95 = np.array([], dtype=float) - lo95 = np.array([], dtype=float) - hi95 = np.array([], dtype=float) - - winkler_90 = ( - float(np.mean(winkler_interval_score(y90, lo90, hi90, alpha=0.10))) - if y90.size - else float("inf") - ) - winkler_95 = ( - float(np.mean(winkler_interval_score(y95, lo95, hi95, alpha=0.05))) - if y95.size - else float("inf") - ) +def _mean_winkler( + y_true: np.ndarray, + lower: np.ndarray, + upper: np.ndarray, + *, + alpha: float, +) -> float: + if not y_true.size: + return float("inf") + return float(np.mean(winkler_interval_score(y_true, lower, upper, alpha=alpha))) + + +def _violation_rate(y_true: np.ndarray, lower: np.ndarray, upper: np.ndarray) -> float: + if not y_true.size: + return float("nan") + return float(np.mean((y_true < lower) | (y_true > upper))) + - # Cross-check: MAPIE native regression_mwi_score (should match manual Winkler) - mapie_mwi_90: float | None = None - if _MAPIE_MWI_AVAILABLE and y90.size: - try: - mapie_mwi_90 = _compute_mapie_mwi_score( - y90, - lo90, - hi90, - confidence_level=0.90, +def _mapie_mwi_cross_check( + *, + y_true: np.ndarray, + lower: np.ndarray, + upper: np.ndarray, + winkler_score: float, + confidence_level: float, +) -> float | None: + if not (_MAPIE_MWI_AVAILABLE and y_true.size): + return None + try: + mapie_score = _compute_mapie_mwi_score( + y_true, + lower, + upper, + confidence_level=confidence_level, + ) + delta = abs(mapie_score - winkler_score) + if delta > 0.01: + logger.warning( + f"MAPIE MWI ({mapie_score:.4f}) deviates from manual Winkler " + f"({winkler_score:.4f}) by {delta:.4f} -- check score definition." ) - delta = abs(mapie_mwi_90 - winkler_90) - if delta > 0.01: - logger.warning( - f"MAPIE MWI ({mapie_mwi_90:.4f}) deviates from manual Winkler " - f"({winkler_90:.4f}) by {delta:.4f} — check score definition." - ) - else: - logger.info(f"MAPIE MWI cross-check OK: {mapie_mwi_90:.4f} ≈ {winkler_90:.4f}") - except Exception as exc: - logger.warning(f"MAPIE MWI cross-check failed: {exc}") - violation_rate_90 = float(np.mean((y90 < lo90) | (y90 > hi90))) if y90.size else float("nan") - violation_rate_95 = float(np.mean((y95 < lo95) | (y95 > hi95))) if y95.size else float("nan") + else: + logger.info(f"MAPIE MWI cross-check OK: {mapie_score:.4f} ~= {winkler_score:.4f}") + return mapie_score + except Exception as exc: + logger.warning(f"MAPIE MWI cross-check failed: {exc}") + return None + + +def _evaluate_check_frame(frame: pd.DataFrame) -> dict[str, Any]: + gate_pass = bool(frame["passed"].all()) + failing_material = frame.loc[~frame["passed"], "metric"].astype(str).tolist() + methodological_status = ( + "not_needed_material_gate_pass" if gate_pass else "blocked_material_gate_failures" + ) + return { + "strict_overall_pass": gate_pass, + "gate_overall_pass": gate_pass, + "non_statistical_checks_pass": gate_pass, + "diagnostic_statistical_pass": True, + "failing_checks": failing_material, + "failing_statistical_checks": [], + "gate_failing_checks": failing_material, + "failing_non_statistical_checks": failing_material, + "diagnostic_failing_checks": [], + "methodological_justification_pass": gate_pass, + "methodological_justification_status": methodological_status, + } + +def _winkler_90_check(policy: dict[str, Any], metrics: dict[str, float]) -> dict[str, object]: max_winkler_90 = float(policy.get("max_winkler_90", float("inf"))) - max_winkler_95 = float(policy.get("max_winkler_95", float("inf"))) - enable_compensated_winkler_90 = bool(policy.get("enable_compensated_winkler_90", False)) - compensated_winkler_90_max = float(policy.get("compensated_winkler_90_max", max_winkler_90)) - compensated_min_coverage_90 = float( + enable_compensated = bool(policy.get("enable_compensated_winkler_90", False)) + compensated_threshold = float(policy.get("compensated_winkler_90_max", max_winkler_90)) + compensated_min_coverage = float( policy.get("compensated_min_coverage_90", policy["target_coverage_90_min"]) ) - compensated_min_group_coverage_90 = float( - policy.get( - "compensated_min_group_coverage_90", - policy["min_group_coverage_90_min"], - ) + compensated_min_group_coverage = float( + policy.get("compensated_min_group_coverage_90", policy["min_group_coverage_90_min"]) ) - compensated_max_avg_width_90 = float( + compensated_max_avg_width = float( policy.get("compensated_max_avg_width_90", policy["max_avg_width_90"]) ) - winkler_90_raw_pass = bool(winkler_90 <= max_winkler_90) - winkler_90_compensated_pass = bool( - enable_compensated_winkler_90 - and (not winkler_90_raw_pass) - and winkler_90 <= compensated_winkler_90_max - and coverage_90 >= compensated_min_coverage_90 - and min_group_coverage_90 >= compensated_min_group_coverage_90 - and avg_width_90 <= compensated_max_avg_width_90 - and critical_alerts <= float(policy["max_critical_alerts"]) - ) - winkler_90_policy_pass = bool(winkler_90_raw_pass or winkler_90_compensated_pass) - winkler_90_policy_mode = ( - "strict" - if winkler_90_raw_pass - else "compensated_band" - if winkler_90_compensated_pass - else "strict" + raw_pass = bool(metrics["winkler_90"] <= max_winkler_90) + compensated_pass = bool( + enable_compensated + and (not raw_pass) + and metrics["winkler_90"] <= compensated_threshold + and metrics["coverage_90"] >= compensated_min_coverage + and metrics["min_group_coverage_90"] >= compensated_min_group_coverage + and metrics["avg_width_90"] <= compensated_max_avg_width + and metrics["critical_alerts"] <= float(policy["max_critical_alerts"]) ) - winkler_90_check = _check("winkler_90", winkler_90, max_winkler_90, "<=", "quality") - winkler_90_check["passed"] = bool(winkler_90_policy_pass) - winkler_90_check["policy_mode"] = str(winkler_90_policy_mode) - winkler_90_check["raw_threshold"] = float(max_winkler_90) - winkler_90_check["raw_passed"] = bool(winkler_90_raw_pass) - winkler_90_check["compensated_band_enabled"] = bool(enable_compensated_winkler_90) - winkler_90_check["compensated_threshold"] = float(compensated_winkler_90_max) - winkler_90_check["compensated_passed"] = bool(winkler_90_compensated_pass) - - checks = [ + policy_pass = bool(raw_pass or compensated_pass) + policy_mode = "strict" if raw_pass else "compensated_band" if compensated_pass else "strict" + check = _check("winkler_90", metrics["winkler_90"], max_winkler_90, "<=", "quality") + check["passed"] = bool(policy_pass) + check["policy_mode"] = str(policy_mode) + check["raw_threshold"] = float(max_winkler_90) + check["raw_passed"] = bool(raw_pass) + check["compensated_band_enabled"] = bool(enable_compensated) + check["compensated_threshold"] = float(compensated_threshold) + check["compensated_passed"] = bool(compensated_pass) + return check + + +def _policy_checks(policy: dict[str, Any], metrics: dict[str, float]) -> list[dict[str, object]]: + return [ _check( - "coverage_90", coverage_90, float(policy["target_coverage_90_min"]), ">=", "portfolio" + "coverage_90", + metrics["coverage_90"], + float(policy["target_coverage_90_min"]), + ">=", + "portfolio", ), _check( - "coverage_95", coverage_95, float(policy["target_coverage_95_min"]), ">=", "portfolio" + "coverage_95", + metrics["coverage_95"], + float(policy["target_coverage_95_min"]), + ">=", + "portfolio", ), _check( "min_group_coverage_90", - min_group_coverage_90, + metrics["min_group_coverage_90"], float(policy["min_group_coverage_90_min"]), ">=", "group", ), - _check("avg_width_90", avg_width_90, float(policy["max_avg_width_90"]), "<=", "portfolio"), + _check( + "avg_width_90", + metrics["avg_width_90"], + float(policy["max_avg_width_90"]), + "<=", + "portfolio", + ), _check( "critical_alerts", - float(critical_alerts), + metrics["critical_alerts"], float(policy["max_critical_alerts"]), "<=", "monitoring", ), _check( "total_alerts", - float(total_alerts), + metrics["total_alerts"], float(policy["max_total_alerts"]), "<=", "monitoring", ), _check( "warning_alerts", - float(warning_alerts), + metrics["warning_alerts"], float(policy["max_warning_alerts"]), "<=", "monitoring", ), - winkler_90_check, - _check("winkler_95", winkler_95, max_winkler_95, "<=", "quality"), + _winkler_90_check(policy, metrics), + _check( + "winkler_95", + metrics["winkler_95"], + float(policy.get("max_winkler_95", float("inf"))), + "<=", + "quality", + ), ] - checks_df = pd.DataFrame(checks) - def _evaluate_check_frame(frame: pd.DataFrame) -> dict[str, Any]: - gate_pass = bool(frame["passed"].all()) - failing_material = frame.loc[~frame["passed"], "metric"].astype(str).tolist() - methodological_status = ( - "not_needed_material_gate_pass" if gate_pass else "blocked_material_gate_failures" - ) - return { - "strict_overall_pass": gate_pass, - "gate_overall_pass": gate_pass, - "non_statistical_checks_pass": gate_pass, - "diagnostic_statistical_pass": True, - "failing_checks": failing_material, - "failing_statistical_checks": [], - "gate_failing_checks": failing_material, - "failing_non_statistical_checks": failing_material, - "diagnostic_failing_checks": [], - "methodological_justification_pass": gate_pass, - "methodological_justification_status": methodological_status, - } + +def _latest_backtest_month(backtest_monthly: pd.DataFrame) -> object | None: + if backtest_monthly.empty: + return None + return backtest_monthly.sort_values("month").iloc[-1]["month"] + + +def main( + config_path: str = DEFAULT_POLICY_CONFIG, + run_tag: str | None = None, + sensitivity_config_path: str | None = None, + artifact_namespace: str | None = None, +) -> None: + cfg = _load_policy_config( + config_path=config_path, + sensitivity_config_path=sensitivity_config_path, + artifact_namespace=artifact_namespace, + ) + + policy = dict(cast(Mapping[str, Any], cfg["policy"])) + artifacts = dict(cast(Mapping[str, Any], cfg["artifacts"])) + output = dict(cast(Mapping[str, Any], cfg["output"])) + resolved_run_tag = resolve_run_tag( + run_tag, + fallback_candidates=[resolve_official_baseline_run_tag()], + require_explicit=True, + ) + + with open(artifacts["conformal_results_path"], "rb") as results_handle: + results = cast(dict[str, Any], pickle.load(results_handle)) + group_metrics = pd.read_parquet(artifacts["group_metrics_path"]) + backtest_monthly = pd.read_parquet(artifacts["backtest_monthly_path"]) + alerts_path = Path(artifacts["backtest_alerts_path"]) + alerts = _load_alerts(alerts_path) + intervals_path = Path( + artifacts.get("intervals_path", "data/processed/conformal_intervals_mondrian.parquet") + ) + intervals_df = pd.read_parquet(intervals_path) + lgd_ead_status_path = Path("models/conformal_lgd_ead_status.json") + lgd_ead_status = ( + json.loads(lgd_ead_status_path.read_text(encoding="utf-8")) + if lgd_ead_status_path.exists() + else {"available": False, "reason": "missing_status_artifact"} + ) + + metrics_90 = results.get("metrics_90", {}) + metrics_95 = results.get("metrics_95", {}) + + coverage_90 = float(metrics_90.get("empirical_coverage", 0.0)) + coverage_95 = float(metrics_95.get("empirical_coverage", 0.0)) + avg_width_90 = float(metrics_90.get("avg_interval_width", 999.0)) + min_group_coverage_90 = float(group_metrics.get("coverage_90", pd.Series([0.0])).min()) + critical_alerts = int((alerts.get("severity", pd.Series([], dtype=str)) == "critical").sum()) + warning_alerts = int((alerts.get("severity", pd.Series([], dtype=str)) == "warning").sum()) + total_alerts = len(alerts) + + y90, lo90, hi90 = _interval_arrays(intervals_df, lower_col="pd_low_90", upper_col="pd_high_90") + y95, lo95, hi95 = _interval_arrays(intervals_df, lower_col="pd_low_95", upper_col="pd_high_95") + winkler_90 = _mean_winkler(y90, lo90, hi90, alpha=0.10) + winkler_95 = _mean_winkler(y95, lo95, hi95, alpha=0.05) + mapie_mwi_90 = _mapie_mwi_cross_check( + y_true=y90, + lower=lo90, + upper=hi90, + winkler_score=winkler_90, + confidence_level=0.90, + ) + violation_rate_90 = _violation_rate(y90, lo90, hi90) + violation_rate_95 = _violation_rate(y95, lo95, hi95) + + metrics = { + "coverage_90": coverage_90, + "coverage_95": coverage_95, + "avg_width_90": avg_width_90, + "min_group_coverage_90": min_group_coverage_90, + "critical_alerts": float(critical_alerts), + "warning_alerts": float(warning_alerts), + "total_alerts": float(total_alerts), + "winkler_90": winkler_90, + "winkler_95": winkler_95, + } + checks = _policy_checks(policy, metrics) + checks_df = pd.DataFrame(checks) + winkler_90_check = next(check for check in checks if check["metric"] == "winkler_90") + winkler_90_raw_pass = bool(winkler_90_check.get("raw_passed", False)) + winkler_90_policy_pass = bool(winkler_90_check.get("passed", False)) + winkler_90_policy_mode = str(winkler_90_check.get("policy_mode", "strict")) + winkler_90_compensated_pass = bool(winkler_90_check.get("compensated_passed", False)) + compensated_winkler_90_max = _safe_float(winkler_90_check.get("compensated_threshold", np.nan)) evaluation = _evaluate_check_frame(checks_df) strict_overall_pass = bool(evaluation["strict_overall_pass"]) @@ -382,11 +460,7 @@ def _evaluate_check_frame(frame: pd.DataFrame) -> dict[str, Any]: methodological_status = str(evaluation["methodological_justification_status"]) overall_pass = gate_overall_pass - latest_month = ( - backtest_monthly.sort_values("month").iloc[-1]["month"] - if not backtest_monthly.empty - else None - ) + latest_month = _latest_backtest_month(backtest_monthly) out_status = { "overall_pass": overall_pass, diff --git a/src/evaluation/backtesting.py b/src/evaluation/backtesting.py index 5d051c7..9978662 100644 --- a/src/evaluation/backtesting.py +++ b/src/evaluation/backtesting.py @@ -34,11 +34,13 @@ def cohort_analysis( for cohort, group in df.groupby(cohort_col): if len(group) < 50: continue - metrics = classification_metrics( - group[y_true_col].values, - group[y_prob_col].values, + metrics: dict[str, object] = dict( + classification_metrics( + group[y_true_col].values, + group[y_prob_col].values, + ) ) - metrics["cohort"] = cohort + metrics["cohort"] = str(cohort) metrics["n_loans"] = len(group) metrics["default_rate"] = group[y_true_col].mean() results.append(metrics) @@ -502,19 +504,19 @@ def drift_monitoring_report( if not rows: return pd.DataFrame( - columns=[ - "feature", - "train_n", - "test_n", - "psi", - "ks_statistic", - "ks_pvalue", - "cvm_statistic", - "cvm_pvalue", - "pass_psi", - "pass_ks", - "pass_cvm", - ] + { + "feature": pd.Series(dtype="object"), + "train_n": pd.Series(dtype="int64"), + "test_n": pd.Series(dtype="int64"), + "psi": pd.Series(dtype="float64"), + "ks_statistic": pd.Series(dtype="float64"), + "ks_pvalue": pd.Series(dtype="float64"), + "cvm_statistic": pd.Series(dtype="float64"), + "cvm_pvalue": pd.Series(dtype="float64"), + "pass_psi": pd.Series(dtype="bool"), + "pass_ks": pd.Series(dtype="bool"), + "pass_cvm": pd.Series(dtype="bool"), + } ) out = pd.DataFrame(rows).sort_values("psi", ascending=False).reset_index(drop=True) return out @@ -566,8 +568,8 @@ def filter_high_psi_features( ) psi_table = pd.DataFrame(psi_records).sort_values("psi", ascending=False).reset_index(drop=True) - stable = [r["feature"] for r in psi_records if r["stable"]] - drifted = [r["feature"] for r in psi_records if not r["stable"]] + stable = [str(r["feature"]) for r in psi_records if r["stable"]] + drifted = [str(r["feature"]) for r in psi_records if not r["stable"]] if drifted: logger.warning( diff --git a/src/evaluation/explainability.py b/src/evaluation/explainability.py index 7b5b489..ee2b6f8 100644 --- a/src/evaluation/explainability.py +++ b/src/evaluation/explainability.py @@ -157,14 +157,14 @@ def pairwise_shap_redundancy( ) if not rows: return pd.DataFrame( - columns=[ - "feature_a", - "feature_b", - "shap_spearman", - "value_spearman", - "redundancy_flag", - "relation_type", - ] + { + "feature_a": pd.Series(dtype="object"), + "feature_b": pd.Series(dtype="object"), + "shap_spearman": pd.Series(dtype="float64"), + "value_spearman": pd.Series(dtype="float64"), + "redundancy_flag": pd.Series(dtype="bool"), + "relation_type": pd.Series(dtype="object"), + } ) return ( pd.DataFrame(rows) diff --git a/src/evaluation/fairness.py b/src/evaluation/fairness.py index 70b67cd..e5d68fc 100644 --- a/src/evaluation/fairness.py +++ b/src/evaluation/fairness.py @@ -45,8 +45,8 @@ def demographic_parity_difference( rates = list(group_rates.values()) dpd = max(rates) - min(rates) - max_group = max(group_rates, key=group_rates.get) # type: ignore[arg-type] - min_group = min(group_rates, key=group_rates.get) # type: ignore[arg-type] + max_group = max(group_rates, key=lambda key: group_rates[key]) + min_group = min(group_rates, key=lambda key: group_rates[key]) return { "dpd": dpd, @@ -310,18 +310,18 @@ def fairness_threshold_frontier( ) if not rows: return pd.DataFrame( - columns=[ - "attribute", - "threshold", - "is_primary_threshold", - "dpd", - "eo_gap", - "dir", - "passed_dpd", - "passed_eo", - "passed_dir", - "passed_all", - ] + { + "attribute": pd.Series(dtype="object"), + "threshold": pd.Series(dtype="float64"), + "is_primary_threshold": pd.Series(dtype="bool"), + "dpd": pd.Series(dtype="float64"), + "eo_gap": pd.Series(dtype="float64"), + "dir": pd.Series(dtype="float64"), + "passed_dpd": pd.Series(dtype="bool"), + "passed_eo": pd.Series(dtype="bool"), + "passed_dir": pd.Series(dtype="bool"), + "passed_all": pd.Series(dtype="bool"), + } ) return pd.DataFrame(rows).sort_values(["attribute", "threshold"]).reset_index(drop=True) diff --git a/src/evaluation/model_shift.py b/src/evaluation/model_shift.py index c120ddc..b9f9d57 100644 --- a/src/evaluation/model_shift.py +++ b/src/evaluation/model_shift.py @@ -20,79 +20,124 @@ def interpret_model_shift( calibration_gap_delta_max: float, ) -> dict[str, Any]: """Distinguish structural shift from predictive degradation.""" - structural_level = "none" + structural_level = _structural_shift_level( + c2st_auc=c2st_auc, + score_psi=score_psi, + distribution_warning_ratio=distribution_warning_ratio, + score_psi_max=score_psi_max, + ) + predictive_level = _predictive_degradation_level( + auc_delta=auc_delta, + brier_increase=brier_increase, + calibration_gap_delta=calibration_gap_delta, + auc_delta_max=auc_delta_max, + brier_increase_max=brier_increase_max, + calibration_gap_delta_max=calibration_gap_delta_max, + ) + shift_type = _shift_type(structural_level, predictive_level) + + return { + "shift_type": shift_type, + "structural_shift_level": structural_level, + "predictive_degradation_level": predictive_level, + "governance_posture": _governance_posture(shift_type, predictive_level), + "c2st_materiality": str(c2st_materiality), + "pvalue_interpretation": _pvalue_note(shift_type), + } + + +def _structural_shift_level( + *, + c2st_auc: float, + score_psi: float, + distribution_warning_ratio: float, + score_psi_max: float, +) -> str: if c2st_auc >= 0.70 or score_psi >= max(score_psi_max * 1.5, 0.20): - structural_level = "severe" - elif c2st_auc >= 0.60 or score_psi >= score_psi_max: - structural_level = "high" - elif c2st_auc >= 0.55 or score_psi >= max(score_psi_max * 0.7, 0.10): - structural_level = "moderate" - elif c2st_auc >= 0.52 or distribution_warning_ratio > 0.05: - structural_level = "low" - - predictive_level = "none" - if ( - auc_delta >= auc_delta_max * 1.5 - or brier_increase >= brier_increase_max * 1.5 - or calibration_gap_delta >= calibration_gap_delta_max * 1.5 - ): - predictive_level = "severe" - elif ( - auc_delta > auc_delta_max - or brier_increase > brier_increase_max - or calibration_gap_delta > calibration_gap_delta_max - ): - predictive_level = "high" - elif ( - auc_delta >= auc_delta_max * 0.75 - or brier_increase >= brier_increase_max * 0.75 - or calibration_gap_delta >= calibration_gap_delta_max * 0.75 - ): - predictive_level = "moderate" - elif auc_delta > 0.0 or brier_increase > 0.0 or calibration_gap_delta > 0.0: - predictive_level = "low" + return "severe" + if c2st_auc >= 0.60 or score_psi >= score_psi_max: + return "high" + if c2st_auc >= 0.55 or score_psi >= max(score_psi_max * 0.7, 0.10): + return "moderate" + if c2st_auc >= 0.52 or distribution_warning_ratio > 0.05: + return "low" + return "none" + + +def _predictive_degradation_level( + *, + auc_delta: float, + brier_increase: float, + calibration_gap_delta: float, + auc_delta_max: float, + brier_increase_max: float, + calibration_gap_delta_max: float, +) -> str: + metrics = ( + (auc_delta, auc_delta_max), + (brier_increase, brier_increase_max), + (calibration_gap_delta, calibration_gap_delta_max), + ) + if _any_metric_crosses(metrics, multiplier=1.5, strict=False): + return "severe" + if _any_metric_crosses(metrics, multiplier=1.0, strict=True): + return "high" + if _any_metric_crosses(metrics, multiplier=0.75, strict=False): + return "moderate" + if any(value > 0.0 for value, _ in metrics): + return "low" + return "none" + + +def _any_metric_crosses( + metrics: tuple[tuple[float, float], ...], + *, + multiplier: float, + strict: bool, +) -> bool: + for value, limit in metrics: + threshold = limit * multiplier + crosses = value > threshold if strict else value >= threshold + if crosses: + return True + return False + +def _shift_type(structural_level: str, predictive_level: str) -> str: if structural_level != "none" and predictive_level in {"none", "low"}: - shift_type = "structural_shift_only" - elif structural_level == "none" and predictive_level != "none": - shift_type = "predictive_degradation" - elif structural_level != "none" and predictive_level != "none": - shift_type = "mixed_shift" - else: - shift_type = "stable" + return "structural_shift_only" + if structural_level == "none" and predictive_level != "none": + return "predictive_degradation" + if structural_level != "none" and predictive_level != "none": + return "mixed_shift" + return "stable" + +def _governance_posture(shift_type: str, predictive_level: str) -> str: if shift_type == "mixed_shift" or predictive_level in {"high", "severe"}: - governance_posture = "candidate_gate" - elif shift_type == "structural_shift_only": - governance_posture = "warning_only" - else: - governance_posture = "monitor" + return "candidate_gate" + if shift_type == "structural_shift_only": + return "warning_only" + return "monitor" - pvalue_note = ( - "Distribution p-values are informative, but they can trigger on small effects at large N; " - "the governance posture should be anchored on materiality, predictive loss, and C2ST/score shift." - ) + +def _pvalue_note(shift_type: str) -> str: if shift_type == "structural_shift_only": - pvalue_note = ( + return ( "KS/CvM or C2ST detect structural population change without clear predictive degradation. " "Treat this as representativeness pressure and monitoring, not an automatic retraining gate." ) - elif shift_type == "predictive_degradation": - pvalue_note = ( + if shift_type == "predictive_degradation": + return ( "Predictive degradation is visible even without strong structural shift. " "Operational metrics dominate p-value interpretation in this case." ) - elif shift_type == "mixed_shift": - pvalue_note = ( + if shift_type == "mixed_shift": + return ( "Both structural shift and predictive degradation are present. " "This combination deserves the strongest governance posture." ) - - return { - "shift_type": shift_type, - "structural_shift_level": structural_level, - "predictive_degradation_level": predictive_level, - "governance_posture": governance_posture, - "c2st_materiality": str(c2st_materiality), - "pvalue_interpretation": pvalue_note, - } + return ( + "Distribution p-values are informative, but they can trigger on small effects at large N; " + "the governance posture should be anchored on materiality, predictive loss, and C2ST/score shift." + ) diff --git a/src/features/feature_config_io.py b/src/features/feature_config_io.py index 7a54b91..0dd38f2 100644 --- a/src/features/feature_config_io.py +++ b/src/features/feature_config_io.py @@ -208,8 +208,8 @@ def _frame_to_config(frame: pd.DataFrame) -> dict[str, Any]: cfg[str(section)] = [json.loads(raw) for raw in group["value_json"]] elif kind == "dict": cfg[str(section)] = { - str(row.key): json.loads(str(row.value_json)) - for row in group.itertuples(index=False) + str(row["key"]): json.loads(str(row["value_json"])) + for row in group.to_dict("records") } elif kind == "scalar": if len(group) != 1: diff --git a/src/features/feature_engineering.py b/src/features/feature_engineering.py index df5ec69..5918ac9 100644 --- a/src/features/feature_engineering.py +++ b/src/features/feature_engineering.py @@ -797,7 +797,8 @@ def save_feature_artifacts( feature_config=feature_config, ) atomic_write_parquet(manifest, out_dir / "feature_manifest_v2.parquet", index=False) - atomic_write_text( - out_dir / "feature_manifest_v2.json", manifest.to_json(orient="records", indent=2) - ) + manifest_json = manifest.to_json(orient="records", indent=2) + if manifest_json is None: + raise RuntimeError("pandas returned no feature manifest JSON payload") + atomic_write_text(out_dir / "feature_manifest_v2.json", manifest_json) logger.info("Saved canonical feature artifacts to {}", out_dir) diff --git a/src/features/tabprep_challenger.py b/src/features/tabprep_challenger.py index 104ac27..6d65c55 100644 --- a/src/features/tabprep_challenger.py +++ b/src/features/tabprep_challenger.py @@ -14,7 +14,7 @@ import itertools from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass, field -from typing import Any +from typing import Any, cast import numpy as np import pandas as pd @@ -486,14 +486,14 @@ def _select_arithmetic_specs(self, frame: pd.DataFrame, y: pd.Series) -> list[Fe triple_candidates: list[FeatureSpec] = [] triple_base = numeric[: min(20, len(numeric))] - for sources in itertools.combinations(triple_base, 3): + for triple_sources in itertools.combinations(triple_base, 3): for operation in ("product3", "sum3", "ratio_sum"): - name = _make_feature_name("tp_ar", operation, sources) + name = _make_feature_name("tp_ar", operation, triple_sources) spec = FeatureSpec( name=name, generator="arithmetic", operation=operation, - source_features=sources, + source_features=triple_sources, score=0.0, selected_rank=0, ) @@ -953,6 +953,7 @@ def _fit_groupby_state( if operation == "pct_rank": state.global_quantiles = _quantile_grid(values) for group, group_values in tmp.groupby("group", observed=True)["value"]: + group_values = cast(pd.Series, group_values) if len(group_values) >= int(min_support): state.group_quantiles[str(group)] = _quantile_grid(group_values) return state diff --git a/src/models/calibration.py b/src/models/calibration.py index fa821f2..06266cc 100644 --- a/src/models/calibration.py +++ b/src/models/calibration.py @@ -29,7 +29,8 @@ def transform(self, scores: np.ndarray) -> np.ndarray: scores_arr = np.clip(np.asarray(scores, dtype=float), 1e-6, 1.0 - 1e-6) logits = np.log(scores_arr / (1.0 - scores_arr)) shifted = 1.0 / (1.0 + np.exp(-(logits + self.delta))) - return cast(np.ndarray, np.clip(np.asarray(shifted, dtype=float), 0.0, 1.0)) + clipped: np.ndarray = np.clip(np.asarray(shifted, dtype=float), 0.0, 1.0) + return clipped def predict(self, scores: np.ndarray) -> np.ndarray: return self.transform(scores) @@ -51,11 +52,12 @@ def __init__(self, temperature: float = 1.0) -> None: @staticmethod def _logit(scores: np.ndarray) -> np.ndarray: clipped = np.clip(np.asarray(scores, dtype=float).reshape(-1), 1e-6, 1.0 - 1e-6) - return np.log(clipped / (1.0 - clipped)) + logits: np.ndarray = np.log(clipped / (1.0 - clipped)) + return logits @staticmethod def _sigmoid(logits: np.ndarray) -> np.ndarray: - return 1.0 / (1.0 + np.exp(-logits)) + return cast(np.ndarray, 1.0 / (1.0 + np.exp(-logits))) def fit(self, y_prob_raw: np.ndarray, y_true: np.ndarray) -> TemperatureScalingCalibrator: logits = self._logit(y_prob_raw) @@ -76,10 +78,8 @@ def predict(self, y_prob_raw: np.ndarray) -> np.ndarray: if not self._is_fitted: raise RuntimeError("TemperatureScalingCalibrator is not fitted.") logits = self._logit(y_prob_raw) - return cast( - np.ndarray, - np.clip(self._sigmoid(logits / max(self.temperature, 1e-3)), 0.0, 1.0), - ) + clipped: np.ndarray = np.clip(self._sigmoid(logits / max(self.temperature, 1e-3)), 0.0, 1.0) + return clipped class QuadraticLogitCalibrator: @@ -130,7 +130,11 @@ def adaptive_calibration_error(y_true: np.ndarray, y_prob: np.ndarray, n_bins: i order = np.argsort(p_arr, kind="mergesort") y_sorted = y_arr[order] p_sorted = p_arr[order] - bins = [chunk for chunk in np.array_split(np.arange(len(y_sorted)), max(1, int(n_bins))) if len(chunk)] + bins = [ + chunk + for chunk in np.array_split(np.arange(len(y_sorted)), max(1, int(n_bins))) + if len(chunk) + ] ace = 0.0 for idx in bins: bin_acc = float(y_sorted[idx].mean()) diff --git a/src/models/conformal_adapters.py b/src/models/conformal_adapters.py index 293c59b..8918709 100644 --- a/src/models/conformal_adapters.py +++ b/src/models/conformal_adapters.py @@ -29,7 +29,7 @@ import numpy as np import pandas as pd -from sklearn.base import BaseEstimator, RegressorMixin +from sklearn.base import BaseEstimator, ClassifierMixin, RegressorMixin # Canonical module path that pickled instances must report. _PICKLE_MODULE = "src.models.conformal" @@ -60,7 +60,7 @@ def predict(self, X: Any) -> np.ndarray: return apply_probability_calibrator(self.calibrator, raw) -class PrefitClassifierAdapter(BaseEstimator): +class PrefitClassifierAdapter(ClassifierMixin, BaseEstimator): """Small sklearn-style adapter for prefit classifiers inside MAPIE checks.""" def __init__(self, classifier: Any, n_features_in: int | None = None) -> None: @@ -80,9 +80,7 @@ def _is_minimal_probe(self, X: pd.DataFrame) -> bool: if X.shape[0] != 1 or X.shape[1] != self.n_features_in_: return False numeric = X.apply(pd.to_numeric, errors="coerce") - return bool( - np.isfinite(numeric.to_numpy()).all() and np.allclose(numeric.to_numpy(), 0.0) - ) + return bool(np.isfinite(numeric.to_numpy()).all() and np.allclose(numeric.to_numpy(), 0.0)) def predict(self, X: Any) -> np.ndarray: X_df = pd.DataFrame(X) if not isinstance(X, pd.DataFrame) else X diff --git a/src/models/conformal_tuning.py b/src/models/conformal_tuning.py index 7ac6d99..e32226f 100644 --- a/src/models/conformal_tuning.py +++ b/src/models/conformal_tuning.py @@ -10,6 +10,7 @@ from __future__ import annotations +from dataclasses import dataclass from typing import Any, cast import numpy as np @@ -542,6 +543,215 @@ def temporal_stability_summary( } +@dataclass(frozen=True) +class _ShrinkContext: + y_true: np.ndarray + y_pred: np.ndarray + base_intervals: np.ndarray + groups: pd.Series + temporal_segments: pd.Series | None + issue_dates: pd.Series | np.ndarray | None + target_coverage: float + min_group_coverage_target: float + max_monthly_gap_target: float | None + alpha: float + + +@dataclass(frozen=True) +class _ShrinkCandidate: + scope: str + key: str + factor: float + accepted: bool + intervals: np.ndarray + group_factors: dict[str, float] + temporal_factors: dict[str, float] + metrics: dict[str, float] + + +def _active_widening_factors(factors: dict[str, float] | None) -> dict[str, float]: + return {str(k): float(v) for k, v in (factors or {}).items() if float(v) > 1.0} + + +def _apply_shrink_factors( + context: _ShrinkContext, + group_factors: dict[str, float], + temporal_factors: dict[str, float], +) -> np.ndarray: + intervals = context.base_intervals.copy() + if group_factors: + intervals = apply_group_multipliers( + context.y_pred, + intervals, + context.groups, + group_factors, + ) + if temporal_factors and context.temporal_segments is not None: + intervals = apply_group_multipliers( + context.y_pred, + intervals, + context.temporal_segments, + temporal_factors, + ) + return intervals + + +def _shrink_metrics(context: _ShrinkContext, intervals: np.ndarray) -> dict[str, float]: + temporal = temporal_stability_summary( + context.y_true, + intervals, + context.issue_dates, + target_coverage=context.target_coverage, + freq="M", + ) + return { + "coverage": empirical_interval_coverage(context.y_true, intervals), + "min_group_coverage": min_group_interval_coverage( + context.y_true, + intervals, + context.groups, + ), + "avg_width": average_interval_width(intervals), + "winkler_90": mean_winkler_score(context.y_true, intervals, alpha=context.alpha), + "max_monthly_gap": float(temporal["max_monthly_gap"]), + "stability_over_time": float(temporal["stability_over_time"]), + } + + +def _shrink_constraints_ok(context: _ShrinkContext, metrics: dict[str, float]) -> bool: + if float(metrics["coverage"]) < context.target_coverage: + return False + if float(metrics["min_group_coverage"]) < context.min_group_coverage_target: + return False + return not ( + context.max_monthly_gap_target is not None + and np.isfinite(context.max_monthly_gap_target) + and float(metrics["max_monthly_gap"]) > context.max_monthly_gap_target + ) + + +def _next_lower_factor(value: float, grid: tuple[float, ...]) -> float | None: + ordered = sorted({round(float(x), 6) for x in grid if float(x) <= float(value) + 1e-9}) + current = round(float(value), 6) + if current not in ordered: + ordered.append(current) + ordered = sorted(set(ordered)) + idx = ordered.index(current) + if idx == 0: + return None + return float(ordered[idx - 1]) + + +def _with_reduced_factor( + factors: dict[str, float], + key: str, + next_factor: float, +) -> dict[str, float]: + reduced = dict(factors) + if next_factor <= 1.0: + reduced.pop(key, None) + else: + reduced[key] = float(next_factor) + return reduced + + +def _evaluate_shrink_candidate( + context: _ShrinkContext, + *, + scope: str, + key: str, + factor: float, + group_factors: dict[str, float], + temporal_factors: dict[str, float], +) -> _ShrinkCandidate: + intervals = _apply_shrink_factors(context, group_factors, temporal_factors) + metrics = _shrink_metrics(context, intervals) + return _ShrinkCandidate( + scope=scope, + key=key, + factor=float(factor), + accepted=_shrink_constraints_ok(context, metrics), + intervals=intervals, + group_factors=dict(group_factors), + temporal_factors=dict(temporal_factors), + metrics=metrics, + ) + + +def _build_shrink_attempts( + context: _ShrinkContext, + *, + scope: str, + group_factors: dict[str, float], + temporal_factors: dict[str, float], + multiplier_grid: tuple[float, ...], +) -> list[_ShrinkCandidate]: + active_factors = group_factors if scope == "group" else temporal_factors + candidates: list[_ShrinkCandidate] = [] + for key, value in list(active_factors.items()): + next_factor = _next_lower_factor(value, multiplier_grid) + if next_factor is None: + continue + trial_group = ( + _with_reduced_factor(group_factors, key, next_factor) + if scope == "group" + else dict(group_factors) + ) + trial_temporal = ( + _with_reduced_factor(temporal_factors, key, next_factor) + if scope == "temporal" + else dict(temporal_factors) + ) + candidates.append( + _evaluate_shrink_candidate( + context, + scope=scope, + key=key, + factor=next_factor, + group_factors=trial_group, + temporal_factors=trial_temporal, + ) + ) + return candidates + + +def _shrink_candidate_is_better( + candidate: _ShrinkCandidate, + best_candidate: _ShrinkCandidate | None, +) -> bool: + if not candidate.accepted: + return False + if best_candidate is None: + return True + candidate_width = float(candidate.metrics["avg_width"]) + best_width = float(best_candidate.metrics["avg_width"]) + if candidate_width < best_width: + return True + return bool( + np.isclose(candidate_width, best_width) + and float(candidate.metrics["winkler_90"]) < float(best_candidate.metrics["winkler_90"]) + ) + + +def _shrink_report_row( + *, + stage: str, + factor_scope: str, + factor_key: str, + candidate_factor: float, + accepted: bool, + metrics: dict[str, float], +) -> dict[str, Any]: + return { + "stage": stage, + "factor_scope": factor_scope, + "factor_key": factor_key, + "candidate_factor": candidate_factor, + "accepted": accepted, + **metrics, + } + + def shrink_group_multipliers( *, y_true: np.ndarray, @@ -560,222 +770,114 @@ def shrink_group_multipliers( temporal_multiplier_grid: tuple[float, ...] = (1.0, 1.02, 1.05, 1.08, 1.12, 1.16, 1.20), ) -> tuple[np.ndarray, dict[str, float], dict[str, float], pd.DataFrame]: """Greedily shrink learned widening factors while preserving constraints.""" - group_factors_cur = { - str(k): float(v) for k, v in (group_factors or {}).items() if float(v) > 1.0 - } - temporal_factors_cur = { - str(k): float(v) for k, v in (temporal_factors or {}).items() if float(v) > 1.0 - } - base = np.asarray(base_intervals, dtype=float) - y_pred_arr = np.asarray(y_pred, dtype=float) - y_true_arr = np.asarray(y_true, dtype=float) - group_series = pd.Series(groups).fillna("UNKNOWN").astype(str).reset_index(drop=True) + group_factors_cur = _active_widening_factors(group_factors) + temporal_factors_cur = _active_widening_factors(temporal_factors) temporal_series = ( pd.Series(temporal_segments).fillna("UNKNOWN").astype(str).reset_index(drop=True) if temporal_segments is not None else None ) + context = _ShrinkContext( + y_true=np.asarray(y_true, dtype=float), + y_pred=np.asarray(y_pred, dtype=float), + base_intervals=np.asarray(base_intervals, dtype=float), + groups=pd.Series(groups).fillna("UNKNOWN").astype(str).reset_index(drop=True), + temporal_segments=temporal_series, + issue_dates=issue_dates, + target_coverage=float(target_coverage), + min_group_coverage_target=float(min_group_coverage_target), + max_monthly_gap_target=max_monthly_gap_target, + alpha=float(alpha), + ) - def _apply_all( - gf: dict[str, float], - tf: dict[str, float], - ) -> np.ndarray: - intervals = base.copy() - if gf: - intervals = apply_group_multipliers(y_pred_arr, intervals, group_series, gf) - if tf and temporal_series is not None: - intervals = apply_group_multipliers(y_pred_arr, intervals, temporal_series, tf) - return intervals - - def _metrics(intervals: np.ndarray) -> dict[str, float]: - temporal = temporal_stability_summary( - y_true_arr, - intervals, - issue_dates, - target_coverage=float(target_coverage), - freq="M", - ) - return { - "coverage": empirical_interval_coverage(y_true_arr, intervals), - "min_group_coverage": min_group_interval_coverage(y_true_arr, intervals, group_series), - "avg_width": average_interval_width(intervals), - "winkler_90": mean_winkler_score(y_true_arr, intervals, alpha=alpha), - "max_monthly_gap": float(temporal["max_monthly_gap"]), - "stability_over_time": float(temporal["stability_over_time"]), - } - - def _constraints_ok(metrics: dict[str, float]) -> bool: - if float(metrics["coverage"]) < float(target_coverage): - return False - if float(metrics["min_group_coverage"]) < float(min_group_coverage_target): - return False - return not ( - max_monthly_gap_target is not None - and np.isfinite(max_monthly_gap_target) - and float(metrics["max_monthly_gap"]) > float(max_monthly_gap_target) - ) - - current_intervals = _apply_all(group_factors_cur, temporal_factors_cur) - current_metrics = _metrics(current_intervals) + current_intervals = _apply_shrink_factors( + context, + group_factors_cur, + temporal_factors_cur, + ) + current_metrics = _shrink_metrics(context, current_intervals) report_rows: list[dict[str, Any]] = [ - { - "stage": "initial", - "factor_scope": "all", - "factor_key": "all", - "candidate_factor": np.nan, - "accepted": True, - **current_metrics, - } + _shrink_report_row( + stage="initial", + factor_scope="all", + factor_key="all", + candidate_factor=np.nan, + accepted=True, + metrics=current_metrics, + ) ] - if not _constraints_ok(current_metrics): + if not _shrink_constraints_ok(context, current_metrics): report_rows.append( - { - "stage": "initial_infeasible", - "factor_scope": "all", - "factor_key": "all", - "candidate_factor": np.nan, - "accepted": False, - **current_metrics, - } + _shrink_report_row( + stage="initial_infeasible", + factor_scope="all", + factor_key="all", + candidate_factor=np.nan, + accepted=False, + metrics=current_metrics, + ) ) return current_intervals, group_factors_cur, temporal_factors_cur, pd.DataFrame(report_rows) - def _next_lower(value: float, grid: tuple[float, ...]) -> float | None: - ordered = sorted({round(float(x), 6) for x in grid if float(x) <= float(value) + 1e-9}) - current = round(float(value), 6) - if current not in ordered: - ordered.append(current) - ordered = sorted(set(ordered)) - idx = ordered.index(current) - if idx == 0: - return None - return float(ordered[idx - 1]) - while True: - best_candidate: dict[str, Any] | None = None - - for key, value in list(group_factors_cur.items()): - next_factor = _next_lower(value, group_multiplier_grid) - if next_factor is None: - continue - trial_group = dict(group_factors_cur) - if next_factor <= 1.0: - trial_group.pop(key, None) - else: - trial_group[key] = next_factor - trial_intervals = _apply_all(trial_group, temporal_factors_cur) - trial_metrics = _metrics(trial_intervals) - accepted = _constraints_ok(trial_metrics) - candidate: dict[str, Any] = { - "scope": "group", - "key": key, - "factor": next_factor, - "accepted": accepted, - "intervals": trial_intervals, - "group_factors": trial_group, - "temporal_factors": dict(temporal_factors_cur), - "metrics": trial_metrics, - } - if accepted and ( - best_candidate is None - or float(candidate["metrics"]["avg_width"]) - < float(best_candidate["metrics"]["avg_width"]) - or ( - np.isclose( - float(candidate["metrics"]["avg_width"]), - float(best_candidate["metrics"]["avg_width"]), - ) - and float(candidate["metrics"]["winkler_90"]) - < float(best_candidate["metrics"]["winkler_90"]) - ) - ): - best_candidate = candidate - report_rows.append( - { - "stage": "attempt", - "factor_scope": "group", - "factor_key": key, - "candidate_factor": float(next_factor), - "accepted": bool(accepted), - **trial_metrics, - } + attempts = _build_shrink_attempts( + context, + scope="group", + group_factors=group_factors_cur, + temporal_factors=temporal_factors_cur, + multiplier_grid=group_multiplier_grid, + ) + attempts.extend( + _build_shrink_attempts( + context, + scope="temporal", + group_factors=group_factors_cur, + temporal_factors=temporal_factors_cur, + multiplier_grid=temporal_multiplier_grid, ) - - for key, value in list(temporal_factors_cur.items()): - next_factor = _next_lower(value, temporal_multiplier_grid) - if next_factor is None: - continue - trial_temporal = dict(temporal_factors_cur) - if next_factor <= 1.0: - trial_temporal.pop(key, None) - else: - trial_temporal[key] = next_factor - trial_intervals = _apply_all(group_factors_cur, trial_temporal) - trial_metrics = _metrics(trial_intervals) - accepted = _constraints_ok(trial_metrics) - candidate = { - "scope": "temporal", - "key": key, - "factor": next_factor, - "accepted": accepted, - "intervals": trial_intervals, - "group_factors": dict(group_factors_cur), - "temporal_factors": trial_temporal, - "metrics": trial_metrics, - } - if accepted and ( - best_candidate is None - or float(candidate["metrics"]["avg_width"]) - < float(best_candidate["metrics"]["avg_width"]) - or ( - np.isclose( - float(candidate["metrics"]["avg_width"]), - float(best_candidate["metrics"]["avg_width"]), - ) - and float(candidate["metrics"]["winkler_90"]) - < float(best_candidate["metrics"]["winkler_90"]) - ) - ): + ) + best_candidate: _ShrinkCandidate | None = None + for candidate in attempts: + if _shrink_candidate_is_better(candidate, best_candidate): best_candidate = candidate report_rows.append( - { - "stage": "attempt", - "factor_scope": "temporal", - "factor_key": key, - "candidate_factor": float(next_factor), - "accepted": bool(accepted), - **trial_metrics, - } + _shrink_report_row( + stage="attempt", + factor_scope=candidate.scope, + factor_key=candidate.key, + candidate_factor=float(candidate.factor), + accepted=bool(candidate.accepted), + metrics=candidate.metrics, + ) ) - if best_candidate is None: break - current_intervals = np.asarray(best_candidate["intervals"], dtype=float) - group_factors_cur = dict(best_candidate["group_factors"]) - temporal_factors_cur = dict(best_candidate["temporal_factors"]) - current_metrics = dict(best_candidate["metrics"]) + current_intervals = np.asarray(best_candidate.intervals, dtype=float) + group_factors_cur = dict(best_candidate.group_factors) + temporal_factors_cur = dict(best_candidate.temporal_factors) + current_metrics = dict(best_candidate.metrics) report_rows.append( - { - "stage": "accepted", - "factor_scope": str(best_candidate["scope"]), - "factor_key": str(best_candidate["key"]), - "candidate_factor": float(best_candidate["factor"]), - "accepted": True, - **current_metrics, - } + _shrink_report_row( + stage="accepted", + factor_scope=best_candidate.scope, + factor_key=best_candidate.key, + candidate_factor=float(best_candidate.factor), + accepted=True, + metrics=current_metrics, + ) ) report_rows.append( - { - "stage": "final", - "factor_scope": "all", - "factor_key": "all", - "candidate_factor": np.nan, - "accepted": True, - **current_metrics, - } + _shrink_report_row( + stage="final", + factor_scope="all", + factor_key="all", + candidate_factor=np.nan, + accepted=True, + metrics=current_metrics, + ) ) report = pd.DataFrame(report_rows) return current_intervals, group_factors_cur, temporal_factors_cur, report diff --git a/src/models/optuna_tuning.py b/src/models/optuna_tuning.py index f05996f..2c5a724 100644 --- a/src/models/optuna_tuning.py +++ b/src/models/optuna_tuning.py @@ -3,6 +3,8 @@ from __future__ import annotations import gc +from dataclasses import dataclass +from functools import partial from pathlib import Path from typing import Any @@ -17,6 +19,16 @@ SEARCH_SPACE_VERSION = "cb_space_v2" _JOURNAL_STORAGE_PREFIXES = ("journal+file:", "journalfile:", "journal:") +CategoricalChoice = None | bool | int | float | str + + +@dataclass(frozen=True) +class _SelectedModelFit: + model: CatBoostClassifier + selected_params: dict[str, Any] + resolved_params: dict[str, Any] + validation_auc: float + best_iteration: int def _is_journal_storage_url(url: str) -> bool: @@ -33,6 +45,18 @@ def _journal_path_from_storage_url(url: str) -> str: return url +def _categorical_choices(raw: Any) -> list[CategoricalChoice]: + choices: list[CategoricalChoice] = [] + for value in list(raw): + if value is None or isinstance(value, bool | int | float | str): + choices.append(value) + else: + raise TypeError(f"Unsupported Optuna categorical choice {value!r}") + if not choices: + raise ValueError("Optuna categorical choices must not be empty") + return choices + + def resolve_optuna_study_name( study_name: str | None, *, @@ -103,629 +127,707 @@ def _build_optuna_sampler_pruner( return sampler_obj, pruner_obj -def train_catboost_tuned_optuna( - X_train: pd.DataFrame, - y_train: pd.Series, - X_val: pd.DataFrame, - y_val: pd.Series, - X_test: pd.DataFrame | None = None, - y_test: pd.Series | None = None, +def _align_feature_vector( + raw: Any, *, - cat_features: list[str] | None = None, - base_params: dict[str, Any] | None = None, - n_trials: int = 100, - sampler: str = "tpe", - pruner: str = "median", - timeout_minutes: int = 0, - n_startup_trials: int = 40, - multivariate_tpe: bool = True, - group_tpe: bool = True, - warn_independent_sampling: bool = True, - constant_liar: bool = False, - pruner_n_startup_trials: int = 20, - pruner_n_warmup_steps: int = 50, - use_pruning_callback: bool = True, - study_storage: str | None = None, - study_name: str | None = None, - load_if_exists: bool = True, - refit_full_train: bool = True, - gc_after_trial: bool = True, - storage_heartbeat_interval: int = 0, - storage_grace_period: int = 0, - sqlite_timeout_seconds: int = 60, - retry_failed_trials: int = 0, - n_jobs: int = 1, - sample_weight: np.ndarray | None = None, - eval_sample_weight: np.ndarray | None = None, - search_space_mode: str = "global", - local_refine_space: dict[str, Any] | None = None, - constraints_policy: dict[str, Any] | None = None, - search_space_version: str = SEARCH_SPACE_VERSION, - enqueue_trials: list[dict[str, Any]] | None = None, -) -> tuple[CatBoostClassifier, dict[str, Any]]: - """Tune CatBoost with Optuna and return best fitted model and metadata.""" - import optuna + feature_order: list[str], + default: float, +) -> list[float] | Any: + if not isinstance(raw, dict): + return raw + raw_by_feature = {str(feature): float(value) for feature, value in raw.items()} + return [float(raw_by_feature.get(feature, default)) for feature in feature_order] - if cat_features is None: - cat_features = [c for c in CATEGORICAL_FEATURES if c in X_train.columns] - base = _catboost_base_params(base_params) - base["verbose"] = 0 - has_monotone_constraints = bool(str(base.get("monotone_constraints", "")).strip()) - if has_monotone_constraints: - base["grow_policy"] = "SymmetricTree" - search_space_mode_resolved = str(search_space_mode or "global").strip().lower() or "global" - local_refine_space = dict(local_refine_space or {}) - if has_monotone_constraints: - local_refine_space["grow_policy"] = ["SymmetricTree"] - constraints_policy = dict(constraints_policy or {}) - enqueue_trials = list(enqueue_trials or []) - feature_order = [str(col) for col in X_train.columns] +def _normalize_feature_penalty_params( + params: dict[str, Any], + *, + feature_order: list[str], +) -> dict[str, Any]: + normalized = dict(params) + if "feature_weights" in normalized: + normalized["feature_weights"] = _align_feature_vector( + normalized["feature_weights"], + feature_order=feature_order, + default=1.0, + ) + if "first_feature_use_penalties" in normalized: + normalized["first_feature_use_penalties"] = _align_feature_vector( + normalized["first_feature_use_penalties"], + feature_order=feature_order, + default=0.0, + ) + return normalized + + +def _local_choice(trial: Any, name: str, spec: Any, default: Any) -> Any: + if spec is None: + return default + if isinstance(spec, dict): + if spec.get("choices") is not None: + return trial.suggest_categorical(name, _categorical_choices(spec["choices"])) + low = spec.get("low") + high = spec.get("high") + step = spec.get("step") + log = bool(spec.get("log", False)) + if low is None or high is None: + return default + if isinstance(low, int) and isinstance(high, int) and not log: + return trial.suggest_int(name, int(low), int(high), step=int(step or 1)) + return trial.suggest_float( + name, + float(low), + float(high), + step=None if log else (float(step) if step is not None else None), + log=log, + ) + if isinstance(spec, list): + return trial.suggest_categorical(name, _categorical_choices(spec)) + return spec - def _align_feature_vector(raw: Any, *, default: float) -> list[float] | Any: - if not isinstance(raw, dict): - return raw - raw_by_feature = {str(feature): float(value) for feature, value in raw.items()} - return [float(raw_by_feature.get(feature, default)) for feature in feature_order] - - def _normalize_feature_penalty_params(params: dict[str, Any]) -> dict[str, Any]: - normalized = dict(params) - if "feature_weights" in normalized: - normalized["feature_weights"] = _align_feature_vector( - normalized["feature_weights"], - default=1.0, - ) - if "first_feature_use_penalties" in normalized: - normalized["first_feature_use_penalties"] = _align_feature_vector( - normalized["first_feature_use_penalties"], - default=0.0, - ) - return normalized - incumbent_metrics: dict[str, float] = {} +def _apply_local_feature_priors( + trial: Any, + params: dict[str, Any], + *, + local_refine_space: dict[str, Any], +) -> None: + feature_weights_cfg = dict(local_refine_space.get("feature_weights", {}) or {}) + if feature_weights_cfg: + weights: dict[str, float] = {} + for feature, spec in feature_weights_cfg.items(): + value = float(_local_choice(trial, f"feature_weight__{feature}", spec, 1.0)) + weights[str(feature)] = value + if any(abs(value - 1.0) > 1e-12 for value in weights.values()): + params["feature_weights"] = weights + penalties_cfg = dict(local_refine_space.get("first_feature_use_penalties", {}) or {}) + if penalties_cfg: + penalties: dict[str, float] = {} + for feature, spec in penalties_cfg.items(): + value = float(_local_choice(trial, f"first_use_penalty__{feature}", spec, 0.0)) + penalties[str(feature)] = value + if any(abs(value) > 1e-12 for value in penalties.values()): + params["first_feature_use_penalties"] = penalties + penalties_coeff_spec = local_refine_space.get("penalties_coefficient") + if penalties_coeff_spec is not None: + params["penalties_coefficient"] = float( + _local_choice(trial, "penalties_coefficient", penalties_coeff_spec, 1.0) + ) - def _local_choice(trial: optuna.Trial, name: str, spec: Any, default: Any) -> Any: - if spec is None: - return default - if isinstance(spec, dict): - if spec.get("choices") is not None: - return trial.suggest_categorical(name, list(spec["choices"])) - low = spec.get("low") - high = spec.get("high") - step = spec.get("step") - log = bool(spec.get("log", False)) - if low is None or high is None: - return default - if isinstance(low, int) and isinstance(high, int) and not log: - return trial.suggest_int(name, int(low), int(high), step=int(step or 1)) - return trial.suggest_float( - name, - float(low), - float(high), - step=None if log else (float(step) if step is not None else None), - log=log, - ) - if isinstance(spec, list): - return trial.suggest_categorical(name, list(spec)) - return spec - - def _apply_local_feature_priors(trial: optuna.Trial, params: dict[str, Any]) -> None: - feature_weights_cfg = dict(local_refine_space.get("feature_weights", {}) or {}) - if feature_weights_cfg: - weights: dict[str, float] = {} - for feature, spec in feature_weights_cfg.items(): - value = float(_local_choice(trial, f"feature_weight__{feature}", spec, 1.0)) - weights[str(feature)] = value - if any(abs(value - 1.0) > 1e-12 for value in weights.values()): - params["feature_weights"] = weights - penalties_cfg = dict(local_refine_space.get("first_feature_use_penalties", {}) or {}) - if penalties_cfg: - penalties: dict[str, float] = {} - for feature, spec in penalties_cfg.items(): - value = float(_local_choice(trial, f"first_use_penalty__{feature}", spec, 0.0)) - penalties[str(feature)] = value - if any(abs(value) > 1e-12 for value in penalties.values()): - params["first_feature_use_penalties"] = penalties - penalties_coeff_spec = local_refine_space.get("penalties_coefficient") - if penalties_coeff_spec is not None: - params["penalties_coefficient"] = float( - _local_choice(trial, "penalties_coefficient", penalties_coeff_spec, 1.0) - ) - def _materialize_study_params(sampled_params: dict[str, Any]) -> dict[str, Any]: - params = {**base} - feature_weights: dict[str, float] = {} - penalties: dict[str, float] = {} +def _materialize_study_params( + sampled_params: dict[str, Any], + *, + base: dict[str, Any], + has_monotone_constraints: bool, +) -> dict[str, Any]: + params = {**base} + feature_weights: dict[str, float] = {} + penalties: dict[str, float] = {} + + for key, value in dict(sampled_params or {}).items(): + key_str = str(key) + if key_str.startswith("feature_weight__"): + feature_name = key_str.split("__", 1)[1] + feature_weights[feature_name] = float(value) + continue + if key_str.startswith("first_use_penalty__"): + feature_name = key_str.split("__", 1)[1] + penalties[feature_name] = float(value) + continue + params[key_str] = value + + if feature_weights and any(abs(weight - 1.0) > 1e-12 for weight in feature_weights.values()): + params["feature_weights"] = feature_weights + else: + params.pop("feature_weights", None) + if penalties and any(abs(weight) > 1e-12 for weight in penalties.values()): + params["first_feature_use_penalties"] = penalties + else: + params.pop("first_feature_use_penalties", None) - for key, value in dict(sampled_params or {}).items(): - key_str = str(key) - if key_str.startswith("feature_weight__"): - feature_name = key_str.split("__", 1)[1] - feature_weights[feature_name] = float(value) - continue - if key_str.startswith("first_use_penalty__"): - feature_name = key_str.split("__", 1)[1] - penalties[feature_name] = float(value) - continue - params[key_str] = value + if str(params.get("bootstrap_type", "")).strip() == "Bayesian": + params.pop("subsample", None) + else: + params.pop("bagging_temperature", None) - if feature_weights and any( - abs(weight - 1.0) > 1e-12 for weight in feature_weights.values() - ): - params["feature_weights"] = feature_weights - else: - params.pop("feature_weights", None) - if penalties and any(abs(weight) > 1e-12 for weight in penalties.values()): - params["first_feature_use_penalties"] = penalties - else: - params.pop("first_feature_use_penalties", None) + if str(params.get("grow_policy", "")).strip() == "Lossguide": + params.pop("depth", None) + else: + params.pop("max_leaves", None) + if has_monotone_constraints: + params["grow_policy"] = "SymmetricTree" + params.pop("max_leaves", None) - if str(params.get("bootstrap_type", "")).strip() == "Bayesian": - params.pop("subsample", None) - else: - params.pop("bagging_temperature", None) + if str(params.get("task_type", "")).strip().upper() == "GPU": + params.pop("rsm", None) - if str(params.get("grow_policy", "")).strip() == "Lossguide": - params.pop("depth", None) - else: - params.pop("max_leaves", None) - if has_monotone_constraints: - params["grow_policy"] = "SymmetricTree" - params.pop("max_leaves", None) + return {key: value for key, value in params.items() if value is not None} + + +def _sanitize_enqueued_trial( + raw_params: dict[str, Any], + *, + base: dict[str, Any], + search_space_mode_resolved: str, + has_monotone_constraints: bool, +) -> dict[str, Any]: + """Keep only parameters that are actually sampled by the active Optuna space.""" + allowed = { + "bootstrap_type", + "grow_policy", + "learning_rate", + "l2_leaf_reg", + "min_data_in_leaf", + "random_strength", + "border_count", + "leaf_estimation_iterations", + "rsm", + "depth", + "max_leaves", + "subsample", + "bagging_temperature", + } + if search_space_mode_resolved == "local_refine": + allowed.add("iterations") + params: dict[str, Any] = {} + for key, value in dict(raw_params or {}).items(): + key_str = str(key) + if key_str in allowed or key_str.startswith(("feature_weight__", "first_use_penalty__")): + params[key_str] = value - if str(params.get("task_type", "")).strip().upper() == "GPU": - params.pop("rsm", None) + if has_monotone_constraints: + params["grow_policy"] = "SymmetricTree" + params.pop("max_leaves", None) + if str(params.get("grow_policy", base.get("grow_policy", "SymmetricTree"))) == "Lossguide": + params.pop("depth", None) + else: + params.pop("max_leaves", None) + if str(params.get("bootstrap_type", base.get("bootstrap_type", "MVS"))) == "Bayesian": + params.pop("subsample", None) + else: + params.pop("bagging_temperature", None) + if str(base.get("task_type", "")).strip().upper() == "GPU": + params.pop("rsm", None) + return {key: value for key, value in params.items() if value is not None} + + +def _trial_params_match(left: dict[str, Any], right: dict[str, Any]) -> bool: + if set(left) != set(right): + return False + for key, left_value in left.items(): + right_value = right.get(key) + if right_value is None: + if left_value is not None: + return False + continue + try: + if abs(float(left_value) - float(right_value)) > 1e-12: + return False + except (TypeError, ValueError): + if str(left_value) != str(right_value): + return False + return True - return {key: value for key, value in params.items() if value is not None} - def _sanitize_enqueued_trial(raw_params: dict[str, Any]) -> dict[str, Any]: - """Keep only parameters that are actually sampled by the active Optuna space.""" - allowed = { - "bootstrap_type", - "grow_policy", +def _enqueue_prior_trials( + study: Any, + *, + enqueue_trials: list[dict[str, Any]], + base: dict[str, Any], + search_space_mode_resolved: str, + has_monotone_constraints: bool, +) -> int: + enqueued = 0 + existing = [dict(trial.params) for trial in study.trials] + for raw_params in enqueue_trials: + params = _sanitize_enqueued_trial( + raw_params, + base=base, + search_space_mode_resolved=search_space_mode_resolved, + has_monotone_constraints=has_monotone_constraints, + ) + if not params: + continue + if any(_trial_params_match(params, trial_params) for trial_params in existing): + continue + try: + study.enqueue_trial(params, skip_if_exists=True) + except TypeError: + study.enqueue_trial(params) + existing.append(params) + enqueued += 1 + return enqueued + + +def _local_refine_params( + trial: Any, + *, + base: dict[str, Any], + local_refine_space: dict[str, Any], + feature_order: list[str], + is_gpu: bool, +) -> dict[str, Any]: + params = {**base} + fixed_params = dict(local_refine_space.get("fixed_params", {}) or {}) + params.update(fixed_params) + + params["iterations"] = int( + _local_choice( + trial, + "iterations", + local_refine_space.get("iterations"), + base.get("iterations", 3000), + ) + ) + params["learning_rate"] = float( + _local_choice( + trial, "learning_rate", + local_refine_space.get("learning_rate"), + base.get("learning_rate", 0.03), + ) + ) + params["l2_leaf_reg"] = float( + _local_choice( + trial, "l2_leaf_reg", + local_refine_space.get("l2_leaf_reg"), + base.get("l2_leaf_reg", 3.0), + ) + ) + params["min_data_in_leaf"] = int( + _local_choice( + trial, "min_data_in_leaf", + local_refine_space.get("min_data_in_leaf"), + base.get("min_data_in_leaf", 64), + ) + ) + params["random_strength"] = float( + _local_choice( + trial, "random_strength", + local_refine_space.get("random_strength"), + base.get("random_strength", 1e-6), + ) + ) + params["border_count"] = int( + _local_choice( + trial, "border_count", - "leaf_estimation_iterations", - "rsm", - "depth", - "max_leaves", - "subsample", - "bagging_temperature", - } - if search_space_mode_resolved == "local_refine": - allowed.add("iterations") - params: dict[str, Any] = {} - for key, value in dict(raw_params or {}).items(): - key_str = str(key) - if ( - key_str in allowed - or key_str.startswith(("feature_weight__", "first_use_penalty__")) - ): - params[key_str] = value - - if has_monotone_constraints: - params["grow_policy"] = "SymmetricTree" - params.pop("max_leaves", None) - if str(params.get("grow_policy", base.get("grow_policy", "SymmetricTree"))) == "Lossguide": - params.pop("depth", None) - else: - params.pop("max_leaves", None) - if str(params.get("bootstrap_type", base.get("bootstrap_type", "MVS"))) == "Bayesian": - params.pop("subsample", None) - else: - params.pop("bagging_temperature", None) - if str(base.get("task_type", "")).strip().upper() == "GPU": - params.pop("rsm", None) - return {key: value for key, value in params.items() if value is not None} - - def _trial_params_match(left: dict[str, Any], right: dict[str, Any]) -> bool: - if set(left) != set(right): - return False - for key, left_value in left.items(): - right_value = right.get(key) - if right_value is None: - if left_value is not None: - return False - continue - try: - if abs(float(left_value) - float(right_value)) > 1e-12: - return False - except (TypeError, ValueError): - if str(left_value) != str(right_value): - return False - return True - - def _enqueue_prior_trials(study: optuna.Study) -> int: - enqueued = 0 - existing = [dict(trial.params) for trial in study.trials] - for raw_params in enqueue_trials: - params = _sanitize_enqueued_trial(raw_params) - if not params: - continue - if any(_trial_params_match(params, trial_params) for trial_params in existing): - continue - try: - study.enqueue_trial(params, skip_if_exists=True) - except TypeError: - study.enqueue_trial(params) - existing.append(params) - enqueued += 1 - return enqueued - - def _local_refine_params(trial: optuna.Trial, *, is_gpu: bool) -> dict[str, Any]: - params = {**base} - fixed_params = dict(local_refine_space.get("fixed_params", {}) or {}) - params.update(fixed_params) - - params["iterations"] = int( - _local_choice( - trial, - "iterations", - local_refine_space.get("iterations"), - base.get("iterations", 3000), - ) + local_refine_space.get("border_count"), + base.get("border_count", 128), ) - params["learning_rate"] = float( - _local_choice( - trial, - "learning_rate", - local_refine_space.get("learning_rate"), - base.get("learning_rate", 0.03), - ) + ) + params["leaf_estimation_iterations"] = int( + _local_choice( + trial, + "leaf_estimation_iterations", + local_refine_space.get("leaf_estimation_iterations"), + base.get("leaf_estimation_iterations", 3), ) - params["l2_leaf_reg"] = float( - _local_choice( - trial, - "l2_leaf_reg", - local_refine_space.get("l2_leaf_reg"), - base.get("l2_leaf_reg", 3.0), - ) + ) + params["bootstrap_type"] = _local_choice( + trial, + "bootstrap_type", + local_refine_space.get("bootstrap_type"), + base.get("bootstrap_type", "MVS"), + ) + params["grow_policy"] = _local_choice( + trial, + "grow_policy", + local_refine_space.get("grow_policy"), + base.get("grow_policy", "SymmetricTree"), + ) + + if is_gpu: + params.pop("rsm", None) + else: + params["rsm"] = float( + _local_choice(trial, "rsm", local_refine_space.get("rsm"), base.get("rsm", 1.0)) ) - params["min_data_in_leaf"] = int( - _local_choice( - trial, - "min_data_in_leaf", - local_refine_space.get("min_data_in_leaf"), - base.get("min_data_in_leaf", 64), - ) + if str(params.get("grow_policy", "SymmetricTree")) == "Lossguide": + params.pop("depth", None) + params["max_leaves"] = int( + _local_choice(trial, "max_leaves", local_refine_space.get("max_leaves"), 32) ) - params["random_strength"] = float( - _local_choice( - trial, - "random_strength", - local_refine_space.get("random_strength"), - base.get("random_strength", 1e-6), - ) + else: + params["depth"] = int( + _local_choice(trial, "depth", local_refine_space.get("depth"), base.get("depth", 8)) ) - params["border_count"] = int( + params.pop("max_leaves", None) + if str(params.get("bootstrap_type", "MVS")) == "Bayesian": + params.pop("subsample", None) + bagging_spec = local_refine_space.get("bagging_temperature", {"low": 0.0, "high": 10.0}) + params["bagging_temperature"] = float( _local_choice( trial, - "border_count", - local_refine_space.get("border_count"), - base.get("border_count", 128), + "bagging_temperature", + bagging_spec, + base.get("bagging_temperature", 1.0), ) ) - params["leaf_estimation_iterations"] = int( + else: + params.pop("bagging_temperature", None) + params["subsample"] = float( _local_choice( trial, - "leaf_estimation_iterations", - local_refine_space.get("leaf_estimation_iterations"), - base.get("leaf_estimation_iterations", 3), + "subsample", + local_refine_space.get("subsample"), + base.get("subsample", 0.8), ) ) - params["bootstrap_type"] = _local_choice( - trial, - "bootstrap_type", - local_refine_space.get("bootstrap_type"), - base.get("bootstrap_type", "MVS"), - ) - params["grow_policy"] = _local_choice( - trial, - "grow_policy", - local_refine_space.get("grow_policy"), - base.get("grow_policy", "SymmetricTree"), - ) + _apply_local_feature_priors(trial, params, local_refine_space=local_refine_space) + return _normalize_feature_penalty_params(params, feature_order=feature_order) - if is_gpu: - params.pop("rsm", None) - else: - params["rsm"] = float( - _local_choice(trial, "rsm", local_refine_space.get("rsm"), base.get("rsm", 1.0)) - ) - if str(params.get("grow_policy", "SymmetricTree")) == "Lossguide": - params.pop("depth", None) - params["max_leaves"] = int( - _local_choice(trial, "max_leaves", local_refine_space.get("max_leaves"), 32) - ) - else: - params["depth"] = int( - _local_choice(trial, "depth", local_refine_space.get("depth"), base.get("depth", 8)) - ) - params.pop("max_leaves", None) - if str(params.get("bootstrap_type", "MVS")) == "Bayesian": - params.pop("subsample", None) - bagging_spec = local_refine_space.get("bagging_temperature", {"low": 0.0, "high": 10.0}) - params["bagging_temperature"] = float( - _local_choice( - trial, - "bagging_temperature", - bagging_spec, - base.get("bagging_temperature", 1.0), - ) - ) - else: - params.pop("bagging_temperature", None) - params["subsample"] = float( - _local_choice( - trial, - "subsample", - local_refine_space.get("subsample"), - base.get("subsample", 0.8), - ) - ) - _apply_local_feature_priors(trial, params) - return _normalize_feature_penalty_params(params) - - use_multivariate = bool(multivariate_tpe) - use_group_tpe = bool(group_tpe and use_multivariate) - constraints_func = None - if constraints_policy: - max_brier_delta = constraints_policy.get("max_brier_delta") - max_ece_delta = constraints_policy.get("max_ece_delta") - min_auc_delta = constraints_policy.get("min_auc_delta") - - def constraints_func(frozen_trial: optuna.trial.FrozenTrial) -> list[float]: - attrs = frozen_trial.user_attrs - violations: list[float] = [] - if max_brier_delta is not None: - ceiling = float(incumbent_metrics.get("validation_brier", 0.0)) + float( - max_brier_delta - ) - violations.append(float(attrs.get("validation_brier", float("inf"))) - ceiling) - if max_ece_delta is not None: - ceiling = float(incumbent_metrics.get("validation_ece", 0.0)) + float(max_ece_delta) - violations.append(float(attrs.get("validation_ece", float("inf"))) - ceiling) - if min_auc_delta is not None: - floor = float(incumbent_metrics.get("validation_auc", 0.0)) + float(min_auc_delta) - violations.append(floor - float(attrs.get("validation_auc", float("-inf")))) - return violations - sampler_obj, pruner_obj = _build_optuna_sampler_pruner( - optuna, - sampler=sampler, - pruner=pruner, - n_startup_trials=n_startup_trials, - multivariate_tpe=use_multivariate, - group_tpe=use_group_tpe, - constant_liar=constant_liar, - pruner_n_startup_trials=pruner_n_startup_trials, - pruner_n_warmup_steps=pruner_n_warmup_steps, - constraints_func=constraints_func, +def _global_search_params( + trial: Any, + *, + base: dict[str, Any], + has_monotone_constraints: bool, +) -> dict[str, Any]: + bootstrap_type = trial.suggest_categorical("bootstrap_type", ["Bayesian", "Bernoulli", "MVS"]) + grow_policy_choices = ( + ["SymmetricTree"] + if has_monotone_constraints + else ["SymmetricTree", "Depthwise", "Lossguide"] ) + grow_policy = trial.suggest_categorical("grow_policy", grow_policy_choices) + params = { + **base, + "learning_rate": trial.suggest_float("learning_rate", 0.005, 0.20, log=True), + "l2_leaf_reg": trial.suggest_float("l2_leaf_reg", 0.5, 100.0, log=True), + "min_data_in_leaf": trial.suggest_int("min_data_in_leaf", 20, 500), + "random_strength": trial.suggest_float("random_strength", 1e-9, 10.0, log=True), + "border_count": trial.suggest_int("border_count", 64, 254), + "bootstrap_type": bootstrap_type, + "grow_policy": grow_policy, + "leaf_estimation_iterations": trial.suggest_int("leaf_estimation_iterations", 1, 10), + "random_seed": int(base.get("random_seed", 42)), + } + if str(base.get("task_type", "")).strip().upper() == "GPU": + params.pop("rsm", None) + else: + params["rsm"] = trial.suggest_float("rsm", 0.5, 1.0) + if grow_policy == "Lossguide": + params["max_leaves"] = trial.suggest_int("max_leaves", 16, 64) + params.pop("depth", None) + else: + params["depth"] = trial.suggest_int("depth", 4, 10) + if bootstrap_type == "Bayesian": + params.pop("subsample", None) + params["bagging_temperature"] = trial.suggest_float("bagging_temperature", 0.0, 10.0) + else: + params.pop("bagging_temperature", None) + params["subsample"] = trial.suggest_float("subsample", 0.5, 0.95) + return params - train_pool = Pool(X_train, y_train, cat_features=cat_features, weight=sample_weight) - val_pool = Pool(X_val, y_val, cat_features=cat_features, weight=eval_sample_weight) - if constraints_policy: - incumbent_model = CatBoostClassifier(**base) - incumbent_model.fit(train_pool, eval_set=val_pool, use_best_model=True) - incumbent_y_val_prob = incumbent_model.predict_proba(X_val)[:, 1] - incumbent_metrics = { - "validation_auc": float(roc_auc_score(y_val, incumbent_y_val_prob)), - "validation_brier": float(brier_score_loss(y_val, incumbent_y_val_prob)), - "validation_ece": float(expected_calibration_error(y_val, incumbent_y_val_prob)), - } - - def objective(trial: optuna.Trial) -> float: - is_gpu = str(base.get("task_type", "")).strip().upper() == "GPU" - if search_space_mode_resolved == "local_refine": - params = _local_refine_params(trial, is_gpu=is_gpu) - else: - bootstrap_type = trial.suggest_categorical( - "bootstrap_type", ["Bayesian", "Bernoulli", "MVS"] - ) - grow_policy_choices = ( - ["SymmetricTree"] - if has_monotone_constraints - else ["SymmetricTree", "Depthwise", "Lossguide"] + +def _constraints_violations( + frozen_trial: Any, + *, + incumbent_metrics: dict[str, float], + constraints_policy: dict[str, Any], +) -> list[float]: + attrs = frozen_trial.user_attrs + violations: list[float] = [] + max_brier_delta = constraints_policy.get("max_brier_delta") + max_ece_delta = constraints_policy.get("max_ece_delta") + min_auc_delta = constraints_policy.get("min_auc_delta") + if max_brier_delta is not None: + ceiling = float(incumbent_metrics.get("validation_brier", 0.0)) + float(max_brier_delta) + violations.append(float(attrs.get("validation_brier", float("inf"))) - ceiling) + if max_ece_delta is not None: + ceiling = float(incumbent_metrics.get("validation_ece", 0.0)) + float(max_ece_delta) + violations.append(float(attrs.get("validation_ece", float("inf"))) - ceiling) + if min_auc_delta is not None: + floor = float(incumbent_metrics.get("validation_auc", 0.0)) + float(min_auc_delta) + violations.append(floor - float(attrs.get("validation_auc", float("-inf")))) + return violations + + +def _incumbent_validation_metrics( + *, + base: dict[str, Any], + train_pool: Pool, + val_pool: Pool, + X_val: pd.DataFrame, + y_val: pd.Series, +) -> dict[str, float]: + incumbent_model = CatBoostClassifier(**base) + incumbent_model.fit(train_pool, eval_set=val_pool, use_best_model=True) + incumbent_y_val_prob = incumbent_model.predict_proba(X_val)[:, 1] + y_val_array = np.asarray(y_val, dtype=int) + return { + "validation_auc": float(roc_auc_score(y_val, incumbent_y_val_prob)), + "validation_brier": float(brier_score_loss(y_val, incumbent_y_val_prob)), + "validation_ece": float(expected_calibration_error(y_val_array, incumbent_y_val_prob)), + } + + +def _catboost_pruning_callbacks( + trial: Any, *, use_pruning_callback: bool +) -> tuple[Any | None, list[Any]]: + if not use_pruning_callback: + return None, [] + try: + from optuna_integration.catboost import CatBoostPruningCallback + + pruning_callback = CatBoostPruningCallback(trial, "AUC") + return pruning_callback, [pruning_callback] + except Exception as exc: # pragma: no cover - optional integration path + if trial.number == 0: + logger.warning( + "CatBoost pruning callback unavailable; disabling pruning callback: {}", exc ) - grow_policy = trial.suggest_categorical("grow_policy", grow_policy_choices) - params = { - **base, - "learning_rate": trial.suggest_float("learning_rate", 0.005, 0.20, log=True), - "l2_leaf_reg": trial.suggest_float("l2_leaf_reg", 0.5, 100.0, log=True), - "min_data_in_leaf": trial.suggest_int("min_data_in_leaf", 20, 500), - "random_strength": trial.suggest_float("random_strength", 1e-9, 10.0, log=True), - "border_count": trial.suggest_int("border_count", 64, 254), - "bootstrap_type": bootstrap_type, - "grow_policy": grow_policy, - "leaf_estimation_iterations": trial.suggest_int( - "leaf_estimation_iterations", 1, 10 - ), - "random_seed": int(base.get("random_seed", 42)), - } - if is_gpu: - params.pop("rsm", None) - else: - params["rsm"] = trial.suggest_float("rsm", 0.5, 1.0) - if grow_policy == "Lossguide": - params["max_leaves"] = trial.suggest_int("max_leaves", 16, 64) - params.pop("depth", None) - else: - params["depth"] = trial.suggest_int("depth", 4, 10) - if bootstrap_type == "Bayesian": - params.pop("subsample", None) - params["bagging_temperature"] = trial.suggest_float( - "bagging_temperature", 0.0, 10.0 - ) - else: - params.pop("bagging_temperature", None) - params["subsample"] = trial.suggest_float("subsample", 0.5, 0.95) - - params = _normalize_feature_penalty_params(params) - model = CatBoostClassifier(**params) - pruning_callback = None - callbacks: list[Any] = [] - if use_pruning_callback: - try: - from optuna_integration.catboost import CatBoostPruningCallback - - pruning_callback = CatBoostPruningCallback(trial, "AUC") - callbacks = [pruning_callback] - except Exception as exc: # pragma: no cover - optional integration path - if trial.number == 0: - logger.warning( - "CatBoost pruning callback unavailable; disabling pruning callback: {}", exc - ) - pruning_callback = None - callbacks = [] + return None, [] - model.fit( - train_pool, - eval_set=val_pool, - use_best_model=True, - callbacks=callbacks or None, + +def _catboost_objective( + trial: Any, + *, + base: dict[str, Any], + search_space_mode_resolved: str, + local_refine_space: dict[str, Any], + feature_order: list[str], + has_monotone_constraints: bool, + train_pool: Pool, + val_pool: Pool, + X_val: pd.DataFrame, + y_val: pd.Series, + use_pruning_callback: bool, +) -> float: + is_gpu = str(base.get("task_type", "")).strip().upper() == "GPU" + if search_space_mode_resolved == "local_refine": + params = _local_refine_params( + trial, + base=base, + local_refine_space=local_refine_space, + feature_order=feature_order, + is_gpu=is_gpu, + ) + else: + params = _global_search_params( + trial, + base=base, + has_monotone_constraints=has_monotone_constraints, ) - if pruning_callback is not None: - pruning_callback.check_pruned() + params = _normalize_feature_penalty_params(params, feature_order=feature_order) + model = CatBoostClassifier(**params) + pruning_callback, callbacks = _catboost_pruning_callbacks( + trial, + use_pruning_callback=use_pruning_callback, + ) + model.fit( + train_pool, + eval_set=val_pool, + use_best_model=True, + callbacks=callbacks or None, + ) - val_auc = model.get_best_score().get("validation", {}).get("AUC") - if val_auc is None: - y_val_prob = model.predict_proba(X_val)[:, 1] - val_auc = roc_auc_score(y_val, y_val_prob) - else: - y_val_prob = model.predict_proba(X_val)[:, 1] - val_brier = float(brier_score_loss(y_val, y_val_prob)) - val_ece = float(expected_calibration_error(y_val, y_val_prob)) + if pruning_callback is not None: + pruning_callback.check_pruned() + + val_auc = model.get_best_score().get("validation", {}).get("AUC") + y_val_prob = model.predict_proba(X_val)[:, 1] + if val_auc is None: + val_auc = roc_auc_score(y_val, y_val_prob) + y_val_array = np.asarray(y_val, dtype=int) + trial.set_user_attr("best_iteration", int(model.get_best_iteration())) + trial.set_user_attr("validation_auc", float(val_auc)) + trial.set_user_attr("validation_brier", float(brier_score_loss(y_val_array, y_val_prob))) + trial.set_user_attr( + "validation_ece", + float(expected_calibration_error(y_val_array, y_val_prob)), + ) + return float(val_auc) - trial.set_user_attr("best_iteration", int(model.get_best_iteration())) - trial.set_user_attr("validation_auc", float(val_auc)) - trial.set_user_attr("validation_brier", val_brier) - trial.set_user_attr("validation_ece", val_ece) - return float(val_auc) +def _create_optuna_study( + optuna_module: Any, + *, + sampler_obj: Any, + pruner_obj: Any, + study_storage: str | None, + study_name: str | None, + load_if_exists: bool, + search_space_version: str, + storage_heartbeat_interval: int, + storage_grace_period: int, + sqlite_timeout_seconds: int, + retry_failed_trials: int, +) -> tuple[Any, Any | None]: create_study_kwargs: dict[str, Any] = { "direction": "maximize", "sampler": sampler_obj, "pruner": pruner_obj, } retry_callback = None - if study_storage: - storage_obj: Any = study_storage - hb_interval = max(0, int(storage_heartbeat_interval)) - hb_grace = max(0, int(storage_grace_period)) - storage_text = str(study_storage) - if _is_journal_storage_url(storage_text): - journal_path = _journal_path_from_storage_url(storage_text) - Path(journal_path).parent.mkdir(parents=True, exist_ok=True) - from src.utils.optuna_storage import _make_journal_storage - - storage_obj = _make_journal_storage( - Path(journal_path), - study_name=resolve_optuna_study_name( - study_name, - search_space_version=search_space_version, - ), - ) - else: - # For long-running trials on SQLite, use a longer connection timeout and - # heartbeat to recover stale RUNNING trials after crashes/restarts. - if storage_text.startswith(("sqlite:///", "sqlite+pysqlite:///")): - engine_kwargs = {"connect_args": {"timeout": max(1, int(sqlite_timeout_seconds))}} - else: - engine_kwargs = None - if hb_interval > 0 or hb_grace > 0: - try: - heartbeat_cb = None - failed_cb = None - if int(retry_failed_trials) > 0: - retry_factory = getattr( - optuna.storages, - "RetryHeartbeatStaleTrialCallback", - None, - ) - if retry_factory is not None: - heartbeat_cb = retry_factory(max_retry=int(retry_failed_trials)) - retry_callback = heartbeat_cb - else: # pragma: no cover - compatibility with older Optuna. - failed_cb = optuna.storages.RetryFailedTrialCallback( - max_retry=int(retry_failed_trials) - ) - retry_callback = failed_cb - storage_kwargs: dict[str, Any] = { - "url": storage_text, - "engine_kwargs": engine_kwargs, - "heartbeat_interval": hb_interval or None, - "grace_period": hb_grace or None, - } - if heartbeat_cb is not None: - storage_kwargs["heartbeat_stale_trial_callback"] = heartbeat_cb - elif failed_cb is not None: - storage_kwargs["failed_trial_callback"] = failed_cb - storage_obj = optuna.storages.RDBStorage(**storage_kwargs) - except Exception as exc: - logger.warning( - "Optuna RDBStorage heartbeat/retry setup failed; falling back to storage " - "URL. reason={}", - exc, - ) - create_study_kwargs["storage"] = storage_obj - create_study_kwargs["study_name"] = resolve_optuna_study_name( - study_name, - search_space_version=search_space_version, + if not study_storage: + return optuna_module.create_study(**create_study_kwargs), retry_callback + + storage_obj: Any = study_storage + hb_interval = max(0, int(storage_heartbeat_interval)) + hb_grace = max(0, int(storage_grace_period)) + storage_text = str(study_storage) + if _is_journal_storage_url(storage_text): + journal_path = _journal_path_from_storage_url(storage_text) + Path(journal_path).parent.mkdir(parents=True, exist_ok=True) + from src.utils.optuna_storage import _make_journal_storage + + storage_obj = _make_journal_storage( + Path(journal_path), + study_name=resolve_optuna_study_name( + study_name, + search_space_version=search_space_version, + ), + ) + else: + engine_kwargs = ( + {"connect_args": {"timeout": max(1, int(sqlite_timeout_seconds))}} + if storage_text.startswith(("sqlite:///", "sqlite+pysqlite:///")) + else None ) - create_study_kwargs["load_if_exists"] = bool(load_if_exists) + if hb_interval > 0 or hb_grace > 0: + try: + heartbeat_cb = None + failed_cb = None + if int(retry_failed_trials) > 0: + retry_factory = getattr( + optuna_module.storages, + "RetryHeartbeatStaleTrialCallback", + None, + ) + if retry_factory is not None: + heartbeat_cb = retry_factory(max_retry=int(retry_failed_trials)) + retry_callback = heartbeat_cb + else: # pragma: no cover - compatibility with older Optuna. + failed_cb = optuna_module.storages.RetryFailedTrialCallback( + max_retry=int(retry_failed_trials) + ) + retry_callback = failed_cb + storage_kwargs: dict[str, Any] = { + "url": storage_text, + "engine_kwargs": engine_kwargs, + "heartbeat_interval": hb_interval or None, + "grace_period": hb_grace or None, + } + if heartbeat_cb is not None: + storage_kwargs["heartbeat_stale_trial_callback"] = heartbeat_cb + elif failed_cb is not None: + storage_kwargs["failed_trial_callback"] = failed_cb + storage_obj = optuna_module.storages.RDBStorage(**storage_kwargs) + except Exception as exc: + logger.warning( + "Optuna RDBStorage heartbeat/retry setup failed; falling back to storage " + "URL. reason={}", + exc, + ) + + create_study_kwargs["storage"] = storage_obj + create_study_kwargs["study_name"] = resolve_optuna_study_name( + study_name, + search_space_version=search_space_version, + ) + create_study_kwargs["load_if_exists"] = bool(load_if_exists) + return optuna_module.create_study(**create_study_kwargs), retry_callback + + +def _local_refine_base_trial_params(base: dict[str, Any]) -> dict[str, Any]: + return { + key: value + for key, value in { + "iterations": int(base.get("iterations", 3000)), + "learning_rate": float(base.get("learning_rate", 0.03)), + "l2_leaf_reg": float(base.get("l2_leaf_reg", 3.0)), + "min_data_in_leaf": int(base.get("min_data_in_leaf", 64)), + "random_strength": float(base.get("random_strength", 1e-6)), + "border_count": int(base.get("border_count", 128)), + "leaf_estimation_iterations": int(base.get("leaf_estimation_iterations", 3)), + "bootstrap_type": str(base.get("bootstrap_type", "MVS")), + "grow_policy": str(base.get("grow_policy", "SymmetricTree")), + "rsm": None + if str(base.get("task_type", "")).strip().upper() == "GPU" + else float(base.get("rsm", 1.0)), + "depth": None + if str(base.get("grow_policy", "SymmetricTree")) == "Lossguide" + else int(base.get("depth", 8)), + "max_leaves": int(base.get("max_leaves", 32)) + if str(base.get("grow_policy", "SymmetricTree")) == "Lossguide" + else None, + "subsample": None + if str(base.get("bootstrap_type", "MVS")) == "Bayesian" + else float(base.get("subsample", 0.8)), + "bagging_temperature": float(base.get("bagging_temperature", 1.0)) + if str(base.get("bootstrap_type", "MVS")) == "Bayesian" + else None, + }.items() + if value is not None + } + - study = optuna.create_study(**create_study_kwargs) - n_enqueued_prior_trials = _enqueue_prior_trials(study) - if search_space_mode_resolved == "local_refine" and bool( +def _enqueue_local_refine_base_trial( + study: Any, + *, + base: dict[str, Any], + local_refine_space: dict[str, Any], + search_space_mode_resolved: str, +) -> None: + if search_space_mode_resolved != "local_refine" or not bool( local_refine_space.get("enqueue_base_trial", True) ): - try: - study.enqueue_trial( - { - key: value - for key, value in { - "iterations": int(base.get("iterations", 3000)), - "learning_rate": float(base.get("learning_rate", 0.03)), - "l2_leaf_reg": float(base.get("l2_leaf_reg", 3.0)), - "min_data_in_leaf": int(base.get("min_data_in_leaf", 64)), - "random_strength": float(base.get("random_strength", 1e-6)), - "border_count": int(base.get("border_count", 128)), - "leaf_estimation_iterations": int( - base.get("leaf_estimation_iterations", 3) - ), - "bootstrap_type": str(base.get("bootstrap_type", "MVS")), - "grow_policy": str(base.get("grow_policy", "SymmetricTree")), - "rsm": None - if str(base.get("task_type", "")).strip().upper() == "GPU" - else float(base.get("rsm", 1.0)), - "depth": None - if str(base.get("grow_policy", "SymmetricTree")) == "Lossguide" - else int(base.get("depth", 8)), - "max_leaves": int(base.get("max_leaves", 32)) - if str(base.get("grow_policy", "SymmetricTree")) == "Lossguide" - else None, - "subsample": None - if str(base.get("bootstrap_type", "MVS")) == "Bayesian" - else float(base.get("subsample", 0.8)), - "bagging_temperature": float(base.get("bagging_temperature", 1.0)) - if str(base.get("bootstrap_type", "MVS")) == "Bayesian" - else None, - }.items() - if value is not None - } - ) - except Exception as exc: - logger.warning("Optuna enqueue_trial for local_refine skipped: {}", exc) - if retry_callback is not None and hasattr(optuna.storages, "fail_stale_trials"): - try: - optuna.storages.fail_stale_trials(study) - except Exception as exc: - logger.warning("Optuna stale-trial recovery skipped: {}", exc) + return + try: + study.enqueue_trial(_local_refine_base_trial_params(base)) + except Exception as exc: + logger.warning("Optuna enqueue_trial for local_refine skipped: {}", exc) + + +def _complete_trial_value(trial: Any) -> float: + value = trial.value + if value is None: + raise ValueError("COMPLETE Optuna trial has no objective value.") + return float(value) + + +def _complete_trials(study: Any, optuna_module: Any) -> list[Any]: + return [ + trial + for trial in study.trials + if trial.state == optuna_module.trial.TrialState.COMPLETE and trial.value is not None + ] + + +def _selected_optuna_trial(study: Any, optuna_module: Any) -> tuple[Any, float, bool]: + complete_trials = _complete_trials(study, optuna_module) + if not complete_trials: + raise ValueError("Optuna study has no COMPLETE trials available for model selection.") + try: + selected_trial = study.best_trial + constrained_best_trial = True + except ValueError as exc: + selected_trial = max(complete_trials, key=_complete_trial_value) + constrained_best_trial = False + logger.warning( + "Optuna study has no feasible trial under constraints; using best COMPLETE trial " + "by validation AUC as fallback. reason={}", + exc, + ) + return selected_trial, _complete_trial_value(selected_trial), constrained_best_trial + + +def _run_or_reuse_study( + study: Any, + *, + objective: Any, + n_trials: int, + timeout_minutes: int, + gc_after_trial: bool, + n_jobs: int, +) -> None: timeout = None if timeout_minutes <= 0 else int(timeout_minutes * 60) requested_trials = int(n_trials) if requested_trials > 0: @@ -739,7 +841,9 @@ def objective(trial: optuna.Trial) -> float: ) else: complete_trials = [ - t for t in study.trials if t.state.name == "COMPLETE" and t.value is not None + trial + for trial in study.trials + if trial.state.name == "COMPLETE" and trial.value is not None ] if not complete_trials: raise ValueError( @@ -753,63 +857,252 @@ def objective(trial: optuna.Trial) -> float: ) gc.collect() - complete_trials = [ - trial - for trial in study.trials - if trial.state == optuna.trial.TrialState.COMPLETE and trial.value is not None - ] - if not complete_trials: - raise ValueError("Optuna study has no COMPLETE trials available for model selection.") - constrained_best_trial = True - try: - selected_trial = study.best_trial - except ValueError as exc: - selected_trial = max(complete_trials, key=lambda trial: float(trial.value)) - constrained_best_trial = False - logger.warning( - "Optuna study has no feasible trial under constraints; using best COMPLETE trial " - "by validation AUC as fallback. reason={}", - exc, - ) - selected_params = dict(selected_trial.params) - selected_value = float(selected_trial.value) - best_params = _materialize_study_params(selected_params) +def _fit_selected_catboost_model( + *, + selected_trial: Any, + base: dict[str, Any], + has_monotone_constraints: bool, + feature_order: list[str], + train_pool: Pool, + val_pool: Pool, + X_train: pd.DataFrame, + y_train: pd.Series, + X_val: pd.DataFrame, + y_val: pd.Series, + cat_features: list[str], + sample_weight: np.ndarray | None, + eval_sample_weight: np.ndarray | None, + refit_full_train: bool, +) -> _SelectedModelFit: + selected_params = dict(selected_trial.params) + best_params = _materialize_study_params( + selected_params, + base=base, + has_monotone_constraints=has_monotone_constraints, + ) best_params["verbose"] = 100 - catboost_best_params = _normalize_feature_penalty_params(best_params) + catboost_best_params = _normalize_feature_penalty_params( + best_params, + feature_order=feature_order, + ) selection_model = CatBoostClassifier(**catboost_best_params) selection_model.fit(train_pool, eval_set=val_pool, use_best_model=True) y_val_prob = selection_model.predict_proba(X_val)[:, 1] - val_auc = roc_auc_score(y_val, y_val_prob) + val_auc = float(roc_auc_score(y_val, y_val_prob)) best_iteration = int(selection_model.get_best_iteration()) - if refit_full_train: - full_X = pd.concat([X_train, X_val], axis=0).reset_index(drop=True) - full_y = pd.concat([y_train, y_val], axis=0).reset_index(drop=True) - full_weight = None - if sample_weight is not None and eval_sample_weight is not None: - full_weight = np.concatenate( - [ - np.asarray(sample_weight, dtype=float), - np.asarray(eval_sample_weight, dtype=float), - ] + if not refit_full_train: + return _SelectedModelFit( + model=selection_model, + selected_params=selected_params, + resolved_params=best_params, + validation_auc=val_auc, + best_iteration=best_iteration, + ) + + full_X = pd.concat([X_train, X_val], axis=0).reset_index(drop=True) + full_y = pd.concat([y_train, y_val], axis=0).reset_index(drop=True) + full_weight = None + if sample_weight is not None and eval_sample_weight is not None: + full_weight = np.concatenate( + [ + np.asarray(sample_weight, dtype=float), + np.asarray(eval_sample_weight, dtype=float), + ] + ) + full_pool = Pool(full_X, full_y, cat_features=cat_features, weight=full_weight) + refit_params = { + key: value for key, value in catboost_best_params.items() if key != "early_stopping_rounds" + } + if best_iteration > 0: + refit_params["iterations"] = best_iteration + 1 + best_model = CatBoostClassifier(**refit_params) + best_model.fit(full_pool) + return _SelectedModelFit( + model=best_model, + selected_params=selected_params, + resolved_params=best_params, + validation_auc=val_auc, + best_iteration=best_iteration, + ) + + +def train_catboost_tuned_optuna( + X_train: pd.DataFrame, + y_train: pd.Series, + X_val: pd.DataFrame, + y_val: pd.Series, + X_test: pd.DataFrame | None = None, + y_test: pd.Series | None = None, + *, + cat_features: list[str] | None = None, + base_params: dict[str, Any] | None = None, + n_trials: int = 100, + sampler: str = "tpe", + pruner: str = "median", + timeout_minutes: int = 0, + n_startup_trials: int = 40, + multivariate_tpe: bool = True, + group_tpe: bool = True, + warn_independent_sampling: bool = True, + constant_liar: bool = False, + pruner_n_startup_trials: int = 20, + pruner_n_warmup_steps: int = 50, + use_pruning_callback: bool = True, + study_storage: str | None = None, + study_name: str | None = None, + load_if_exists: bool = True, + refit_full_train: bool = True, + gc_after_trial: bool = True, + storage_heartbeat_interval: int = 0, + storage_grace_period: int = 0, + sqlite_timeout_seconds: int = 60, + retry_failed_trials: int = 0, + n_jobs: int = 1, + sample_weight: np.ndarray | None = None, + eval_sample_weight: np.ndarray | None = None, + search_space_mode: str = "global", + local_refine_space: dict[str, Any] | None = None, + constraints_policy: dict[str, Any] | None = None, + search_space_version: str = SEARCH_SPACE_VERSION, + enqueue_trials: list[dict[str, Any]] | None = None, +) -> tuple[CatBoostClassifier, dict[str, Any]]: + """Tune CatBoost with Optuna and return best fitted model and metadata.""" + import optuna + + if cat_features is None: + cat_features = [c for c in CATEGORICAL_FEATURES if c in X_train.columns] + + base = _catboost_base_params(base_params) + base["verbose"] = 0 + has_monotone_constraints = bool(str(base.get("monotone_constraints", "")).strip()) + if has_monotone_constraints: + base["grow_policy"] = "SymmetricTree" + search_space_mode_resolved = str(search_space_mode or "global").strip().lower() or "global" + local_refine_space = dict(local_refine_space or {}) + if has_monotone_constraints: + local_refine_space["grow_policy"] = ["SymmetricTree"] + constraints_policy = dict(constraints_policy or {}) + enqueue_trials = list(enqueue_trials or []) + feature_order = [str(col) for col in X_train.columns] + + train_pool = Pool(X_train, y_train, cat_features=cat_features, weight=sample_weight) + val_pool = Pool(X_val, y_val, cat_features=cat_features, weight=eval_sample_weight) + incumbent_metrics: dict[str, float] = {} + if constraints_policy: + incumbent_metrics.update( + _incumbent_validation_metrics( + base=base, + train_pool=train_pool, + val_pool=val_pool, + X_val=X_val, + y_val=y_val, ) - full_pool = Pool(full_X, full_y, cat_features=cat_features, weight=full_weight) - refit_params = { - k: v for k, v in catboost_best_params.items() if k != "early_stopping_rounds" - } - if best_iteration > 0: - refit_params["iterations"] = best_iteration + 1 - best_model = CatBoostClassifier(**refit_params) - best_model.fit(full_pool) - else: - best_model = selection_model + ) + + use_multivariate = bool(multivariate_tpe) + use_group_tpe = bool(group_tpe and use_multivariate) + constraints_func = ( + partial( + _constraints_violations, + incumbent_metrics=incumbent_metrics, + constraints_policy=constraints_policy, + ) + if constraints_policy + else None + ) + + sampler_obj, pruner_obj = _build_optuna_sampler_pruner( + optuna, + sampler=sampler, + pruner=pruner, + n_startup_trials=n_startup_trials, + multivariate_tpe=use_multivariate, + group_tpe=use_group_tpe, + constant_liar=constant_liar, + pruner_n_startup_trials=pruner_n_startup_trials, + pruner_n_warmup_steps=pruner_n_warmup_steps, + constraints_func=constraints_func, + ) + + objective = partial( + _catboost_objective, + base=base, + search_space_mode_resolved=search_space_mode_resolved, + local_refine_space=local_refine_space, + feature_order=feature_order, + has_monotone_constraints=has_monotone_constraints, + train_pool=train_pool, + val_pool=val_pool, + X_val=X_val, + y_val=y_val, + use_pruning_callback=use_pruning_callback, + ) + + study, retry_callback = _create_optuna_study( + optuna, + sampler_obj=sampler_obj, + pruner_obj=pruner_obj, + study_storage=study_storage, + study_name=study_name, + load_if_exists=load_if_exists, + search_space_version=search_space_version, + storage_heartbeat_interval=storage_heartbeat_interval, + storage_grace_period=storage_grace_period, + sqlite_timeout_seconds=sqlite_timeout_seconds, + retry_failed_trials=retry_failed_trials, + ) + n_enqueued_prior_trials = _enqueue_prior_trials( + study, + enqueue_trials=enqueue_trials, + base=base, + search_space_mode_resolved=search_space_mode_resolved, + has_monotone_constraints=has_monotone_constraints, + ) + _enqueue_local_refine_base_trial( + study, + base=base, + local_refine_space=local_refine_space, + search_space_mode_resolved=search_space_mode_resolved, + ) + if retry_callback is not None and hasattr(optuna.storages, "fail_stale_trials"): + try: + optuna.storages.fail_stale_trials(study) + except Exception as exc: + logger.warning("Optuna stale-trial recovery skipped: {}", exc) + _run_or_reuse_study( + study, + objective=objective, + n_trials=n_trials, + timeout_minutes=timeout_minutes, + gc_after_trial=gc_after_trial, + n_jobs=n_jobs, + ) + + selected_trial, selected_value, constrained_best_trial = _selected_optuna_trial(study, optuna) + selected_fit = _fit_selected_catboost_model( + selected_trial=selected_trial, + base=base, + has_monotone_constraints=has_monotone_constraints, + feature_order=feature_order, + train_pool=train_pool, + val_pool=val_pool, + X_train=X_train, + y_train=y_train, + X_val=X_val, + y_val=y_val, + cat_features=cat_features, + sample_weight=sample_weight, + eval_sample_weight=eval_sample_weight, + refit_full_train=refit_full_train, + ) metrics: dict[str, Any] = { - "validation_auc": float(val_auc), - "best_iteration": best_iteration, - "best_params": selected_params, - "best_params_resolved": best_params, + "validation_auc": float(selected_fit.validation_auc), + "best_iteration": selected_fit.best_iteration, + "best_params": selected_fit.selected_params, + "best_params_resolved": selected_fit.resolved_params, "hpo_trials_executed": len(study.trials), "hpo_best_validation_auc": selected_value, "hpo_selected_trial_number": int(selected_trial.number), @@ -822,15 +1115,15 @@ def objective(trial: optuna.Trial) -> float: "enqueued_prior_trials": n_enqueued_prior_trials, } if X_test is not None and y_test is not None: - y_test_prob = best_model.predict_proba(X_test)[:, 1] + y_test_prob = selected_fit.model.predict_proba(X_test)[:, 1] metrics["auc_roc"] = float(roc_auc_score(y_test, y_test_prob)) logger.info( "CatBoost tuned — val_AUC: " - f"{val_auc:.4f}, best_trial_val_AUC: {selected_value:.4f}, " + f"{selected_fit.validation_auc:.4f}, best_trial_val_AUC: {selected_value:.4f}, " f"trials={len(study.trials)}, multivariate_tpe={use_multivariate}, group_tpe={use_group_tpe}" ) - return best_model, metrics + return selected_fit.model, metrics def export_hpo_visualizations( diff --git a/src/optimization/certificate_semantics.py b/src/optimization/certificate_semantics.py new file mode 100644 index 0000000..345d9df --- /dev/null +++ b/src/optimization/certificate_semantics.py @@ -0,0 +1,260 @@ +"""Shared semantics for the paper-facing funded-set decision certificate.""" + +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np +import pandas as pd + +IJDS_DECLARED_ALPHA_GRID: tuple[float, ...] = ( + 0.01, + 0.03, + 0.05, + 0.07, + 0.10, + 0.12, + 0.15, + 0.20, +) +IJDS_DECLARED_ALPHA_GRID_CSV = ",".join(f"{alpha:.2f}" for alpha in IJDS_DECLARED_ALPHA_GRID) + + +@dataclass(frozen=True) +class FundedCertificateMetrics: + """Exact funded-set metrics and their policy-aware upper bounds. + + ``endpoint_budget`` and ``markov_loss_threshold`` are exact for the supplied + funded weights. ``endpoint_budget_upper`` and ``markov_loss_cap`` additionally + use the declared effective-PD constraint and its optional solver slack. + """ + + alpha: float + risk_tolerance: float + n_funded: int + weighted_outcome: float + weighted_miscoverage: float + weighted_coverage: float + empirical_coverage_funded: float + weighted_pd_point: float + weighted_pd_effective: float + endpoint_budget: float + gamma_cp: float + gamma_internalized: float + gamma_residual: float + effective_constraint_slack: float + effective_constraint_excess: float + realized_risk_tolerance_excess: float + sqrt_alpha: float + endpoint_budget_upper: float + markov_loss_threshold: float + markov_loss_cap: float + + +def _one_dimensional_float_array(values: np.ndarray, *, name: str) -> np.ndarray: + array = np.asarray(values, dtype=float) + if array.ndim != 1: + raise ValueError(f"{name} must be one-dimensional, got shape={array.shape}") + if not np.all(np.isfinite(array)): + raise ValueError(f"{name} contains non-finite values.") + return array + + +def _validate_probability_array(values: np.ndarray, *, name: str) -> None: + tolerance = 1e-10 + if np.any(values < -tolerance) or np.any(values > 1.0 + tolerance): + raise ValueError(f"{name} must stay on the [0, 1] probability scale.") + + +def compute_funded_certificate_metrics( + weights: np.ndarray, + outcomes: np.ndarray, + pd_point: np.ndarray, + pd_high: np.ndarray, + pd_effective: np.ndarray, + *, + alpha: float, + risk_tolerance: float, + pd_cap_slack: float = 0.0, + funded_tolerance: float = 1e-8, +) -> FundedCertificateMetrics: + """Compute the exact and policy-aware funded-set certificate. + + The effective PD vector may come from any policy family. This is important + for capped and tail-focused policies, where the linear-blend shortcut + ``(1 - gamma) * Gamma_CP`` is not generally the residual endpoint premium. + """ + arrays = { + "weights": _one_dimensional_float_array(weights, name="weights"), + "outcomes": _one_dimensional_float_array(outcomes, name="outcomes"), + "pd_point": _one_dimensional_float_array(pd_point, name="pd_point"), + "pd_high": _one_dimensional_float_array(pd_high, name="pd_high"), + "pd_effective": _one_dimensional_float_array(pd_effective, name="pd_effective"), + } + lengths = {len(values) for values in arrays.values()} + if len(lengths) != 1: + shapes = {name: values.shape for name, values in arrays.items()} + raise ValueError(f"Certificate arrays must have the same length: {shapes}") + + weights_arr = arrays["weights"] + outcomes_arr = arrays["outcomes"] + point_arr = arrays["pd_point"] + high_arr = arrays["pd_high"] + effective_arr = arrays["pd_effective"] + if np.any(weights_arr < 0.0): + raise ValueError("weights must be nonnegative.") + weight_sum = float(weights_arr.sum()) + if not np.isclose(weight_sum, 1.0, rtol=0.0, atol=1e-8): + raise ValueError(f"weights must sum to one, got {weight_sum:.12g}") + + for name, values in ( + ("outcomes", outcomes_arr), + ("pd_point", point_arr), + ("pd_high", high_arr), + ("pd_effective", effective_arr), + ): + _validate_probability_array(values, name=name) + order_tolerance = 1e-10 + if np.any(high_arr + order_tolerance < point_arr): + raise ValueError("pd_high must be at least pd_point for every row.") + if np.any(effective_arr + order_tolerance < point_arr) or np.any( + effective_arr > high_arr + order_tolerance + ): + raise ValueError("pd_effective must lie between pd_point and pd_high.") + + alpha_value = float(alpha) + if not 0.0 < alpha_value < 1.0: + raise ValueError(f"alpha must lie in (0, 1), got {alpha_value}") + tolerance_value = float(risk_tolerance) + if not 0.0 <= tolerance_value <= 1.0: + raise ValueError(f"risk_tolerance must lie in [0, 1], got {tolerance_value}") + cap_slack = float(pd_cap_slack) + if not np.isfinite(cap_slack) or cap_slack < 0.0: + raise ValueError(f"pd_cap_slack must be finite and nonnegative, got {cap_slack}") + + funded = weights_arr > float(funded_tolerance) + miscoverage = outcomes_arr > high_arr + weighted_outcome = float(weights_arr @ outcomes_arr) + weighted_miscoverage = float(weights_arr @ miscoverage.astype(float)) + weighted_point = float(weights_arr @ point_arr) + weighted_effective = float(weights_arr @ effective_arr) + endpoint_budget = float(weights_arr @ high_arr) + gamma_cp = float(weights_arr @ np.clip(high_arr - point_arr, 0.0, 1.0)) + gamma_internalized = float(weights_arr @ np.clip(effective_arr - point_arr, 0.0, 1.0)) + gamma_residual = float(weights_arr @ np.clip(high_arr - effective_arr, 0.0, 1.0)) + if not np.isclose( + gamma_cp, + gamma_internalized + gamma_residual, + rtol=0.0, + atol=1e-9, + ): + raise ValueError("Conformal-premium decomposition is not internally consistent.") + + effective_cap = tolerance_value + cap_slack + effective_constraint_slack = max(0.0, effective_cap - weighted_effective) + effective_constraint_excess = max(0.0, weighted_effective - effective_cap) + endpoint_budget_upper = effective_cap + gamma_residual + sqrt_alpha = float(np.sqrt(alpha_value)) + empirical_coverage = float(1.0 - miscoverage[funded].mean()) if funded.any() else float("nan") + + return FundedCertificateMetrics( + alpha=alpha_value, + risk_tolerance=tolerance_value, + n_funded=int(funded.sum()), + weighted_outcome=weighted_outcome, + weighted_miscoverage=weighted_miscoverage, + weighted_coverage=1.0 - weighted_miscoverage, + empirical_coverage_funded=empirical_coverage, + weighted_pd_point=weighted_point, + weighted_pd_effective=weighted_effective, + endpoint_budget=endpoint_budget, + gamma_cp=gamma_cp, + gamma_internalized=gamma_internalized, + gamma_residual=gamma_residual, + effective_constraint_slack=effective_constraint_slack, + effective_constraint_excess=effective_constraint_excess, + realized_risk_tolerance_excess=max(0.0, weighted_outcome - tolerance_value), + sqrt_alpha=sqrt_alpha, + endpoint_budget_upper=endpoint_budget_upper, + markov_loss_threshold=endpoint_budget + sqrt_alpha, + markov_loss_cap=endpoint_budget_upper + sqrt_alpha, + ) + + +def add_policy_aware_bound_columns(frame: pd.DataFrame) -> pd.DataFrame: + """Add exact and policy-aware bound columns to alpha-grid evaluation rows. + + Historical pool93 evaluations already store the sufficient statistics for + this decomposition. Rehydrating the bounds from those columns corrects + capped/tail policy semantics without re-solving any portfolio. + """ + required = { + "alpha", + "weighted_pd_high", + "weighted_pd_constraint_used", + } + missing = sorted(required.difference(frame.columns)) + if missing: + raise ValueError(f"Bound-evaluation frame is missing required columns: {missing}") + tolerance_column = "risk_tolerance" if "risk_tolerance" in frame.columns else "tau" + if tolerance_column not in frame.columns: + raise ValueError("Bound-evaluation frame requires risk_tolerance or tau.") + + work = frame.copy() + if "realized_risk_tolerance_excess" not in work.columns and "violation" in work.columns: + work["realized_risk_tolerance_excess"] = pd.to_numeric( + work["violation"], errors="raise" + ).astype(float) + if ( + "empirical_risk_excess_leq_alpha" not in work.columns + and "bound_a_expected_violation_leq_alpha" in work.columns + ): + work["empirical_risk_excess_leq_alpha"] = work[ + "bound_a_expected_violation_leq_alpha" + ].astype(bool) + alpha = pd.to_numeric(work["alpha"], errors="raise").astype(float) + endpoint = pd.to_numeric(work["weighted_pd_high"], errors="raise").astype(float) + effective = pd.to_numeric(work["weighted_pd_constraint_used"], errors="raise").astype(float) + tolerance = pd.to_numeric(work[tolerance_column], errors="raise").astype(float) + slack = ( + pd.to_numeric(work["pd_cap_slack"], errors="raise").astype(float) + if "pd_cap_slack" in work.columns + else pd.Series(0.0, index=work.index, dtype=float) + ) + numeric = pd.concat( + { + "alpha": alpha, + "endpoint": endpoint, + "effective": effective, + "tolerance": tolerance, + "slack": slack, + }, + axis=1, + ) + if not np.isfinite(numeric.to_numpy(dtype=float)).all(): + raise ValueError("Bound-evaluation frame contains non-finite certificate inputs.") + if (alpha <= 0.0).any() or (alpha >= 1.0).any(): + raise ValueError("Bound-evaluation alpha values must lie in (0, 1).") + if (slack < 0.0).any(): + raise ValueError("Bound-evaluation pd_cap_slack must be nonnegative.") + + residual = endpoint - effective + if (residual < -1e-9).any(): + raise ValueError("weighted_pd_high must be at least weighted_pd_constraint_used.") + residual = residual.clip(lower=0.0) + effective_cap = tolerance + slack + sqrt_alpha = np.sqrt(alpha) + work["gamma_residual"] = residual + if "weighted_pd_point" in work.columns: + point = pd.to_numeric(work["weighted_pd_point"], errors="raise").astype(float) + work["gamma_internalized"] = (effective - point).clip(lower=0.0) + elif "gamma_cp" in work.columns: + gamma_cp = pd.to_numeric(work["gamma_cp"], errors="raise").astype(float) + work["gamma_internalized"] = (gamma_cp - residual).clip(lower=0.0) + work["effective_constraint_slack"] = (effective_cap - effective).clip(lower=0.0) + work["effective_constraint_excess"] = (effective - effective_cap).clip(lower=0.0) + work["endpoint_budget"] = endpoint + work["endpoint_budget_upper"] = effective_cap + residual + work["markov_loss_threshold"] = endpoint + sqrt_alpha + work["markov_loss_cap"] = work["endpoint_budget_upper"] + sqrt_alpha + return work diff --git a/src/optimization/cuopt_adapter.py b/src/optimization/cuopt_adapter.py index dbca887..f5e9e57 100644 --- a/src/optimization/cuopt_adapter.py +++ b/src/optimization/cuopt_adapter.py @@ -17,17 +17,17 @@ import numpy as np import pandas as pd from loguru import logger -from scipy.sparse import csr_matrix + +from src.optimization.portfolio_model import _portfolio_lp_components, _PortfolioLpComponents def _require_cuopt() -> Any: try: - from cuopt import linear_programming as lp_api # type: ignore[import-not-found] + return importlib.import_module("cuopt").linear_programming except Exception as exc: # pragma: no cover - exercised in RAPIDS env only raise RuntimeError( "solver_backend='cuopt' requested but native cuOpt Python API is not available." ) from exc - return lp_api def _extract_primal_solution(solution: Any, n_vars: int) -> np.ndarray: @@ -109,112 +109,33 @@ def _unique_cuopt_log_file(log_dir: str | Path, *, random_seed: int | None) -> s return str(target / f"cuopt_seed-{seed_token}_pid-{os.getpid()}_{time.time_ns()}.log") -def solve_portfolio_cuopt_native( - *, - loans: pd.DataFrame, - pd_point: np.ndarray, - pd_high: np.ndarray, - lgd: np.ndarray, - int_rates: np.ndarray, - total_budget: float = 1_000_000, - max_concentration: float = 0.25, - max_portfolio_pd: float = 0.10, - robust: bool = True, - uncertainty_aversion: float = 0.0, - min_budget_utilization: float = 0.0, - pd_cap_slack_penalty: float = 0.0, - pd_constraint_override: np.ndarray | None = None, - time_limit: int = 300, - random_seed: int | None = None, - presolve: int | None = 1, - cuopt_parameters: dict[str, Any] | None = None, -) -> dict[str, Any]: - """Solve the portfolio LP natively with cuOpt.""" - lp_api = _require_cuopt() - - n = len(loans) - if n == 0: - raise ValueError("Cannot solve empty portfolio.") - - loan_amounts = ( - loans["loan_amnt"].to_numpy(dtype=float) - if "loan_amnt" in loans.columns - else np.ones(n, dtype=float) * 10_000.0 - ) - point = np.asarray(pd_point, dtype=float) - high = np.asarray(pd_high, dtype=float) - lgd_arr = np.asarray(lgd, dtype=float) - rates = np.asarray(int_rates, dtype=float) - pd_constraint = ( - np.asarray(pd_constraint_override, dtype=float) - if pd_constraint_override is not None - else (high if robust else point) - ) - pd_uncertainty = np.clip(high - point, 0.0, 1.0) - - use_pd_slack = float(pd_cap_slack_penalty) > 0 - obj = loan_amounts * (rates - point * lgd_arr - uncertainty_aversion * pd_uncertainty * lgd_arr) - - rows: list[np.ndarray] = [] - rhs: list[float] = [] - row_types: list[str] = [] - - # Budget cap - rows.append(loan_amounts.astype(float)) - rhs.append(float(total_budget)) - row_types.append("L") - - # Optional minimum budget utilization - min_budget_utilization = float(np.clip(min_budget_utilization, 0.0, 1.0)) - if min_budget_utilization > 0: - rows.append((-loan_amounts).astype(float)) - rhs.append(float(-min_budget_utilization * total_budget)) - row_types.append("L") - - # Portfolio PD cap: sum(x_i * loan_i * (pd_i - max_pd)) - slack <= 0 - pd_row = loan_amounts * (pd_constraint - float(max_portfolio_pd)) - rows.append(pd_row.astype(float)) - rhs.append(0.0) - row_types.append("L") - - if "purpose" in loans.columns: - purposes = loans["purpose"].fillna("unknown").astype(str) - top_purposes = purposes.unique() - for purpose in top_purposes: - mask = (purposes == purpose).to_numpy(dtype=float) - row = loan_amounts * (mask - float(max_concentration)) - rows.append(row.astype(float)) - rhs.append(0.0) - row_types.append("L") - - A = np.vstack(rows).astype(np.float64) - - var_lb = np.zeros(n + int(use_pd_slack), dtype=np.float64) - var_ub = np.ones(n + int(use_pd_slack), dtype=np.float64) - if use_pd_slack: - slack_col = np.zeros((A.shape[0], 1), dtype=np.float64) - pd_cap_row_idx = 2 if min_budget_utilization > 0 else 1 - slack_col[pd_cap_row_idx, 0] = -1.0 - A = np.hstack([A, slack_col]) - obj = np.concatenate([obj.astype(np.float64), np.array([-float(pd_cap_slack_penalty)])]) - var_ub[-1] = float(total_budget) - else: - obj = obj.astype(np.float64) - - A_csr = csr_matrix(A) +def _cuopt_data_model(lp_api: Any, components: _PortfolioLpComponents) -> Any: + matrix = components.a_ub.tocsr() dm = lp_api.DataModel() dm.set_csr_constraint_matrix( - A_csr.data.astype(np.float64), - A_csr.indices.astype(np.int32), - A_csr.indptr.astype(np.int32), + matrix.data.astype(np.float64), + matrix.indices.astype(np.int32), + matrix.indptr.astype(np.int32), ) - dm.set_constraint_bounds(np.asarray(rhs, dtype=np.float64)) - dm.set_row_types(np.asarray(row_types)) - dm.set_objective_coefficients(obj) + dm.set_constraint_bounds(components.rhs.astype(np.float64)) + dm.set_row_types(np.asarray(["L"] * len(components.rhs))) + dm.set_objective_coefficients(components.objective_coefficients.astype(np.float64)) dm.set_maximize(True) - dm.set_variable_lower_bounds(var_lb) - dm.set_variable_upper_bounds(var_ub) + bounds = np.asarray(components.bounds, dtype=np.float64) + dm.set_variable_lower_bounds(bounds[:, 0]) + dm.set_variable_upper_bounds(bounds[:, 1]) + return dm + + +def _cuopt_solver_settings( + lp_api: Any, + *, + time_limit: int, + random_seed: int | None, + presolve: int | None, + cuopt_parameters: dict[str, Any] | None, +) -> tuple[Any, dict[str, Any], dict[str, str]]: settings = lp_api.SolverSettings() requested_parameters = { _normalize_parameter_name(k): _coerce_setting_value(v) @@ -237,6 +158,7 @@ def solve_portfolio_cuopt_native( for name, value in requested_parameters.items(): if value is not None: applied_parameters[name] = value + rejected_parameters: dict[str, str] = {} critical_parameters = { "time_limit", @@ -253,19 +175,23 @@ def solve_portfolio_cuopt_native( rejected_parameters[name] = str(exc) if name in critical_parameters: raise + return settings, applied_parameters, rejected_parameters - solution = lp_api.Solve(dm, settings) - termination_reason = str(solution.get_termination_reason()) - if "Optimal" not in termination_reason and "Feasible" not in termination_reason: - raise RuntimeError( - f"cuOpt solve did not produce an acceptable solution: {termination_reason}" - ) - primal = _extract_primal_solution(solution, n + int(use_pd_slack)) - x = primal[:n] - pd_cap_slack = float(primal[-1]) if use_pd_slack else 0.0 - allocation = {i: float(x[i]) for i in range(n)} - total_allocated = float(np.sum(x * loan_amounts)) +def _cuopt_result_payload( + *, + solution: Any, + primal: np.ndarray, + components: _PortfolioLpComponents, + termination_reason: str, + applied_parameters: dict[str, Any], + rejected_parameters: dict[str, str], +) -> dict[str, Any]: + x = np.clip(primal[: components.n], 0.0, 1.0) + pd_cap_slack = float(primal[-1]) if components.use_pd_slack else 0.0 + allocation = {i: float(x[i]) for i in range(components.n)} + allocation_vector = x.astype(float) + total_allocated = float(np.sum(x * components.loan_amounts)) n_funded = int(np.sum(x > 0.01)) obj_value = float(solution.get_primal_objective()) @@ -273,13 +199,14 @@ def solve_portfolio_cuopt_native( "Portfolio solved (cuopt_native): obj={:,.2f}, funded={}/{}, allocated={:,.0f}, pd_cap_slack={:.4f}", obj_value, n_funded, - n, + components.n, total_allocated, pd_cap_slack, ) return { "allocation": allocation, + "allocation_vector": allocation_vector, "objective_value": obj_value, "n_funded": n_funded, "total_allocated": total_allocated, @@ -292,3 +219,67 @@ def solve_portfolio_cuopt_native( "cuopt_log_file": applied_parameters.get("log_file", ""), "cuopt_rejected_parameters": rejected_parameters, } + + +def solve_portfolio_cuopt_native( + *, + loans: pd.DataFrame, + pd_point: np.ndarray, + pd_high: np.ndarray, + lgd: np.ndarray, + int_rates: np.ndarray, + total_budget: float = 1_000_000, + max_concentration: float = 0.25, + max_portfolio_pd: float = 0.10, + robust: bool = True, + uncertainty_aversion: float = 0.0, + min_budget_utilization: float = 0.0, + pd_cap_slack_penalty: float = 0.0, + pd_constraint_override: np.ndarray | None = None, + time_limit: int = 300, + random_seed: int | None = None, + presolve: int | None = 1, + cuopt_parameters: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Solve the portfolio LP natively with cuOpt.""" + lp_api = _require_cuopt() + components = _portfolio_lp_components( + loans=loans, + pd_point=pd_point, + pd_high=pd_high, + lgd=lgd, + int_rates=int_rates, + total_budget=total_budget, + max_concentration=max_concentration, + max_portfolio_pd=max_portfolio_pd, + robust=robust, + uncertainty_aversion=uncertainty_aversion, + min_budget_utilization=min_budget_utilization, + pd_cap_slack_penalty=pd_cap_slack_penalty, + pd_constraint_override=pd_constraint_override, + ) + dm = _cuopt_data_model(lp_api, components) + settings, applied_parameters, rejected_parameters = _cuopt_solver_settings( + lp_api, + time_limit=time_limit, + random_seed=random_seed, + presolve=presolve, + cuopt_parameters=cuopt_parameters, + ) + + solution = lp_api.Solve(dm, settings) + termination_reason = str(solution.get_termination_reason()) + if "Optimal" not in termination_reason and "Feasible" not in termination_reason: + raise RuntimeError( + f"cuOpt solve did not produce an acceptable solution: {termination_reason}" + ) + + primal = _extract_primal_solution(solution, components.n + int(components.use_pd_slack)) + return _cuopt_result_payload( + solution=solution, + primal=primal, + components=components, + termination_reason=termination_reason, + applied_parameters=applied_parameters, + rejected_parameters=rejected_parameters, + ) diff --git a/src/optimization/input_alignment.py b/src/optimization/input_alignment.py new file mode 100644 index 0000000..2091af6 --- /dev/null +++ b/src/optimization/input_alignment.py @@ -0,0 +1,195 @@ +"""Deterministic alignment of candidate loans and conformal interval rows.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal + +import numpy as np +import pandas as pd + +AlignmentMode = Literal["id", "row_number", "position"] + + +@dataclass(frozen=True) +class AlignedCandidateIntervals: + """Candidate and interval frames that share the same sampled row order.""" + + candidates: pd.DataFrame + intervals: pd.DataFrame + mode: AlignmentMode + available_rows: int + + @property + def selected_rows(self) -> int: + return len(self.candidates) + + +def _normalized_limit(max_candidates: int | None) -> int | None: + if max_candidates is None or int(max_candidates) <= 0: + return None + return int(max_candidates) + + +def _sample_positions( + row_count: int, + *, + max_candidates: int | None, + random_state: int, +) -> np.ndarray: + if row_count <= 0: + raise ValueError("Candidate/interval alignment produced zero rows.") + limit = _normalized_limit(max_candidates) + sample_size = row_count if limit is None else min(row_count, limit) + if sample_size == row_count: + return np.arange(row_count, dtype=int) + positions = np.random.default_rng(random_state).choice( + row_count, + size=sample_size, + replace=False, + ) + return np.sort(positions.astype(int, copy=False)) + + +def _require_unique_columns(frame: pd.DataFrame, *, source: str) -> None: + duplicated = frame.columns[frame.columns.duplicated()].tolist() + if duplicated: + raise ValueError(f"{source} contains duplicate column names: {duplicated}") + + +def _validated_string_key(series: pd.Series, *, source: str) -> pd.Series: + if series.isna().any(): + raise ValueError(f"{source} alignment key contains missing values.") + keys = series.astype(str) + if keys.str.strip().eq("").any(): + raise ValueError(f"{source} alignment key contains blank values.") + duplicated = keys[keys.duplicated(keep=False)] + if not duplicated.empty: + examples = duplicated.drop_duplicates().head(3).tolist() + raise ValueError(f"{source} alignment key is not unique; examples={examples}") + return keys + + +def _validated_row_number_key(series: pd.Series) -> pd.Series: + if series.isna().any(): + raise ValueError("interval _row_number contains missing values.") + numeric = pd.to_numeric(series, errors="raise") + values = numeric.to_numpy(dtype=float) + if not np.isfinite(values).all() or not np.equal(values, np.floor(values)).all(): + raise ValueError("interval _row_number must contain finite integers.") + keys = pd.Series(values.astype(np.int64), index=series.index) + duplicated = keys[keys.duplicated(keep=False)] + if not duplicated.empty: + examples = duplicated.drop_duplicates().head(3).tolist() + raise ValueError(f"interval _row_number is not unique; examples={examples}") + return keys + + +def _unused_column_name(base: str, used: set[str]) -> str: + name = base + suffix = 1 + while name in used: + name = f"{base}_{suffix}" + suffix += 1 + used.add(name) + return name + + +def _align_by_key( + candidates: pd.DataFrame, + intervals: pd.DataFrame, + *, + candidate_keys: pd.Series, + interval_keys: pd.Series, + mode: AlignmentMode, + max_candidates: int | None, + random_state: int, +) -> AlignedCandidateIntervals: + used_columns = set(candidates.columns) | set(intervals.columns) + join_column = _unused_column_name("__crpto_alignment_key", used_columns) + interval_aliases = { + column: _unused_column_name(f"__crpto_interval_{index}", used_columns) + for index, column in enumerate(intervals.columns) + } + + candidate_work = candidates.copy() + candidate_work[join_column] = candidate_keys.to_numpy() + interval_work = intervals.rename(columns=interval_aliases).copy() + interval_work[join_column] = interval_keys.to_numpy() + merged = candidate_work.merge( + interval_work, + on=join_column, + how="inner", + sort=False, + validate="one_to_one", + ) + positions = _sample_positions( + len(merged), + max_candidates=max_candidates, + random_state=random_state, + ) + sampled = merged.iloc[positions].reset_index(drop=True) + aligned_candidates = sampled.loc[:, list(candidates.columns)].copy() + aligned_intervals = sampled.loc[:, list(interval_aliases.values())].rename( + columns={alias: source for source, alias in interval_aliases.items()} + ) + return AlignedCandidateIntervals( + candidates=aligned_candidates, + intervals=aligned_intervals, + mode=mode, + available_rows=len(merged), + ) + + +def align_candidate_intervals( + candidates: pd.DataFrame, + intervals: pd.DataFrame, + *, + max_candidates: int | None, + random_state: int, + allow_row_number: bool = True, +) -> AlignedCandidateIntervals: + """Align and sample candidate/interval rows under a strict one-to-one contract. + + Stable IDs take precedence. Legacy interval artifacts may instead provide a + zero-based ``_row_number``. A positional fallback is retained for older + artifacts, but samples from the full alignable universe rather than taking + a deterministic prefix. + """ + _require_unique_columns(candidates, source="candidates") + _require_unique_columns(intervals, source="intervals") + + if "id" in candidates.columns and "id" in intervals.columns: + return _align_by_key( + candidates, + intervals, + candidate_keys=_validated_string_key(candidates["id"], source="candidate id"), + interval_keys=_validated_string_key(intervals["id"], source="interval id"), + mode="id", + max_candidates=max_candidates, + random_state=random_state, + ) + + if allow_row_number and "_row_number" in intervals.columns: + return _align_by_key( + candidates, + intervals, + candidate_keys=pd.Series(np.arange(len(candidates), dtype=np.int64)), + interval_keys=_validated_row_number_key(intervals["_row_number"]), + mode="row_number", + max_candidates=max_candidates, + random_state=random_state, + ) + + available_rows = min(len(candidates), len(intervals)) + positions = _sample_positions( + available_rows, + max_candidates=max_candidates, + random_state=random_state, + ) + return AlignedCandidateIntervals( + candidates=candidates.iloc[positions].reset_index(drop=True).copy(), + intervals=intervals.iloc[positions].reset_index(drop=True).copy(), + mode="position", + available_rows=available_rows, + ) diff --git a/src/optimization/portfolio_model.py b/src/optimization/portfolio_model.py index ce180f6..ebc9138 100644 --- a/src/optimization/portfolio_model.py +++ b/src/optimization/portfolio_model.py @@ -10,6 +10,8 @@ from __future__ import annotations import os +from collections.abc import Mapping +from dataclasses import dataclass from typing import Any, cast import numpy as np @@ -22,6 +24,45 @@ from src.optimization.policy import PolicyMode, resolve_policy_mode +@dataclass(frozen=True) +class _PortfolioLpComponents: + n: int + loan_amounts: np.ndarray + objective_coefficients: np.ndarray + a_ub: csr_matrix + rhs: np.ndarray + bounds: list[tuple[float, float]] + use_pd_slack: bool + + +def solution_allocation_vector(solution: Mapping[str, Any], n_items: int) -> np.ndarray: + """Return a validated dense allocation vector from any solver payload.""" + n = int(n_items) + if n < 0: + raise ValueError(f"n_items must be nonnegative, got {n}") + + raw_vector = solution.get("allocation_vector") + if raw_vector is not None: + allocation = np.asarray(raw_vector, dtype=float) + else: + raw_mapping = solution.get("allocation") + if not isinstance(raw_mapping, Mapping): + raise TypeError( + "Solver result must contain allocation_vector or an allocation mapping." + ) + allocation = np.fromiter( + (float(raw_mapping.get(i, 0.0)) for i in range(n)), + dtype=float, + count=n, + ) + + if allocation.shape != (n,): + raise ValueError(f"Solver allocation shape mismatch: {allocation.shape} != {(n,)}") + if not np.all(np.isfinite(allocation)): + raise ValueError("Solver allocation contains non-finite values.") + return allocation + + def compute_effective_pd( pd_point: np.ndarray, pd_high: np.ndarray, @@ -61,71 +102,93 @@ def compute_effective_pd( if mode is PolicyMode.HARD_WORST_CASE: return high if mode is PolicyMode.BLENDED_UNCERTAINTY: - weight = float(np.clip(gamma, 0.0, 1.0)) - return cast( - np.ndarray, - np.clip(point + weight * np.clip(high - point, 0.0, 1.0), 0.0, 1.0), - ) + return _blend_pd(point, _pd_delta(point, high), gamma) if mode is PolicyMode.CAPPED_BLENDED_UNCERTAINTY: - weight = float(np.clip(gamma, 0.0, 1.0)) - delta = np.clip(high - point, 0.0, 1.0) - q = 1.0 if delta_cap_quantile is None else float(np.clip(delta_cap_quantile, 0.0, 1.0)) - delta_cap = float(np.quantile(delta, q)) if len(delta) else 0.0 - return cast(np.ndarray, np.clip(point + weight * np.minimum(delta, delta_cap), 0.0, 1.0)) + delta = _pd_delta(point, high) + cap = _quantile_or_zero(delta, _clipped_quantile(delta_cap_quantile, default=1.0)) + return _blend_pd(point, np.minimum(delta, cap), gamma) if mode is PolicyMode.TAIL_BLENDED_UNCERTAINTY: - weight = float(np.clip(gamma, 0.0, 1.0)) - delta = np.clip(high - point, 0.0, 1.0) - q = 0.9 if tail_focus_quantile is None else float(np.clip(tail_focus_quantile, 0.0, 1.0)) - cutoff = float(np.quantile(delta, q)) if len(delta) else 0.0 - local_delta = np.where(delta >= cutoff, delta, 0.0) - return np.clip(point + weight * local_delta, 0.0, 1.0) + delta = _pd_delta(point, high) + local_delta = _tail_delta( + delta=delta, + score=delta, + q=_clipped_quantile(tail_focus_quantile, default=0.9), + ) + return _blend_pd(point, local_delta, gamma) if mode is PolicyMode.SEGMENT_TAIL_BLENDED_UNCERTAINTY: - weight = float(np.clip(gamma, 0.0, 1.0)) - delta = np.clip(high - point, 0.0, 1.0) - q = 0.9 if tail_focus_quantile is None else float(np.clip(tail_focus_quantile, 0.0, 1.0)) - global_cutoff = float(np.quantile(delta, q)) if len(delta) else 0.0 - if segment_labels is None or len(segment_labels) != len(delta): - local_delta = np.where(delta >= global_cutoff, delta, 0.0) - return np.clip(point + weight * local_delta, 0.0, 1.0) - - labels = pd.Series(np.asarray(segment_labels, dtype=object)).fillna("unknown").astype(str) - local_delta = np.zeros_like(delta) - for label in labels.unique(): - mask = labels == label - mask_arr = mask.to_numpy(dtype=bool) - seg_delta = delta[mask_arr] - if len(seg_delta) < int(max(min_segment_size, 1)): - cutoff = global_cutoff - else: - cutoff = float(np.quantile(seg_delta, q)) - local_delta[mask_arr] = np.where(seg_delta >= cutoff, seg_delta, 0.0) - return np.clip(point + weight * local_delta, 0.0, 1.0) + delta = _pd_delta(point, high) + q = _clipped_quantile(tail_focus_quantile, default=0.9) + local_delta = _segment_tail_delta( + delta=delta, + score=delta, + q=q, + segment_labels=segment_labels, + min_segment_size=min_segment_size, + ) + return _blend_pd(point, local_delta, gamma) if mode is PolicyMode.SEGMENT_RELATIVE_TAIL_BLENDED_UNCERTAINTY: - weight = float(np.clip(gamma, 0.0, 1.0)) - delta = np.clip(high - point, 0.0, 1.0) + delta = _pd_delta(point, high) relative_width = delta / np.maximum(point, 1e-4) - q = 0.9 if tail_focus_quantile is None else float(np.clip(tail_focus_quantile, 0.0, 1.0)) - global_cutoff = float(np.quantile(relative_width, q)) if len(relative_width) else 0.0 - if segment_labels is None or len(segment_labels) != len(delta): - local_delta = np.where(relative_width >= global_cutoff, delta, 0.0) - return np.clip(point + weight * local_delta, 0.0, 1.0) - - labels = pd.Series(np.asarray(segment_labels, dtype=object)).fillna("unknown").astype(str) - local_delta = np.zeros_like(delta) - for label in labels.unique(): - mask = labels == label - mask_arr = mask.to_numpy(dtype=bool) - seg_delta = delta[mask_arr] - seg_relative = relative_width[mask_arr] - if len(seg_relative) < int(max(min_segment_size, 1)): - cutoff = global_cutoff - else: - cutoff = float(np.quantile(seg_relative, q)) - local_delta[mask_arr] = np.where(seg_relative >= cutoff, seg_delta, 0.0) - return np.clip(point + weight * local_delta, 0.0, 1.0) + q = _clipped_quantile(tail_focus_quantile, default=0.9) + local_delta = _segment_tail_delta( + delta=delta, + score=relative_width, + q=q, + segment_labels=segment_labels, + min_segment_size=min_segment_size, + ) + return _blend_pd(point, local_delta, gamma) raise ValueError(f"Unhandled policy mode: {mode!r}") # safety net, unreachable +def _pd_delta(point: np.ndarray, high: np.ndarray) -> np.ndarray: + return cast(np.ndarray, np.clip(high - point, 0.0, 1.0)) + + +def _clipped_quantile(value: float | None, *, default: float) -> float: + raw = default if value is None else float(value) + return float(np.clip(raw, 0.0, 1.0)) + + +def _quantile_or_zero(values: np.ndarray, q: float) -> float: + return float(np.quantile(values, q)) if len(values) else 0.0 + + +def _blend_pd(point: np.ndarray, delta: np.ndarray, gamma: float) -> np.ndarray: + weight = float(np.clip(gamma, 0.0, 1.0)) + return np.clip(point + weight * delta, 0.0, 1.0) + + +def _tail_delta(*, delta: np.ndarray, score: np.ndarray, q: float) -> np.ndarray: + cutoff = _quantile_or_zero(score, q) + return np.where(score >= cutoff, delta, 0.0) + + +def _segment_tail_delta( + *, + delta: np.ndarray, + score: np.ndarray, + q: float, + segment_labels: np.ndarray | None, + min_segment_size: int, +) -> np.ndarray: + if segment_labels is None or len(segment_labels) != len(delta): + return _tail_delta(delta=delta, score=score, q=q) + + labels = pd.Series(np.asarray(segment_labels, dtype=object)).fillna("unknown").astype(str) + local_delta = np.zeros_like(delta) + global_cutoff = _quantile_or_zero(score, q) + min_size = int(max(min_segment_size, 1)) + for label in labels.unique(): + mask_arr = (labels == label).to_numpy(dtype=bool) + segment_score = score[mask_arr] + cutoff = ( + global_cutoff if len(segment_score) < min_size else _quantile_or_zero(segment_score, q) + ) + local_delta[mask_arr] = np.where(segment_score >= cutoff, delta[mask_arr], 0.0) + return local_delta + + def build_portfolio_model( loans: pd.DataFrame, pd_point: np.ndarray, @@ -264,49 +327,102 @@ def solve_portfolio( ) -> dict[str, Any]: """Solve portfolio optimization with HiGHS (default) or optional cuOpt.""" backend = solver_backend.strip().lower() + results = _solve_pyomo_backend( + model, + backend=backend, + time_limit=time_limit, + threads=threads, + ) + solution = _pyomo_portfolio_solution(model, backend=backend, results=results) + logger.info( + "Portfolio solved ({}): obj={:,.2f}, funded={}/{}, allocated={:,.0f}, pd_cap_slack={:.4f}", + backend, + solution["objective_value"], + solution["n_funded"], + len(solution["allocation"]), + solution["total_allocated"], + solution["pd_cap_slack"], + ) + return solution + + +def _solve_pyomo_backend( + model: pyo.ConcreteModel, + *, + backend: str, + time_limit: int, + threads: int, +) -> Any: if backend == "highs": - from pyomo.contrib.appsi.solvers import Highs - - solver = Highs() - solver.config.time_limit = time_limit - _ = threads # reserved for future HiGHS appsi configuration - results = solver.solve(model) - elif backend == "cuopt": - solver = pyo.SolverFactory("cuopt") - if solver is None or not solver.available(False): - raise RuntimeError( - "solver_backend='cuopt' requested but Pyomo cuOpt solver is not available " - "in this environment." - ) - _ = (time_limit, threads) # backend-specific options vary by cuOpt deployment - results = solver.solve(model) - else: - raise ValueError(f"Unsupported solver_backend={solver_backend!r}. Use 'highs' or 'cuopt'.") + return _solve_pyomo_highs(model, time_limit=time_limit, threads=threads) + if backend == "cuopt": + return _solve_pyomo_cuopt(model, time_limit=time_limit, threads=threads) + raise ValueError(f"Unsupported solver_backend={backend!r}. Use 'highs' or 'cuopt'.") + + +def _solve_pyomo_highs( + model: pyo.ConcreteModel, + *, + time_limit: int, + threads: int, +) -> Any: + from pyomo.contrib.appsi.solvers import Highs + + solver = Highs() + solver.config.time_limit = time_limit + _ = threads # reserved for future HiGHS appsi configuration + return solver.solve(model) - allocation = {i: pyo.value(model.x[i]) for i in model.I} - obj_value = pyo.value(model.obj) + +def _solve_pyomo_cuopt( + model: pyo.ConcreteModel, + *, + time_limit: int, + threads: int, +) -> Any: + solver = pyo.SolverFactory("cuopt") + if solver is None or not solver.available(False): + raise RuntimeError( + "solver_backend='cuopt' requested but Pyomo cuOpt solver is not available " + "in this environment." + ) + _ = (time_limit, threads) # backend-specific options vary by cuOpt deployment + return solver.solve(model) + + +def _pyomo_portfolio_solution( + model: pyo.ConcreteModel, + *, + backend: str, + results: Any, +) -> dict[str, Any]: + index_set = cast(Any, model.I) + decision_vars = cast(Any, model.x) + loan_amount_param = cast(Any, model.loan_amnt) + allocation = {i: pyo.value(decision_vars[i]) for i in index_set} + obj_value = pyo.value(cast(Any, model.obj)) n_funded = sum(1 for v in allocation.values() if v > 0.01) - total_allocated = sum(allocation[i] * pyo.value(model.loan_amnt[i]) for i in model.I) - pd_cap_slack = float(pyo.value(model.pd_cap_slack)) if hasattr(model, "pd_cap_slack") else 0.0 - termination = getattr(results, "termination_condition", None) - if termination is None and hasattr(results, "solver"): - termination = getattr(results.solver, "termination_condition", None) + total_allocated = sum(allocation[i] * pyo.value(loan_amount_param[i]) for i in index_set) + pd_cap_slack = ( + float(pyo.value(cast(Any, model.pd_cap_slack))) if hasattr(model, "pd_cap_slack") else 0.0 + ) - solution = { + return { "allocation": allocation, "objective_value": float(obj_value), "n_funded": int(n_funded), "total_allocated": float(total_allocated), - "solver_status": str(termination) if termination is not None else "unknown", + "solver_status": _pyomo_termination_status(results), "solver_backend": backend, "pd_cap_slack": pd_cap_slack, } - logger.info( - f"Portfolio solved ({backend}): obj={obj_value:,.2f}, funded={n_funded}/{len(allocation)}, " - f"allocated={total_allocated:,.0f}, pd_cap_slack={pd_cap_slack:.4f}" - ) - return solution + +def _pyomo_termination_status(results: Any) -> str: + termination = getattr(results, "termination_condition", None) + if termination is None and hasattr(results, "solver"): + termination = getattr(results.solver, "termination_condition", None) + return str(termination) if termination is not None else "unknown" def solve_portfolio_highs_sparse( @@ -334,62 +450,21 @@ def solve_portfolio_highs_sparse( model construction. The CRPTO exact rerank solves thousands of very similar large LPs, so avoiding per-check symbolic model creation is material. """ - n = len(loans) - if n == 0: - raise ValueError("Cannot solve empty portfolio.") - - loan_amounts = ( - pd.to_numeric(loans["loan_amnt"], errors="coerce").fillna(1.0).to_numpy(dtype=float) - if "loan_amnt" in loans.columns - else np.ones(n, dtype=float) - ) - point = np.asarray(pd_point, dtype=float) - high = np.asarray(pd_high, dtype=float) - lgd_arr = np.asarray(lgd, dtype=float) - rates = np.asarray(int_rates, dtype=float) - pd_constraint = ( - np.asarray(pd_constraint_override, dtype=float) - if pd_constraint_override is not None - else (high if robust else point) - ) - pd_uncertainty = np.clip(high - point, 0.0, 1.0) - - objective = loan_amounts * ( - rates - point * lgd_arr - float(uncertainty_aversion) * pd_uncertainty * lgd_arr + components = _portfolio_lp_components( + loans=loans, + pd_point=pd_point, + pd_high=pd_high, + lgd=lgd, + int_rates=int_rates, + total_budget=total_budget, + max_concentration=max_concentration, + max_portfolio_pd=max_portfolio_pd, + robust=robust, + uncertainty_aversion=uncertainty_aversion, + min_budget_utilization=min_budget_utilization, + pd_cap_slack_penalty=pd_cap_slack_penalty, + pd_constraint_override=pd_constraint_override, ) - use_pd_slack = float(pd_cap_slack_penalty) > 0.0 - - rows: list[np.ndarray] = [] - rhs: list[float] = [] - - rows.append(loan_amounts.astype(float)) - rhs.append(float(total_budget)) - - min_budget_utilization = float(np.clip(min_budget_utilization, 0.0, 1.0)) - if min_budget_utilization > 0: - rows.append((-loan_amounts).astype(float)) - rhs.append(float(-min_budget_utilization * total_budget)) - - pd_cap_row_idx = len(rows) - rows.append((loan_amounts * (pd_constraint - float(max_portfolio_pd))).astype(float)) - rhs.append(0.0) - - if "purpose" in loans.columns: - purposes = loans["purpose"].fillna("unknown").astype(str) - for purpose in purposes.unique(): - mask = (purposes == purpose).to_numpy(dtype=float) - rows.append((loan_amounts * (mask - float(max_concentration))).astype(float)) - rhs.append(0.0) - - A_ub = csr_matrix(np.vstack(rows).astype(float)) - c = -objective.astype(float) - bounds: list[tuple[float, float | None]] = [(0.0, 1.0)] * n - if use_pd_slack: - slack_col = np.zeros((A_ub.shape[0], 1), dtype=float) - slack_col[pd_cap_row_idx, 0] = -1.0 - A_ub = hstack([A_ub, csr_matrix(slack_col)], format="csr") - c = np.concatenate([c, np.array([float(pd_cap_slack_penalty)], dtype=float)]) - bounds.append((0.0, float(total_budget))) options: dict[str, Any] = { "time_limit": float(time_limit), @@ -398,10 +473,10 @@ def solve_portfolio_highs_sparse( } _ = threads # SciPy's HiGHS wrapper does not expose a documented threads option. result = linprog( - c, - A_ub=A_ub, - b_ub=np.asarray(rhs, dtype=float), - bounds=bounds, + -components.objective_coefficients.astype(float), + A_ub=components.a_ub, + b_ub=components.rhs, + bounds=components.bounds, method="highs", options=options, ) @@ -412,33 +487,23 @@ def solve_portfolio_highs_sparse( ) primal = np.asarray(result.x, dtype=float) - alloc = np.clip(primal[:n], 0.0, 1.0) - pd_cap_slack = float(primal[-1]) if use_pd_slack else 0.0 - total_allocated = float(np.sum(alloc * loan_amounts)) - obj_value = float(np.sum(alloc * objective) - float(pd_cap_slack_penalty) * pd_cap_slack) - n_funded = int(np.sum(alloc > 0.01)) - allocation = {i: float(value) for i, value in enumerate(alloc) if value > 1e-12} + summary = _portfolio_solution_summary(primal, components) solver_status = "optimal" if bool(result.success) else str(result.message) logger.info( "Portfolio solved (highs_sparse): obj={:,.2f}, funded={}/{}, allocated={:,.0f}, " "pd_cap_slack={:.4f}, status={}", - obj_value, - n_funded, - n, - total_allocated, - pd_cap_slack, + summary["objective_value"], + summary["n_funded"], + components.n, + summary["total_allocated"], + summary["pd_cap_slack"], solver_status, ) return { - "allocation": allocation, - "allocation_vector": alloc, - "objective_value": obj_value, - "n_funded": n_funded, - "total_allocated": total_allocated, + **summary, "solver_status": solver_status, "solver_backend": "highs_sparse", - "pd_cap_slack": pd_cap_slack, "highs_status": int(result.status), "highs_message": str(result.message), "highs_iterations": int(getattr(result, "nit", 0) or 0), @@ -472,21 +537,177 @@ def solve_portfolio_highspy_native( HiGHS' own thread/parallel options while preserving the same LP algebra. """ try: - import highspy # type: ignore[import-not-found] + import highspy except Exception as exc: # pragma: no cover - optional runtime dependency raise RuntimeError( "solver_backend='highspy' requested but highspy is unavailable." ) from exc + components = _portfolio_lp_components( + loans=loans, + pd_point=pd_point, + pd_high=pd_high, + lgd=lgd, + int_rates=int_rates, + total_budget=total_budget, + max_concentration=max_concentration, + max_portfolio_pd=max_portfolio_pd, + robust=robust, + uncertainty_aversion=uncertainty_aversion, + min_budget_utilization=min_budget_utilization, + pd_cap_slack_penalty=pd_cap_slack_penalty, + pd_constraint_override=pd_constraint_override, + ) + lp = _build_highspy_lp(highspy, components) + solver = highspy.Highs() + _configure_highspy_solver(highspy, solver, time_limit=time_limit, threads=threads) + + status = solver.passModel(lp) + if status != highspy.HighsStatus.kOk: + raise RuntimeError(f"highspy failed to accept portfolio LP: {status}") + run_status = solver.run() + if run_status == highspy.HighsStatus.kError: + raise RuntimeError(f"highspy failed while solving portfolio LP: {run_status}") + + model_status = solver.getModelStatus() + status_text = str(solver.modelStatusToString(model_status)) + if "Optimal" not in status_text: + raise RuntimeError( + "highspy did not solve portfolio LP to optimality: " + f"run_status={run_status}, model_status={status_text}" + ) + + solution = solver.getSolution() + primal = np.asarray(solution.col_value, dtype=float) + if len(primal) < components.n: + raise RuntimeError( + f"highspy primal solution has length {len(primal)}; expected >= {components.n}." + ) + summary = _portfolio_solution_summary(primal, components) + info = solver.getInfo() + + logger.info( + "Portfolio solved (highspy): obj={:,.2f}, funded={}/{}, allocated={:,.0f}, " + "pd_cap_slack={:.4f}, status={}", + summary["objective_value"], + summary["n_funded"], + components.n, + summary["total_allocated"], + summary["pd_cap_slack"], + status_text, + ) + return { + **summary, + "solver_status": status_text, + "solver_backend": "highspy", + "highs_model_status": status_text, + "highs_simplex_iterations": int(getattr(info, "simplex_iteration_count", 0) or 0), + "highs_ipm_iterations": int(getattr(info, "ipm_iteration_count", 0) or 0), + } + + +def _build_highspy_lp(highspy: Any, components: _PortfolioLpComponents) -> Any: + a_ub: csc_matrix = components.a_ub.tocsc() + bounds = np.asarray(components.bounds, dtype=np.double) + lp = highspy.HighsLp() + lp.num_col_ = int(a_ub.shape[1]) + lp.num_row_ = int(a_ub.shape[0]) + lp.col_cost_ = components.objective_coefficients.astype(np.double).tolist() + lp.col_lower_ = bounds[:, 0].tolist() + lp.col_upper_ = bounds[:, 1].tolist() + lp.row_lower_ = np.full(a_ub.shape[0], -highspy.kHighsInf, dtype=np.double).tolist() + lp.row_upper_ = components.rhs.astype(np.double).tolist() + lp.sense_ = highspy.ObjSense.kMaximize + lp.a_matrix_.format_ = highspy.MatrixFormat.kColwise + lp.a_matrix_.num_col_ = int(a_ub.shape[1]) + lp.a_matrix_.num_row_ = int(a_ub.shape[0]) + lp.a_matrix_.start_ = a_ub.indptr.astype(np.int32).tolist() + lp.a_matrix_.index_ = a_ub.indices.astype(np.int32).tolist() + lp.a_matrix_.value_ = a_ub.data.astype(np.double).tolist() + return lp + + +def _configure_highspy_solver( + highspy: Any, + solver: Any, + *, + time_limit: int, + threads: int, +) -> None: + if _env_int("HIGHS_RESET_GLOBAL_SCHEDULER", 1) and hasattr(solver, "resetGlobalScheduler"): + solver.resetGlobalScheduler(True) + options: dict[str, bool | int | float | str] = { + "output_flag": False, + "log_to_console": False, + "time_limit": float(time_limit), + "presolve": _env_str("HIGHS_PRESOLVE", "on"), + "threads": max(1, int(threads)), + "parallel": _env_str("HIGHS_PARALLEL", "choose"), + "solver": _env_str("HIGHS_SOLVER", "choose"), + "simplex_strategy": _env_int("HIGHS_SIMPLEX_STRATEGY", 0), + } + for name, value in options.items(): + status = _set_highs_option(solver, name, value) + if status != highspy.HighsStatus.kOk and name == "threads": + status = _retry_highspy_threads_option(solver, name, value) + if status != highspy.HighsStatus.kOk: + logger.warning("HiGHS rejected option {}={!r}: {}", name, value, status) + + +def _set_highs_option( + highs_solver: Any, + name: str, + value: bool | int | float | str, +) -> Any: + return highs_solver.setOptionValue(name, value) + + +def _retry_highspy_threads_option( + solver: Any, + name: str, + value: bool | int | float | str, +) -> Any: + if hasattr(solver, "resetGlobalScheduler"): + solver.resetGlobalScheduler(True) + return _set_highs_option(solver, name, value) + + +def _env_str(name: str, default: str) -> str: + return str(os.environ.get(name, default)).strip() or default + + +def _env_int(name: str, default: int) -> int: + raw = os.environ.get(name) + if raw is None or not str(raw).strip(): + return int(default) + try: + return int(raw) + except ValueError: + logger.warning("Ignoring invalid {}={!r}; using {}", name, raw, default) + return int(default) + + +def _portfolio_lp_components( + *, + loans: pd.DataFrame, + pd_point: np.ndarray, + pd_high: np.ndarray, + lgd: np.ndarray, + int_rates: np.ndarray, + total_budget: float, + max_concentration: float, + max_portfolio_pd: float, + robust: bool, + uncertainty_aversion: float, + min_budget_utilization: float, + pd_cap_slack_penalty: float, + pd_constraint_override: np.ndarray | None, +) -> _PortfolioLpComponents: n = len(loans) if n == 0: raise ValueError("Cannot solve empty portfolio.") - loan_amounts = ( - pd.to_numeric(loans["loan_amnt"], errors="coerce").fillna(1.0).to_numpy(dtype=float) - if "loan_amnt" in loans.columns - else np.ones(n, dtype=float) - ) + loan_amounts = _loan_amounts(loans, n) point = np.asarray(pd_point, dtype=float) high = np.asarray(pd_high, dtype=float) lgd_arr = np.asarray(lgd, dtype=float) @@ -497,17 +718,64 @@ def solve_portfolio_highspy_native( else (high if robust else point) ) pd_uncertainty = np.clip(high - point, 0.0, 1.0) - objective = loan_amounts * ( rates - point * lgd_arr - float(uncertainty_aversion) * pd_uncertainty * lgd_arr ) + + rows, rhs, pd_cap_row_idx = _portfolio_constraint_rows( + loans=loans, + loan_amounts=loan_amounts, + pd_constraint=pd_constraint, + total_budget=total_budget, + max_concentration=max_concentration, + max_portfolio_pd=max_portfolio_pd, + min_budget_utilization=min_budget_utilization, + ) + a_ub = csr_matrix(np.vstack(rows).astype(float)) + objective_coefficients = objective.astype(float) + bounds = [(0.0, 1.0)] * n use_pd_slack = float(pd_cap_slack_penalty) > 0.0 + if use_pd_slack: + slack_col = np.zeros((a_ub.shape[0], 1), dtype=float) + slack_col[pd_cap_row_idx, 0] = -1.0 + a_ub = hstack([a_ub, csr_matrix(slack_col)], format="csr") + objective_coefficients = np.concatenate( + [objective_coefficients, np.array([-float(pd_cap_slack_penalty)], dtype=float)] + ) + bounds.append((0.0, float(total_budget))) - rows: list[np.ndarray] = [] - rhs: list[float] = [] + return _PortfolioLpComponents( + n=n, + loan_amounts=loan_amounts, + objective_coefficients=objective_coefficients, + a_ub=a_ub, + rhs=np.asarray(rhs, dtype=float), + bounds=bounds, + use_pd_slack=use_pd_slack, + ) + + +def _loan_amounts(loans: pd.DataFrame, n: int) -> np.ndarray: + if "loan_amnt" not in loans.columns: + return np.ones(n, dtype=float) + return cast( + np.ndarray, + pd.to_numeric(loans["loan_amnt"], errors="coerce").fillna(1.0).to_numpy(dtype=float), + ) - rows.append(loan_amounts.astype(float)) - rhs.append(float(total_budget)) + +def _portfolio_constraint_rows( + *, + loans: pd.DataFrame, + loan_amounts: np.ndarray, + pd_constraint: np.ndarray, + total_budget: float, + max_concentration: float, + max_portfolio_pd: float, + min_budget_utilization: float, +) -> tuple[list[np.ndarray], list[float], int]: + rows: list[np.ndarray] = [loan_amounts.astype(float)] + rhs: list[float] = [float(total_budget)] min_budget_utilization = float(np.clip(min_budget_utilization, 0.0, 1.0)) if min_budget_utilization > 0: @@ -524,124 +792,24 @@ def solve_portfolio_highspy_native( mask = (purposes == purpose).to_numpy(dtype=float) rows.append((loan_amounts * (mask - float(max_concentration))).astype(float)) rhs.append(0.0) + return rows, rhs, pd_cap_row_idx - A_ub: csc_matrix = csr_matrix(np.vstack(rows).astype(float)).tocsc() - col_cost = objective.astype(np.double) - col_lower = np.zeros(n + int(use_pd_slack), dtype=np.double) - col_upper = np.ones(n + int(use_pd_slack), dtype=np.double) - if use_pd_slack: - slack_col = np.zeros((A_ub.shape[0], 1), dtype=float) - slack_col[pd_cap_row_idx, 0] = -1.0 - A_ub = hstack([A_ub, csc_matrix(slack_col)], format="csc") - col_cost = np.concatenate( - [col_cost, np.array([-float(pd_cap_slack_penalty)], dtype=np.double)] - ) - col_upper[-1] = float(total_budget) - inf = highspy.kHighsInf - lp = highspy.HighsLp() - lp.num_col_ = int(A_ub.shape[1]) - lp.num_row_ = int(A_ub.shape[0]) - lp.col_cost_ = col_cost - lp.col_lower_ = col_lower - lp.col_upper_ = col_upper - lp.row_lower_ = np.full(A_ub.shape[0], -inf, dtype=np.double) - lp.row_upper_ = np.asarray(rhs, dtype=np.double) - lp.sense_ = highspy.ObjSense.kMaximize - lp.a_matrix_.format_ = highspy.MatrixFormat.kColwise - lp.a_matrix_.num_col_ = int(A_ub.shape[1]) - lp.a_matrix_.num_row_ = int(A_ub.shape[0]) - lp.a_matrix_.start_ = A_ub.indptr.astype(np.int32) - lp.a_matrix_.index_ = A_ub.indices.astype(np.int32) - lp.a_matrix_.value_ = A_ub.data.astype(np.double) - - def _env_str(name: str, default: str) -> str: - return str(os.environ.get(name, default)).strip() or default - - def _env_int(name: str, default: int) -> int: - raw = os.environ.get(name) - if raw is None or not str(raw).strip(): - return int(default) - try: - return int(raw) - except ValueError: - logger.warning("Ignoring invalid {}={!r}; using {}", name, raw, default) - return int(default) - - solver = highspy.Highs() - if _env_int("HIGHS_RESET_GLOBAL_SCHEDULER", 1) and hasattr(solver, "resetGlobalScheduler"): - solver.resetGlobalScheduler(True) - options: dict[str, object] = { - "output_flag": False, - "log_to_console": False, - "time_limit": float(time_limit), - "presolve": _env_str("HIGHS_PRESOLVE", "on"), - "threads": max(1, int(threads)), - "parallel": _env_str("HIGHS_PARALLEL", "choose"), - "solver": _env_str("HIGHS_SOLVER", "choose"), - "simplex_strategy": _env_int("HIGHS_SIMPLEX_STRATEGY", 0), - } - for name, value in options.items(): - status = solver.setOptionValue(name, value) - if ( - status != highspy.HighsStatus.kOk - and name == "threads" - and hasattr(solver, "resetGlobalScheduler") - ): - solver.resetGlobalScheduler(True) - status = solver.setOptionValue(name, value) - if status != highspy.HighsStatus.kOk: - logger.warning("HiGHS rejected option {}={!r}: {}", name, value, status) - - status = solver.passModel(lp) - if status != highspy.HighsStatus.kOk: - raise RuntimeError(f"highspy failed to accept portfolio LP: {status}") - run_status = solver.run() - if run_status == highspy.HighsStatus.kError: - raise RuntimeError(f"highspy failed while solving portfolio LP: {run_status}") - - model_status = solver.getModelStatus() - status_text = str(solver.modelStatusToString(model_status)) - if "Optimal" not in status_text: - raise RuntimeError( - "highspy did not solve portfolio LP to optimality: " - f"run_status={run_status}, model_status={status_text}" - ) - - solution = solver.getSolution() - primal = np.asarray(solution.col_value, dtype=float) - if len(primal) < n: - raise RuntimeError(f"highspy primal solution has length {len(primal)}; expected >= {n}.") - alloc = np.clip(primal[:n], 0.0, 1.0) - pd_cap_slack = float(primal[-1]) if use_pd_slack else 0.0 - total_allocated = float(np.sum(alloc * loan_amounts)) - obj_value = float(np.sum(alloc * objective) - float(pd_cap_slack_penalty) * pd_cap_slack) - n_funded = int(np.sum(alloc > 0.01)) - allocation = {i: float(value) for i, value in enumerate(alloc) if value > 1e-12} - info = solver.getInfo() - - logger.info( - "Portfolio solved (highspy): obj={:,.2f}, funded={}/{}, allocated={:,.0f}, " - "pd_cap_slack={:.4f}, status={}", - obj_value, - n_funded, - n, - total_allocated, - pd_cap_slack, - status_text, - ) +def _portfolio_solution_summary( + primal: np.ndarray, + components: _PortfolioLpComponents, +) -> dict[str, Any]: + alloc = np.clip(primal[: components.n], 0.0, 1.0) + pd_cap_slack = float(primal[-1]) if components.use_pd_slack else 0.0 + total_allocated = float(np.sum(alloc * components.loan_amounts)) + objective_value = float(np.dot(primal, components.objective_coefficients)) return { - "allocation": allocation, + "allocation": {i: float(value) for i, value in enumerate(alloc) if value > 1e-12}, "allocation_vector": alloc, - "objective_value": obj_value, - "n_funded": n_funded, + "objective_value": objective_value, + "n_funded": int(np.sum(alloc > 0.01)), "total_allocated": total_allocated, - "solver_status": status_text, - "solver_backend": "highspy", "pd_cap_slack": pd_cap_slack, - "highs_model_status": status_text, - "highs_simplex_iterations": int(getattr(info, "simplex_iteration_count", 0) or 0), - "highs_ipm_iterations": int(getattr(info, "ipm_iteration_count", 0) or 0), } diff --git a/src/utils/mlflow_tracing.py b/src/utils/mlflow_tracing.py index 980bc40..18db545 100644 --- a/src/utils/mlflow_tracing.py +++ b/src/utils/mlflow_tracing.py @@ -102,7 +102,8 @@ def decorator(fn: T) -> T: if not _HAS_MLFLOW or not hasattr(mlflow, "trace"): return fn try: # pragma: no cover — depends on installed MLflow version. - wrapped = mlflow.trace(name=name or fn.__name__)(fn) + span_name = name or getattr(fn, "__name__", "trace") + wrapped = mlflow.trace(name=span_name)(fn) return cast(T, wrapped) except Exception: return fn diff --git a/src/utils/script_helpers.py b/src/utils/script_helpers.py index c381c01..72656e9 100644 --- a/src/utils/script_helpers.py +++ b/src/utils/script_helpers.py @@ -76,6 +76,8 @@ def write_table( match, so frozen tables under ``reports/crpto/tables`` keep their manifest hashes when regenerated from unchanged inputs. """ + table_dir = table_dir.resolve() + root = root.resolve() table_dir.mkdir(parents=True, exist_ok=True) csv_path = table_dir / f"{name}.csv" tex_path = table_dir / f"{name}.tex" @@ -101,6 +103,35 @@ def artifact_path(path_like: str | Path) -> Path: return (Path(root) / path) if root else path +def resolve_repo_artifact_path( + path_like: str | Path, + *, + root: Path = REPO_ROOT, +) -> Path: + """Resolve relative or foreign-OS paths that point inside this repository. + + Experiment manifests may have been written from WSL and therefore contain + ``/mnt/c/...//...`` paths. The artifact identity is repository-relative, + so a native Windows replay should resolve the suffix under its current root. + Absolute paths outside this repository are left unchanged. + """ + path = Path(path_like) + if path.exists(): + return path.resolve() + + normalized_parts = [part for part in str(path_like).replace("\\", "/").split("/") if part] + root_name = root.name.casefold() + matching_indices = [ + index for index, part in enumerate(normalized_parts) if part.casefold() == root_name + ] + if matching_indices: + suffix = normalized_parts[matching_indices[-1] + 1 :] + return root.joinpath(*suffix) + if not path.is_absolute(): + return root / path + return path + + def first_existing(*paths: Path) -> Path: """Return the first existing path, falling back to the last candidate.""" if not paths: diff --git a/tests/test_evaluation/test_model_shift.py b/tests/test_evaluation/test_model_shift.py index 8454e31..ba18f8a 100644 --- a/tests/test_evaluation/test_model_shift.py +++ b/tests/test_evaluation/test_model_shift.py @@ -39,3 +39,42 @@ def test_interpret_model_shift_detects_mixed_shift() -> None: assert out["shift_type"] == "mixed_shift" assert out["governance_posture"] == "candidate_gate" + + +def test_interpret_model_shift_detects_predictive_degradation_without_structural_shift() -> None: + out = interpret_model_shift( + c2st_auc=0.50, + c2st_materiality="none", + score_psi=0.02, + auc_delta=0.08, + brier_increase=0.001, + calibration_gap_delta=0.001, + distribution_warning_ratio=0.0, + score_psi_max=0.15, + auc_delta_max=0.05, + brier_increase_max=0.02, + calibration_gap_delta_max=0.02, + ) + + assert out["shift_type"] == "predictive_degradation" + assert out["governance_posture"] == "candidate_gate" + assert "Operational metrics dominate" in str(out["pvalue_interpretation"]) + + +def test_interpret_model_shift_keeps_stable_cases_on_monitor_posture() -> None: + out = interpret_model_shift( + c2st_auc=0.50, + c2st_materiality="none", + score_psi=0.01, + auc_delta=0.0, + brier_increase=0.0, + calibration_gap_delta=0.0, + distribution_warning_ratio=0.0, + score_psi_max=0.15, + auc_delta_max=0.05, + brier_increase_max=0.02, + calibration_gap_delta_max=0.02, + ) + + assert out["shift_type"] == "stable" + assert out["governance_posture"] == "monitor" diff --git a/tests/test_experiments/test_champion_reopen_orchestration.py b/tests/test_experiments/test_champion_reopen_orchestration.py index 84e1345..5f7dca4 100644 --- a/tests/test_experiments/test_champion_reopen_orchestration.py +++ b/tests/test_experiments/test_champion_reopen_orchestration.py @@ -123,6 +123,7 @@ def test_portfolio_command_separates_proxy_and_exact_sampling(tmp_path) -> None: "solver_backend": "cuopt", "exact_solver_backend": "highs", "frontier_only": True, + "exact_python_executable": "python-exact", }, "frontier": { "proxy_candidates_per_conformal_finalist": 100000, @@ -147,6 +148,11 @@ def test_portfolio_command_separates_proxy_and_exact_sampling(tmp_path) -> None: "gamma_neighbors": "0.45", "policy_modes": "blended_uncertainty", }, + "cuopt": { + "method": "concurrent", + "num_gpus": 1, + "extra_parameters": {"tolerance": "tight"}, + }, }, conformal_intervals_path=tmp_path / "intervals.parquet", run_label="unit", @@ -160,4 +166,8 @@ def test_portfolio_command_separates_proxy_and_exact_sampling(tmp_path) -> None: assert command[command.index("--exact-checkpoint-every") + 1] == "25" assert command[command.index("--exact-threads") + 1] == "8" assert command[command.index("--budget-profiles") + 1] == "free" + assert command[command.index("--exact-python-executable") + 1] == "python-exact" + assert command[command.index("--cuopt-method") + 1] == "concurrent" + assert command[command.index("--cuopt-num-gpus") + 1] == "1" + assert "tolerance=tight" in command assert "--frontier-only" in command diff --git a/tests/test_manifest_regression.py b/tests/test_manifest_regression.py index 807076c..6cbc9be 100644 --- a/tests/test_manifest_regression.py +++ b/tests/test_manifest_regression.py @@ -50,6 +50,15 @@ "models/experiments/champion_reopen/" "champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/" "portfolio/pool93_ijds_consolidated_governance.json", + "models/experiments/champion_reopen/" + "champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/" + "portfolio/pool93_ijds_consolidated_frontier.json", + "models/experiments/champion_reopen/" + "champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/" + "portfolio/pool93_ijds_consolidated_governance.json", + "models/experiments/champion_reopen/" + "champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/" + "portfolio/pool93_point_pd_baseline_audit.json", ) diff --git a/tests/test_models/test_conformal_tuning.py b/tests/test_models/test_conformal_tuning.py index 08f1a78..8ae730e 100644 --- a/tests/test_models/test_conformal_tuning.py +++ b/tests/test_models/test_conformal_tuning.py @@ -385,6 +385,65 @@ def test_reduces_width_while_preserving_constraints(self): assert all(f <= 1.20 for f in shrunk_group_factors.values()) assert shrunk_temporal_factors == {} + def test_temporal_factors_shrink_when_base_intervals_are_feasible(self): + n = 48 + y_pred = np.full(n, 0.50) + y_true = np.full(n, 0.50) + base_intervals = np.column_stack([np.full(n, 0.45), np.full(n, 0.55)]) + groups = np.array(["A"] * n) + temporal_segments = np.array(["early"] * (n // 2) + ["late"] * (n // 2)) + issue_dates = pd.Series(pd.date_range("2024-01-01", periods=n, freq="D")) + + widened, group_factors, temporal_factors, report = shrink_group_multipliers( + y_true=y_true, + y_pred=y_pred, + base_intervals=base_intervals, + groups=groups, + issue_dates=issue_dates, + group_factors=None, + temporal_segments=temporal_segments, + temporal_factors={"early": 1.20, "late": 1.20}, + target_coverage=0.90, + min_group_coverage_target=0.88, + temporal_multiplier_grid=(1.0, 1.20), + ) + + accepted_temporal = report[ + (report["stage"] == "accepted") & (report["factor_scope"] == "temporal") + ] + + assert np.allclose(widened, base_intervals) + assert group_factors == {} + assert temporal_factors == {} + assert set(accepted_temporal["factor_key"]) == {"early", "late"} + + def test_returns_initial_infeasible_report_without_shrinking(self): + n = 20 + y_pred = np.full(n, 0.50) + y_true = np.full(n, 0.95) + base_intervals = np.column_stack([np.full(n, 0.45), np.full(n, 0.55)]) + groups = np.array(["A"] * (n // 2) + ["B"] * (n // 2)) + issue_dates = pd.Series(pd.date_range("2024-01-01", periods=n, freq="D")) + + _widened, group_factors, temporal_factors, report = shrink_group_multipliers( + y_true=y_true, + y_pred=y_pred, + base_intervals=base_intervals, + groups=groups, + issue_dates=issue_dates, + group_factors={"A": 1.20}, + temporal_segments=None, + temporal_factors=None, + target_coverage=0.90, + min_group_coverage_target=0.88, + group_multiplier_grid=(1.0, 1.20), + ) + + assert report["stage"].tolist() == ["initial", "initial_infeasible"] + assert report["accepted"].tolist() == [True, False] + assert group_factors == {"A": 1.20} + assert temporal_factors == {} + # --------------------------------------------------------------------------- # to_python_scalar diff --git a/tests/test_optimization/test_certificate_semantics.py b/tests/test_optimization/test_certificate_semantics.py new file mode 100644 index 0000000..3b17f8e --- /dev/null +++ b/tests/test_optimization/test_certificate_semantics.py @@ -0,0 +1,164 @@ +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest +import yaml + +from src.optimization.certificate_semantics import ( + IJDS_DECLARED_ALPHA_GRID, + IJDS_DECLARED_ALPHA_GRID_CSV, + add_policy_aware_bound_columns, + compute_funded_certificate_metrics, +) + +ROOT = Path(__file__).resolve().parents[2] + + +def test_ijds_alpha_grid_matches_profile_and_claim_registry() -> None: + profile_path = ROOT / "configs" / "profiles" / "search_portfolio_pool93_stage1_claim_26_06.yaml" + profile = yaml.safe_load(profile_path.read_text(encoding="utf-8")) + claims = (ROOT / "docs" / "research" / "active_claims_2026-07-04.md").read_text( + encoding="utf-8" + ) + claim_grid = ", ".join(f"{alpha:.2f}" for alpha in IJDS_DECLARED_ALPHA_GRID) + + assert profile["grids"]["alpha_grid"] == IJDS_DECLARED_ALPHA_GRID_CSV + assert f"`A = {{{claim_grid}}}`" in claims + + +def test_linear_blend_certificate_decomposition() -> None: + weights = np.array([0.25, 0.75]) + point = np.array([0.10, 0.20]) + high = np.array([0.50, 0.60]) + gamma = 0.50 + effective = point + gamma * (high - point) + + metrics = compute_funded_certificate_metrics( + weights, + outcomes=np.array([0.0, 1.0]), + pd_point=point, + pd_high=high, + pd_effective=effective, + alpha=0.01, + risk_tolerance=float(weights @ effective), + ) + + assert metrics.gamma_cp == pytest.approx(0.40) + assert metrics.gamma_internalized == pytest.approx(gamma * metrics.gamma_cp) + assert metrics.gamma_residual == pytest.approx((1.0 - gamma) * metrics.gamma_cp) + assert metrics.endpoint_budget == pytest.approx(float(weights @ high)) + assert metrics.endpoint_budget_upper == pytest.approx(metrics.endpoint_budget) + assert metrics.markov_loss_threshold == pytest.approx(metrics.endpoint_budget + 0.10) + assert metrics.markov_loss_cap == pytest.approx(metrics.markov_loss_threshold) + + +def test_tail_policy_uses_actual_residual_not_linear_blend_shortcut() -> None: + weights = np.array([0.50, 0.50]) + point = np.array([0.10, 0.10]) + high = np.array([0.50, 0.90]) + gamma = 0.50 + effective = np.array([0.10, 0.50]) + + metrics = compute_funded_certificate_metrics( + weights, + outcomes=np.array([0.0, 1.0]), + pd_point=point, + pd_high=high, + pd_effective=effective, + alpha=0.01, + risk_tolerance=float(weights @ effective), + ) + + linear_shortcut = (1.0 - gamma) * metrics.gamma_cp + assert metrics.gamma_cp == pytest.approx(0.60) + assert metrics.gamma_internalized == pytest.approx(0.20) + assert metrics.gamma_residual == pytest.approx(0.40) + assert linear_shortcut == pytest.approx(0.30) + assert metrics.gamma_residual > linear_shortcut + assert metrics.endpoint_budget_upper == pytest.approx(metrics.endpoint_budget) + + +def test_policy_cap_slack_is_included_in_endpoint_upper() -> None: + metrics = compute_funded_certificate_metrics( + weights=np.array([1.0]), + outcomes=np.array([1.0]), + pd_point=np.array([0.10]), + pd_high=np.array([0.60]), + pd_effective=np.array([0.40]), + alpha=0.04, + risk_tolerance=0.35, + pd_cap_slack=0.05, + ) + + assert metrics.endpoint_budget == pytest.approx(0.60) + assert metrics.gamma_residual == pytest.approx(0.20) + assert metrics.endpoint_budget_upper == pytest.approx(0.60) + assert metrics.markov_loss_cap == pytest.approx(0.80) + assert metrics.effective_constraint_excess == pytest.approx(0.0) + + +@pytest.mark.parametrize( + ("weights", "match"), + [ + (np.array([0.25, 0.25]), "sum to one"), + (np.array([1.1, -0.1]), "nonnegative"), + ], +) +def test_certificate_rejects_invalid_weights(weights: np.ndarray, match: str) -> None: + with pytest.raises(ValueError, match=match): + compute_funded_certificate_metrics( + weights=weights, + outcomes=np.array([0.0, 1.0]), + pd_point=np.array([0.10, 0.20]), + pd_high=np.array([0.30, 0.40]), + pd_effective=np.array([0.20, 0.30]), + alpha=0.01, + risk_tolerance=0.30, + ) + + +def test_bound_frame_rehydrates_tail_policy_without_linear_shortcut() -> None: + frame = pd.DataFrame( + { + "alpha": [0.01], + "risk_tolerance": [0.20], + "gamma": [0.50], + "gamma_cp": [0.60], + "weighted_pd_point": [0.10], + "weighted_pd_constraint_used": [0.20], + "weighted_pd_high": [0.60], + "pd_cap_slack": [0.0], + } + ) + + result = add_policy_aware_bound_columns(frame).iloc[0] + + assert result["gamma_internalized"] == pytest.approx(0.10) + assert result["gamma_residual"] == pytest.approx(0.40) + assert result["endpoint_budget"] == pytest.approx(0.60) + assert result["endpoint_budget_upper"] == pytest.approx(0.60) + assert result["markov_loss_threshold"] == pytest.approx(0.70) + assert result["markov_loss_cap"] == pytest.approx(0.70) + assert pytest.approx(0.50) == 0.20 + (1.0 - 0.50) * 0.60 + + +def test_bound_frame_separates_exact_threshold_from_slack_upper() -> None: + frame = pd.DataFrame( + { + "alpha": [0.04], + "tau": [0.50], + "weighted_pd_constraint_used": [0.40], + "weighted_pd_high": [0.60], + } + ) + + result = add_policy_aware_bound_columns(frame).iloc[0] + + assert result["effective_constraint_slack"] == pytest.approx(0.10) + assert result["endpoint_budget"] == pytest.approx(0.60) + assert result["endpoint_budget_upper"] == pytest.approx(0.70) + assert result["markov_loss_threshold"] == pytest.approx(0.80) + assert result["markov_loss_cap"] == pytest.approx(0.90) diff --git a/tests/test_optimization/test_cuopt_adapter.py b/tests/test_optimization/test_cuopt_adapter.py new file mode 100644 index 0000000..70d6d8c --- /dev/null +++ b/tests/test_optimization/test_cuopt_adapter.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +from typing import Any + +import numpy as np +import pandas as pd +import pytest + +import src.optimization.cuopt_adapter as cuopt_adapter + + +class _FakeDataModel: + def set_csr_constraint_matrix( + self, + data: np.ndarray, + indices: np.ndarray, + indptr: np.ndarray, + ) -> None: + self.data = data + self.indices = indices + self.indptr = indptr + + def set_constraint_bounds(self, bounds: np.ndarray) -> None: + self.constraint_bounds = bounds + + def set_row_types(self, row_types: np.ndarray) -> None: + self.row_types = row_types + + def set_objective_coefficients(self, objective: np.ndarray) -> None: + self.objective = objective + + def set_maximize(self, maximize: bool) -> None: + self.maximize = maximize + + def set_variable_lower_bounds(self, bounds: np.ndarray) -> None: + self.variable_lower_bounds = bounds + + def set_variable_upper_bounds(self, bounds: np.ndarray) -> None: + self.variable_upper_bounds = bounds + + +class _FakeSettings: + def __init__(self) -> None: + self.parameters: dict[str, Any] = {} + + def set_parameter(self, name: str, value: Any) -> None: + self.parameters[name] = value + + +class _FakeSolution: + def __init__( + self, + primal: np.ndarray, + *, + objective: float = 12.5, + termination_reason: str = "Optimal", + ) -> None: + self._primal = primal + self._objective = objective + self._termination_reason = termination_reason + + def get_primal_solution(self) -> np.ndarray: + return self._primal + + def get_primal_objective(self) -> float: + return self._objective + + def get_termination_reason(self) -> str: + return self._termination_reason + + +class _FakeLpApi: + def __init__(self, solution: _FakeSolution) -> None: + self.solution = solution + self.data_model: _FakeDataModel | None = None + self.settings: _FakeSettings | None = None + + def DataModel(self) -> _FakeDataModel: + self.data_model = _FakeDataModel() + return self.data_model + + def SolverSettings(self) -> _FakeSettings: + self.settings = _FakeSettings() + return self.settings + + def Solve(self, data_model: _FakeDataModel, settings: _FakeSettings) -> _FakeSolution: + self.solved_data_model = data_model + self.solved_settings = settings + return self.solution + + +def test_cuopt_native_uses_shared_lp_components_and_solver_settings( + monkeypatch: pytest.MonkeyPatch, + tmp_path, +) -> None: + loans = pd.DataFrame( + { + "loan_amnt": [100.0, 200.0, 300.0], + "purpose": ["debt", "home", "home"], + } + ) + fake_api = _FakeLpApi(_FakeSolution(np.array([0.50, 0.25, 0.00, 3.00]))) + monkeypatch.setattr(cuopt_adapter, "_require_cuopt", lambda: fake_api) + + result = cuopt_adapter.solve_portfolio_cuopt_native( + loans=loans, + pd_point=np.array([0.02, 0.04, 0.06]), + pd_high=np.array([0.03, 0.05, 0.08]), + lgd=np.array([0.45, 0.45, 0.45]), + int_rates=np.array([0.12, 0.14, 0.16]), + total_budget=1_000.0, + max_concentration=0.75, + max_portfolio_pd=0.10, + robust=True, + min_budget_utilization=0.20, + pd_cap_slack_penalty=0.50, + time_limit=15, + random_seed=9, + presolve=0, + cuopt_parameters={ + "log_to_console": "true", + "log_dir": str(tmp_path), + "num_cpu_threads": "2", + }, + ) + + assert fake_api.data_model is not None + assert fake_api.settings is not None + assert fake_api.data_model.maximize is True + assert fake_api.data_model.constraint_bounds[0] == pytest.approx(1_000.0) + assert fake_api.data_model.variable_lower_bounds.tolist() == [0.0, 0.0, 0.0, 0.0] + assert fake_api.data_model.variable_upper_bounds.tolist() == [1.0, 1.0, 1.0, 1_000.0] + assert set(fake_api.data_model.row_types.tolist()) == {"L"} + assert fake_api.settings.parameters["time_limit"] == 15 + assert fake_api.settings.parameters["random_seed"] == 9 + assert fake_api.settings.parameters["presolve"] == 0 + assert fake_api.settings.parameters["log_to_console"] is True + assert fake_api.settings.parameters["num_cpu_threads"] == 2 + assert str(tmp_path) in str(fake_api.settings.parameters["log_file"]) + + assert result["solver_backend"] == "cuopt" + assert result["allocation"] == {0: 0.5, 1: 0.25, 2: 0.0} + assert np.allclose(result["allocation_vector"], np.array([0.5, 0.25, 0.0])) + assert result["total_allocated"] == pytest.approx(100.0) + assert result["pd_cap_slack"] == pytest.approx(3.0) + assert result["cuopt_log_file"] == fake_api.settings.parameters["log_file"] + + +def test_cuopt_native_rejects_non_feasible_termination( + monkeypatch: pytest.MonkeyPatch, +) -> None: + loans = pd.DataFrame({"loan_amnt": [100.0], "purpose": ["debt"]}) + fake_api = _FakeLpApi(_FakeSolution(np.array([0.0]), termination_reason="Infeasible")) + monkeypatch.setattr(cuopt_adapter, "_require_cuopt", lambda: fake_api) + + with pytest.raises(RuntimeError, match="did not produce an acceptable solution"): + cuopt_adapter.solve_portfolio_cuopt_native( + loans=loans, + pd_point=np.array([0.02]), + pd_high=np.array([0.03]), + lgd=np.array([0.45]), + int_rates=np.array([0.12]), + ) diff --git a/tests/test_optimization/test_input_alignment.py b/tests/test_optimization/test_input_alignment.py new file mode 100644 index 0000000..5616148 --- /dev/null +++ b/tests/test_optimization/test_input_alignment.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from src.optimization.input_alignment import align_candidate_intervals + + +def test_id_alignment_preserves_left_order_and_interval_payload() -> None: + candidates = pd.DataFrame( + { + "id": ["b", "a", "c"], + "grade": ["B", "A", "C"], + "y_pred": [9.0, 8.0, 7.0], + } + ) + intervals = pd.DataFrame( + { + "id": ["a", "b", "c"], + "grade": ["score_q01", "score_q02", "score_q03"], + "y_pred": [0.1, 0.2, 0.3], + } + ) + + aligned = align_candidate_intervals( + candidates, + intervals, + max_candidates=None, + random_state=42, + ) + + assert aligned.mode == "id" + assert aligned.available_rows == 3 + assert aligned.selected_rows == 3 + assert aligned.candidates["id"].tolist() == ["b", "a", "c"] + assert aligned.candidates["grade"].tolist() == ["B", "A", "C"] + assert aligned.intervals["id"].tolist() == ["b", "a", "c"] + assert aligned.intervals["grade"].tolist() == ["score_q02", "score_q01", "score_q03"] + np.testing.assert_allclose(aligned.intervals["y_pred"], [0.2, 0.1, 0.3]) + + +def test_id_alignment_sampling_is_sorted_and_reproducible() -> None: + candidates = pd.DataFrame({"id": np.arange(20), "row": np.arange(20)}) + intervals = pd.DataFrame( + { + "id": np.arange(19, -1, -1), + "y_pred": np.arange(19, -1, -1) / 100.0, + } + ) + expected_positions = np.sort(np.random.default_rng(17).choice(20, size=6, replace=False)) + + first = align_candidate_intervals( + candidates, + intervals, + max_candidates=6, + random_state=17, + ) + second = align_candidate_intervals( + candidates, + intervals, + max_candidates=6, + random_state=17, + ) + + assert first.candidates["row"].tolist() == expected_positions.tolist() + pd.testing.assert_frame_equal(first.candidates, second.candidates) + pd.testing.assert_frame_equal(first.intervals, second.intervals) + + +def test_row_number_alignment_reorders_interval_rows_without_losing_source_columns() -> None: + candidates = pd.DataFrame({"loan": ["a", "b", "c"], "grade": ["A", "B", "C"]}) + intervals = pd.DataFrame( + { + "_row_number": [2, 0, 1], + "grade": ["q3", "q1", "q2"], + "y_pred": [0.3, 0.1, 0.2], + } + ) + + aligned = align_candidate_intervals( + candidates, + intervals, + max_candidates=0, + random_state=42, + ) + + assert aligned.mode == "row_number" + assert aligned.candidates["loan"].tolist() == ["a", "b", "c"] + assert aligned.intervals["_row_number"].tolist() == [0, 1, 2] + assert aligned.intervals["grade"].tolist() == ["q1", "q2", "q3"] + + +def test_positional_fallback_samples_from_full_alignable_universe() -> None: + candidates = pd.DataFrame({"row": np.arange(10)}) + intervals = pd.DataFrame({"y_pred": np.arange(10) / 100.0}) + expected_positions = np.sort(np.random.default_rng(7).choice(10, size=4, replace=False)) + + aligned = align_candidate_intervals( + candidates, + intervals, + max_candidates=4, + random_state=7, + ) + + assert aligned.mode == "position" + assert aligned.available_rows == 10 + assert aligned.candidates["row"].tolist() == expected_positions.tolist() + np.testing.assert_allclose( + aligned.intervals["y_pred"], + expected_positions / 100.0, + ) + assert max(expected_positions) >= 4 + + +@pytest.mark.parametrize( + ("candidate_ids", "interval_ids", "message"), + [ + (["a", "a"], ["a", "b"], "candidate id alignment key is not unique"), + (["a", "b"], ["a", "a"], "interval id alignment key is not unique"), + (["a", None], ["a", "b"], "candidate id alignment key contains missing"), + (["a", "b"], ["a", None], "interval id alignment key contains missing"), + ], +) +def test_id_alignment_rejects_ambiguous_keys( + candidate_ids: list[str | None], + interval_ids: list[str | None], + message: str, +) -> None: + candidates = pd.DataFrame({"id": candidate_ids}) + intervals = pd.DataFrame({"id": interval_ids, "y_pred": [0.1, 0.2]}) + + with pytest.raises(ValueError, match=message): + align_candidate_intervals( + candidates, + intervals, + max_candidates=None, + random_state=42, + ) + + +def test_id_alignment_rejects_disjoint_universes() -> None: + candidates = pd.DataFrame({"id": ["a", "b"]}) + intervals = pd.DataFrame({"id": ["c", "d"], "y_pred": [0.1, 0.2]}) + + with pytest.raises(ValueError, match="produced zero rows"): + align_candidate_intervals( + candidates, + intervals, + max_candidates=None, + random_state=42, + ) diff --git a/tests/test_optimization/test_policy.py b/tests/test_optimization/test_policy.py index 1212627..63a05eb 100644 --- a/tests/test_optimization/test_policy.py +++ b/tests/test_optimization/test_policy.py @@ -159,6 +159,43 @@ def test_every_policy_produces_pd_in_unit_interval(mode: PolicyMode) -> None: assert np.all((out >= 0.0) & (out <= 1.0)) +def test_segment_tail_blended_uses_segment_cutoffs_when_segments_are_large() -> None: + point = np.full(6, 0.10) + high = point + np.array([0.01, 0.04, 0.07, 0.02, 0.06, 0.10]) + labels = np.array(["A", "A", "A", "B", "B", "B"]) + + out = compute_effective_pd( + point, + high, + policy_mode=PolicyMode.SEGMENT_TAIL_BLENDED_UNCERTAINTY, + gamma=0.5, + tail_focus_quantile=0.5, + segment_labels=labels, + min_segment_size=3, + ) + + expected_delta = np.array([0.00, 0.04, 0.07, 0.00, 0.06, 0.10]) + assert np.allclose(out, point + 0.5 * expected_delta) + + +def test_segment_relative_tail_blended_ranks_by_relative_width() -> None: + point = np.array([0.10, 0.20, 0.10, 0.20]) + high = np.array([0.12, 0.26, 0.15, 0.22]) + labels = np.array(["A", "A", "B", "B"]) + + out = compute_effective_pd( + point, + high, + policy_mode=PolicyMode.SEGMENT_RELATIVE_TAIL_BLENDED_UNCERTAINTY, + gamma=1.0, + tail_focus_quantile=0.5, + segment_labels=labels, + min_segment_size=1, + ) + + assert np.allclose(out, np.array([0.10, 0.26, 0.15, 0.20])) + + def test_legacy_string_aliases_still_work() -> None: """Backward compatibility: passing a legacy string must equal passing the enum.""" rng = np.random.default_rng(seed=7) diff --git a/tests/test_optimization/test_portfolio_model.py b/tests/test_optimization/test_portfolio_model.py index e609f9d..53ed6ce 100644 --- a/tests/test_optimization/test_portfolio_model.py +++ b/tests/test_optimization/test_portfolio_model.py @@ -5,7 +5,10 @@ import pytest import src.optimization.portfolio_model as portfolio_model -from src.optimization.portfolio_model import optimize_portfolio_allocation +from src.optimization.portfolio_model import ( + optimize_portfolio_allocation, + solution_allocation_vector, +) def _toy_loans() -> tuple[pd.DataFrame, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: @@ -28,6 +31,36 @@ def _weighted(values: np.ndarray, allocation: np.ndarray, amounts: np.ndarray) - return float(np.sum(allocation * amounts * values) / exposure) +def test_solution_allocation_vector_normalizes_dense_and_sparse_payloads() -> None: + dense = solution_allocation_vector( + {"allocation_vector": np.array([0.25, 0.0, 1.0])}, + 3, + ) + sparse = solution_allocation_vector({"allocation": {0: 0.25, 2: 1.0}}, 3) + + np.testing.assert_allclose(dense, [0.25, 0.0, 1.0]) + np.testing.assert_allclose(sparse, dense) + + +@pytest.mark.parametrize( + ("solution", "n_items", "error", "match"), + [ + ({"allocation_vector": [0.5, 0.5]}, 3, ValueError, "shape mismatch"), + ({"allocation": {0: float("nan")}}, 1, ValueError, "non-finite"), + ({}, 1, TypeError, "allocation_vector"), + ({"allocation": {}}, -1, ValueError, "nonnegative"), + ], +) +def test_solution_allocation_vector_rejects_invalid_payloads( + solution: dict[str, object], + n_items: int, + error: type[Exception], + match: str, +) -> None: + with pytest.raises(error, match=match): + solution_allocation_vector(solution, n_items) + + def test_highs_sparse_respects_portfolio_constraints() -> None: loans, pd_point, pd_low, pd_high, lgd, int_rates = _toy_loans() result = optimize_portfolio_allocation( @@ -80,6 +113,32 @@ def test_highs_sparse_matches_pyomo_highs_objective_on_toy_lp() -> None: assert sparse["total_allocated"] == pytest.approx(pyomo["total_allocated"], rel=1e-6) +def test_highs_sparse_matches_pyomo_when_pd_slack_is_enabled() -> None: + loans, pd_point, pd_low, pd_high, lgd, int_rates = _toy_loans() + kwargs = { + "loans": loans, + "pd_point": pd_point, + "pd_low": pd_low, + "pd_high": pd_high, + "lgd": lgd, + "int_rates": int_rates, + "total_budget": 3500, + "max_concentration": 0.75, + "max_portfolio_pd": 0.03, + "robust": True, + "min_budget_utilization": 0.80, + "pd_cap_slack_penalty": 0.01, + "pd_constraint_override": pd_high, + } + + sparse = optimize_portfolio_allocation(**kwargs, solver_backend="highs_sparse") + pyomo = optimize_portfolio_allocation(**kwargs, solver_backend="highs_pyomo") + + assert sparse["objective_value"] == pytest.approx(pyomo["objective_value"], rel=1e-6) + assert sparse["total_allocated"] == pytest.approx(pyomo["total_allocated"], rel=1e-6) + assert float(sparse["pd_cap_slack"]) == pytest.approx(float(pyomo["pd_cap_slack"]), rel=1e-6) + + def test_highspy_matches_sparse_highs_objective_on_toy_lp() -> None: pytest.importorskip("highspy") loans, pd_point, pd_low, pd_high, lgd, int_rates = _toy_loans() diff --git a/tests/test_pool93_body_claim_sync.py b/tests/test_pool93_body_claim_sync.py index dcb8d56..39b8bce 100644 --- a/tests/test_pool93_body_claim_sync.py +++ b/tests/test_pool93_body_claim_sync.py @@ -1,7 +1,7 @@ """Drift guard for the promoted pool93 IJDS body claim. The paper body point (A35 "Body/default balanced point") lives in the pool93 -governance sidecars and the A35-A39 tables, all generated outside the DVC DAG +governance sidecars and the A35-A40 tables, all generated outside the DVC DAG by the champion-reopen experiment scripts. The IJDS manuscript embeds those numbers as hand-written prose/Markdown, so a regenerated CSV or a retyped figure can silently desync the submission from its evidence. These tests lock @@ -27,6 +27,7 @@ PAPER = REPO / "paper" / "CRPTO_ijds.qmd" SUPPLEMENT = REPO / "paper" / "supplement_ijds.qmd" +SUBMISSION = REPO / "paper" / "submission" / "CRPTO_ijds_submission.tex" TERMINAL_GOVERNANCE = ( REPO @@ -42,10 +43,11 @@ / "models" / "experiments" / "champion_reopen" - / "champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive" + / "champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2" / "portfolio" / "pool93_ijds_consolidated_governance.json" ) +POINT_BASELINE_AUDIT = CONSOLIDATED_GOVERNANCE.with_name("pool93_point_pd_baseline_audit.json") PROMOTION = REPO / "models" / "final_project_promotion.json" MANIFEST = REPO / "EXTRACTION_MANIFEST.json" @@ -55,6 +57,7 @@ "crpto_tableA37_pool93_body_tail_risk", "crpto_tableA38_pool93_body_cluster_bound_audit", "crpto_tableA39_pool93_body_bootstrap_metrics", + "crpto_tableA40_pool93_point_baseline", ) TERMINAL_RUN_TAG = "champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal" @@ -63,9 +66,11 @@ EXPECTED_BODY = { "return": 184832.475845, "Gamma_CP": 0.162616, + "Gamma_internalized": 0.089032, + "Gamma_residual": 0.073584, "V": 0.03535, - "Markov_cap": 0.34508374, - "endpoint_budget_upper": 0.24508374, + "Markov_threshold": 0.345084, + "endpoint_budget": 0.245084, "risk_tolerance": 0.1715, "gamma": 0.5475, "uncertainty_aversion": 0.05, @@ -105,6 +110,10 @@ def _text(path: Path) -> str: return path.read_text(encoding="utf-8") +def _normalized_manuscript_text(path: Path) -> str: + return _text(path).replace("{,}", ",").replace(r"\$", "$") + + def _body_row_a35() -> dict[str, str]: rows = _read_rows("crpto_tableA35_pool93_ijds_frontier.csv") matches = [r for r in rows if r["role"] == "Body/default balanced point"] @@ -118,11 +127,13 @@ def test_pool93_claim_governance_matches_expected_body_point() -> None: body = consolidated["selected_candidates"]["paper_body"] assert body["return"] == pytest.approx(EXPECTED_BODY["return"], abs=1e-6) assert body["Gamma_CP"] == pytest.approx(EXPECTED_BODY["Gamma_CP"], abs=1e-9) - assert body["V"] == pytest.approx(EXPECTED_BODY["V"], abs=1e-9) - assert body["Markov_cap"] == pytest.approx(EXPECTED_BODY["Markov_cap"], abs=1e-9) - assert body["endpoint_budget_upper"] == pytest.approx( - EXPECTED_BODY["endpoint_budget_upper"], abs=1e-9 + assert body["Gamma_internalized"] == pytest.approx( + EXPECTED_BODY["Gamma_internalized"], abs=1e-9 ) + assert body["Gamma_residual"] == pytest.approx(EXPECTED_BODY["Gamma_residual"], abs=1e-9) + assert body["V"] == pytest.approx(EXPECTED_BODY["V"], abs=1e-9) + assert body["Markov_threshold"] == pytest.approx(EXPECTED_BODY["Markov_threshold"], abs=1e-9) + assert body["endpoint_budget"] == pytest.approx(EXPECTED_BODY["endpoint_budget"], abs=1e-9) assert body["risk_tolerance"] == EXPECTED_BODY["risk_tolerance"] assert body["gamma"] == EXPECTED_BODY["gamma"] assert body["uncertainty_aversion"] == EXPECTED_BODY["uncertainty_aversion"] @@ -138,7 +149,10 @@ def test_pool93_claim_governance_matches_expected_body_point() -> None: assert float(row["realized_return"]) == pytest.approx(body["return"], abs=1e-6) assert float(row["Gamma_CP_alpha01"]) == pytest.approx(body["Gamma_CP"], abs=1e-9) assert float(row["V_alpha01"]) == pytest.approx(body["V"], abs=1e-9) - assert float(row["Markov_cap_alpha01"]) == pytest.approx(body["Markov_cap"], abs=1e-9) + assert float(row["Gamma_residual_alpha01"]) == pytest.approx(body["Gamma_residual"], abs=1e-9) + assert float(row["Markov_threshold_alpha01"]) == pytest.approx( + body["Markov_threshold"], abs=1e-9 + ) assert row["alpha_grid_pass"] == body["alpha_pass"] @@ -156,7 +170,7 @@ def test_pool93_consolidated_governance_frontier_counts() -> None: def test_pool93_tables_exist() -> None: - """A35-A39 evidence tables exist in both CSV and TEX form.""" + """A35-A40 evidence tables exist in both CSV and TEX form.""" missing = [ f"{stem}.{ext}" for stem in POOL93_TABLE_STEMS @@ -169,13 +183,14 @@ def test_pool93_tables_exist() -> None: def test_pool93_paper_anchors_match_csvs() -> None: """Body-claim numbers printed in the paper surfaces derive from A35/A39.""" row = _body_row_a35() - budget = float(row["endpoint_budget_upper_alpha01"]) + budget = float(row["endpoint_budget_alpha01"]) deterministic_bound = budget + float(row["V_alpha01"]) paper_anchors = [ f"${float(row['realized_return']):,.2f}", f"{float(row['V_alpha01']):.6f}", f"{float(row['Gamma_CP_alpha01']):.6f}", - f"{float(row['Markov_cap_alpha01']):.6f}", + f"{float(row['Gamma_residual_alpha01']):.6f}", + f"{float(row['Markov_threshold_alpha01']):.6f}", f"{budget:.6f}", f"{deterministic_bound:.6f}", ] @@ -195,9 +210,37 @@ def test_pool93_paper_anchors_match_csvs() -> None: missing.extend( f"{a} missing in {SUPPLEMENT.name}" for a in supplement_anchors if a not in supplement_text ) + submission_text = _normalized_manuscript_text(SUBMISSION) + missing.extend( + f"{a} missing in {SUBMISSION.name}" for a in paper_anchors if a not in submission_text + ) assert not missing, "pool93 body-claim drift:\n" + "\n".join(missing) +def test_pool93_matched_point_baseline_agrees_across_surfaces() -> None: + """A40 audit, table, body, supplement, and submission share one contrast.""" + audit = _load_json(POINT_BASELINE_AUDIT) + table = {row["policy"]: row for row in _read_rows("crpto_tableA40_pool93_point_baseline.csv")} + point = audit["point_pd_baseline"] + selected = audit["selected_crpto"] + contrasts = audit["contrasts"] + + assert float(table["Point-PD two-stage LP"]["realized_return"]) == pytest.approx( + point["realized_return"], abs=1e-6 + ) + assert float(table["Selected CRPTO"]["realized_return"]) == pytest.approx( + selected["realized_return"], abs=1e-6 + ) + assert contrasts["realized_return_cost_pct"] == pytest.approx(5.8749883793) + assert contrasts["weighted_default_rate_reduction"] == pytest.approx(0.08305) + assert contrasts["markov_threshold_reduction"] == pytest.approx(0.4354954304) + + anchors = ("196,369.14", "5.875", "8.305", "43.55") + for surface in (PAPER, SUPPLEMENT, SUBMISSION): + text = _normalized_manuscript_text(surface) + assert all(anchor in text for anchor in anchors), surface + + def test_pool93_two_tag_scheme_is_coherent() -> None: """The frozen rebaseline chain stays the declared return floor for pool93.""" promotion = _load_json(PROMOTION) @@ -224,7 +267,12 @@ def test_pool93_manifest_block_agrees() -> None: assert point["realized_total_return"] == pytest.approx(body["return"], abs=1e-6) assert point["alpha01_gamma_cp"] == pytest.approx(body["Gamma_CP"], abs=1e-9) assert point["alpha01_weighted_miscoverage_V"] == pytest.approx(body["V"], abs=1e-9) - assert point["markov_cap_alpha01"] == pytest.approx(body["Markov_cap"], abs=1e-9) + assert point["alpha01_gamma_internalized"] == pytest.approx( + body["Gamma_internalized"], abs=1e-9 + ) + assert point["alpha01_gamma_residual"] == pytest.approx(body["Gamma_residual"], abs=1e-9) + assert point["endpoint_budget_alpha01"] == pytest.approx(body["endpoint_budget"], abs=1e-9) + assert point["markov_threshold_alpha01"] == pytest.approx(body["Markov_threshold"], abs=1e-9) assert point["declared_return_floor"] == EXPECTED_FLOOR assert ( block["frontier_counts"]["deduped_semantic_policies"] diff --git a/tests/test_publication_integrity.py b/tests/test_publication_integrity.py new file mode 100644 index 0000000..c736c1f --- /dev/null +++ b/tests/test_publication_integrity.py @@ -0,0 +1,7 @@ +from __future__ import annotations + +from scripts.check_publication_integrity import check_publication_integrity + + +def test_active_ijds_publication_surfaces_are_claim_synchronized() -> None: + assert check_publication_integrity() == [] diff --git a/tests/test_publication_targets.py b/tests/test_publication_targets.py index 4f29ac4..7f886ba 100644 --- a/tests/test_publication_targets.py +++ b/tests/test_publication_targets.py @@ -46,11 +46,13 @@ def test_journal_strengthening_pack_classifies_current_and_backlog_items() -> No backlog = pack["backlog_not_blocking"] assert "no longer a blanket exclusion" in boundary - assert "future work" in boundary + assert "outside the submitted claim" in boundary + assert "not acceptance criteria" in boundary assert set(included) == { "regret_auditability_frontier", "tail_risk_oce_cvar_diagnostic", "pool93_frontier_and_selected_allocation", + "matched_point_pd_baseline", "robust_satisficing_margins", "dependence_aware_bound", "tail_satisficing_challenger_audit", @@ -63,6 +65,7 @@ def test_journal_strengthening_pack_classifies_current_and_backlog_items() -> No assert included["pool93_frontier_and_selected_allocation"]["status"] == ( "include_body_and_supplement" ) + assert included["matched_point_pd_baseline"]["status"] == ("include_body_and_supplement") assert included["robust_satisficing_margins"]["status"] == ("include_supplement_or_short_body") assert included["dependence_aware_bound"]["status"] == "include_theory_appendix_or_caveat" assert included["tail_satisficing_challenger_audit"]["status"] == "include_supplement" @@ -76,6 +79,7 @@ def test_journal_strengthening_pack_classifies_current_and_backlog_items() -> No assert "reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.csv" in ( pool93_artifacts ) + assert "reports/crpto/tables/crpto_tableA40_pool93_point_baseline.csv" in (pool93_artifacts) for artifact in pool93_artifacts: assert Path(artifact).exists(), artifact multidataset_artifacts = included["multidataset_external_replication"]["artifacts"] diff --git a/tests/test_scripts/test_benchmark_conformal_variants.py b/tests/test_scripts/test_benchmark_conformal_variants.py index d662c25..b3f55e2 100644 --- a/tests/test_scripts/test_benchmark_conformal_variants.py +++ b/tests/test_scripts/test_benchmark_conformal_variants.py @@ -2,7 +2,7 @@ from __future__ import annotations -from scripts.benchmark_conformal_variants import _build_output_paths +from scripts.benchmark_conformal_variants import _build_output_paths, _normalize_search_space def test_build_output_paths_uses_namespaced_shadow_locations() -> None: @@ -15,3 +15,24 @@ def test_build_output_paths_uses_namespaced_shadow_locations() -> None: .as_posix() .endswith("models/conformal_gap/abc_def/conformal_variant_selection_status.json") ) + + +def test_normalize_search_space_dedupes_and_applies_defaults() -> None: + space = _normalize_search_space( + calibration_size_fractions=(0.25, 1.5, 0.5), + partition_candidates=(" grade ", "grade", ""), + partition_probability_sources=("RAW", "raw"), + n_score_bins_candidates=(0, 10), + fallback_modes=("", "Grade_Then_Global", "grade_then_global"), + score_scale_families=("",), + min_group_sizes=None, + min_group_size_default=500, + ) + + assert space.partition_candidates == ("grade",) + assert space.partition_probability_sources == ("raw",) + assert space.n_score_bins_candidates == (10,) + assert space.fallback_modes == ("grade_then_global",) + assert space.score_scale_families == ("none",) + assert space.min_group_sizes == (500,) + assert space.calibration_size_fractions == (0.25, 0.5) diff --git a/tests/test_scripts/test_benchmark_pd_set_prediction.py b/tests/test_scripts/test_benchmark_pd_set_prediction.py new file mode 100644 index 0000000..2188404 --- /dev/null +++ b/tests/test_scripts/test_benchmark_pd_set_prediction.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import numpy as np + +from scripts.benchmark_pd_set_prediction import ( + _build_output_paths, + _promotion_gate, + _set_benchmark_settings, +) + + +def test_build_output_paths_uses_namespaced_shadow_locations() -> None: + paths = _build_output_paths("set/audit") + + assert ( + paths["cases"] + .as_posix() + .endswith("data/processed/conformal_gap/set_audit/pd_set_prediction_cases.parquet") + ) + assert ( + paths["status"] + .as_posix() + .endswith("models/conformal_gap/set_audit/pd_set_prediction_status.json") + ) + + +def test_set_benchmark_settings_normalizes_inputs_and_fallback() -> None: + settings = _set_benchmark_settings( + alpha=0.1, + method="lac", + methods=("lac", "aps", "lac"), + partitions=("global", "global", "grade"), + partition_probability_source="RAW", + n_score_bins=10, + min_group_size=500, + fallback_mode="score_only", + calibration_size_fractions=(0.25, 1.5, 0.50), + prob_cal_lookup={"raw": np.array([0.1]), "calibrated": np.array([0.2])}, + ) + + assert settings.methods == ("lac", "aps") + assert settings.partitions == ("global", "grade") + assert settings.partition_probability_source == "raw" + assert settings.effective_fallback_mode == "global_only" + assert settings.calibration_size_fractions == (0.25, 0.50) + + +def test_promotion_gate_requires_coverage_grade_a_and_breadth() -> None: + gate = _promotion_gate( + {"set_coverage": 0.90}, + [ + {"slice_value": "A", "singleton_rate": 0.81}, + {"slice_value": "B", "singleton_rate": 0.41}, + {"slice_value": "C", "singleton_rate": 0.42}, + {"slice_value": "D", "singleton_rate": 0.43}, + ], + ) + + assert gate["pass"] is True + assert gate["grade_a_singleton_rate"] == 0.81 + assert gate["grades_with_singleton_above_40pct"] == 4 diff --git a/tests/test_scripts/test_build_crpto_journal_package.py b/tests/test_scripts/test_build_crpto_journal_package.py index 7cc4bab..88a919d 100644 --- a/tests/test_scripts/test_build_crpto_journal_package.py +++ b/tests/test_scripts/test_build_crpto_journal_package.py @@ -2,6 +2,7 @@ import subprocess import sys +import time from collections.abc import Iterator from contextlib import contextmanager from pathlib import Path @@ -9,6 +10,25 @@ import pytest +def _restore_payload(path: Path, payload: bytes | None) -> None: + for attempt in range(5): + try: + if payload is None: + path.unlink(missing_ok=True) + return + + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(f".{path.name}.restore.{attempt}.tmp") + tmp.write_bytes(payload) + tmp.replace(path) + return + except OSError: + tmp.unlink(missing_ok=True) if "tmp" in locals() else None + if attempt == 4: + raise + time.sleep(0.2 * (attempt + 1)) + + @contextmanager def _preserve_files(paths: list[Path]) -> Iterator[None]: snapshots = {path: path.read_bytes() if path.exists() else None for path in paths} @@ -16,11 +36,7 @@ def _preserve_files(paths: list[Path]) -> Iterator[None]: yield finally: for path, payload in snapshots.items(): - if payload is None: - path.unlink(missing_ok=True) - else: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(payload) + _restore_payload(path, payload) def test_build_crpto_journal_package_script_runs() -> None: diff --git a/tests/test_scripts/test_build_papers_tesis_deep_audit.py b/tests/test_scripts/test_build_papers_tesis_deep_audit.py new file mode 100644 index 0000000..ace49b5 --- /dev/null +++ b/tests/test_scripts/test_build_papers_tesis_deep_audit.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +from pathlib import Path + +from scripts.build_papers_tesis_deep_audit import write_audit + + +def _row( + *, + relative_path: str, + decision: str, + action_required: str = "none_now", + bib_status: str = "existing", +) -> dict[str, object]: + return { + "folder": relative_path.split("/", 1)[0], + "relative_path": relative_path, + "title": f"Title {relative_path}", + "status": "published", + "primary_domain": "conformal risk control", + "bib_key": "paperkey", + "bib_status": bib_status, + "core_concepts": "coverage; decision risk", + "key_claims": "claim summary", + "conclusions": "use with finite-grid boundary", + "figures_tables_useful": "inspiration only", + "limitations": "not CRPTO evidence", + "decision": decision, + "action_required": action_required, + "crpto_value": "supports CRPTO framing", + "extended_lab_value": "future-work value", + "evidence_gate": "do not reopen champion", + "artifact_sink": "docs/research/example.md", + "stop_rule": "keep as literature unless claim changes", + "implementation_or_experiment": "none", + } + + +def test_write_audit_builds_editorial_sections(tmp_path: Path) -> None: + rows = [ + _row(relative_path="paper/promote.pdf", decision="promote_crpto_body"), + _row(relative_path="paper/append.pdf", decision="append_crpto_related_work"), + _row( + relative_path="supplement/experiment.pdf", + decision="append_tail_risk", + action_required="experiment_completed_appendix_diagnostic", + ), + _row( + relative_path="tesis/future.pdf", + decision="park_future_work", + bib_status="needs_bib_if_cited", + ), + ] + curated_visual_rows = [ + { + "relative_path": "paper/promote.pdf", + "caption_type": "figure", + "caption_index": 1, + "editorial_sink": "own schematic", + "why_useful": "layout inspiration", + "claim_boundary": "do not reproduce", + } + ] + audit_path = tmp_path / "audit.md" + + write_audit( + audit_path, + rows, + tmp_path / "matrix.csv", + tmp_path / "captions.csv", + tmp_path / "visuals.csv", + curated_visual_rows, + ) + + text = audit_path.read_text(encoding="utf-8") + + assert "# Papers_tesis Deep Audit" in text + assert "## Lectura integrada para Paper CRPTO" in text + assert "paper/promote.pdf" in text + assert "paper/append.pdf" in text + assert "supplement/experiment.pdf" in text + assert "tesis/future.pdf" in text + assert "| action_required | n |" in text + assert "experiment_completed_appendix_diagnostic" in text + assert "needs_bib_if_cited" in text + assert "no cambia el champion CRPTO" in text diff --git a/tests/test_scripts/test_build_pool93_ijds_consolidated_frontier.py b/tests/test_scripts/test_build_pool93_ijds_consolidated_frontier.py new file mode 100644 index 0000000..1e870c7 --- /dev/null +++ b/tests/test_scripts/test_build_pool93_ijds_consolidated_frontier.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +from pathlib import Path + +import pandas as pd +import pytest + +from scripts.search import build_pool93_ijds_consolidated_frontier as frontier + + +def test_load_leaderboards_rehydrates_policy_aware_tail_bound( + tmp_path: Path, + monkeypatch, +) -> None: + leaderboard_path = tmp_path / "leaderboard.parquet" + bound_path = tmp_path / "bound.parquet" + pd.DataFrame( + { + "local_candidate_id": [7], + "semantic_policy_key": ["tail-policy"], + "alpha01_endpoint_budget_upper": [0.50], + "alpha01_markov_loss_cap": [0.60], + } + ).to_parquet(leaderboard_path, index=False) + pd.DataFrame( + { + "local_candidate_id": [7], + "semantic_policy_key": ["tail-policy"], + "alpha": [0.01], + "risk_tolerance": [0.20], + "gamma_cp": [0.60], + "weighted_pd_point": [0.10], + "weighted_pd_constraint_used": [0.20], + "weighted_pd_high": [0.60], + "pd_cap_slack": [0.0], + } + ).to_parquet(bound_path, index=False) + monkeypatch.setattr(frontier, "_leaderboard_path", lambda _tag: leaderboard_path) + monkeypatch.setattr(frontier, "_bound_eval_path", lambda _tag: bound_path) + + row = frontier._load_leaderboards(["unit-tag"]).iloc[0] + + assert row["alpha01_gamma_internalized"] == pytest.approx(0.10) + assert row["alpha01_gamma_residual"] == pytest.approx(0.40) + assert row["alpha01_endpoint_budget"] == pytest.approx(0.60) + assert row["alpha01_endpoint_budget_upper"] == pytest.approx(0.60) + assert row["alpha01_markov_loss_threshold"] == pytest.approx(0.70) + assert row["alpha01_markov_loss_cap"] == pytest.approx(0.70) + + +def test_body_selection_uses_exact_markov_threshold() -> None: + eligible = pd.DataFrame( + { + "semantic_policy_key": ["tail", "linear"], + "alpha01_realized_total_return": [220_000.0, 190_000.0], + "alpha01_markov_loss_threshold": [0.70, 0.34], + } + ) + + selected = frontier._body_candidate(eligible, markov_threshold=0.35) + + assert selected["semantic_policy_key"] == "linear" + + +def test_threshold_frontier_does_not_mislabel_tail_policy_as_under_half() -> None: + eligible = pd.DataFrame( + { + "semantic_policy_key": ["tail", "linear"], + "alpha01_realized_total_return": [220_000.0, 190_000.0], + "alpha01_markov_loss_threshold": [0.70, 0.45], + } + ) + + selected = frontier._best_under_threshold(eligible, 0.50) + + assert selected is not None + assert selected["semantic_policy_key"] == "linear" diff --git a/tests/test_scripts/test_build_pool93_point_baseline_audit.py b/tests/test_scripts/test_build_pool93_point_baseline_audit.py new file mode 100644 index 0000000..715c911 --- /dev/null +++ b/tests/test_scripts/test_build_pool93_point_baseline_audit.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import pandas as pd +import pytest + +from scripts.search.build_pool93_point_baseline_audit import ( + _comparison_table, + _format_comparison_tex, +) + + +def test_comparison_table_reports_return_cost_and_certificate_metrics() -> None: + point = { + "realized_return": 200.0, + "expected_return_net_point": 210.0, + "certificate": { + "n_funded": 10, + "weighted_outcome": 0.12, + "weighted_miscoverage": 0.11, + "gamma_cp": 0.50, + "endpoint_budget": 0.60, + "markov_loss_threshold": 0.70, + }, + } + selected = { + "realized_return": 180.0, + "expected_return_net_point": 170.0, + "certificate": { + "n_funded": 14, + "weighted_outcome": 0.04, + "weighted_miscoverage": 0.03, + "gamma_cp": 0.16, + "endpoint_budget": 0.25, + "markov_loss_threshold": 0.35, + }, + } + + table = _comparison_table(point, selected).set_index("policy") + + assert isinstance(table, pd.DataFrame) + assert table.loc["Point-PD two-stage LP", "return_cost_vs_point_pct"] == pytest.approx(0.0) + assert table.loc["Selected CRPTO", "return_cost_vs_point_pct"] == pytest.approx(10.0) + assert table.loc["Selected CRPTO", "Markov_threshold_alpha01"] == pytest.approx(0.35) + assert table.loc["Selected CRPTO", "weighted_default_rate"] == pytest.approx(0.04) + + tex = _format_comparison_tex(table.reset_index()) + assert "Policy & Realized return & Weighted default" in tex + assert "Selected CRPTO & \\$180.00 & 0.040000 & 0.160000 & 0.250000 & 0.350000" in tex + assert "expected\\_return\\_net\\_point" not in tex diff --git a/tests/test_scripts/test_compile_ijds_submission.py b/tests/test_scripts/test_compile_ijds_submission.py new file mode 100644 index 0000000..a096ab9 --- /dev/null +++ b/tests/test_scripts/test_compile_ijds_submission.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +from scripts.compile_ijds_submission import LatexScan + + +def test_latex_scan_ok_property_flags_clean_build() -> None: + scan = LatexScan(pages=27, blg_warnings=(), log_failures=()) + + assert scan.ok + + +def test_latex_scan_ok_property_rejects_warnings_or_log_failures() -> None: + assert not LatexScan(pages=27, blg_warnings=("Warning--empty journal",), log_failures=()).ok + assert not LatexScan(pages=27, blg_warnings=(), log_failures=("undefined references",)).ok diff --git a/tests/test_scripts/test_export_pool93_policy_aware_frontier.py b/tests/test_scripts/test_export_pool93_policy_aware_frontier.py new file mode 100644 index 0000000..c548491 --- /dev/null +++ b/tests/test_scripts/test_export_pool93_policy_aware_frontier.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from scripts.search.export_pool93_policy_aware_frontier import ROLE_ORDER, build_table + + +def test_build_table_orders_roles_and_uses_exact_threshold() -> None: + rows = [] + for index, role in enumerate(reversed(ROLE_ORDER), start=1): + rows.append( + { + "role": role, + "run_label": "unit", + "local_candidate_id": index, + "family": "family", + "risk_tolerance": 0.17, + "policy_mode": "blended_uncertainty", + "gamma": 0.5, + "uncertainty_aversion": 0.1, + "return": 180_000.0 + index, + "return_floor_surplus": 9_000.0, + "Gamma_CP": 0.2, + "Gamma_residual": 0.1, + "V": 0.03, + "endpoint_budget": 0.24, + "endpoint_budget_upper": 0.24, + "Markov_threshold": 0.34, + "Markov_cap": 0.34, + "alpha_pass": "8/8", + "n_funded_mean": 300.0, + } + ) + + table = build_table({"rows": rows}) + + assert len(table) == len(ROLE_ORDER) + assert table.iloc[0]["role"] == "Minimum Markov-threshold endpoint" + assert table.iloc[-1]["role"] == "Max-return economic endpoint" + assert table["Markov_threshold_alpha01"].eq(0.34).all() + assert "Gamma_residual_alpha01" in table.columns diff --git a/tests/test_scripts/test_generate_conformal_intervals_cli.py b/tests/test_scripts/test_generate_conformal_intervals_cli.py index 572aa5d..b58f4dc 100644 --- a/tests/test_scripts/test_generate_conformal_intervals_cli.py +++ b/tests/test_scripts/test_generate_conformal_intervals_cli.py @@ -4,13 +4,19 @@ import scripts.generate_conformal_intervals as conformal_script from scripts.generate_conformal_intervals import ( + _apply_global_rebalance, + _apply_learned_floor_policy, + _build_conformal_artifact_tables, _build_tuning_split, + _can_use_temporal_segments, _parse_bool_tuple, _parse_float_tuple, _parse_int_tuple, _parse_str_tuple, _resolve_tuning_grid, + _select_alpha_95, _select_best_tuning_config, + _tuning_total_candidates, ) @@ -77,6 +83,27 @@ def test_resolve_tuning_grid_uses_current_defaults_for_empty_inputs() -> None: assert grid.scaled_scores_options == () +def test_tuning_total_candidates_counts_cartesian_grid() -> None: + grid = _resolve_tuning_grid( + partition="grade", + partition_candidates=("grade", "score_bin"), + partition_probability_sources=("raw",), + n_score_bins_candidates=(5, 10), + fallback_modes=("global",), + score_scale_families=("none",), + scaled_scores_options=(True, False), + ) + + assert ( + _tuning_total_candidates( + grid, + alpha_candidates_90=(0.10, 0.09), + min_group_sizes=(200, 500), + ) + == 32 + ) + + def test_build_tuning_split_materializes_fit_and_holdout( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -125,6 +152,145 @@ def fake_split_calibration_for_tuning(**kwargs): assert split.issue_test.isna().tolist() == [False, True] +def test_apply_global_rebalance_disabled_preserves_current_metrics() -> None: + y_int = conformal_script.np.array([[0.1, 0.4], [0.2, 0.7]]) + metrics = {"empirical_coverage": 0.9} + group_metrics = conformal_script.pd.DataFrame({"group": ["A"], "coverage": [0.9]}) + + result = _apply_global_rebalance( + enabled=False, + min_factor=0.75, + max_factor=1.05, + step=0.01, + y_int_tune_working=y_int, + y_pred_tune=conformal_script.np.array([0.25, 0.45]), + y_tune=conformal_script.pd.Series([0.0, 1.0]), + y_int_90=y_int, + y_pred_90=conformal_script.np.array([0.25, 0.45]), + y_eval_90=conformal_script.pd.Series([0.0, 1.0]), + group_tune=conformal_script.pd.Series(["A", "B"]), + eval_groups_90=conformal_script.pd.Series(["A", "B"]), + alpha_target_90=0.10, + target_coverage_90=0.90, + min_group_coverage_target=0.88, + metrics_90=metrics, + group_metrics_90=group_metrics, + ) + + assert result.y_intervals is y_int + assert result.metrics is metrics + assert result.group_metrics is group_metrics + assert result.factor == 1.0 + assert result.diagnostics == {"enabled": False, "applied": False} + + +def test_can_use_temporal_segments_requires_enabled_dates_and_matching_lengths() -> None: + issue_dates = conformal_script.pd.Series(["2020-01-01", "2020-02-01"]) + groups = conformal_script.pd.Series(["A", "B"]) + + assert _can_use_temporal_segments( + enabled=True, + issue_tune=issue_dates, + eval_issue=issue_dates, + group_tune=groups, + eval_groups=groups, + ) + assert not _can_use_temporal_segments( + enabled=False, + issue_tune=issue_dates, + eval_issue=issue_dates, + group_tune=groups, + eval_groups=groups, + ) + assert not _can_use_temporal_segments( + enabled=True, + issue_tune=conformal_script.pd.Series([None, None]), + eval_issue=issue_dates, + group_tune=groups, + eval_groups=groups, + ) + assert not _can_use_temporal_segments( + enabled=True, + issue_tune=issue_dates, + eval_issue=issue_dates, + group_tune=groups, + eval_groups=conformal_script.pd.Series(["A"]), + ) + + +def test_apply_learned_floor_policy_applies_group_temporal_and_global_factors() -> None: + adjusted = _apply_learned_floor_policy( + y_pred=conformal_script.np.array([0.5, 0.5]), + y_intervals=conformal_script.np.array([[0.4, 0.6], [0.4, 0.6]]), + groups=conformal_script.pd.Series(["A", "B"]), + group_multipliers={"A": 2.0}, + temporal_segments=conformal_script.pd.Series(["A|2020Q1", "B|2020Q1"]), + temporal_segment_multipliers={"B|2020Q1": 3.0}, + global_rebalance_factor=0.5, + ) + + conformal_script.np.testing.assert_allclose( + adjusted, + conformal_script.np.array([[0.4, 0.6], [0.35, 0.65]]), + ) + + +def test_build_conformal_artifact_tables_preserves_holdout_metadata() -> None: + tables = _build_conformal_artifact_tables( + y_eval_90=conformal_script.pd.Series([0.0, 1.0]), + y_pred_90=conformal_script.np.array([0.2, 0.8]), + y_int_90=conformal_script.np.array([[0.1, 0.4], [0.6, 0.9]]), + y_int_95=conformal_script.np.array([[0.0, 0.5], [0.5, 1.0]]), + eval_groups_90=conformal_script.pd.Series(["A", "B"]), + eval_temporal_segments=conformal_script.pd.Series(["A|2020Q1", "B|2020Q1"]), + evaluation_scope_key="holdout", + test_df=conformal_script.pd.DataFrame({"id": ["test-1"], "loan_amnt": [9000.0]}), + cal_df=conformal_script.pd.DataFrame( + { + "id": ["cal-0", "cal-1", "cal-2"], + "loan_amnt": [1000.0, 2000.0, 3000.0], + } + ), + idx_cal_tune=conformal_script.np.array([1, 2]), + group_metrics_90=conformal_script.pd.DataFrame( + { + "group": ["A", "B"], + "coverage": [0.9, 1.0], + "avg_width": [0.3, 0.3], + "median_width": [0.3, 0.3], + } + ), + group_metrics_95=conformal_script.pd.DataFrame( + { + "group": ["A", "B"], + "coverage": [0.95, 1.0], + "avg_width": [0.5, 0.5], + "median_width": [0.5, 0.5], + } + ), + coverage_floor_report=conformal_script.pd.DataFrame( + { + "group": ["A", "B"], + "coverage_before": [0.9, 1.0], + "coverage_after": [0.92, 1.0], + "multiplier": [1.02, 1.0], + "adjusted": [True, False], + } + ), + width_attr_rows=[{"dataset_scope": "holdout", "stage": "base_interval"}], + ) + + assert tables.intervals["_row_number"].tolist() == [0, 1] + assert tables.intervals["id"].tolist() == ["cal-1", "cal-2"] + assert tables.intervals["loan_amnt"].tolist() == [2000.0, 3000.0] + assert tables.intervals["temporal_segment"].tolist() == ["A|2020Q1", "B|2020Q1"] + assert tables.group_metrics["coverage_95"].tolist() == [0.95, 1.0] + assert tables.group_metrics["adjusted"].tolist() == [True, False] + assert tables.width_attribution.to_dict(orient="records") == [ + {"dataset_scope": "holdout", "stage": "base_interval"} + ] + + def test_build_tuning_split_rejects_empty_holdout(monkeypatch: pytest.MonkeyPatch) -> None: def fake_split_calibration_for_tuning(**kwargs): return conformal_script.np.array([0, 1]), conformal_script.np.array([], dtype=int) @@ -148,6 +314,48 @@ def fake_split_calibration_for_tuning(**kwargs): ) +def test_select_alpha_95_uses_holdout_gap_then_width(monkeypatch: pytest.MonkeyPatch) -> None: + def fake_create_intervals(**kwargs): + alpha = float(kwargs["alpha"]) + return ( + conformal_script.np.array([0.2, 0.3]), + conformal_script.np.array([[0.0, alpha], [0.1, alpha + 0.1]]), + {}, + ) + + def fake_validate_coverage(*args, **kwargs): + alpha = float(kwargs["alpha"]) + return { + "coverage_gap": {0.04: 0.03, 0.05: 0.01, 0.06: 0.01}[alpha], + "avg_interval_width": {0.04: 0.20, 0.05: 0.18, 0.06: 0.16}[alpha], + } + + monkeypatch.setattr( + conformal_script, + "create_pd_intervals_mondrian_from_predictions", + fake_create_intervals, + ) + monkeypatch.setattr(conformal_script, "validate_coverage", fake_validate_coverage) + + selected_alpha = _select_alpha_95( + alpha_95=0.05, + alpha_candidates_95=(0.04, 0.05, 0.06), + interval_fit_pred=conformal_script.np.array([0.1, 0.2]), + interval_tune_pred=conformal_script.np.array([0.2, 0.3]), + y_cal_fit=conformal_script.pd.Series([0.0, 1.0]), + y_tune=conformal_script.pd.Series([0.0, 1.0]), + group_cal_fit_holdout=conformal_script.pd.Series(["A", "B"]), + group_tune=conformal_script.pd.Series(["A", "B"]), + best_cfg={ + "min_group_size": 200, + "scaled_scores": False, + "score_scale_family": "none", + }, + ) + + assert selected_alpha == 0.06 + + def test_select_best_tuning_config_materializes_promoted_config() -> None: rows = [ { diff --git a/tests/test_scripts/test_generate_crpto_figures.py b/tests/test_scripts/test_generate_crpto_figures.py new file mode 100644 index 0000000..9dc5c5b --- /dev/null +++ b/tests/test_scripts/test_generate_crpto_figures.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import pandas as pd + +from scripts.generate_crpto_figures import ( + PALETTE, + _alpha_annotation_offset, + _alpha_pareto_column_map, + _alpha_pareto_missing_columns, + _alpha_pareto_subframe, + _alpha_pareto_variant_styles, + _alpha_tick_labels, +) + + +def test_alpha_pareto_column_map_detects_semantic_columns() -> None: + df = pd.DataFrame( + columns=[ + "method_name", + "alpha_level", + "empirical_coverage", + "mean_width", + "n_eligible_loans", + ] + ) + + columns = _alpha_pareto_column_map(df) + + assert columns == { + "variant": "method_name", + "alpha": "alpha_level", + "coverage": "empirical_coverage", + "width": "mean_width", + "eligible": "n_eligible_loans", + } + assert _alpha_pareto_missing_columns(columns) == [] + + +def test_alpha_pareto_variant_styles_label_mondrian_and_global() -> None: + colors, labels = _alpha_pareto_variant_styles(["mondrian", "global"]) + + assert labels == { + "mondrian": "Mondrian CP", + "global": "Global Split-CP", + } + assert colors == { + "mondrian": PALETTE["blue"], + "global": PALETTE["orange"], + } + + +def test_alpha_pareto_subframe_sorts_alpha_and_formats_labels() -> None: + df = pd.DataFrame( + { + "variant": ["global", "mondrian", "global"], + "alpha": [0.2, 0.1, 0.05], + "coverage": [0.93, 0.91, 0.90], + } + ) + + sub = _alpha_pareto_subframe( + df, + variant_col="variant", + alpha_col="alpha", + variant="global", + ) + + assert list(sub["alpha"]) == [0.05, 0.2] + assert _alpha_tick_labels(sub["alpha"]) == ["0.05", "0.2"] + assert _alpha_annotation_offset(0, 2) == (4, 4) + assert _alpha_annotation_offset(1, 2) == (-24, -8) diff --git a/tests/test_scripts/test_generate_governance_status.py b/tests/test_scripts/test_generate_governance_status.py new file mode 100644 index 0000000..1ea0a13 --- /dev/null +++ b/tests/test_scripts/test_generate_governance_status.py @@ -0,0 +1,166 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np +import pandas as pd + +from scripts.generate_governance_status import ( + GovernanceOutputPaths, + GovernanceThresholds, + _build_explanation_drift_report, + _build_governance_status, + _drift_breach_metrics, +) + + +def _test_thresholds() -> GovernanceThresholds: + return GovernanceThresholds( + psi_threshold=0.25, + ks_pvalue_min=0.01, + cvm_pvalue_min=0.01, + c2st_auc_max=0.60, + max_feature_breach_ratio=0.60, + c2st_max_rows=50_000, + score_psi_max=0.15, + auc_delta_max=0.05, + brier_increase_max=0.02, + calibration_gap_delta_max=0.02, + performance_max_rows=100_000, + min_rank_overlap_top10=0.60, + max_explanation_shap_psi=0.25, + min_reason_code_stability=0.55, + explanation_min_rows_per_slice=80, + psi_bins=10, + random_state=42, + ) + + +def _test_paths() -> GovernanceOutputPaths: + return GovernanceOutputPaths( + drift_path=Path("data/processed/drift_monitoring.parquet"), + status_path=Path("models/governance_status.json"), + explanation_drift_path=Path("data/processed/explanation_drift.parquet"), + fairness_status_path=Path("models/fairness_audit_status.json"), + fairness_frontier_path=Path("data/processed/fairness_threshold_frontier.parquet"), + challenger_report_path=Path("models/challenger_promotion_report.json"), + model_shift_status_path=Path("models/model_shift_status.json"), + ) + + +def test_build_explanation_drift_report_emits_overall_and_grade_rows() -> None: + rows: list[dict[str, object]] = [] + periods = ["2020Q1", "2020Q2", "2020Q3"] + for period in periods: + for grade in ["A", "B"]: + for idx in range(20): + rows.append( + { + "issue_quarter": period, + "grade": grade, + "pd_calibrated": 0.20, + "shap_dti": 0.30 + 0.001 * idx, + "shap_income": 0.10 + 0.001 * idx, + } + ) + shap_raw = pd.DataFrame(rows) + + report = _build_explanation_drift_report( + shap_raw, + primary_threshold=0.50, + min_rank_overlap_top10=0.50, + max_shap_psi=10.0, + min_reason_code_stability=0.50, + min_rows_per_slice=20, + ) + + assert set(report["segment_type"]) == {"overall", "grade"} + assert set(report["segment"]) == {"all", "A", "B"} + assert set(report["comparison_period"]) == {"2020Q3"} + assert report["passed_all"].all() + assert np.isfinite(report["max_shap_psi_top5"]).all() + + details = json.loads(str(report.loc[report["segment"] == "all", "feature_psi_details"].iloc[0])) + assert {row["feature"] for row in details} == {"dti", "income"} + + +def test_governance_status_helpers_preserve_public_contract() -> None: + thresholds = _test_thresholds() + drift_df = pd.DataFrame( + { + "pass_psi": [True, False], + "pass_ks": [True, True], + "pass_cvm": [True, False], + "psi": [0.05, 0.20], + "ks_pvalue": [0.50, 0.40], + "cvm_pvalue": [0.60, 0.30], + "feature": ["a", "b"], + } + ) + c2st = { + "c2st_auc": 0.55, + "materiality": "moderate", + "effective_driver_count": 1, + "top_drivers": ["a"], + "n_rows": 100, + } + performance = { + "score_psi": 0.10, + "auc_delta_train_to_test": 0.02, + "brier_increase_train_to_test": 0.01, + "calibration_gap_delta": 0.01, + } + metrics = _drift_breach_metrics(drift_df, c2st, performance, thresholds) + explanation_drift = pd.DataFrame( + { + "passed_all": [True], + "pass_reason_code_stability": [True], + "rank_overlap_top10": [0.80], + "max_shap_psi_top5": [0.10], + "reason_code_match_rate": [0.90], + } + ) + status = _build_governance_status( + config_path="configs/mrm_policy.yaml", + resolved_run_tag="test-run", + paths=_test_paths(), + thresholds=thresholds, + drift_df=drift_df, + explanation_drift=explanation_drift, + fairness_status={"overall_pass": True, "primary_threshold": 0.42}, + challenger_report={"challenger_promotable": True}, + metrics=metrics, + model_shift={"shift_type": "stable", "governance_posture": "monitor"}, + ) + + assert metrics["psi_breaches"] == 1 + assert metrics["pass_predictive_drift"] is True + assert status["overall_pass"] is True + assert status["checks"]["pass_explainability"] is True + assert status["summary"]["fairness_primary_threshold"] == 0.42 + assert status["summary"]["challenger_promotable"] is True + assert Path(status["artifacts"]["model_shift_status_path"]) == Path( + "models/model_shift_status.json" + ) + + +def test_build_explanation_drift_report_requires_enough_recent_rows() -> None: + shap_raw = pd.DataFrame( + { + "issue_quarter": ["2020Q1", "2020Q2"], + "pd_calibrated": [0.20, 0.20], + "shap_dti": [0.1, 0.2], + } + ) + + report = _build_explanation_drift_report( + shap_raw, + primary_threshold=0.50, + min_rank_overlap_top10=0.50, + max_shap_psi=10.0, + min_reason_code_stability=0.50, + min_rows_per_slice=20, + ) + + assert report.empty diff --git a/tests/test_scripts/test_optimize_portfolio_tradeoff.py b/tests/test_scripts/test_optimize_portfolio_tradeoff.py index bfcd56b..3ed4526 100644 --- a/tests/test_scripts/test_optimize_portfolio_tradeoff.py +++ b/tests/test_scripts/test_optimize_portfolio_tradeoff.py @@ -2,14 +2,111 @@ import numpy as np import pandas as pd +import pytest +import scripts.optimize_portfolio_tradeoff as tradeoff_module +from scripts.optimize_portfolio import _align_candidates_and_intervals from scripts.optimize_portfolio_tradeoff import ( + _align_loans_and_intervals, _build_policy_grid, _prepare_tradeoff_inputs, _select_champion_policy, + _solve_single, ) +def test_portfolio_alignment_wrappers_share_strict_id_contract() -> None: + candidates = pd.DataFrame( + { + "id": ["b", "a", "c"], + "grade": ["B", "A", "C"], + "y_pred": [9.0, 8.0, 7.0], + } + ) + intervals = pd.DataFrame( + { + "id": ["a", "b", "c"], + "grade": ["score_q01", "score_q02", "score_q03"], + "y_pred": [0.1, 0.2, 0.3], + "pd_low_90": [0.05, 0.15, 0.25], + "pd_high_90": [0.15, 0.25, 0.35], + } + ) + + tradeoff_loans, tradeoff_intervals = _align_loans_and_intervals( + candidates, + intervals, + max_candidates=0, + random_state=42, + ) + optimizer_loans, pd_point, pd_low, pd_high = _align_candidates_and_intervals( + candidates, + intervals, + max_candidates=0, + random_state=42, + ) + + assert tradeoff_loans["id"].tolist() == ["b", "a", "c"] + assert tradeoff_intervals["grade"].tolist() == [ + "score_q02", + "score_q01", + "score_q03", + ] + pd.testing.assert_frame_equal(tradeoff_loans, optimizer_loans) + np.testing.assert_allclose(pd_point, [0.2, 0.1, 0.3]) + np.testing.assert_allclose(pd_low, [0.15, 0.05, 0.25]) + np.testing.assert_allclose(pd_high, [0.25, 0.15, 0.35]) + + +def test_nonrobust_solve_uses_point_pd_contract(monkeypatch: pytest.MonkeyPatch) -> None: + captured: dict[str, object] = {} + + def _fake_optimize_portfolio_allocation(**kwargs: object) -> dict[str, object]: + captured.update(kwargs) + return { + "allocation": {1: 0.5}, + "objective_value": 10.0, + "n_funded": 1, + "pd_cap_slack": 0.0, + "solver_status": "optimal", + } + + monkeypatch.setattr( + tradeoff_module, + "optimize_portfolio_allocation", + _fake_optimize_portfolio_allocation, + ) + loans = pd.DataFrame({"loan_amnt": [1000.0, 2000.0]}) + pd_point = np.array([0.10, 0.20]) + pd_high = np.array([0.40, 0.50]) + + result, allocation = _solve_single( + loans=loans, + pd_point=pd_point, + pd_low=np.array([0.05, 0.10]), + pd_high=pd_high, + lgd=np.array([0.45, 0.45]), + int_rates=np.array([0.10, 0.12]), + default_flag=np.array([0, 0]), + total_budget=1000.0, + max_concentration=1.0, + risk_tolerance=0.25, + robust=False, + uncertainty_aversion=0.0, + min_budget_utilization=0.0, + pd_cap_slack_penalty=0.0, + time_limit=10, + threads=1, + policy_mode="hard_worst_case", + gamma=1.0, + ) + + np.testing.assert_allclose(captured["pd_constraint_override"], pd_point) + np.testing.assert_allclose(allocation, [0.0, 0.5]) + assert result["policy_mode"] == "point_estimate" + assert result["gamma"] == 0.0 + + def test_build_policy_grid_preserves_tradeoff_frontier_contract() -> None: grid = _build_policy_grid() diff --git a/tests/test_scripts/test_pool93_local_refinement_grid.py b/tests/test_scripts/test_pool93_local_refinement_grid.py new file mode 100644 index 0000000..55c4087 --- /dev/null +++ b/tests/test_scripts/test_pool93_local_refinement_grid.py @@ -0,0 +1,267 @@ +from __future__ import annotations + +import hashlib + +import pandas as pd +import pytest + +from scripts.search.run_pool93_ijds_local_refinement import ( + DEFAULT_ALPHA_GRID, + _build_parser, + _claim_summary, + _generate_candidate_grid, + _manifest_payload, + _pending_refinement_tasks, + _resolve_paths, +) +from src.optimization.certificate_semantics import IJDS_DECLARED_ALPHA_GRID + +EXPECTED_PROFILE_FINGERPRINTS = { + "stage1": (1236, "4f4fa9791ad71b3901f0af5aa55ff616426700e3245e6d5fd4cd5e923086f6ad"), + "expanded": (7463, "a0bb03c5ee6491b2f9e1032e50c12c8aae6937a84206a989889acbb4212d371a"), + "claim_expanded": ( + 3659, + "8345545b20e93985462a84e92bf503417911386ab1aaf0194a657aab64d8d329", + ), + "claim_micro": (2949, "1e4083c8b8e200c5689566a06da1e18591a11b219b13ffea5c65d575dfb796bc"), + "claim_micro_ext": ( + 4407, + "3cc8a45b2ca0fde9a2f12cd6229b94cc7bf0119e3eeb7e742d6ce2f7bf601d08", + ), + "claim_bound_closure": ( + 1653, + "8d8ef58d92049809406049ff22b81831a9607c0203a2c401fb026826f2a9acee", + ), + "claim_bound_floor_closure": ( + 2343, + "9cb18594d1ec323cab23ffcdb4c96e481b2fb0798a18c4df73b796737e6b72c5", + ), + "claim_bound_terminal": ( + 37068, + "6d75ef0b7c083f9f60dfc834a50cb5da10873223dffe340665ef130e6c4c88ac", + ), +} + + +def test_pool93_default_alpha_grid_uses_shared_certificate_semantics() -> None: + assert list(IJDS_DECLARED_ALPHA_GRID) == DEFAULT_ALPHA_GRID + + +def _synthetic_anchor_rows() -> pd.DataFrame: + base_fields = { + "tail_focus_quantile": 1.0, + "min_budget_utilization": 0.0, + "pd_cap_slack_penalty": 0.0, + } + return pd.DataFrame( + [ + { + **base_fields, + "candidate_rank": 96, + "risk_tolerance": 0.156, + "policy_mode": "tail_blended_uncertainty", + "gamma": 0.46, + "uncertainty_aversion": 0.125, + "delta_cap_quantile": 1.0, + }, + { + **base_fields, + "candidate_rank": 219, + "risk_tolerance": 0.171, + "policy_mode": "blended_uncertainty", + "gamma": 0.45, + "uncertainty_aversion": 0.1, + "delta_cap_quantile": 1.0, + }, + { + **base_fields, + "candidate_rank": 223, + "risk_tolerance": 0.173, + "policy_mode": "capped_blended_uncertainty", + "gamma": 0.40, + "uncertainty_aversion": 0.1, + "delta_cap_quantile": 0.95, + }, + ] + ) + + +def _semantic_fingerprint(frame: pd.DataFrame) -> str: + blob = "\n".join(frame["semantic_policy_key"].astype(str)).encode() + return hashlib.sha256(blob).hexdigest() + + +@pytest.mark.parametrize("profile", EXPECTED_PROFILE_FINGERPRINTS) +def test_pool93_local_refinement_grid_is_stable_by_profile(profile: str) -> None: + expected_rows, expected_sha = EXPECTED_PROFILE_FINGERPRINTS[profile] + + candidates = _generate_candidate_grid( + _synthetic_anchor_rows(), + profile=profile, + solver_backend="highspy", + ) + + assert len(candidates) == expected_rows + assert _semantic_fingerprint(candidates) == expected_sha + assert candidates["local_candidate_id"].tolist() == list(range(1, expected_rows + 1)) + assert candidates["semantic_policy_key"].is_unique + + +def test_pool93_local_refinement_rejects_unknown_profile() -> None: + with pytest.raises(ValueError, match="profile must be one of"): + _generate_candidate_grid( + _synthetic_anchor_rows(), + profile="broad_new_search", + solver_backend="highspy", + ) + + +def test_pool93_manifest_paths_and_pending_tasks_are_coherent(tmp_path) -> None: + args = _build_parser().parse_args( + [ + "--run-tag", + "unit/run", + "--output-dir", + str(tmp_path / "out"), + "--model-dir", + str(tmp_path / "model"), + "--source-bound-eval", + str(tmp_path / "source.parquet"), + "--source-selection", + str(tmp_path / "selection.json"), + ] + ) + paths = _resolve_paths(args, run_tag="unit_run") + candidates = pd.DataFrame({"local_candidate_id": [1, 2]}) + + pending = _pending_refinement_tasks( + candidates=candidates, + alpha_grid=[0.01, 0.03], + completed_keys={(1, 0.01)}, + ) + manifest = _manifest_payload( + args=args, + paths=paths, + run_tag="unit_run", + source_bound_eval=tmp_path / "source.parquet", + source_selection=tmp_path / "selection.json", + conformal_intervals_path="data/processed/conformal.parquet", + anchor_ranks=[96, 219, 223], + alpha_grid=[0.01, 0.03], + ) + + assert [(row["local_candidate_id"], alpha) for row, alpha in pending] == [ + (1, 0.03), + (2, 0.01), + (2, 0.03), + ] + assert manifest["candidates_path"] == str(paths.candidates_path) + assert manifest["claim_summary_path"] == str(paths.claim_summary_path) + assert manifest["run_tag"] == "unit_run" + + +def test_claim_summary_exposes_finite_grid_and_balanced_claim() -> None: + leaderboard = pd.DataFrame( + [ + { + "claim_rank": 1, + "local_candidate_id": 1, + "local_family": "endpoint", + "anchor_rank": 96, + "source_reason": "max_return", + "risk_tolerance": 0.18, + "policy_mode": "tail_blended_uncertainty", + "gamma": 0.40, + "delta_cap_quantile": 1.0, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.05, + "alpha01_realized_total_return": 190000.0, + "alpha01_gamma_cp": 0.20, + "alpha01_weighted_miscoverage_V": 0.05, + "alpha01_weighted_pd_true": 0.11, + "alpha01_empirical_coverage_funded": 0.92, + "alpha01_exact_pass": True, + "all_alpha_pass": True, + "alpha_exact_pass_count": 2, + "alpha_exact_check_count": 2, + "alpha_mean_gamma_cp": 0.18, + "alpha_mean_weighted_miscoverage_V": 0.04, + "n_funded_mean": 50, + "allocator_backends": "highspy", + }, + { + "claim_rank": 2, + "local_candidate_id": 2, + "local_family": "body", + "anchor_rank": 219, + "source_reason": "balanced", + "risk_tolerance": 0.17, + "policy_mode": "capped_blended_uncertainty", + "gamma": 0.55, + "delta_cap_quantile": 0.975, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.05, + "alpha01_realized_total_return": 185000.0, + "alpha01_gamma_cp": 0.10, + "alpha01_weighted_miscoverage_V": 0.03, + "alpha01_weighted_pd_true": 0.10, + "alpha01_empirical_coverage_funded": 0.94, + "alpha01_exact_pass": True, + "all_alpha_pass": True, + "alpha_exact_pass_count": 2, + "alpha_exact_check_count": 2, + "alpha_mean_gamma_cp": 0.11, + "alpha_mean_weighted_miscoverage_V": 0.03, + "n_funded_mean": 48, + "allocator_backends": "highspy", + }, + { + "claim_rank": 3, + "local_candidate_id": 3, + "local_family": "failed", + "anchor_rank": 223, + "source_reason": "not_all_alpha", + "risk_tolerance": 0.16, + "policy_mode": "blended_uncertainty", + "gamma": 0.60, + "delta_cap_quantile": 1.0, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.10, + "alpha01_realized_total_return": 200000.0, + "alpha01_gamma_cp": 0.09, + "alpha01_weighted_miscoverage_V": 0.02, + "alpha01_weighted_pd_true": 0.09, + "alpha01_empirical_coverage_funded": 0.95, + "alpha01_exact_pass": True, + "all_alpha_pass": False, + "alpha_exact_pass_count": 1, + "alpha_exact_check_count": 2, + "alpha_mean_gamma_cp": 0.10, + "alpha_mean_weighted_miscoverage_V": 0.02, + "n_funded_mean": 45, + "allocator_backends": "highspy", + }, + ] + ) + bound_eval = pd.DataFrame( + { + "alpha": [0.01, 0.01, 0.03], + "all_bounds_hold": [True, False, True], + "violation": [0.0, 0.02, 0.0], + "gamma_cp": [0.1, 0.2, 0.15], + "weighted_miscoverage_V": [0.03, 0.05, 0.04], + } + ) + + summary = _claim_summary(leaderboard, bound_eval, alpha_grid=[0.03, 0.01, 0.01]) + + assert summary["finite_grid_policy"]["alpha_grid"] == [0.01, 0.03] + assert summary["n_policies"] == 3 + assert summary["n_all_alpha_passers"] == 2 + assert summary["n_all_alpha_passers_above_return_floor"] == 2 + assert summary["max_return_claim"]["local_candidate_id"] == 1 + assert summary["best_gamma_cp_return_floor_claim"]["local_candidate_id"] == 2 + assert summary["best_weighted_miscoverage_return_floor_claim"]["local_candidate_id"] == 2 + assert summary["balanced_return_bound_claim"]["local_candidate_id"] == 2 + assert summary["by_family"]["failed"]["n_all_alpha_passers"] == 0 + assert summary["by_alpha"]["0.01"]["n_checks"] == 2 diff --git a/tests/test_scripts/test_retired_search_entrypoints.py b/tests/test_scripts/test_retired_search_entrypoints.py new file mode 100644 index 0000000..3a2b2b9 --- /dev/null +++ b/tests/test_scripts/test_retired_search_entrypoints.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from scripts.search import run_conformal_search, run_portfolio_search + + +def test_retired_conformal_search_entrypoint_is_actionable(capsys) -> None: + assert run_conformal_search.main([]) == 2 + + captured = capsys.readouterr() + assert "retired" in captured.err + assert "run_conformal_reopen_search.py" in captured.err + + +def test_retired_portfolio_search_entrypoint_is_actionable(capsys) -> None: + assert run_portfolio_search.main([]) == 2 + + captured = capsys.readouterr() + assert "retired" in captured.err + assert "run_pool93_ijds_local_refinement.py" in captured.err diff --git a/tests/test_scripts/test_run_comparison.py b/tests/test_scripts/test_run_comparison.py new file mode 100644 index 0000000..30c945b --- /dev/null +++ b/tests/test_scripts/test_run_comparison.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +from scripts.run_comparison import _collect_status_metadata, _gate_ab_no_regression + + +def _ab_status(control: float, robust: float, **extra: object) -> dict[str, object]: + return { + "metrics_a": {"total_return": control}, + "metrics_b": {"total_return": robust}, + **extra, + } + + +def _status(run_tag: str) -> dict[str, str]: + return { + "schema_version": "test.1", + "generated_at_utc": "2026-07-08T20:00:00+00:00", + "run_tag": run_tag, + } + + +def test_collect_status_metadata_passes_when_core_statuses_match() -> None: + cur = { + "dvc_metrics_meta": _status("run-a"), + "pipeline_summary": _status("run-a"), + "conformal_status": _status("run-a"), + "fairness_status": _status("run-a"), + "governance_status": _status("run-a"), + "ab_simulation_status": _status("run-a"), + } + + metadata = _collect_status_metadata(cur, expected_run_tag="run-a") + + assert metadata["passed"] is True + assert metadata["all_have_metadata"] is True + assert metadata["run_tags_observed"] == ["run-a"] + assert metadata["mismatched_run_tag_artifacts"] == [] + assert len(metadata["critical_artifacts"]) == 6 + + +def test_collect_status_metadata_allows_causal_insights_only_mismatch() -> None: + cur = { + "dvc_metrics_meta": _status("run-a"), + "pipeline_summary": _status("run-a"), + "conformal_status": _status("run-a"), + "fairness_status": _status("run-a"), + "governance_status": _status("run-a"), + "ab_simulation_status": _status("run-a"), + "causal_effect_status": _status("older-causal-run"), + } + + metadata = _collect_status_metadata(cur, expected_run_tag="run-a") + + assert metadata["passed"] is True + assert metadata["run_tag_matches_expected"] is False + assert metadata["run_tag_matches_expected_operational"] is True + assert metadata["causal_only_mismatch"] is True + assert metadata["non_causal_mismatched_run_tag_artifacts"] == [] + + +def test_gate_ab_no_regression_passes_when_current_self_gate_passes() -> None: + base = {"ab_simulation_status": _ab_status(100.0, 120.0)} + cur = { + "ab_simulation_status": _ab_status( + 101.0, + 119.0, + no_regression={ + "passed": True, + "diff_total_return": 18.0, + "tolerance_total_return": 5.0, + }, + comparison={"p_value": 0.12, "significant": False}, + n_candidates_used=50, + ) + } + + gate = _gate_ab_no_regression(base, cur) + + assert gate.name == "ab_no_regression" + assert gate.passed is True + assert gate.details["checks"]["self_no_regression_ok"] is True + assert gate.details["warnings"]["control_vs_baseline_warning"] is False + assert gate.details["diagnostics"]["gate_mode"] == "no_regression" + + +def test_gate_ab_no_regression_allows_selective_ambiguity_cross_gate() -> None: + base = {"ab_simulation_status": _ab_status(100.0, 120.0)} + cur = { + "ab_simulation_status": _ab_status( + 100.0, + 80.0, + decision_scenario="selective_ambiguity_defer", + no_regression={"passed": False, "diff_total_return": -20.0}, + cross_scenario_gate={"passed": True}, + ) + } + + gate = _gate_ab_no_regression(base, cur) + + assert gate.passed is True + assert gate.details["checks"]["self_no_regression_ok"] is True + assert gate.details["checks"]["cross_scenario_gate_ok"] is True + assert gate.details["warnings"]["robust_vs_baseline_warning"] is True diff --git a/tests/test_scripts/test_run_conformal_reopen_search.py b/tests/test_scripts/test_run_conformal_reopen_search.py index d684f32..d1348b3 100644 --- a/tests/test_scripts/test_run_conformal_reopen_search.py +++ b/tests/test_scripts/test_run_conformal_reopen_search.py @@ -1,8 +1,23 @@ from __future__ import annotations +import json +from pathlib import Path +from typing import Any + import pandas as pd -from scripts.search.run_conformal_reopen_search import _phase2_top_designs +from scripts.search import run_conformal_reopen_search as reopen +from scripts.search.run_conformal_reopen_search import ( + Phase1ConfirmationResult, + _maybe_apply_phase2, + _phase1_from_resume, + _phase2_metric_blocked, + _phase2_run_reason, + _phase2_top_designs, + _rank_phase2_candidates, + _run_phase1_oot_confirmation, + _run_phase2_search, +) def _design(*, partition: str, alpha: float, width: float, rank: int) -> dict[str, object]: @@ -61,3 +76,322 @@ def test_phase2_top_designs_falls_back_to_inner_when_oot_empty() -> None: assert source == "phase1_inner_aggregate" assert top.iloc[0]["partition"] == "score_decile_mondrian" assert set(top["phase2_design_source"]) == {"phase1_inner_aggregate"} + + +def _passing_policy(*, width: float = 0.42) -> dict[str, object]: + return { + "overall_pass": True, + "strict_overall_pass": True, + "methodological_justification_pass": True, + "coverage_90": 0.90, + "avg_width_90": width, + "min_group_coverage_90": 0.89, + "warning_alerts": 0, + "total_alerts": 0, + } + + +def test_phase1_resume_result_preserves_source_paths_and_winner( + tmp_path: Path, + monkeypatch: object, +) -> None: + aggregate = pd.DataFrame( + [ + _design(partition="grade", alpha=0.10, width=0.40, rank=1), + _design(partition="score_decile_mondrian", alpha=0.09, width=0.41, rank=2), + ] + ) + source_paths = { + "inner_aggregate": tmp_path / "source_aggregate.parquet", + "inner_search": tmp_path / "source_inner.parquet", + } + aggregate.to_parquet(source_paths["inner_aggregate"], index=False) + + shortlist = aggregate.head(1).copy() + + def fake_paths(_run_tag: str) -> dict[str, Path]: + return source_paths + + def fake_shortlist( + *, source_run_tag: str, top_k_inner: int + ) -> tuple[pd.DataFrame, dict[str, Any]]: + assert source_run_tag == "source-run" + assert top_k_inner == 1 + return shortlist, {"source": "source-run"} + + monkeypatch.setattr(reopen, "_reopen_artifact_paths", fake_paths) + monkeypatch.setattr(reopen, "_build_resume_shortlist", fake_shortlist) + + result = _phase1_from_resume( + resume_from_run_tag="source-run", + top_k_inner=1, + output_paths={"phase1_shortlist": tmp_path / "shortlist.parquet"}, + ) + + assert result.aggregate_path == str(source_paths["inner_aggregate"]) + assert result.inner_search_path == str(source_paths["inner_search"]) + assert result.inner_search_winner["partition"] == "grade" + assert result.resume_meta == {"source": "source-run"} + assert (tmp_path / "shortlist.parquet").exists() + + +def test_phase1_oot_confirmation_writes_ranked_candidate_frame( + tmp_path: Path, + monkeypatch: object, +) -> None: + shortlist = pd.DataFrame( + [ + _design(partition="grade", alpha=0.10, width=0.50, rank=1), + _design(partition="score_decile_mondrian", alpha=0.09, width=0.42, rank=2), + ] + ) + + def fake_run_candidate(**kwargs: Any) -> dict[str, object]: + rank = int(kwargs["rank"]) + return { + "namespace": f"candidate-{rank}", + "policy_status": _passing_policy(width=0.50 if rank == 1 else 0.42), + "set_status": {"summary": {"set_coverage": 0.91, "singleton_rate": 0.73}}, + "selection_status": {"promotion_pass": rank == 2, "selected_variant": "variant"}, + } + + monkeypatch.setattr(reopen, "_run_phase1_oot_candidate", fake_run_candidate) + + result = _run_phase1_oot_confirmation( + run_tag="test-run", + env={}, + shortlist=shortlist, + output_paths={"phase1_final_candidates": tmp_path / "phase1_final.parquet"}, + alpha_candidates_95=[0.05], + partition_candidates=["grade"], + partition_probability_sources=["calibrated"], + n_score_bins_candidates=[10], + fallback_modes=["grade_then_global"], + score_scale_families=["none"], + calibration_fractions=[1.0], + sidecar_cfg={}, + validation_cfg={}, + ) + + assert result.best_namespace == "candidate-2" + assert result.final_namespace == "candidate-2" + assert result.final_decision == "promotable_for_followup" + assert result.frame.iloc[0]["avg_width_90"] == 0.42 + assert (tmp_path / "phase1_final.parquet").exists() + + +def test_maybe_apply_phase2_respects_phase1_only() -> None: + phase1 = Phase1ConfirmationResult( + candidates=[], + frame=pd.DataFrame([{"namespace": "phase1"}]), + best_namespace="phase1", + final_policy=_passing_policy(), + final_sets={"summary": {"set_coverage": 0.91}}, + final_decision="promotable_for_followup", + final_namespace="phase1", + ) + + result = _maybe_apply_phase2( + run_tag="run", + upstream_run_tag="upstream", + env={}, + aggregated=pd.DataFrame(), + phase1=phase1, + phase1_only=True, + force_phase2=True, + alpha_candidates_95=[], + tuning_holdout_ratios=[], + inner_random_states=[], + partition_candidates=[], + partition_probability_sources=[], + n_score_bins_candidates=[], + fallback_modes=[], + score_scale_families=[], + calibration_fractions=[], + phase2_cfg={"enabled": True}, + sidecar_cfg={}, + validation_cfg={}, + ) + + assert result.final_namespace == "phase1" + assert result.final_decision == "promotable_for_followup" + assert result.phase2_summary is None + assert _phase2_run_reason(force_phase2=True, phase2_always_evaluate=True) == "forced" + + +def test_phase2_metric_gate_and_ranking_are_explicit() -> None: + assert _phase2_metric_blocked( + calibration_metrics={"ece": 0.12, "brier_score": 0.21}, + baseline_metrics={"ece": 0.10, "brier_score": 0.20}, + max_metric_degradation={"ece": 0.01, "brier_score": 0.02}, + ) + assert not _phase2_metric_blocked( + calibration_metrics={"ece": 0.105}, + baseline_metrics={"ece": 0.10}, + max_metric_degradation={"ece": 0.01}, + ) + + ranked = _rank_phase2_candidates( + pd.DataFrame( + [ + { + "artifact_namespace": "wide", + "calibrator_method": "platt", + "holdout_coverage": 0.91, + "holdout_width": 0.30, + "calibrator_ece": 0.01, + "calibrator_adaptive_ece": 0.01, + "calibrator_brier": 0.10, + "calibrator_phi_brier": 0.20, + "selection_rank": 1, + }, + { + "artifact_namespace": "centered", + "calibrator_method": "isotonic", + "holdout_coverage": 0.90, + "holdout_width": 0.50, + "calibrator_ece": 0.02, + "calibrator_adaptive_ece": 0.02, + "calibrator_brier": 0.10, + "calibrator_phi_brier": 0.10, + "selection_rank": 2, + }, + ] + ) + ) + + assert ranked.iloc[0]["artifact_namespace"] == "centered" + + +def _phase2_paths(tmp_path: Path) -> dict[str, Path]: + data_dir = tmp_path / "data" + models_dir = tmp_path / "models" + data_dir.mkdir() + models_dir.mkdir() + return { + "data_dir": data_dir, + "models_dir": models_dir, + "phase2_search": data_dir / "phase2.parquet", + "phase2_progress": models_dir / "phase2_progress.json", + } + + +def test_run_phase2_search_records_metric_gate_skip( + tmp_path: Path, + monkeypatch: Any, +) -> None: + paths = _phase2_paths(tmp_path) + aggregate = pd.DataFrame([_design(partition="grade", alpha=0.10, width=0.40, rank=1)]) + + def fake_fit_calibrator( + *, method: str, output_path: Path, upstream_run_tag: str + ) -> tuple[str, dict[str, float]]: + del output_path, upstream_run_tag + if method == "venn_abers": + return method, {"ece": 0.10} + return method, {"ece": 0.20} + + monkeypatch.setattr(reopen, "_reopen_artifact_paths", lambda _run_tag: paths) + monkeypatch.setattr(reopen, "_fit_calibrator", fake_fit_calibrator) + + decision, policy, sets, summary = _run_phase2_search( + run_tag="phase2-skip", + upstream_run_tag="upstream", + env={}, + aggregated=aggregate, + phase1_candidates_frame=pd.DataFrame(), + alpha_candidates_95=[0.05], + tuning_holdout_ratios=[0.2], + inner_random_states=[42], + partition_candidates=["grade"], + partition_probability_sources=["calibrated"], + n_score_bins_candidates=[10], + fallback_modes=["grade_then_global"], + score_scale_families=["none"], + calibration_fractions=[1.0], + phase2_cfg={"calibrators": ["platt"], "max_metric_degradation": {"ece": 0.01}}, + sidecar_cfg={}, + validation_cfg={}, + ) + + progress = json.loads(paths["phase2_progress"].read_text(encoding="utf-8")) + assert decision == "policy_review_candidate" + assert policy == {} + assert sets == {} + assert summary is not None + assert summary["status"] == "no_noninferior_calibrator_candidate" + assert progress["skipped"][0]["reason"] == "metric_degradation_gate" + assert paths["phase2_search"].exists() + + +def test_run_phase2_search_ranks_and_confirms_best_candidate( + tmp_path: Path, + monkeypatch: Any, +) -> None: + paths = _phase2_paths(tmp_path) + aggregate = pd.DataFrame( + [ + _design(partition="grade", alpha=0.10, width=0.50, rank=1), + _design(partition="score_decile_mondrian", alpha=0.09, width=0.42, rank=2), + ] + ) + + def fake_fit_calibrator( + *, method: str, output_path: Path, upstream_run_tag: str + ) -> tuple[str, dict[str, float]]: + del output_path, upstream_run_tag + return method, {"ece": 0.02, "adaptive_ece": 0.03, "brier_score": 0.10} + + def fake_resolve_run_paths(namespace: str) -> dict[str, Path]: + return {"results": tmp_path / f"{namespace}.pkl"} + + def fake_load_pickle(path: Path) -> dict[str, dict[str, float]]: + if "rank-2" in path.name: + return {"metrics_90": {"empirical_coverage": 0.90, "avg_interval_width": 0.44}} + return {"metrics_90": {"empirical_coverage": 0.88, "avg_interval_width": 0.30}} + + def fake_final_candidate(**kwargs: Any) -> dict[str, object]: + assert kwargs["phase_prefix"] == "phase2" + assert kwargs["design"]["selection_rank"] == 2 + assert str(kwargs["calibrator_override_path"]).endswith("platt.pkl") + return { + "namespace": "phase2-final", + "policy_status": _passing_policy(width=0.44), + "set_status": {"summary": {"set_coverage": 0.91}}, + } + + monkeypatch.setattr(reopen, "_reopen_artifact_paths", lambda _run_tag: paths) + monkeypatch.setattr(reopen, "_fit_calibrator", fake_fit_calibrator) + monkeypatch.setattr(reopen, "_run_python", lambda *_args, **_kwargs: None) + monkeypatch.setattr(reopen, "_resolve_run_paths", fake_resolve_run_paths) + monkeypatch.setattr(reopen, "_load_pickle", fake_load_pickle) + monkeypatch.setattr(reopen, "_run_phase1_oot_candidate", fake_final_candidate) + + decision, policy, sets, summary = _run_phase2_search( + run_tag="phase2-success", + upstream_run_tag="upstream", + env={}, + aggregated=aggregate, + phase1_candidates_frame=pd.DataFrame(), + alpha_candidates_95=[0.05], + tuning_holdout_ratios=[0.2], + inner_random_states=[42], + partition_candidates=["grade"], + partition_probability_sources=["calibrated"], + n_score_bins_candidates=[10], + fallback_modes=["grade_then_global"], + score_scale_families=["none"], + calibration_fractions=[1.0], + phase2_cfg={"calibrators": ["platt"], "max_metric_degradation": {}}, + sidecar_cfg={}, + validation_cfg={}, + ) + + phase2_search = pd.read_parquet(paths["phase2_search"]) + assert decision == "promotable_for_followup" + assert policy["avg_width_90"] == 0.44 + assert sets["summary"]["set_coverage"] == 0.91 + assert summary is not None + assert summary["final_namespace"] == "phase2-final" + assert summary["best_candidate"]["selection_rank"] == 2 + assert phase2_search.iloc[0]["selection_rank"] == 2 diff --git a/tests/test_scripts/test_run_crpto_vs_spo_stability.py b/tests/test_scripts/test_run_crpto_vs_spo_stability.py index a415e66..7f483c0 100644 --- a/tests/test_scripts/test_run_crpto_vs_spo_stability.py +++ b/tests/test_scripts/test_run_crpto_vs_spo_stability.py @@ -1,8 +1,15 @@ from __future__ import annotations import json +from argparse import Namespace from pathlib import Path +import numpy as np +import pandas as pd +import pytest + +from scripts import run_crpto_vs_spo_stability as stability_mod + def test_crpto_vs_spo_stability_artifacts_exist() -> None: status_path = Path("data/processed/crpto_vs_spo_stability.json") @@ -10,3 +17,59 @@ def test_crpto_vs_spo_stability_artifacts_exist() -> None: status = json.loads(status_path.read_text(encoding="utf-8")) assert status.get("schema_version") assert Path("reports/crpto/figures/crpto_fig11_crpto_stability.png").exists() + + +def test_period_sample_seed_is_stable_and_distinct_by_period() -> None: + seed = 42 + + assert stability_mod._period_sample_seed(seed, "2018H1") == 100_042 + assert stability_mod._period_sample_seed(seed, "2018H1") == 100_042 + assert stability_mod._period_sample_seed(seed, "2020") == 500_042 + + +def test_detail_rows_and_summary_payload_preserve_period_contract() -> None: + periods = list(stability_mod.PERIODS) + test = pd.DataFrame({"default_flag": [0, 1, 0, 1, 0]}) + period_masks = { + period: np.array([idx == pos for idx in range(len(periods))]) + for pos, period in enumerate(periods) + } + regrets = stability_mod._init_period_regrets() + regrets["2018H1"]["two_stage"] = [2.0, 4.0] + regrets["2018H1"]["spo_plus"] = [1.0, 2.0] + regrets["2018H1"]["conformal_robust"] = [3.0, float("nan")] + coverage = { + "2018H1": { + "coverage_90": 0.91, + "coverage_95": 0.96, + "avg_width_90": 0.42, + "min_grade_coverage_90": 0.89, + } + } + + rows = stability_mod._detail_rows( + test=test, + period_masks=period_masks, + per_period_regrets=regrets, + period_coverage=coverage, + ) + summary = stability_mod._summary_payload( + run_tag="run-test", + args=Namespace(n_items=50, budget=15, n_train=800, epochs=50, seeds=2), + n_features=3, + feature_names=["a", "b", "c"], + rows=rows, + per_period_regrets=regrets, + total_time=12.34, + ) + + first = rows[0] + assert first["period"] == "2018H1" + assert first["two_stage_mean_regret"] == 3.0 + assert first["spo_plus_mean_regret"] == 1.5 + assert first["conformal_robust_mean_regret"] == 3.0 + assert first["spo_improvement_pct"] == pytest.approx(49.999999983333336) + assert summary["config"]["n_features"] == 3 + assert summary["per_period"]["2018H1"]["coverage_90"] == 0.91 + assert summary["per_period"]["2018H1"]["spo_improvement_vs_ts_pct"] == 50.0 + assert summary["stability_summary"]["coverage_always_above_target"] is True diff --git a/tests/test_scripts/test_run_fairness_audit.py b/tests/test_scripts/test_run_fairness_audit.py index 8ab46d3..f3322b5 100644 --- a/tests/test_scripts/test_run_fairness_audit.py +++ b/tests/test_scripts/test_run_fairness_audit.py @@ -4,6 +4,7 @@ import json +import numpy as np import pandas as pd import pytest import yaml @@ -184,3 +185,57 @@ def test_run_fairness_auto_selects_threshold_and_writes_decision_policy(tmp_path assert status["prediction_threshold_source"] == "decision_policy_artifact_auto_selected" assert status["decision_policy"]["path"].endswith("fairness_decision_policy.json") assert decision_policy["global_threshold"] in [0.35, 0.4, 0.45, 0.5] + + +def test_shap_helpers_detect_categorical_columns_and_fill_missing_values() -> None: + class DummyModel: + def __init__(self) -> None: + self.feature_names_ = ["grade", "purpose", "rate"] + + def get_cat_feature_indices(self) -> list[int]: + return [0] + + frame = pd.DataFrame( + { + "grade": ["A", None, "B"], + "purpose": ["debt", "car", None], + "rate": [0.1, np.nan, 0.3], + } + ) + + cat_names = fairness_mod._catboost_cat_feature_names(DummyModel(), frame) + prepared = fairness_mod._prepare_catboost_shap_frame(frame, cat_names) + + assert cat_names == ["grade", "purpose"] + assert prepared.loc[1, "grade"] == "missing" + assert prepared.loc[2, "purpose"] == "missing" + assert prepared.loc[1, "rate"] == 0.0 + + +def test_shap_attribute_result_reports_group_drivers_and_pairwise_diffs() -> None: + shap_matrix = np.vstack( + [ + np.tile([3.0, 1.0, 0.5], (10, 1)), + np.tile([1.0, 4.0, 0.5], (10, 1)), + ] + ) + sample_idx = np.arange(20) + groups = { + "home_ownership": np.array(["A"] * 10 + ["B"] * 10), + "home_ownership__x__grade": np.array(["skip"] * 20), + } + + results = fairness_mod._shap_attribute_results( + groups, + sample_idx=sample_idx, + shap_matrix=shap_matrix, + feature_names=["dti", "income", "grade"], + ) + + assert len(results) == 1 + result = results[0] + assert result["attribute"] == "home_ownership" + assert result["groups_analyzed"] == ["A", "B"] + assert result["top5_per_group"]["A"][0]["feature"] == "dti" + assert result["top5_per_group"]["B"][0]["feature"] == "income" + assert result["pairwise_feature_diffs"][0]["top_driving_features"][0]["feature"] == "income" diff --git a/tests/test_scripts/test_run_portfolio_bound_aware_search.py b/tests/test_scripts/test_run_portfolio_bound_aware_search.py index 25d2020..6df7fb5 100644 --- a/tests/test_scripts/test_run_portfolio_bound_aware_search.py +++ b/tests/test_scripts/test_run_portfolio_bound_aware_search.py @@ -1,13 +1,25 @@ from __future__ import annotations import pandas as pd +import pytest from scripts.search.run_portfolio_bound_aware_search import ( _aggregate_exact_results, + _budget_profiles, + _build_grid_spec, + _build_parser, _build_stratified_shortlist, _policy_semantic_key, + _resolve_run_paths, + _sanitize_run_label, + _search_space_payload, + _selection_context_payload, _targeted_policy_grid, ) +from src.optimization.certificate_semantics import ( + IJDS_DECLARED_ALPHA_GRID, + IJDS_DECLARED_ALPHA_GRID_CSV, +) def _frontier_row( @@ -227,3 +239,113 @@ def test_targeted_policy_grid_includes_segment_tail_families() -> None: "segment_tail_blended_uncertainty", "segment_relative_tail_blended_uncertainty", } + assert grid == [ + ("blended_uncertainty", 0.5, 1.0, 1.0), + ("capped_blended_uncertainty", 0.5, 0.75, 1.0), + ("tail_blended_uncertainty", 0.5, 1.0, 0.9), + ("segment_tail_blended_uncertainty", 0.5, 1.0, 0.9), + ("segment_relative_tail_blended_uncertainty", 0.5, 1.0, 0.9), + ] + + +def test_budget_profiles_are_explicit_and_reject_unknown_tokens() -> None: + profiles = _budget_profiles("free,floored") + + assert [profile["name"] for profile in profiles] == ["free_budget", "floored_budget"] + assert profiles[0]["min_budget_utilization"] == 0.0 + assert profiles[1]["pd_cap_slack_penalty"] == 1.5 + with pytest.raises(ValueError, match="Unsupported budget profile"): + _budget_profiles("free,aggressive") + + +def test_bound_aware_default_uses_declared_ijds_alpha_grid() -> None: + args = _build_parser().parse_args(["--conformal-intervals-path", "intervals.parquet"]) + + assert args.alpha_grid == IJDS_DECLARED_ALPHA_GRID_CSV + + +def test_grid_spec_separates_proxy_and_exact_sampling_contracts() -> None: + args = _build_parser().parse_args( + [ + "--conformal-intervals-path", + "intervals.parquet", + "--risk-grid", + "0.16,0.17", + "--random-states", + "42,52", + "--exact-random-states", + "62,72,82", + "--max-candidates", + "5000", + "--exact-max-candidates", + "0", + ] + ) + + grid = _build_grid_spec(args) + + assert grid.random_states == [42, 52] + assert grid.exact_random_states == [62, 72, 82] + assert grid.exact_max_candidates == 0 + assert grid.alpha_grid == list(IJDS_DECLARED_ALPHA_GRID) + assert grid.bound_total_checks(shortlist_size=5) == 5 * 8 * 3 + assert grid.frontier_total_units == 2 * 2 * (1 + grid.policy_grid_count) + + +def test_selection_context_payload_keeps_frontier_and_exact_paths_together(tmp_path) -> None: + args = _build_parser().parse_args( + [ + "--conformal-intervals-path", + str(tmp_path / "intervals.parquet"), + "--run-label", + "unit/run", + "--output-dir", + str(tmp_path / "out"), + "--model-dir", + str(tmp_path / "model"), + "--random-states", + "42,52", + "--exact-random-states", + "42", + "--policy-modes", + "blended_uncertainty", + ] + ) + run_label = _sanitize_run_label(args.run_label) + paths = _resolve_run_paths(args, run_label=run_label) + search_space = _search_space_payload( + args=args, + risk_values=[0.16], + aversion_values=[0.0], + gamma_values=[0.5], + delta_cap_quantiles=[1.0], + tail_focus_quantiles=[1.0], + budget_profiles=_budget_profiles(args.budget_profiles), + alpha_grid=[0.01], + random_states=[42, 52], + exact_random_states=[42], + exact_max_candidates=0, + policy_modes=["blended_uncertainty"], + cuopt_parameters={"method": "concurrent"}, + incumbent_risk_neighbors=[0.16], + incumbent_gamma_neighbors=[0.5], + incumbent_policy_modes=["blended_uncertainty"], + ) + + context = _selection_context_payload( + args=args, + paths=paths, + run_label=run_label, + search_space=search_space, + exact_max_candidates=0, + random_states=[42, 52], + exact_random_states=[42], + alpha_grid=[0.01], + ) + + assert run_label == "unit_run" + assert context["search_space"]["random_states"] == [42, 52] + assert context["search_space"]["exact_random_states"] == [42] + assert context["shortlist_path"] == str(paths.shortlist_path) + assert context["selection_path"] == str(paths.selection_path) + assert context["resource_snapshot_path"] == str(paths.resource_path) diff --git a/tests/test_scripts/test_run_portfolio_bound_exact_eval.py b/tests/test_scripts/test_run_portfolio_bound_exact_eval.py index 2f492d7..203aa63 100644 --- a/tests/test_scripts/test_run_portfolio_bound_exact_eval.py +++ b/tests/test_scripts/test_run_portfolio_bound_exact_eval.py @@ -10,9 +10,12 @@ _context_exact_threads, _context_max_candidates, _context_random_states, + _exact_eval_plan, _load_completed_bound_eval, _load_partial_bound_eval, _repo_relative, + _resume_exact_rows, + _search_space_payload, _shortlist_exact_path, _validate_alpha_grid_supported, ) @@ -84,6 +87,28 @@ def test_exact_context_overrides_proxy_sampling() -> None: assert _context_random_states(context) == [42, 52, 62] +def test_exact_plan_dedupes_full_universe_random_states(monkeypatch) -> None: + monkeypatch.delenv("EXACT_THREADS", raising=False) + context = { + "alpha_grid": [0.01, 0.03], + "max_candidates": 100000, + "exact_max_candidates": 0, + "random_states": [42], + "requested_exact_random_states": "42,52,62", + "exact_checkpoint_every": 7, + "exact_threads": 3, + } + + plan = _exact_eval_plan(context=context, shortlist_rows=5) + + assert plan.requested_random_states == [42, 52, 62] + assert plan.random_states == [42] + assert plan.full_universe_seed_deduped is True + assert plan.expected_checks == 10 + assert plan.exact_threads == 3 + assert plan.checkpoint_every == 7 + + def test_exact_threads_can_come_from_environment(monkeypatch) -> None: monkeypatch.setenv("EXACT_THREADS", "8") @@ -108,6 +133,34 @@ def test_repo_relative_keeps_artifact_paths_standalone() -> None: assert _repo_relative(path) == "models/portfolio_bound_aware/selection.json" +def test_resume_exact_rows_filters_out_irrelevant_seeds(tmp_path) -> None: + path = tmp_path / "bound_eval.parquet" + pd.DataFrame( + { + "candidate_rank": [1, 1, 2], + "eval_random_state": [42, 52, 42], + "alpha": [0.01, 0.01, 0.03], + "all_bounds_hold": [True, True, False], + } + ).to_parquet(path, index=False) + + rows, keys = _resume_exact_rows(bound_eval_path=path, random_states=[42]) + + assert len(rows) == 2 + assert keys == {(1, 42, 0.01), (2, 42, 0.03)} + + +def test_search_space_payload_records_requested_alpha_grid() -> None: + context = {"search_space": {"alpha_grid": [0.01], "mode": ["capped"]}} + + payload = _search_space_payload(context=context, alpha_grid=[0.01, 0.03]) + + assert payload["alpha_grid"] == [0.01, 0.03] + assert payload["requested_alpha_grid"] == [0.01] + assert payload["effective_alpha_grid"] == [0.01, 0.03] + assert payload["mode"] == ["capped"] + + def test_validate_alpha_grid_supported_accepts_sweep_values(tmp_path, monkeypatch) -> None: sweep_path = tmp_path / "data" / "processed" / "alpha_sweep_pareto_mondrian.parquet" sweep_path.parent.mkdir(parents=True) diff --git a/tests/test_scripts/test_run_regret_auditability_sandbox.py b/tests/test_scripts/test_run_regret_auditability_sandbox.py index 8161439..23b5bbb 100644 --- a/tests/test_scripts/test_run_regret_auditability_sandbox.py +++ b/tests/test_scripts/test_run_regret_auditability_sandbox.py @@ -6,9 +6,15 @@ import pytest import yaml +import scripts.search.run_regret_auditability_sandbox as sandbox from scripts.search.run_regret_auditability_sandbox import ( FEATURE_PROFILES, MONOTONIC_POLICIES, + PORTFOLIO_ALPHA_GRID, + PhaseCommand, + _pd_validation_policy, + _pending_commands_for_group, + _phase_command_groups, _rank_pd_candidate_rows, assert_safe_output_path, build_phase_commands, @@ -117,6 +123,69 @@ def test_resume_manifest_loading_roundtrip(tmp_path: Path) -> None: assert load_resume_manifest(tmp_path / "missing.json") == {} +def _command(name: str, phase: str, output: Path) -> PhaseCommand: + return PhaseCommand( + name=name, + phase=phase, + command=["python", "-c", "pass"], + outputs=[str(output)], + checkpoint=str(output), + env={}, + max_workers=1, + cpu_threads=1, + ) + + +def test_phase_command_groups_keep_consecutive_phase_batches(tmp_path: Path) -> None: + commands = [ + _command("a", "pd-smoke", tmp_path / "a"), + _command("b", "pd-smoke", tmp_path / "b"), + _command("c", "conformal", tmp_path / "c"), + _command("d", "pd-smoke", tmp_path / "d"), + ] + + groups = _phase_command_groups(commands) + + assert [[command.name for command in group] for group in groups] == [["a", "b"], ["c"], ["d"]] + + +def test_pending_commands_skip_completed_outputs_on_resume( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + completed_output = tmp_path / "done.txt" + completed_output.write_text("ok", encoding="utf-8") + pending_output = tmp_path / "todo.txt" + monkeypatch.setattr(sandbox, "_log_command_to_mlflow", lambda **_: None) + + pending, skipped = _pending_commands_for_group( + artifact_root=tmp_path, + log_path=tmp_path / "command_log.csv", + group=[ + _command("done", "pd-smoke", completed_output), + _command("todo", "pd-smoke", pending_output), + ], + resume=True, + ) + + assert skipped == 1 + assert [command.name for command in pending] == ["todo"] + assert "skipped_completed" in (tmp_path / "command_log.csv").read_text(encoding="utf-8") + + +def test_pd_validation_policy_scales_by_phase() -> None: + smoke_replay, smoke_walk_forward = _pd_validation_policy("pd-smoke") + broad_replay, broad_walk_forward = _pd_validation_policy("pd-broad") + refine_replay, refine_walk_forward = _pd_validation_policy("pd-refine") + + assert smoke_replay["top_k_trials"] == 1 + assert smoke_walk_forward is False + assert broad_replay["seeds"] == [42, 52, 62] + assert broad_walk_forward is True + assert refine_replay["top_k_trials"] == 30 + assert refine_walk_forward is True + + def test_pd_candidate_ranking_prefers_auc_then_calibration() -> None: ranked = _rank_pd_candidate_rows( [ @@ -172,6 +241,9 @@ def test_build_portfolio_command_uses_external_output_dirs(tmp_path: Path) -> No command = commands[0] assert "--output-dir" in command.command assert "--model-dir" in command.command + alpha_grid_index = command.command.index("--alpha-grid") + 1 + assert command.command[alpha_grid_index] == PORTFOLIO_ALPHA_GRID + assert PORTFOLIO_ALPHA_GRID == "0.01,0.03,0.05,0.07,0.10,0.12,0.15,0.20" assert all(str(tmp_path) in output for output in command.outputs) diff --git a/tests/test_scripts/test_run_ty_advisory.py b/tests/test_scripts/test_run_ty_advisory.py new file mode 100644 index 0000000..71171e9 --- /dev/null +++ b/tests/test_scripts/test_run_ty_advisory.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from scripts.run_ty_advisory import TY_REQUIREMENT, build_ty_command, iter_python_files + + +def test_active_ty_scope_excludes_archived_optional_and_protected_paths() -> None: + files = set(iter_python_files(scope="active")) + + assert "scripts/generate_conformal_intervals.py" not in files + assert "scripts/train_pd_model.py" not in files + assert "scripts/run_spo_real.py" not in files + assert "src/optimization/cuopt_adapter.py" not in files + assert all(not path.startswith("scripts/archive/") for path in files) + assert all(not path.startswith("scripts/experiments/") for path in files) + assert all( + not (path.startswith("scripts/search/run_") and path.endswith(".py")) for path in files + ) + + +def test_active_ty_scope_keeps_live_ijds_helpers() -> None: + files = set(iter_python_files(scope="active")) + + assert "scripts/compile_ijds_submission.py" in files + assert "scripts/search/build_pool93_body_allocation_audit.py" in files + assert "src/optimization/portfolio_model.py" in files + + +def test_full_ty_scope_keeps_every_python_file_under_src_and_scripts() -> None: + files = set(iter_python_files(scope="full")) + + assert "scripts/generate_conformal_intervals.py" in files + assert "scripts/train_pd_model.py" in files + assert "src/optimization/cuopt_adapter.py" in files + + +def test_ty_command_pins_version_and_keeps_daily_scope_advisory() -> None: + command = build_ty_command( + uvx="uvx", + files=["src/example.py"], + fail_on_diagnostics=False, + ) + + assert command[:4] == ["uvx", "--from", TY_REQUIREMENT, "ty"] + assert "--exit-zero" in command + assert command[-1] == "src/example.py" + + +def test_ty_command_can_block_the_submission_gate() -> None: + command = build_ty_command( + uvx="uvx", + files=["src/example.py"], + fail_on_diagnostics=True, + ) + + assert "--exit-zero" not in command diff --git a/tests/test_scripts/test_simulate_ab_test.py b/tests/test_scripts/test_simulate_ab_test.py new file mode 100644 index 0000000..93907e1 --- /dev/null +++ b/tests/test_scripts/test_simulate_ab_test.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import json + +import pandas as pd +import pytest + +from scripts.simulate_ab_test import _resolve_robust_policy + + +def test_resolve_robust_policy_uses_guardrail_champion_priority(tmp_path, monkeypatch) -> None: + monkeypatch.chdir(tmp_path) + model_dir = tmp_path / "models" + model_dir.mkdir() + (model_dir / "champion_portfolio_policy.json").write_text( + json.dumps( + { + "selected_policy": {"risk_tolerance": 0.10, "gamma": 0.1}, + "selected_policy_balanced_robustness": { + "risk_tolerance": 0.11, + "policy_mode": "balanced", + "gamma": 0.2, + }, + "selected_policy_guardrail_robustness": { + "risk_tolerance": 0.12, + "policy_mode": "guardrail", + "gamma": 0.3, + }, + } + ), + encoding="utf-8", + ) + + policy = _resolve_robust_policy( + max_portfolio_pd=0.20, + policy_selector="guardrail_robustness", + ) + + assert policy["source"] == "champion_policy_artifact::guardrail_robustness" + assert policy["policy_mode"] == "guardrail" + assert policy["risk_tolerance"] == 0.12 + assert policy["gamma"] == 0.3 + + +def test_resolve_robust_policy_uses_summary_when_champion_missing(tmp_path, monkeypatch) -> None: + monkeypatch.chdir(tmp_path) + summary_path = tmp_path / "data" / "processed" / "portfolio_robustness_summary.parquet" + summary_path.parent.mkdir(parents=True) + pd.DataFrame( + [ + { + "risk_tolerance": 0.15, + "best_robust_lambda": 0.2, + "best_robust_min_budget_utilization": 0.90, + "best_robust_pd_cap_slack_penalty": 1.0, + "best_robust_return": 100.0, + "best_robust_policy_mode": "hard_worst_case", + "best_robust_gamma": 0.8, + "best_robust_delta_cap_quantile": 0.95, + }, + { + "risk_tolerance": 0.18, + "best_robust_lambda": 0.4, + "best_robust_min_budget_utilization": 0.91, + "best_robust_pd_cap_slack_penalty": 2.0, + "best_robust_return": 200.0, + "best_robust_policy_mode": "blended_uncertainty", + "best_robust_gamma": 0.6, + "best_robust_delta_cap_quantile": 0.90, + }, + ] + ).to_parquet(summary_path, index=False) + + policy = _resolve_robust_policy(max_portfolio_pd=0.17) + + assert policy["source"] == "portfolio_robustness_summary" + assert policy["risk_tolerance"] == 0.15 + assert policy["uncertainty_aversion"] == 0.2 + assert policy["gamma"] == 0.8 + + +def test_resolve_robust_policy_explicit_champion_only_requires_artifact( + tmp_path, monkeypatch +) -> None: + monkeypatch.chdir(tmp_path) + + with pytest.raises(FileNotFoundError): + _resolve_robust_policy( + max_portfolio_pd=0.20, + policy_selector="explicit_champion_only", + ) diff --git a/tests/test_scripts/test_train_pd_model_config_overrides.py b/tests/test_scripts/test_train_pd_model_config_overrides.py index 484a06b..93799aa 100644 --- a/tests/test_scripts/test_train_pd_model_config_overrides.py +++ b/tests/test_scripts/test_train_pd_model_config_overrides.py @@ -1,5 +1,9 @@ from __future__ import annotations +import json +import sys +import types + import numpy as np import pytest @@ -7,11 +11,14 @@ from scripts.train_pd_model import ( _apply_cli_overrides, _apply_pd_replay_manifest, + _gate_tier, _load_training_splits, _normalize_sample_size, _prepare_training_inputs, + _replay_selection_policy, _sample_training_splits, _select_calibration_from_backtest, + _summarize_replayed_trials, ) @@ -219,6 +226,45 @@ def fake_resolve_feature_sets(*args, **kwargs): assert replay_stable_core_meta == {"enabled": False, "excluded_features": []} +def test_replay_trial_summary_prioritizes_gate_then_ece_then_auc() -> None: + rows = [ + { + "trial_number": 1, + "validation_auc": 0.80, + "validation_brier": 0.20, + "validation_ece": 0.05, + "gate_attrs_present": True, + "gate_all_pass": False, + }, + { + "trial_number": 2, + "validation_auc": 0.79, + "validation_brier": 0.21, + "validation_ece": 0.06, + "gate_attrs_present": True, + "gate_all_pass": True, + }, + { + "trial_number": 3, + "validation_auc": 0.90, + "validation_brier": 0.19, + "validation_ece": 0.01, + "gate_attrs_present": False, + "gate_all_pass": None, + }, + ] + + summary = _summarize_replayed_trials(rows, prioritize_gate_pass=True) + + assert summary["trial_number"].tolist() == [2, 3, 1] + assert summary["gate_tier"].tolist() == [0, 1, 2] + assert _gate_tier(True, prioritize_gate_pass=True) == 0 + assert _gate_tier(None, prioritize_gate_pass=True) == 1 + assert _gate_tier(False, prioritize_gate_pass=True) == 2 + assert _gate_tier(False, prioritize_gate_pass=False) == 1 + assert _replay_selection_policy(True)["rank_order"][0] == "gate_tier(pass->unknown->fail)" + + def test_load_training_splits_uses_config_paths_and_normalizes( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -342,3 +388,102 @@ def test_prepare_training_inputs_builds_model_ready_frames() -> None: assert x_test_cb["grade"].tolist() == ["B", "UNKNOWN"] assert x_test_lr.isna().sum().sum() == 0 assert lr_fill["score"] == x_train_fit_lr["score"].median() + + +def test_evaluate_walk_forward_stage_reports_disabled_without_training( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fail_if_called(*args, **kwargs): + raise AssertionError("walk-forward evaluator should not run when disabled") + + monkeypatch.setattr(pd_train, "_evaluate_walk_forward_auc", fail_if_called) + + report = pd_train._evaluate_walk_forward_stage( + enabled=False, + walk_cfg={"n_windows": 5}, + train=pd_train.pd.DataFrame(), + catboost_features=[], + categorical_features=[], + model_params={}, + ) + + assert report == { + "enabled": False, + "reason": "disabled_in_config", + "n_windows_requested": 5, + "n_windows_used": 0, + "folds": [], + } + + +def test_evaluate_walk_forward_stage_normalizes_config( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: dict[str, object] = {} + + def fake_evaluate(train_df: pd_train.pd.DataFrame, **kwargs): + captured["train_rows"] = len(train_df) + captured.update(kwargs) + return {"enabled": True, "n_windows_used": 1} + + monkeypatch.setattr(pd_train, "_evaluate_walk_forward_auc", fake_evaluate) + + report = pd_train._evaluate_walk_forward_stage( + enabled=True, + walk_cfg={ + "n_windows": "4", + "min_train_rows": "12", + "window_rows": "6", + "date_col": "issue_d", + "max_rows": "0", + }, + train=pd_train.pd.DataFrame({"x": [1, 2]}), + catboost_features=["x"], + categorical_features=[], + model_params={"depth": 4}, + ) + + assert report == {"enabled": True, "n_windows_used": 1} + assert captured["features"] == ["x"] + assert captured["target"] == pd_train.TARGET + assert captured["params"] == {"depth": 4} + assert captured["n_windows"] == 4 + assert captured["min_train_rows"] == 12 + assert captured["window_rows"] == 6 + assert captured["max_rows"] is None + + +def test_export_shap_feature_importance_writes_summary( + tmp_path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + fake_catboost = types.ModuleType("catboost") + + class FakePool: + def __init__(self, frame, cat_features): + self.frame = frame + self.cat_features = cat_features + + fake_catboost.Pool = FakePool + monkeypatch.setitem(sys.modules, "catboost", fake_catboost) + + class FakeModel: + def get_feature_importance(self, *, type: str, data: FakePool): + assert type == "ShapValues" + assert data.cat_features == ["grade"] + return np.array([[0.1, 0.4, 0.5], [0.3, -0.2, 0.5]]) + + shap_dir = tmp_path / "shap" + result = pd_train._export_shap_feature_importance( + cb_tuned_model=FakeModel(), + X_test_cb=pd_train.pd.DataFrame({"score": [1.0, 2.0], "grade": ["A", "B"]}), + categorical_features=["grade"], + catboost_features=["score", "grade"], + shap_dir=shap_dir, + ) + + summary = json.loads((shap_dir / "shap_feature_importance.json").read_text(encoding="utf-8")) + assert result == {"exported": True, "n_features": 2, "path": str(shap_dir)} + assert (shap_dir / "shap_values_test.npz").exists() + assert summary["expected_value"] == 0.5 + assert summary["top_features"][0]["feature"] == "grade" diff --git a/tests/test_scripts/test_validate_conformal_policy.py b/tests/test_scripts/test_validate_conformal_policy.py index e5ef542..2078000 100644 --- a/tests/test_scripts/test_validate_conformal_policy.py +++ b/tests/test_scripts/test_validate_conformal_policy.py @@ -65,6 +65,52 @@ def _fake_legacy_score(*_args, **_kwargs): assert score == pytest.approx(1.20) +def test_interval_arrays_filters_invalid_interval_rows() -> None: + intervals = pd.DataFrame( + { + "y_true": [0.1, None, 0.3, 0.4], + "pd_low_90": [0.0, 0.1, np.nan, 0.2], + "pd_high_90": [0.2, 0.3, 0.4, 0.6], + } + ) + + y_true, lower, upper = policy_mod._interval_arrays( + intervals, + lower_col="pd_low_90", + upper_col="pd_high_90", + ) + + assert list(y_true) == [0.1, 0.4] + assert list(lower) == [0.0, 0.2] + assert list(upper) == [0.2, 0.6] + + +def test_winkler_90_check_allows_documented_compensated_band() -> None: + policy = { + "target_coverage_90_min": 0.90, + "min_group_coverage_90_min": 0.88, + "max_avg_width_90": 0.80, + "max_critical_alerts": 0, + "max_winkler_90": 1.00, + "enable_compensated_winkler_90": True, + "compensated_winkler_90_max": 1.20, + } + metrics = { + "winkler_90": 1.10, + "coverage_90": 0.91, + "min_group_coverage_90": 0.89, + "avg_width_90": 0.70, + "critical_alerts": 0.0, + } + + check = policy_mod._winkler_90_check(policy, metrics) + + assert check["passed"] is True + assert check["raw_passed"] is False + assert check["compensated_passed"] is True + assert check["policy_mode"] == "compensated_band" + + def test_validate_conformal_policy_includes_material_gate_checks(tmp_path) -> None: data_dir = tmp_path / "data" / "processed" model_dir = tmp_path / "models" diff --git a/tests/test_utils/test_script_helpers.py b/tests/test_utils/test_script_helpers.py index 0ab892b..201d56d 100644 --- a/tests/test_utils/test_script_helpers.py +++ b/tests/test_utils/test_script_helpers.py @@ -17,6 +17,7 @@ parse_percent_series, policy_matches, resolve_interval_columns, + resolve_repo_artifact_path, try_load_json, write_json, write_table, @@ -77,6 +78,23 @@ def test_artifact_path_with_env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) assert artifact_path("models/x.json") == tmp_path / "models/x.json" +def test_resolve_repo_artifact_path_maps_wsl_manifest_path(tmp_path: Path) -> None: + root = tmp_path / "Paper_CRPTO" + expected = root / "data" / "processed" / "intervals.parquet" + + resolved = resolve_repo_artifact_path( + "/mnt/c/Users/carlos/Documents/Paper_CRPTO/data/processed/intervals.parquet", + root=root, + ) + + assert resolved == expected + + +def test_resolve_repo_artifact_path_anchors_relative_path(tmp_path: Path) -> None: + root = tmp_path / "repo" + assert resolve_repo_artifact_path("models/a.json", root=root) == root / "models" / "a.json" + + def test_first_existing_prefers_existing(tmp_path: Path) -> None: existing = tmp_path / "exact.parquet" existing.touch() From 4134a6d32156fb02d7c7a2787383f20caea315eb Mon Sep 17 00:00:00 2001 From: Carlos Alfredo Vergara Rojas Date: Thu, 9 Jul 2026 21:31:55 -0500 Subject: [PATCH 3/7] refactor: simplify IJDS policy and evidence --- .codex/skills/crpto/SKILL.md | 266 +-- AGENTS.md | 4 +- CLAUDE.md | 42 +- EXTRACTION_MANIFEST.md | 12 +- README.md | 30 +- configs/crpto_publication_targets.yaml | 43 +- ...ijds_calibration_selected_simple90_v6.yaml | 45 + ...mpion_reopen_ijds_exact_alpha_grid_v1.yaml | 19 + docs/SCOPE_AND_GOVERNANCE.md | 51 +- docs/refactor/README.md | 2 +- .../ijds_tooling_decisions_2026-07-09.md | 122 ++ .../ijds_tooling_refactor_lab_2026-07-08.md | 419 ----- docs/research/README.md | 13 +- docs/research/active_claims_2026-07-04.md | 355 ++-- ..._alpha_calibration_selection_2026-07-09.md | 69 + justfile | 18 +- .../calibration_selected_policy_summary.json | 149 ++ .../portfolio/ijds_policy_governance.json | 114 ++ .../conformal/exact_alpha_grid_summary.json | 244 +++ paper/CRPTO.qmd | 93 +- paper/CRPTO_ijds.qmd | 1507 +++++----------- paper/README.md | 23 +- paper/ijds.css | 20 + paper/submission/CLAIM_AUDIT_MATRIX.md | 130 +- .../submission/COVER_LETTER_AND_DISCLOSURE.md | 149 +- paper/submission/CRPTO_ijds_submission.tex | 1511 +++++------------ .../IJDS_SUBMISSION_ROADMAP_2026-08-10.md | 105 +- paper/submission/README.md | 344 ++-- paper/submission/REPRODUCIBILITY_PACKAGE.md | 205 +-- .../submission/SCHOLARONE_FINAL_CHECKLIST.md | 166 +- paper/supplement_ijds.qmd | 1277 ++++---------- .../crpto_tableA35_exact_alpha_grid.csv | 9 + .../crpto_tableA35_exact_alpha_grid.tex | 14 + ...o_tableA36_calibration_policy_selector.csv | 10 + ...o_tableA36_calibration_policy_selector.tex | 15 + ...libration_selected_temporal_evaluation.csv | 19 + ...libration_selected_temporal_evaluation.tex | 24 + ...leA38_calibration_selected_grade_audit.csv | 7 + ...leA38_calibration_selected_grade_audit.tex | 12 + ...ableA39_calibration_selected_bootstrap.csv | 6 + ...ableA39_calibration_selected_bootstrap.tex | 11 + ...40_calibration_selected_point_baseline.csv | 4 + ...40_calibration_selected_point_baseline.tex | 9 + scripts/README.md | 18 +- scripts/analyze_crpto_evidence.py | 24 +- ...uild_ijds_calibration_selected_evidence.py | 398 +++++ scripts/check_publication_integrity.py | 156 +- scripts/experiments/ijds_policy_support.py | 176 ++ ..._calibration_selected_policy_challenger.py | 440 +++++ .../run_ijds_exact_alpha_grid_challenger.py | 291 ++++ scripts/optimize_portfolio_tradeoff.py | 61 +- scripts/run_ty_advisory.py | 11 +- scripts/simulate_ab_test.py | 80 +- scripts/validate_alpha_gamma_bound.py | 27 +- src/models/conformal_alpha_grid.py | 211 +++ src/optimization/policy.py | 45 + src/optimization/policy_evaluation.py | 109 ++ src/optimization/policy_selection.py | 166 ++ tests/test_crpto_final_sync.py | 18 +- ..._calibration_selected_policy_challenger.py | 80 + .../test_ijds_exact_alpha_grid_challenger.py | 58 + tests/test_ijds_active_claim_sync.py | 132 ++ .../test_models/test_conformal_alpha_grid.py | 79 + tests/test_optimization/test_policy.py | 34 +- .../test_policy_evaluation.py | 109 ++ .../test_policy_selection.py | 106 ++ tests/test_pool93_body_claim_sync.py | 277 +-- tests/test_publication_targets.py | 28 +- ...uild_ijds_calibration_selected_evidence.py | 105 ++ .../test_optimize_portfolio_tradeoff.py | 4 +- tests/test_scripts/test_run_ty_advisory.py | 7 +- tests/test_supplement_table_sync.py | 48 +- 72 files changed, 5660 insertions(+), 5325 deletions(-) create mode 100644 configs/experiments/champion_reopen_ijds_calibration_selected_simple90_v6.yaml create mode 100644 configs/experiments/champion_reopen_ijds_exact_alpha_grid_v1.yaml create mode 100644 docs/refactor/ijds_tooling_decisions_2026-07-09.md delete mode 100644 docs/refactor/ijds_tooling_refactor_lab_2026-07-08.md create mode 100644 docs/research/ijds_exact_alpha_calibration_selection_2026-07-09.md create mode 100644 models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6/portfolio/calibration_selected_policy_summary.json create mode 100644 models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6/portfolio/ijds_policy_governance.json create mode 100644 models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1/conformal/exact_alpha_grid_summary.json create mode 100644 reports/crpto/tables/crpto_tableA35_exact_alpha_grid.csv create mode 100644 reports/crpto/tables/crpto_tableA35_exact_alpha_grid.tex create mode 100644 reports/crpto/tables/crpto_tableA36_calibration_policy_selector.csv create mode 100644 reports/crpto/tables/crpto_tableA36_calibration_policy_selector.tex create mode 100644 reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.csv create mode 100644 reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.tex create mode 100644 reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.csv create mode 100644 reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.tex create mode 100644 reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.csv create mode 100644 reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.tex create mode 100644 reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.csv create mode 100644 reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.tex create mode 100644 scripts/build_ijds_calibration_selected_evidence.py create mode 100644 scripts/experiments/ijds_policy_support.py create mode 100644 scripts/experiments/run_ijds_calibration_selected_policy_challenger.py create mode 100644 scripts/experiments/run_ijds_exact_alpha_grid_challenger.py create mode 100644 src/models/conformal_alpha_grid.py create mode 100644 src/optimization/policy_evaluation.py create mode 100644 src/optimization/policy_selection.py create mode 100644 tests/test_experiments/test_ijds_calibration_selected_policy_challenger.py create mode 100644 tests/test_experiments/test_ijds_exact_alpha_grid_challenger.py create mode 100644 tests/test_ijds_active_claim_sync.py create mode 100644 tests/test_models/test_conformal_alpha_grid.py create mode 100644 tests/test_optimization/test_policy_evaluation.py create mode 100644 tests/test_optimization/test_policy_selection.py create mode 100644 tests/test_scripts/test_build_ijds_calibration_selected_evidence.py diff --git a/.codex/skills/crpto/SKILL.md b/.codex/skills/crpto/SKILL.md index 10a10e6..3394e9e 100644 --- a/.codex/skills/crpto/SKILL.md +++ b/.codex/skills/crpto/SKILL.md @@ -1,13 +1,10 @@ # CRPTO Skill -Use this skill when working inside `C:\Users\carlos\Documents\Paper_CRPTO`. -CRPTO is a standalone academic paper project for Conformal Robust -Predict-Then-Optimize in credit risk. The repository is public-facing, but the -paper champion artifacts are frozen. +Use this skill inside `C:\Users\carlos\Documents\Paper_CRPTO`. CRPTO is a +single-author IJDS paper and reproducibility bundle, not a production service. +Prefer simple code, frozen evidence, and one coherent manuscript claim. -## Required Context - -Before structural work, read these files in order: +## Read First 1. `docs/ACADEMIC_CONTEXT.md` 2. `docs/SCOPE_AND_GOVERNANCE.md` @@ -15,66 +12,45 @@ Before structural work, read these files in order: 4. `CONTRIBUTING.md` 5. `EXTRACTION_MANIFEST.md` 6. `configs/crpto_publication_targets.yaml` -7. `docs/research/README.md` - -Use the local project context over generic habits. This is a single-author, -static-dataset, no-production academic repo. Keep code simple, functional, and -close to the existing style. - -## Platform - -- Work Windows-first in PowerShell. -- Prefer `uv run ...` for Python, Quarto, dbt, DVC, MLflow, pytest, and ruff. -- The local venv is `.venv/Scripts/python.exe`. -- Do not introduce Unix-only shell assumptions. - -## Champion Rules - -The current IJDS paper-facing body claim is the promoted pool93 finite-grid -frontier closure. It is a deterministic policy-grid re-evaluation over the -frozen upstream PD/calibration/conformal artifacts, not a retraining run. - -- Terminal run tag: - `champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal` -- Active certificate tag: - `champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2` -- Body point source run: - `champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext` -- Policy family: `claim_micro_ext_body_cap345` -- Policy mode: `capped_blended_uncertainty` -- Robust return: `$184,832.48` -- Return-floor surplus: `$14,367.94` -- `V(alpha=0.01)`: `0.035350` -- `Gamma_CP(alpha=0.01)`: `0.162616` -- `Gamma_internalized(alpha=0.01)`: `0.089032` -- `Gamma_residual(alpha=0.01)`: `0.073584` -- Exact endpoint budget at `alpha=0.01`: `0.245083866` -- Exact Markov loss threshold at `alpha=0.01`: `0.345083866` -- Realized risk-tolerance excess: `0.0` -- Declared alpha-grid pass: `8/8` -- Consolidated frontier: `50,010` deduplicated semantic policies, `27,508` - eligible all-alpha above-floor policies. -- Terminal exact search: `37,068/37,068` all-alpha passers and `296,544` - completed exact candidate-alpha checks. -- Matched A40 point-PD baseline: `5.875%` realized-return cost, `0.08305` - weighted default/miscoverage reduction, and `0.435495` threshold reduction. - -The frozen upstream baseline remains retained for provenance and as the -declared return floor: - -- Run tag: `ijds-rebaseline-2026-06-07` -- Policy: `bound_aware_276k_economic_champion` -- Robust return: `$170,464.54` -- `V(alpha=0.01)`: `0.028875` -- `Gamma_CP(alpha=0.01)`: `0.187987` -- Exact pass: `True` -- Former robust region: `45/45` - -The older run tag `paper-thesis-final-economic-2026-04-06` is historical -provenance only. Do not use it as the active body claim. - -Never overwrite these frozen artifacts unless the user explicitly asks for a -champion rebuild: + +Use Windows PowerShell and `uv run`. Do not introduce Unix-only workflow +assumptions. + +## Active IJDS Policy + +- Run tag: + `champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6` +- Exact conformal replay: target `alpha=0.10`, frozen used alpha `0.095`. +- Decision score: `q=(p+u)/2`. +- Risk tolerance: `tau=0.17`. +- Objective: point-PD expected net return; conformal `q` is the risk guardrail. +- Selector: nine round-number policies on the temporal calibration holdout; + five satisfy full budget, effective-PD, and `0.60` threshold screens; no + outcome-derived selector columns. +- Full OOT: 276,869 candidates, 308 funded, `$179,327.59` realized return, + `0.039375` weighted default, `0.036875` weighted miscoverage. +- `Gamma_CP=0.176102`, `Gamma_residual=0.088051`, endpoint `0.258051`. +- Markov sensitivity: threshold `0.574279` with probability bound `0.316228` + under weighted funded-set validity. +- Matched point-PD: `$196,369.14`, `0.118400` weighted default, endpoint + `0.921317`, threshold `1.237545`. + +The exact paper-facing source is: + +`models/experiments/champion_reopen//portfolio/ijds_policy_governance.json` + +A35--A40 provide exact-alpha, selector, temporal, grade, bootstrap, and matched +comparison evidence. The final selector does not read OOT outcomes, but prior +project development inspected the static OOT corpus. Say "retrospective +lockbox replay," not "preregistered," "prospective," or "untouched holdout." + +Do not revive these as active claims: approximate alpha-0.01 scaling, `8/8`, +the 50,010-policy frontier, `0.345084`, capped/tail-selected policies, or OOT +outcome-selected hyperparameters. + +## Frozen Provenance + +Never overwrite the manifest-protected upstream or historical pool93 files: - `models/pd_canonical.cbm` - `models/pd_canonical_calibrator.pkl` @@ -82,121 +58,65 @@ champion rebuild: - `models/conformal_policy_status.json` - `data/processed/conformal_intervals_mondrian.parquet` - `data/processed/portfolio_bound_aware/rank1_alpha01_bound_aware_276k_full_2026-04-05-1734/` -- `reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv` -- `reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.csv` -- `reports/crpto/tables/crpto_tableA37_pool93_body_tail_risk.csv` -- `reports/crpto/tables/crpto_tableA38_pool93_body_cluster_bound_audit.csv` -- `reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.csv` -- `reports/crpto/tables/crpto_tableA40_pool93_point_baseline.csv` -- `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_frontier.json` -- `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_governance.json` -- `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_point_pd_baseline_audit.json` -- `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal/portfolio/pool93_ijds_claim_governance.json` +- historical `crpto_tableA35..A40_pool93_*` files +- historical pool93 governance JSON files - `EXTRACTION_MANIFEST.json` -Protected DVC stages: - -- Do not run without explicit permission: `crpto.portfolio.bound_exact_eval` - and any Optuna/HPO or policy search. -- Treat as protected revalidation stages: `crpto.pd.champion`, - `crpto.conformal.intervals`, `crpto.conformal.validation`, and - `crpto.portfolio.optimization`. If the user explicitly permits them, run them - as a drift check and validate hashes before/after. -- Safe paper stages: `crpto.paper.export_tables`, `crpto.paper.evidence`, - `crpto.paper.journal_package`, `crpto.paper.tail_satisficing_audit`, - `crpto.paper.figures`, `crpto.paper.spo_stability`, and `crpto.book.render`. - -If the user says a change may touch the champion, isolate the work: - -- Use a dedicated branch and a clear run label. -- Write new outputs under a new path; do not replace frozen champion files. -- Produce a drift report with expected tolerances. -- Run `just validate-champion` before and after. -- Do not update `EXTRACTION_MANIFEST.json` unless the user explicitly approves - a new frozen release. - -## Current Journal Scope - -The active paper scope is the IJDS pool93 certificate plus bounded diagnostics: - -- A19/Fig15: regret-auditability frontier in the main narrative. -- A35: promoted pool93 finite-grid return-bound frontier. -- A36: funded-set grade composition audit for the selected pool93 allocation. -- A37: pool93 selected-allocation LGD/CVaR/OCE tail-risk repricing. -- A38: pool93 selected-allocation cluster-bound sensitivity; Markov remains - the body theorem because the cluster thresholds are not tighter here. -- A39: fixed-allocation bootstrap diagnostic; it does not resample model, - solver, conformal intervals or policy search. -- A40: matched point-PD baseline with candidates and operating constraints fixed; - one frozen OOT trade-off, not a causal or universal-dominance claim. -- A20--A22: legacy tail-risk/OCE/CVaR diagnostic package retained in the - supplement, not as the promoted pool93 selector. -- A23--A24: multi-distribution/online coverage diagnostics, not universal - conditional-coverage claims. -- A25--A34: Prosper/Freddie external economic replication, not new Lending Club - champions. - -Do not re-run champion search, HPO, conformal interval generation or protected -portfolio stages to support this pack. Use existing artifacts unless the user -explicitly asks for a new isolated experiment with a claim target, evidence -gate, artifact sink and stop rule. - -## Objective Experiments - -OCE/CVaR/satisficing can be implemented as an isolated research objective or -scoring layer. The default rule is: - -- It may read existing allocations, predictions, intervals, or shortlist - artifacts. -- It may generate new diagnostic tables, figures, configs, tests, and docs. -- It must not replace the champion objective, rank-1 policy, or frozen outputs. -- A tail-satisficing challenger audit may read or re-score frozen allocations - under a new paper/audit stage if outputs are new and the status marks - `champion_promotion_changed=false`. It must not replace the A35 pool93 body - selector without a new promotion protocol. -- If used for a new search, store results under a new experiment path and make - the comparison explicit against the active pool93 body claim and the frozen - upstream rebaseline. - -## Submission Closeout - -For the current submission, keep these gates visible: - -- Consolidate A19/Fig15, A20--A40, paper/supplement, docs, manifest hashes and - `dvc.lock`. -- Sweep the manuscript for stale numbers, captions, body-vs-appendix placement, - and IJDS length. -- Convert the final `.qmd` into the official IJDS LaTeX template when the PDF - is ready for submission. -- Keep GitHub/DagsHub/MLflow disclosure compatible with double-anonymous - review: anonymize in the manuscript, reveal in cover letter or after review - according to venue policy. -- Create the reproducible release tag/bundle only after the final PDF passes. -- Treat `dvc push` as optional and credential-dependent. - -## Standard Checks - -Use focused checks while editing, then close with the strongest feasible set: +New experiments must use a distinct run tag and write under +`data/processed/experiments/champion_reopen/` and +`models/experiments/champion_reopen/`. Never replace frozen paths. + +Protected DVC stages are `crpto.pd.champion`, +`crpto.conformal.intervals`, `crpto.conformal.validation`, +`crpto.portfolio.optimization`, and `crpto.portfolio.bound_exact_eval`. +Experiments may read their outputs. Run a protected stage only with explicit +permission and a drift report. + +## Method Boundary + +The submitted method has one linear policy. Capped, tail, OCE/CVaR, SPO+, +multi-distribution, online, causal, and external-data variants are comparators +or diagnostics, not additional CRPTO methods. + +Keep these distinctions explicit: + +- exact conformal quantile replay versus approximate width scaling; +- point PD in the economic objective versus conformal `q` in the constraint; +- deterministic `weighted outcome <= B_u + V` versus the + assumption-conditional Markov statement; +- calibration-only final ranking versus historical OOT-aware development; +- full-OOT averages versus temporal heterogeneity; +- retrospective contrasts versus causal or universal dominance. + +## Paper Workflow + +Safe paper work may regenerate active A35--A40, figures, Quarto outputs, and the +official IJDS PDF. Keep body, supplement, submission TeX, governance JSON, and +claim-sync tests numerically aligned. + +Standard closeout: ```powershell -uv run python scripts/build_crpto_journal_package.py -uv run pytest tests/test_publication_targets.py -q -uv run pytest tests/test_scripts/test_build_crpto_journal_package.py -q -uv run pytest tests/test_pool93_body_claim_sync.py -q -just smoke +uv run python scripts/build_ijds_calibration_selected_evidence.py +uv run pytest tests/test_ijds_active_claim_sync.py -q just lint +just type-check +just type-advisory-full +just smoke just validate-champion just paper-submission +just paper-submission-official uv run dvc status --no-updates ``` -For a full milestone, also run `just test`, `just type-check`, and the relevant -Quarto render. Do not bypass hooks. +Run `just drift-gate` after changes to conformal or PD semantics. Do not bypass +hooks, commit secrets, or alter `EXTRACTION_MANIFEST.json`. -## Writing And Docs +## Writing -- Spanish for book, paper prose, and research notes. -- English for code, identifiers, CI, and docstrings. -- Do not reformat the whole book in one pass. -- Avoid broad refactors unless they directly reduce risk for the current task. -- Never commit secrets, `.env`, local credentials, or heavyweight raw data. +- Paper and code identifiers are English; project notes may be Spanish. +- Lead with data, method, decision, and managerial implication. +- Report the price of robustness and temporal failures as prominently as wins. +- Treat reproducibility as evidence quality, not as the sole novelty. +- Keep the main IJDS body within 25 pages; move proofs and diagnostics to the + separate supplement. diff --git a/AGENTS.md b/AGENTS.md index 9311ff3..6bdc9f6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,8 +10,8 @@ Lectura obligatoria, en este orden: 1. [`CLAUDE.md`](CLAUDE.md) — reglas de operación, champion congelado (esquema dual-tag pool93), stages prohibidos, comandos y convenciones. 2. [`docs/research/active_claims_2026-07-04.md`](docs/research/active_claims_2026-07-04.md) - — registro vigente de claims del paper (body point pool93, denominadores - finite-grid, reopen gate). + — registro vigente de claims del paper (replay exacto al 90%, selector de + calibración 3x3, política lineal 50/50 y reopen gate). 3. [`.codex/skills/crpto/SKILL.md`](.codex/skills/crpto/SKILL.md) — skill de Codex con el detalle operativo del certificado y sus artefactos. diff --git a/CLAUDE.md b/CLAUDE.md index 0097bbb..4845186 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -33,25 +33,30 @@ El "champion congelado" se refiere al **pipeline de búsqueda** que produjo las ## Champion congelado — NO RE-CORRER -Esquema dual-tag (detalle en `docs/SCOPE_AND_GOVERNANCE.md`). Pool93 es una -re-evaluación determinista de una grilla finita de políticas sobre los mismos -intervalos conformal congelados; **no regenera ningún artefacto upstream**. +El modelo PD, calibrador, intervalos y bundle pool93 del manifest permanecen +congelados. El body IJDS usa un replay exacto y una política nueva bajo un run +tag aislado; **no regenera ni sobreescribe ningún artefacto upstream**. -**Body claim del paper IJDS (pool93, activo):** +**Body claim del paper IJDS (activo):** | Campo | Valor | | --- | --- | -| Certificate tag | `champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2` | -| Source policy run | `champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext` | -| Policy mode | `capped_blended_uncertainty` (familia `claim_micro_ext_body_cap345`) | -| Retorno robusto | `$184,832.48` | -| V(α=0.01) | `0.035350` | -| Γ_CP(α=0.01) | `0.162616` | -| Γ_int / Γ_res (α=0.01) | `0.089032 / 0.073584` | -| Endpoint / Markov threshold (α=0.01) | `0.245084 / 0.345084` | -| Alpha grid | `8/8`, exceso realizado sobre τ `0.0` | -| Baseline A40 | costo de retorno `5.875%`; reducción default/V `8.305` pp | -| Evidencia | A35–A40 + JSONs de gobernanza/certificado pool93 | +| Run tag | `champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6` | +| Conformal | exact replay at target `alpha=0.10` (frozen used alpha `0.095`) | +| Policy | `q=(p+u)/2`, `tau=0.17`, point-PD economic objective | +| Selector | 9 round-number policies on the temporal calibration holdout; 5 eligible; no outcome-derived selector columns | +| Realized return | `$179,327.59` on a `$1M` budget | +| Weighted default / miscoverage | `0.039375 / 0.036875` | +| Gamma_CP / Gamma_residual | `0.176102 / 0.088051` | +| Endpoint / Markov threshold | `0.258051 / 0.574279` | +| Matched point-PD A40 | return cost `8.678%`; default reduction `7.9025` pp; threshold reduction `66.3266` pp | +| Evidence | exact alpha A35 + calibration selector A36 + temporal/funded-set/baseline A37--A40 | + +The exact policy-facing quantities come from +`models/experiments/champion_reopen//portfolio/ijds_policy_governance.json`. +The primary claim is the simple calibration-selected guardrail and its exact +funded-set audit. Markov remains an assumption-conditional sensitivity, not the +headline novelty. **Cadena upstream congelada (histórica; su retorno es el return floor declarado del pool93):** @@ -65,7 +70,8 @@ intervalos conformal congelados; **no regenera ningún artefacto upstream**. | Exact pass | `True` | | Región robusta | `45/45` | -Artefactos congelados cuyos hashes están en `EXTRACTION_MANIFEST.json` y **no se tocan** sin permiso: +Artefactos históricos congelados cuyos hashes están en +`EXTRACTION_MANIFEST.json` y **no se tocan** sin permiso: - `models/pd_canonical.cbm` - `models/pd_canonical_calibrator.pkl` @@ -80,7 +86,9 @@ Artefactos congelados cuyos hashes están en `EXTRACTION_MANIFEST.json` y **no s - `models/experiments/champion_reopen/...__pool93__ijds-certificate-semantics-v2/portfolio/pool93_point_pd_baseline_audit.json` - `EXTRACTION_MANIFEST.json` -La sincronía del body claim con el paper la vigila `tests/test_pool93_body_claim_sync.py`. +La sincronía del body claim activo con el paper la vigila +`tests/test_ijds_active_claim_sync.py`. `tests/test_pool93_body_claim_sync.py` +queda limitado a la integridad de procedencia histórica. Stages DVC que regeneran estos artefactos (`crpto.pd.champion`, `crpto.conformal.intervals`, `crpto.conformal.validation`, `crpto.portfolio.optimization`, `crpto.portfolio.bound_exact_eval`) **no se ejecutan** sin permiso. Validar con `crpto-validate-champion` antes de cualquier merge. diff --git a/EXTRACTION_MANIFEST.md b/EXTRACTION_MANIFEST.md index 4da809f..d67cfb1 100644 --- a/EXTRACTION_MANIFEST.md +++ b/EXTRACTION_MANIFEST.md @@ -9,13 +9,13 @@ exists, and how `tests/test_manifest_regression.py` enforces it. ## TL;DR - **Schema version**: 6 (top-level key `schema_version`). -- **Dual-tag governance**: - - frozen upstream baseline: `ijds-rebaseline-2026-06-07`; - - active IJDS certificate semantics: - `champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2`. +- **Manifest scope**: frozen upstream baseline plus the historical pool93 + promotion recorded at extraction time. The current IJDS manuscript policy is + newer and is governed separately by + `docs/research/active_claims_2026-07-04.md`. - **187 critical files** are hashed under `critical_hashes` (SHA256 + byte count). -- **Pool93 body claim**: return `$184,832.48`, `V(alpha=0.01)=0.035350`, +- **Historical pool93 claim**: return `$184,832.48`, `V(alpha=0.01)=0.035350`, `Gamma_CP=0.162616`, `Gamma_res=0.073584`, endpoint `0.245084`, exact Markov loss threshold `0.345084`, realized risk-tolerance excess `0.0`, and declared alpha-grid pass `8/8`. @@ -50,7 +50,7 @@ exists, and how `tests/test_manifest_regression.py` enforces it. | `generated_at_utc` | When the manifest was produced. | | `summary` | Free-text human description of the extraction scope. | | `champion_metrics` | Frozen upstream baseline numbers retained as provenance and as the declared return floor. | -| `pool93_ijds_promotion` | Active IJDS metadata for the policy-aware frontier, semantic audit, selected body point, and A40 matched baseline. | +| `pool93_ijds_promotion` | Historical IJDS metadata frozen at extraction time; retained as provenance, not the current manuscript claim. | | `critical_hashes` | Map `relative_path → {sha256, bytes, hash_source}` for every file the paper depends on. | | `validation_results` | Output of the extraction-time guardrail tests. | | `files` | Inventory of files copied/created during extraction. | diff --git a/README.md b/README.md index 5b8384c..ed16e40 100644 --- a/README.md +++ b/README.md @@ -8,22 +8,19 @@ Pipeline de investigación y libro Quarto que acompañan el paper **CRPTO**, una | Campo | Valor | | --- | --- | -| Certificate tag | `champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2` | -| Policy family | `claim_micro_ext_body_cap345` | -| Policy mode | `capped_blended_uncertainty` | -| Retorno robusto | **$184,832.48** | -| `V(α=0.01)` | `0.035350` | -| `Γ_CP(α=0.01)` | `0.162616` | -| `Γ_int / Γ_res` (`α=0.01`) | `0.089032 / 0.073584` | -| Endpoint / Markov threshold | `0.245084 / 0.345084` | -| Alpha-grid pass | `8/8` | -| Frontera consolidada | `50,010` políticas semánticas; `27,508` elegibles sobre el return floor | -| Baseline A40 | costo de retorno `5.875%`; reducción default/V `8.305` pp | +| Run tag | `champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6` | +| Conformal | replay exacto al `90%` (`alpha=0.10`, used `0.095`) | +| Política | `q=(p+u)/2`, `tau=0.17`; PD puntual en el objetivo y `q` en el guardrail | +| Selector | grilla redonda `3x3` en calibración; `5/9` elegibles; sin outcomes OOT | +| Retorno realizado | **$179,327.59** | +| Default / miscoverage ponderados | `0.039375 / 0.036875` | +| `Gamma_CP / Gamma_residual` | `0.176102 / 0.088051` | +| Endpoint / contabilidad observada / umbral condicional | `0.258051 / 0.294926 / 0.574279` | +| Baseline A40 | `$196,369.14`; costo de retorno `8.678%`; reducción de default `7.9025` pp | Hashes SHA256 de los artefactos críticos están en [`EXTRACTION_MANIFEST.json`](EXTRACTION_MANIFEST.json). Verifica con `just validate-champion` o el skill `/crpto-validate-champion`. -El rebaseline `ijds-rebaseline-2026-06-07` se conserva como upstream congelado -y return floor histórico (`$170,464.54`, `45/45`), no como el claim activo del -manuscrito IJDS. +El rebaseline y la frontera pool93 anterior se conservan como procedencia +congelada, no como claims activos del manuscrito IJDS. ## Requisitos del sistema @@ -67,6 +64,7 @@ just book-clean # borra _book/, _freeze/, .quarto/ # Pipeline de paper (no toca el champion) just paper-export # tablas + figuras + evidence + journal + libro +just ijds-evidence # A35--A40 y gobernanza de la política IJDS activa just tables # solo CSVs just figures # solo PNGs/PDFs @@ -75,12 +73,12 @@ just lint # ruff check + format check just fmt # ruff fix + format just type-check # mypy src scripts just type-advisory # ty sobre ruta activa IJDS, no bloqueante -just type-advisory-full # ty sobre src/scripts completos, deuda opcional/historica +just type-advisory-full # ty sobre src/scripts completos; bloquea el cierre IJDS just api-docs-core # pdoc local para modulos core, salida ignorada just hooks-check # valida hooks con pre-commit y prek just smoke # tests críticos rápidos just test # suite completa -just submission-check # cierre IJDS: claims, lint, type, smoke, champion y PDF oficial +just submission-check # cierre IJDS: claims, lint, type, suite completa, champion y PDF oficial # DVC just dvc-status # drift detection diff --git a/configs/crpto_publication_targets.yaml b/configs/crpto_publication_targets.yaml index af7752d..76f3c7c 100644 --- a/configs/crpto_publication_targets.yaml +++ b/configs/crpto_publication_targets.yaml @@ -36,11 +36,12 @@ primary_target: - "Freeze a release tag and reproducibility bundle after anonymity decision." current_paper_scope: include: - - "Pool93 finite-grid return-bound frontier as the active IJDS body claim." - - "Frozen upstream PD/calibration/conformal chain retained as provenance and return floor." - - "A3--A40 as supplement/appendix evidence generated from frozen artifacts." + - "Exact 90% conformal replay plus a calibration-selected 3x3 linear policy grid as the active IJDS body claim." + - "One interpretable policy, q=(p+u)/2 with tau=0.17, and a matched point-PD comparator." + - "Frozen upstream PD/calibration/conformal chain and historical pool93 frontier retained only as provenance." + - "A3--A34 as supporting diagnostics and A35--A40 as the active exact-alpha/selector/evaluation bundle." - "Regret-auditability frontier as the body-level SPO+/CRPTO trade-off." - - "OCE/CVaR and robust satisficing as diagnostics, not new selectors." + - "OCE/CVaR, robust satisficing, external replications, and SPO+ as diagnostics or comparators, not additional CRPTO methods." - "Cluster-aware dependence caveat/proposition in the theory supplement." - "Prosper/Freddie external economic replication without reopening the Lending Club champion." - "Quarto/DVC/DagsHub/MLflow companion after anonymity handling." @@ -62,34 +63,30 @@ journal_strengthening_pack: requires_new_run: false tail_risk_oce_cvar_diagnostic: status: include_supplement - role: "Tail-risk audit of legacy diagnostic surfaces plus A37 selected pool93 allocation repricing under LGD alternatives." + role: "Legacy OCE/CVaR sensitivity retained in the supplement; it does not select or redefine the active linear policy." artifacts: - reports/crpto/tables/crpto_tableA12_tail_risk_oce_cvar.csv - reports/crpto/tables/crpto_tableA20_tail_satisficing_challenger_audit.csv - - reports/crpto/tables/crpto_tableA37_pool93_body_tail_risk.csv requires_new_run: false - pool93_frontier_and_selected_allocation: + exact_alpha_calibration_selected_policy: status: include_body_and_supplement - role: "Active IJDS policy-aware certificate: A35 exact frontier in the body, A36--A39 selected-allocation audits, and A40 matched point-PD audit." + role: "Active IJDS evidence: exact alpha replay, nine-policy calibration selector, temporal/funded-set audits, bootstrap, and matched comparisons." artifacts: - - reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv - - reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.csv - - reports/crpto/tables/crpto_tableA37_pool93_body_tail_risk.csv - - reports/crpto/tables/crpto_tableA38_pool93_body_cluster_bound_audit.csv - - reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.csv - - reports/crpto/tables/crpto_tableA40_pool93_point_baseline.csv - - models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal/portfolio/pool93_ijds_claim_governance.json - - models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_frontier.json - - models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_governance.json - - models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_point_pd_baseline_audit.json + - reports/crpto/tables/crpto_tableA35_exact_alpha_grid.csv + - reports/crpto/tables/crpto_tableA36_calibration_policy_selector.csv + - reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.csv + - reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.csv + - reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.csv + - reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.csv + - models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6/portfolio/ijds_policy_governance.json requires_new_run: false matched_point_pd_baseline: status: include_body_and_supplement - role: "A40 matched Lending Club baseline: same candidates and operating constraints, point PD as the only decision-semantic change." + role: "A40 matched Lending Club baseline: same candidates, tau=0.17 and operating constraints; point PD replaces the conformal guardrail." artifacts: - - reports/crpto/tables/crpto_tableA40_pool93_point_baseline.csv - - reports/crpto/tables/crpto_tableA40_pool93_point_baseline.tex - - models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_point_pd_baseline_audit.json + - reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.csv + - reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.tex + - models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6/portfolio/ijds_policy_governance.json requires_new_run: false robust_satisficing_margins: status: include_supplement_or_short_body @@ -107,7 +104,7 @@ journal_strengthening_pack: requires_new_run: false tail_satisficing_challenger_audit: status: include_supplement - role: "Journal-only legacy audit that ranks tail-satisficing alternatives without promoting them over the pool93 body point." + role: "Journal-only legacy audit; it is not a selector for the active 50/50 linear policy." artifacts: - reports/crpto/tables/crpto_tableA20_tail_satisficing_challenger_audit.csv - models/crpto_tail_satisficing_audit_status.json diff --git a/configs/experiments/champion_reopen_ijds_calibration_selected_simple90_v6.yaml b/configs/experiments/champion_reopen_ijds_calibration_selected_simple90_v6.yaml new file mode 100644 index 0000000..6e98e4a --- /dev/null +++ b/configs/experiments/champion_reopen_ijds_calibration_selected_simple90_v6.yaml @@ -0,0 +1,45 @@ +schema_version: "2026-07-09.6" +run_tag: "champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6" + +source: + upstream_canonical_run_tag: "champion-reopen-2026-06-19__hpo-wave1__pool93__seed42" + conformal_results_path: "models/conformal_gap/champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1/conformal_results_mondrian.pkl" + conformal_intervals_path: "data/processed/conformal_gap/champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1/conformal_intervals_mondrian.parquet" + exact_alpha_grid_path: "data/processed/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1/conformal/exact_alpha_grid.parquet" + +design: + alpha: 0.10 + budget: 1000000.0 + max_concentration: 0.25 + lgd: 0.45 + period_order: ["2018H1", "2018H2", "2019H1", "2019H2", "2020+"] + combine_years_from: 2020 + markov_threshold_cap: 0.60 + selection_min_budget_utilization: 0.999 + selection_rule: "maximize expected point-PD objective on the calibration holdout under a 0.60 endpoint-plus-Markov screen, the effective-PD cap, and full budget use" + +policy_grid: + family: "simple_linear_conformal_guardrail" + risk_tolerances: [0.15, 0.17, 0.19] + gammas: [0.25, 0.50, 0.75] + uncertainty_aversions: [0.0] + +incumbent_policy: + risk_tolerance: 0.17 + gamma: 0.75 + uncertainty_aversion: 0.0 + +execution: + solver_backend: "highspy" + time_limit: 300 + threads: 1 + random_seed: 42 + +claim_boundary: >- + The final tagged policy rule ranks nine round-number candidates on the + calibration development block without reading default, realized-return, or + other outcome-derived selector columns, and then freezes the selected policy + before OOT evaluation. Conformal endpoints themselves use calibration labels, + as required. Earlier project development inspected this static OOT corpus, so + the evaluation is a transparent retrospective lockbox replay rather than a + pristine prospective trial, causal estimate, or live-deployment guarantee. diff --git a/configs/experiments/champion_reopen_ijds_exact_alpha_grid_v1.yaml b/configs/experiments/champion_reopen_ijds_exact_alpha_grid_v1.yaml new file mode 100644 index 0000000..8f447b4 --- /dev/null +++ b/configs/experiments/champion_reopen_ijds_exact_alpha_grid_v1.yaml @@ -0,0 +1,19 @@ +schema_version: "2026-07-09.1" +run_tag: "champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1" + +source: + upstream_canonical_run_tag: "champion-reopen-2026-06-19__hpo-wave1__pool93__seed42" + conformal_namespace: "champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1" + conformal_results_path: "models/conformal_gap/champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1/conformal_results_mondrian.pkl" + conformal_intervals_path: "data/processed/conformal_gap/champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1/conformal_intervals_mondrian.parquet" + +design: + alpha_grid: [0.01, 0.03, 0.05, 0.07, 0.10, 0.12, 0.15, 0.20] + alpha_mapping: "proportional frozen conservative ratio alpha_used/alpha_target from the selected 90% recipe" + replay_tolerance: 1.0e-12 + +claim_boundary: >- + Recomputes finite-sample Mondrian quantiles for every declared alpha while + freezing the selected partition, calibration fit/holdout split, score scale, + and holdout-learned widening factors. It is a retrospective OOT audit, not a + post-selection or live-deployment guarantee. diff --git a/docs/SCOPE_AND_GOVERNANCE.md b/docs/SCOPE_AND_GOVERNANCE.md index f69258e..349e6a6 100644 --- a/docs/SCOPE_AND_GOVERNANCE.md +++ b/docs/SCOPE_AND_GOVERNANCE.md @@ -25,30 +25,33 @@ CRPTO does not cover: - New model-training research unless it is explicitly isolated from the frozen champion or run under a new tag. -## Frozen champion contract - -The current IJDS paper-facing CRPTO body point is the promoted pool93 -finite-grid frontier closure: - -- certificate tag: `champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2` -- source policy run: `champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext` -- body/default policy mode: `capped_blended_uncertainty` (family - `claim_micro_ext_body_cap345`, selected from the consolidated frontier) -- robust return: `$184,832.48` -- `V(alpha=0.01)=0.035350` -- `Gamma_CP(alpha=0.01)=0.162616` -- `Gamma_internalized(alpha=0.01)=0.089032` -- `Gamma_residual(alpha=0.01)=0.073584` -- exact endpoint budget at `alpha=0.01`: `0.245084` -- exact Markov loss threshold at `alpha=0.01`: `0.345084` -- realized risk-tolerance excess: `0.0` -- declared alpha-grid pass: `8/8` -- main paper-facing artifacts: A35 finite-grid frontier, A36 funded-set grade - audit, A37 selected-allocation tail-risk repricing, A38 cluster-bound audit, - A39 fixed-allocation bootstrap diagnostic, and A40 matched point-PD audit. - -The previous IJDS rebaseline is retained as historical provenance, not as the -active body claim: +## Active paper contract + +The current IJDS body point is the simple calibration-selected 90% guardrail: + +- run tag: + `champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6` +- exact conformal replay: target `alpha=0.10`, frozen used alpha `0.095`; +- policy: `q=(p+u)/2`, `tau=0.17`, with point PD in the economic objective + and conformal `q` in the risk constraint; +- selector: nine round-number policies on the temporal calibration holdout, + five eligible under the `0.60` endpoint-plus-Markov screen, no + outcome-derived selector columns; +- realized return: `$179,327.59`; +- weighted default and miscoverage: `0.039375` and `0.036875`; +- `Gamma_CP=0.176102`, `Gamma_residual=0.088051`; +- endpoint budget `0.258051`; assumption-conditional Markov threshold + `0.574279`; +- paper artifacts: A35 exact-alpha audit, A36 calibration selector, A37 + temporal evaluation, A38 grade audit, A39 bootstrap, and A40 matched + comparisons. + +The selector is outcome-free with respect to OOT policy ranking, but earlier +project development inspected the static OOT corpus. The paper must describe +the result as a retrospective lockbox replay, not a pristine prospective trial. + +The previous IJDS rebaseline and pool93 frontier are retained as historical +provenance, not as active body claims: - run tag: `ijds-rebaseline-2026-06-07` - policy: `bound_aware_276k_economic_champion` diff --git a/docs/refactor/README.md b/docs/refactor/README.md index 4fed59d..b8d70ab 100644 --- a/docs/refactor/README.md +++ b/docs/refactor/README.md @@ -21,7 +21,7 @@ Each plan documents: | [`CONFORMAL_REFACTOR_PLAN.md`](CONFORMAL_REFACTOR_PLAN.md) | Yes (calibrator pickle) | Full public split executed 2026-06-13; `src.models.conformal` is now a package facade with strict-typed submodules. | | [`MAPIE_MIGRATION_PLAN.md`](MAPIE_MIGRATION_PLAN.md) | Yes (intervals parquet) | Runtime is already MAPIE 1.x and the drift report is green; protected reruns still require explicit approval. | | [`archive/FEATURE_CONFIG_PARQUET_PLAN.md`](archive/FEATURE_CONFIG_PARQUET_PLAN.md) | Yes (downstream stages) | Executed 2026-06-13 and archived; `feature_config.pkl` retired from the live DVC DAG and manifest. | -| [`ijds_tooling_refactor_lab_2026-07-08.md`](ijds_tooling_refactor_lab_2026-07-08.md) | No (tooling/refactor only) | Active and full `ty` advisory scopes are clean; `pyrefly` is experimental; `pdoc`/`prek` are optional local helpers before IJDS submission. | +| [`ijds_tooling_decisions_2026-07-09.md`](ijds_tooling_decisions_2026-07-09.md) | No (tooling/refactor only) | Final live contract: `uv`, Ruff, mypy, `ty`, pytest, `just`, DVC and hook validation; Pyrefly and Commitizen are not adopted. | Executed lanes now in `main`: diff --git a/docs/refactor/ijds_tooling_decisions_2026-07-09.md b/docs/refactor/ijds_tooling_decisions_2026-07-09.md new file mode 100644 index 0000000..c0038b5 --- /dev/null +++ b/docs/refactor/ijds_tooling_decisions_2026-07-09.md @@ -0,0 +1,122 @@ +# IJDS Tooling and Refactor Decisions - 2026-07-09 + +This is the final decision record for the IJDS code and manuscript workflow. +It replaces the iterative tooling lab from 2026-07-08, whose useful decisions +have been implemented. Scientific history remains in `docs/research/`; this +file describes only the live engineering contract. + +## Objective + +Keep one auditable route from frozen upstream artifacts to the submitted +claim. Prefer small, explicit modules and named commands over additional +frameworks, parallel implementations, or hidden manuscript-time computation. + +## Adopted tools + +| Tool | Role | Decision | +|---|---|---| +| `uv` | Python environment, lockfile, commands | Sole Python package/runtime interface. | +| Ruff | lint and formatting | Sole linter and formatter. | +| mypy | stable gradual type gate | Required by `just type-check`. | +| ty | fast independent type audit | Active and full scopes; full scope blocks submission closeout. | +| pytest | behavioral and claim-sync tests | Required for focused, smoke, and full suites. | +| just | named local workflow | Sole human-facing command menu. | +| DVC | frozen artifact lineage | Keep for scientific provenance; never use it as a general task runner. | +| pre-commit + prek | hook compatibility and fast config validation | Keep both checks; do not create a second hook policy. | +| pdoc | optional local API browsing | Ephemeral via `uv run --with pdoc`; no project dependency. | + +`ty` complements rather than replaces mypy. The active scope includes the +exact-alpha replay and calibration-selected policy modules. Its full clean +scope is useful as an independent submission check, while mypy remains the +stable repository contract. + +## Rejected additions + +| Tool | Decision | Reason | +|---|---|---| +| Pyrefly | Do not adopt | It duplicated type checking and produced substantially more migration noise than actionable signal. | +| Commitizen | Do not adopt | Commit-message automation does not improve scientific validity or the one-author release flow. | +| Permanent pdoc dependency | Do not adopt | Generated API pages are useful locally but are not a publication artifact. | +| A second task runner | Do not adopt | `just` already exposes the complete Windows-first workflow. | +| Automatic semantic versioning | Do not adopt | Run tags, Git commits, and evidence hashes are the relevant scientific identifiers. | + +## Live methodology path + +1. `src/models/conformal_alpha_grid.py` exactly replays the frozen 90% + intervals and reports the alpha sensitivity. +2. `src/optimization/policy_evaluation.py` uses point PD in the economic + objective and an effective PD only in the risk constraint. +3. `src/optimization/policy_selection.py` defines the nine-cell round-number + grid and rejects selectors containing outcome-derived columns. +4. `scripts/experiments/ijds_policy_support.py` owns shared alignment, solving, + and evaluation for the active challengers. +5. `scripts/build_ijds_calibration_selected_evidence.py` materializes A35-A40 + from versioned experiment outputs. Manuscript rendering does not solve or + retune portfolios. + +The active run is +`champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6`. +The exact-alpha run is +`champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1`. + +## Deliberate simplifications + +- One active policy family: `q=(p+u)/2`, `tau=0.17`, `gamma=0.50`. +- Point PD remains the economic objective; uncertainty is a feasibility + guardrail. +- One deterministic 3x3 calibration selector, with an outcome-column denylist. +- One A35-A40 active evidence bundle. +- One body, one supplement, and one official submission TeX source. +- No nested temporal selector, effective-PD objective branch, active cap/tail + variants, or manuscript-time optimizer. +- Historical A1-A34 tables remain diagnostics and provenance, not competing + active claims. + +## Named commands + +```powershell +just ijds-evidence +just ijds-active-replay +just lint +just type-check +just type-advisory-full +just smoke +just validate-champion +just drift-gate +just test +just submission-check +``` + +`just submission-check` is the ordinary release gate and includes the full +pytest suite. `just +ijds-active-replay` is intentionally separate because it recomputes exact +interval grids and solves experiment portfolios. + +## Compilation contract + +The official source first attempts `latexmk`. On the current Windows TinyTeX +installation its `runscript.tlu` wrapper may fail, so the documented robust +fallback is: + +```text +pdflatex -> bibtex -> pdflatex -> pdflatex +``` + +The first pass writes citation keys to `.aux`, BibTeX writes `.bbl`, and the +last two passes stabilize citations, cross-references, and pagination. This is +one compilation workflow, not three independent builds. + +## Drift policy + +Experimental drift is allowed only under a new run tag. Promotion requires: + +- no writes to manifest-listed or protected champion artifacts; +- an explicit scientific reason and comparator; +- claim-sync and publication-integrity tests; +- `just validate-champion`; +- `just drift-gate` when PD or conformal paths change; +- regenerated tables and a visually inspected PDF. + +Light drift is not itself a benefit. A challenger is promoted only when it +improves the submitted method or its defensibility enough to justify the extra +surface area. Otherwise it is removed from the live path. diff --git a/docs/refactor/ijds_tooling_refactor_lab_2026-07-08.md b/docs/refactor/ijds_tooling_refactor_lab_2026-07-08.md deleted file mode 100644 index 16f6782..0000000 --- a/docs/refactor/ijds_tooling_refactor_lab_2026-07-08.md +++ /dev/null @@ -1,419 +0,0 @@ -# IJDS tooling and refactor lab - 2026-07-08 - -## Decision - -This branch keeps the CRPTO workflow intentionally narrow for IJDS. The daily -path should optimize for claim integrity, reproducible paper outputs and low -maintenance, not for adopting every new Python tool. - -## Tooling decisions - -| Tool | Decision | Rationale | -| --- | --- | --- | -| `uv` | Keep as canonical environment/package runner. | Already matches the repo, lockfile and Windows-first workflow. Official docs position it as a fast Python package/project manager: . | -| `ruff` | Keep as formatter/linter gate. | It already replaces separate formatter/import/lint tools with one fast gate: . | -| `ty` | Use a pinned daily advisory and a blocking final full-scope gate. | `just type-advisory` keeps the active IJDS-safe surface visible without competing with `mypy`; `just type-advisory-full` is clean and now blocks `submission-check` on future diagnostics. Official docs: . | -| `pyrefly` | Do not gate before submission. Use only for targeted experiments. | `uvx pyrefly` works (`1.1.1`), but active-scope trial produced 56 diagnostics, mostly pandas/matplotlib inference noise. Pyrefly is stable and fast, but not yet lower-maintenance than `ty` for this repo: . | -| `pdoc` | Add optional local API-doc recipe only. | `just api-docs-core` builds ignored docs for core optimization/calibration/evaluation modules. Useful for technical inspection, not IJDS-critical. Docs: . | -| `prek` | Add compatibility validation, not a full migration. | `just hooks-check` validates the existing `.pre-commit-config.yaml` with both `pre-commit` and `prek`. `prek` is a fast drop-in alternative, but changing hook execution semantics before submission is unnecessary. Docs: . | -| `commitizen` | Do not adopt before IJDS submission. | Commitizen helps teams enforce conventional commits/changelogs, but CRPTO is single-author and the paper/release checklist matters more than semantic-version automation. Docs: . | - -## Implemented process simplifications - -- Added `scripts/run_ty_advisory.py` so pinned `ty==0.0.57` has two deterministic - scopes: active IJDS path and full repository debt. -- Updated `just type-advisory` to use the active scope. Current result: - `ty advisory clean` over 102 active files. -- Added `just type-advisory-full`; after the final cleanup pass it is also - clean over 131 files. Optional TabPFN/SPO/cuOpt dependencies now load through - explicit optional-import helpers, and retired generic search entrypoints fail - with actionable messages instead of unresolved imports. The previously noisy - pandas/PD/conformal typing issues in protected scripts were removed without - changing drift. -- Updated `just submission-check` to enforce the full `ty` scope now that it is - clean and cheap. `mypy` remains the stable contractual type gate; `ty` adds a - second fast regression check without introducing Pyrefly's duplicate noise. -- Added `just api-docs-core`; generated output lives in ignored - `reports/api-docs/`. -- Added `just hooks-check`; both `pre-commit validate-config` and - `prek validate-config` pass. -- Added `just complexity-report` as an explicit `radon` report over `src/` and - `scripts/`. It is intentionally a refactor radar, not a submission gate, - because remaining long scripts include historical/protected search - entrypoints that should not be rewritten before IJDS without a concrete - claim-risk reduction. -- Split the policy arithmetic inside - `src.optimization.portfolio_model.compute_effective_pd` into small helpers - for clipped deltas, quantiles, blending, and tail selection. The public API - and policy semantics are unchanged, but the main policy resolver no longer - appears in the complexity report. -- Split `src.evaluation.model_shift.interpret_model_shift` into named - structural-shift, predictive-degradation, shift-type, governance-posture and - p-value-note helpers. This keeps the MRM/governance semantics auditable and - removes the module from the C-or-higher complexity report. -- Split `src.models.conformal_tuning.shrink_group_multipliers` into a small - immutable shrink context plus private helpers for factor normalization, - interval application, metric calculation, constraint checks, candidate - generation, tie-breaking and reporting. The public API and greedy policy are - unchanged, but the conformal shrink step is now easier to audit against the - paper's coverage/fairness claims. -- Extracted the shared continuous-portfolio LP algebra in - `src.optimization.portfolio_model` so SciPy HiGHS and native highspy consume - the same constraint matrix, bounds and objective coefficients. This removes - duplicated budget/PD/purpose/slack construction. -- Rewired the native cuOpt adapter to consume that same shared LP component - builder instead of reconstructing budget, concentration, PD-cap and slack - rows locally. This keeps the optional GPU backend aligned with the canonical - CPU formulation and removes another D-level complexity hotspot. -- Split `src.models.optuna_tuning.train_catboost_tuned_optuna` from a single - F-level HPO orchestrator into explicit helpers for local/global search-space - materialization, feature-prior normalization, constraints, incumbent metrics, - objective evaluation, storage/study creation, trial enqueueing, trial - selection and final fit/refit. The CatBoost/Optuna defaults and public return - shape are preserved, while the HPO path is now much easier to inspect when - defending the frozen model recipe. -- Split `scripts.build_papers_tesis_deep_audit.write_audit` into small Markdown - section builders for inventory, paper-facing literature, extended thesis - lanes, visual curation, bibliography control and closeout. This keeps the - local literature audit regenerable without one large opaque memo writer. -- Split `scripts.generate_governance_status._build_explanation_drift_report` - into helpers for recent-period selection, segment construction, SHAP ranking, - feature PSI details and per-segment pass/fail rows. This clarifies the MRM - explanation-drift logic without changing the governance artifact contract. -- Split the `scripts.generate_governance_status` orchestrator into explicit - threshold/path dataclasses plus helpers for train/test loading, JSON sidecar - reads, drift metrics, model-shift interpretation and status serialization. - This keeps `governance_status.json`, `model_shift_status.json` and parquet - paths unchanged while removing the remaining E-level main function. -- Split `scripts.run_comparison._gate_ab_no_regression` into helpers for A/B - return extraction, current self-gate evaluation and baseline comparison - warnings. The gate still passes only on the self no-regression rule, with the - documented selective-ambiguity cross-scenario exception preserved. -- Split the remaining D-level comparison-report helpers in - `scripts.run_comparison`: artifact/status metadata now has explicit source, - observation, timestamp-skew and run-tag-coherence helpers, and comparison - report writing now separates gate execution, gate field extraction, quality - contract construction and JSON/Markdown emission. This makes the promotion - evidence easier to inspect without changing the gate semantics. -- Split `scripts.generate_crpto_figures._crpto_fig8_alpha_pareto` into helpers - for semantic column detection, variant styling, alpha sorting, tick labels and - annotation offsets. This keeps the IJDS alpha-sweep figure logic auditable - without changing the plotted data or the figure contract. -- Split `scripts.run_fairness_audit` so SHAP per-group interpretation, - CatBoost SHAP preparation, Fairlearn sidecar bootstrap summaries, primary - status construction and SHAP status writing are isolated helpers. This turns - the fairness audit script from a mixed I/O/analysis block into a clearer - pipeline while preserving thresholds, sidecar paths and JSON/parquet - contracts. -- Split `scripts.validate_conformal_policy` into helpers for config/sensitivity - loading, namespace application, alert fallback, valid interval extraction, - Winkler/MAPIE cross-checks, compensated Winkler policy handling, material - check construction and latest-month selection. The validation contract and - output schema stay the same, but the conformal promotion gate is now much - easier to audit against the IJDS coverage/width/group-coverage claims. -- Split `scripts.run_crpto_vs_spo_stability` into import-safe optional SPO - loading plus helpers for period masks, deterministic per-period sampling, - coverage aggregation, detail rows and summary JSON. The output contract stays - the same, but tests can now import the module without PyEPO/Torch, and period - sampling no longer depends on Python's randomized `hash()` seed. -- Split `scripts.select_economic_portfolio_policy` into explicit selector - settings, decision inputs, candidate evaluation, hard-filter eligibility, - A/B-like ranking, fallback construction and payload serialization helpers. - This preserves the champion-policy/status JSON contracts while making the - economic selector auditable as a sequence of declared gates instead of one - long mixed orchestration block. -- Split `scripts.simulate_ab_test._resolve_robust_policy` into champion-policy - selection, selected-policy normalization, robustness-summary validation, - summary row choice and default fallback helpers. This makes the A/B audit - policy precedence explicit: champion artifact first, summary second, fallback - last, with `explicit_champion_only` still failing loudly when the artifact is - absent. -- Split `scripts.benchmark_conformal_variants` into benchmark-data loading, - normalized search-space construction, variant accumulation, global/Mondrian/ - cross-conformal appenders, calibration-size sensitivity rows, final frame - assembly and artifact writing. The benchmark still writes the same parquet - and JSON surfaces, but the conformal experiment is now inspectable as stages - instead of one F-level orchestrator. -- Split `scripts.benchmark_pd_set_prediction` into set-benchmark data loading, - settings normalization, per-variant prediction, calibration-size sensitivity, - benchmark matrix assembly, slice summaries, promotion-gate calculation, - status payload construction and artifact writing. The binary set-prediction - sidecar remains a triage/abstention diagnostic, but its evidence path is now - explicit and no longer an E-level `main`. -- Split the high-risk paths in `scripts.train_pd_model`: config defaults, - CLI overrides, replay expectation checks, calibration backtests, Optuna seed - replay, tuned-CatBoost/HPO orchestration, decision-threshold resolution, - MAPIE statistical calibration tests, walk-forward diagnostics and SHAP export - now live behind named helpers. This preserves the training contracts while - making the PD replay and evidence path easier to audit. The script still has - a long main orchestrator, but it is now C-level instead of D/F-level and no - D-level helper remains. -- Split `scripts.generate_conformal_intervals` so feature resolution, - contract-matrix alignment, tuning-grid normalization, 90% Mondrian tuning - search, 90% coverage-floor/shrinkback evidence, optional global rebalance, - 95% alpha selection and final artifact-table/payload persistence are - explicit helpers. The conformal interval generator's `main` is now B-level: - it reads like the paper's certificate sequence instead of mixing tuning, - coverage policy and artifact-writing branches. -- Split `scripts.search.run_pool93_ijds_local_refinement` so the IJDS finite - policy-grid construction is organized by declared profile/family rather than - one F-level function. The candidate generator is now covered by per-profile - semantic-key fingerprints, including the terminal `37,068`-policy surface, - so future edits cannot silently change the paper-facing grid denominators. -- Split the `scripts.search.run_pool93_ijds_local_refinement` entrypoint into - parser, path, conformal-source, candidate, manifest, resume, pending-task, - progress persistence, serial/parallel execution and final-output helpers. - The candidate-grid fingerprints remain unchanged for every declared profile, - including the terminal `37,068`-policy surface, and no exact refinement run - was executed. -- Split `scripts.search.run_portfolio_bound_exact_eval` into explicit context - paths, exact-evaluation plan, resume/cache handling, pending-task iteration, - selection payload writing and final status helpers. This removes the D-level - `main()` from the exact finite-grid evaluator without running the protected - `crpto.portfolio.bound_exact_eval` search stage or changing artifact paths. -- Split `scripts.search.run_portfolio_bound_aware_search` so parser - construction, typed grid/execution state, run paths, budget profiles, - search-space payloads, selection context, frontier artifact writes, - frontier-only completion, external exact delegation, in-process exact - evaluation, success/failure cleanup and selection output writes are explicit - helpers. The protected search stage was not executed; the refactor clarifies - the finite-grid certificate plumbing and leaves no C-or-higher block in the - file. The targeted policy-family grid is table-driven so segment-tail - families are easier to audit. -- Added `src.optimization.certificate_semantics` as the code-level source of - truth for the eight IJDS alpha levels. Pool93 refinement, the bound-aware CLI - default and the regret-auditability portfolio command now consume the same - tuple/CSV contract. A sync test checks the code constant against the - paper-facing search profile and active claim registry. -- Split `scripts.search.run_regret_auditability_sandbox` so sandbox-local PD - config snapshots are assembled by feature/profile, model params, - Venn-Abers calibration, HPO/warm-start, validation, output paths, threshold - disablement and sandbox metadata helpers. The resumable command scheduler now - separates phase grouping, resume skips, launch, completion logging and - PD-phase winner selection. Command planning is now split into PD incumbent, - PD lane, conformal, portfolio and metrics builders; the former C(20) - `build_phase_commands` no longer appears in the C-level report. The portfolio - phase now defaults to the declared eight-level IJDS alpha grid instead of an - older seven-level exploratory grid. No sandbox commands or protected stages - were run. -- Split `scripts.search.run_conformal_reopen_search` so parser construction, - resume-vs-fresh phase-1 materialization, OOT confirmation and optional phase-2 - promotion are explicit helpers with small dataclass handoffs. This removes the - last live D-level search orchestrator without executing the reopen search or - touching frozen conformal artifacts. -- Split the phase-2 calibrator tournament inside - `scripts.search.run_conformal_reopen_search` into explicit helpers for method - normalization, progress-state writes, baseline metric fitting, degradation - gating, holdout candidate execution, candidate ranking and final OOT - confirmation. The phase-2 search no longer appears in the C-or-higher report, - and no reopen search was executed. -- Split `scripts.experiments.run_champion_claim_max_downstream._portfolio_command` - into base-command, frontier-option, execution-option and cuOpt-option - helpers. The downstream watcher remains an isolated experiment lane, but its - portfolio search command is now inspectable and covered for proxy-vs-exact - sampling, exact-python and cuOpt flags. -- Split the legacy Pyomo `solve_portfolio` wrapper into backend solving, - result extraction and termination-status helpers. After this pass, - `src.optimization.portfolio_model` no longer appears in the C-or-higher - complexity report. - -## Code refactor stance - -The useful pre-submission refactor lane is not a broad rewrite. It is: - -1. Keep `mypy` as the contractual gate. -2. Keep `ty` active scope clean so new IJDS-path issues stand out. -3. Convert pandas/Pyomo dynamic edges only where the change is local and tests - can cover it. -4. Keep full-scope historical/protected diagnostics visible through - `just type-advisory-full`, but do not install optional TabPFN/SPO/cuOpt - stacks unless an isolated experiment needs them. -5. Stop live-code complexity cleanup at this point: `src` and active `scripts/` - no longer have D-or-higher radon findings. The only remaining D-level report - is in `scripts/archive/`, so further pre-submission refactors should happen - only when they reduce a concrete claim-risk or maintenance burden. - -## Current validation evidence - -- Focused `ruff` and `mypy` checks passed for edited modules. -- Focused tests passed for conformal adapters, calibration pickle compatibility, - TabPrep challengers, MLflow tracing, MRM report generation and the new - `ty` wrapper. -- Focused policy/portfolio tests pass, including exact regression checks for - segment-tail and segment-relative-tail effective-PD semantics. -- Focused model-shift tests pass, including structural-only, predictive-only, - mixed and stable governance postures. -- Focused conformal-tuning tests pass, including new regression coverage for - temporal-factor shrinkage and the initial-infeasible report path. -- Focused portfolio tests pass for sparse HiGHS vs Pyomo equivalence, native - highspy vs sparse HiGHS equivalence, native fallback behavior and the PD - slack/min-budget case. -- Focused cuOpt adapter tests pass with a fake cuOpt API, covering shared LP - matrix handoff, solver settings, generated log files, allocation payloads, - PD slack and non-feasible termination handling without requiring RAPIDS on - Windows. -- Focused PD-model tests pass, including small real CatBoost/Optuna runs for - tuned-vs-default predictions and local-refine materialization. -- Focused literature-audit memo tests pass for editorial sections, experiment - rows, bibliography-status counts and the no-champion-change boundary. -- Focused governance tests pass for overall/grade explanation-drift rows, - insufficient-support empty reports and the public governance-status summary, - checks and artifact-path contract. -- Focused run-comparison tests pass for ordinary A/B no-regression and the - selective-ambiguity cross-gate exception, plus artifact metadata coherence - and the causal/CATE insights-only run-tag mismatch exception. -- Focused CRPTO figure tests pass for alpha-sweep column detection, variant - labels/colors, alpha sorting, tick labels and annotation offsets. -- Focused fairness-audit tests pass for threshold resolution, auto-selected - decision policy writing, SHAP categorical detection/fill behavior and - per-group SHAP driver summaries. -- Focused conformal-policy validation tests pass for MAPIE current/legacy MWI - signatures, valid interval extraction, compensated Winkler gates, material - status JSON fields, official-baseline run-tag fallback, artifact namespaces - and sensitivity overrides. -- Focused CRPTO-vs-SPO stability tests pass for artifact presence, - deterministic per-period sampling seeds and summary/detail aggregation. -- Focused economic-selector tests pass for robust promotion, fallback, - breadth-aware v2 selection, A/B-like v3 ranking and breadth hard filters. -- Focused A/B policy-resolution tests pass for guardrail champion priority, - summary fallback when no champion artifact exists and explicit champion-only - missing-artifact failure. -- Focused conformal-variant benchmark tests pass for namespaced shadow output - paths and search-space normalization/deduplication. -- Focused PD set-prediction tests pass for namespaced shadow output paths, - settings normalization/fallback coercion and the guardrail promotion gate. -- Focused PD training config tests pass for CLI/replay overrides, feature - resolution, split loading/sampling and the new Optuna replay gate-tier - ranking contract, plus walk-forward stage normalization and SHAP summary - export. -- Focused conformal interval CLI tests pass for tuple parsers, tuning-grid - normalization, tuning candidate counts, split materialization, global - rebalance no-op behavior, 95% alpha tie-breaking, tuning-selection - materialization, learned floor-policy application, temporal-segment - eligibility and final artifact-table metadata preservation. -- Focused pool93 local-refinement tests pass for all declared candidate-grid - profiles (`stage1`, `expanded`, `claim_expanded`, `claim_micro`, - `claim_micro_ext`, `claim_bound_closure`, `claim_bound_floor_closure`, - `claim_bound_terminal`) using stable semantic-key fingerprints and for the - finite-grid claim-summary protocol. Additional helper tests cover manifest - path coherence and pending candidate-alpha task construction after the - entrypoint split. `mypy` is clean for - `scripts/search/run_pool93_ijds_local_refinement.py`, and the former D-level - `main()` no longer appears in the C-or-higher report. -- Focused exact-eval tests pass for completed-cache reuse, partial-cache - resume filtering, full-universe seed deduplication, alpha-grid payload - normalization and priority-context ordering. `mypy` is clean for - `scripts/search/run_portfolio_bound_exact_eval.py`, and `radon` reports the - exact-eval `main()` as A-level with no C-or-higher blocks. -- Focused bound-aware search tests pass for shortlist preservation, exact - aggregation ranking, table-driven policy-grid order, budget-profile parsing, - shared alpha-grid defaults, separated proxy/exact sampling, exact-work counts - and selection-context path/search-space coherence. `mypy` is clean for - `scripts/search/run_portfolio_bound_aware_search.py`, and `radon` reports no - C-or-higher blocks in that file. -- Focused champion-reopen orchestration tests pass for paper-facing downstream - candidate selection and the portfolio command builder, including separated - proxy/exact sampling plus cuOpt option propagation. `mypy` is clean for - `scripts/experiments/run_champion_claim_max_downstream.py`, and its former - D-level `_portfolio_command` no longer appears in the C-or-higher report. -- Focused regret-auditability sandbox tests pass for protected-output rejection, - sandbox lane materialization, PD snapshot writing, external output-dir - command planning, declared alpha-grid propagation, phase grouping, resume - skip behavior and validation-policy scaling. `mypy` is clean for - `scripts/search/run_regret_auditability_sandbox.py`; its former D-level - `write_pd_config_snapshot` and `_run_commands` plus the former C-level - `build_phase_commands` no longer appear at those thresholds. -- Focused conformal-reopen tests pass for phase-2 design fallback, resume - source-path preservation, OOT confirmation ranking, phase1-only phase-2 skip - behavior, explicit calibrator metric-gate skips and final phase-2 candidate - ranking/confirmation. `mypy` is clean for - `scripts/search/run_conformal_reopen_search.py`, and `radon` reports no - D-or-higher blocks in that file. -- `uvx radon cc src -s -n D` returns no findings after the conformal tuning, - portfolio, cuOpt and Optuna refactors. -- `uvx radon cc scripts/run_comparison.py -s -n D` returns no findings after - the comparison metadata/report split. -- `uvx radon cc scripts/generate_crpto_figures.py -s -n D` returns no findings - after the alpha/Pareto figure refactor. -- `uvx radon cc scripts/run_fairness_audit.py -s -n D` returns no findings - after the fairness SHAP/Fairlearn sidecar split. -- `uvx radon cc scripts/validate_conformal_policy.py -s -n D` returns no - findings after the conformal validation split. -- `uvx radon cc scripts/generate_governance_status.py -s -n D` returns no - findings after the governance status orchestration split. -- `uvx radon cc scripts/run_crpto_vs_spo_stability.py -s -n D` returns no - findings after the optional-dependency and aggregation split. -- `uvx radon cc scripts/select_economic_portfolio_policy.py -s -n D` returns - no findings after the selector orchestration split; `main` is now A-level. -- `uvx radon cc scripts/simulate_ab_test.py -s -n D` returns no findings after - the robust-policy resolver split. -- `uvx radon cc scripts/benchmark_conformal_variants.py -s -n D` returns no - findings after the benchmark orchestration split. -- `uvx radon cc scripts/benchmark_pd_set_prediction.py -s -n D` returns no - findings after the set-prediction sidecar split. -- `uvx radon cc scripts/train_pd_model.py -s -n D` now returns no findings; - `main` is C(19) and all helper functions are below D-level after the - PD-training orchestration split. -- `uvx radon cc scripts/generate_conformal_intervals.py -s -a` now reports - average complexity A after the conformal generator split; `main` dropped - from 83 to B(9). The remaining C-level logic is localized in - `_build_90_interval_evidence` and `_load_conformal_inputs`. -- `uvx radon cc scripts/search/run_pool93_ijds_local_refinement.py -s -n C` - no longer reports `_generate_candidate_grid` or `main` as D-level; candidate - generation, claim summarization and entrypoint flow are helperized and - fingerprint-tested. -- `just complexity-report` now reports only - `scripts/archive/search/monitor_regret_auditability.py::render` at D-level; - the active `src/` and `scripts/` surfaces are clear of D-or-higher findings. -- `just type-advisory` passes clean. -- `just type-advisory-full` passes clean; the latest report is written to - `reports/ci/ty-advisory-full.txt`. -- `just drift-gate` stayed bit-exact after touching PD/conformal scripts: - max absolute diffs for predictions, intervals and score-band edges were - `0.000e+00`. -- `just submission-check` passes with the full `ty` advisory, body/supplement - Quarto renders, and the official IJDS LaTeX fallback build; the current - official PDF has 28 pages, References begin on page 24, and the build is - citation/reference clean. -- `just test` passes. - -## Remaining caution - -This branch touched `src/models/optuna_tuning.py`, conformal adapter code and -the conformal tuning shrink path in small interface/refactor-only ways. The -latest `just drift-gate` stayed bit-exact, but keep it in the promotion -checklist alongside the standard submission gates because these modules sit -close to the paper's certificate. - -## Portfolio input and baseline semantics audit (2026-07-09) - -- Added `src/optimization/input_alignment.py` as the single alignment contract - for the canonical and trade-off portfolio entrypoints. It enforces one-to-one - ID or `_row_number` matching, preserves interval-origin columns, rejects - duplicate/missing keys, and makes positional sampling reproducible over the - full universe. -- Replayed canonical alignments at 17, 5,000 and 276,869 rows. ID and - `pd_high_90` fingerprints were bit-identical before and after the refactor in - both entrypoints. -- Added `portfolio_model.solution_allocation_vector` as the validated contract - for dense and sparse solver payloads. Migrated the primary optimizer, - trade-off wrapper, economic selector, evidence audit, alpha--gamma validator, - body-allocation audit and pool93 refinement. Their previous all-row indexing - or local fallback copies were incompatible or redundant under the modern - sparse solver payload. -- Corrected `robust=False` semantics: it now forces `point_estimate`, `gamma=0` - and point PD in the optimization constraint. Previously an endpoint override - had precedence, so the stored `nonrobust` baseline was still constrained by - `pd_high`. -- The preliminary read-only comparison at `tau=0.175` was superseded by the - matched A40 audit: same 276,869-candidate universe, `$1M` budget, - `tau=0.1715`, concentration and LGD contracts, and solver settings. Point-PD - earns `$196,369.14`; the selected CRPTO allocation earns `$184,832.48`, a - cost of `$11,536.66` (`5.875%`) alongside an 8.305 pp reduction in realized - weighted default and a 43.55 pp reduction in the exact loss threshold. The - historical signed-price interpretation and preliminary unmatched comparison - are retired from active paper surfaces; protected historical tables remain - provenance. -- Focused tests cover source-column preservation, deterministic sampling, - key-integrity failures, wrapper parity, effective nonrobust policy metadata, - and dense/sparse allocation payloads. diff --git a/docs/research/README.md b/docs/research/README.md index 9b572aa..567aa29 100644 --- a/docs/research/README.md +++ b/docs/research/README.md @@ -9,9 +9,11 @@ perenne y lo que el código lee/escribe. ## Registros activos (referenciados por el código o el paper) - `active_claims_2026-07-04.md` — source-of-truth operativo del claim IJDS - activo: dual-tag governance, punto pool93, semántica de grilla finita, - denominadores de frontera y stop rules para no reabrir búsquedas sin un - claim que pueda cambiar. + activo: replay conformal exacto al 90%, selector de calibración 3x3, + política lineal 50/50, evidencia A35--A40 y stop rules. +- `ijds_exact_alpha_calibration_selection_2026-07-09.md` — closeout que + documenta por qué se retiró el alpha-0.01 aproximado y cómo se eligió la + política simple sin outcomes OOT en el selector final. - `crpto_p1_evidence_2026-05-04.md` — evidencia P1 alrededor del champion congelado (escrito por `scripts/analyze_crpto_evidence.py`). - `crpto_journal_package_2026-05-04.md` — tablas A12–A34 y figuras journal @@ -51,9 +53,8 @@ perenne y lo que el código lee/escribe. local para nuevas referencias IJDS: contextual optimization, incertidumbre de credit scoring, conformal no-exchangeable, post-selection y comparadores decision-calibrated 2026. -- `pool93_certificate_semantics_v2_2026-07-09.md` - auditoría consolidada de la - descomposición policy-aware, corrección exacta de la frontera A35 y baseline - point-PD emparejada A40; reemplaza el memo preliminar de baseline. +- `pool93_certificate_semantics_v2_2026-07-09.md` - auditoría histórica de la + frontera pool93 y su baseline; conserva procedencia, no claims activos. ## Registros de gobernanza (decisiones; no se re-ejecutan sin permiso) diff --git a/docs/research/active_claims_2026-07-04.md b/docs/research/active_claims_2026-07-04.md index 9e77c51..2107f46 100644 --- a/docs/research/active_claims_2026-07-04.md +++ b/docs/research/active_claims_2026-07-04.md @@ -1,219 +1,160 @@ # CRPTO Active Claim Registry - 2026-07-09 -This registry is the current source of truth for paper-facing CRPTO claims. It -supersedes older research notes that centered the `45/45` local region or the -`paper-thesis-final-economic-2026-04-06` run as the active manuscript result. -Those notes remain useful as provenance, not as active operating instructions. - -## Active Body Claim - -CRPTO is an auditable conformal-robust credit-portfolio decision certificate: -a frozen calibrated PD model feeds Mondrian conformal upper endpoints, an -exact finite policy-grid portfolio search exposes a return-bound frontier, and -the selected funded set is audited on the full OOT universe. - -Current body/default point: - -- terminal run tag: - `champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal` -- active certificate-semantics tag: - `champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2` -- body point source run: - `champion-reopen-2026-06-19__pool93__ijds-claim-micro-ext` -- policy family: `claim_micro_ext_body_cap345` -- policy mode: `capped_blended_uncertainty` -- risk tolerance: `0.1715` -- gamma: `0.5475` -- delta cap quantile: `0.975` -- uncertainty aversion: `0.05` -- realized return on a 1M budget: `$184,832.48` -- return-floor surplus: `$14,367.94` -- `V(alpha=0.01)`: `0.035350` -- `Gamma_CP(alpha=0.01)`: `0.162616` -- `Gamma_internalized(alpha=0.01)`: `0.089032` -- `Gamma_residual(alpha=0.01)`: `0.073584` -- exact endpoint budget at `alpha=0.01`: `0.245083866` (`0.245084` paper rounding) -- exact Markov loss threshold at `alpha=0.01`: `0.345083866` (`0.345084` paper rounding) -- realized risk-tolerance excess: `0.0` -- declared alpha-grid pass: `8/8` -- fixed-allocation bootstrap return interval: - `$167,963.20`--`$198,650.47` - -Do not describe this as a newly retrained champion. The active pool93 claim is a -deterministic policy-grid re-evaluation over the same frozen upstream PD model, -calibrator and conformal interval outputs. - -## Active Evidence - -| Claim | Decision | Destination | Evidence | Stop Rule | -|---|---|---|---|---| -| CRPTO is a decision certificate, not a classifier leaderboard. | Promote | body | `paper/CRPTO_ijds.qmd`, Figure 1, exact certificate table, A35 | Do not reopen unless the decision certificate changes. | -| The pool93 body point is selected from a finite exact return-bound frontier. | Promote | body + A35 | `crpto_tableA35_pool93_ijds_frontier.csv`, certificate-semantics-v2 frontier/governance JSON | Do not run more portfolio search unless a new result can lower the exact threshold at the same return or materially lift return under the declared threshold. | -| The selected allocation has inspectable business composition and tail profile. | Append | supplement A36--A39 | A36 grade audit, A37 LGD/CVaR/OCE repricing, A38 cluster-bound audit, A39 bootstrap | Diagnostics only; do not use as hidden selector. | -| The conformal decision has a matched point-PD baseline. | Promote | body + supplement A40 | A40 table and `pool93_point_pd_baseline_audit.json` | Treat as one frozen OOT trade-off; do not claim causal or universal dominance. | -| The former `45/45` rebaseline remains provenance and return floor. | Archive/Append | provenance/supplement | `EXTRACTION_MANIFEST.json`, `ijds_rebaseline_2026-06-07.md` | Do not use as active headline except to explain the declared floor. | -| External Prosper/Freddie runs support recipe transfer. | Append | body short paragraph + supplement A25--A34 | external replication tables and figures | Do not promote as new Lending Club certificates. | - -## Finite-Grid Semantics - -The declared alpha grid is: - +This file is the source of truth for IJDS-facing claims. The active result is a +simple, calibration-selected 90% conformal guardrail. Older pool93 frontier +files remain immutable provenance under `EXTRACTION_MANIFEST.json`; they are no +longer evidence for the manuscript's main claim. + +## Active Decision + +- Run tag: + `champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6` +- Conformal target: `alpha = 0.10`; frozen conservative alpha used by the + recipe: `0.095`. +- Partition: five score-quantile Mondrian cells on calibrated PD (the frozen + artifact retains the historical label `score_decile_mondrian`). +- Decision score: `q_i = p_i + 0.50 (u_i - p_i) = (p_i + u_i) / 2`. +- Portfolio risk tolerance: `tau = 0.17`. +- Economic objective: expected point-PD net return, `c_i - p_i L`, with + `L = 0.45`. Conformal `q_i` enters the risk constraint, not the objective. +- Budget: `$1,000,000`; maximum concentration: `0.25`. + +The policy is selected from nine round-number candidates: +`tau in {0.15, 0.17, 0.19}` crossed with +`gamma in {0.25, 0.50, 0.75}`. The final tagged selector uses the temporal +calibration holdout only, requires full budget use, enforces the effective-PD +cap and `B_u + sqrt(0.10) <= 0.60`, and maximizes expected point-PD objective. +Five of nine candidates are eligible; the selected policy is +`tau = 0.17, gamma = 0.50`. + +The conformal recipe uses `142,550` calibration-fit rows. Policy selection is +performed on a later `35,638`-row calibration holdout covering November and +December 2017. The policy-ranking artifact contains no defaults, realized +returns, miscoverage, or other outcome-derived selector columns. Conformal +endpoints themselves use calibration labels, as required. + +## Full OOT Result + +The fixed policy is evaluated on `276,869` loans from January 2018 through +September 2020: + +| Quantity | Value | +|---|---:| +| Funded loans | `308` | +| Allocated budget | `$1,000,000` | +| Expected point-PD objective | `$168,271.56` | +| Realized return | `$179,327.59` | +| Weighted default rate | `0.039375` | +| Weighted miscoverage `V` | `0.036875` | +| Weighted point PD | `0.081949` | +| Weighted decision score | `0.170000` | +| `Gamma_CP` | `0.176102` | +| `Gamma_internalized` | `0.088051` | +| `Gamma_residual` | `0.088051` | +| Endpoint budget `B_u` | `0.258051` | +| Observed accounting bound `B_u + V` | `0.294926` | +| Markov event threshold `B_u + sqrt(alpha)` | `0.574279` | + +The fixed-allocation bootstrap return interval is +`$162,706.17`--`$193,924.74` (`5,000` draws). It resamples funded-loan +contributions only; it does not resample the model, conformal recipe, selector, +or optimizer. + +## Matched Baseline + +The point-PD comparator uses the same `276,869` candidates, budget, +concentration cap, `tau = 0.17`, LGD, solver, and point-PD economic objective. +It earns `$196,369.14`, funds `225` loans, and has weighted default `0.118400`, +miscoverage `0.041900`, endpoint budget `0.921317`, and Markov threshold +`1.237545`. + +Relative to that comparator, selected CRPTO gives up `$17,041.55` (`8.678%`) +of realized return, reduces weighted default by `7.9025` percentage points, +reduces weighted miscoverage by `0.5025` percentage points, and lowers the +endpoint-plus-Markov threshold by `66.3266` percentage points. These are +retrospective OOT contrasts, not causal effects or universal dominance. + +A more conservative `gamma = 0.75` comparator earns `$172,939.50`, with +weighted default `0.035875` and threshold `0.516624`. It shows the remaining +within-CRPTO trade-off: the selected 50/50 policy earns `$6,388.08` more at +`0.35` percentage points more weighted default and a `0.057655` higher +threshold. + +## Exact Alpha Evidence + +The exact replay reproduces the stored 90% reference intervals to numerical +precision (`max abs error <= 6.67e-16`). Other alpha levels are sensitivity +rows under the same frozen widening recipe, not separately selected policies. +The declared sensitivity grid is `A = {0.01, 0.03, 0.05, 0.07, 0.10, 0.12, 0.15, 0.20}`. -Therefore, the maximum possible alpha-grid pass for a single policy under the -current evidence bundle is `8/8`. A different maximum would require a new, -explicitly declared alpha grid and regenerated exact validation outputs. - -For the terminal endpoint run: - -- `n_policies = 37,068` -- `|A| = 8` -- maximum exact candidate-alpha checks: - `37,068 * 8 = 296,544` -- observed completion: `296,544/296,544` -- all-alpha passers: `37,068/37,068` -- all-alpha passers above the declared return floor: `14,814/37,068` - -For the consolidated frontier: - -- raw rows: `51,678` -- duplicate semantic rows removed: `1,668` -- maximum deduplicated semantic policies in this consolidated evidence file: - `50,010` -- eligible all-alpha above-floor policies: `27,508/50,010` -- nonpass or below-floor policies: `22,502/50,010` - -The v2 policy-aware rehydration uses the stored exact endpoint budget instead of -the linear-only residual shortcut. It changes neither denominator nor the body -selection, but changes 10,423 policy thresholds materially. Of these, 2,866 -tail/segment-tail policies were understated; the maximum understatement was -`0.241324`, and 716 policies formerly labeled at or below `0.50` exceed `0.50` -on the exact endpoint scale. The max-return endpoint is therefore `0.697056`, -not the retired linear-shortcut value. - -These denominators are finite-grid denominators, not continuous optimality -claims. If a later, separately tagged run adds new policy families, gamma -values, alpha levels or solvers, the denominators can grow under that run tag; -they must not be mixed with the current frozen denominators. - -## How To Present The Denominators - -The result should be presented as a finite-grid decision certificate, using the -same style that robust optimization and conformal risk-control papers use for -tunable risk levels: state the declared grid, report the denominator, expose the -trade-off frontier, and separate the selected body point from endpoints. - -Recommended body wording: - -> The selected policy is chosen from a declared finite policy-grid frontier. The -> consolidated frontier contains 50,010 deduplicated semantic policies, of which -> 27,508 both pass every declared alpha level and exceed the return floor. The -> terminal exact endpoint search evaluates 37,068 policies across eight alpha -> levels, for 296,544 candidate-alpha checks, and all terminal policies pass the -> all-alpha audit. These counts certify the declared finite search surface; they -> are not a continuous global-optimality claim. - -Use `pool93` only for run tags, file names, governance JSONs, and internal -provenance. The body manuscript should say selected policy, selected decision, -or declared finite-grid frontier unless the run label is needed to disambiguate -an evidence source. - -Report the screenshot numbers with explicit denominators: - -- `alpha_grid_pass = 8/8`: one selected policy passed all eight declared alpha - levels. The current maximum is eight because the alpha grid has eight - levels. -- `50,010` semantic policies: the maximum deduplicated policy denominator in - the current consolidated frontier. It equals `51,678` raw rows minus `1,668` - duplicate semantic policies. -- `27,508` eligible all-alpha above-floor policies: the number of consolidated - semantic policies that satisfy both gates: all-alpha pass and nonnegative - surplus over the declared return floor. -- `37,068/37,068` terminal all-alpha passers: every policy in the terminal - endpoint search passed all eight declared alpha checks. -- `296,544/296,544` exact terminal checks: the run completed all - `37,068 * 8` policy-alpha evaluations. This is primarily a completion - denominator; because all 37,068 policies are all-alpha passers, it also means - there were no failed alpha cells inside the terminal surface. - -Do not present these as: - -- proof over all possible policy hyperparameters; -- proof over all possible alpha values in `(0, 1)`; -- a live-production coverage guarantee after adaptive policy selection; -- evidence that more policies are always better. - -## Baseline Semantics Boundary - -The frozen Lending Club field `price_of_robustness=-10.56%` is historical -provenance, not an active IJDS claim. Its stored `nonrobust` solve inherited an -endpoint constraint and therefore was not a point-PD comparator. A40 replaces -that field with a matched two-stage LP at the selected policy's `tau=0.1715`, -holding 276,869 candidates, budget, concentration, LGD, solver, and operating -constraints fixed. The point-PD allocation earns `$196,369.14`; selected CRPTO -earns `$184,832.48`, a cost of `$11,536.66` (`5.875%`). CRPTO reduces weighted -default/miscoverage by `0.08305` and the exact Markov threshold by `0.435495`. -See `pool93_certificate_semantics_v2_2026-07-09.md`. - -This correction does not alter the selected pool93 allocation, its realized -return, `V`, `Gamma_CP`, exact Markov threshold, zero realized risk-tolerance -excess, alpha-grid pass, or finite-grid denominators. The active Lending Club -comparison is A40, interpreted jointly with the A35 exact return--bound -frontier. Frozen Table 0/Table 1/A2 fields remain untouched for -manifest provenance and must not be cited as evidence of robust dominance over -a point estimate. Historical A/B proxy flags that inherited that comparator are -also non-promoted. - -The IJDS framing should emphasize data + methodology + decision + implication: -the finite-grid frontier is the decision object, the exact checks are the -auditable computation, and the endpoints expose the price of robustness. +Do not reinstate the former `alpha = 0.01` headline. Its exact intervals have +average width `0.9882`, and `93.54%` of OOT upper endpoints equal one. That +setting is nearly vacuous for portfolio discrimination. The active method uses +the conventional 90% level because it is the recipe's selected and exactly +replayed reference level. ## Theory Boundary -The paper-facing theorem uses deterministic accounting plus a distribution-free -Markov step under weighted funded-set validity. The body keeps Markov because -it is the weakest defensible assumption for the current selected allocation. -A38 reports cluster-aware thresholds as sensitivity; none is tighter than -Markov for the observed exposure concentration. - -Every paper-facing policy now uses the policy-aware decomposition -`Gamma_CP = Gamma_internalized + Gamma_residual`, with exact endpoint budget -`B_u = sum(w*q) + Gamma_residual`. The shortcut -`Gamma_residual = (1-gamma) * Gamma_CP` is valid only for a pure linear blend. -It remains numerically valid for the selected capped policy because its row-level -cap is inactive on all 314 funded rows, but it must not be applied to tail or -segment-tail policies. - -Do not claim: - -- universal conditional coverage; -- global optimum over continuous policy parameters; -- future live-deployment validity; -- a CVaR/OCE/bootstrap-selected champion; -- that `8/8` is an external standard rather than the declared grid. - -Literature-informed boundary added after the 2026-07-08 corpus scan: - -- Contextual optimization and credit-scoring uncertainty papers support the - framing of CRPTO as prediction-to-decision data science, but they do not - change the certificate object. -- Non-exchangeable conformal risk control, valid selection among conformal sets, - inverse/decision-calibrated robustness, and learned decision-aware conformal - sets are outside the submitted claim unless rerun under a new tag with an - explicit selection/calibration design. -- The current finite-grid frontier is strong audit evidence for the declared - frozen surface; it is not a stability-based or independent-recalibration - theorem for selecting among many conformal sets. +For a fixed funded set, the deterministic accounting identity + +`weighted outcome <= B_u + V` + +holds without a statistical assumption. In the active OOT allocation its +right-hand side is `0.294926`, while the observed weighted outcome is +`0.039375`. + +The Markov statement is secondary. If one additionally assumes weighted +funded-set validity, `E[V] <= alpha`, then + +`P(weighted outcome >= B_u + sqrt(alpha)) <= sqrt(alpha)`. + +At `alpha = 0.10`, this gives threshold `0.574279` and probability bound +`0.316228`. It is deliberately reported as a weak, assumption-conditional +sensitivity, not as a deterministic risk cap, nominal selected-set coverage, +or the paper's primary novelty. + +## Evidence Contract + +The active evidence bundle is: + +- `models/experiments/champion_reopen//portfolio/ijds_policy_governance.json` +- A35: exact alpha replay and saturation audit. +- A36: nine-policy calibration selector. +- A37: full-OOT and temporal fixed-policy evaluation. +- A38: selected funded-set grade composition. +- A39: fixed-allocation bootstrap. +- A40: selected, more-conservative, and matched point-PD comparison. + +The manuscript must say explicitly that earlier project development inspected +this static OOT corpus. The final tagged rule is outcome-free with respect to +its policy-ranking code path, but the evaluation is a transparent retrospective +lockbox replay, not a pristine prospective trial. + +## Retired Headline Claims + +The following remain provenance only and must not appear as active results: + +- alpha-0.01 endpoints obtained by cross-family average-width scaling; +- the `8/8` approximate alpha-grid pass; +- the 50,010-policy frontier as the active selector; +- the `0.345084` Markov threshold; +- capped/tail-focused policy families as the selected method; +- the exploratory 25-policy `gamma = 0.35`, threshold-cap `0.65` challenger; +- policy hyperparameters chosen from OOT realized outcomes. + +The frozen upstream model, calibrator, interval artifacts, historical pool93 +tables, and `EXTRACTION_MANIFEST.json` remain untouched. ## Reopen Gate -A new search is justified only if it can plausibly change one of these claims: +Reopen the active method only for one of four reasons: -1. same or higher return with materially lower exact Markov threshold or `Gamma_CP`; -2. much higher return under the same declared threshold; -3. a denser predeclared alpha grid that materially strengthens the certificate; -4. a nested/prospective evaluation design that reduces post-selection risk; -5. a reviewer-requested diagnostic that closes a specific objection. +1. a calibration-only rule materially improves return at the same `0.60` + screen; +2. a simpler rule matches the selected policy within prespecified tolerances; +3. a valid selected-set or prospective protocol materially strengthens the + statistical claim; +4. an IJDS reviewer requests a specific additional test. -Otherwise, append the idea to research notes after submission and keep the -current pool93 frontier closed. +Otherwise, keep one method, one policy, and one manuscript narrative. diff --git a/docs/research/ijds_exact_alpha_calibration_selection_2026-07-09.md b/docs/research/ijds_exact_alpha_calibration_selection_2026-07-09.md new file mode 100644 index 0000000..082c656 --- /dev/null +++ b/docs/research/ijds_exact_alpha_calibration_selection_2026-07-09.md @@ -0,0 +1,69 @@ +# IJDS Exact-Alpha and Calibration-Selection Closeout + +Date: 2026-07-09 + +## Why the active claim changed + +The previous paper-facing alpha sweep did not recompute conformal quantiles at +each alpha. It scaled stored 90% row radii using average-width ratios from a +different conformal family. The resulting alpha-0.01 endpoints were useful as +an exploratory approximation, but they did not support the manuscript's +"exact alpha" language. The former `8/8`, `0.345084` threshold, and 50,010-grid +selection claims were therefore retired rather than cosmetically relabeled. + +## Exact replay + +`src/models/conformal_alpha_grid.py` reconstructs the frozen score-decile +Mondrian recipe from its result payload and recomputes quantiles for each alpha. +At the reference 90% level, replay matches the stored point, low, and high +vectors to at most `6.67e-16`. The exact alpha sweep also exposes why +`alpha = 0.01` is not decision-useful here: average interval width is `0.9882` +and `93.54%` of upper endpoints equal one. + +## Final policy protocol + +The final run is +`champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6`. +It uses the exactly replayed 90% interval recipe and a nine-policy round-number +grid: + +- `tau in {0.15, 0.17, 0.19}`; +- `gamma in {0.25, 0.50, 0.75}`; +- linear `q = p + gamma(u-p)` only; +- no cap, tail rule, or uncertainty-aversion penalty; +- point-PD expected net return in the objective; +- conformal `q` only in the portfolio-risk constraint. + +The policy-ranking code reads no default, realized-return, or miscoverage +columns. It requires at least 99.9% budget use, feasibility of the effective-PD +cap, and `B_u + sqrt(0.10) <= 0.60` on the calibration holdout. Five candidates +are eligible. Maximizing expected point-PD objective selects +`tau = 0.17, gamma = 0.50`, the interpretable midpoint +`q = (p+u)/2`. + +## What was learned from the challengers + +- The exploratory 25-policy selector chose `gamma = 0.35` and earned + `$181,758.69` OOT, but its threshold was `0.646094` and its policy grid and + cap were harder to defend. +- The final 50/50 policy earns `$179,327.59`, only `$2,431.11` less, while + lowering weighted default from `0.04855` to `0.039375` and the threshold from + `0.646094` to `0.574279`. +- A 75% blend is safer (`0.035875` default; `0.516624` threshold) but earns + `$6,388.08` less than the selected midpoint. +- The matched point-PD policy earns more on the full OOT panel, but its default + rate and endpoint audit are substantially worse. It also beats CRPTO in some + temporal slices, so no universal dominance claim is supportable. + +## Interpretation + +The scientific upgrade is not a larger search. It is a smaller and auditable +decision rule, an exact conformal replay at the level actually used, separation +of point-PD economics from conformal feasibility, and a policy selector whose +inputs can be inspected for outcome leakage. This is the active IJDS narrative. + +The historical OOT panel was inspected during earlier project development. +Accordingly, v6 is described as a retrospective lockbox replay with an +OOT-outcome-column-free final selector conditional on the frozen conformal +recipe, not as a pristine prospective holdout or +preregistered trial. diff --git a/justfile b/justfile index 0112faa..1337709 100644 --- a/justfile +++ b/justfile @@ -44,7 +44,7 @@ complexity-report: uvx radon cc src scripts -s -n D api-docs-core: - uv run --with pdoc pdoc src.optimization.portfolio_model src.models.calibration src.evaluation.backtesting src.evaluation.fairness --docformat google --output-directory reports/api-docs --no-browser + uv run --with pdoc pdoc src.optimization.portfolio_model src.optimization.policy_evaluation src.optimization.policy_selection src.models.conformal_alpha_grid src.models.calibration src.evaluation.backtesting src.evaluation.fairness --docformat google --output-directory reports/api-docs --no-browser hooks-check: uv run pre-commit validate-config @@ -77,7 +77,19 @@ evidence: journal-package: uv run python scripts/build_crpto_journal_package.py -paper-export: tables figures evidence journal-package book +ijds-evidence: + uv run python scripts/build_ijds_calibration_selected_evidence.py + +# Explicit methodology replays. These write only to versioned experiment paths. +ijds-exact-alpha: + uv run python scripts/experiments/run_ijds_exact_alpha_grid_challenger.py --config configs/experiments/champion_reopen_ijds_exact_alpha_grid_v1.yaml + +ijds-policy-challenger: + uv run python scripts/experiments/run_ijds_calibration_selected_policy_challenger.py --config configs/experiments/champion_reopen_ijds_calibration_selected_simple90_v6.yaml + +ijds-active-replay: ijds-exact-alpha ijds-policy-challenger ijds-evidence + +paper-export: tables figures evidence journal-package ijds-evidence book # IJDS-oriented manuscript body (HTML writing preview). paper-ijds: @@ -95,7 +107,7 @@ paper-submission-official: @uv run python scripts/compile_ijds_submission.py # Final local IJDS gate before freezing or uploading. -submission-check: publication-integrity lint type-check type-advisory-full smoke validate-champion paper-submission paper-submission-official +submission-check: ijds-evidence publication-integrity lint type-check type-advisory-full test validate-champion paper-submission paper-submission-official # IJDS-oriented manuscript body (local HTML-print PDF verification draft). paper-ijds-pdf: diff --git a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6/portfolio/calibration_selected_policy_summary.json b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6/portfolio/calibration_selected_policy_summary.json new file mode 100644 index 0000000..fe63910 --- /dev/null +++ b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6/portfolio/calibration_selected_policy_summary.json @@ -0,0 +1,149 @@ +{ + "allocation_path": "data\\processed\\experiments\\champion_reopen\\champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6\\portfolio\\calibration_selected_policy_full_oot_allocations.parquet", + "calibration_metadata": { + "calibration_fit_rows": 142550, + "calibration_selection_end": "2017-12-01", + "calibration_selection_rows": 35638, + "calibration_selection_start": "2017-11-01", + "conformal_results_path": "models\\conformal_gap\\champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1\\conformal_results_mondrian.pkl", + "partition": "score_decile_mondrian", + "target_alpha": 0.1, + "used_alpha": 0.095 + }, + "claim_boundary": "The final tagged policy rule ranks nine round-number candidates on the calibration development block without reading default, realized-return, or other outcome-derived selector columns, and then freezes the selected policy before OOT evaluation. Conformal endpoints themselves use calibration labels, as required. Earlier project development inspected this static OOT corpus, so the evaluation is a transparent retrospective lockbox replay rather than a pristine prospective trial, causal estimate, or live-deployment guarantee.", + "config_path": "configs\\experiments\\champion_reopen_ijds_calibration_selected_simple90_v6.yaml", + "config_sha256": "cddf7daf67c91b7715e676b8a9e8dcba08e8d3a9499b8fe7cdffe51cfe690646", + "contrasts": { + "2020+": { + "default_delta_vs_incumbent": -0.010000000000000009, + "default_delta_vs_point": 0.06687499999999999, + "return_cost_vs_point": 118939.59578289572, + "return_delta_vs_incumbent": 16526.81992213076, + "selected_markov_threshold": 0.580689837874186, + "selected_realized_return": 99689.53961659266, + "selected_weighted_outcome": 0.08377499999999999, + "threshold_delta_vs_point": -0.5969627812854879 + }, + "full_oot": { + "default_delta_vs_incumbent": 0.003500000000000003, + "default_delta_vs_point": -0.07902500000000001, + "return_cost_vs_point": 17041.554867740255, + "return_delta_vs_incumbent": 6388.080277992645, + "selected_markov_threshold": 0.5742788554403055, + "selected_realized_return": 179327.5851322598, + "selected_weighted_outcome": 0.03937500000000001, + "threshold_delta_vs_point": -0.663266349105067 + } + }, + "design": { + "alpha": 0.1, + "budget": 1000000.0, + "combine_years_from": 2020, + "lgd": 0.45, + "markov_threshold_cap": 0.6, + "max_concentration": 0.25, + "period_order": [ + "2018H1", + "2018H2", + "2019H1", + "2019H2", + "2020+" + ], + "selection_min_budget_utilization": 0.999, + "selection_rule": "maximize expected point-PD objective on the calibration holdout under a 0.60 endpoint-plus-Markov screen, the effective-PD cap, and full budget use" + }, + "evaluation_path": "data\\processed\\experiments\\champion_reopen\\champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6\\portfolio\\calibration_selected_policy_oot_evaluation.csv", + "generated_at_utc": "2026-07-10T02:25:53.745446+00:00", + "grid_size": 9, + "incumbent_policy": { + "candidate_id": "linear-006", + "delta_cap_quantile": 1.0, + "gamma": 0.75, + "min_budget_utilization": 0.0, + "pd_cap_slack_penalty": 0.0, + "policy_mode": "blended_uncertainty", + "risk_tolerance": 0.17, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.0 + }, + "recipe": { + "partition": "score_decile_mondrian", + "partition_probability_source": "calibrated", + "reference_target_alpha": 0.1, + "reference_used_alpha": 0.095 + }, + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6", + "schema_version": "2026-07-09.6", + "selected_calibration_metrics": { + "candidate_id": "linear-005", + "delta_cap_quantile": 1.0, + "effective_pd_cap_slack": 0.0, + "endpoint_budget": 0.26104699571114653, + "expected_objective": 110346.16233639097, + "gamma": 0.5, + "gamma_cp": 0.18209399142229304, + "gamma_internalized": 0.09104699571114652, + "gamma_residual": 0.09104699571114651, + "markov_loss_threshold": 0.5772747617279845, + "min_budget_utilization": 0.0, + "n_funded": 211, + "n_panel": 35638, + "objective_risk_mode": "point_pd_plus_aversion", + "pd_cap_slack_penalty": 0.0, + "policy_mode": "blended_uncertainty", + "risk_tolerance": 0.17, + "solver_status": "Optimal", + "tail_focus_quantile": 1.0, + "total_allocated": 1000000.0, + "uncertainty_aversion": 0.0, + "weighted_pd_effective": 0.17, + "weighted_pd_point": 0.07895300428885349 + }, + "selected_policy": { + "candidate_id": "linear-005", + "delta_cap_quantile": 1.0, + "gamma": 0.5, + "min_budget_utilization": 0.0, + "pd_cap_slack_penalty": 0.0, + "policy_mode": "blended_uncertainty", + "risk_tolerance": 0.17, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.0 + }, + "selection_audit": { + "markov_threshold_cap": 0.6, + "min_budget_utilization": 0.999, + "n_eligible": 5, + "n_total": 9, + "outcome_columns_used": 0, + "selected_candidate_id": "linear-005", + "selection_rule": "max_expected_objective_under_ex_ante_screen" + }, + "selector_columns": [ + "candidate_id", + "risk_tolerance", + "gamma", + "uncertainty_aversion", + "policy_mode", + "delta_cap_quantile", + "tail_focus_quantile", + "min_budget_utilization", + "pd_cap_slack_penalty", + "solver_status", + "objective_risk_mode", + "expected_objective", + "n_panel", + "n_funded", + "total_allocated", + "weighted_pd_point", + "weighted_pd_effective", + "gamma_cp", + "gamma_internalized", + "gamma_residual", + "endpoint_budget", + "markov_loss_threshold", + "effective_pd_cap_slack" + ], + "selector_forbidden_columns_present": [], + "source_commit": "17811d8fb4c5dfc0035f86ac7095088533bfec5b" +} diff --git a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6/portfolio/ijds_policy_governance.json b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6/portfolio/ijds_policy_governance.json new file mode 100644 index 0000000..a012946 --- /dev/null +++ b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6/portfolio/ijds_policy_governance.json @@ -0,0 +1,114 @@ +{ + "bootstrap_return_interval": { + "n_draws": 5000, + "p025": 162706.17200230644, + "p975": 193924.73990027257 + }, + "claim_boundary": "The final tagged policy rule ranks nine round-number candidates on the calibration development block without reading default, realized-return, or other outcome-derived selector columns, and then freezes the selected policy before OOT evaluation. Conformal endpoints themselves use calibration labels, as required. Earlier project development inspected this static OOT corpus, so the evaluation is a transparent retrospective lockbox replay rather than a pristine prospective trial, causal estimate, or live-deployment guarantee.", + "exact_alpha_reference_replay": { + "high_max_abs": 6.661338147750939e-16, + "low_max_abs": 3.3306690738754696e-16, + "pass": true, + "point_max_abs": 4.440892098500626e-16, + "tolerance": 1e-12 + }, + "full_oot": { + "Gamma_CP": 0.1761021788469351, + "Gamma_internalized": 0.0880510894234675, + "Gamma_residual": 0.0880510894234675, + "endpoint_budget": 0.2580510894234676, + "expected_objective": 168271.56287282018, + "markov_loss_threshold": 0.5742788554403055, + "markov_tail_probability_bound": 0.31622776601683794, + "n_candidates": 276869, + "n_funded": 308, + "observed_accounting_bound": 0.2949260894234676, + "realized_return": 179327.5851322598, + "total_allocated": 1000000.0, + "weighted_default_rate": 0.039375, + "weighted_miscoverage": 0.036875, + "weighted_pd_effective": 0.17, + "weighted_pd_point": 0.0819489105765324 + }, + "generated_at_utc": "2026-07-10T02:25:53.745446+00:00", + "paper_tables": { + "alpha": [ + "reports/crpto/tables/crpto_tableA35_exact_alpha_grid.csv", + "reports/crpto/tables/crpto_tableA35_exact_alpha_grid.tex" + ], + "baseline": [ + "reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.csv", + "reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.tex" + ], + "bootstrap": [ + "reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.csv", + "reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.tex" + ], + "grade": [ + "reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.csv", + "reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.tex" + ], + "selector": [ + "reports/crpto/tables/crpto_tableA36_calibration_policy_selector.csv", + "reports/crpto/tables/crpto_tableA36_calibration_policy_selector.tex" + ], + "temporal": [ + "reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.csv", + "reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.tex" + ] + }, + "point_pd_contrast": { + "endpoint_budget": 0.9213174385285344, + "markov_loss_threshold": 1.2375452045453723, + "realized_return": 196369.14000000004, + "selected_default_reduction": 0.07902500000000001, + "selected_return_cost": 17041.554867740255, + "selected_return_cost_pct": 0.0867832637436832, + "selected_threshold_reduction": 0.6632663491050668, + "weighted_default_rate": 0.1184, + "weighted_miscoverage": 0.0419 + }, + "retired_active_claims": [ + "alpha01 intervals obtained by cross-family average-width scaling", + "8/8 approximate alpha-grid pass as a headline certificate", + "50,010-policy frontier as the active selector", + "0.345084 Markov threshold", + "capped_blended_uncertainty with delta_cap_quantile=0.975", + "OOT-outcome-selected portfolio hyperparameters", + "the exploratory 25-policy gamma=0.35, threshold-cap=0.65 challenger" + ], + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6", + "schema_version": "2026-07-09.6", + "selected_policy": { + "candidate_id": "linear-005", + "delta_cap_quantile": 1.0, + "gamma": 0.5, + "min_budget_utilization": 0.0, + "pd_cap_slack_penalty": 0.0, + "policy_mode": "blended_uncertainty", + "risk_tolerance": 0.17, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.0 + }, + "selection_protocol": { + "calibration_metadata": { + "calibration_fit_rows": 142550, + "calibration_selection_end": "2017-12-01", + "calibration_selection_rows": 35638, + "calibration_selection_start": "2017-11-01", + "conformal_results_path": "models\\conformal_gap\\champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1\\conformal_results_mondrian.pkl", + "partition": "score_decile_mondrian", + "target_alpha": 0.1, + "used_alpha": 0.095 + }, + "markov_threshold_cap": 0.6, + "min_budget_utilization": 0.999, + "n_eligible": 5, + "n_total": 9, + "outcome_columns_used": 0, + "selected_candidate_id": "linear-005", + "selection_rule": "max_expected_objective_under_ex_ante_screen", + "selector_forbidden_columns_present": [] + }, + "status": "active_ijds_policy" +} diff --git a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1/conformal/exact_alpha_grid_summary.json b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1/conformal/exact_alpha_grid_summary.json new file mode 100644 index 0000000..70b3255 --- /dev/null +++ b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1/conformal/exact_alpha_grid_summary.json @@ -0,0 +1,244 @@ +{ + "schema_version": "2026-07-09.1", + "generated_at_utc": "2026-07-10T00:28:51.509823+00:00", + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1", + "source_commit": "17811d8fb4c5dfc0035f86ac7095088533bfec5b", + "config_path": "configs\\experiments\\champion_reopen_ijds_exact_alpha_grid_v1.yaml", + "config_sha256": "c2d2af0d1c0e0f83ef8612888f9b12d423063a601f04c8c51c9680e92327ef3f", + "source": { + "upstream_canonical_run_tag": "champion-reopen-2026-06-19__hpo-wave1__pool93__seed42", + "conformal_namespace": "champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1", + "conformal_results_path": "models/conformal_gap/champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1/conformal_results_mondrian.pkl", + "conformal_intervals_path": "data/processed/conformal_gap/champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1/conformal_intervals_mondrian.parquet", + "conformal_results_sha256": "cb8db92908739c37a11f373b9802f6b74831d8d9c26c4824dae586423d3cb07b", + "conformal_intervals_sha256": "3784cc68a5b72cf290c9678e6cd5d30f7b794dac4c1eedaf393fe9f72cc0c3a1" + }, + "recipe": { + "partition": "score_decile_mondrian", + "partition_probability_source": "calibrated", + "n_score_bins": 5, + "fallback_mode": "score_only", + "score_scale_family": "bernoulli_sqrt", + "min_group_size": 100, + "reference_target_alpha": 0.1, + "reference_used_alpha": 0.095, + "calibration_fraction": 0.75, + "tuning_holdout_ratio": 0.2, + "tuning_random_state": 42, + "group_multipliers": { + "score_q00": 1.02, + "score_q01": 1.05, + "score_q02": 1.05, + "score_q03": 1.02, + "score_q04": 1.02 + }, + "temporal_segment_multipliers": {}, + "temporal_segment_freq": "Q", + "global_rebalance_factor": 1.0 + }, + "alpha_mapping": "proportional frozen conservative ratio alpha_used/alpha_target from the selected 90% recipe", + "reference_replay": { + "point_max_abs": 4.440892098500626e-16, + "low_max_abs": 3.3306690738754696e-16, + "high_max_abs": 6.661338147750939e-16, + "tolerance": 1e-12, + "pass": true + }, + "alpha_summaries": [ + { + "target_alpha": 0.01, + "used_alpha": 0.0095, + "target_coverage": 0.99, + "empirical_coverage": 0.9967204706919157, + "coverage_gap": 0.006720470691915725, + "avg_width": 0.9882149910656284, + "median_width": 1.0, + "min_partition_coverage": 0.989628743645835, + "min_grade_coverage": 0.9722703639514731, + "high_endpoint_mean": 0.9883685381679577, + "high_endpoint_min": 0.28857911767290634, + "high_endpoint_p01": 0.6715297528015821, + "high_endpoint_p10": 1.0, + "high_endpoint_at_one_rate": 0.9354243342519386, + "partition_count": 5, + "group_quantiles": { + "score_q00": 4.620892417510161, + "score_q01": 2.9770143672127003, + "score_q02": 2.2280905155147157, + "score_q03": 1.7679793270133157, + "score_q04": 1.379561550348839 + } + }, + { + "target_alpha": 0.03, + "used_alpha": 0.028499999999999998, + "target_coverage": 0.97, + "empirical_coverage": 0.9884783056246818, + "coverage_gap": 0.01847830562468178, + "avg_width": 0.9697789837688854, + "median_width": 1.0, + "min_partition_coverage": 0.9697990273996318, + "min_grade_coverage": 0.9664933564413634, + "high_endpoint_mean": 0.9699657056860221, + "high_endpoint_min": 0.22825533153683755, + "high_endpoint_p01": 0.5334804414885259, + "high_endpoint_p10": 0.9018799702618802, + "high_endpoint_at_one_rate": 0.8323141991338864, + "partition_count": 5, + "group_quantiles": { + "score_q00": 3.642519971959776, + "score_q01": 2.8072631501367455, + "score_q02": 2.1737923639215784, + "score_q03": 1.7387907144323151, + "score_q04": 1.3580168082764283 + } + }, + { + "target_alpha": 0.05, + "used_alpha": 0.0475, + "target_coverage": 0.95, + "empirical_coverage": 0.9728246932664907, + "coverage_gap": 0.022824693266490725, + "avg_width": 0.9535799768723978, + "median_width": 1.0, + "min_partition_coverage": 0.9505830880848586, + "min_grade_coverage": 0.9525750124329999, + "high_endpoint_mean": 0.953801858012357, + "high_endpoint_min": 0.20305073151598757, + "high_endpoint_p01": 0.47580041346523627, + "high_endpoint_p10": 0.806835962194862, + "high_endpoint_at_one_rate": 0.7181555175913519, + "partition_count": 5, + "group_quantiles": { + "score_q00": 3.233734520299044, + "score_q01": 2.6752015014256125, + "score_q02": 2.1212263259384927, + "score_q03": 1.7077473640552945, + "score_q04": 1.3388102247632694 + } + }, + { + "target_alpha": 0.07, + "used_alpha": 0.0665, + "target_coverage": 0.9299999999999999, + "empirical_coverage": 0.9552712654721186, + "coverage_gap": 0.025271265472118665, + "avg_width": 0.795666199875876, + "median_width": 1.0, + "min_partition_coverage": 0.9347822665680426, + "min_grade_coverage": 0.938442835829143, + "high_endpoint_mean": 0.7959333620592705, + "high_endpoint_min": 0.023668097607277484, + "high_endpoint_p01": 0.06528823147517372, + "high_endpoint_p10": 0.13040211717455735, + "high_endpoint_at_one_rate": 0.6135500904760013, + "partition_count": 5, + "group_quantiles": { + "score_q00": 0.32438420480886127, + "score_q01": 2.567436984095836, + "score_q02": 2.0727963588460026, + "score_q03": 1.6764580225891925, + "score_q04": 1.3186561259152434 + } + }, + { + "target_alpha": 0.1, + "used_alpha": 0.095, + "target_coverage": 0.9, + "empirical_coverage": 0.9348356081757077, + "coverage_gap": 0.03483560817570763, + "avg_width": 0.7888790790793794, + "median_width": 1.0, + "min_partition_coverage": 0.9263099219620958, + "min_grade_coverage": 0.9267968822615554, + "high_endpoint_mean": 0.7892193805747533, + "high_endpoint_min": 0.023406884837288408, + "high_endpoint_p01": 0.06469045329656556, + "high_endpoint_p10": 0.12941711012832322, + "high_endpoint_at_one_rate": 0.5178730735474177, + "partition_count": 5, + "group_quantiles": { + "score_q00": 0.3201476773467629, + "score_q01": 2.4331722444504513, + "score_q02": 2.007709882293012, + "score_q03": 1.633544476383572, + "score_q04": 1.2899294761760491 + } + }, + { + "target_alpha": 0.12, + "used_alpha": 0.11399999999999999, + "target_coverage": 0.88, + "empirical_coverage": 0.9182645944471934, + "coverage_gap": 0.038264594447193434, + "avg_width": 0.7831737841625059, + "median_width": 0.9867966540367542, + "min_partition_coverage": 0.906874767744333, + "min_grade_coverage": 0.9057375416605808, + "high_endpoint_mean": 0.7835760369425744, + "high_endpoint_min": 0.023236857746810564, + "high_endpoint_p01": 0.06430135101621326, + "high_endpoint_p10": 0.12877595509528877, + "high_endpoint_at_one_rate": 0.4545109781160043, + "partition_count": 5, + "group_quantiles": { + "score_q00": 0.31739006163700956, + "score_q01": 2.3534507550187485, + "score_q02": 1.9655367760210574, + "score_q03": 1.6066047959404717, + "score_q04": 1.269964951004726 + } + }, + { + "target_alpha": 0.15, + "used_alpha": 0.1425, + "target_coverage": 0.85, + "empirical_coverage": 0.8860977574231856, + "coverage_gap": 0.03609775742318566, + "avg_width": 0.6461762449744832, + "median_width": 0.94425532567159, + "min_partition_coverage": 0.8635623982976592, + "min_grade_coverage": 0.8631010078209027, + "high_endpoint_mean": 0.6466929852254412, + "high_endpoint_min": 0.02298905825369661, + "high_endpoint_p01": 0.06373426874967675, + "high_endpoint_p10": 0.1278415281651632, + "high_endpoint_at_one_rate": 0.33485872380078663, + "partition_count": 5, + "group_quantiles": { + "score_q00": 0.31337107987851737, + "score_q01": 0.4420899636645897, + "score_q02": 1.9079870902279796, + "score_q03": 1.5675398154360425, + "score_q04": 1.2412696239135141 + } + }, + { + "target_alpha": 0.2, + "used_alpha": 0.19, + "target_coverage": 0.8, + "empirical_coverage": 0.8493800317117481, + "coverage_gap": 0.049380031711748096, + "avg_width": 0.6365853175970958, + "median_width": 0.9128005695874017, + "min_partition_coverage": 0.8293756967670011, + "min_grade_coverage": 0.828006532910199, + "high_endpoint_mean": 0.6373641323567814, + "high_endpoint_min": 0.022569416686283303, + "high_endpoint_p01": 0.06277393066265638, + "high_endpoint_p10": 0.1262591020808178, + "high_endpoint_at_one_rate": 0.24466805601204902, + "partition_count": 5, + "group_quantiles": { + "score_q00": 0.30656504576058813, + "score_q01": 0.43573433937656825, + "score_q02": 1.8175729021078393, + "score_q03": 1.5052283270500615, + "score_q04": 1.194771966347316 + } + } + ], + "grid_path": "data\\processed\\experiments\\champion_reopen\\champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1\\conformal\\exact_alpha_grid.parquet", + "grid_rows": 276869, + "claim_boundary": "Recomputes finite-sample Mondrian quantiles for every declared alpha while freezing the selected partition, calibration fit/holdout split, score scale, and holdout-learned widening factors. It is a retrospective OOT audit, not a post-selection or live-deployment guarantee." +} diff --git a/paper/CRPTO.qmd b/paper/CRPTO.qmd index 09e3487..7274b92 100644 --- a/paper/CRPTO.qmd +++ b/paper/CRPTO.qmd @@ -14,53 +14,46 @@ execute: # Resumen -CRPTO integra prediccion de default calibrada, intervalos conformales Mondrian y -optimizacion robusta de portafolio. La superficie IJDS activa promueve el punto -pool93 body/default: retorno robusto `$184,832.48`, `V=0.035350`, -`Gamma_CP=0.162616`, `Gamma_res=0.073584`, endpoint `0.245084`, umbral Markov -`0.345084`, exceso realizado sobre `tau` igual a cero y pass `8/8` sobre -la grilla alpha declarada. La frontera A35 y el audit de composicion A36 son los -artefactos paper-facing principales para el cierre pool93; A37--A39 agregan la -repricing de tail-risk, el audit cluster-bound y el bootstrap de contribuciones -de la asignacion seleccionada; A40 agrega la baseline point-PD emparejada. - -# Manuscrito - -Este archivo es el punto de entrada generico del manuscrito standalone. La -decision editorial activa esta en -`../configs/crpto_publication_targets.yaml`: escribir primero para **INFORMS -Journal on Data Science** y mantener **European Journal of Operational -Research** como pivote principal. - -Los borradores de trabajo son: - -- `CRPTO_ijds.qmd`: cuerpo anonimo IJDS, conceptualmente limitado a 25 paginas - excluyendo referencias y appendix. -- `supplement_ijds.qmd`: online supplement IJDS con A3--A40, robustez, - reproducibilidad, MRM y fairness. - -La version larga y sus apendices estan en `book/`. - -# Artefactos principales - -Esquema dual-tag: el body claim del paper vive en los sidecars pool93; la -cadena upstream congelada (rebaseline 2026-06-07) define el return floor -declarado. `crpto_table0_key_metrics.csv` y `final_project_promotion.json` -documentan esa cadena historica, no el body point. - -Body claim pool93 (autoritativos): - -- `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2/portfolio/pool93_ijds_consolidated_governance.json` -- `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal/portfolio/pool93_ijds_claim_governance.json` -- `reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv` (y A36--A40) - -Cadena upstream congelada (historica, return floor): - -- `models/final_project_promotion.json` -- `models/champion_portfolio_policy.json` -- `reports/crpto/tables/crpto_table0_key_metrics.csv` - -Figuras de entrada: - -- `reports/crpto/figures/crpto_fig1_journal_pipeline.png` -- `reports/crpto/figures/crpto_fig12_crpto_conceptual_pipeline.png` +CRPTO convierte una PD calibrada en una decision de portafolio auditable. La +version IJDS activa reproduce exactamente el endpoint conformal al 90%, usa la +regla simple `q=(p+u)/2` con `tau=0.17`, y selecciona esa politica dentro de una +grilla redonda `3x3` usando el holdout temporal de calibracion. El ranking final +no lee outcomes OOT. + +En 276,869 prestamos OOT, la politica financia 308 prestamos y obtiene +`$179,327.59` sobre `$1M`, con default ponderado `0.039375`, miscoverage +`0.036875` y endpoint `0.258051`. La baseline point-PD emparejada obtiene +`$196,369.14` y default `0.118400`. El costo observado de retorno es `8.678%` y +la reduccion de default es `7.9025` puntos porcentuales. El paper reporta tambien +los periodos donde point-PD domina; no afirma superioridad universal. + +# Fuentes del manuscrito + +- `CRPTO_ijds.qmd`: body anonimo y fuente narrativa activa. +- `supplement_ijds.qmd`: pruebas y evidencia A35--A40, mas diagnosticos + historicos A1--A34. +- `submission/CRPTO_ijds_submission.tex`: handoff compacto bajo `informs4`. +- `../docs/research/active_claims_2026-07-04.md`: contrato numerico vigente. + +# Evidencia activa + +- A35: replay exacto y saturacion por alpha. +- A36: selector de calibracion de nueve politicas. +- A37: evaluacion OOT total y temporal. +- A38: composicion por grado de credito. +- A39: bootstrap de asignacion fija. +- A40: politica seleccionada, blend conservador y baseline point-PD. + +La gobernanza vive en +`models/experiments/champion_reopen//portfolio/ijds_policy_governance.json`. +Los artefactos historicos protegidos por `EXTRACTION_MANIFEST.json` permanecen +intactos como procedencia, no como claims activos. + +# Comandos + +```powershell +just ijds-evidence +just paper-submission +just paper-submission-official +just submission-check +``` diff --git a/paper/CRPTO_ijds.qmd b/paper/CRPTO_ijds.qmd index be56261..5bcb9fd 100644 --- a/paper/CRPTO_ijds.qmd +++ b/paper/CRPTO_ijds.qmd @@ -1,5 +1,5 @@ --- -title: "CRPTO: Conformal Robust Predict-Then-Optimize for Auditable Credit Portfolio Decisions" +title: "CRPTO: A Calibration-Selected Conformal Guardrail for Auditable Credit Portfolio Decisions" author: "Anonymous" date: today lang: en @@ -25,1138 +25,465 @@ execute: # Abstract -Credit allocation is a data-science-for-decisions problem: default probabilities -matter only after they shape which loans are funded under a budget and risk -appetite. We introduce Conformal Robust Predict-Then-Optimize (CRPTO), a -post-hoc decision certificate that maps a frozen calibrated -probability-of-default model through Mondrian conformal intervals into robust -portfolio constraints and an empirical funded-set audit. A policy-aware -decomposition separates the conformal premium internalized by the optimizer -from the residual premium needed to recover the exact upper-endpoint budget. -On a 276,869-loan out-of-time Lending Club evaluation, the selected policy earns -`$184.8K` on a `$1M` budget while passing the declared eight-level alpha grid -($V(0.01)=0.035350$, $\Gamma_{\mathrm{CP}}=0.162616$, exact Markov loss -threshold `0.345084`, zero realized risk-tolerance excess). Against a matched -point-PD two-stage LP with the same candidates, budget, concentration cap, and -risk tolerance, CRPTO gives up `5.87%` of realized return while reducing the -weighted default rate by `8.305` percentage points and the loss threshold by -`43.55` percentage points. The consolidated finite frontier contains `50,010` -deduplicated semantic policies, of which `27,508` pass all declared alpha levels -and exceed the return floor. Frozen Prosper and Freddie/Mendeley applications -test recipe transfer and preserve the predeclared global conformal gates with -positive robust LP objectives. CRPTO therefore makes predictive uncertainty -decision-useful as an auditable return--risk frontier, with a distribution-free -Markov bound under weighted funded-set validity and an explicit separation -between deterministic accounting and its statistical assumption. - -**Keywords:** conformal prediction; robust optimization; predict-then-optimize; -credit risk; portfolio optimization; reproducible data science. +Credit models matter only through the decisions they change. We study how +finite-sample predictive uncertainty can constrain a loan portfolio after a +probability-of-default (PD) model has been frozen. Conformal Robust +Predict-Then-Optimize (CRPTO) recomputes a 90% Mondrian conformal upper endpoint +exactly, forms the transparent decision score $q_i=(p_i+u_i)/2$, and places +$q_i$ in a portfolio-risk constraint while retaining point PD in the economic +objective. Nine round-number policies are ranked on a temporal calibration +holdout without default, realized-return, or miscoverage columns; the selected +policy is then frozen and replayed on 276,869 out-of-time Lending Club loans. +It funds 308 loans and earns `$179,327.59` on a `$1M` budget, with weighted +default `0.039375`, weighted miscoverage `0.036875`, and conformal endpoint +budget `0.258051`. A matched point-PD allocation earns `$196,369.14` but has +weighted default `0.118400` and endpoint budget `0.921317`. Thus CRPTO pays +`8.678%` of realized return for a `7.9025` percentage-point default reduction; +the advantage reverses in some temporal slices, so we do not claim universal +dominance. The contribution is a small, auditable prediction-to-decision +guardrail with an exact replay, an inspectable selector, and explicit statistical +boundaries, rather than another credit-scoring leaderboard or a collection of +policy variants. + +**Keywords:** conformal prediction; predict-then-optimize; credit risk; +portfolio optimization; calibration; reproducible data science. # Introduction -Credit allocation is a contextual optimization problem in credit form: a lender -first estimates a probability of default (PD), then chooses which loans to fund -under a budget and risk appetite [@sadana2025contextual]. The modeling -literature has become very good at the first step: calibration, discrimination, -and backtesting are now standard ingredients of credit-risk model validation -[@lessmann2015; @chen2024creditrisk]. Recent credit-scoring work also prices -predictive uncertainty and parameter uncertainty through profit, rejection, or -multiobjective risk metrics [@xu2025profit_uncertainty_credit; -@xu2024profit_risk_credit]. The second step is less settled. Once a calibrated -PD enters an optimizer, uncertainty is often treated as a reporting diagnostic -rather than as a constraint that can change the funded set. - -That separation is uncomfortable in auditable credit decisions. A portfolio -policy can have a reasonable average PD and still concentrate probability mass -in loans where the model is most uncertain. Conversely, a policy that is too -conservative can pass every risk check while destroying economic value. The -scientific question in this paper is therefore not whether one can build a -slightly better credit classifier. It is whether finite-sample predictive -uncertainty can be carried into a robust portfolio decision in a way that is -transparent enough for a reviewer to audit. This has practical stakes, but it is -not automatic: conformal sets can be valid without being decision-useful unless -the downstream action and objective are explicit -[@hullman2025conformal_human_decision]. In a pre-registered randomized trial, -conformal prediction sets improved human decision making relative to fixed-size -sets with the same coverage [@cresswell2024]. CRPTO takes that -committee-facing idea into a credit portfolio setting, where the uncertainty -summary must change a funding decision or it is just another report. - -CRPTO answers this question with a post-hoc, reproducible pipeline. It starts -from a calibrated CatBoost PD model, constructs Mondrian conformal intervals -over PD-scale predictions, and maps the upper conformal endpoint into robust -portfolio constraints. The pipeline is modular by design: the predictive model, -conformal layer, optimization policy, and paper outputs each have separate -contracts. That separation lets the paper ask whether a frozen prediction -system can be converted into a defendable decision system without reopening -hyperparameter search whenever the manuscript or appendix changes. - -The empirical setting is the Lending Club retail-loan panel, with an -out-of-time evaluation set of 276,869 loans. The consolidated frontier contains -50,010 deduplicated semantic policies, of which 27,508 pass every declared -alpha level and exceed the return floor. From that declared finite frontier, -the selected policy is the body/default balanced point at the approximately -`0.345` return-bound lens, with exact Markov loss threshold `0.345084`; it is neither a -continuous global optimum nor the economic endpoint. The selected point earns -`$184.8K` on a `$1M` budget and passes the exact empirical funded-set audit at -$\alpha = 0.01$. The headline result is not a single lucky allocation. It is a -reproducible bridge from calibrated probabilistic learning to robust, auditable -credit portfolio choice, with the return-bound frontier reported rather than -hidden behind one selected point. - -To address the natural "single dataset" concern without reopening the Lending Club -champion, we also freeze two external economic replications: Prosper final-status -marketplace loans and a Freddie/Mendeley single-family mortgage panel with -out-of-sample and out-of-time splits. These replications are not new champions; -they test whether the same PD-to-conformal-to-LP recipe remains economically -usable on different credit products. - -The paper makes four contributions. First, it gives a CRPTO construction for -credit portfolios: frozen calibrated PD, Mondrian conformal uncertainty, robust -budgeted optimization, and an exact post-allocation audit. Second, it proves a -distribution-free Markov bound under weighted funded-set validity (Theorem 1) -and introduces a policy-aware decomposition -$\Gamma_{\mathrm{CP}}=\Gamma_{\mathrm{int}}+\Gamma_{\mathrm{res}}$ that recovers -the exact upper-endpoint budget for linear, capped, and tail-focused policies. -Supplement propositions show that Markov is optimal under the stated -first-moment assumption (A.1) and locate the cluster structure that would -tighten it (A.2). Third, it reports the selected Lending Club decision as part -of a declared finite-grid return-bound frontier and compares it with a matched -point-PD allocation; the body point, surrounding policies, exact alpha checks, -and external Prosper/Freddie economic replications are all generated from frozen -evidence. Fourth, it packages the result as a reproducible IJDS decision -artifact, with tables, figures, governance files, and claim-sync checks designed -to keep the statistical boundary visible. The key claim is narrow: CRPTO maps a -frozen calibrated PD model into a robust funded set, reports how much conformal -premium is internalized or left residual, and audits the promoted Lending Club -allocation and its finite-grid frontier exactly. Adjacent methods enter only to -locate and stress-test this single claim, not to create additional acceptance -criteria. - -Read as data science for decisions, the paper's four components are explicit. -The data component is a static Lending Club OOT panel, with Prosper and -Freddie/Mendeley replications used only as frozen external stress tests. The -model component is the CRPTO bridge from calibrated PD to Mondrian conformal -uncertainty to robust LP. The decision component is the funded set under a -budget and risk cap. The implication is a reproducible audit surface for model -risk committees: the paper reports not only return, but also the conformal -premium, exact funded-set miscoverage, region stability, and the boundary beyond -which a new protocol would be required. - -![CRPTO's central move is operational rather than decorative: frozen calibrated PD models become Mondrian conformal intervals, the upper-risk signal enters a robust portfolio optimizer, and the selected funding policy is audited against exact funded-set controls.](../reports/crpto/figures/crpto_fig1_journal_pipeline.png){#fig-crpto-pipeline width="94%" fig-alt="Four-stage CRPTO pipeline from frozen calibrated PD to Mondrian conformal intervals, robust portfolio optimization, and selected auditable policy."} +Credit allocation is a contextual optimization problem. A lender estimates a +PD and then chooses which loans to fund under capital, concentration, and risk +constraints [@sadana2025contextual]. Credit-scoring research has made the first +step increasingly reliable through discrimination benchmarks, probability +calibration, and cost-aware evaluation [@lessmann2015; @chen2024creditrisk; +@yang2025costaware]. Yet a calibrated probability does not specify how much +model uncertainty a portfolio should bear. A point-PD optimizer can concentrate +capital in loans that look attractive precisely where the predictive model is +least certain. + +Conformal prediction offers finite-sample coverage language under explicit +exchangeability conditions [@vovk2005; @angelopoulos2023], while robust +optimization makes uncertainty operational through feasibility sets +[@bertsimas2004; @goldfarb2003robustportfolio]. Joining the two is attractive, +but an applied paper still has to answer three questions. Which conformal level +is informative rather than nearly vacuous? How is a policy selected without +using the outcomes on which it is later reported? And what economic value is +lost when conformal uncertainty actually changes the funded set? + +CRPTO answers those questions with one deliberately simple policy. A frozen, +calibrated CatBoost model produces $p_i$. An exactly replayed 90% Mondrian +recipe produces upper endpoint $u_i$. The portfolio uses their midpoint, -# Related Work +$$ +q_i = p_i + 0.5(u_i-p_i) = \frac{p_i+u_i}{2}, +$$ -CRPTO builds on conformal prediction, especially split conformal methods and -their risk-control extensions [@vovk2005; @angelopoulos2023; -@angelopoulos2024risk]. The relevant property is not that conformal intervals -are the narrowest possible uncertainty summaries. It is that they provide -distribution-free coverage language under explicit exchangeability assumptions, -and that this language can be audited without trusting a fully parametric -posterior. Mondrian and group-conditional variants are especially natural in -credit because risk grades are already used as business and governance -partitions [@bostrom2021; @gibbs2024]. The paper keeps the stronger localized, -weighted, and conditional claims separate because recent impossibility and -weighted-coverage results show that those guarantees require extra structure -[@barber2021limits; @barber2023beyond; @jonkers2024wcps]. - -The second foundation is robust optimization. Classical robust optimization -frames uncertainty as a set against which a decision must remain feasible, with -the price of robustness made visible as a design trade-off [@bertsimas2004]. -Robust portfolio selection makes that trade-off operational for allocation -under parameter uncertainty [@goldfarb2003robustportfolio], whereas -distributionally robust optimization broadens the uncertainty object toward -moment or ambiguity sets [@delage2010dro]. Data-driven robust, -contextual-optimization, and prescriptive-analytics work then connects -predictive models to downstream decisions while keeping the -uncertainty-to-action contract explicit [@bertsimas2018datadriven; -@bertsimas2020prescriptive; @sadana2025contextual]. Recent work connects -conformal prediction and robust optimization more directly by using conformal -uncertainty sets in downstream decisions [@johnstone2021; @patel2024; -@sun2024ptc; @hu2026crc]. That line certifies the uncertainty set *before* the decision: -coverage of the conformal region is the guarantee, and the downstream decision -inherits it. CRPTO follows this line but audits the other side of the decision -as well: after the optimizer selects a funded set, the realized weighted -miscoverage $V(\alpha)$ and the deterministic budget identity of Theorem 1 are -evaluated exactly on that funded set, so the certificate concerns the -allocation actually purchased rather than the input set alone. Its empirical -emphasis also differs: the uncertainty set is not an abstract benchmark -instance, but a credit-risk interval model with auditable lineage, paper tables, -and model-risk documentation. - -The third foundation is predict-then-optimize and decision-focused learning. -IJDS work on causal decision making sharpens the same warning: once an estimate -feeds an action, the relevant target can become the assignment rule rather than -only the intermediate effect-size estimate [@fernandezloria2022causaldecision]. -SPO+ and modern decision-focused learning -ask models to respect the loss surface induced by the downstream decision -[@elmachtoub2022; @donti2017; @mandi2024]. CRPTO is more -conservative. It does not retrain the PD model end-to-end through the optimizer. -Instead, it asks what can be achieved when a calibrated predictive system is -already frozen and the decision layer must remain explainable to credit-risk -reviewers. Robust losses for decision-focused learning [@schutte2024robust] -share this protective intent, but operate at training time rather than as a -post-hoc auditable constraint. - -The fourth foundation is machine learning and optimization for credit -decisions. Credit-scoring benchmarks define the performance frontier on retail -panels [@lessmann2015; @ayari2026; @xia2017]. Recent IJDS credit-risk work -shows how richer data structures such as firm graphs can improve rating -prediction [@das2023creditgraph], and cost-aware calibration work makes explicit -why probability quality matters when predictions feed asymmetric downstream -decisions [@yang2025costaware]. EJOR/Omega credit-scoring work similarly moves -from discrimination to economic uncertainty, using profit-based uncertainty, -rejection, parameter uncertainty, and multiobjective profit/risk metrics -[@xu2025profit_uncertainty_credit; @xu2024profit_risk_credit]. IJDS decision -papers also sharpen the distinction between an accurate intermediate estimate -and an effective automated decision -[@fernandezloria2022causaldecision; @fernandezloria2025observational], while -replication-robust analytics markets show the journal's appetite for robust, -reproducible decision systems [@falconer2026replication]. Work on fintech -lending and consumer-credit allocation studies platform structure, credit -invisibility, measurement noise, and scorecard equity -[@jagtiani2019altdata; @albanesi2024credit; @khandani2010consumer; -@fuster2022predictably]. -In the P2P/Lending Club decision neighborhood, prior work studies -instance-based investment support, P2P portfolio selection, profit scoring, -robust credit portfolio optimization, and multi-objective AI/OR funding -policies [@guo2016p2p; @zhao2016p2pportfolio; @serrano2016profitscoring; -@chi2019p2p; @babaei2020p2p; @aior2025lendingclub]. Recent ordinal conformal -credit-scoring work also means that the safe claim is not "no conformal -prediction in credit" [@kawasumi2026ordinal]. CRPTO does not compete on raw -ranking against this literature; its champion AUC is mid-range. -The contribution is the auditable bridge from a calibrated, frozen PD model to a -conformal robust portfolio decision, not another point on the credit-scoring -leaderboard. +inside the risk constraint. The economic objective remains point-PD expected +net return. This separation is important: $p_i$ prices expected loss, while +$q_i$ limits the amount of uncertainty the funded portfolio may carry. It also +removes the capped, tail-focused, and uncertainty-penalty branches that made an +earlier research frontier difficult to explain and easy to misread. + +The empirical design uses a temporal Lending Club panel. The conformal recipe +is fit inside the calibration period, and a later calibration holdout ranks a +declared $3\times3$ grid of round-number risk tolerances and conformal weights. +The final ranking artifact contains no defaults, realized returns, or +miscoverage. The fixed decision rule is then evaluated on loans originated from +January 2018 through September 2020. Earlier project development did inspect +this static OOT corpus, so we describe the final run as a transparent +retrospective lockbox replay, not as a pristine prospective or preregistered +trial. + +The paper makes three contributions. First, it gives an auditable +prediction-to-decision construction in which the economic objective and the +conformal guardrail have separate, inspectable roles. Second, it replaces +approximate cross-alpha scaling with an exact replay of the frozen conformal +recipe and a calibration-only final policy selector. Third, it reports the +price and limits of that guardrail against matched point-PD and more-conservative +comparators, including temporal slices where CRPTO wins and slices where it +does not. The novelty is the closed, inspectable decision protocol for a frozen +credit model, not a claim that conformal prediction, robust optimization, or +credit scoring is individually new. + +![CRPTO carries a frozen calibrated PD through an exact conformal replay, a simple portfolio guardrail, and a funded-set audit.](../reports/crpto/figures/crpto_fig1_journal_pipeline.png){#fig-crpto-pipeline width="92%" fig-alt="Four-stage CRPTO pipeline from calibrated PD to conformal intervals, portfolio allocation, and funded-set audit."} + +# Related Work -| IJDS precedent | Lesson for this submission | CRPTO extension | +CRPTO sits at the intersection of conformal prediction, robust optimization, +and decision-focused learning. Split conformal methods provide marginal +coverage without a parametric posterior, while Mondrian variants condition on +declared partitions [@vovk2005; @bostrom2021]. Conditional and weighted +extensions require additional structure, and exact conditional coverage is +generally unavailable without restrictive assumptions [@barber2021limits; +@barber2023beyond; @jonkers2024wcps]. We therefore distinguish population or +partition coverage from coverage after a portfolio has adaptively reweighted +the loans. + +Data-driven robust and contextual optimization translate predictive +uncertainty into decisions [@bertsimas2018datadriven; +@bertsimas2020prescriptive; @sadana2025contextual]. Recent work uses conformal +sets directly in robust optimization [@johnstone2021; @patel2024; +@sun2024ptc; @hu2026crc]. CRPTO is an applied complement: it retains a frozen +credit PD model, exposes the exact funded rows and uncertainty premium, and +compares the resulting allocation with a matched point-PD decision. + +Decision-focused learning and SPO+ train predictions against downstream regret +[@donti2017; @elmachtoub2022; @mandi2024]. That is a different institutional +choice. CRPTO asks what can be done after a calibrated model already exists and +must remain unchanged for governance. A synthetic SPO+ comparison is retained +in the supplement because it confirms the expected distinction: decision- +focused training improves its own regret objective, whereas CRPTO produces an +auditable uncertainty-constrained funded set. + +Credit-allocation research already covers profit scoring, P2P investment +recommendation, robust loan portfolios, rejection, and multiobjective risk +[@guo2016p2p; @zhao2016p2pportfolio; @serrano2016profitscoring; @chi2019p2p; +@babaei2020p2p; @xu2025profit_uncertainty_credit; +@xu2024profit_risk_credit]. Conformal credit scoring also means that the safe +claim is not "first use of conformal prediction in credit" +[@kawasumi2026ordinal]. The remaining gap is a file-backed protocol that shows +exactly how a conformal endpoint changes a budgeted credit decision and what +that change costs. + +| Literature family | Existing contribution | CRPTO boundary | |---|---|---| -| AI/OR perspective [@wiberg2025ai_or] | AI methods and OR structure strengthen each other when rigor and interpretability remain visible. | Makes the AI-to-OR bridge an auditable credit decision certificate. | -| Credit graph ML [@das2023creditgraph] | IJDS accepts credit-risk ML when data, method, and reproducibility are explicit. | Moves from rating prediction to a funded portfolio certificate. | -| Cost-aware calibration [@yang2025costaware] | Calibration matters because downstream miscalibration costs are asymmetric. | Prices uncertainty inside a budgeted allocation and exact audit. | -| Causal decision papers [@fernandezloria2022causaldecision; @fernandezloria2025observational] | Decision quality is not the same as estimating an intermediate quantity. | Evaluates the funded decision, not only PD quality. | -| Replication-robust analytics [@falconer2026replication] | Robustness and reproducibility are first-class IJDS concerns. | Supplies frozen evidence, exact checks, and a reproducibility harness. | - -: IJDS decision-science precedents and the CRPTO extension. - -Finally, recent work on conformal model selection for robust optimization, -non-exchangeable conformal risk control, valid selection among conformal sets, -multi-distribution conformal validity, online conformal portfolio methods, -end-to-end conformal risk training, robust conformal decision certificates, and -conformal satisficing fixes the boundary around the single IJDS claim -[@bao2025croms; @yang2026multidistribution; @liu2026portfolio; -@farinhas2024nonexchangeable_crc; @hegazy2025valid_selection_conformal_sets; -@zhou2025credo; @zhou2026creme; @zhao2025robust]. We use those ideas where they -can be evaluated from the frozen CRPTO evidence: -OCE/CVaR [@rockafellar2000cvar; @bental2007oce] appears as a tail-risk -diagnostic, robust satisficing appears as committee-style margin evidence, and -SPO+ motivates the regret-auditability frontier. The external Prosper and -Freddie/Mendeley runs are a separate, frozen replication protocol rather than a -new method-changing search. The remaining variants--optimized OCE/CVaR -objectives, non-exchangeable recalibration, formal post-selection conformal-set -selection, online protocols, causal layers, and hybrid decision-focused -training--are explicitly outside the submitted claim rather than hidden -acceptance criteria. - -## Closest Work Boundary - -Table 1 states the novelty boundary directly. CRPTO is not first in any broad -individual family; its claim is the combination of calibrated PD, conformal -uncertainty, robust credit-portfolio optimization, exact funded-set validation, -and reproducible governance. The row labels aggregate the closest -families discussed above: P2P/Lending Club OR [@guo2016p2p; -@zhao2016p2pportfolio; @serrano2016profitscoring; @chi2019p2p; @babaei2020p2p; -@aior2025lendingclub], conformal credit scoring [@kawasumi2026ordinal], -conformal robust optimization [@johnstone2021; @patel2024; @hu2026crc], -decision-focused learning [@elmachtoub2022; @mandi2024; @schutte2024robust], -and conformal finance portfolios [@noguer2024portfolio; @kato2025; -@liu2026portfolio]. - -| Neighboring literature | What it already contributes | What CRPTO adds | Why not enough for auditable credit decisions | -|---|---|---|---| -| P2P/Lending Club OR | Credit investment recommendation and robust/multi-objective funding models. | Conformal PD uncertainty as the uncertainty set plus exact funded-set validation. | Leaves a gap between prediction uncertainty and a post-allocation certificate that a credit committee can inspect. | -| Profit/risk credit scoring | Economic uncertainty, rejection, and cost-sensitive profit metrics for credit decisions. | Portfolio-level conformal premium and exact funded-set audit. | Still evaluates the classifier or rejection rule before a funded portfolio certificate. | -| Conformal credit scoring | Conformal intervals for ordinal credit scores. | A downstream robust portfolio decision, not only score uncertainty. | Stops at score uncertainty; it does not audit a budgeted funded set or economic policy. | -| Conformal robust optimization | Conformal sets used in downstream robust decisions. | A frozen Lending Club credit-risk model stack with paper-facing audit trail. | Establishes the decision logic, but not the credit-specific PD lineage and funded-set certificate. | -| Decision-focused learning | Training losses aligned with downstream regret. | A post-hoc governance path for institutions with existing calibrated PD models. | Improves training-time regret, but does not certify risk controls after a frozen production-style PD model. | -| Conformal finance portfolios | Portfolio use cases for financial markets. | Retail credit payoffs, default risk, funded-set checks, and model-risk documentation. | Market-return portfolios have different payoffs and governance obligations than binary default-driven credit funding. | - -: Closest work boundary for CRPTO. - -The novelty is the closed loop, not any single row of the table: a frozen -calibrated credit model becomes conformal upper endpoints, those endpoints -enter a robust funded-set decision, and the purchased allocation is audited -after the decision with file-backed governance. None of the neighboring lines -jointly supplies that frozen-model-to-funded-set certificate for a credit -portfolio. +| Credit scoring and calibration | Accurate, calibrated PD and cost-aware prediction. | Treats PD as an input contract; does not claim AUC leadership. | +| P2P and robust credit portfolios | Economic loan selection under risk and uncertainty. | Uses an exact conformal endpoint as a simple portfolio guardrail. | +| Conformal robust optimization | Coverage-backed uncertainty sets for downstream decisions. | Adds a frozen credit stack, funded-set audit, and matched economic comparator. | +| Decision-focused learning | Training-time reduction of decision regret. | Keeps the predictive model frozen and emphasizes post-hoc auditability. | +| Valid conformal selection | Corrects validity after selecting among sets or models [@hegazy2025valid_selection_conformal_sets]. | Motivates our narrow claim; formal selected-set validity is not asserted. | + +: Closest-work boundary for the submitted CRPTO claim. + +# Data and Evaluation Design + +The data are Lending Club retail loans originated from 2007 through 2020. The +feature contract contains only information available at origination. The model +and evaluation pipeline use temporal rather than random splits. + +| Split | Period | Loans | Role | +|---|---|---:|---| +| Train | Jun 2007--Mar 2017 | `1,346,311` | Fit the PD model and calibrator. | +| Calibration pool | Mar--Dec 2017 | `237,584` | Develop and freeze conformal and policy rules. | +| OOT evaluation | Jan 2018--Sep 2020 | `276,869` | Freeze-then-evaluate portfolio decisions. | + +: Temporal Lending Club design. + +The frozen conformal recipe uses the most recent 75% of the calibration pool +(`178,188` rows). Within that subset, `142,550` rows estimate conformal +quantiles and `35,638` later rows, from November and December 2017, form the +temporal development holdout. The conformal recipe uses calibration labels, as +conformal prediction requires. Policy ranking on the development holdout uses +only candidate parameters, solver status, expected point-PD objective, budget +use, effective-PD exposure, and conformal endpoint summaries. A schema guard +rejects outcome, default, realized-return, and miscoverage columns. + +The final OOT panel covers several regimes, including the 2020 disruption. We +report the full panel and five temporal slices (`2018H1`, `2018H2`, `2019H1`, +`2019H2`, and `2020+`). Each slice solves the same fixed policy on a fresh `$1M` +budget, so slice returns are comparable stress evaluations rather than +components that sum to the full-panel return. # Method -## Calibrated PD Layer - -The predictive layer estimates a one-period default probability for each loan. -The champion model is a CatBoost classifier trained on the frozen feature -contract and calibrated before it is exposed to conformal and optimization -layers. The paper reports discrimination and probability quality together: -the PD layer reaches AUC `0.7139`, Brier score `0.1544`, and expected -calibration error approximately `0.0070` on the paper-facing evaluation -summary. These numbers matter because the optimizer consumes probabilities, not -rankings alone. - -Calibration is treated as a contract. The downstream layers do not receive a -free-form classifier; they receive a calibrated PD vector, feature metadata, -and a model contract that fixes feature order and categorical handling. This -prevents a common reproducibility failure in applied predict-then-optimize -studies: a table can change because a preprocessing file changed, even -though the optimization code did not change. - -## Mondrian Conformal Layer - -For each loan `i`, let `p_hat_i` denote the calibrated PD. The conformal layer -forms prediction intervals on the PD scale and records the upper endpoint -`u_i(alpha)`. Operationally, CRPTO evaluates Mondrian partitions rather than a -single global interval. The selected uncertainty layer uses a -score-decile-based Mondrian partition selected by out-of-time interval quality, -while grade-based partitions remain the natural governance baseline. The -score-decile choice keeps each cell well populated even at the tight -$\alpha = 0.01$, where a per-cell $99\%$ upper endpoint needs on the order of -$100$ calibration points; finer grade-period partitions are sparser, and the -supplement flags where small cells weaken coverage (notably the external Freddie -panel). - -The resulting conformal summary is more than a scalar coverage -number. The paper-facing metrics include 90% coverage `0.9297`, 95% coverage -`0.9664`, average 90% interval width `0.7842`, minimum group 90% coverage -`0.9190`, and 90% Winkler score `1.1107` for the promoted conformal winner. -Because the outcome is binary and the intervals live on the clipped PD scale, -raw width is not a standalone utility claim. Its decision role is relative: -upper endpoints rank loans by protected downside risk, the funded-set audit -shows mean upper endpoints rising from `0.12529` in A-B to `0.52587` in E-G, and -the promotion gate uses Winkler score, funded-set miscoverage, and -$\Gamma_{\mathrm{CP}}$ rather than treating narrowness as the objective. -The full gate also scores material coverage, group coverage, interval width, and -alert rate, but not exact-nominal-coverage backtests: conformal intervals -over-cover by design, so testing for exact nominal coverage would penalize the -safety margin the method is meant to provide on a large out-of-time sample. - -## Robust Portfolio Layer - -The decision variable $x_i$ is the allocation fraction for each eligible loan; -$x_i a_i$ is the funded exposure. The optimizer maximizes expected net -economic return under a `$1M` budget and policy constraints that replace point -PD with a declared effective decision score $q_i(\alpha;\theta)$. Every policy -used in the frontier satisfies -$\hat p_i\leq q_i(\alpha;\theta)\leq u_i(\alpha)$; $\theta$ identifies whether -the policy is a linear blend, capped blend, or tail-focused blend. The selected -body point has `risk_tolerance = 0.1715`, -`policy_mode = capped_blended_uncertainty`, `gamma = 0.5475`, and -`uncertainty_aversion = 0.05`. - -Schematically, the robust decision layer solves a budgeted allocation problem -of the form +## Calibrated PD and Exact Conformal Replay + +Let $p_i\in[0,1]$ be the calibrated PD for loan $i$. The predictive layer is a +frozen CatBoost classifier with AUC `0.7139`, Brier score `0.1544`, and expected +calibration error about `0.0070` on the paper-facing evaluation. These values +are not presented as a leaderboard result; probability quality matters because +the downstream objective consumes PD directly. + +The conformal recipe partitions calibrated scores into five score-quantile +Mondrian cells. For calibration loan $j$, it computes a scaled residual $$ -\begin{aligned} -\max_x\quad & \sum_i x_i a_i \left(c_i - q_i(\alpha;\theta)\,L\right) \\ -\text{s.t.}\quad & \sum_i x_i a_i \le B,\\ -& \sum_i x_i a_i q_i(\alpha;\theta) - \le \tau \sum_i x_i a_i,\\ -& 0 \le x_i \le \bar x_i, -\end{aligned} +s_j = \frac{|Y_j-p_j|}{\sqrt{p_j(1-p_j)}}, $$ -where `a_i` is exposure, `c_i` is the loan coupon (interest rate), `L` is the -loss-given-default (`L = 0.45` in the frozen evaluation), and $\tau$ is the -risk-tolerance cap. The linear member of the policy family is +with numerical clipping near zero. Each cell uses the finite-sample `higher` +quantile at the frozen conservative level `0.095` for the target +$\alpha=0.10$. Recorded group and temporal widening factors are then applied +without narrowing. The upper endpoint is +$u_i=\min\{1,p_i+\widehat q_{g(i)}\sqrt{p_i(1-p_i)}\}$ after those frozen +adjustments. + +The replay implementation reconstructs this recipe from its result payload. At +the reference 90% level, it reproduces the stored point, lower, and upper +vectors with maximum absolute error below `6.67e-16`. This matters because an +earlier exploratory analysis scaled 90% row radii using average widths from a +different conformal family. Those cross-alpha values are retained only as +historical provenance and are not used here. + +## One Portfolio Policy + +Let $a_i$ be loan amount, $x_i\in[0,1]$ the funded fraction, $r_i$ the coupon, +$L=0.45$ loss given default, $B=1{,}000{,}000$, and $\tau$ the portfolio risk +tolerance. CRPTO solves $$ -q_i(\alpha;\gamma) -= \hat p_i + \gamma\left(u_i(\alpha)-\hat p_i\right) +\begin{aligned} +\max_x\quad & \sum_i x_i a_i(r_i-p_iL) \\ +\text{s.t.}\quad +& \sum_i x_i a_i \le B, \\ +& \sum_i x_i a_i q_i \le \tau\sum_i x_i a_i, \\ +& 0\le x_i\le \bar x_i, +\end{aligned} $$ -on the PD scale. Capped and tail-focused members transform that score while -remaining between the point PD and upper endpoint. For the selected allocation, -the cap is inactive on all 314 funded rows, so its effective score equals the -linear expression above; that fact is audited rather than assumed for other -frontier policies. The objective is the *expected* net return -`c_i - q_i * L`; the headline realized return is the -post-hoc accounting of the same funded set on observed defaults -(a funded loan earns `c_i * a_i` if it survives and loses `L * a_i` if it -defaults). Separating the optimized expectation from the realized accounting is -deliberate: the optimizer never sees outcomes, so the realized figure is an -out-of-sample audit of the policy, not the objective it maximized. Additional -operational filters and caps live in the frozen policy configuration; the body -displays the core statistical-to-decision contract because that is the reusable -CRPTO pattern. - -The lowercase $\gamma$ is a policy parameter; the uppercase quantities are -post-allocation funded-set metrics. With weights $w_i$ proportional to funded -exposure, define +with the existing concentration and eligibility constraints. The selected +policy uses $\tau=0.17$ and $q_i=(p_i+u_i)/2$. The objective is intentionally +based on $p_i$, not $q_i$: expected economics and uncertainty feasibility are +separate contracts. The matched point-PD baseline changes only the risk score +to $q_i=p_i$ while holding candidates, budget, concentration, LGD, solver, and +risk tolerance fixed. + +For funded-exposure weights +$w_i=x_ia_i/\sum_jx_ja_j$, define $$ \begin{aligned} -\Gamma_{\mathrm{CP}}(\alpha) - &= \sum_i w_i\bigl(u_i(\alpha)-\hat p_i\bigr),\\ -\Gamma_{\mathrm{int}}(\alpha) - &= \sum_i w_i\bigl(q_i(\alpha;\theta)-\hat p_i\bigr),\\ -\Gamma_{\mathrm{res}}(\alpha) - &= \sum_i w_i\bigl(u_i(\alpha)-q_i(\alpha;\theta)\bigr). +\Gamma_{\mathrm{CP}} &= \sum_iw_i(u_i-p_i),\\ +\Gamma_{\mathrm{int}} &= \sum_iw_i(q_i-p_i),\\ +\Gamma_{\mathrm{res}} &= \sum_iw_i(u_i-q_i),\\ +B_u &= \sum_iw_i u_i. \end{aligned} $$ -Thus $\Gamma_{\mathrm{CP}}=\Gamma_{\mathrm{int}}+\Gamma_{\mathrm{res}}$: -the first component is uncertainty internalized by the decision score and the -second is the residual needed to recover the endpoint budget. At -$\alpha=0.01$, the selected point has -$\Gamma_{\mathrm{CP}}=0.162616$, $\Gamma_{\mathrm{int}}=0.089032$, -$\Gamma_{\mathrm{res}}=0.073584$, and weighted miscoverage $V=0.035350$. -Its realized weighted default rate is also `0.035350`, below $\tau=0.1715$, -so its realized risk-tolerance excess is zero. This terminology matters: -zero excess is an empirical audit result, not a violation metric for the -deterministic identity below. - -# Theory - -The theoretical role of conformal prediction in CRPTO is modest and explicit. -For a fixed allocation evaluated on exchangeable calibration/test data, -conformal coverage controls the expected rate at which outcomes fall outside -the constructed uncertainty intervals. When funded-set weights are -non-negative and normalized, this yields a weighted miscoverage quantity -$V(\alpha)$ that can be monitored after the decision. Two statements are kept -separate. The deterministic portfolio identity (Theorem 1(i)) holds for any -allocation and needs no distributional assumption; it is the accounting the -exact certificate verifies. The probabilistic statement (Theorem 1(ii)) is a -Markov argument that requires weighted funded-set validity, -$E[V(\alpha)] \leq \alpha$, stated as Assumption 1. This is a modeling -assumption, not a property the single frozen draw establishes: on the selected -funded set the realized weighted miscoverage is $V(0.01) = 0.035350$, *above* -the nominal $\alpha = 0.01$, which is the expected price of evaluating an -adaptively selected subportfolio rather than a fresh population. What the paper -certifies is therefore the exact accounting together with the safety level -$V \leq \sqrt{\alpha}$ that Markov delivers, not a claim that the funded set -attains nominal $\alpha$-coverage or that post-selection evaluation creates a -stronger conformal guarantee. Recent work on valid selection among conformal -sets reinforces this boundary: selecting the most attractive set or policy after -seeing multiple valid candidates is itself a statistical operation that needs -its own protocol [@hegazy2025valid_selection_conformal_sets]. - -| Claim component | CRPTO evidence | Boundary | -|---|---|---| -| Deterministic portfolio identity | Exact funded-set audit computes $B_u$, $V$, and the conformal-premium decomposition after allocation. | Does not require a new statistical guarantee. | -| Split/Mondrian validity | OOT coverage, minimum group coverage, and temporal diagnostics. | Not exact conditional coverage for every borrower profile. | -| Weighted funded-set validity | Exact alpha-safe certificate plus A23 weighted/multi-distribution stress evidence. | Assumption for the theorem; empirical audit after frozen selection. | -| Post-selection robustness | Nested search chain plus a declared finite-grid frontier with 50,010 deduplicated semantic policies and 27,508 all-alpha above-floor policies. | Strong audit evidence, not a prospective live-selection guarantee or a continuous-region theorem. | -| Formal selected-set validity | A future nested or stability-based selection protocol could target this directly [@hegazy2025valid_selection_conformal_sets]. | Not claimed by the current frozen frontier. | +For the midpoint policy, +$\Gamma_{\mathrm{int}}=\Gamma_{\mathrm{res}}=Gamma_{\mathrm{CP}}/2$. +This identity is one reason to prefer the midpoint over nonlinear caps or tail +rules: every quantity has a direct interpretation. + +## Calibration-Only Final Selector -: Assumption-to-evidence map for the CRPTO bound. +The candidate set crosses +$\tau\in\{0.15,0.17,0.19\}$ with +$\gamma\in\{0.25,0.50,0.75\}$ in +$q_i=p_i+\gamma(u_i-p_i)$. A candidate is eligible when the solver is optimal, +at least 99.9% of the budget is allocated, the effective-PD cap holds, and -The figure below is the paper's main guardrail against overclaiming: it -separates what is deterministic, what is assumed, and what is empirically -certified after the frozen selection. +$$ +B_u+\sqrt{0.10}\le 0.60 +$$ -![The bound claim stack separates deterministic accounting, the weighted-validity assumption, and the frozen exact certificate.](../reports/crpto/figures/crpto_fig20_bound_claim_layers.png){#fig-bound-claim-stack width="94%" fig-alt="Four-block bound claim stack separating conformal endpoint, deterministic identity, weighted validity assumption, and exact frozen certificate."} +on the calibration development holdout. Among eligible candidates, the rule +maximizes expected point-PD objective. Five of nine candidates pass, and the +selected candidate is $\tau=0.17,\gamma=0.50$. The screen value `0.60` is a +declared decision threshold for the final protocol, not an estimated property +of OOT outcomes. -Dependence is handled conservatively rather than assumed away. The main bound -does not require loan-level independence. Temporal structure is addressed by -the out-of-time design and backtests; any sharper concentration argument is -kept in the supplement, where the extra independence structure is stated -explicitly. +| Candidate | $\tau$ | $\gamma$ | Calibration expected objective | $B_u+\sqrt{0.10}$ | Status | +|---|---:|---:|---:|---:|---| +| Higher-return, low guardrail | `0.17` | `0.25` | `$121,761.88` | `0.708835` | Ineligible | +| Selected midpoint | `0.17` | `0.50` | `$110,346.16` | `0.577275` | Selected | +| More-conservative blend | `0.17` | `0.75` | `$104,272.78` | `0.519696` | Eligible | +| Higher risk tolerance | `0.19` | `0.50` | `$113,591.27` | `0.611017` | Ineligible | -The compact validity ladder below fixes that boundary. CRPTO uses the first two -levels as evidence, states weighted funded-set validity as the theorem's -portfolio-level assumption, reports multi-distribution checks as diagnostics, -and leaves online/live control for a new protocol. +: Calibration selector examples. A36 reports all nine candidates. -| Validity level | What it supports | CRPTO status | -|---|---|---| -| Marginal split conformal | Population-level coverage under exchangeability. | Core interval guarantee. | -| Mondrian/group conformal | Coverage within declared partitions. | Used for score-decile and grade audits. | -| Weighted/localized coverage | Coverage under weights, local neighborhoods, or selected groups. | Explicit theorem assumption plus A23 diagnostic evidence. | -| Non-exchangeable conformal risk control | Loss control under weighted relevance or time/source shift [@farinhas2024nonexchangeable_crc]. | Future recalibration path; A23--A24 are read-only diagnostics. | -| Post-selection conformal validity | Coverage after selecting among multiple valid sets or policies [@hegazy2025valid_selection_conformal_sets]. | Future protocol; current frontier is an exact finite-grid audit. | -| Multi-distribution coverage | Robustness across multiple source distributions. | Read-only stress evidence, not recalibration. | -| Online/adaptive coverage | Sequential alpha updates under live drift. | A24 replay only; not a live deployment claim. | - -: Coverage-validity ladder used in the paper claims. - -The bound is therefore read as three linked objects: a deterministic -accounting identity, an explicitly stated statistical assumption, and the -frozen empirical certificate that verifies the promoted decision exactly on -the 276,869-loan OOT evaluation. We now state the first two formally. - -**Setup.** Fix the promoted allocation $x$, chosen without access to OOT -labels, with exposures $a_i > 0$ and funded-set weights -$w_i = x_i a_i / \sum_j x_j a_j$, so that $w_i \geq 0$ and $\sum_i w_i = 1$. -For each funded loan let $Y_i \in [0, 1]$ be the realized outcome on the PD -scale, $u_i(\alpha) \in [0, 1]$ its upper conformal endpoint, and -$Z_i(\alpha) = \mathbf{1}\{Y_i > u_i(\alpha)\}$ the miscoverage indicator. -The weighted funded-set miscoverage is -$V(\alpha) = \sum_i w_i Z_i(\alpha)$. Probabilities and expectations below are -taken over the exchangeable calibration/test draw, conditional on the frozen -recipe, declared partitions, and allocation rule. - -**Assumption 1 (weighted funded-set validity).** -$E[V(\alpha)] \leq \alpha$ under the funded-set weights $w$ for that draw. This -means the expected weighted miss rate on the funded dollars is no larger than -the nominal conformal level. It is the explicit price of evaluating a selected -portfolio rather than a single population, and it does not follow from marginal -split conformal. The -funded-set weights $w_i \propto x_i a_i$ are chosen by the optimizer and depend -on the conformal endpoints $u_i(\alpha)$, so they are not measurable with -respect to the Mondrian partition and inherit no per-cell coverage guarantee. -The assumption is therefore stated, audited empirically after the frozen -selection, and never silently upgraded to a guarantee -- and the audit does not -find it slack: the realized $V(0.01) = 0.035350$ exceeds $\alpha = 0.01$, so the -operative safety level is the weaker $V \leq \sqrt{\alpha}$. - -**Theorem 1 (distribution-free Markov bound under weighted funded-set validity).** -Let $B_u(\alpha) = \sum_i w_i u_i(\alpha)$ be the weighted conformal -upper-endpoint budget of the funded set. Then: - -(i) *(deterministic)* $\;\sum_i w_i Y_i \leq B_u(\alpha) + V(\alpha)$ always; - -(ii) *(statistical)* under Assumption 1, for every $t > 0$, -$P(V(\alpha) \geq t) \leq \alpha / t$, and in particular +## Accounting and Statistical Boundary + +Let $Z_i=\mathbf 1\{Y_i>u_i\}$ and +$V=\sum_iw_iZ_i$. Because $Y_i\le u_i+Z_i$ for every funded loan, $$ -P\!\left(\sum_i w_i Y_i \;\geq\; B_u(\alpha) + \sqrt{\alpha}\right) - \;\leq\; \sqrt{\alpha}. +\sum_iw_iY_i\le B_u+V $$ -*Proof sketch.* Since $Y_i \leq u_i(\alpha) + Z_i(\alpha)$ for every loan, -part (i) follows by taking the $w$-weighted sum; it is portfolio accounting and -needs no probability. Part (ii) is Markov's inequality applied to the -nonnegative variable $V(\alpha)$ with $E[V(\alpha)] \leq \alpha$ [@ghosh2002], -combined with (i). The full proof is in Online Supplement Appendix A. $\square$ +holds deterministically. It is an exact funded-set accounting statement, not a +coverage theorem. + +If one additionally assumes weighted funded-set validity, +$\mathbb E[V]\le\alpha$, Markov's inequality gives -**The optimizer's cap versus the endpoint budget.** The robust layer -constrains the policy-specific effective score, not $B_u(\alpha)$ directly: -$\sum_i w_i q_i(\alpha;\theta)\leq\tau+s$, where $s\geq0$ is any recorded -solver cap slack. The policy-aware decomposition gives $$ -B_u(\alpha) -= \sum_i w_i q_i(\alpha;\theta)+\Gamma_{\mathrm{res}}(\alpha) -\;\leq\;\tau+s+\Gamma_{\mathrm{res}}(\alpha). +\Pr\!\left(\sum_iw_iY_i\ge B_u+\sqrt{\alpha}\right) +\le\sqrt{\alpha}. $$ -For a pure linear blend only, -$\Gamma_{\mathrm{res}}=(1-\gamma)\Gamma_{\mathrm{CP}}$. The selected capped -policy has no active row-level cap on its funded set, its effective-score cap -binds with $s=0$, and therefore the same numerical shortcut happens to hold: -$B_u(0.01)=0.1715+0.073584=0.245084$. The deterministic identity gives -$\sum_i w_iY_i\leq0.245084+V(0.01)=0.280434$, while the observed left-hand side -is `0.035350`. The exact Markov loss threshold is -$T_{0.01}=B_u(0.01)+\sqrt{0.01}=0.345084$; under Assumption 1, -$P(\sum_iw_iY_i\geq T_{0.01})\leq0.10$. It is a probabilistic event threshold, -not a deterministic risk cap. This policy-aware form is essential for capped -and tail-focused frontier points, where the linear shortcut need not hold. - -**Remark 1 (why $t = \sqrt{\alpha}$, and why Markov).** -The choice $t = \sqrt{\alpha}$ is made for interpretability, not optimality: -it gives the clean reading "miscoverage exceeds $\sqrt{\alpha}$ with -probability at most $\sqrt{\alpha}$" (for $\alpha = 0.01$, a $0.10$ excess -with probability at most $0.10$). Markov is deliberately the weakest -defensible argument: it uses only the first moment. Supplement -Propositions A.1--A.2 separate the boundary: under Assumption 1 alone the -best second-moment (Cantelli) threshold is *worse* than Markov, while explicit -cross-cluster structure is the extra condition under which Hoeffding-style -tightening becomes available [@hoeffding1963; @boucheron2013concentration]. We -keep those tightenings in the online supplement (A21) rather than in the -body, because the contribution here is the auditable decision construction, -not the sharpest possible tail bound. The exact certificate in this paper is -the empirical audit of the frozen selected policy, not a stronger -post-selection conformal theorem. - -The theorem and the two supplement propositions should be read as one small -triptych. Theorem 1 gives the paper's guarantee once weighted funded-set -validity is accepted. Proposition A.1 shows that, without additional structure, -Markov is not a placeholder for a missing second-moment bound; it is the sharp -first-moment statement. Proposition A.2 then asks what extra structure would -buy a tighter threshold. In this temporal credit panel the defensible version is -cross-period or period-grade independence after the frozen recipe and allocation -are fixed: within a period, grade, or period-grade cell, defaults and interval -misses may remain dependent. The observed funded set is too exposure -concentrated for that cluster argument to tighten the headline bound, which is -why the body keeps Markov and the supplement reports the cluster calculation as -a sensitivity check. - -# Experimental Design - -The empirical study uses Lending Club retail-loan data covering originations -from 2007 through 2020. The raw panel is cleaned into a static feature store and -split temporally with a January 2018 cutoff, so that calibration and evaluation -use only loans originated after the training window. The calibration block plays -the dual role required by split conformal: it is held out from training and used -exclusively to fit the conformal quantiles. The final evaluation set contains -276,869 loans, large enough to stress both probability calibration and -decision-level robustness. -| Split | Period | Loans | Role | -|---|---|---:|---| -| Train | Jun 2007 -- Mar 2017 | `1,346,311` | Fit PD model and monotonic constraints. | -| Calibration | Mar 2017 -- Dec 2017 | `237,584` | Fit Mondrian conformal quantiles (held out from training). | -| Test (OOT) | Jan 2018 -- Sep 2020 | `276,869` | Evaluate coverage, run the portfolio decision, and audit the exact funded set. | - -: Temporal out-of-time split. The January 2018 cutoff keeps the test window -(including the 2020 COVID regime) strictly after training and calibration, so no -future information leaks into the funded-set decision. The displayed periods are -monthly vintage labels; split assignment is row-disjoint in code, so the shared -March 2017 label marks the internal cutoff rather than duplicated loans. - -The out-of-time design is adversarial to the method: the test window -spans an expansion (2018--2019) and a regime break (2020 COVID), so coverage and -the funded-set certificate are measured under a documented distribution shift -rather than on a random split that would let the model see the future. - -The design distinguishes three kinds of computation. Predictive and conformal -searches choose models, calibration, partitions, and policy families. Those -searches are frozen for this manuscript. Paper-facing reruns regenerate tables, -figures, evidence summaries, and manuscript surfaces from frozen inputs. This -separation is central to the reproducibility claim: the manuscript can evolve -without quietly reopening the policy search that selected the promoted result. -Governance keeps the two certified results distinguishable: the frozen upstream -record defines the declared return floor, while the selected-policy governance -files define the body point and its frontier. Automated claim-sync tests assert -that every body-claim number printed in the manuscript and supplement matches -those files. A validation harness additionally rebuilds the promoted conformal -intervals from the frozen PD binaries and recorded recipe and verifies the -published endpoints and coverage summaries. Gradient-boosted retraining is not -presented as the routine reproduction target across machines; retraining would -be a new research run, not the routine reproduction path for this submission. - -All primary evidence objects are represented as files with explicit ownership: model -binaries, calibration objects, conformal intervals, portfolio allocations, -tables, figures, and status reports. The anonymous submission describes the -bundle without revealing author identity. Repository and remote-storage URLs -will be disclosed according to the journal's double-anonymous and -data/code-disclosure policy. - -The body-supplement split is fixed before submission. The body keeps the CRPTO -pipeline, the alpha-to-portfolio link, the finite-grid frontier, and the -core metrics, plus the compact regret-auditability frontier. The online -supplement carries A3--A40, the conformal finalist ablation, funded-set loan -audit, tail-risk diagnostics, satisficing margins, dependence diagnostics, the CVaR/OCE -tail-constrained re-optimization (A22), the multi-distribution (A23) and -online ACI-stability (A24) diagnostics, the external economic replication -tables (A25--A34), the selected-policy frontier and funded-set audits -(A35--A40), MRM/fairness material, and reproduction commands. This keeps -the IJDS body focused while preserving the audit trail that reviewers need. - -## Multi-Dataset External Replication Protocol - -The external-replication layer is narrower than a new benchmark -campaign. We reuse the frozen CRPTO recipe--CatBoost PD, calibration, -train-only WOE/IV feature screening, Mondrian conformal intervals, and the same -bound-aware robust LP--on two credit datasets with economic fields. Prosper -contributes final-status marketplace loans with observed outcome, loan amount, -yield/rate, and a temporal OOT window [@prosperLoanData]. Freddie/Mendeley -contributes processed single-family mortgage panels derived from the Freddie Mac -loan-level ecosystem, including 12--60 month default windows and train/OOS/OOT -splits [@freddieMacSfLoanLevel; @mushava2023classimbalance]. Home Credit was -audited but not promoted because it lacks a clean investment-return and exposure -contract comparable to Lending Club, Prosper, and Freddie. - -The replication gate is fixed before inclusion: global 90% coverage must meet -target, the $\alpha=0.01$ conformal coverage must be reportable, and the LP must -return a positive robust objective when economic fields are available. This gate -does not re-promote the Lending Club champion and does not claim a new exact -funded-set theorem for every external portfolio. Its purpose is empirical: -address whether the method survives two materially different credit products. +This assumption does not follow from marginal split conformal because the +optimizer chooses the weights using $p_i$ and $u_i$. We therefore treat the +Markov expression as a secondary, assumption-conditional sensitivity. The +primary evidence is the exact replay, OOT-outcome-column-free policy ranking +conditional on the frozen conformal recipe, and observed funded-set audit. +Full proofs and sharper-assumption diagnostics are in the online supplement. # Results -The results section is ordered around the reviewer decision object: first the -selected-policy certificate, then the A35 finite-grid frontier that prevents a -singleton reading, the A40 matched point-PD baseline, and finally the external -recipe-transfer checks. The core -metric table summarizes the paper-facing metrics. The calibrated PD -layer is not sold as a leaderboard model: AUC `0.7139` is sufficient only -because the downstream decision consumes calibrated probabilities, not rankings -alone. Its Brier score `0.1544` and ECE near `0.0070` are therefore as important -as discrimination. The conformal layer over-covers marginally at the reported -levels (90% coverage `0.9297`, 95% coverage `0.9664` for the conformal winner). -The portfolio layer then turns this uncertainty into an exact finite-grid -return-bound frontier. The selected policy passes the -$V \leq \sqrt{\alpha}$ certificate at the tightest reported level and has zero -realized risk-tolerance excess. - -| Layer | Metric | Value | -|---|---:|---:| -| PD | AUC | `0.7139` | -| PD | Brier score | `0.1544` | -| PD | ECE | `0.0070` | -| Conformal | Coverage 90% | `0.9297` | -| Conformal | Coverage 95% | `0.9664` | -| Conformal | Minimum group coverage 90% | `0.9190` | -| Portfolio | Body-point robust return | `$184,832.48` | -| Portfolio | Weighted realized default rate | `0.035350` | -| Portfolio | $V(\alpha = 0.01)$ | `0.035350` | -| Portfolio | $\Gamma_{\mathrm{CP}}(\alpha = 0.01)$ | `0.162616` | -| Portfolio | $\Gamma_{\mathrm{res}}(\alpha = 0.01)$ | `0.073584` | -| Portfolio | Endpoint budget $B_u(\alpha = 0.01)$ | `0.245084` | -| Portfolio | Exact Markov loss threshold | `0.345084` | -| Portfolio | Realized risk-tolerance excess | `0.0` | -| Portfolio | Declared alpha-grid pass | `8/8` | - -: Frozen paper-facing metrics by layer. - -The exact certificate is an accounting claim. Here "exact" means the quantities -are computed directly on the frozen OOT funded set rather than approximated by a -surrogate table or visual proxy, and the deterministic identity requires no -distributional assumption. The declared empirical `pass` combines -$V(\alpha)\leq\sqrt{\alpha}$ with realized risk-tolerance excess no larger than -$\alpha$; for the selected point that excess is zero. This screen is *not* a -claim of nominal $\alpha$-coverage, which the selected funded set does not attain -($V=0.035350>\alpha=0.01$), and the excess criterion is not presented as a new -probabilistic theorem. - -| $\alpha$ | $\Gamma_{\mathrm{CP}}$ | $\Gamma_{\mathrm{res}}$ | $V(\alpha)$ | $B_u$ | Markov threshold | Risk excess | Pass | -|---:|---:|---:|---:|---:|---:|---:|:---:| -| `0.01` | `0.162616` | `0.073584` | `0.035350` | `0.245084` | `0.345084` | `0.00000` | yes | - -: Exact certificate for the selected body point. The Markov column is the exact -event threshold $B_u+\sqrt{\alpha}$, not a deterministic cap. - -This makes $\Gamma_{\mathrm{CP}}$ more than a diagnostic line item. It is the -amount of conformal robustness the optimizer accepts in order to keep the funded -set inside the $\sqrt{\alpha}$-safe region. A credit reviewer can therefore read -$\Gamma_{\mathrm{CP}}=0.162616$ as the total conformal premium carried by the -selected body point, $\Gamma_{\mathrm{res}}=0.073584$ as the part not internalized -by its decision score, $V=0.035350$ as the realized weighted noncoverage audit, -and `0.345084` as the loss level whose exceedance probability is bounded by -`0.10` under Assumption 1. - -The central empirical object is now a return-bound frontier rather than a single -winner. The consolidated frontier surface deduplicates 51,678 raw rows into -50,010 semantic policies; 27,508 policies both pass every level in the declared -alpha grid and exceed the declared return floor. The terminal endpoint search -alone evaluates 37,068 policies and 296,544 exact alpha checks, with -37,068/37,068 policies passing all eight alpha levels. - -The provenance of the frontier deserves one explicit sentence, because two -certified numbers coexist in the project's governance record. The declared -return floor `$170,464.54` is itself the realized return of the previously -certified bound-aware allocation on the same frozen chain, retained as the -frozen upstream baseline; the declared frontier is a deterministic re-evaluation -of a pre-declared finite policy grid over the *same* frozen PD model and -Mondrian conformal intervals, so no upstream object (model, calibrator, or -interval) was regenerated when the body point moved from the floor to -`$184.8K`. The earlier certificate is not discarded: it becomes the floor -that every eligible frontier policy must beat, which is why the frontier is -reported with the floor surplus rather than as a replacement champion. - -| Policy role | Source | Realized return | $V(0.01)$ | $\Gamma_{\mathrm{CP}}$ | Markov threshold | Pass | -|---|---|---:|---:|---:|---:|:---:| -| Minimum Markov-threshold endpoint | terminal | `$170,467.27` | `0.031875` | `0.095719` | `0.273036` | `8/8` | -| Low-threshold balanced endpoint | terminal | `$171,006.20` | `0.031875` | `0.097190` | `0.274789` | `8/8` | -| Highest return under threshold <= `0.30` | bound-closure | `$174,552.51` | `0.035875` | `0.120988` | `0.299997` | `8/8` | -| Highest return under threshold <= `0.345` | micro-ext | `$184,800.41` | `0.035350` | `0.162562` | `0.344997` | `8/8` | -| Body/default balanced point | micro-ext | `$184,832.48` | `0.035350` | `0.162616` | `0.345084` | `8/8` | -| Highest return under threshold <= `0.36` | micro-ext | `$186,050.73` | `0.037750` | `0.174600` | `0.358685` | `8/8` | -| Max-return economic endpoint | micro-ext | `$223,458.14` | `0.069575` | `0.457438` | `0.697056` | `8/8` | - -: Pool93 finite-grid return-bound frontier. Each threshold is computed from the -exact funded-set endpoint budget. `terminal` rows come from the conservative -endpoint sweep; `bound-closure` and `micro-ext` denote frozen local grids. All -rows are finite-grid points, not continuous optima. - -The table gives the manuscript its decision geometry. The body/default point is -not the highest-return point and not the tightest-bound point; it is the -balanced point selected by the declared return-bound lens. The neighboring -strict `<= 0.345` row is reported separately because it is a different finite -policy: it earns `$184,800.41` at threshold `0.344997`, whereas the body/default -row earns `$184,832.48` at threshold `0.345084`. The endpoint at `0.273036` -shows how conservative the certified frontier can become while preserving the -return floor, and the `$223.5K` endpoint shows the return available when the -committee accepts a threshold of `0.697056`. The latter policy is tail-focused; -its residual premium cannot be recovered with the linear -$(1-\gamma)\Gamma_{\mathrm{CP}}$ shortcut. The supplement reports the full -policy-aware frontier and traceability details. - -## Matched Point-PD Baseline - -To isolate what the conformal decision layer buys, we solve a matched two-stage -LP on the same 276,869 candidates with the same `$1M` budget, concentration cap, -$\tau=0.1715$, LGD, solver, and two-stage objective. The only change is that the -baseline uses calibrated point PD in both its objective and risk constraint. -Neither optimizer sees OOT outcomes; defaults enter only in the frozen post-hoc -audit. - -| Policy | Realized return | Funded | Weighted default / $V$ | $\Gamma_{\mathrm{CP}}$ | $B_u$ | Markov threshold | +## Exact 90% Conformal Evidence + +At the active target $\alpha=0.10$, exact OOT coverage is `0.934836`, average +interval width is `0.788879`, minimum score-partition coverage is `0.926310`, +minimum letter-grade coverage is `0.926797`, and `51.7873%` of upper endpoints +equal one. The intervals are conservative and broad, which is expected for a +binary outcome on the probability scale. + +The exact alpha sensitivity explains why the paper does not headline a 99% +interval. At target alpha `0.01`, empirical coverage is `0.996720`, but average +width is `0.988215` and `93.5424%` of upper endpoints equal one. Such endpoints +carry almost no ranking information for a portfolio. The selected 90% level is +the frozen recipe's reference level and preserves materially more decision +resolution. A35 reports the complete sensitivity from alpha `0.01` to `0.20`. + +## Full OOT Funded-Set Audit + +The fixed midpoint policy allocates the full `$1M` budget across 308 loans. Its +expected point-PD objective is `$168,271.56`; realized return is `$179,327.59`. +The weighted default rate is `0.039375`, and weighted miscoverage is `0.036875`. + +| Quantity | OOT value | +|---|---:| +| Weighted point PD | `0.081949` | +| Weighted midpoint score | `0.170000` | +| $\Gamma_{\mathrm{CP}}$ | `0.176102` | +| $\Gamma_{\mathrm{int}}$ | `0.088051` | +| $\Gamma_{\mathrm{res}}$ | `0.088051` | +| Endpoint budget $B_u$ | `0.258051` | +| Observed accounting bound $B_u+V$ | `0.294926` | +| Conditional Markov threshold | `0.574279` | + +: Exact full-OOT audit of the selected policy. + +The observed weighted outcome `0.039375` is well below the exact accounting +right-hand side `0.294926`. Under the additional weighted-validity assumption, +the `0.574279` event threshold has probability bound +$\sqrt{0.10}=0.316228$. This loose probability statement is not interpreted as +a direct default cap. The operational controls are $\tau=0.17$, the midpoint +score, and the exact funded-set diagnostics. + +The fixed-allocation bootstrap gives a 95% return interval of +`$162,706.17`--`$193,924.74` from 5,000 funded-loan resamples. It does not +resample the model, conformal recipe, policy selector, or optimizer and is +therefore a contribution-level stability diagnostic rather than a full +pipeline confidence interval. + +## Matched Point-PD and More-Conservative Comparators + +| Policy | Funded | Realized return | Weighted default | Miscoverage | $B_u$ | Conditional threshold | |---|---:|---:|---:|---:|---:|---:| -| Point-PD two-stage LP | `$196,369.14` | `225` | `0.118400` | `0.526736` | `0.680579` | `0.780579` | -| Selected CRPTO | `$184,832.48` | `314` | `0.035350` | `0.162616` | `0.245084` | `0.345084` | - -: Matched point-PD baseline on the frozen Lending Club OOT panel. Weighted -default and $V$ coincide in these two funded sets because each observed default -lies above its conformal upper endpoint. Full fields are in A40. - -CRPTO gives up `$11,536.66`, or `5.875%`, of the baseline's realized return. In -exchange, the weighted default rate and miscoverage fall by `0.08305` (8.305 -percentage points), and the exact Markov loss threshold falls by `0.435495` -(43.55 percentage points). Both allocations have zero realized -risk-tolerance excess because their default rates remain below $\tau$, but the -point-PD allocation fails the tightest Markov safety screen -($V=0.1184>\sqrt{0.01}=0.10$). This is the paper's measured price of robustness -on Lending Club: a return--auditability trade-off under one frozen OOT design, -not causal evidence or universal dominance. - -The funded-set under-coverage remains structural rather than a calibration-draw -effect. With $n_{\mathrm{cal}} = 237{,}584$ calibration loans, the -split-conformal conditional-coverage result makes marginal coverage highly -stable around the nominal level [@vovk2005; @angelopoulos2023]. The residual -funded-set $V$ is a test-side, portfolio-selection quantity. That is why the -paper reads the safety level at $\sqrt{\alpha}$ under Assumption 1, and why the -frontier reports $V$, $\Gamma_{\mathrm{CP}}$, residual premium, endpoint budget, -exact Markov threshold, and return together instead of promoting a standalone -coverage number. - -The compact reviewer checks below summarize the body-level defense. The -supplement expands the same structure into traceability and guardrail references. - -| Reviewer concern | Body answer | Primary evidence | -|---|---|---| -| "This is only a classifier." | The claim is decision auditability, not AUC leadership. | Exact funded-set certificate and A35 frontier. | -| "CP + RO already exists." | CRPTO instantiates the idea for frozen credit PD models, funded-set governance, and Lending Club payoffs. | Closest-work boundary and bound claim stack. | -| "The robust policy has no matched baseline." | Holding the candidate set and operating constraints fixed, A40 quantifies a 5.875% return cost against 8.305 percentage points less weighted default/miscoverage. | Matched point-PD table and A40 audit. | -| "Adaptive selection breaks coverage." | The theorem states weighted funded-set validity as an assumption and then audits the frozen selection exactly. | Assumption map, validity ladder, and A23 diagnostics. | -| "The selected policy is cherry-picked." | The selected point comes from a consolidated finite frontier with 50,010 deduplicated semantic policies and 27,508 eligible all-alpha above-floor policies. | Frontier table and governance files. | - -: Reviewer claim checks in the main manuscript. - -## Multi-Dataset External Economic Replication - -The natural generalization question after the 276,869-loan Lending Club audit is -whether the recipe still works outside the champion panel. The table below -answers that question without changing the champion: the same frozen recipe is -applied to two external credit products. Prosper is a -marketplace personal-loan panel with final statuses and a full OOT economic -candidate universe. Freddie FM48 is a collateralized mortgage panel, using the -48-month red+green default window with provided train/OOS/OOT structure. Both -pass the conformal gates and both return positive robust LP objectives. - -| Dataset | Product | Rows | Default | AUC | Cov. 90% | Cov. alpha01 | OOT cand. | Robust LP | -|---|---|---:|---:|---:|---:|---:|---:|---:| -| Prosper | Marketplace personal loans | `54,807` | `30.92%` | `0.7074` | `0.9205` | `0.9943` | `10,531` | `$199,419` | -| Freddie FM48 | Single-family mortgages | `3,173,355` | `1.45%` | `0.7839` | `0.9745` | `0.9907` | `1,396,053` | `$1,291,228` | - -: External economic replications using the frozen CRPTO recipe. The table is -generated from `crpto_tableA25_external_replication_gate.csv`; Home Credit is -archived but not promoted because it lacks a comparable economic -exposure/return contract. - -Figure @fig-external-replication summarizes the same result visually. Prosper -uses its full `10,531`-loan OOT economic universe. Freddie is evaluated on -`1,396,053` OOT economic candidates. A sparse all-candidate LP solves the full -Freddie universe and returns the same robust objective as the top screens; the -worst funded loan has rank `551`, with zero funded loans outside the top-250,000 -screen. The supplement reports this as an exhaustiveness audit rather than -as an unverified shortlist caveat. - -![External CRPTO replications preserve the predeclared global conformal gates and produce positive robust LP value on two materially different credit products.](../reports/crpto/figures/crpto_fig22_external_replication.png){#fig-external-replication width="94%" fig-alt="Two-panel external replication figure. The left panel shows 90 percent and alpha 0.01 coverage for Prosper and Freddie FM48 above target lines; the right panel shows positive robust LP objective values and OOT candidate counts."} - -The external layer adds a secondary economic pattern. Across Prosper and three -Freddie default-window applications, the signed robust premium ranges from -`+1.00%` to `+9.46%` and is ordered by panel default rate. Four applications -cannot establish a scaling law, so A34 and its companion figure remain in the -supplement as mechanism-consistent diagnostics rather than a body claim. The -matched Lending Club comparison is instead A40 above, whose point-PD baseline -holds the candidate universe and operating constraints fixed. The external -recipe therefore transfers as an economic audit protocol, while the exact -funded-set certificate remains the Lending Club object. - -# Robustness and Comparators - -The first robustness concern is temporal leakage. CRPTO addresses it through -out-of-time splits, temporal backtesting, and a strict distinction between -calibration, test, and paper-facing outputs. The paper does not claim that a -Lending Club static panel substitutes for future originations after the retail -platform closed. It claims that within the available historical panel, the -promoted policy survives the documented temporal validation design. The strict -temporal holdout in supplement A9 reports both temporal slices passing the exact -check, strengthening that validation claim without reopening the champion. - -The second concern is whether conformal uncertainty is doing decision work or -only adding conservative decoration. The answer is visible in the portfolio -frontier. Policies are evaluated by return, exact alpha pass/fail, weighted -miscoverage, and $\Gamma_{\mathrm{CP}}$; the promoted point is the body/default -balanced policy on that finite-grid frontier, while the strict `<= 0.345` threshold -policy is reported separately. This differs from a workflow where conformal -intervals are plotted after the optimizer has already chosen a point-PD -allocation. - -The supplement also carries the reviewer-facing robustness checks: nested -holdout, strict temporal holdout, exact evaluation of conformal finalists, -uncertainty-set baselines, and the selected-policy finite-grid frontier. -Diagnostics that depend on the selected funded-loan composition are split -explicitly: selected-allocation checks stay diagnostic, while legacy -tail-frontier tables remain diagnostic machinery rather than body selectors. - -The third concern is whether a decision-focused or SPO+ model would be a -stronger baseline. CRPTO treats SPO+ as an important comparator, but not as the -same governance object. Decision-focused training can reduce regret relative -to an optimization loss, while CRPTO prioritizes calibrated uncertainty, -traceable risk controls, and exact funded-set checks. The manuscript therefore -does not claim to dominate SPO+ on every regret metric; it claims a different -auditability/economic trade-off. - -| Baseline family | What it optimizes or reports | CRPTO comparison | -|---|---|---| -| Two-stage predict-then-optimize | Point-PD allocation after prediction. | Adds conformal uncertainty, exact funded-set audit, and frontier denominators. | -| SPO+ / decision-focused learning | Training-time regret on a decision-loss surface. | SPO+ owns low regret; CRPTO owns auditable funded-set risk controls. | -| P2P profit scoring | Economic loan selection. | Adds portfolio-level conformal premium and alpha-safe audit. | -| P2P robust portfolio optimization | Robust allocation under credit uncertainty. | Calibrates uncertainty with conformal intervals and exposes finite-grid selection. | -| Cost-aware calibration | Probability calibration under asymmetric costs. | Carries calibrated uncertainty into a budgeted portfolio decision. | - -: Baseline map for reviewer interpretation. - -## Regret-Auditability Frontier - -The SPO+ comparator makes the trade-off sharp. In the committed A19/Fig. 15 -results, SPO+ reduces mean regret from `0.425896` to `0.216837`, a `49.09%` -improvement over the two-stage baseline (Wilcoxon `p = 1.39e-164`). A later -PyEPO 1.3.7 paired rerun independently reports the same conclusion -under a slightly different protocol (`0.358073` to `0.184366`, `48.51%`; -Wilcoxon `p = 3.80e-163`), so we treat it as a curated closeout note rather -than as the numeric source for A19. The CRPTO robust point has higher decision -regret (`0.947429`) because it is not trained to minimize regret; it is -constructed to expose and control predictive uncertainty before funding. The -frontier is therefore not a single leaderboard. It asks what the method buys -besides regret: temporal coverage above target, an exact funded-set -$\alpha = 0.01$ pass, and a finite-grid return-bound frontier. This is the cleanest -comparator story in the paper. SPO+ answers "how much regret can training -remove?" CRPTO answers "what can a reviewer verify after a calibrated PD model -is frozen?" Those are complementary questions, and the frontier makes the -trade-off explicit rather than burying it in a single score. - -| Method | Mean regret | Regret delta vs. two-stage | Realized funding value | Verifiable risk controls | +| Selected 50/50 CRPTO | `308` | `$179,327.59` | `0.039375` | `0.036875` | `0.258051` | `0.574279` | +| More-conservative 75% blend | `312` | `$172,939.50` | `0.035875` | `0.035875` | `0.200396` | `0.516624` | +| Point-PD matched-$\tau$ | `225` | `$196,369.14` | `0.118400` | `0.041900` | `0.921317` | `1.237545` | + +: Full-OOT matched decision comparison. + +Relative to point PD, selected CRPTO gives up `$17,041.55`, or `8.678%` of +realized return. Weighted default falls by `7.9025` percentage points, +miscoverage falls by `0.5025` percentage points, and the endpoint-plus-Markov +threshold falls by `66.3266` percentage points. The default contrast is much +larger than the miscoverage contrast: conformal robustness changes which loans +are funded, but it does not create a dramatic selected-set coverage gain. + +The 75% blend reveals the remaining internal trade-off. It lowers default by +`0.35` percentage points and the threshold by `0.057655` relative to the +selected midpoint, but costs another `$6,388.08` in realized return. The +calibration selector chooses the midpoint because it is the highest expected- +objective candidate under the declared screen, not because it dominates every +risk metric. + +## Temporal Heterogeneity + +The full-panel average hides substantial regime dependence. + +| Period | CRPTO return | Point-PD return | CRPTO default | Point-PD default | |---|---:|---:|---:|---:| -| Two-stage baseline | `0.425896` | `0.0%` | not certified | `0/3` | -| SPO+ | `0.216837` | `49.09% lower` | not certified | `0/3` | -| CRPTO robust | `0.947429` | `122.46% higher` | `$184.8K` | `3/3` | - -: Regret-auditability frontier. Regret is the A19/PyEPO synthetic -decision-loss scale, not realized dollar return on the `$1M` funded portfolio. - -The regret column must be read with its protocol in mind, and the last two -columns prevent a one-dimensional reading. Mean regret comes from a separate -decision-regret experiment (A19/PyEPO) run on small synthetic optimization -instances (50 items, budget 15, five seeds), which scores each method on a -normalized decision-loss scale rather than on the `$1M` funded portfolio. That -experiment is not the same object as the funded-set economics: it -isolates training-time decision quality, so SPO+ is the low-regret method by -construction because it is trained to minimize exactly that loss. The -external price-of-robustness diagnostic reported in the supplement and the higher CRPTO regret here -are therefore not in tension; they are two different measurements (a real -`$1M` funded set versus a synthetic regret benchmark). The right-hand columns -report what the credit decision actually delivers: only CRPTO produces a -budgeted funded set with a certified realized return (`$184.8K` on the -`$1M` budget) and the three verifiable risk controls (exact $\alpha$-safe pass, -weighted-miscoverage audit, and finite-grid return-bound frontier). The two regret-trained -comparators optimize a loss surface but do not emit an auditable funded-set -certificate. The regret comparison is therefore about the synthetic benchmark -task, not the quality of the funded loans in the `$1M` credit portfolio. - -![The regret-auditability frontier shows the paper's trade-off in one panel: SPO+ is the low-regret corner, while CRPTO robust is the auditable-risk-control corner with all three verifiable checks passing.](../reports/crpto/figures/crpto_fig15_regret_auditability_frontier.png){#fig-regret-auditability width="72%" fig-alt="Scatter plot comparing two-stage, SPO+, and CRPTO robust by mean decision regret and number of verifiable risk-control checks passed."} - -CRPTO is also close in spirit to a recent line that gives distribution-free, -finite-sample guarantees jointly on miscoverage and decision regret, and that -traces a miscoverage--regret Pareto frontier for robust predict-then-optimize -policies [@zhou2025credo; @zhou2026creme]. That work is the general theory of -the frontier in @fig-regret-auditability: it shows how to calibrate a robustness -level against a cost--risk preference for an abstract optimization family. CRPTO -is the complementary, applied object. It does not propose a new frontier -estimator; it instantiates one corner of that frontier on a frozen, -production-style credit-risk model, with a named conformal robustness premium -$\Gamma_{\mathrm{CP}}$, an exact funded-set certificate, grade-level audits, and -model-risk lineage that a credit committee can inspect. A reviewer who knows the -general theory should therefore read CRPTO not as a competing estimator but as -the credit-decision instantiation that connects that theory to an auditable -Lending Club funded set. - -## Tail Risk and Distribution Robustness - -Two reviewer questions deserve a body-level answer rather than an appendix-only -one: what does the decision give up on the tail, and does its coverage hold once -the evaluation is sliced by grade? The return-bound frontier answers the first -question directly: lower Markov loss thresholds are available at lower return, while the -selected policy sits on the declared frontier. The supplement then checks the -funded-grade mix, selected-allocation tail repricing, dependence sensitivity, -and fixed-allocation bootstrap interval. These rows explain the selected -policy's risk profile; they do not add a hidden CVaR/OCE or bootstrap selector. - -| Reviewer question | Body answer | Boundary | -|---|---|---| -| Is the point only high-return? | The frontier shows safer lower-return choices and higher-return looser-threshold choices. | Finite grid, not a continuous optimum. | -| What loans does it fund? | The supplement reports funded exposure by grade. | Business mix, not protected-class fairness certification. | -| What happens in the tail? | The selected allocation is repriced under LGD and tail summaries. | Risk profile only; tail risk is not the selector. | -| Does dependence change the bound? | Cluster sensitivity recomputes tighter assumptions. | Sensitivity only; Markov remains the body theorem. | -| Is the return estimate fragile? | A fixed-allocation bootstrap reports contribution intervals. | Does not resample the model, solver, or search. | -| Does coverage survive slices? | Grade, distribution, and online-style checks stress the frozen intervals. | Diagnostics, not universal conditional or live coverage. | - -: Tail-risk and distribution-robustness checks for the selected decision. - -The second question is distribution robustness across grades. On the frozen -intervals, the worst per-grade 90% coverage is grade E at `0.9004`, still above -the `0.90` target. Supplement A23 reports marginal coverage `0.9293` on its -multi-distribution evaluation slice, while the promoted interval summary in -Table 3 reports `0.9297`; these are distinct cuts through the same -intervals. No grade falls below target, so the conservative marginal -coverage is not hiding a failing segment; the thinnest grade$\times$vintage -cells identify where a group-weighted or multi-distribution recalibration would -require a separately tagged protocol, so they are not promoted as a present -guarantee. - -## Managerial Implication - -For a credit-risk committee, CRPTO turns a model into a decision -conversation. The committee can pick a risk cap $\tau$, inspect how a policy -parameter $\gamma$ changes the funded set, separate the total conformal premium -into internalized and residual components, and compare $V(\alpha)$ with the -stated bound tolerance. On this evaluation the matched choice is concrete: -accept `5.875%` less realized return than the point-PD LP in exchange for `8.305` -percentage points less weighted default/miscoverage and a `43.55` percentage -point lower Markov loss threshold. The method therefore supports a practical -trade-off rather than promising a free robustness premium. -If the committee wants lower regret, the SPO+ corner is visible; if it wants -stronger validity language, the validity ladder states the new calibration -protocol that would be required. That separation is the managerial value of the -paper. - -The online supplement contains the robustness package: nested holdout, -segment-period sensitivity, decision-aware conformal selector checks, -synthetic-shift diagnostics, conformal finalist exact evaluation, tail-risk -OCE/CVaR diagnostics, satisficing margins, dependency clusters, bootstrap -funded-set metrics, budget/LGD/cap sensitivity, and finite-grid frontier -summaries by policy family, plus the A19 regret-auditability frontier, A20 -tail-risk diagnostic audit, A21 cluster-bound tightening audit, A22 CVaR/OCE -tail-constrained re-optimization, and A23--A24 multi-distribution and online -(ACI) conformal-stability diagnostics, plus A25--A34 external economic -replication, exhaustiveness, interval, subperiod, and sensitivity audits on -Prosper and Freddie/Mendeley, A35--A39 selected-policy frontier, composition, -tail-risk, concentration, and bootstrap audits, and A40 matched point-PD -comparison. - -# Reproducibility and Companion - -The project is built as an executable research bundle. Source code, manuscript -files, DVC metadata, tables, figures, and status reports are versioned together; -heavy data and model files live outside Git and are verified by hashes. The -paper-facing commands regenerate tables, figures, evidence summaries, and -rendered manuscript surfaces from frozen inputs; protected model-search stages -are not rerun as part of ordinary paper revision. - -Governance keeps the two certified results distinguishable. The frozen upstream -record defines the declared return floor, while the selected-policy governance -files define the body point and its frontier. Automated claim-sync tests assert -that every body-claim number printed in the manuscript and supplement matches -those files, so the two records cannot silently diverge. - -The certificate chain carries a stronger but narrower property: a validation -harness rebuilds the promoted conformal intervals from the frozen PD binaries -and recorded recipe and verifies the published endpoints and coverage summaries. -Gradient-boosted retraining is not presented as the routine reproduction target -across machines; retraining would be a new research run, not the routine -reproduction path for this submission. - -For double-anonymous review, this manuscript omits author-identifying URLs and -uses neutral language around repository ownership. The data/code companion can -be disclosed in the cover letter, supplement, or post-review reproducibility bundle -according to the venue policy. At acceptance, the reproducibility package is -designed to include public code, Quarto sources, pointers for processed data -and model files, raw Lending Club source instructions rather than secrets or -redistributed credentials, and the commands used to regenerate paper tables, -figures, and rendered manuscript surfaces. The important reproducibility -property is that the manuscript does not depend on hidden spreadsheet edits: -the reported numbers come from versioned evidence and guardrail tests. - -# Discussion - -CRPTO is useful precisely because it stays close to the operational reality of -credit-risk analytics. Many institutions already have calibrated PD models and -portfolio policies. Replacing them with an end-to-end decision-focused learner -may be scientifically attractive but organizationally difficult. CRPTO offers a -middle path: keep the predictive model auditable, quantify uncertainty with a -finite-sample conformal layer, and make the optimizer pay attention to the -upper end of plausible default risk. - -The policy-aware decomposition is more than a notation change. A linear blend -can recover its residual endpoint premium from $\gamma$ alone, but capped and -tail-focused policies cannot. Computing $\Gamma_{\mathrm{res}}$ from the funded -rows makes every frontier point comparable on the same exact endpoint scale and -prevents an attractive tail policy from appearing safer merely because its -decision score is nonlinear. - -The external replications also change how to read the price of robustness. -Across the frozen external applications, the premium is ordered by panel default -risk rather than by discrimination. That pattern reframes robustness as a -panel-specific premium to be measured, not a fixed toll to be assumed. It also -tempers the contribution: the matched Lending Club audit observes a `5.875%` -return cost, while the external applications report different premiums under -their frozen contracts. The transferable claim is therefore a reproducible way -to measure a return--risk trade-off, not a universal free lunch. - -The limits are equally important. CRPTO does not prove that any one public -dataset is a universal proxy for modern credit origination, even after the -Prosper and Freddie/Mendeley replications. Those external runs strengthen the -empirical defense across credit products, but they remain static historical -replications, not new exact funded-set certificates and not a prospective -deployment. The paper also does not claim legal -fair-lending certification because the public data lack direct protected -attributes. It does not assert that robust conformal policies dominate all -decision-focused learners on regret, and it does not treat synthetic shift or -external static panels as substitutes for live post-deployment monitoring. Its -conformal guarantee is marginal or partitioned by the chosen Mondrian design, -not exact conditional coverage for every borrower profile. The external panels -make this concrete: on Freddie the all-group minimum coverage is driven by tiny -sparse Mondrian cells, and the high-default red segment misses the strict -$\alpha = 0.01$ gate at `0.9850`; both are reported as sensitivity -evidence rather than promoted as conditional guarantees. Finally, this is not -an online deployment study: there are no new post-2020 Lending Club retail -originations, no live monitoring loop, and no end-to-end utility-directed -conformal learner replacing the frozen PD model. - -The manuscript is deliberately written as one IJDS paper rather than a bundle of -method variants. Adjacent methods enter only when they make the submitted -certificate easier to evaluate: OCE/CVaR as a tail-risk audit, satisficing as -margin evidence, SPO+ as the low-regret corner of the regret-auditability -frontier, and dependence as a formal caveat. Table @tbl-upgrade-map states that -single-submission boundary directly. - -| Adjacent path | What this paper uses | Boundary for the submitted claim | -|---|---|---| -| Tail-aware selection | A20--A22 and A37 show the selected decision's tail profile and available return-tail trade-off. | The promoted selector remains the declared return-bound finite frontier. | -| Prospective selection | Nested holdout and the finite declared grid reduce post-selection ambiguity. | The paper does not claim a fully prospective selection/evaluation trial. | -| Multi-distribution or online validity | A23--A24 diagnose grade, distribution, and vintage stress on frozen intervals. | The conformal layer is not recalibrated for multi-source or live sequential validity. | -| Decision-focused conformal learning | A19 shows SPO+ as the low-regret comparator and CRPTO as the auditable corner. | The PD model remains frozen; no end-to-end learner is promoted. | - -: Single-submission boundary map for the current CRPTO paper. {#tbl-upgrade-map} - -Optimized OCE/CVaR objectives, online conformal methods, multi-distribution -conformal validity, utility-directed or decision-theoretic conformal variants -[@cortesgomez2025utility; @lekeufack2023cdt], causal variants, broader -asset-class panels, and prospective multi-period origination studies are all -valuable comparators. In this submission, their role is to make the frozen -CRPTO contribution easier to locate: an auditable post-hoc -predict-then-optimize certificate, not a universal decision-learning framework -and not a collection of additional promoted methods. +| 2018H1 | `$92,530.73` | `$118,101.99` | `0.106703` | `0.190825` | +| 2018H2 | `$156,185.51` | `$95,603.58` | `0.026725` | `0.236728` | +| 2019H1 | `$123,590.69` | `$144,281.46` | `0.077325` | `0.170275` | +| 2019H2 | `$110,251.95` | `$256,966.20` | `0.103250` | `0.023775` | +| 2020+ | `$99,689.54` | `$218,629.14` | `0.083775` | `0.016900` | + +: Fixed-policy temporal stress evaluation; each row uses a fresh `$1M` budget. + +CRPTO is economically and statistically attractive in 2018H2, when the +point-PD portfolio concentrates heavily in realized defaults. In 2019H2 and +2020+, the point-PD policy earns much more and defaults less. The honest result +is therefore a full-period return-risk trade-off with temporal heterogeneity, +not a universal robustness premium. This pattern also cautions against treating +a static conformal guardrail as a substitute for live recalibration. + +## Funded-Set Composition + +The selected allocation is concentrated in grades C and D: they represent +`31.36%` and `58.68%` of funded exposure. Grade F is only `1.30%` of exposure +but has a high realized default rate (`0.269231`), illustrating why aggregate +metrics should be accompanied by composition. A38 reports every funded grade +and reconciles to the full `$1M` allocation. Because Lending Club does not +provide the protected attributes needed for a legal fair-lending audit, this is +a business-risk composition check, not a fairness certification. + +# Robustness, Comparators, and Implications + +The online supplement contains the evidence that helps a reviewer challenge +the one active method without turning the paper into several methods: exact +alpha saturation (A35), all nine selector cells (A36), temporal evaluation +(A37), grade composition (A38), fixed-allocation bootstrap (A39), and matched +comparators (A40). Earlier OCE/CVaR, dependence, online-style, SPO+, Prosper, +and Freddie/Mendeley analyses are retained as diagnostics or external context. +They do not select the active policy or create additional claims. + +For a credit committee, the useful control is not the Markov threshold by +itself. The committee first chooses a conventional conformal level, a +transparent blend $\gamma$, and a portfolio tolerance $\tau$. It can then read +the economic cost, endpoint exposure, realized default, and temporal behavior +together. In this application the midpoint rule has a clear interpretation: +half of the conformal premium enters the enforceable risk score and half remains +visible as residual endpoint exposure. + +The matched baseline turns that interpretation into a decision. A committee +that accepts the full-period evidence pays `8.678%` of realized return for a +`7.9025` percentage-point reduction in weighted default. A more conservative +committee can move to the 75% blend and pay an additional `$6.4K`. A committee +that emphasizes the 2019H2 or 2020 regimes should reject the static CRPTO rule +or require a new temporal recalibration protocol. The method exposes that +choice rather than hiding it behind a single score. + +# Reproducibility and Limitations + +The research bundle versions code, manuscript sources, configurations, tables, +figures, and governance files together. Heavy data and model artifacts are +stored separately and checked by hashes. The active evidence builder regenerates +A35--A40 from the frozen exact-alpha and policy outputs. Claim-sync tests verify +that the body, supplement, and official IJDS LaTeX manuscript share the same +policy, selector counts, and numeric anchors. + +Several limitations bound the contribution. First, Lending Club retail +origination ended in 2020; the panel is historical and cannot establish live +post-2020 performance. Second, the final tagged selector is outcome-free with +respect to OOT ranking, but earlier project development inspected the same +static OOT corpus. The evaluation is therefore retrospective, not a new +prospective holdout. Third, the conformal intervals are broad because the +outcome is binary; at 90%, more than half of OOT upper endpoints equal one. +Fourth, marginal or Mondrian coverage does not imply validity under +optimizer-selected funded weights. The Markov result is conditional on an +explicit weighted-validity assumption. Fifth, temporal slices show that the +point-PD comparator can dominate both return and default. Finally, public data +do not support a legal fair-lending certification or a causal interpretation +of policy contrasts. + +These limitations suggest a focused next step rather than a larger current +paper: a genuinely prospective or formally selection-valid protocol that +freezes conformal and policy choices before a new evaluation period +[@farinhas2024nonexchangeable_crc; @hegazy2025valid_selection_conformal_sets]. +It is not required to interpret the present retrospective decision audit. + +For double-anonymous review, author-identifying repository URLs are omitted +from the manuscript. The code/data companion and artifact-access instructions +can be disclosed under the journal's policy without embedding credentials or +identity in the anonymous PDF. # Conclusion -Can finite predictive uncertainty change a funding decision in a way a reviewer -can audit? CRPTO's answer is yes, provided the statistical boundary is stated -and the decision record is frozen. On the Lending Club out-of-time panel the -selected policy earns `$184,832.48` on a `$1M` budget while passing the exact -empirical $\alpha = 0.01$ funded-set audit, and it lies on a declared -finite-grid return-bound frontier with 50,010 deduplicated semantic policies and -27,508 all-alpha above-floor policies rather than at a single lucky point. A -matched point-PD LP shows the price explicitly: `5.875%` less realized return for -`8.305` percentage points less weighted default/miscoverage and a `43.55` -percentage point lower exact Markov loss threshold. The policy-aware residual -premium makes that certificate valid across the linear, capped, and tail-focused -frontier, while Prosper and Freddie/Mendeley show that the recipe can be audited -on other credit products. The contribution is one auditable post-hoc decision -certificate, not a new end-to-end learner, live-deployment study, or portfolio -of additional methods; every reported number is regenerable from frozen -evidence. - -# References +CRPTO shows how a frozen credit model can become an auditable portfolio +decision without adding a maze of policy variants. An exact 90% conformal +replay produces $u_i$; the midpoint $q_i=(p_i+u_i)/2$ constrains risk; and a +nine-cell calibration selector fixes $\tau=0.17$ without reading OOT outcomes. +On the full OOT panel, the policy earns `$179,327.59`, with weighted default +`0.039375`, miscoverage `0.036875`, $\Gamma_{\mathrm{CP}}=0.176102$, +$\Gamma_{\mathrm{res}}=0.088051$, endpoint `0.258051`, observed accounting +bound `0.294926`, and conditional Markov threshold `0.574279`. Against the +matched point-PD allocation, it pays `8.678%` of realized return for a +`7.9025` percentage-point default reduction. Temporal reversals keep the claim +narrow: CRPTO is an inspectable retrospective return-risk guardrail, not a +universal winner, prospective deployment guarantee, or new credit-scoring +leaderboard. diff --git a/paper/README.md b/paper/README.md index e23cf4f..f594e16 100644 --- a/paper/README.md +++ b/paper/README.md @@ -34,22 +34,19 @@ the submission-shaped versions are written. IJDS disclosure form. - `submission/SCHOLARONE_FINAL_CHECKLIST.md`: final upload/proof checklist. -Selected P2/P3-inspired diagnostics are part of this single IJDS submission -only when they defend the promoted decision certificate: -regret-auditability, OCE/CVaR tail risk, robust satisficing margins, -multi-distribution diagnostics, online replay diagnostics, the pool93 A35 -finite-grid frontier, the A36 funded-set grade audit, the A37--A39 selected -pool93 tail-risk, cluster-bound, and bootstrap audits, and external Prosper/Freddie economic -replications. A40 is the matched point-PD baseline with candidates and operating -constraints fixed. Tail-risk row-level repricing is supplement evidence for the -selected pool93 allocation, not a hidden promotion criterion. Prospective -live online validation, causal variants, new multi-dataset protocols beyond the -frozen Prosper/Freddie replications, production, and package tracks remain -outside the submitted claim. +The active paper has one method: exact 90% conformal replay, the midpoint +guardrail `q=(p+u)/2`, `tau=0.17`, and a nine-cell calibration selector. A35 is +the exact-alpha audit, A36 is the selector, A37 is temporal evaluation, A38 is +letter-grade composition, A39 is the fixed-allocation bootstrap, and A40 is the +matched point-PD comparison. OCE/CVaR, SPO+, satisficing, online-style checks, +and Prosper/Freddie replications remain supplement diagnostics; they do not +select or redefine the midpoint policy. Prospective validation, causal variants, +live recalibration, production, and package tracks remain outside the claim. ## Render Commands -```bash +```powershell +just ijds-evidence just paper-ijds just paper-ijds-supplement just paper-submission diff --git a/paper/ijds.css b/paper/ijds.css index f2c21a4..134aee5 100644 --- a/paper/ijds.css +++ b/paper/ijds.css @@ -18,3 +18,23 @@ overflow-x: auto; } } + +@media print { + html, + body, + main.content, + #quarto-appendix { + height: auto !important; + min-height: 0 !important; + margin-bottom: 0 !important; + padding-bottom: 0 !important; + } + + #quarto-content { + grid-template-rows: [content-top] auto [content-bottom] 0 [page-bottom] !important; + } + + #quarto-appendix { + break-after: avoid-page; + } +} diff --git a/paper/submission/CLAIM_AUDIT_MATRIX.md b/paper/submission/CLAIM_AUDIT_MATRIX.md index 2c1e2e6..7aa5e42 100644 --- a/paper/submission/CLAIM_AUDIT_MATRIX.md +++ b/paper/submission/CLAIM_AUDIT_MATRIX.md @@ -1,85 +1,45 @@ -# CRPTO Claim Audit Matrix - -This matrix is the pre-submission reviewer-defense layer. It maps each visible -claim to evidence, artifact provenance, and the boundary that prevents -overclaiming. - -| Claim | Body location | Evidence/artifact | Reviewer risk | Boundary language | -|---|---|---|---|---| -| CRPTO is a data-science decision method, not a classifier leaderboard. | Abstract, Introduction, Related Work. | Pipeline Figure 1, exact certificate table, funded-set audit. | Desk screen reads it as applied ML. | State the central object as an auditable conformal-robust decision certificate. | -| The predictive input is frozen and calibrated. | Method, Results, Supplement Appendix E. | `models/pd_canonical.cbm`, `models/pd_canonical_calibrator.pkl`, Table 0 metrics, E3/E4 PD stability diagnostics. | Reviewer asks whether results depend on hidden retraining. | PD artifact is consumed, not re-searched, by paper renders; E3/E4 are non-promoted T1 diagnostics, not a new champion. | -| The conformal layer is conservative on OOT data. | Method, Results, Supplement A23-A24. | `conformal_intervals_mondrian.parquet`, coverage metrics, group audits. | Reviewer expects conditional coverage. | Claim marginal/Mondrian coverage; stronger local/multi-distribution claims are future work. | -| The 90% intervals are useful despite large raw width. | Method, funded-set audit, Tables A35-A40. | Average width `0.7842`, Winkler `1.1107`, body-point `Gamma_CP=0.162616`, and A40's matched reduction in weighted default/miscoverage from `0.118400` to `0.035350`; A37--A39 profile tail risk, concentration, and bootstrap uncertainty. | Reviewer says intervals are too wide on the `[0,1]` PD scale. | Width is not promoted alone; the decision layer is judged through exact endpoint budgets, funded-set miscoverage, and the measured return--risk trade-off. | -| The promoted Lending Club body point passes the alpha-grid funded-set audit. | Results, Supplement A35-A39. | Body point: return `$184,832.48`, `V=0.035350`, `Gamma_CP=0.162616`, `Gamma_res=0.073584`, endpoint `0.245084`, exact Markov threshold `0.345084`, zero realized risk-tolerance excess, `8/8` alpha pass; A36--A39 profile the allocation. | "Exact" may be misunderstood as universal validity. | Exact means direct accounting on frozen funded-set outputs (Theorem 1(i)); statistical interpretation uses weighted funded-set validity, stated as Assumption 1. The threshold is probabilistic, not a deterministic cap. | -| The result is not a single lucky point. | Results, Supplement A35. | Consolidated pool93 frontier: `50,010` deduplicated semantic policies, `27,508` eligible all-alpha above-floor policies; terminal endpoint search: `37,068/37,068` all-alpha passers. | Reviewer asks whether one selected policy was cherry-picked. | The frontier is a declared finite policy-grid surface, not all possible continuous policy values and not a future-selection guarantee. | -| The conformal decision has a matched point-PD baseline. | Results, Supplement A40. | Same 276,869 candidates, `$1M` budget, concentration cap, `tau=0.1715`, LGD and solver. CRPTO costs `5.875%` realized return, reduces weighted default/miscoverage by `8.305` pp, and lowers the exact Markov threshold by `43.55` pp. | Reviewer asks whether the baseline is incomparable or labels entered optimization. | Only the effective PD semantics change; neither optimizer sees OOT labels. This is one frozen OOT trade-off, not causal or universal dominance evidence. | -| Prosper/Freddie show transfer of the recipe. | Results, Supplement A25-A34. | External replication gate, candidate sensitivity, all-candidate LP exhaustiveness. | Reviewer reads them as new champion certificates. | They are frozen external economic replications and exhaustiveness audits only. | -| The price of robustness is economically interpretable. | Results, Discussion. | A40 reports a matched Lending Club cost of `5.875%`; external premiums range from `+1.00%` to `+9.46%` across four frozen applications. | Sign convention, incomparable baselines, or over-reading four external cases as a scaling law. | Exclude the historical Lending Club `-10.56%` field; use A40 for Lending Club and describe the external ordering as a four-case diagnostic only. | -| SPO+ is a comparator, not a replacement champion. | Robustness and Comparators. | A19/Fig. 15 committed regret-auditability artifact. | Reviewer says CRPTO has higher regret. | SPO+ optimizes synthetic regret; CRPTO emits funded-set risk controls. | -| Tail-risk alternatives exist and the selected point has been repriced. | Tail Risk section, Supplement A20-A22 and A37-A39. | CVaR/OCE and tail-constrained challenger tables; pool93 body repricing has baseline LGD return `$184,832.48`, realized CVaR95 `0.276211`, decision-time CVaR95 `0.218140`, no cluster-bound threshold tighter than Markov, and fixed-allocation bootstrap diagnostics. | Reviewer asks why not select lowest-tail policy. | The body selector is the finite-grid return-bound point; tail-risk/bootstrap tables are documented trade-offs and selected-point diagnostics, not hidden promotion criteria. | -| Fairness/MRM claims are limited. | Supplement Appendix D, Discussion. | Proxy/intersectional diagnostics and MRM scope. | Reviewer asks for statutory fair-lending proof. | Public data lack direct protected attributes; no legal certification claim. | -| The paper is reproducible. | Body design paragraph, Supplement E, submission docs. | `just smoke`, `just validate-champion`, DVC metadata, lockfile, manifest. | Reviewer asks how to reproduce without secrets. | Raw-data instructions and DVC pointers are disclosed through the accepted-paper package or an editor-approved review bundle. | -| The certificate chain is exactly recomputable from frozen artifacts. | Body design paragraph in the official `.tex`, Supplement E, submission docs. | Drift harness `tests/test_models/test_conformal_mapie_drift.py` (zero max abs diff per loan and per Mondrian cell, identical re-learned floor multipliers); `scripts/rebuild_test_predictions_from_frozen.py` hard-asserts score/interval identity. | Reviewer reads "reproducible" as including GBM retraining. | The verified property is prediction-to-decision: frozen PD binaries through conformal intervals to the certificate, under the locked dependency stack. Gradient-boosted retraining is not bit-reproducible, which is why the predictive layer ships as a hash-verified frozen binary; E3/E4 support this choice without becoming routine reruns. | - -## Reviewer Objection Bank - -| Objection | Short response | -|---|---| -| "CP + RO already exists." | CRPTO instantiates the bridge for frozen credit PD artifacts, funded-set economics, exact audit, and reproducible governance. | -| "CP + RO is a direct combination, not a theory contribution." | Theorem 1 separates deterministic funded-set accounting from the Markov step under weighted validity, while the policy-aware decomposition `Gamma_CP = Gamma_int + Gamma_res` makes exact endpoint accounting valid for linear, capped, and tail-focused policies. The contribution remains the auditable decision-certificate bridge, not a stronger conditional-coverage theorem. | -| "Adaptive selection breaks conformal validity." | Correct concern, and exactly why the paper isolates it: Assumption 1 states weighted funded-set validity explicitly, Theorem 1 separates the deterministic identity from the Markov step, and the exact audit checks the selected frozen funded set after the fact. | -| "This is one dataset." | Lending Club carries the certificate; Prosper/Freddie test transfer of the recipe without claiming new certificates. | -| "The AUC is not high enough." | The paper is not a credit-scoring leaderboard; calibrated probabilities are inputs to an auditable decision. | -| "Why distribute a binary instead of asking readers to retrain?" | The certified object is the frozen prediction-to-decision chain. E3/E4 show seed and temporal PD diagnostics are stable enough to support the binary choice, while the exact artifact hashes keep the submitted certificate fixed. | -| "The intervals are too wide to use." | Raw width is expected on a binary PD-scale interval; the paper evaluates whether upper endpoints rank downside risk and produce a funded set that passes Winkler, funded-set miscoverage, and exact alpha-safe checks. | -| "SPO+ has lower regret." | Correct; the paper reports a frontier where SPO+ buys regret and CRPTO buys verifiable risk controls; the pool93 frontier updates the funding certificate, not the SPO+ regret experiment. | -| "Why not live validation?" | Lending Club retail originations ended in 2020; prospective live validation is future protocol, not hidden current evidence. | - -## Response-Ready Reviewer Paragraphs - -**Why not SPO+ as the main method?** SPO+ is the right comparator for -training-time decision regret, and the manuscript reports that comparison -directly. The point of CRPTO is different: it asks what can be certified after a -calibrated PD model is frozen and the decision layer must remain auditable. On -the A19 regret scale SPO+ owns the low-regret corner; CRPTO owns the funded-set -risk-control corner with a dollar-valued allocation, conformal premium, -finite-grid denominator, and exact post-allocation audit. - -**Why not CVaR/OCE as the selector?** CVaR and OCE are useful tail-risk -diagnostics, but making either one the promoted selector would define a new -objective and require a new predeclared search/audit protocol. The current -submission deliberately promotes the finite-grid return-bound point and then -reprices that selected allocation under LGD, CVaR, OCE, cluster, and bootstrap -stress checks. This keeps tail risk visible without turning a diagnostic table -into a hidden promotion criterion. - -**Is the selected point cherry-picked?** The selected policy is not a singleton -chosen after looking at one lucky allocation. It sits on a declared finite-grid -frontier: 50,010 deduplicated semantic policies are reported, 27,508 both pass -all declared alpha levels and exceed the return floor, and the terminal endpoint -search completes 296,544 exact policy-alpha checks. The body/default point and -the strict `<=0.345` neighboring point are separated explicitly to avoid -rounding-based overclaiming. - -**What happens under dependence?** The body theorem uses the weakest -distribution-free Markov step under the stated weighted funded-set validity -assumption and does not require loan-level independence. The supplement prices -stronger assumptions through cluster-aware sensitivity tables; those rows show -what a reviewer would gain by accepting additional structure, but none becomes -the body guarantee. Dependence therefore appears as an assumption boundary, not -as an unstated theorem condition. - -**Is this a live-production guarantee?** No. Lending Club retail originations -ended in 2020, and the manuscript is explicit that the evidence is a frozen -historical decision certificate, not a prospective control system. The -contribution is reproducible prediction-to-decision governance on the available -out-of-time panel; online conformal control, prospective validation, and live -monitoring are future protocols. - -**What can be reproduced under double-anonymous review?** During anonymous -review, the manuscript and supplement describe the companion package without -author-identifying repository URLs. The reproducible object is the -prediction-to-decision chain from frozen PD artifacts and conformal intervals to -tables, figures, exact checks, and manifest validation. Protected searches and -retraining are intentionally excluded from routine reproduction because they -would change the submitted certificate rather than verify it. +# IJDS Claim Audit Matrix + +This matrix is the editorial guardrail for the active calibration-selected +midpoint policy. Numeric authority is +`ijds_policy_governance.json` plus A35--A40. + +| Claim | Evidence | Main reviewer risk | Defensible wording | +|---|---|---|---| +| The 90% conformal recipe is replayed exactly. | A35; stored endpoint replay max error `6.67e-16`. | "Exact" is mistaken for universal conditional validity. | Exact refers to numerical reconstruction of the frozen finite-sample recipe. Coverage remains marginal/Mondrian under its assumptions. | +| The 99% setting is not decision-useful here. | A35: width `0.988215`; `93.5424%` of upper endpoints equal one. | Reviewer thinks 90% was chosen only to improve economics. | The 90% level is the frozen recipe's reference level and preserves materially more ranking resolution; alpha sensitivity is fully reported. | +| The final policy is simple. | `q=(p+u)/2`, `tau=0.17`; one linear policy family. | Complexity or hidden nonlinear logic. | Point PD prices expected loss; the midpoint is used only in the risk constraint. | +| Final ranking does not use OOT outcomes. | A36; nine candidates, five eligible, zero forbidden selector columns. | Historical OOT inspection makes "untouched holdout" false. | Call it a retrospective lockbox replay with an outcome-free final ranking code path, not preregistration or a pristine prospective trial. | +| The selector has a declared rule. | A36: full budget, effective-PD feasibility, threshold `<=0.60`, then maximum expected objective. | The `0.60` screen appears arbitrary. | Present it as a committee risk preference in the tagged final protocol and report the complete 3x3 grid. | +| The selected funded set has an exact accounting audit. | Full OOT: return `$179,327.59`, default `0.039375`, miscoverage `0.036875`, endpoint `0.258051`, `B_u+V=0.294926`. | Accounting is confused with a coverage theorem. | The identity is deterministic after outcomes; it is not nominal selected-set coverage. | +| The Markov statement is secondary and conditional. | Threshold `0.574279`; tail-probability bound `0.316228` under weighted validity. | Bound is too loose or presented as a hard risk cap. | Use it as sensitivity only. Operational controls are `tau`, midpoint exposure, and observed funded-set metrics. | +| A40 is a matched point-PD comparison. | Same candidates, budget, concentration, LGD, solver, and `tau=0.17`. | Comparator changes multiple semantics or sees labels. | Only the risk score changes; neither optimization reads OOT outcomes. | +| Robustness has a measured price. | A40: `8.678%` realized-return cost and `7.9025` percentage-point default reduction. | Default and miscoverage reductions are conflated. | Report default reduction (`7.9025` pp) separately from miscoverage reduction (`0.5025` pp). | +| The midpoint is not the safest CRPTO policy. | A40: 75% blend return `$172,939.50`, default `0.035875`, threshold `0.516624`. | Selected point is sold as dominant. | It is the highest calibration expected-objective candidate under the declared screen. | +| Performance is temporally heterogeneous. | A37: CRPTO wins strongly in 2018H2; point PD dominates 2019H2 and 2020+. | Full-panel result is overgeneralized. | State the reversals in body and abstract; no universal dominance claim. | +| Funded-set composition is correctly labeled. | A38 uses letter grade recovered from `sub_grade` and stores conformal group separately. | Score-quantile groups are mistaken for loan grades. | Call A38 a business composition audit, not fairness certification. | +| Bootstrap uncertainty is bounded. | A39 return interval `$162,706.17`--`$193,924.74`, 5,000 draws. | Interval is read as full model/selection uncertainty. | It is a fixed-allocation funded-loan contribution bootstrap only. | +| Earlier methods do not multiply the contribution. | Supplement A1--A34. | Paper reads as several papers or an uncontrolled tournament. | OCE/CVaR, SPO+, online-style checks, and external replications are diagnostics or context, not active selectors. | +| Reproducibility is substantive evidence quality. | Run tags, configs, A35--A40 builder, claim-sync tests, manifest validation. | Tooling is presented as the only novelty. | Lead with decision method and empirical implication; reproducibility makes them auditable. | + +## Do Not Claim + +- pristine prospective or preregistered OOT evaluation; +- nominal conformal validity for optimizer-selected funded weights; +- universal dominance over point PD or decision-focused learning; +- causal return or default effects; +- legal fair-lending certification; +- live post-2020 Lending Club performance; +- that external Prosper/Freddie diagnostics replicate the final midpoint + selector exactly. + +## Required Headline Numbers + +- selected return: `$179,327.59`; +- selected default / miscoverage: `0.039375 / 0.036875`; +- `Gamma_CP / Gamma_residual`: `0.176102 / 0.088051`; +- endpoint / observed accounting / conditional threshold: + `0.258051 / 0.294926 / 0.574279`; +- point-PD return: `$196,369.14`; +- return cost / default reduction: `8.678% / 7.9025` pp; +- selector: `5/9` eligible, selected `tau=0.17`, `gamma=0.50`. diff --git a/paper/submission/COVER_LETTER_AND_DISCLOSURE.md b/paper/submission/COVER_LETTER_AND_DISCLOSURE.md index 9b65f95..540762a 100644 --- a/paper/submission/COVER_LETTER_AND_DISCLOSURE.md +++ b/paper/submission/COVER_LETTER_AND_DISCLOSURE.md @@ -1,85 +1,82 @@ # IJDS Cover Letter and Disclosure Draft -This file is for the editor-facing submission package. It is not part of the -double-anonymous reviewer packet unless the submission system explicitly asks -for the corresponding disclosure fields. +Editor-facing material only. Do not include it in the double-anonymous reviewer +packet unless ScholarOne requests the corresponding disclosure text. -## Cover Letter Core Paragraph +## Cover Letter Dear Editors, -We submit "CRPTO: Conformal Robust Predict-Then-Optimize for Auditable Credit -Portfolio Decisions" for consideration at the INFORMS Journal on Data Science. -The paper studies credit allocation as data science for decisions rather than -as a predictive leaderboard. Its data component is a static Lending Club -out-of-time panel, supported by frozen Prosper and Freddie/Mendeley external -economic replications. Its method maps a frozen calibrated probability-of-default -artifact through Mondrian conformal intervals into a robust portfolio decision. -Its decision object is the funded set under a budget and risk cap, and its main -implication is an auditable model-risk surface: the promoted Lending Club body -point earns `$184.8K` on a `$1M` budget while passing an exact empirical -alpha-grid funded-set audit, and the declared pool93 finite-grid frontier -contains 50,010 deduplicated semantic policies with 27,508 all-alpha -above-floor policies. -An opt-in drift harness verifies that the prediction-to-decision certificate -chain regenerates bit-exactly from the frozen artifacts under the locked stack. -The contribution is intended for settings where decision auditability, -reproducibility, and model-risk governance matter as much as predictive rank. +We submit "CRPTO: A Calibration-Selected Conformal Guardrail for Auditable +Credit Portfolio Decisions" for consideration at the *INFORMS Journal on Data +Science*. The paper treats credit allocation as data science for decisions, +not as a credit-scoring leaderboard. A frozen calibrated PD model is combined +with an exactly replayed 90% Mondrian conformal endpoint. The resulting +midpoint score, `q=(p+u)/2`, constrains a `$1M` portfolio while point PD remains +in the expected-return objective. + +The final policy is selected from nine round-number candidates on a temporal +calibration holdout. Its ranking artifact contains no OOT default, +realized-return, or miscoverage fields. On 276,869 out-of-time Lending Club +loans, the fixed policy earns `$179,327.59`, with weighted default `0.039375`. +A matched point-PD allocation earns `$196,369.14` with weighted default +`0.118400`. The paper reports both the `8.678%` return cost and the `7.9025` +percentage-point default reduction, together with temporal periods in which +the point-PD decision performs better. We therefore position CRPTO as an +auditable retrospective return-risk guardrail, not as a universal winner or +prospective deployment guarantee. + +The submission contributes an explicit prediction-to-decision contract, an +exact conformal replay, an inspectable calibration selector, matched economic +comparisons, and a file-backed reproducibility package. These features align +with IJDS's emphasis on data, innovative methodology, decision relevance, and +reproducible evidence. + +Sincerely, + +[Author details supplied separately] ## Data and Code Availability -The submission body and supplement are double-anonymous. During review, the -manuscript refers to a reproducible companion package without exposing -author-identifying URLs. The submission will complete the IJDS Data and Code -Disclosure Form and acknowledge the accepted-paper reproducibility workflow. -After the venue permits disclosure, the companion can include: - -- public source code and Quarto manuscript sources; -- DVC metadata and pointers for processed artifacts and frozen model files; -- MLflow/DagsHub lineage for the CRPTO runs, subject to credential-free access - rules; -- raw-data source instructions from `RAW_DATA_SOURCE_NOTES.md` rather than - redistributed raw CSVs when source terms or file size make rehosting - inappropriate; -- Prosper and Freddie/Mendeley source notes for the external replication layer; -- the frozen extraction manifest and guardrail tests used to verify the - promoted frontier; -- the drift harness that recomputes the conformal interval and certificate - chain from frozen PD artifacts with zero endpoint drift under the locked stack; -- commands for regenerating paper tables, figures, HTML previews, and local - IJDS PDF verification drafts. - -No secrets, tokens, private DVC credentials, or local machine paths should be -included in the reviewer packet. - -If ScholarOne asks for the disclosure option in prose, the intended answer is: -code and manuscript sources will be released at acceptance; raw data are -public-source or source-controlled and therefore disclosed through source -instructions plus DVC pointers/processed artifacts when the journal workflow and -source terms permit. - -## Double-Anonymous Handling - -- Upload `paper/CRPTO_ijds.pdf` as the local body preview only if the official - template PDF is not yet required. -- Upload `paper/supplement_ijds.pdf` as the local supplement preview only if - the official workflow accepts HTML-print verification drafts. -- Keep public repository, DagsHub, MLflow, personal site, affiliation, and - author-identifying acknowledgements out of reviewer-facing files. -- Use this file or the submission system fields for disclosure timing, not the - anonymous manuscript body. - -## Editorial Fit - -The paper is positioned as data science for decisions: conformal prediction is -not only an uncertainty report, and robust optimization is not only a portfolio -heuristic. The central object is an executable, auditable decision recipe whose -numbers are backed by frozen artifacts, manifest regression tests, and -submission-ready tables and figures. - -The highest-risk interpretive boundary is also stated explicitly in the paper: -the Lending Club exact funded-set certificate is an exact accounting audit on -the frozen promoted portfolio under a declared weighted-validity assumption. The -Prosper and Freddie/Mendeley results are external economic replications and -exhaustiveness audits, not new exact funded-set certificates or prospective live -deployment guarantees. +The body and supplement are double-anonymous. During review they refer to a +reproducible companion without exposing author-identifying URLs. The IJDS Data +and Code Disclosure Form will state that, when venue policy permits, the +companion includes: + +- source code, configurations, and manuscript sources; +- A35--A40 evidence tables and active governance metadata; +- DVC metadata and artifact pointers for large processed data and model files; +- source instructions for Lending Club, Prosper, and Freddie/Mendeley data + rather than unauthorized redistribution; +- exact-alpha and calibration-selector replay commands; +- manifest, claim-sync, and publication-integrity tests; +- commands for regenerating tables, previews, and the official-template PDF. + +No secrets, tokens, private storage credentials, local usernames, or machine +paths belong in the reviewer package. + +Suggested ScholarOne prose: + +> Code, configurations, manuscript sources, and evidence-generation scripts +> will be released under the journal's accepted-paper reproducibility process. +> Public-source raw data are disclosed through source instructions; large or +> license-constrained artifacts are provided through documented pointers and +> integrity hashes where redistribution terms permit. + +## Anonymity Handling + +- Upload the `informs4` PDF built with `dblanonrev` as the manuscript. +- Upload the anonymous supplement separately. +- Keep the title page, affiliation, acknowledgements, repository ownership, + and personal URLs outside reviewer-facing files. +- Use this file and ScholarOne fields for editor-only disclosure timing. + +## Editorial Boundary + +The final ranking code is outcome-free with respect to OOT policy selection, +but earlier project development inspected the static OOT corpus. The manuscript +therefore says "retrospective lockbox replay," not "preregistered" or +"untouched holdout." Marginal/Mondrian coverage is not promoted to nominal +validity under optimizer-selected funded weights. OCE/CVaR, SPO+, online-style +checks, and external datasets remain diagnostics or context rather than +additional active methods. diff --git a/paper/submission/CRPTO_ijds_submission.tex b/paper/submission/CRPTO_ijds_submission.tex index f469a88..30b9e55 100644 --- a/paper/submission/CRPTO_ijds_submission.tex +++ b/paper/submission/CRPTO_ijds_submission.tex @@ -1,40 +1,20 @@ %% ===================================================================== %% CRPTO -- INFORMS Journal on Data Science (IJDS) submission manuscript %% ===================================================================== -%% Double-anonymous submission body in the official INFORMS class. The prose is -%% ported from `paper/CRPTO_ijds.qmd`, then manually compacted for the official -%% IJDS template/page budget. Do not regenerate this file mechanically from QMD -%% after freeze; port substantive claim changes deliberately and rebuild. +%% Double-anonymous official-template handoff synchronized with +%% paper/CRPTO_ijds.qmd. Build with latexmk, or use the Windows fallback: %% -%% REQUIRED (NOT vendored in this repo, per the no-vendoring rule): -%% - informs4.cls INFORMS document class -%% - informs2014.bst INFORMS BibTeX style -%% `informs4` is NOT on CTAN/TeX Live; download both from the INFORMS author -%% portal or Overleaf and drop them next to this file, then build: -%% https://pubsonline.informs.org/authorportal/latex-style-files -%% https://www.overleaf.com/latex/templates/template-for-informs-journal-on-data-science/sbthszxgycfn -%% if (-not $env:WINDIR) { $env:WINDIR = $env:SystemRoot } % PowerShell -%% latexmk -pdf -gg -interaction=nonstopmode CRPTO_ijds_submission.tex -%% If a TeX Live update leaves a LaTeX support-file mismatch, run once: -%% fmtutil-sys --byfmt pdflatex -%% If the local TinyTeX wrapper still fails, use the verified Windows fallback: %% pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex %% bibtex CRPTO_ijds_submission %% pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex %% pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex -%% The first pdflatex writes .aux, bibtex writes .bbl, the second pdflatex reads -%% bibliography/cross-reference data, and the third stabilizes references and -%% pagination. On 2026-07-07 the local Codex shell needed WINDIR initialized -%% from SystemRoot before TinyTeX wrappers would run. %% -%% `dblanonrev` keeps the manuscript anonymous (IJDS uses double-anonymous -%% review for submissions on/after 2025-01-01). Do NOT add author names, -%% affiliations, acknowledgements, or repository URLs to this body. +%% Pass 1 writes the .aux file, BibTeX writes the .bbl, pass 2 resolves the +%% bibliography and cross-references, and pass 3 stabilizes labels and pages. %% ===================================================================== \documentclass[ijds,dblanonrev]{informs4} -% --- Spacing / theorem / equation conventions (IJDS template defaults) --- -\usepackage{eqndefns-left} % flags display equations that exceed text width +\usepackage{eqndefns-left} \RequirePackage{tgtermes} \RequirePackage{newtxtext} \RequirePackage{newtxmath} @@ -44,9 +24,8 @@ \TheoremsNumberedThrough \EquationsNumberedThrough \ECRepeatTheorems -\MANUSCRIPTNO{} % left blank for initial submission +\MANUSCRIPTNO{} -% --- Math + graphics packages used by the manuscript --- \usepackage{amsmath,amssymb,mathtools} \usepackage{booktabs} \usepackage{graphicx} @@ -63,1117 +42,493 @@ \begin{document} \RUNAUTHOR{Anonymous} -\RUNTITLE{Conformal Robust Predict-Then-Optimize for Credit Portfolios} +\RUNTITLE{A Calibration-Selected Conformal Guardrail for Credit Portfolios} -\TITLE{CRPTO: Conformal Robust Predict-Then-Optimize for Auditable Credit +\TITLE{CRPTO: A Calibration-Selected Conformal Guardrail for Auditable Credit Portfolio Decisions} -% Authors hidden under dblanonrev; keep the block empty/anonymous. \ARTICLEAUTHORS{% \AUTHOR{} \AFF{} } \ABSTRACT{% -Credit allocation is a data-science-for-decisions problem: default probabilities -matter only after they shape which loans are funded under a budget and risk -appetite. We introduce Conformal Robust Predict-Then-Optimize (CRPTO), a -post-hoc decision certificate that maps a frozen calibrated probability-of-default -model through Mondrian conformal intervals into robust portfolio constraints and -an empirical funded-set audit. A policy-aware decomposition separates the -conformal premium internalized by the optimizer from the residual premium needed -to recover the exact upper-endpoint budget. On a 276{,}869-loan out-of-time -Lending Club evaluation, the selected policy earns \$184.8K on a \$1M budget -while passing the declared eight-level alpha grid ($V(0.01)=0.035350$, -$\Gamma_{\mathrm{CP}}=0.162616$, exact Markov loss threshold $0.345084$, zero -realized risk-tolerance excess). Against a matched point-PD two-stage LP with the -same candidates, budget, concentration cap, and risk tolerance, CRPTO gives up -$5.87\%$ of realized return while reducing the weighted default rate by $8.305$ -percentage points and the loss threshold by $43.55$ percentage points. The -consolidated frontier contains 50{,}010 deduplicated policies, of which 27{,}508 -pass all declared alpha levels and exceed the return floor. Frozen Prosper and -Freddie/Mendeley applications preserve the predeclared global conformal gates -with positive robust LP objectives. CRPTO therefore makes predictive uncertainty -decision-useful as an auditable return--risk frontier, with a distribution-free -Markov bound under weighted funded-set validity and an explicit separation -between deterministic accounting and its statistical assumption.% +Credit models matter only through the decisions they change. We study how +finite-sample predictive uncertainty can constrain a loan portfolio after a +probability-of-default (PD) model has been frozen. Conformal Robust +Predict-Then-Optimize (CRPTO) recomputes a 90\% Mondrian conformal upper endpoint +exactly, forms the transparent decision score $q_i=(p_i+u_i)/2$, and places +$q_i$ in a portfolio-risk constraint while retaining point PD in the economic +objective. Nine round-number policies are ranked on a temporal calibration +holdout without default, realized-return, or miscoverage columns; the selected +policy is then frozen and replayed on 276{,}869 out-of-time Lending Club loans. +It funds 308 loans and earns \$179{,}327.59 on a \$1M budget, with weighted +default 0.039375, weighted miscoverage 0.036875, and conformal endpoint budget +0.258051. A matched point-PD allocation earns \$196{,}369.14 but has weighted +default 0.118400 and endpoint budget 0.921317. Thus CRPTO pays 8.678\% of +realized return for a 7.9025 percentage-point default reduction; the advantage +reverses in some temporal slices, so we do not claim universal dominance. The +contribution is a small, auditable prediction-to-decision guardrail with an +exact replay, an inspectable selector, and explicit statistical boundaries.% } -\KEYWORDS{conformal prediction; robust optimization; predict-then-optimize; -credit risk; portfolio optimization; reproducible data science} +\KEYWORDS{conformal prediction; predict-then-optimize; credit risk; portfolio +optimization; calibration; reproducible data science} \maketitle -% ===================================================================== \section{Introduction}\label{sec:intro} -Credit allocation is a contextual optimization problem in credit form. A lender -first estimates a probability of default (PD), then chooses which loans to fund -under a budget and risk appetite. This places the paper in the broader -prescriptive-analytics literature on decision making under uncertainty -\citep{sadana2025contextual}. The credit-risk literature has become very good at -the first step: calibration, discrimination, and backtesting are now standard -ingredients of model validation \citep{lessmann2015,chen2024creditrisk}. Recent -credit-scoring work also evaluates uncertainty through economic metrics. The -second step is less settled. Once a calibrated PD enters an optimizer, -uncertainty is often treated as a reporting diagnostic rather than as a -constraint that can change the funded set. - -That separation is uncomfortable in auditable credit decisions. A portfolio policy -can have a reasonable average PD and still concentrate probability mass in loans -where the model is most uncertain. Conversely, a policy that is too conservative -can pass every risk check while destroying economic value. The scientific question -in this paper is therefore not whether one can build a slightly better credit -classifier. It is whether finite-sample predictive uncertainty can be carried into -a robust portfolio decision in a way that is transparent enough for a reviewer to -audit. This has practical stakes, but it is not automatic: conformal sets must be -tied to a downstream action and objective to become decision-useful. In a -pre-registered randomized trial, conformal prediction sets improved human -decision making relative to fixed-size sets with the same coverage -\citep{cresswell2024}. CRPTO takes that -committee-facing idea into a credit portfolio setting, where the uncertainty -summary must change a funding decision or it is just another report. - -CRPTO answers this question with a post-hoc, reproducible pipeline. It starts from -a calibrated CatBoost PD model, constructs Mondrian conformal intervals over -PD-scale predictions, and maps the upper conformal endpoint into robust portfolio -constraints. The pipeline is modular by design: the predictive model, conformal -layer, optimization policy, and paper outputs each have separate contracts. -That separation lets the paper ask whether a frozen prediction system can be -converted into a defendable decision system without reopening hyperparameter -search whenever the manuscript or appendix changes. - -The empirical setting is the Lending Club retail-loan panel, with an out-of-time -evaluation set of 276{,}869 loans. The consolidated frontier contains 50{,}010 -deduplicated semantic policies, of which 27{,}508 pass every declared alpha -level and exceed the return floor. From that declared finite frontier, the -selected policy is the body/default balanced point at the approximately -$0.345$ return-bound lens, with exact Markov loss threshold $0.345084$; it is neither a continuous -global optimum nor the economic endpoint. The selected point earns \$184.8K on -a \$1M budget and passes the exact empirical funded-set audit at -$\alpha=0.01$. The headline result is not a single lucky allocation. It is a -reproducible bridge from calibrated probabilistic learning to robust, auditable -credit portfolio choice, with the return-bound frontier reported rather than -hidden behind one selected point. - -To address the natural single-dataset concern without reopening the Lending Club -champion, the paper also freezes two external economic replications: Prosper -final-status marketplace loans and a Freddie/Mendeley single-family mortgage -panel with out-of-sample and out-of-time splits. These replications are not new -champions; they test whether the same PD-to-conformal-to-LP recipe remains -economically usable on different credit products. - -The paper makes four contributions. First, it gives a CRPTO construction for -credit portfolios: frozen calibrated PD, Mondrian conformal uncertainty, robust -budgeted optimization, and an exact post-allocation audit. Second, it proves a -distribution-free Markov bound under weighted funded-set validity -(Theorem~\ref{thm:funded-set-bound}) -and introduces a policy-aware decomposition -$\Gamma_{\mathrm{CP}}=\Gamma_{\mathrm{int}}+\Gamma_{\mathrm{res}}$ that recovers -the exact upper-endpoint budget for linear, capped, and tail-focused policies. -Supplement propositions show that Markov is optimal under the stated first-moment -assumption (A.1) and locate the cluster structure that would tighten it (A.2). -Third, it reports the selected Lending Club decision as part of a declared -finite-grid return-bound frontier and compares it with a matched point-PD -allocation; all rows and exact alpha checks are generated from frozen evidence. -Fourth, it packages the result as a reproducible IJDS decision artifact, with -tables, figures, governance files, and claim-sync checks designed to keep the -statistical boundary visible. The key claim is narrow: CRPTO maps a frozen -calibrated PD model into a robust funded set, reports the portfolio-level -conformal premium and its internalized/residual components, and audits the -promoted Lending Club allocation and its finite-grid frontier exactly. Adjacent methods enter the manuscript only to locate and -stress-test this single claim, not to create additional acceptance criteria. - -Read as data science for decisions, the paper's four components are explicit: -the data are a static Lending Club OOT panel plus Prosper/Freddie external stress -tests; the model is the CRPTO bridge from calibrated PD to Mondrian conformal -uncertainty to robust LP; the decision is the funded set under budget and risk -caps; and the implication is a reproducible audit surface for model-risk -committees. +Credit allocation is a contextual optimization problem. A lender estimates a +probability of default (PD) and then chooses which loans to fund under capital, +concentration, and risk constraints \citep{sadana2025contextual}. Credit-scoring +research has strengthened the first step through discrimination benchmarks, +probability calibration, and cost-aware evaluation +\citep{lessmann2015,chen2024creditrisk,yang2025costaware}. Yet a calibrated +probability does not specify how much model uncertainty a portfolio should +bear. A point-PD optimizer can concentrate capital in loans that appear +attractive where the predictive model is least certain. + +Conformal prediction offers finite-sample coverage language under explicit +exchangeability conditions \citep{vovk2005,angelopoulos2023}, while robust +optimization makes uncertainty operational through feasibility sets +\citep{bertsimas2004,goldfarb2003robustportfolio}. Joining them is attractive, +but an applied decision paper still has to answer three questions. Which +conformal level is informative rather than nearly vacuous? How is a policy +selected without using the outcomes on which it is later reported? What +economic value is lost when conformal uncertainty actually changes the funded +set? + +CRPTO answers those questions with one deliberately simple policy. A frozen, +calibrated CatBoost model produces $p_i$. An exactly replayed 90\% Mondrian +recipe produces upper endpoint $u_i$. The portfolio uses their midpoint, +\begin{equation}\label{eq:midpoint} +q_i=p_i+0.5(u_i-p_i)=\frac{p_i+u_i}{2}, +\end{equation} +inside the risk constraint. The economic objective remains point-PD expected +net return. This separation is substantive: $p_i$ prices expected loss, while +$q_i$ limits the uncertainty the funded portfolio may carry. It also removes +nonlinear caps, tail rules, and uncertainty penalties that made an earlier +research frontier difficult to explain. + +The empirical design uses a temporal Lending Club panel. The conformal recipe +is fit inside the calibration period, and a later calibration holdout ranks a +declared $3\times3$ grid of round-number risk tolerances and conformal weights. +The final ranking artifact contains no defaults, realized returns, or +miscoverage. The fixed rule is then evaluated on January 2018 through September +2020 originations. Earlier project development inspected this static OOT +corpus, so the final run is a transparent retrospective lockbox replay, not a +pristine prospective or preregistered trial. + +The paper makes three contributions. First, it gives an auditable +prediction-to-decision construction in which the economic objective and the +conformal guardrail have separate roles. Second, it replaces approximate +cross-alpha scaling with an exact replay of the frozen conformal recipe and a +calibration-only final policy selector. Third, it reports the price and limits +of that guardrail against matched point-PD and more-conservative comparators, +including temporal slices where CRPTO wins and slices where it does not. The +novelty is the closed, inspectable decision protocol for a frozen credit model, +not a claim that conformal prediction, robust optimization, or credit scoring +is individually new. \begin{figure}[t] - \centering - \includegraphics[width=\textwidth]{crpto_fig1_journal_pipeline.png} - \caption{CRPTO maps frozen calibrated PD models through a Mondrian conformal - uncertainty layer into an auditable robust portfolio decision and exact - funded-set certificate.} - \label{fig:pipeline} +\centering +\includegraphics[width=0.92\textwidth]{crpto_fig1_journal_pipeline.png} +\caption{CRPTO carries a frozen calibrated PD through an exact conformal +replay, a simple portfolio guardrail, and a funded-set audit.} +\label{fig:pipeline} \end{figure} -% ===================================================================== \section{Related Work}\label{sec:related} -CRPTO builds on conformal prediction, especially split conformal methods and their -risk-control extensions \citep{vovk2005,angelopoulos2023,angelopoulos2024foundations,angelopoulos2024risk,bates2021rcps}. -The relevant property is not that conformal intervals are the narrowest possible -uncertainty summaries. It is that they provide distribution-free coverage language -under explicit exchangeability assumptions, and that this language can be audited -without trusting a fully parametric posterior. Mondrian and group-conditional -variants are especially natural in credit because risk grades are already used as -business and governance partitions \citep{bostrom2021,gibbs2024,zhou2024}. - -The second foundation is robust optimization and contextual optimization. -Classical robust optimization frames uncertainty as a set against which a decision -must remain feasible, with the price of robustness made visible as a design -trade-off \citep{bertsimas2004}. Robust portfolio selection makes that trade-off -operational for allocation under parameter uncertainty -\citep{goldfarb2003robustportfolio}, whereas distributionally robust optimization -broadens the uncertainty object toward moment or ambiguity sets -\citep{delage2010dro}. Recent work connects conformal prediction and robust -optimization more directly by using conformal uncertainty sets in downstream -decisions \citep{johnstone2021,patel2024,sun2024ptc,hu2026crc}. That -line certifies the uncertainty set \emph{before} the decision: coverage of the -conformal region is the guarantee, and the downstream decision inherits it. -CRPTO follows this line but audits the other side of the decision as well: -after the optimizer selects a funded set, the realized weighted miscoverage -$V(\alpha)$ and the deterministic budget identity of -Theorem~\ref{thm:funded-set-bound} -are evaluated exactly on that funded set, so the certificate concerns the -allocation actually purchased rather than the input set alone. Its empirical -emphasis also differs: the uncertainty set is -not an abstract benchmark instance, but a credit-risk interval model with -auditable lineage, paper tables, and model-risk documentation. - -The third foundation is predict-then-optimize and decision-focused learning. SPO+ -and modern decision-focused learning ask models to respect the loss surface induced -by the downstream decision \citep{elmachtoub2022,donti2017,mandi2024}. IJDS work on -causal decision making sharpens the same warning: once an estimate feeds an -action, the relevant target can become the assignment rule rather than only the -intermediate effect-size estimate \citep{fernandezloria2022causaldecision}. CRPTO is more -conservative. It does not retrain the PD model end-to-end through the optimizer. -Instead, it asks what can be achieved when a calibrated predictive system is -already frozen and the decision layer must remain explainable to credit-risk -reviewers. Robust losses for decision-focused learning \citep{schutte2024robust} -share this protective intent, but operate at training time rather than as a -post-hoc auditable constraint. - -The fourth foundation is machine learning and optimization for credit -decisions. Credit-scoring benchmarks define the performance frontier on retail -panels \citep{lessmann2015,ayari2026,boosting2025default,xia2017}. Recent IJDS -credit-risk work shows how richer data structures such as firm graphs can improve -rating prediction \citep{das2023creditgraph}, and cost-aware calibration work makes -explicit why probability quality matters when predictions feed asymmetric -downstream decisions \citep{yang2025costaware}. Recent EJOR credit-scoring work -similarly moves from discrimination to economic uncertainty, evaluating uncertainty -through profit and rejection objectives. -IJDS decision papers also sharpen -the distinction between an accurate intermediate estimate and an effective -automated decision -\citep{fernandezloria2022causaldecision,fernandezloria2025observational}, while -replication-robust analytics markets show the journal's appetite for robust, -reproducible decision systems \citep{falconer2026replication}. Work specific to -the Lending Club platform spans alternative-data fintech lending -\citep{jagtiani2019altdata}, scorecard equity \citep{albanesi2024credit}, -two-stage learning under fragmentary data \citep{zheng2026twostage}, P2P -portfolio selection and profit scoring -\citep{zhao2016p2pportfolio,serrano2016profitscoring}, and operations-research -treatments of the same investment-decision problem CRPTO studies -\citep{aior2025lendingclub}. CRPTO does not compete on raw ranking against this -literature; its champion AUC is mid-range. The contribution is the -auditable bridge from a calibrated, frozen PD model to a robust portfolio decision, -not another point on the credit-scoring leaderboard. - -Finally, recent work on conformal model selection for robust optimization, -multi-distribution conformal validity, online conformal portfolio methods, -end-to-end conformal risk training, and conformal satisficing -\citep{bao2025croms,yang2026multidistribution,liu2026portfolio,yeh2025training,zhao2025robust} -fixes the boundary around the single IJDS claim. We use those ideas where -they can be evaluated from the frozen CRPTO evidence: OCE/CVaR -\citep{rockafellar2000cvar,bental2007oce} appears as a tail-risk diagnostic, robust -satisficing appears as committee-style margin evidence, and SPO+ motivates the -regret-auditability frontier. The method-changing variants---optimized OCE/CVaR -objectives, non-exchangeable recalibration, formal post-selection conformal-set -selection, online or multi-dataset protocols, causal layers, and hybrid -decision-focused training---are outside the submitted claim rather than hidden acceptance -criteria. - -\FloatBarrier -\subsection{Closest Work Boundary}\label{sec:closest} - -CRPTO is not first in any broad individual family; its claim is the combination of -calibrated PD, conformal uncertainty, robust credit-portfolio optimization, exact -funded-set validation, and reproducible governance. - -\begin{table}[h] - \centering - \small - \caption{Closest work boundary for CRPTO.} - \label{tab:closest-work} - \begin{tabular}{p{0.23\textwidth}p{0.23\textwidth}p{0.23\textwidth}p{0.23\textwidth}} - \toprule - Neighboring literature & What it contributes & What CRPTO adds & Why not enough \\ - \midrule - P2P/Lending Club OR & Robust or multi-objective funding models. & Conformal PD uncertainty plus exact funded-set validation. & Leaves a gap between prediction uncertainty and a post-allocation certificate. \\ - Conformal credit scoring & Intervals for score uncertainty. & A downstream robust portfolio decision. & Does not audit a budgeted funded set or economic policy. \\ - Conformal robust optimization & Conformal sets in robust decisions. & Frozen credit-risk lineage, PD governance and funded-set certificate. & Not credit-specific enough for PD lineage. \\ - Decision-focused learning & Regret-aligned training. & Post-hoc governance for existing calibrated PD models. & Does not certify risk controls after a frozen PD model. \\ - Conformal finance portfolios & Financial portfolio applications. & Retail-credit payoffs, default risk and model-risk documentation. & Market-return portfolios face different payoffs and controls. \\ - \bottomrule - \end{tabular} -\end{table} - -The novelty is the closed loop, not any single row of the table: a frozen calibrated -credit model becomes conformal upper endpoints, those endpoints enter a robust -funded-set decision, and the purchased allocation is audited after the decision with -file-backed governance. None of the neighboring lines jointly supplies that -frozen-model-to-funded-set certificate for a credit portfolio. - -% ===================================================================== -\section{Method}\label{sec:method} - -\subsection{Calibrated PD Layer}\label{sec:method-pd} - -The predictive layer estimates a one-period default probability for each loan. The -champion model is a CatBoost classifier trained on the frozen feature contract and -calibrated before it is exposed to conformal and optimization layers. The paper -reports discrimination and probability quality together: the PD layer reaches - AUC $0.7139$, Brier score $0.1544$, and expected calibration error approximately - $0.0070$ on the paper-facing evaluation summary. These numbers matter because the -optimizer consumes probabilities, not rankings alone. - -Calibration is treated as a contract. The downstream layers do not receive a -free-form classifier; they receive a calibrated PD vector, feature metadata, and a -model contract that fixes feature order and categorical handling. This prevents a -common reproducibility failure in applied predict-then-optimize studies: a table can -change because a preprocessing file changed, even though the optimization code -did not change. - -\subsection{Mondrian Conformal Layer}\label{sec:method-cp} - -For each loan $i$, let $\hat{p}_i$ denote the calibrated PD. The conformal layer -forms prediction intervals on the PD scale and records the upper endpoint -$u_i(\alpha)$. Operationally, CRPTO evaluates Mondrian partitions rather than a -single global interval. The selected uncertainty layer uses a score-decile-based -Mondrian partition selected by out-of-time interval quality, while grade-based -partitions remain the natural governance baseline. The score-decile choice keeps -each cell well populated even at the tight $\alpha=0.01$, where a per-cell $99\%$ -upper endpoint needs on the order of $100$ calibration points; finer grade-period -partitions are sparser, and the supplement flags where small cells weaken coverage -(notably the external Freddie panel). - -The resulting conformal summary is more than a scalar coverage number. -The paper-facing metrics include 90\% coverage $0.9297$, 95\% coverage $0.9664$, -average 90\% interval width $0.7842$, minimum group 90\% coverage $0.9190$, and 90\% - Winkler score $1.1107$ for the promoted conformal winner. Because the outcome is - binary and intervals live on the clipped PD scale, raw width is not a standalone - utility claim. Its decision role is relative: upper endpoints rank loans by - protected downside risk, the funded-set audit shows mean upper endpoints rising - from $0.12529$ in A--B to $0.52587$ in E--G, and the promotion gate uses Winkler - score, funded-set miscoverage, and $\Gamma_{\mathrm{CP}}$ rather than treating - narrowness as the objective. The full gate also scores - material coverage, group coverage, interval width, and alert rate, but not - exact-nominal-coverage backtests: conformal intervals over-cover by design, so - testing for exact nominal coverage would penalize the safety margin the method is - meant to provide on a large out-of-time sample. - -\subsection{Robust Portfolio Layer}\label{sec:method-lp} - -The decision variable $x_i$ is the allocation fraction for each eligible loan; -$x_i a_i$ is the funded exposure. The optimizer maximizes expected net economic -return under a \$1M budget and policy constraints that replace point PD with a -declared effective decision score $q_i(\alpha;\theta)$. Every frontier policy -satisfies $\hat p_i\le q_i(\alpha;\theta)\le u_i(\alpha)$; $\theta$ identifies a -linear, capped, or tail-focused blend. The selected body point has -$\texttt{risk\_tolerance}=0.1715$, -$\texttt{policy\_mode}=\texttt{capped\_blended\_uncertainty}$, -$\gamma=0.5475$, and $\texttt{uncertainty\_aversion}=0.05$. - -Schematically, the robust decision layer solves -\[ -\begin{aligned} -\max_x\quad & \sum_i x_i a_i \left(c_i - q_i(\alpha;\theta)\,L\right) \\ -\text{s.t.}\quad & \sum_i x_i a_i \le B,\\ -& \sum_i x_i a_i q_i(\alpha;\theta) - \le \tau \sum_i x_i a_i,\\ -& 0 \le x_i \le \bar x_i, -\end{aligned} -\] -where $a_i$ is exposure, $c_i$ is the loan coupon, $L$ is the loss-given-default -($L=0.45$ in the frozen evaluation), and $\tau$ is the risk-tolerance cap. The -linear policy member is -\[ -q_i(\alpha;\gamma)=\hat p_i+\gamma\left(u_i(\alpha)-\hat p_i\right). -\] -Capped and tail-focused members transform that score while remaining between -the point PD and upper endpoint. For the selected allocation, the cap is inactive -on all 314 funded rows, so its effective score equals the linear expression; this -is audited rather than assumed for other policies. The objective is the -\emph{expected} net return $c_i-q_iL$; the headline realized return is -the post-hoc accounting of the same funded set on observed -defaults (a funded loan earns $c_i a_i$ if it survives and loses $L a_i$ if it -defaults). Separating the optimized expectation from the realized accounting is -deliberate: the optimizer never sees outcomes, so the realized figure is an -out-of-sample audit of the policy, not the objective it maximized. Additional -operational filters and caps live in the frozen policy configuration; the -manuscript displays the core statistical-to-decision contract because that is -the reusable CRPTO pattern. - -The lowercase $\gamma$ is a policy parameter; the uppercase quantities are -post-allocation funded-set metrics: -\[ -\begin{aligned} -\Gamma_{\mathrm{CP}}(\alpha)&=\sum_iw_i(u_i-\hat p_i),\\ -\Gamma_{\mathrm{int}}(\alpha)&=\sum_iw_i(q_i-\hat p_i),\\ -\Gamma_{\mathrm{res}}(\alpha)&=\sum_iw_i(u_i-q_i). -\end{aligned} -\] -Thus $\Gamma_{\mathrm{CP}}=\Gamma_{\mathrm{int}}+\Gamma_{\mathrm{res}}$. At -$\alpha=0.01$, the selected point has $\Gamma_{\mathrm{CP}}=0.162616$, -$\Gamma_{\mathrm{int}}=0.089032$, $\Gamma_{\mathrm{res}}=0.073584$, and -$V=0.035350$. Its weighted realized default rate is also $0.035350$, below -$\tau=0.1715$, so realized risk-tolerance excess is zero. This is an empirical -audit result, not a violation metric for the deterministic identity below. - -% ===================================================================== -\section{Theory}\label{sec:theory} - -The theoretical role of conformal prediction in CRPTO is modest and explicit. For a -fixed allocation evaluated on exchangeable calibration/test data, conformal coverage -controls the expected rate at which outcomes fall outside the constructed uncertainty -intervals. When funded-set weights are non-negative and normalized, this yields a -weighted miscoverage quantity $V(\alpha)$ that can be monitored after the decision. -Two statements are kept separate. The deterministic portfolio identity -(Theorem~\ref{thm:funded-set-bound}(i)) holds for any allocation and needs no -distributional assumption; it is the accounting the exact certificate verifies. The -probabilistic statement (Theorem~\ref{thm:funded-set-bound}(ii)) is a Markov argument -that requires weighted funded-set validity, $\mathbb{E}[V(\alpha)]\le\alpha$, stated as -Assumption~\ref{asm:weighted-validity}. This is a modeling assumption, not a property -the single frozen draw establishes: on the selected funded set the realized weighted -miscoverage is $V(0.01)=0.035350$, \emph{above} the nominal $\alpha=0.01$, the expected -price of evaluating an adaptively selected subportfolio rather than a fresh population. -What the paper certifies is therefore the exact accounting together with the safety -level $V\le\sqrt{\alpha}$ that Markov delivers, not a claim that the funded set attains -nominal $\alpha$-coverage or that post-selection evaluation creates a stronger -conformal guarantee. This boundary also covers conformal-set selection: choosing -the most attractive set or policy after seeing multiple valid candidates is -itself a statistical operation that needs its own protocol. +CRPTO sits at the intersection of conformal prediction, robust optimization, +and decision-focused learning. Split conformal methods provide marginal +coverage without a parametric posterior, while Mondrian variants condition on +declared partitions \citep{vovk2005,bostrom2021}. Conditional and weighted +extensions require additional structure, and exact conditional coverage is +generally unavailable without restrictive assumptions +\citep{barber2021limits,barber2023beyond,jonkers2024wcps}. We therefore +distinguish population or partition coverage from coverage after a portfolio +has adaptively reweighted the loans. + +Data-driven robust and contextual optimization translate predictive +uncertainty into decisions +\citep{bertsimas2018datadriven,bertsimas2020prescriptive,sadana2025contextual}. +Recent work uses conformal sets in robust optimization +\citep{johnstone2021,patel2024,sun2024ptc,hu2026crc}. CRPTO is an applied +complement: it retains a frozen credit PD model, exposes the funded rows and +uncertainty premium, and compares the resulting allocation with a matched +point-PD decision. + +Decision-focused learning and SPO+ train predictions against downstream regret +\citep{donti2017,elmachtoub2022,mandi2024}. That is a different institutional +choice. CRPTO asks what can be done after a calibrated model already exists and +must remain frozen for governance. A synthetic SPO+ diagnostic remains in the +online supplement; it is not mixed with the real-dollar portfolio results. + +Credit-allocation research already covers profit scoring, P2P investment +recommendation, robust loan portfolios, rejection, and multiobjective risk +\citep{guo2016p2p,zhao2016p2pportfolio,serrano2016profitscoring,chi2019p2p, +babaei2020p2p,xu2025profit_uncertainty_credit,xu2024profit_risk_credit}. +Conformal credit scoring also means that the safe claim is not first use of +conformal prediction in credit \citep{kawasumi2026ordinal}. The remaining gap +is a file-backed protocol that shows exactly how a conformal endpoint changes a +budgeted credit decision and what that change costs. -\begin{figure}[t] - \centering - \includegraphics[width=0.95\textwidth]{crpto_fig20_bound_claim_layers.pdf} - \caption{The CRPTO bound claim stack separates conformal endpoints, - deterministic portfolio accounting, the weighted-validity assumption, and the - frozen exact certificate.} - \label{fig:bound-stack} -\end{figure} - -The bound is read as three linked objects: a deterministic accounting identity, -an explicitly stated statistical assumption, and the frozen empirical certificate -that verifies the promoted decision exactly on the 276{,}869-loan OOT evaluation. -We now state the first two formally. Fix the promoted allocation $x$, chosen -without access to OOT labels, with exposures $a_i>0$ and funded-set weights -$w_i=x_i a_i/\sum_j x_j a_j$, so that $w_i\ge 0$ and $\sum_i w_i=1$. For each -funded loan let $Y_i\in[0,1]$ be the realized outcome on the PD scale, -$u_i(\alpha)\in[0,1]$ its upper conformal endpoint, and -$Z_i(\alpha)=\mathbf{1}\{Y_i>u_i(\alpha)\}$ the miscoverage indicator. The -weighted funded-set miscoverage is $V(\alpha)=\sum_i w_i Z_i(\alpha)$. -Probabilities and expectations below are taken over the exchangeable -calibration/test draw, conditional on the frozen recipe, declared partitions, -and allocation rule. - -\begin{assumption}[Weighted funded-set validity]\label{asm:weighted-validity} -$\mathbb{E}[V(\alpha)]\le\alpha$ under the funded-set weights $w$ for that draw; -in words, the expected weighted miss rate on the funded dollars is no larger -than the nominal conformal level. -\end{assumption} - -Assumption~\ref{asm:weighted-validity} is the explicit price of evaluating a selected -portfolio rather than a single population, and it does not follow from marginal split -conformal. The funded-set weights $w_i\propto x_i a_i$ are chosen by the optimizer and -depend on the conformal endpoints $u_i(\alpha)$, so they are not measurable with respect -to the Mondrian partition and inherit no per-cell coverage guarantee. The assumption is -therefore stated, audited empirically after the frozen selection, and never silently -upgraded to a guarantee---and the audit does not find it slack: the realized -$V(0.01)=0.035350$ exceeds $\alpha=0.01$, so the operative safety level is the weaker -$V\le\sqrt{\alpha}$. - -\begin{theorem}[Distribution-free funded-set risk bound]\label{thm:funded-set-bound} -Let $B_u(\alpha)=\sum_i w_i u_i(\alpha)$ be the weighted conformal -upper-endpoint budget of the funded set. Then -(i) deterministically, $\sum_i w_i Y_i \le B_u(\alpha) + V(\alpha)$; and -(ii) under Assumption~\ref{asm:weighted-validity}, for every $t>0$, -$\mathbb{P}(V(\alpha)\ge t)\le\alpha/t$, and in particular -\[ -\mathbb{P}\Bigl(\textstyle\sum_i w_i Y_i \;\ge\; B_u(\alpha)+\sqrt{\alpha}\Bigr) - \;\le\; \sqrt{\alpha}. -\] -\end{theorem} - -\proof{Proof sketch.} -Since $Y_i\le u_i(\alpha)+Z_i(\alpha)$ for every loan, part~(i) follows by -taking the $w$-weighted sum; it is portfolio accounting and needs no -probability. Part~(ii) is Markov's inequality applied to the nonnegative -variable $V(\alpha)$ with $\mathbb{E}[V(\alpha)]\le\alpha$ \citep{ghosh2002}, -combined with~(i). The full proof is in Online Supplement Appendix~A. \Halmos -\endproof - -\begin{remark}[The optimizer's cap versus the endpoint budget]\label{rem:endpoint-budget} -The robust layer constrains the policy-specific effective score, not -$B_u(\alpha)$ directly: $\sum_iw_iq_i(\alpha;\theta)\le\tau+s$, where $s\ge0$ -is recorded solver cap slack. The policy-aware decomposition gives -\[ -B_u(\alpha)=\sum_iw_iq_i(\alpha;\theta)+\Gamma_{\mathrm{res}}(\alpha) - \;\le\;\tau+s+\Gamma_{\mathrm{res}}(\alpha). -\] -For a pure linear blend only, -$\Gamma_{\mathrm{res}}=(1-\gamma)\Gamma_{\mathrm{CP}}$. The selected capped -policy has no active row-level cap on its funded set, its effective-score cap -binds with $s=0$, and therefore $B_u(0.01)=0.1715+0.073584=0.245084$. The -deterministic identity gives -$\sum_iw_iY_i\le0.245084+V(0.01)=0.280434$, while the observed left-hand side is -$0.035350$. The exact Markov loss threshold is -$T_{0.01}=B_u(0.01)+\sqrt{0.01}=0.345084$; under -Assumption~\ref{asm:weighted-validity}, -$\mathbb{P}(\sum_iw_iY_i\ge T_{0.01})\le0.10$. It is a probabilistic event -threshold, not a deterministic risk cap. This policy-aware form is essential for -capped and tail-focused frontier points, where the linear shortcut need not hold. -\end{remark} - -\begin{remark}[Why $t=\sqrt{\alpha}$, and why Markov]\label{rem:sqrt-alpha} -The choice $t=\sqrt{\alpha}$ is made for interpretability, not optimality: it -reads cleanly as ``miscoverage exceeds $\sqrt{\alpha}$ with probability at most -$\sqrt{\alpha}$'' (for $\alpha=0.01$, a $0.10$ excess with probability at most -$0.10$). Markov is deliberately the weakest defensible argument: it uses only -the first moment. Supplement Propositions~A.1--A.2 separate the boundary: -under Assumption~1 alone the best second-moment (Cantelli) threshold is -\emph{worse} than Markov, while explicit cross-cluster structure is the extra -condition under which Hoeffding-style tightening becomes available -\citep{hoeffding1963,boucheron2013concentration}. Those tightenings are kept -in the online supplement (A21) rather than the body, because the contribution -is the auditable decision construction, not the sharpest tail bound. The exact -certificate in this paper is the empirical audit of the frozen selected policy, -not a stronger post-selection conformal theorem. -\end{remark} - -The theorem and the two supplement propositions should be read as one small -triptych. Theorem~\ref{thm:funded-set-bound} gives the paper's guarantee once -weighted funded-set validity is accepted. Supplement Proposition~A.1 shows -that, without additional structure, Markov is not a placeholder for a missing -second-moment bound; it is the sharp first-moment statement. Supplement -Proposition~A.2 then asks what extra structure would buy a tighter threshold. -In this temporal credit panel the defensible version is cross-period or -period-grade independence after the frozen recipe and allocation are fixed: -within a period, grade, or period-grade cell, defaults and interval misses may -remain dependent. The observed funded set is too exposure concentrated for that -cluster argument to tighten the headline bound, which is why the body keeps -Markov and the supplement reports the cluster calculation as a sensitivity -check. - -% ===================================================================== -\section{Experimental Design}\label{sec:design} - -The empirical study uses Lending Club retail-loan data covering originations from -2007 through 2020. The raw panel is cleaned into a static feature store and split -temporally with a January 2018 cutoff, so calibration and evaluation use only loans -originated after the training window. The calibration block plays the dual role -required by split conformal: held out from training and used only to fit the -conformal quantiles. The final evaluation set contains 276{,}869 loans, large enough -to stress both probability calibration and decision-level robustness. - -\begin{table}[h!] - \centering - \caption{Temporal out-of-time split. The January 2018 cutoff keeps the test - window (including the 2020 COVID regime) strictly after training and calibration. - Monthly vintage labels are shown for readability; split assignment is - row-disjoint in code, so the shared March 2017 label marks the internal cutoff - rather than duplicated loans.} - \label{tab:split} - \resizebox{\textwidth}{!}{% - \begin{tabular}{llrl} - \toprule - Split & Period & Loans & Role \\ - \midrule - Train & Jun 2007 -- Mar 2017 & 1{,}346{,}311 & Fit PD model and monotonic constraints. \\ - Calibration & Mar 2017 -- Dec 2017 & 237{,}584 & Fit Mondrian conformal quantiles (held out). \\ - Test (OOT) & Jan 2018 -- Sep 2020 & 276{,}869 & Coverage, portfolio decision, exact funded-set audit. \\ - \bottomrule - \end{tabular} - }% +\begin{table}[t] +\centering +\small +\caption{Closest-work boundary for CRPTO.} +\label{tab:closest} +\begin{tabular}{p{0.20\textwidth}p{0.34\textwidth}p{0.37\textwidth}} +\toprule +Family & Existing contribution & CRPTO boundary \\ +\midrule +Credit scoring & Calibrated and cost-aware PD & PD is an input contract; no AUC-leadership claim. \\ +Robust credit portfolios & Economic selection under uncertainty & Exact conformal endpoint becomes a simple guardrail. \\ +Conformal robust optimization & Coverage-backed uncertainty sets & Frozen credit stack, funded-set audit, matched economics. \\ +Decision-focused learning & Training-time regret reduction & Frozen predictor and post-hoc auditability. \\ +Valid conformal selection & Validity after model or set selection & Motivates the boundary; selected-set validity is not claimed. \\ +\bottomrule +\end{tabular} \end{table} -The out-of-time design is adversarial: the test window spans an -expansion (2018--2019) and a regime break (2020 COVID), so coverage and the -funded-set certificate are measured under a documented distribution shift rather -than on a random split that would let the model see the future. - -The design distinguishes three kinds of computation. Predictive and conformal -searches choose models, calibration, partitions, and policy families. Those -searches are frozen for this manuscript. Paper-facing reruns regenerate tables, -figures, evidence summaries, and manuscript surfaces from frozen inputs. This -separation is central to the reproducibility claim: the manuscript can evolve -without quietly reopening the policy search that selected the promoted result. -Governance keeps the two certified results distinguishable: the frozen upstream -record defines the declared return floor, while the selected-policy governance -files define the body point and its frontier. Automated claim-sync tests assert -that every body-claim number printed in the manuscript and supplement matches -those files. A validation harness additionally rebuilds the promoted conformal -intervals from the frozen PD binaries and recorded recipe and verifies the -published endpoints and coverage summaries. Gradient-boosted retraining is not -presented as the routine reproduction target across machines; retraining would -be a new research run, not the routine reproduction path for this submission. - -All primary evidence objects are represented as files with explicit ownership: model -binaries, calibration objects, conformal intervals, portfolio allocations, a -YAML/Parquet feature contract, tables, figures, and status reports. The anonymous -submission describes the bundle without -revealing author identity. Repository and remote-storage URLs will be disclosed -according to the journal's double-anonymous and data/code-disclosure policy. - -The body-supplement split is fixed before submission. The body keeps the CRPTO -pipeline, the alpha-to-portfolio link, the finite-grid frontier, and the -core metrics, plus the compact regret-auditability frontier. The online supplement carries -A3--A40, the conformal finalist ablation, funded-set loan audit, tail-risk -diagnostics, satisficing margins, dependence diagnostics, the CVaR/OCE -tail-constrained re-optimization (A22), the multi-distribution (A23) and online -ACI-stability (A24) diagnostics, the multi-dataset external economic replication -tables (A25--A34), the selected-policy frontier and funded-set audits -(A35--A40), MRM/fairness material, and reproduction commands. This keeps the -IJDS body focused while preserving the audit trail that reviewers need. - -\subsection{Multi-Dataset External Replication Protocol} - -The external-replication layer is narrower than a new benchmark -campaign. We reuse the frozen CRPTO recipe---CatBoost PD, calibration, train-only -WOE/IV feature screening, Mondrian conformal intervals, and the same bound-aware -robust LP---on two credit datasets with economic fields. Prosper contributes -final-status marketplace loans with observed outcome, loan amount, yield/rate, -and a temporal OOT window. Freddie/Mendeley contributes processed single-family -mortgage panels derived from the Freddie Mac loan-level ecosystem, including -12--60 month default windows and train/OOS/OOT splits. Home Credit was audited -but not promoted because it lacks a clean investment-return and exposure contract -comparable to Lending Club, Prosper, and Freddie. This gate does not re-promote -the Lending Club champion and does not claim a new exact funded-set theorem for -every external portfolio. - -% ===================================================================== -\section{Results}\label{sec:results} +\section{Data and Evaluation Design}\label{sec:data} -Table~\ref{tab:core} summarizes the paper-facing metrics. The calibrated PD -layer is not sold as a leaderboard model: AUC $0.7139$ is sufficient only -because the downstream decision consumes calibrated probabilities, not rankings -alone. Its Brier score $0.1544$ and ECE near $0.0070$ are therefore as important -as discrimination. The conformal layer over-covers marginally at the reported -levels (90\% coverage $0.9297$, 95\% coverage $0.9664$ for the conformal winner). -The portfolio layer then turns this uncertainty into an exact finite-grid -return-bound frontier. The selected policy passes the -$V\le\sqrt{\alpha}$ certificate at the tightest reported level and has zero -realized risk-tolerance excess. - -The results are ordered around four reviewer questions: what certificate the -selected policy carries, where it sits on the finite-grid frontier, what it buys -relative to a matched point-PD LP, and whether the same recipe transfers to other -credit panels without changing the Lending Club champion. +The data are Lending Club retail loans originated from 2007 through 2020. The +feature contract contains only origination-time information. Model development +and evaluation use temporal rather than random splits. \begin{table}[t] - \centering - \caption{Frozen paper-facing core metrics by layer.} - \label{tab:core} - \begin{tabular}{llr} - \toprule - Layer & Metric & Value \\ - \midrule - PD & AUC & $0.7139$ \\ - PD & Brier score & $0.1544$ \\ - PD & ECE & $0.0070$ \\ - Conformal & Coverage 90\% & $0.9297$ \\ - Conformal & Coverage 95\% & $0.9664$ \\ - Conformal & Minimum group coverage 90\% & $0.9190$ \\ - Portfolio & Body-point robust return & \$184{,}832.48 \\ - Portfolio & Weighted realized default & $0.035350$ \\ - Portfolio & $V(\alpha=0.01)$ & $0.035350$ \\ - Portfolio & $\Gamma_{\mathrm{CP}}(\alpha=0.01)$ & $0.162616$ \\ - Portfolio & $\Gamma_{\mathrm{res}}(\alpha=0.01)$ & $0.073584$ \\ - Portfolio & Endpoint budget $B_u(0.01)$ & $0.245084$ \\ - Portfolio & Exact Markov loss threshold & $0.345084$ \\ - Portfolio & Realized risk-tolerance excess & $0.0$ \\ - Portfolio & Declared alpha-grid pass & $8/8$ \\ - \bottomrule - \end{tabular} +\centering +\small +\caption{Temporal Lending Club design.} +\label{tab:splits} +\begin{tabular}{lrrp{0.42\textwidth}} +\toprule +Split & Period & Loans & Role \\ +\midrule +Train & Jun 2007--Mar 2017 & 1{,}346{,}311 & Fit PD model and calibrator. \\ +Calibration pool & Mar--Dec 2017 & 237{,}584 & Develop and freeze conformal and policy rules. \\ +OOT evaluation & Jan 2018--Sep 2020 & 276{,}869 & Freeze-then-evaluate portfolio decisions. \\ +\bottomrule +\end{tabular} \end{table} -The exact certificate is an accounting claim. Here ``exact'' means the quantities -are computed directly on the frozen OOT funded set rather than approximated by a -surrogate table or visual proxy, and the deterministic identity requires no -distributional assumption. The declared empirical pass combines -$V(\alpha)\le\sqrt{\alpha}$ with realized risk-tolerance excess no larger than -$\alpha$; for the selected point that excess is zero. This screen is \emph{not} -a claim of nominal $\alpha$-coverage, which the funded set does not attain -($V=0.035350>\alpha=0.01$), and the excess criterion is not a new probabilistic -theorem. - -This makes $\Gamma_{\mathrm{CP}}$ more than a diagnostic line item. It is the -amount of conformal robustness carried by the funded set. A reviewer can read -$\Gamma_{\mathrm{CP}}=0.162616$ as the total premium, -$\Gamma_{\mathrm{res}}=0.073584$ as the part not internalized by the decision -score, $V=0.035350$ as realized weighted noncoverage, and $0.345084$ as the loss -level whose exceedance probability is bounded by $0.10$ under -Assumption~\ref{asm:weighted-validity}. - -\begin{table}[t] - \centering - \caption{Exact certificate for the promoted funded set. The Markov column is - the exact event threshold $B_u+\sqrt{\alpha}$, not a deterministic cap.} - \label{tab:exact-certificate} - \resizebox{\textwidth}{!}{% - \begin{tabular}{rrrrrrrr} - \toprule - $\alpha$ & $\Gamma_{\mathrm{CP}}$ & $\Gamma_{\mathrm{res}}$ & $V(\alpha)$ & $B_u$ & Markov threshold & Risk excess & Pass \\ - \midrule - $0.01$ & $0.162616$ & $0.073584$ & $0.035350$ & $0.245084$ & $0.345084$ & $0.00000$ & yes \\ - \bottomrule - \end{tabular} - }% -\end{table} +The frozen conformal recipe uses the most recent 75\% of the calibration pool +(178{,}188 rows). Within that subset, 142{,}550 rows estimate conformal +quantiles and 35{,}638 later rows, from November and December 2017, form the +temporal development holdout. Conformal endpoints use calibration labels, as +conformal prediction requires. Policy ranking uses candidate settings, solver +status, expected point-PD objective, budget use, effective-PD exposure, and +endpoint summaries. A schema guard rejects default, outcome, realized-return, +and miscoverage fields. -The central empirical object is now a return-bound frontier rather than a single -winner. The consolidated frontier surface deduplicates 51{,}678 raw rows into -50{,}010 semantic policies; 27{,}508 policies both pass every level in the -declared alpha grid and exceed the declared return floor. The terminal endpoint -search alone evaluates 37{,}068 policies and 296{,}544 exact alpha checks, with -37{,}068/37{,}068 policies passing all eight alpha levels. - -The provenance of the frontier deserves one explicit sentence, because two -certified numbers coexist in the project's governance record. The declared -return floor \$170{,}464.54 is itself the realized return of the previously -certified bound-aware allocation on the same frozen chain, retained as the -frozen upstream baseline; the declared frontier is a deterministic re-evaluation -of a pre-declared finite policy grid over the \emph{same} frozen PD model -and Mondrian conformal intervals, so no upstream object (model, calibrator, -or interval) was regenerated when the body point moved from the floor to -\$184.8K. The earlier certificate is not discarded: it becomes the floor -that every eligible frontier policy must beat, which is why the frontier is -reported with the floor surplus rather than as a replacement champion. +The final OOT panel covers several regimes, including the 2020 disruption. We +report the full panel and five temporal slices. Each slice solves the same fixed +policy on a fresh \$1M budget; slice returns are comparable stress evaluations, +not components that sum to the full-panel value. -\begin{table}[t] - \centering - \caption{Pool93 finite-grid return-bound frontier. Each threshold uses the exact - funded-set endpoint budget. \texttt{terminal}, \texttt{bound-closure}, and - \texttt{micro-ext} denote frozen finite grids, not continuous optima.} - \label{tab:pool93-frontier} - \resizebox{\textwidth}{!}{% - \begin{tabular}{llrrrrr} - \toprule - Policy role & Source & Realized return & $V(0.01)$ & - $\Gamma_{\mathrm{CP}}$ & Markov threshold & Pass \\ - \midrule - Minimum Markov-threshold endpoint & terminal & \$170{,}467.27 & $0.031875$ & $0.095719$ & $0.273036$ & $8/8$ \\ - Low-threshold balanced endpoint & terminal & \$171{,}006.20 & $0.031875$ & $0.097190$ & $0.274789$ & $8/8$ \\ - Highest return under threshold $\le 0.30$ & bound-closure & \$174{,}552.51 & $0.035875$ & $0.120988$ & $0.299997$ & $8/8$ \\ - Highest return under threshold $\le 0.345$ & micro-ext & \$184{,}800.41 & $0.035350$ & $0.162562$ & $0.344997$ & $8/8$ \\ - Body/default balanced point & micro-ext & \$184{,}832.48 & $0.035350$ & $0.162616$ & $0.345084$ & $8/8$ \\ - Highest return under threshold $\le 0.36$ & micro-ext & \$186{,}050.73 & $0.037750$ & $0.174600$ & $0.358685$ & $8/8$ \\ - Max-return economic endpoint & micro-ext & \$223{,}458.14 & $0.069575$ & $0.457438$ & $0.697056$ & $8/8$ \\ - \bottomrule - \end{tabular} - }% -\end{table} +\section{Method}\label{sec:method} -Table~\ref{tab:pool93-frontier} gives the manuscript its decision geometry. The -body/default point is not the highest-return point and not the tightest-bound -point; it is the balanced point selected by the declared return-bound lens. The -strict $\le 0.345$ row is reported separately because it is a different finite -policy: it earns \$184{,}800.41 at threshold $0.344997$, whereas the -body/default row earns \$184{,}832.48 at threshold $0.345084$. The endpoint at -$0.273036$ shows how conservative the frontier can become while preserving the -return floor, and the \$223.5K tail-focused endpoint requires a threshold of -$0.697056$. Its residual premium cannot be recovered with the linear -$(1-\gamma)\Gamma_{\mathrm{CP}}$ shortcut. The supplement reports the full -policy-aware frontier and traceability details. - -\subsection{Matched Point-PD Baseline}\label{sec:point-baseline} - -To isolate what the conformal decision layer buys, we solve a matched two-stage -LP on the same 276{,}869 candidates with the same \$1M budget, concentration cap, -$\tau=0.1715$, LGD, solver, and operating constraints. The only change is that -the baseline uses calibrated point PD in both its objective and risk constraint. -Neither optimizer sees OOT outcomes; defaults enter only in the frozen post-hoc -audit. +\subsection{Calibrated PD and Exact Conformal Replay} + +Let $p_i\in[0,1]$ be calibrated PD. The frozen CatBoost layer has AUC 0.7139, +Brier score 0.1544, and expected calibration error about 0.0070. These values +are not a leaderboard claim; probability quality matters because the economic +objective consumes PD. + +The conformal recipe partitions calibrated scores into five score-quantile +Mondrian cells. Calibration loan $j$ receives scaled residual +\begin{equation}\label{eq:score} +s_j=\frac{|Y_j-p_j|}{\sqrt{\max\{p_j(1-p_j),10^{-6}\}}}. +\end{equation} +Each cell uses the finite-sample higher quantile at frozen used alpha 0.095 for +target $\alpha=0.10$. Recorded group and temporal factors may widen but never +narrow an interval. The resulting upper endpoint is clipped to $[0,1]$. + +The replay reconstructs every setting from the frozen result payload. At the +90\% reference level it reproduces stored point, lower, and upper vectors with +maximum absolute error below $6.67\times10^{-16}$. This matters because an +earlier exploratory sweep scaled 90\% radii with average widths from another +conformal family; those values are not used in this manuscript. + +\subsection{One Portfolio Policy} + +Let $a_i$ be loan amount, $x_i\in[0,1]$ the funded fraction, $r_i$ the coupon, +$L=0.45$ loss given default, $B=1{,}000{,}000$, and $\tau$ risk tolerance. +CRPTO solves +\begin{equation}\label{eq:portfolio} +\begin{aligned} +\max_x\quad & \sum_i x_i a_i(r_i-p_iL) \\ +\text{s.t.}\quad +& \sum_i x_i a_i\le B, \\ +& \sum_i x_i a_iq_i\le\tau\sum_i x_i a_i, \\ +& 0\le x_i\le\bar x_i, +\end{aligned} +\end{equation} +with the existing eligibility and concentration constraints. The selected +policy uses $\tau=0.17$ and Equation~\eqref{eq:midpoint}. The objective uses +$p_i$, not $q_i$: expected economics and uncertainty feasibility are separate +contracts. The matched point-PD baseline changes only the risk score to +$q_i=p_i$ while holding candidates, budget, concentration, LGD, solver, and +risk tolerance fixed. + +For funded-exposure weights $w_i=x_ia_i/\sum_jx_ja_j$, define +\begin{equation}\label{eq:gamma} +\begin{aligned} +\Gamma_{\mathrm{CP}}&=\sum_iw_i(u_i-p_i),\\ +\Gamma_{\mathrm{int}}&=\sum_iw_i(q_i-p_i),\\ +\Gamma_{\mathrm{res}}&=\sum_iw_i(u_i-q_i),\\ +B_u&=\sum_iw_iu_i. +\end{aligned} +\end{equation} +For the midpoint policy, +$\Gamma_{\mathrm{int}}=\Gamma_{\mathrm{res}}=\Gamma_{\mathrm{CP}}/2$. + +\subsection{Calibration-Only Final Selector} + +The candidate set crosses $\tau\in\{0.15,0.17,0.19\}$ with +$\gamma\in\{0.25,0.50,0.75\}$ in +$q_i=p_i+\gamma(u_i-p_i)$. A candidate is eligible when the solver is optimal, +at least 99.9\% of budget is allocated, the effective-PD cap holds, and +\begin{equation}\label{eq:screen} +B_u+\sqrt{0.10}\le0.60 +\end{equation} +on the development holdout. Among eligible candidates, the rule maximizes +expected point-PD objective. Five of nine pass; the selected row is +$\tau=0.17,\gamma=0.50$. \begin{table}[t] - \centering - \caption{Matched point-PD baseline on the frozen Lending Club OOT panel.} - \label{tab:point-baseline} - \resizebox{\textwidth}{!}{% - \begin{tabular}{lrrrrrr} - \toprule - Policy & Realized return & Funded & Weighted default / $V$ & - $\Gamma_{\mathrm{CP}}$ & $B_u$ & Markov threshold \\ - \midrule - Point-PD two-stage LP & \$196{,}369.14 & 225 & $0.118400$ & $0.526736$ & $0.680579$ & $0.780579$ \\ - Selected CRPTO & \$184{,}832.48 & 314 & $0.035350$ & $0.162616$ & $0.245084$ & $0.345084$ \\ - \bottomrule - \end{tabular} - }% +\centering +\small +\caption{Calibration selector examples; the supplement reports all nine rows.} +\label{tab:selector} +\resizebox{\textwidth}{!}{% +\begin{tabular}{lrrrrl} +\toprule +Candidate & $\tau$ & $\gamma$ & Expected objective & Screen value & Status \\ +\midrule +Low guardrail & 0.17 & 0.25 & \$121{,}761.88 & 0.708835 & Ineligible \\ +Selected midpoint & 0.17 & 0.50 & \$110{,}346.16 & 0.577275 & Selected \\ +Conservative blend & 0.17 & 0.75 & \$104{,}272.78 & 0.519696 & Eligible \\ +Higher tolerance & 0.19 & 0.50 & \$113{,}591.27 & 0.611017 & Ineligible \\ +\bottomrule +\end{tabular}} \end{table} -CRPTO gives up \$11{,}536.66, or $5.875\%$, of the baseline's realized return. -In exchange, weighted default/miscoverage falls by $0.08305$ and the exact -Markov loss threshold falls by $0.435495$. Both allocations have zero realized -risk-tolerance excess because their default rates remain below $\tau$, but the -point-PD allocation fails the tightest Markov safety screen -($0.1184>\sqrt{0.01}=0.10$). This is a frozen OOT return--auditability trade-off, -not causal evidence or universal dominance. Full fields are in Supplement A40. - -The funded-set under-coverage remains structural rather than a calibration-draw -effect. With $n_{\mathrm{cal}}=237{,}584$ calibration loans, the split-conformal -conditional-coverage result makes marginal coverage highly stable around the -nominal level \citep{vovk2005,angelopoulos2023}. The residual funded-set $V$ is a -test-side, portfolio-selection quantity. That is why the paper reads the safety -level at $\sqrt{\alpha}$ under Assumption~\ref{asm:weighted-validity}, and why - the frontier reports $V$, $\Gamma_{\mathrm{CP}}$, residual premium, endpoint budget, exact Markov threshold, and -return together instead of promoting a standalone coverage number. +The 0.60 value is a declared ex-ante screen in the final tagged protocol, not +an estimate from OOT outcomes. Earlier research iterations used the OOT corpus; +the narrow claim is that this final ranking code path does not. + +\subsection{Accounting and Statistical Boundary} + +Let $Z_i=\mathbf1\{Y_i>u_i\}$ and $V=\sum_iw_iZ_i$. Since +$Y_i\le u_i+Z_i$ for every funded loan, +\begin{equation}\label{eq:accounting} +\sum_iw_iY_i\le B_u+V +\end{equation} +holds deterministically. It is funded-set accounting, not a coverage theorem. + +If one additionally assumes weighted funded-set validity, +$\mathbb E[V]\le\alpha$, Markov's inequality gives +\begin{equation}\label{eq:markov} +\Pr\!\left(\sum_iw_iY_i\ge B_u+\sqrt{\alpha}\right) +\le\sqrt{\alpha}. +\end{equation} +The assumption does not follow from marginal split conformal because the +optimizer chooses $w_i$ from $p_i$ and $u_i$. Equation~\eqref{eq:markov} is +therefore secondary, assumption-conditional sensitivity. The supplement gives +the proof and selected-set boundary \citep{hegazy2025valid_selection_conformal_sets}. -\begin{table}[t] - \centering - \caption{Reviewer claim checks in the main manuscript.} - \label{tab:reviewer-checks} - {% - \small - \setlength{\tabcolsep}{4pt} - \renewcommand{\arraystretch}{1.12} - \begin{tabular}{@{}>{\raggedright\arraybackslash}p{0.20\textwidth}>{\raggedright\arraybackslash}p{0.47\textwidth}>{\raggedright\arraybackslash}p{0.25\textwidth}@{}} - \toprule - Reviewer concern & Body answer & Primary evidence \\ - \midrule - ``This is only a classifier.'' & The claim is decision auditability, not AUC leadership. & Exact funded-set certificate and A35 frontier. \\ - ``CP + RO already exists.'' & CRPTO instantiates the idea for frozen credit PD models, funded-set governance, and Lending Club payoffs. & Closest-work boundary and bound claim stack. \\ - ``There is no matched baseline.'' & A40 holds candidates and operating constraints fixed and quantifies a $5.875\%$ return cost. & Table~\ref{tab:point-baseline} and A40 audit. \\ - ``Adaptive selection breaks coverage.'' & The theorem states weighted funded-set validity as an assumption and then audits the frozen selection exactly. & Assumption map, validity ladder, and A23 diagnostics. \\ - ``The selected policy is cherry-picked.'' & The selected point comes from a consolidated finite frontier with 50{,}010 deduplicated semantic policies and 27{,}508 eligible all-alpha above-floor policies. & Frontier table and governance files. \\ - \bottomrule - \end{tabular} - }% -\end{table} +\section{Results}\label{sec:results} -\FloatBarrier -\subsection{Multi-Dataset External Economic Replication} +\subsection{Exact 90\% Conformal Evidence} -The natural generalization question after the Lending Club audit is whether the -recipe still works outside the champion panel. Table~\ref{tab:external-replication} -answers that question without changing the champion: the same frozen recipe is -applied to two external credit products. -Both pass the conformal gates and both return positive robust LP objectives. +At target $\alpha=0.10$, exact OOT coverage is 0.934836, average width is +0.788879, minimum score-partition coverage is 0.926310, minimum letter-grade +coverage is 0.926797, and 51.7873\% of upper endpoints equal one. The intervals +are conservative and broad, as expected for a binary outcome on the probability +scale. -\begin{table}[t] - \centering - \caption{External economic replications using the frozen CRPTO recipe.} - \label{tab:external-replication} - \resizebox{\textwidth}{!}{% - \begin{tabular}{llrrrrrrr} - \toprule - Dataset & Product & Rows & Default & AUC & Cov. 90\% & Cov. 99\% & OOT cand. & Robust LP \\ - \midrule - Prosper & Marketplace loans & 54{,}807 & 30.92\% & 0.7074 & 0.9205 & 0.9943 & 10{,}531 & \$199{,}419 \\ - Freddie FM48 & Mortgages & 3{,}173{,}355 & 1.45\% & 0.7839 & 0.9745 & 0.9907 & 1{,}396{,}053 & \$1{,}291{,}228 \\ - \bottomrule - \end{tabular} - }% -\end{table} +The exact alpha sensitivity explains why the paper does not headline a 99\% +interval. At target alpha 0.01, coverage is 0.996720, but average width is +0.988215 and 93.5424\% of upper endpoints equal one. Such endpoints carry +almost no ranking information for a portfolio. The selected 90\% level is the +frozen recipe's reference level and preserves more decision resolution. -Prosper uses its full 10{,}531-loan OOT economic universe. Freddie is evaluated -on 1{,}396{,}053 OOT economic candidates; a sparse all-candidate LP returns the -same robust objective as the large top screens, with worst funded rank 551 and -zero funded loans outside the top-250{,}000 screen. This is an exhaustiveness -audit on the external LP solve, not a new exact funded-set certificate for -Prosper or Freddie. - -\begin{figure}[ht] - \centering - \includegraphics[width=0.94\textwidth]{crpto_fig22_external_replication.pdf} - \caption{External CRPTO replications preserve predeclared global conformal gates and positive - robust LP value on two materially different credit products.} - \label{fig:external-replication} -\end{figure} +\subsection{Full OOT Funded-Set Audit} -\FloatBarrier -The external layer adds a secondary economic pattern. Across Prosper and three -Freddie default-window applications, the signed robust premium ranges from -$+1.00\%$ to $+9.46\%$ and is ordered by panel default rate. Four applications -cannot establish a scaling law, so A34 and its companion figure remain in the -online supplement as mechanism-consistent diagnostics rather than a body claim. -The matched Lending Club comparison is instead A40 above, whose point-PD -baseline holds the candidate universe and operating constraints fixed. The -external recipe therefore transfers as an economic audit protocol, while the -exact funded-set certificate remains the Lending Club object. - -% ===================================================================== -\FloatBarrier -\section{Robustness and Comparators}\label{sec:robustness} - -The first robustness concern is temporal leakage. CRPTO addresses it through -out-of-time splits, temporal backtesting, and a strict distinction between -calibration, test, and paper-facing outputs. The paper does not claim that a Lending -Club static panel substitutes for future originations after the retail platform -closed. It claims that within the available historical panel, the promoted policy -survives the documented temporal validation design. The strict temporal holdout -in supplement A9 reports both temporal slices passing the exact check, strengthening -that validation claim without reopening the champion. - -The second concern is whether conformal uncertainty is doing decision work or only -adding conservative decoration. The answer is visible in the portfolio frontier. -Policies are evaluated by return, exact alpha pass/fail, weighted miscoverage, and -$\Gamma_{\mathrm{CP}}$; the promoted point is the body/default balanced policy on -that finite-grid frontier, while the strict $\le 0.345$ threshold policy is reported -separately. This differs from a workflow where conformal intervals are plotted -after the optimizer has already chosen a point-PD allocation. - -The supplement also carries the reviewer-facing robustness checks: nested holdout, -strict temporal holdout, exact evaluation of conformal finalists, uncertainty-set -baselines, and the selected-policy finite-grid frontier. Diagnostics that depend on -the selected funded-loan composition are split explicitly: selected-allocation checks -stay diagnostic, while legacy tail-frontier tables remain -diagnostic machinery rather than body selectors. - -The third concern is whether a decision-focused or SPO+ model would be a stronger -baseline. CRPTO treats SPO+ as an important comparator, but not as the same governance -object. Decision-focused training can reduce regret relative to an optimization loss, -while CRPTO prioritizes calibrated uncertainty, traceable risk controls, and exact -funded-set checks. The manuscript therefore does not claim to dominate SPO+ on every -regret metric; it claims a different auditability/economic trade-off. Relative to -two-stage PTO, SPO+/decision-focused learning, P2P profit scoring, P2P robust -portfolio optimization, and cost-aware calibration, the distinguishing object is -not a new ranking score but a budgeted conformal premium, exact funded-set audit, -and finite-grid frontier denominator. - -\subsection{Regret-Auditability Frontier}\label{sec:regret} - -The SPO+ comparator makes the trade-off sharp. In the committed A19/Figure~\ref{fig:regret} -results, SPO+ reduces mean regret from $0.425896$ to $0.216837$, a $49.09\%$ -improvement over the two-stage baseline (Wilcoxon $p=1.39\times10^{-164}$). A later -PyEPO 1.3.7 paired rerun independently reports the same conclusion under a slightly -different protocol ($0.358073$ to $0.184366$, $48.51\%$; Wilcoxon -$p=3.80\times10^{-163}$), so we treat it as a curated closeout note rather than as the -numeric source for A19. The CRPTO robust point has higher decision regret ($0.947429$) -because it is not trained to minimize regret; it is constructed to expose and control -predictive uncertainty before funding. The frontier is therefore not a single -leaderboard. It asks what the method buys besides regret: temporal coverage above -target, an exact funded-set $\alpha=0.01$ pass, and a finite-grid return-bound -frontier. This is the cleanest comparator story in the paper. SPO+ answers how -much regret training can remove; CRPTO answers what a reviewer can verify after -a calibrated PD model is frozen. +The fixed midpoint policy allocates the full \$1M budget across 308 loans. Its +expected point-PD objective is \$168{,}271.56; realized return is +\$179{,}327.59. Weighted default is 0.039375 and weighted miscoverage is +0.036875. \begin{table}[t] - \centering - \caption{Regret-auditability frontier: CRPTO trades synthetic benchmark regret for - verifiable risk controls. Regret is the A19/PyEPO decision-loss scale, not realized - dollar return on the \$1M funded portfolio.} - \label{tab:regret} - \resizebox{\textwidth}{!}{% - \begin{tabular}{lrrrr} - \toprule - Method & Mean regret & Regret $\Delta$ vs. two-stage & Realized funding value & Verifiable risk controls \\ - \midrule - Two-stage baseline & $0.425896$ & $0.0\%$ & not certified & $0/3$ \\ - SPO+ & $0.216837$ & $49.09\%$ lower & not certified & $0/3$ \\ - CRPTO robust & $0.947429$ & $122.46\%$ higher & \$184.8K & $3/3$ \\ - \bottomrule - \end{tabular} - }% +\centering +\small +\caption{Exact full-OOT audit of the selected midpoint policy.} +\label{tab:active} +\begin{tabular}{lr} +\toprule +Quantity & Value \\ +\midrule +Weighted point PD & 0.081949 \\ +Weighted midpoint score & 0.170000 \\ +$\Gamma_{\mathrm{CP}}$ & 0.176102 \\ +$\Gamma_{\mathrm{int}}$ & 0.088051 \\ +$\Gamma_{\mathrm{res}}$ & 0.088051 \\ +Endpoint budget $B_u$ & 0.258051 \\ +Observed accounting bound $B_u+V$ & 0.294926 \\ +Conditional Markov threshold & 0.574279 \\ +\bottomrule +\end{tabular} \end{table} -The regret column must be read with its protocol in mind, and the last two -columns prevent a one-dimensional reading. Mean regret comes from a separate -decision-regret experiment (A19/PyEPO) run on small synthetic optimization -instances (50 items, budget 15, five seeds), scoring each method on a normalized -decision-loss scale rather than on the \$1M funded portfolio; on that scale SPO+ -is the low-regret method by construction. The external price-of-robustness diagnostic in the supplement -and the higher CRPTO regret here are not in tension---they are two different -measurements (a real \$1M funded set versus a synthetic regret benchmark). The -right-hand columns report what the credit decision actually delivers: only CRPTO -produces a budgeted funded set with a certified realized return (\$184.8K on -the \$1M budget) and the three verifiable risk controls. The regret comparison is -therefore about the synthetic benchmark task, not the quality of the funded loans -in the \$1M credit portfolio. +The observed weighted outcome 0.039375 is below the exact accounting right-hand +side 0.294926. Under the additional weighted-validity assumption, the 0.574279 +event threshold has probability bound $\sqrt{0.10}=0.316228$. This loose +statement is not a direct default cap; the operational controls are $\tau$, +$q_i$, and the funded-set diagnostics. -\begin{figure}[t] - \centering - \includegraphics[width=0.58\textwidth]{crpto_fig15_regret_auditability_frontier.pdf} - \caption{CRPTO sits on the auditable-risk-control corner, while SPO+ occupies the - low-regret corner.} - \label{fig:regret} -\end{figure} +A fixed-allocation bootstrap gives a 95\% return interval of +\$162{,}706.17--\$193{,}924.74 from 5{,}000 funded-loan resamples. It does not +resample the model, conformal recipe, selector, or optimizer. -CRPTO is also close in spirit to a recent line that gives distribution-free, -finite-sample guarantees jointly on miscoverage and decision regret, tracing a -miscoverage--regret Pareto frontier for robust predict-then-optimize policies -\citep{zhou2025credo,zhou2026creme}. That work is the general theory of the -frontier in Figure~\ref{fig:regret}: it calibrates a robustness level against a -cost--risk preference for an abstract optimization family. CRPTO is the -complementary, applied object: it does not propose a new frontier estimator, but -instantiates one corner of that frontier on a frozen, production-style -credit-risk model, with a named conformal robustness premium -$\Gamma_{\mathrm{CP}}$, an exact funded-set certificate, finite-grid portfolio -frontier, and model-risk lineage that a credit committee can inspect. - -\FloatBarrier -\subsection{Tail Risk and Distribution Robustness}\label{sec:tail-dist} - -Two reviewer questions deserve a body-level answer: what does the selected policy -give up on the tail, and does its coverage hold once the evaluation is sliced by -grade? The return-bound frontier answers the first question directly: lower -Markov loss thresholds are available at lower return, while the selected policy sits on the -declared frontier. The supplement then checks the funded-grade mix, -selected-allocation tail repricing, dependence sensitivity, and fixed-allocation -bootstrap interval. These rows explain the selected policy's risk profile; they -do not add a hidden CVaR/OCE or bootstrap selector. +\subsection{Matched Comparators} \begin{table}[t] - \centering - \caption{Tail-risk and distribution-robustness checks for the selected decision.} - \label{tab:tail-scope} - {% - \small - \setlength{\tabcolsep}{4pt} - \renewcommand{\arraystretch}{1.18} - \begin{tabular}{@{}>{\raggedright\arraybackslash}p{0.26\textwidth}>{\raggedright\arraybackslash}p{0.36\textwidth}>{\raggedright\arraybackslash}p{0.30\textwidth}@{}} - \toprule - Reviewer question & Body answer & Boundary \\ - \midrule - Is the point only high-return? & The frontier shows safer lower-return choices and higher-return looser-threshold choices. & Finite grid, not a continuous optimum. \\ - What loans does it fund? & The supplement reports funded exposure by grade. & Business mix, not protected-class fairness certification. \\ - What happens in the tail? & The selected allocation is repriced under LGD and tail summaries. & Risk profile only; tail risk is not the selector. \\ - Does dependence change the bound? & Cluster sensitivity recomputes tighter assumptions. & Sensitivity only; Markov remains the body theorem. \\ - Is the return estimate fragile? & A fixed-allocation bootstrap reports contribution intervals. & Does not resample the model, solver, or search. \\ - Does coverage survive slices? & Grade, distribution, and online-style checks stress the frozen intervals. & Diagnostics, not universal conditional or live coverage. \\ - \bottomrule - \end{tabular} - }% +\centering +\small +\caption{Full-OOT matched decision comparison.} +\label{tab:matched} +\resizebox{\textwidth}{!}{% +\begin{tabular}{lrrrrrr} +\toprule +Policy & Funded & Return & Default & Miscoverage & $B_u$ & Threshold \\ +\midrule +Selected 50/50 CRPTO & 308 & \$179{,}327.59 & 0.039375 & 0.036875 & 0.258051 & 0.574279 \\ +Conservative 75\% blend & 312 & \$172{,}939.50 & 0.035875 & 0.035875 & 0.200396 & 0.516624 \\ +Point-PD matched $\tau$ & 225 & \$196{,}369.14 & 0.118400 & 0.041900 & 0.921317 & 1.237545 \\ +\bottomrule +\end{tabular}} \end{table} -On distribution robustness, the worst per-grade 90\% coverage on the frozen -intervals is grade E at $0.9004$, still above the $0.90$ target. Supplement A23 -reports marginal coverage $0.9293$ on its multi-distribution evaluation slice, -while the promoted interval summary in Table~\ref{tab:core} reports $0.9297$; -these are distinct cuts through the same intervals. No grade falls -below target, so the conservative marginal coverage is not hiding a failing segment; the thinnest -grade$\times$vintage cells identify where a group-weighted or multi-distribution -recalibration would require a separately tagged protocol, so they are not -promoted as a present guarantee. - -\subsection{Managerial Implication}\label{sec:managerial} - -For a credit-risk committee, CRPTO turns a model into a decision -conversation. The committee can pick a risk cap $\tau$, inspect how a policy -parameter $\gamma$ changes the funded set, separate the total conformal premium -into internalized and residual components, and compare $V(\alpha)$ with the -stated bound tolerance. On this evaluation the matched choice is concrete: -accept $5.875\%$ less realized return than the point-PD LP in exchange for -$8.305$ percentage points less weighted default/miscoverage and a $43.55$ -percentage point lower Markov loss threshold. The method supports a practical -trade-off rather than promising a free robustness premium. If the -committee wants lower regret, the SPO+ corner is visible; if it wants stronger -validity language, the validity ladder states the new -calibration protocol that would be required. That separation is the managerial -value of the paper. - -The online supplement contains the robustness package: nested holdout, segment-period -sensitivity, decision-aware conformal selector checks, synthetic-shift diagnostics, -conformal finalist exact evaluation, tail-risk OCE/CVaR diagnostics, satisficing -margins, dependency clusters, bootstrap funded-set metrics, budget/LGD/cap sensitivity, -and finite-grid frontier summaries by policy family, plus the A19 -regret-auditability frontier, A20 tail-risk diagnostic audit, A21 cluster-bound -tightening audit, A22 CVaR/OCE tail-constrained re-optimization, and A23--A24 -multi-distribution and online (ACI) conformal-stability diagnostics, plus -A25--A34 external economic replication, exhaustiveness, interval, subperiod, -and sensitivity audits on Prosper and Freddie/Mendeley, and A35--A39 selected-policy -frontier, composition, tail-risk, concentration, and bootstrap audits, plus A40 -matched point-PD comparison. - -% ===================================================================== -\section{Discussion}\label{sec:discussion} - -CRPTO is useful precisely because it stays close to the operational reality of -credit-risk analytics. Many institutions already have calibrated PD models and -portfolio policies. Replacing them with an end-to-end decision-focused learner may be -scientifically attractive but organizationally difficult. CRPTO offers a middle path: -keep the predictive model auditable, quantify uncertainty with a finite-sample -conformal layer, and make the optimizer pay attention to the upper end of plausible -default risk. - -The policy-aware decomposition is more than a notation change. A linear blend -can recover its residual endpoint premium from $\gamma$ alone, but capped and -tail-focused policies cannot. Computing $\Gamma_{\mathrm{res}}$ from funded rows -makes every frontier point comparable on the same exact endpoint scale and -prevents a nonlinear tail policy from appearing safer because of a shortcut. - -The external replications also change how to read the price of robustness. Across -the frozen external applications, the premium is ordered by panel default risk -rather than by discrimination. That pattern reframes robustness as a -panel-specific premium to be measured, not a fixed toll to be assumed. It also -tempers the contribution: the matched Lending Club audit observes a $5.875\%$ -return cost, while external applications report different premiums under their -frozen contracts. The transferable claim is reproducible measurement of a -return--risk trade-off, not a universal free lunch. - -The limits are equally important. CRPTO does not prove that any one public dataset -is a universal proxy for modern credit origination. The Prosper and -Freddie/Mendeley replications reduce the single-dataset concern, but they are still -static historical panels rather than live deployment evidence. The external panels also make the conditional-coverage -caveat concrete: on Freddie the all-group minimum coverage is driven by tiny sparse -Mondrian cells, and the high-default red segment misses the strict $\alpha = 0.01$ -gate at $0.9850$; both are reported as sensitivity evidence rather than promoted -as conditional guarantees. CRPTO does not claim legal fairness certification, -because the public data lack direct protected attributes. It does not assert that -robust conformal policies dominate all decision-focused learners on regret. - -The manuscript is deliberately written as one IJDS paper rather than a bundle of -method variants. Adjacent methods enter only when they make the submitted -certificate easier to evaluate: OCE/CVaR as a tail-risk audit, satisficing as -margin evidence, SPO+ as the low-regret corner of the regret-auditability -frontier, and dependence as a formal caveat. Table~\ref{tab:upgrade-map} states -that single-submission boundary directly. +Relative to point PD, selected CRPTO gives up \$17{,}041.55, or 8.678\% of +realized return. Weighted default falls by 7.9025 percentage points, +miscoverage by 0.5025 percentage points, and the endpoint-plus-Markov threshold +by 66.3266 percentage points. The default contrast is much larger than the +miscoverage contrast: the guardrail changes the funded loans but does not create +a dramatic selected-set coverage gain. + +The 75\% blend lowers default by 0.35 percentage points and the threshold by +0.057655 relative to the midpoint, but costs another \$6{,}388.08. The +calibration selector chooses the midpoint because it has the highest expected +objective under the declared screen, not because it dominates every risk +metric. + +\subsection{Temporal Heterogeneity} \begin{table}[t] - \centering - \caption{Single-submission boundary map for the current CRPTO paper.} - \label{tab:upgrade-map} - {% - \small - \setlength{\tabcolsep}{4pt} - \renewcommand{\arraystretch}{1.12} - \begin{tabular}{@{}>{\raggedright\arraybackslash}p{0.22\textwidth}>{\raggedright\arraybackslash}p{0.36\textwidth}>{\raggedright\arraybackslash}p{0.34\textwidth}@{}} - \toprule - Adjacent path & What this paper uses & Boundary for the submitted claim \\ - \midrule - Tail-aware selection & A20--A22 and A37 show the selected decision's tail profile and available return-tail trade-off. & The promoted selector remains the declared return-bound finite frontier. \\ - Prospective selection & Nested holdout and the finite declared grid reduce post-selection ambiguity. & The paper does not claim a fully prospective selection/evaluation trial. \\ - Multi-distribution or online validity & A23--A24 diagnose grade, distribution, and vintage stress on frozen intervals. & The conformal layer is not recalibrated for multi-source or live sequential validity. \\ - Decision-focused conformal learning & A19 shows SPO+ as the low-regret comparator and CRPTO as the auditable corner. & The PD model remains frozen; no end-to-end learner is promoted. \\ - \bottomrule - \end{tabular} - }% +\centering +\small +\caption{Fixed-policy temporal stress evaluation; each row uses a fresh \$1M budget.} +\label{tab:temporal} +\begin{tabular}{lrrrr} +\toprule +Period & CRPTO return & Point-PD return & CRPTO default & Point-PD default \\ +\midrule +2018H1 & \$92{,}530.73 & \$118{,}101.99 & 0.106703 & 0.190825 \\ +2018H2 & \$156{,}185.51 & \$95{,}603.58 & 0.026725 & 0.236728 \\ +2019H1 & \$123{,}590.69 & \$144{,}281.46 & 0.077325 & 0.170275 \\ +2019H2 & \$110{,}251.95 & \$256{,}966.20 & 0.103250 & 0.023775 \\ +2020+ & \$99{,}689.54 & \$218{,}629.14 & 0.083775 & 0.016900 \\ +\bottomrule +\end{tabular} \end{table} -Optimized OCE/CVaR objectives, online conformal methods, multi-distribution -conformal validity, utility-directed or decision-theoretic conformal variants -\citep{cortesgomez2025utility,lekeufack2023cdt}, causal variants, broader -asset-class panels, and prospective multi-period origination studies are all -valuable comparators. In this submission, their role is to make the frozen CRPTO -contribution easier to locate: an auditable post-hoc predict-then-optimize -certificate, not a universal decision-learning framework and not a collection of -additional promoted methods. +CRPTO is economically and statistically attractive in 2018H2, when the point- +PD portfolio concentrates in realized defaults. In 2019H2 and 2020+, point PD +earns more and defaults less. The result is therefore a full-period return-risk +trade-off with temporal heterogeneity, not a universal robustness premium. + +\subsection{Funded-Set Composition} + +Grades C and D represent 31.36\% and 58.68\% of funded exposure. Grade F is +only 1.30\% but has realized default 0.269231. The supplement reports every +letter grade and reconciles to the full allocation. Letter grade is recovered +from the original subgrade field; the score-quantile conformal group is stored +separately. This is a business-risk composition audit, not legal fair-lending +certification. + +\section{Robustness, Comparators, and Implications}\label{sec:implications} + +The online supplement keeps diagnostics that help challenge the one active +method without turning the paper into several methods: exact alpha saturation, +all selector cells, temporal results, grade composition, bootstrap, and matched +comparators. Earlier OCE/CVaR, dependence, online-style, SPO+, Prosper, and +Freddie/Mendeley analyses are supporting diagnostics or external context. They +do not select the active policy. + +For a credit committee, the useful control is not the Markov threshold alone. +The committee chooses a conformal level, a transparent blend $\gamma$, and a +portfolio tolerance $\tau$, then reads economic cost, endpoint exposure, +realized default, and temporal behavior together. Here the midpoint has a clear +interpretation: half of the conformal premium enters the enforceable risk score +and half remains visible as residual endpoint exposure. + +The matched baseline turns that interpretation into a decision. A committee +that accepts the full-period evidence pays 8.678\% of realized return for a +7.9025 percentage-point default reduction. A more conservative committee can +use the 75\% blend and pay another roughly \$6.4K. A committee emphasizing +2019H2 or 2020 should reject the static rule or require a new temporal +recalibration protocol. CRPTO exposes that choice rather than hiding it behind +a single score. + +\section{Reproducibility and Limitations}\label{sec:limitations} + +The research bundle versions code, manuscript sources, configurations, tables, +figures, and governance files. Heavy data and model artifacts are stored +separately and checked by hashes. The evidence builder regenerates A35--A40 +from frozen exact-alpha and policy outputs. Claim-sync tests verify that the +body, supplement, and this official TeX surface share policy settings, selector +counts, and numeric anchors. + +Several limitations bound the contribution. Lending Club retail origination +ended in 2020, so the historical panel cannot establish live performance. The +final selector is outcome-free with respect to OOT ranking, but earlier project +development inspected the same OOT corpus. The evaluation is retrospective, +not prospective. The conformal intervals are broad because the outcome is +binary; more than half of 90\% OOT upper endpoints equal one. Mondrian coverage +does not imply validity under optimizer-selected funded weights, so the Markov +statement requires an explicit assumption. Temporal slices show that point PD +can dominate both return and default. Public data also do not support a legal +fair-lending certification or a causal interpretation. + +A focused next step is a genuinely prospective or formally selection-valid +protocol frozen before a new evaluation period +\citep{farinhas2024nonexchangeable_crc,hegazy2025valid_selection_conformal_sets}. +That extension is not hidden as an acceptance criterion for this retrospective +decision audit. + +For double-anonymous review, author-identifying repository URLs are omitted. +Code/data access and artifact instructions can be disclosed under the journal's +policy without embedding credentials or identity in the anonymous PDF. -% ===================================================================== \section{Conclusion}\label{sec:conclusion} -Can finite predictive uncertainty change a funding decision in a way a reviewer -can audit? CRPTO's answer is yes, provided the statistical boundary is stated -and the decision record is frozen. On the Lending Club out-of-time panel the -selected policy earns \$184{,}832.48 on a \$1M budget while passing -the exact empirical $\alpha=0.01$ funded-set audit, and it lies on a declared -finite-grid return-bound frontier with 50{,}010 deduplicated semantic policies -and 27{,}508 all-alpha above-floor policies rather than at a single lucky point. -A matched point-PD LP shows the price explicitly: $5.875\%$ less realized return -for $8.305$ percentage points less weighted default/miscoverage and a $43.55$ -percentage point lower exact Markov loss threshold. The policy-aware residual -premium makes the certificate valid across the linear, capped, and tail-focused -frontier, while Prosper and Freddie/Mendeley show that the recipe can be audited -on other credit products. The contribution is one auditable post-hoc decision -certificate, not a new end-to-end learner, live-deployment study, or portfolio -of additional methods; every reported number is regenerable from frozen evidence. - -% Reproducibility/companion disclosure is kept for the cover letter / non-anonymous -% version, not the double-anonymous body. +CRPTO shows how a frozen credit model can become an auditable portfolio +decision without a maze of policy variants. An exact 90\% conformal replay +produces $u_i$; the midpoint $q_i=(p_i+u_i)/2$ constrains risk; and a nine-cell +calibration selector fixes $\tau=0.17$ without reading OOT outcomes. On the +full panel, the policy earns \$179{,}327.59, with weighted default 0.039375, +miscoverage 0.036875, $\Gamma_{\mathrm{CP}}=0.176102$, +$\Gamma_{\mathrm{res}}=0.088051$, endpoint 0.258051, observed accounting bound +0.294926, and conditional Markov threshold 0.574279. Against matched point PD, +it pays 8.678\% of realized return for a 7.9025 percentage-point default +reduction. Temporal reversals keep the claim narrow: CRPTO is an inspectable +retrospective return-risk guardrail, not a universal winner, prospective +deployment guarantee, or new credit-scoring leaderboard. \bibliographystyle{informs2014} \bibliography{../../book/references} diff --git a/paper/submission/IJDS_SUBMISSION_ROADMAP_2026-08-10.md b/paper/submission/IJDS_SUBMISSION_ROADMAP_2026-08-10.md index 82049be..060d592 100644 --- a/paper/submission/IJDS_SUBMISSION_ROADMAP_2026-08-10.md +++ b/paper/submission/IJDS_SUBMISSION_ROADMAP_2026-08-10.md @@ -1,70 +1,65 @@ # IJDS Submission Roadmap - Target 2026-08-10 -This roadmap keeps the CRPTO submission work aligned with INFORMS Journal on -Data Science rather than with a generic machine-learning or operations-research -paper. The target date is August 10, 2026. IJDS regular submissions are rolling; -the date is an internal quality gate, not an external deadline. +The date is an internal quality gate; IJDS submissions are rolling. -Official sources to recheck before freezing: +Official sources to recheck in the submission week: -- Submission guidelines: -- Data and Code Disclosure Policy: -- Reviewer guidelines: -- LaTeX style files: +- +- +- +- ## Submission Thesis -CRPTO should be read as data science for decisions: - -| IJDS component | CRPTO surface | +| IJDS dimension | CRPTO answer | |---|---| -| Data | Static Lending Club OOT panel, plus Prosper and Freddie/Mendeley frozen external stress tests. | -| Models/algorithms | Calibrated PD, Mondrian conformal intervals, robust LP, exact funded-set audit. | -| Decision relevance | Funding a credit portfolio under budget, risk tolerance, and uncertainty. | -| Implications | Model-risk governance, reproducible auditability, robust-price interpretation, and limits of external transfer. | +| Data | Temporal Lending Club panel with a calibration development block and OOT evaluation. | +| Method | Exact 90% conformal replay and one midpoint portfolio guardrail. | +| Decision | Allocate `$1M` under capital, concentration, and effective-PD constraints. | +| Evidence | Nine-cell calibration selector, matched point-PD decision, temporal reversals, and funded-set audit. | +| Implication | An inspectable price of uncertainty, including cases where the static guardrail should be rejected. | + +## Completed Scientific Refactor + +- Retired approximate cross-alpha headline values. +- Replayed conformal quantiles exactly at every sensitivity alpha. +- Selected the conventional 90% reference level; documented endpoint + saturation at tighter levels. +- Replaced nonlinear/tail policy families with `q=(p+u)/2`. +- Separated point-PD economics from conformal feasibility. +- Reduced policy selection to a round-number `3x3` calibration grid. +- Added schema guards against outcome-derived selector columns. +- Added matched point-PD and 75% blend comparators. +- Promoted temporal reversals and limitations to the body. +- Rebuilt A35--A40 and active claim-sync tests. -## Work Plan +## Remaining Submission Work -| Window | Goal | Required output | +| Window | Deliverable | Exit condition | |---|---|---| -| Jun 9-16 | Editorial contract | Body and supplement explicitly state data-model-decision-implication logic. | -| Jun 17-24 | Claim hardening | Every headline number maps to an artifact and a non-overclaim boundary. | -| Jun 25-Jul 2 | Related-work pressure test | Closest-work table reads as a novelty boundary, not a literature survey. | -| Jul 3-10 | Method and theorem audit | Exact funded-set certificate, weighted-validity assumption, and post-selection boundary are unambiguous. | -| Jul 11-17 | External replication polish | Prosper/Freddie remain evidence of recipe transfer, not new exact certificates. | -| Jul 18-24 | Figures and tables | Captions state takeaway; tables fit IJDS; figures remain readable in grayscale. | -| Jul 25-31 | Reproducibility package | Data/code disclosure plan, commands, hashes, DVC pointers, and raw-data instructions are ready. | -| Aug 1-5 | Official template | Body compiles in `informs4` with `dblanonrev`; current local build is 26 pages total, Section 9 and References start on page 22, so the body remains within the 25-page limit when references are excluded. Final ScholarOne proof still pending. | -| Aug 6-8 | Double-anonymous QA | Metadata, URLs, acknowledgements, local paths, and author signals are removed from reviewer-facing PDFs. | -| Aug 9-10 | Submission freeze | `just lint`, `just smoke`, `just validate-champion`, `just paper-submission-pdf`, and visual QA pass. | +| Jul 9--12 | Code and claim gates | Ruff, mypy, ty, focused tests, smoke, manifest, and drift gate green. | +| Jul 12--18 | PDF editorial QA | Official body and supplement render; no undefined citations; body within 25-page rule; visual QA complete. | +| Jul 18--24 | Reproducibility archive | Sanitized commands, source notes, run tags, hashes, and A35--A40 bundle staged. | +| Jul 25--31 | Anonymous package | Body, supplement, title page, cover letter, and disclosure form separated correctly. | +| Aug 1--8 | Cold review | Read only the generated PDFs; fix clarity, table, and citation defects. | +| Aug 9--10 | ScholarOne freeze | Upload, inspect ScholarOne proof, and submit only after go/no-go checklist. | -## The 15 Improvement Tracks +## Acceptance Risks -| # | Track | Done definition | -|---:|---|---| -| 1 | Central methodological claim | Abstract, introduction, and conclusion describe CRPTO as an auditable conformal-robust decision certificate. | -| 2 | IJDS fit | The body visibly contains data, method, decision, and implication components. | -| 3 | Exact-certificate language | "Exact" is defined as funded-set accounting on frozen OOT outputs, with statistical assumptions stated separately. | -| 4 | Finite-grid frontier | A35 is explained as the final evaluated pool93 finite-grid frontier, not all candidate policies or a continuous region. | -| 5 | External datasets | Prosper/Freddie are frozen external economic replications, not new Lending Club champions. | -| 6 | Related work | The closest-work boundary distinguishes CRPTO from P2P OR, conformal credit scoring, conformal RO, DFL, and financial portfolios. | -| 7 | Figures | Main figures have single-sentence takeaways, readable axes, grayscale-safe contrast, and no unnecessary decorative elements. | -| 8 | Tables | Body tables are compact reviewer evidence; voluminous diagnostics stay in the supplement. | -| 9 | Supplement | A3--A40 are organized as a defense layer with scope caveats. | -| 10 | Reproducibility | Accepted-paper package has code, DVC pointers, manifest, raw-data instructions, and guardrail commands. | -| 11 | Double anonymity | Reviewer-facing body and supplement contain no author URLs, names, local paths, or private remotes. | -| 12 | Official IJDS template | `CRPTO_ijds_submission.tex` is manually synchronized from the pool93 A35--A40 QMD source, keeps the official-template compaction, compiles against the official files, and is rechecked after body edits. | -| 13 | Data/code form | Cover letter and disclosure text acknowledge IJDS accepted-paper reproducibility requirements. | -| 14 | Acceptance-risk audit | A short list of likely reviewer objections has body or supplement responses. | -| 15 | Freeze discipline | Protected champion/search stages are never rerun as routine paper reproduction. | +| Risk | Mitigation in current draft | +|---|---| +| Applied pipeline rather than method | One explicit objective/constraint contract and exact selector protocol. | +| Broad binary conformal intervals | A35 reports width and endpoint saturation; no 99% headline. | +| Adaptive funded-set validity | Deterministic accounting is separated from assumption-conditional Markov language. | +| Historical OOT reuse | "Retrospective lockbox replay" stated in abstract, design, limitations, supplement, and cover letter. | +| Baseline cherry-picking | Same candidates, budget, concentration, LGD, solver, and `tau`; temporal failures are shown. | +| Too many methods | A1--A34 demoted to diagnostics; A35--A40 support one midpoint policy. | +| Reproducibility mistaken for novelty | Decision method and managerial trade-off lead; tooling supports auditability. | +| Page and template risk | Official `informs4` build and visual QA are blocking gates. | -## Current Acceptance Risks +## Freeze Rule -| Risk | Why it matters | Mitigation | -|---|---|---| -| Perceived as applied pipeline | IJDS needs methodological data science, not just a case study. | Keep the decision-certificate framing central. | -| Overreading exact validity | Reviewers may object if "exact" sounds like universal conformal validity. | Define exact as funded-set accounting and state weighted validity separately. | -| External claims too strong | Prosper/Freddie are not new certificates. | Label them as economic replication and exhaustiveness audits. | -| Regret comparator confusion | SPO+ wins regret by design. | Present regret-auditability as a frontier with different governance outputs. | -| Template/page risk | Local HTML-print PDFs are not official. | Keep the `informs4` handoff build current and recheck the ScholarOne proof before submission. | -| Reproducibility policy | IJDS requires disclosure form at submission and archive workflow at acceptance. | Maintain `REPRODUCIBILITY_PACKAGE.md` and cover-letter language. | +After the scientific and PDF gates pass, do not reopen the policy for marginal +OOT gains. Reopen only for a concrete reviewer request, a simpler calibration- +only rule that matches the active result, or a formally stronger prospective or +selection-valid protocol. diff --git a/paper/submission/README.md b/paper/submission/README.md index 701874d..6a084ad 100644 --- a/paper/submission/README.md +++ b/paper/submission/README.md @@ -1,240 +1,128 @@ # IJDS Submission Package -This directory is the handoff checklist for the IJDS submission surfaces. The -source of truth remains: - -- `paper/CRPTO_ijds.qmd` for the anonymous manuscript body. -- `paper/supplement_ijds.qmd` for the anonymous online supplement. -- `paper/submission/COVER_LETTER_AND_DISCLOSURE.md` for editor-facing cover - letter language and data/code disclosure timing. -- `paper/submission/IJDS_SUBMISSION_ROADMAP_2026-08-10.md` for the internal - two-month readiness plan. -- `paper/submission/CLAIM_AUDIT_MATRIX.md` for the claim/evidence/risk map. -- `paper/submission/REPRODUCIBILITY_PACKAGE.md` for the IJDS data/code package - plan. -- `paper/submission/TITLE_PAGE_DRAFT.md` for the separate non-anonymous title - page. -- `paper/submission/DATA_CODE_DISCLOSURE_FORM_DRAFT.md` for official-form - answer drafting. -- `paper/submission/SCHOLARONE_FINAL_CHECKLIST.md` for the final upload/proof - checklist. - -## Render Commands +This directory contains the official-template handoff and editor-facing +submission materials. The synchronized scientific sources are: + +- `paper/CRPTO_ijds.qmd`: anonymous body. +- `paper/supplement_ijds.qmd`: anonymous online supplement. +- `paper/submission/CRPTO_ijds_submission.tex`: manually compacted + `informs4` handoff. +- `paper/submission/CLAIM_AUDIT_MATRIX.md`: active claim/evidence map. +- `paper/submission/REPRODUCIBILITY_PACKAGE.md`: data/code package plan. + +The active manuscript has one policy: exact 90% conformal replay, +`q=(p+u)/2`, `tau=0.17`, and a nine-cell calibration selector. A35--A40 are the +active evidence bundle. Keep body, supplement, TeX, and governance numerically +aligned with `tests/test_ijds_active_claim_sync.py`. + +## Preview ```powershell +just ijds-evidence just paper-submission just paper-submission-pdf ``` -The HTML render is the writing preview. The PDF render is a local HTML-print -verification draft for pagination and visual inspection. The final submission -PDF should be produced with the official INFORMS IJDS LaTeX template and -double-anonymous review option (`dblanonrev`), not with a hand-written local -style. - -Editorial submission notes, venue reminders and page-budget comments belong in -this README or the cover-letter checklist, not in the anonymous manuscript body. - -`COVER_LETTER_AND_DISCLOSURE.md` is intentionally separated from the anonymous -body and supplement. Use it for editor-facing disclosure fields, then keep the -reviewer packet free of public repository URLs, author identity, local paths and -private remote details. +HTML and browser-print PDFs are writing and visual-QA previews. The upload PDF +must come from `CRPTO_ijds_submission.tex` with the official INFORMS class and +the `dblanonrev` option. -## Official Template Sources +## Official Sources -- INFORMS author portal: +- INFORMS style files: - IJDS submission guidelines: -- IJDS data/code disclosure policy: +- IJDS data/code policy: - IJDS reviewer guidelines: -- Overleaf template page: - -Do not vendor private template downloads, reviewer forms, or authenticated -publisher material into this repository. Local copies of the official template -files can live in this directory for compilation, but they are gitignored on -purpose. - -## Official LaTeX Submission Build - -`CRPTO_ijds_submission.tex` is the official-template handoff draft in the -INFORMS class (`\documentclass[ijds,dblanonrev]{informs4}`). The narrative -source remains `paper/CRPTO_ijds.qmd`, but the official `.tex` is now a -manually compacted IJDS-template surface. After freeze, do **not** regenerate it -mechanically from QMD; port substantive claim changes deliberately, then rebuild -and recheck the official-template PDF. The synchronized submission surface should -carry the central IJDS body: title, abstract, -keywords, core sections, the journal pipeline Figure 1, the bound-claim stack, -the A35 policy-aware finite-grid frontier, the A36--A39 selected-allocation -audits and A40 matched point-PD baseline in the supplement, the -regret-auditability comparison, plus the core, exact-certificate, -funded-set audit and regret tables. The -`informs2014.bst` + `../../book/references.bib` bibliography wiring is already -present. Journal figures use PDF/vector exports from `reports/crpto/figures/` -where possible; Figure 1 intentionally uses the PNG export because the vector -PDF crop box cuts the right edge under `informs4`. - -> **`informs4` is not on CTAN/TeX Live.** The class and bibliography style are -> distributed through the INFORMS author portal or the IJDS Overleaf template. -> Local copies are allowed for compilation and are gitignored. Do not commit -> `informs4.cls`, `informs2014.bst`, template PDFs, `.sty` files, or generated -> LaTeX build artifacts. - -Current local build state (verified 2026-07-09): TinyTeX/TeX Live 2026, -`pdflatex`, `bibtex`, and the `listingsutf8` TeX package compile -`CRPTO_ijds_submission.tex` to a 28-page official-template PDF. References -start on page 24, so the body remains inside the IJDS 25-page -initial-submission budget when references are excluded. The -only LaTeX log warnings left are a small `\maketitle` overfull from the -`informs4` anonymous title block and font-size / underfull paragraph warnings, -visually acceptable unless the final ScholarOne proof shows a layout issue. - -`latexmk` remains the preferred command because it automates the required -LaTeX/BibTeX convergence loop. On 2026-07-07, the local Codex PowerShell -environment was missing `WINDIR`, which made TinyTeX wrapper scripts fail with -`runscript.tlu:712: attempt to concatenate a nil value`. Set `WINDIR` from -`SystemRoot` before calling TinyTeX wrappers in that environment. After -`tlmgr update --self --all`, the LaTeX format also had to be refreshed with -`fmtutil-sys --byfmt pdflatex` to resolve an `expl3` format mismatch. - -To produce the official submission PDF: - -1. Download or refresh `informs4.cls` and `informs2014.bst` from the INFORMS - author portal (or Overleaf) and drop them next to - `CRPTO_ijds_submission.tex`. These are gitignored on purpose - (`paper/submission/.gitignore`); do not commit them. -2. Build with `latexmk`. In Codex/PowerShell sessions where `WINDIR` is absent, - initialize it first: - - ```powershell - if (-not $env:WINDIR) { $env:WINDIR = $env:SystemRoot } - latexmk -pdf -gg -interaction=nonstopmode CRPTO_ijds_submission.tex - ``` - - If LaTeX reports mismatched support files after a TeX Live update, rebuild - the local TinyTeX format once: - - ```powershell - if (-not $env:WINDIR) { $env:WINDIR = $env:SystemRoot } - fmtutil-sys --byfmt pdflatex - ``` - - If PowerShell/TinyTeX still fails, use the proven fallback: - - ```powershell - pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex - bibtex CRPTO_ijds_submission - pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex - pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex - ``` - - The three `pdflatex` passes are intentional. The first pass writes the - `.aux` file that BibTeX needs; `bibtex` then writes the `.bbl`; the second - `pdflatex` pass reads the bibliography and updates citations, cross - references, labels, and page anchors; the final pass stabilizes any values - that shifted after the bibliography and floats were inserted. - -3. The `dblanonrev` option keeps the body anonymous; verify against the anonymity - checklist below before uploading. - -## Anonymity Checklist - -- Manuscript metadata uses `author: "Anonymous"`. -- Supplement metadata uses `author: "Anonymous"`. -- Public GitHub, DVC, MLflow, DagsHub and personal URLs are described as a - companion package but not exposed in the double-anonymous body. -- Cover-letter/data-code wording lives in `COVER_LETTER_AND_DISCLOSURE.md`. -- Title-page, acknowledgements and repository disclosure are kept for the cover - letter or for post-acceptance policy, not the anonymous manuscript. -- Local paths, private workspace paths and usernames do not appear in the submitted - sources. - -## SPO+ Numbering Rule - -Use the committed A19/Figure 15 artifact for body claims: - -- Two-stage regret: `0.425896` -- SPO+ regret: `0.216837` -- Relative reduction: `49.09%` -- Wilcoxon: `p = 1.39e-164` - -The PyEPO 1.3.7 closeout remains a curated appendix note: - -- Two-stage regret: `0.358073` -- SPO+ regret: `0.184366` -- Relative reduction: `48.51%` -- Wilcoxon: `p = 3.80e-163` - -These protocols are compatible but not interchangeable. - -## Final Assembly Checklist - -- Recheck the official IJDS sources linked above within the final submission - week; policies and forms can change. -- Verify the Data and Code Disclosure Form language against - `REPRODUCIBILITY_PACKAGE.md` and `DATA_CODE_DISCLOSURE_FORM_DRAFT.md`. -- Convert `TITLE_PAGE_DRAFT.md` into the separate title page requested by - ScholarOne. -- Use `SCHOLARONE_FINAL_CHECKLIST.md` while uploading and reviewing the generated - proof. -- Recheck the official-template page budget if the body changes materially. The - current local official-template build is 28 pages total; References start on - page 24, keeping the body within the 25-page limit when references are - excluded. -- Keep A3--A40 in the online supplement unless a reviewer-facing argument needs - one compact table in the body. -- Preserve CRPTO as the coverage/auditability method and SPO+ as the low-regret - comparator. -- Cross-check every headline claim against `CLAIM_AUDIT_MATRIX.md`. -- Keep `CRPTO_ijds_submission.tex` semantically synchronized with the QMD - whenever the body adds or demotes a figure, table, theorem statement or major - result paragraph. Preserve the manual compaction choices that keep the - official-template PDF inside the IJDS page budget. -- Regenerate previews with `just paper-submission-pdf` before release. -- Run the repository gates: `just lint`, `just smoke`, `just validate-champion`. - -## Final Step - Official Compile - -This remains a final-week action before upload, gated on an explicit decision -to freeze the manuscript. `CRPTO_ijds_submission.tex` already carries the full -ported prose, the economic-anchor ladder, and the temporal-split and tail-risk -tables. Local publisher class/style files are available in this directory and -gitignored; recheck the official source before final submission in case INFORMS -updates the template. - -1. **Confirm closure.** The body content, numbers, and figures are final and the - repository gates pass. -2. **Refresh the official template files if needed** (gitignored on purpose; - never commit): - - `informs4.cls` and `informs2014.bst` from the INFORMS author portal - or the - IJDS Overleaf template (v2.00, 29 Apr 2025, the latest checked locally) - . - - Drop both next to `CRPTO_ijds_submission.tex` if local copies are absent or - stale. The fastest fallback is to paste the `.tex` into the Overleaf - template, which already bundles both files. -3. **Compile:** - - ```powershell - latexmk -pdf -gg -interaction=nonstopmode CRPTO_ijds_submission.tex - ``` - - Use the documented `pdflatex -> bibtex -> pdflatex -> pdflatex` fallback if - the local TinyTeX wrapper fails after the `WINDIR` and format-refresh steps. - - ```powershell - if (-not $env:WINDIR) { $env:WINDIR = $env:SystemRoot } - pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex - bibtex CRPTO_ijds_submission - pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex - pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex - ``` - - The repeated `pdflatex` calls are not redundant: pass 1 creates `.aux`, - BibTeX creates `.bbl`, pass 2 imports bibliography/citation data, and pass 3 - converges final references and pagination. - -4. **Recount the official-template page budget** and demote body floats to the - supplement only if the body exceeds 25 pages excluding references. The local - official-template build is currently 28 pages total; References start on - page 24. The Chrome-print body preview is only a verification proxy. -5. **Verify anonymity** against the checklist above, then upload the body PDF and - submit the title page separately. +- IJDS Overleaf template: + +Local copies of `informs4.cls`, `informs2014.bst`, and related publisher files +are used for compilation and remain gitignored. Recheck official sources during +the final submission week. + +## Official Build + +Run from `paper/submission`: + +```powershell +if (-not $env:WINDIR) { $env:WINDIR = $env:SystemRoot } +latexmk -pdf -gg -interaction=nonstopmode CRPTO_ijds_submission.tex +``` + +`latexmk` is preferred because it automates convergence. Some Windows TinyTeX +installations fail in the wrapper even when `pdflatex`, BibTeX, and TeX Live are +healthy. The robust fallback is: + +```text +pdflatex -> bibtex -> pdflatex -> pdflatex +``` + +```powershell +pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex +bibtex CRPTO_ijds_submission +pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex +pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex +``` + +The three `pdflatex` calls are intentional: + +1. The first pass writes `.aux`, including citation keys and unresolved labels. +2. BibTeX reads `.aux` and writes the formatted `.bbl`. +3. The second LaTeX pass imports `.bbl` and resolves citations and references. +4. The final pass stabilizes labels, float positions, and pagination changed by + the bibliography and cross-references. + +After a TeX Live update, an `expl3` format mismatch can be repaired once with: + +```powershell +if (-not $env:WINDIR) { $env:WINDIR = $env:SystemRoot } +fmtutil-sys --byfmt pdflatex +``` + +The repository wrapper runs `latexmk` first and falls back automatically: + +```powershell +just paper-submission-official +``` + +## Build Acceptance + +The official-template PDF is acceptable only when: + +- `.blg` has no bibliography warnings; +- `.log` has no undefined citations or references; +- the body is at most 25 pages under the IJDS counting rule; +- tables do not overflow or become unreadably small; +- the PDF remains double-anonymous; +- visual inspection confirms that figures, equations, and references render. + +Current verified build (2026-07-09): 12 pages total; References begin on page +10, so the body is 9 pages under the template's pagination. The bibliography is +clean, and visual inspection of all 12 rendered pages found no clipping, +overlap, missing glyphs, or unreadable tables. + +Do not keep a page-count statement in this README without rebuilding the +current TeX. The final compile wrapper records the current count and warning +scan. + +## Anonymity + +- QMD metadata uses `author: "Anonymous"`. +- TeX uses `\documentclass[ijds,dblanonrev]{informs4}`. +- Author names, acknowledgements, repository ownership, personal URLs, local + usernames, and private remotes stay out of the reviewer packet. +- Cover-letter and data/code language lives in + `COVER_LETTER_AND_DISCLOSURE.md` and related editor-facing files. +- `TITLE_PAGE_DRAFT.md` is uploaded separately when ScholarOne requests it. + +## Final Gate + +```powershell +just submission-check +uv run dvc status --no-updates +git status --short +``` + +Use `SCHOLARONE_FINAL_CHECKLIST.md` for upload order and proof review. Preserve +the active midpoint narrative; OCE/CVaR, SPO+, external replications, and other +historical diagnostics remain supplement context rather than additional active +methods. diff --git a/paper/submission/REPRODUCIBILITY_PACKAGE.md b/paper/submission/REPRODUCIBILITY_PACKAGE.md index 3b2d1fd..45b7654 100644 --- a/paper/submission/REPRODUCIBILITY_PACKAGE.md +++ b/paper/submission/REPRODUCIBILITY_PACKAGE.md @@ -1,56 +1,83 @@ # IJDS Reproducibility Package Plan -IJDS requires the Data and Code Disclosure Form at submission and expects -accepted computational papers to upload data/code and complete the journal's -reproducibility workflow unless an exemption applies. CRPTO can satisfy this -without exposing secrets, private credentials, local paths, or -author-identifying repository URLs during double-anonymous review. - Official policy: +CRPTO can support accepted-paper reproduction without exposing credentials, +local paths, or author identity during double-anonymous review. + ## Disclosure Timing -| Stage | What to disclose | What to withhold | +| Stage | Disclose | Withhold | |---|---|---| -| Initial double-anonymous submission | Neutral description of the companion package, data sources, DVC-style artifact validation, and code/data availability timing. | Public GitHub/DagsHub URLs, author identity, local paths, secrets, tokens. | -| If editors request verification during review | An anonymized archive or controlled access bundle with source, tables, figures, tests, and non-identifying artifact metadata. | Credentials, raw private data, non-anonymized remotes. | -| Acceptance | Public source repository, reproducibility commands, DVC pointers or downloadable processed artifacts where permitted, raw-data acquisition instructions, manifest hashes, and final rendered outputs. | Secrets and any data redistribution prohibited by source licenses. | +| Initial submission | Neutral package description, source-data availability, and release timing. | Repository ownership, personal URLs, local paths, secrets. | +| Editor-requested verification | Anonymized source, A35--A40, tests, and sanitized artifact metadata. | Credentials, private remotes, non-anonymous provenance. | +| Acceptance | Public source, environment lock, data instructions, artifact pointers/hashes, and final outputs. | Secrets and data prohibited from redistribution. | ## Package Contents -| Component | Files/directories | Purpose | +| Component | Files | Purpose | |---|---|---| -| Source code | `src/`, `scripts/`, `tests/`, `pyproject.toml`, `uv.lock`, `justfile`. | Rebuild tables, figures, journal package, and validation checks. | -| Manuscript | `paper/CRPTO_ijds.qmd`, `paper/supplement_ijds.qmd`, `paper/submission/`. | Reproduce body, supplement, and submission surfaces. | -| Frozen evidence | `EXTRACTION_MANIFEST.json`, `models/*.json`, `reports/crpto/tables/`, `reports/crpto/figures/`. | Tie claims to immutable metrics and rendered evidence. | -| Data pointers | `.dvc/`, `dvc.yaml`, `dvc.lock`, DVC remote notes. | Retrieve or verify processed artifacts outside Git. | -| Raw-data instructions | `RAW_DATA_SOURCE_NOTES.md`. | Let readers reconstruct inputs without committing raw CSVs or credentials. | -| Guardrails | `just smoke`, `just validate-champion`, publication target tests, DVC status. | Confirm reproducibility without rerunning protected search stages. | +| Environment | `pyproject.toml`, `uv.lock`, `justfile`. | Recreate the Windows-first toolchain. | +| Method source | `src/models/conformal_alpha_grid.py`, `src/optimization/`, active experiment scripts. | Replay exact intervals and solve declared policies. | +| Active config | `configs/experiments/champion_reopen_ijds_calibration_selected_simple90_v6.yaml`. | Fix alpha, 3x3 grid, selector, and solver settings. | +| Active evidence | A35--A40 CSV/TeX files and `ijds_policy_governance.json`. | Tie every paper claim to generated evidence. | +| Manuscript | body QMD, supplement QMD, official TeX. | Reproduce reviewer-facing surfaces. | +| Data pointers | `dvc.yaml`, `dvc.lock`, `.dvc/`, raw-data notes. | Retrieve large artifacts where terms permit. | +| Guardrails | active claim sync, publication integrity, manifest regression. | Detect narrative or historical artifact drift. | + +## Active Artifact Contract + +| Evidence | Path | +|---|---| +| Exact alpha grid | `data/processed/experiments/champion_reopen//conformal/exact_alpha_grid.parquet` | +| Calibration selector | `data/processed/experiments/champion_reopen//portfolio/calibration_policy_selection_grid.parquet` | +| OOT evaluation | `data/processed/experiments/champion_reopen//portfolio/calibration_selected_policy_oot_evaluation.csv` | +| Funded rows | `data/processed/experiments/champion_reopen//portfolio/calibration_selected_policy_full_oot_allocations.parquet` | +| Governance | `models/experiments/champion_reopen//portfolio/ijds_policy_governance.json` | +| Paper tables | `reports/crpto/tables/crpto_tableA35...A40_*` | + +Active run: +`champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6`. + +Exact-alpha run: +`champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1`. -## Accepted-Paper Reproduction Commands +The active policy evidence is intentionally separate from the manifest-protected +historical bundle. `tests/test_ijds_active_claim_sync.py` guards the submitted +claim; `tests/test_manifest_regression.py` guards frozen provenance. The +manifest is not rewritten to make a manuscript update look historical. + +## Reproduction Commands + +Paper-facing reproduction from frozen experiment outputs: ```powershell just setup-base -just smoke -just validate-champion -just tables -just figures -just evidence -just journal-package +just ijds-evidence +uv run pytest tests/test_ijds_active_claim_sync.py -q just paper-submission -just paper-submission-pdf +just paper-submission-official +just validate-champion ``` -Official IJDS-template PDF build, after `paper/submission/CRPTO_ijds_submission.tex` -is synchronized: +Full isolated methodology replay: ```powershell -cd paper/submission -if (-not $env:WINDIR) { $env:WINDIR = $env:SystemRoot } -latexmk -pdf -gg -interaction=nonstopmode CRPTO_ijds_submission.tex +just ijds-active-replay +``` + +The full replay recomputes the exact alpha grid, solves the nine calibration +policies, evaluates the frozen selected policy, and rebuilds A35--A40. It writes +only to versioned experiment paths and does not overwrite the frozen PD model, +calibrator, historical intervals, or manifest. + +Official-template compilation is automated by: + +```powershell +just paper-submission-official ``` -PowerShell/TinyTeX fallback proven in the local Codex environment: +Manual Windows fallback: ```powershell cd paper/submission @@ -61,93 +88,21 @@ pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex ``` -The repeated `pdflatex` calls are the standard LaTeX/BibTeX convergence loop: -first `.aux`, then BibTeX `.bbl`, then citation/cross-reference import, then -final pagination/reference stabilization. As of 2026-07-07, the local Codex -PowerShell environment needed `WINDIR` initialized from `SystemRoot` before -TinyTeX wrappers such as `latexmk` and `fmtutil-sys` would run; after -`tlmgr update --self --all`, `fmtutil-sys --byfmt pdflatex` also refreshed the -LaTeX format to match the updated support files. +## Data and Artifact Boundary -Artifact-aware DVC verification, when credentials or public artifact access are -available: +- Lending Club, Prosper, and Freddie/Mendeley raw data are distributed through + their original sources, not copied into Git. +- Large processed parquet and model binaries use DVC or journal-approved + artifact delivery when source terms permit. +- `EXTRACTION_MANIFEST.json` verifies the historical upstream bundle. +- Review-stage copies of DVC/configuration metadata must remove repository + ownership, remote URLs, credentials, and absolute local paths. +- If a remote is unavailable, provide journal-approved processed artifacts plus + hashes, schema/source notes, and the commands above. -```powershell -uv run dvc status --no-updates -uv run dvc status -c -r dagshub -``` - -## Pool93 Body-Claim Artifacts +## Non-Routine Stages -The paper body point (A35 "Body/default balanced point") and its frontier come -from the pool93 champion-reopen experiments, generated *outside* the DVC DAG by -deterministic re-evaluation of a pre-declared finite policy grid over the -frozen Mondrian conformal intervals. The package therefore includes, verbatim -and frozen: - -| Artifact | Path | Role | -|---|---|---| -| A35 frontier | `reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv` (+ `.tex`) | Consolidated return-bound frontier; body point row. | -| A36 grade audit | `reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.csv` (+ `.tex`) | Funded-set grade composition of the body allocation. | -| A37 tail risk | `reports/crpto/tables/crpto_tableA37_pool93_body_tail_risk.csv` (+ `.tex`) | LGD-grid repricing and CVaR/OCE diagnostics. | -| A38 cluster bounds | `reports/crpto/tables/crpto_tableA38_pool93_body_cluster_bound_audit.csv` (+ `.tex`) | Cluster-aware Hoeffding sensitivity. | -| A39 bootstrap | `reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.csv` (+ `.tex`) | 5,000-draw fixed-allocation bootstrap intervals. | -| Terminal claim governance | `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal/portfolio/pool93_ijds_claim_governance.json` | Declared return floor, claim hierarchy, do-not-claim list. | -| Consolidated governance | `models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive/portfolio/pool93_ijds_consolidated_governance.json` | Authoritative body-point metrics and frontier counts. | - -Run tags: terminal `champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal`; -consolidated `champion-reopen-2026-06-19__pool93__ijds-claim-consolidated-definitive`. -Their hashes are pinned in `EXTRACTION_MANIFEST.json` and enforced by -`tests/test_manifest_regression.py`; body-claim consistency with the manuscript -is enforced by `tests/test_pool93_body_claim_sync.py`. These artifacts are not -regenerated by `just tables`/`just figures` (which rebuild only the -rebaseline-chain tables); they ship as frozen evidence. - -Anonymity note: the governance JSONs embed absolute local paths in their -`source_paths`/`runtime_status` blocks. Copies shipped inside any -review-stage or accepted-paper archive must be path-sanitized (or the fields -dropped); the in-repo originals stay hash-frozen. - -## Hash and DVC Boundary - -- `EXTRACTION_MANIFEST.json` is the source of truth for protected artifact - hashes; `just validate-champion` runs the manifest regression tests. -- DVC metadata (`dvc.yaml`, `dvc.lock`, `.dvc/`) records large data/model - dependencies and outputs without placing raw CSVs, processed parquet files, or - model binaries in Git. -- The accepted-paper archive may include processed/model artifacts directly only - if the journal workflow and source-data terms permit it. Otherwise it should - include DVC pointers, source acquisition notes, and the commands above. -- Review-stage archives must sanitize author-local paths and repository remotes - if they are sent before acceptance. - -## Review-Stage DVC Sanitization Checklist - -Before sending any reviewer-facing archive or editor-requested verification -bundle, inspect and sanitize: - -- `.dvc/config`: remove or replace the configured `dagshub` remote name, - endpoint URL, account/repository slug, and any credential-dependent remote - settings. -- `dvc.lock`: keep stage dependency/output hashes, but inspect text fields for - author-local paths or non-anonymous remote references before packaging. -- Governance JSONs under `models/experiments/champion_reopen/`: drop or redact - `source_paths`, `runtime_status`, and any absolute `C:\Users\...` values in - the copy sent for review. Do not edit the in-repo originals because their - hashes are frozen. -- Submission docs: keep source instructions and DVC-style validation language, - but avoid public GitHub, DagsHub, MLflow, personal, affiliation, or local - machine URLs until the journal workflow allows disclosure. - -If the DVC remote is unavailable during acceptance checks, the contingency is to -provide journal-approved processed artifacts or model files, plus -`EXTRACTION_MANIFEST.json`, `dvc.lock`, raw-data acquisition notes, and the -paper-export commands above so reviewers can verify the paper-facing surfaces. - -## Non-Rerunnable Stages - -The following stages are not part of routine reproduction because they would -change the frozen champion or reopen the protected search: +The active reproduction does not run protected upstream stages: ```text crpto.pd.champion @@ -157,15 +112,15 @@ crpto.portfolio.optimization crpto.portfolio.bound_exact_eval ``` -Paper-facing exports are safe because they consume frozen inputs. Any protected -rerun requires a new branch, drift report, and explicit decision to create a new -run tag. +Those stages would retrain, rewrite frozen artifacts, or reopen historical +search. Any such run requires a distinct tag and drift report. -## Data-License Strategy +## Acceptance Checklist -The current plan is to publish code, derived tables/figures, manifest hashes, -and instructions for obtaining the raw public datasets. Processed artifacts and -models should be provided through DVC or a journal-approved supplement only when -source licenses and file sizes permit. If a dataset cannot be redistributed, the -package should include enough source instructions, schema notes, and commands for -a reader to rebuild comparable artifacts from legally obtained data. +1. Build in the locked `uv` environment. +2. Rebuild A35--A40 and run active claim sync. +3. Validate historical manifest hashes. +4. Compile and visually inspect the official PDF and supplement. +5. Sanitize author identity and local paths in the review archive. +6. Publish source-data acquisition instructions and artifact hashes. +7. Record any unavoidable platform-level numerical differences explicitly. diff --git a/paper/submission/SCHOLARONE_FINAL_CHECKLIST.md b/paper/submission/SCHOLARONE_FINAL_CHECKLIST.md index 34fe81b..3c00d4d 100644 --- a/paper/submission/SCHOLARONE_FINAL_CHECKLIST.md +++ b/paper/submission/SCHOLARONE_FINAL_CHECKLIST.md @@ -1,113 +1,79 @@ # ScholarOne Final Checklist -Use this checklist only after the paper content is frozen and the official IJDS -template files have been downloaded outside Git. - -## Files to Prepare - -| File | Source | Reviewer-facing? | Status | -|---|---|:---:|---| -| Anonymous manuscript PDF | `CRPTO_ijds_submission.tex` compiled with `informs4` and `dblanonrev`. | Yes | Local file ready. Official-template build verified 2026-07-07 (26 pages total; `.blg` warnings 0; `.log` has no undefined citations/references; source/metadata anonymity checks clean). Final ScholarOne proof remains a system-side gate. | -| Anonymous online supplement PDF | `paper/supplement_ijds.qmd` rendered and visually checked. | Yes | Local render and representative page QA verified 2026-07-07; source/metadata anonymity checks clean; final ScholarOne proof pending | -| Separate title page | `TITLE_PAGE_DRAFT.md` converted into the ScholarOne/title-page format. | No | Draft ready for separate upload/copy; complete affiliation and ORCID in ScholarOne if applicable. | -| Data and Code Disclosure Form | Official IJDS form using `DATA_CODE_DISCLOSURE_FORM_DRAFT.md`. | Editor/system | Draft language ready; official ScholarOne form entry remains manual. | -| Cover letter | `COVER_LETTER_AND_DISCLOSURE.md`, shortened if ScholarOne text boxes are tight. | Editor | Draft ready; final paste/proof inside ScholarOne remains manual. | -| Optional reproducibility note | `REPRODUCIBILITY_PACKAGE.md` or excerpted text if requested. | Editor/system | Optional | - -## Official Template Build - -1. Download or refresh `informs4.cls` and `informs2014.bst` from INFORMS/Overleaf. -2. Synchronize `CRPTO_ijds_submission.tex` manually from the pool93 A35--A40 - QMD source while preserving the official-template compaction. -3. Place the template files next to `CRPTO_ijds_submission.tex`; local gitignored copies are already present. -4. Build with `latexmk`. In Codex/PowerShell sessions where `WINDIR` is absent, - initialize it first: - - ```powershell - if (-not $env:WINDIR) { $env:WINDIR = $env:SystemRoot } - latexmk -pdf -gg -interaction=nonstopmode CRPTO_ijds_submission.tex - ``` - - If LaTeX reports a support-file mismatch after a TeX Live update, run - `fmtutil-sys --byfmt pdflatex` once with the same `WINDIR` initialization. - If PowerShell/TinyTeX still fails, use the proven fallback: - - ```powershell - if (-not $env:WINDIR) { $env:WINDIR = $env:SystemRoot } - pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex - bibtex CRPTO_ijds_submission - pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex - pdflatex -interaction=nonstopmode -halt-on-error CRPTO_ijds_submission.tex - ``` - - The three `pdflatex` passes are intentional: the first creates `.aux`, - BibTeX creates `.bbl`, the second imports bibliography and cross-reference - data, and the third stabilizes references and pagination. - -5. Confirm body page count is at most 25 pages excluding references and - appendices. The local official-template build verified on 2026-07-07 is 26 - pages total; Section 9 (Conclusion) and References both start on page 22, so - the manuscript remains comfortably inside the IJDS page budget when - references are excluded. Recount after every official rebuild. - -## Final Local Gates +Use only after the scientific content and official PDFs are frozen. + +## Files + +| File | Reviewer-facing | Local status | +|---|:---:|---| +| Anonymous manuscript PDF from `CRPTO_ijds_submission.tex` | Yes | Rebuild and recheck after every body edit. | +| Anonymous supplement PDF | Yes | Render and visually inspect. | +| Separate title page | No | Complete from `TITLE_PAGE_DRAFT.md`. | +| Data and Code Disclosure Form | Editor/system | Finalize from the draft. | +| Cover letter | Editor | Finalize from `COVER_LETTER_AND_DISCLOSURE.md`. | +| Reproducibility note/archive | Editor/system | Sanitize identity, paths, and remotes. | + +## Official Build ```powershell +just paper-submission-official +``` + +The wrapper tries `latexmk` and falls back to the verified +`pdflatex -> bibtex -> pdflatex -> pdflatex` loop. Accept only when: + +- `.blg` has zero warnings; +- `.log` has no undefined citation/reference warnings; +- body page count satisfies the IJDS 25-page rule; +- figures and tables fit; +- PDF metadata and visible content remain anonymous. + +Current local build (2026-07-09): 12 pages total, with References beginning on +page 10; citation/reference scans are clean. Recount after every substantive +TeX edit. + +## Local Gates + +```powershell +just ijds-evidence +uv run pytest tests/test_ijds_active_claim_sync.py -q +just publication-integrity just lint +just type-check +just type-advisory-full just smoke just validate-champion -uv run pytest tests/test_publication_targets.py -q +just paper-submission +just paper-submission-official uv run dvc status --no-updates -just paper-submission-pdf ``` -Last local closeout on 2026-07-07: `just lint`, `just smoke`, -`just validate-champion`, `uv run pytest tests/test_publication_targets.py -q`, -`uv run pytest tests/test_pool93_body_claim_sync.py -q`, -`just paper-submission-pdf`, and `latexmk -pdf -gg -interaction=nonstopmode -CRPTO_ijds_submission.tex` passed. Representative PDF pages were rendered -locally for body/supplement/submission visual QA. - -`uv run dvc status --no-updates` is intentionally treated as a pipeline-state -report, not a submission blocker. On 2026-07-07 it reported modified deps/outs -for protected and paper/book stages from earlier code/book work, while -`just validate-champion` remained green. Do not resolve that status by -rerunning protected stages during ScholarOne closeout; open a separate pipeline -debt task after submission if needed. - -## QMD-vs-TeX Freeze Rule - -`paper/CRPTO_ijds.qmd` remains the long-form narrative source. The official -`CRPTO_ijds_submission.tex` is a manually compacted INFORMS-template handoff -surface. After freeze, port substantive claim edits from QMD to TeX deliberately -and recompile; do not regenerate the `.tex` mechanically unless you are prepared -to redo the page-budget and visual QA. +`dvc status` is a report, not permission to rerun protected stages. Do not +repair paper-stage drift by overwriting the frozen upstream chain. ## Anonymous PDF QA -- Body PDF has no author names, affiliation, acknowledgements, public repo URLs, - DagsHub/MLflow URLs, local paths, or hidden PDF metadata that identifies the - author. -- Supplement PDF has no author names, affiliation, acknowledgements, public repo - URLs, DagsHub/MLflow URLs, local paths, or hidden PDF metadata. -- References render correctly with `informs2014.bst`. -- Figures are readable in grayscale or black-and-white printing. -- Tables do not overflow page margins. -- Captions state the takeaway and do not overclaim exact validity. -- Prosper/Freddie text remains labeled as external economic replication, not as - new exact funded-set certificates. - -## ScholarOne Proof QA - -ScholarOne generates a proof after upload. Before final submission: - -- **Go/no-go gate:** the ScholarOne-generated proof must be visually checked - before final submission. Broken proof, missing figures, wrong file order, - title-page leakage, or anonymous-body identity leakage means **NO-GO**: - retract, repair locally, regenerate, and upload again. -- Open the generated proof, not only the local PDF. -- Recheck title, abstract, keywords, file order, supplement designation, and - Data/Code Disclosure fields. -- Confirm the title page is not included in the anonymous reviewer PDF. -- Confirm the supplement is designated as online supplemental material. -- Confirm no optional file accidentally reveals identity to reviewers. +- No author names, affiliations, acknowledgements, repository ownership, + personal URLs, local usernames, or private remotes. +- Correct title, abstract, keywords, section order, and supplement designation. +- References use the official INFORMS bibliography style. +- No missing glyphs, clipped figures, overflow tables, orphan headings, or + unreadably small text. +- Active numbers match A35--A40 and governance. +- Temporal reversals and retrospective-design caveat remain visible. +- OCE/CVaR, SPO+, and external datasets remain diagnostics, not active methods. + +## ScholarOne Proof Go/No-Go + +Open the ScholarOne-generated proof, not only the local files. Submission is +**NO-GO** if any of these occur: + +- title page or author identity leaks into reviewer files; +- body/supplement order is wrong; +- figure, equation, table, or bibliography is missing or clipped; +- data/code answers differ from the cover letter; +- page count or anonymous-review option is wrong; +- uploaded PDF differs from the locally validated build. + +Repair locally, rerun the gates, re-upload, and inspect the new proof before +final submission. diff --git a/paper/supplement_ijds.qmd b/paper/supplement_ijds.qmd index 3c6065c..7a978f4 100644 --- a/paper/supplement_ijds.qmd +++ b/paper/supplement_ijds.qmd @@ -1,5 +1,5 @@ --- -title: "Online Supplement for CRPTO" +title: "Online Supplement for CRPTO: A Calibration-Selected Conformal Guardrail for Auditable Credit Portfolio Decisions" author: "Anonymous" date: today lang: en @@ -22,1047 +22,366 @@ execute: warning: false --- -::: {.callout-note} -## Scope - -This online supplement supports the IJDS submission body. It collects proof -details, robustness and external-replication tables A3--A40, reproducibility -commands, model-risk material, fairness diagnostics, and evidence lineage. It -does not introduce a hidden selection criterion or unreported claim family: -A35 is the declared finite-grid frontier consumed by the body, A36 -is the regenerated funded-set grade audit for the selected body point, -and A37--A39 add selected-policy tail-risk, cluster-bound, and fixed-allocation -bootstrap diagnostics from the same selected allocation. A40 is the matched -point-PD baseline used to quantify the Lending Club return--risk trade-off. -::: - -The supplement is organized as a defense layer for the main manuscript rather -than as a second paper. Appendices A--C support the statistical and economic -claims, Appendix D records governance limits, Appendix E gives the reproduction -path, and Appendix F maps the anonymous submission files. Tables and figures are -included only when they either protect a body claim or prevent overreading of the -external and robustness evidence. - -| Reader question | Where to go | What to remember | +# Scope and Evidence Map + +This supplement supports one active method: an exact 90% conformal replay, +the midpoint decision score $q=(p+u)/2$, risk tolerance $\tau=0.17$, and a +nine-cell calibration selector. A35--A40 are the active evidence bundle. +Earlier A1--A34 analyses remain useful as model, robustness, comparison, or +external-transfer diagnostics, but none selects or redefines the submitted +policy. + +| Evidence | Role in the submission | Claim boundary | |---|---|---| -| What is the theorem? | Appendix A | Markov under weighted funded-set validity is the body claim; stronger assumptions stay explicit. | -| Is the selected policy a singleton? | Appendix C, A35 | The frontier is finite-grid and exact, with denominators reported. | -| What does the selected policy fund? | Appendix C, A36--A39 | Composition, tail, concentration, and bootstrap are selected-allocation diagnostics. | -| What does robustness cost on Lending Club? | Appendix C, A40 | The point-PD LP holds the candidate universe and operating constraints fixed. | -| Does the recipe travel? | Appendix C, A25--A34 | Prosper and Freddie/Mendeley are external economic recipe-transfer checks. | -| What can be reproduced? | Appendix E | Routine reproduction rebuilds paper surfaces from frozen evidence and excludes protected searches. | - -: How to read the online supplement. - -::: {.callout-important} -## Journal Strengthening Pack - -Selected former P2/P3 ideas are included here only when they defend the single -IJDS submission from frozen evidence: OCE/CVaR as a tail-risk diagnostic, robust -satisficing as committee-style margins, regret-auditability as the SPO+/CRPTO -comparator, and dependence-aware theory as a caveat/proposition. The -multidataset layer is included as a frozen external economic replication on -Prosper and Freddie/Mendeley: it tests transfer of the CRPTO recipe without -reopening the Lending Club champion. Optimized OCE/CVaR objectives, full -multi-distribution or online conformal prediction, online DFL, causal CRPTO, -multi-period portfolios, production monitoring, and package extraction are -outside the submitted claim and are not acceptance criteria for this paper. -::: - -# Appendix A: Theoretical Details - -The body states the CRPTO guarantee in operational terms. This appendix fixes -notation and records the exact boundary between the distribution-free claim and -the optional tightening arguments. The proof depends on Assumption 1 (weighted -funded-set validity) as a modeling premise for the selected allocation; Appendix -A does not turn the selected funded set into a universal conformal guarantee. - -| Symbol | Meaning | -|---|---| -| `Y_i` | Observed default indicator or bounded loss proxy for loan `i`. | -| `p_hat_i` | Calibrated probability of default used by the decision layer. | -| `u_i(alpha)` | Upper conformal endpoint at miscoverage level `alpha`. | -| `x_i` | Funding decision or allocation weight. | -| `a_i` | Loan amount or exposure. | -| `w_i` | Normalized funded-set weight, `x_i a_i / sum_j x_j a_j`. | -| `gamma` | Blend parameter for linear members of the optimization-policy family, `gamma in [0,1]`. | -| `q_i(alpha;theta)` | Effective PD used by a declared linear, capped, or tail-focused policy; `p_hat_i <= q_i <= u_i`. | -| `tau` | Risk-tolerance cap applied to the effective score, `sum_i w_i q_i <= tau + s`, where `s` is recorded solver slack. | -| `Gamma_CP` | Total portfolio conformal premium after allocation: `sum_i w_i (u_i - p_hat_i)`. | -| `Gamma_int` | Premium internalized by the decision score: `sum_i w_i (q_i - p_hat_i)`. | -| `Gamma_res` | Residual endpoint premium: `sum_i w_i (u_i - q_i)`. | -| `B_u(alpha)` | Exact weighted upper-endpoint budget, `sum_i w_i u_i(alpha) = sum_i w_i q_i + Gamma_res`. | -| `V(alpha)` | Weighted funded-set miscoverage quantity. | - -Prose writes the conformal premium as $\Gamma_{\mathrm{CP}}$; tables sometimes -use the CSV label `Gamma_CP` so regenerated headers remain traceable. - -For a fixed allocation, conformal coverage controls the expected interval miss -indicator under the stated exchangeability design. CRPTO converts loan-level -misses into the funded-set quantity `V(alpha)` using normalized exposure -weights. The expectation in Assumption 1 is over the exchangeable -calibration/test draw, conditional on the frozen recipe, declared partitions, -and allocation rule. The portfolio-level theorem assumes weighted funded-set -validity, `E[V(alpha)] <= alpha`, rather than claiming that marginal split -conformal automatically controls every adaptively selected subportfolio. A -Markov-style argument then gives the main conservative portfolio bound used in -the exact alpha-safe check. - -## Proof of Theorem 1 - -The body states the result as Theorem 1 under Assumption 1 (weighted -funded-set validity). We record the full argument here. - -**Setting.** The allocation $x$ is fixed before OOT labels are revealed. -Funded-set weights satisfy $w_i \geq 0$ and $\sum_i w_i = 1$. Outcomes and -endpoints live on the PD scale, $Y_i \in [0,1]$ and $u_i(\alpha) \in [0,1]$, -with miscoverage indicators $Z_i(\alpha) = \mathbf{1}\{Y_i > u_i(\alpha)\}$ -and $V(\alpha) = \sum_i w_i Z_i(\alpha)$. Write -$B_u(\alpha) = \sum_i w_i u_i(\alpha)$ for the weighted upper-endpoint budget of -the funded set. The robust layer caps a policy-specific effective score -$q_i(\alpha;\theta)$ satisfying -$\hat p_i\leq q_i(\alpha;\theta)\leq u_i(\alpha)$ and -$\sum_iw_iq_i\leq\tau+s$, where $s\geq0$ is recorded solver cap slack. The -deterministic identity below does not use this cap and holds for any allocation. Probabilities -and expectations are over the exchangeable calibration/test draw conditional on -the frozen recipe, partitions, and allocation rule. - -**Step 1 (pointwise domination).** For every funded loan, +| A35 | Exact alpha replay, coverage, width, and endpoint saturation. | Other alpha rows are sensitivities, not additional policies. | +| A36 | All nine calibration selector cells. | No OOT outcome-derived ranking columns. | +| A37 | Full-OOT and fixed-policy temporal evaluation. | Retrospective stress evidence, not prospective performance. | +| A38 | Funded exposure and outcomes by letter grade. | Business composition, not legal fair-lending certification. | +| A39 | Fixed-allocation funded-loan bootstrap. | Does not resample model, conformal recipe, selector, or solver. | +| A40 | Selected, more-conservative, and matched point-PD decisions. | Retrospective contrasts, not causal effects or universal dominance. | -$$ -Y_i \;\leq\; u_i(\alpha) + Z_i(\alpha). -$$ +The active governance file is +`models/experiments/champion_reopen//portfolio/ijds_policy_governance.json`. +The final tagged ranking code does not read OOT defaults or returns. Earlier +project development did inspect the static OOT corpus, so the evaluation is a +retrospective lockbox replay rather than a pristine prospective trial. + +# Appendix A: Exact Conformal and Accounting Details -If $Z_i(\alpha) = 0$ then $Y_i \leq u_i(\alpha)$ by definition of the -indicator. If $Z_i(\alpha) = 1$ then $Y_i \leq 1 \leq u_i(\alpha) + 1$ -because $u_i(\alpha) \geq 0$. Both cases give the displayed inequality. +## A.1 Frozen Recipe Replay -**Step 2 (deterministic identity, Theorem 1(i)).** Taking the $w$-weighted -sum of Step 1, +Let $p_j$ be calibrated PD and $Y_j\in\{0,1\}$ the calibration outcome. The +active recipe uses the scaled nonconformity score $$ -\sum_i w_i Y_i - \;\leq\; \sum_i w_i u_i(\alpha) + \sum_i w_i Z_i(\alpha) - \;=\; B_u(\alpha) + V(\alpha). +s_j=\frac{|Y_j-p_j|}{\sigma(p_j)},\qquad +\sigma(p)=\sqrt{\max\{p(1-p),10^{-6}\}}. $$ -This step uses no probability and no optimality: it is portfolio accounting, -valid for every realization, and it is exactly the identity verified by the -frozen exact funded-set audit. - -**Step 3 (Markov, Theorem 1(ii)).** $V(\alpha)$ is a nonnegative random -variable under the stated draw, and Assumption 1 gives -$E[V(\alpha)] \leq \alpha$. Markov's -inequality yields, for every $t > 0$ [@ghosh2002], +Calibration scores are partitioned into five score-quantile Mondrian cells. +For a cell with $n_g$ calibration rows, the finite-sample quantile level is $$ -P\bigl(V(\alpha) \geq t\bigr) \;\leq\; \frac{E[V(\alpha)]}{t} - \;\leq\; \frac{\alpha}{t}. +\min\left\{1, +\frac{\lceil(n_g+1)(1-\widetilde\alpha)\rceil}{n_g} +\right\}, $$ -**Step 4 (combination).** By Step 2, the event -$\{\sum_i w_i Y_i \geq B_u(\alpha) + t\}$ implies $\{V(\alpha) \geq t\}$, so +with `higher` interpolation and frozen used alpha +$\widetilde\alpha=0.095$ for target $\alpha=0.10$. Sparse cells fall back to +the recorded global recipe, and frozen group/temporal factors may widen but +never narrow an interval. For evaluation loan $i$, $$ -P\!\left(\sum_i w_i Y_i \geq B_u(\alpha) + t\right) \;\leq\; \frac{\alpha}{t}, +[\ell_i,u_i] +=\left[ +\max\{0,p_i-\widehat q_{g(i)}\sigma(p_i)\}, +\min\{1,p_i+\widehat q_{g(i)}\sigma(p_i)\} +\right] $$ -and the choice $t = \sqrt{\alpha}$ gives the body statement: -the endpoint budget $B_u(\alpha)$ is exceeded by more than $\sqrt{\alpha}$ with -probability at most $\sqrt{\alpha}$. $\blacksquare$ +before the recorded widening factors. -**Policy-term decomposition.** The optimizer constrains $q_i$, not -$B_u(\alpha)$ itself. Define +The replay reads all settings from the frozen conformal result payload. At the +90% reference level, maximum absolute differences against the stored artifact +are `4.44e-16` for point predictions, `3.33e-16` for lower endpoints, and +`6.67e-16` for upper endpoints. The active claim therefore uses recomputed +finite-sample quantiles rather than average-width scaling. +## A.2 Policy Algebra + +For a generic linear blend +$q_i=p_i+\gamma(u_i-p_i)$ and funded-exposure weights $w_i$, + +$$ +\Gamma_{\mathrm{CP}}=\sum_iw_i(u_i-p_i), $$ -\Gamma_{\mathrm{int}}(\alpha) -=\sum_iw_i(q_i-\hat p_i),\qquad -\Gamma_{\mathrm{res}}(\alpha) -=\sum_iw_i(u_i-q_i). + +$$ +\Gamma_{\mathrm{int}}=\gamma\Gamma_{\mathrm{CP}},\qquad +\Gamma_{\mathrm{res}}=(1-\gamma)\Gamma_{\mathrm{CP}}. $$ -Then -$\Gamma_{\mathrm{CP}}=\Gamma_{\mathrm{int}}+\Gamma_{\mathrm{res}}$ and +At $\gamma=0.5$, the conformal premium is split equally. The endpoint budget is $$ -B_u(\alpha) -=\sum_iw_iq_i(\alpha;\theta)+\Gamma_{\mathrm{res}}(\alpha) -\leq\tau+s+\Gamma_{\mathrm{res}}(\alpha). +B_u=\sum_iw_iu_i +=\sum_iw_iq_i+\Gamma_{\mathrm{res}}. $$ -For the pure linear blend only, -$\Gamma_{\mathrm{res}}=(1-\gamma)\Gamma_{\mathrm{CP}}$. The selected capped -policy has no active cap among its 314 funded rows, its effective-score cap -binds with $s=0$, and its audit gives -$\Gamma_{\mathrm{int}}=0.089032$ and $\Gamma_{\mathrm{res}}=0.073584$. -Consequently $B_u(0.01)=0.1715+0.073584=0.245084$, and the deterministic -identity reads $\sum_iw_iY_i\leq0.245084+V(0.01)=0.280434$. The observed -weighted outcome is `0.035350`, so realized risk-tolerance excess above -$\tau=0.1715$ is zero. The exact Markov loss threshold is -$B_u+\sqrt{\alpha}=0.345084$; it is an event threshold, not a deterministic -risk cap. This general decomposition is required for capped and tail-focused -policies whose residual premium cannot be inferred from $\gamma$ alone. - -On the promoted draw the realized weighted miscoverage is $V(0.01) = 0.035350$, -which *exceeds* the nominal $\alpha = 0.01$: the funded set under-covers relative -to the marginal conformal level (unweighted funded coverage `0.9427` at -alpha01 for the body point), as expected -for an adaptively selected subportfolio. The deterministic identity needs no -assumption and holds with wide margin; the probabilistic guarantee is therefore -read at the Markov level $V \leq \sqrt{\alpha} = 0.10$, which the realized $V$ -clears, not at the nominal $\alpha$. This is why Assumption 1 is presented as a -modeling premise audited after selection, not as a property the single frozen -draw establishes. The under-coverage is structural, not a calibration-draw -effect: with $n_{\mathrm{cal}} = 237{,}584$ calibration loans, the split-conformal -conditional-coverage result [@vovk2005; @angelopoulos2023] makes the marginal -coverage Beta-distributed about $1 - \alpha$ with standard deviation -$\sqrt{\alpha(1-\alpha)/n_{\mathrm{cal}}} \approx 0.0002$ at $\alpha = 0.01$, so the -endpoints (hence $\Gamma_{\mathrm{CP}}$ and $B_u$) are essentially invariant to the -calibration draw and the residual $V$ variability is test-side and -portfolio-selection driven. A36 regenerates the row-level funded-set grade audit -from the selected allocation. A37 reprices the same selected allocation -under LGD-sensitive CVaR/OCE tail summaries, and A38 recomputes the -concentration-sensitive cluster-bound thresholds from the selected funded -weights. A39 bootstraps funded-loan contributions under the fixed final -allocation. This is an empirical interval diagnostic, not a conformal guarantee, -and it does not resample solver inputs, the PD model, calibration data, or the -policy search. - -These steps leave a clean split. Theorem 1 proves the operational bound under -Assumption 1. Proposition A.1 below shows that Markov is the sharp -first-moment statement when no additional structure is asserted. Proposition -A.2 asks the next natural question: if the funded set is grouped by period, -grade, or period-grade, and dependence is allowed inside each group, what would -independence across groups buy? That cross-cluster assumption is plausible only -as an additional sensitivity condition, not as a theorem premise quietly added -to the body. - -The phrase "exact funded-set certificate" has a narrow meaning throughout the -paper. It is an exact accounting audit on the frozen out-of-time funded set: -given the selected allocation weights, observed outcomes, calibrated PD values, -and conformal upper endpoints, the audit computes `V(alpha)`, -`Gamma_CP(alpha)`, its internalized/residual decomposition, the exact endpoint -budget, and realized risk-tolerance excess directly on the funded loans. It is not a new -distribution-free theorem for arbitrary adaptive portfolios and it is not an -external-dataset certificate. Its statistical interpretation still requires the -weighted funded-set validity assumption in the body; its value is that the -promoted decision is not supported by a proxy metric, a graph, or a manually -transcribed table. - -| Certificate object | Computed from | What it supports | What it does not support | -|---|---|---|---| -| Exact funded-set audit | Frozen Lending Club OOT funded loans, allocations, labels, `p_hat`, `q_i`, and `u_i(alpha)`. | Body point `V(0.01)=0.035350`, `Gamma_CP=0.162616`, `Gamma_res=0.073584`, exact Markov threshold `0.345084`, zero realized risk-tolerance excess, and `8/8` alpha-grid pass. | Universal conditional coverage or live adaptive control. | -| Finite-grid frontier | 51,678 raw rows consolidated into 50,010 deduplicated semantic policies. | 27,508 all-alpha above-floor policies; terminal endpoint search `37,068/37,068` all-alpha passers. | A claim about all continuous policy values or future searches. | -| External exhaustiveness audit | Prosper all-candidate and Freddie capped/all-candidate LP solves. | The reported external LP values are not shortlist effects. | New exact funded-set certificates for Prosper or Freddie. | - -: Certificate taxonomy used in the paper. - -## Finite-Grid Frontier Closure (A35) - -The frontier closure replaces single-policy reporting with a declared finite-grid -frontier. The declared return floor `$170,464.54` is the realized return of the -previously certified bound-aware allocation on the frozen upstream chain; the -frontier re-evaluates a pre-declared finite policy grid deterministically over -the *same* frozen PD model outputs and Mondrian conformal intervals, so promoting -the body point regenerates no upstream model, calibrator, or interval. The consolidated frontier file -deduplicates 51,678 raw rows into 50,010 -semantic policies; 27,508 policies both pass the declared alpha grid and exceed -the return floor. The terminal endpoint run evaluates 37,068 policies and -296,544 exact alpha checks, with 37,068/37,068 all-alpha passers. The alpha grid -is fixed at -`{0.01, 0.03, 0.05, 0.07, 0.10, 0.12, 0.15, 0.20}`. - -The policy-aware v2 reconstruction reads sufficient statistics from six frozen -exact-evaluation runs and performs no new search or LP solve. It preserves the -51,678 raw rows, 50,010 deduplicated policies, 27,508 eligible policies, and the -selected body point. Relative to the former linear-only shortcut, 10,423 -policies receive a materially different exact threshold. The affected nonlinear -families are `tail_blended_uncertainty` and -`segment_relative_tail_blended_uncertainty`: 2,866 tail policies had understated -thresholds, the largest understatement was `0.241324`, and 716 policies formerly -labeled at or below `0.50` exceed `0.50` on the exact endpoint scale. This audit -is why A35 now reports $\Gamma_{\mathrm{res}}$ and $B_u$ explicitly. - -| Role | Return | $\Gamma_{CP}$ | $\Gamma_{res}$ | $V$ | $B_u$ | Markov threshold | Pass | -|---|---:|---:|---:|---:|---:|---:|:---:| -| Minimum Markov-threshold endpoint | `$170,467.27` | `0.095719` | `0.004786` | `0.031875` | `0.173036` | `0.273036` | `8/8` | -| Low-threshold balanced endpoint | `$171,006.20` | `0.097190` | `0.007289` | `0.031875` | `0.174789` | `0.274789` | `8/8` | -| Highest return under threshold <= `0.30` | `$174,552.51` | `0.120988` | `0.030247` | `0.035875` | `0.199997` | `0.299997` | `8/8` | -| Highest return under threshold <= `0.32` | `$179,436.12` | `0.139182` | `0.049410` | `0.035875` | `0.219910` | `0.319910` | `8/8` | -| Strict threshold <= `0.345` body proxy | `$184,800.41` | `0.162562` | `0.072747` | `0.035350` | `0.244997` | `0.344997` | `8/8` | -| Body/default balanced point | `$184,832.48` | `0.162616` | `0.073584` | `0.035350` | `0.245084` | `0.345084` | `8/8` | -| Highest return under threshold <= `0.36` | `$186,050.73` | `0.174600` | `0.082935` | `0.037750` | `0.258685` | `0.358685` | `8/8` | -| Highest return under threshold <= `0.45` | `$198,693.28` | `0.252323` | `0.164010` | `0.045600` | `0.349010` | `0.449010` | `8/8` | -| Max-return economic endpoint | `$223,458.14` | `0.457438` | `0.440181` | `0.069575` | `0.597056` | `0.697056` | `8/8` | - -: Finite-grid return-bound frontier (A35). Source file: -`reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv`. - -## Sharpness of the Distribution-Free Bound - -The body keeps Markov as the headline because it is the distribution-free -first-moment statement under Assumption 1. Earlier concentration tables remain -useful as a diagnostic recipe, but they depend on the funded-loan exposure -weights of the selected allocation. After the selected-policy promotion, the paper-facing -certificate is therefore A35 plus the exact funded-set audit, with A38 providing -the regenerated cluster-bound sensitivity and A39 closing the -fixed-allocation bootstrap diagnostic. A38 confirms that the -period-grade partition has the smallest cluster-aware Hoeffding threshold among -the reported partitions, but it is still looser than Markov at the paper -threshold. The body therefore keeps the first-moment Markov statement rather -than adding unverified cross-cluster independence to the theorem. - -Table A21c extends the same audit to one-sided Cantelli, Bennett and Freedman -variants [@cantelli1929; @bennett1962; @freedman1975]. At `alpha = 0.01`, -weak-variance Cantelli gives `t = 0.0661`, Bennett gives `t = 0.0722`, and -Bernstein/Freedman give `t = 0.0852` under the weaker weighted-validity variance -proxy; the stronger individual-alpha variance mode is sharper but requires an -assumption we do not assert in the theorem. - -The menu also contains the row that justifies the body's choice, which we -state as a formal optimality result rather than a stylistic preference. - -**Proposition A.1 (Markov-optimality under Assumption 1 alone).** -Let $V(\alpha) \in [0,1]$ satisfy only Assumption 1, $E[V(\alpha)] \le \alpha$, -with no further independence, correlation, or variance structure. Then for the -one-sided deviation event $\{V(\alpha) \ge t\}$ at the interpretability level -$\delta = \sqrt{\alpha}$, Markov's threshold $t_{\mathrm M} = \sqrt{\alpha}$ -cannot be improved by any second-moment argument: the variance compatible with -Assumption 1 is at most $\sigma^2 \le \alpha(1-\alpha)$ (attained by a -Bernoulli$(\alpha)$ miscoverage profile), and the sharp one-sided Chebyshev -(Cantelli) threshold under that worst-case variance is -$t_{\mathrm C} = \alpha + 3\sigma = \alpha + 3\sqrt{\alpha(1-\alpha)} > \sqrt{\alpha}$ -for $\alpha \le 0.01$ (numerically $t_{\mathrm C}=0.3085 > 0.1000 = t_{\mathrm M}$). - -*Proof.* Cantelli gives $P(V \ge E[V] + k) \le \sigma^2/(\sigma^2 + k^2)$. -Setting the right side to $\delta = \sqrt{\alpha}$ and $E[V] = \alpha$ yields -$k = \sigma\sqrt{(1-\delta)/\delta}$, so -$t_{\mathrm C} = \alpha + \sigma\sqrt{(1-\sqrt\alpha)/\sqrt\alpha}$. At -$\alpha = 0.01$ this is $\sigma\sqrt{9} = 3\sigma$ above $\alpha$, i.e. -$0.01 + 3(0.0995) = 0.3085$, which exceeds $t_{\mathrm M}=\sqrt{0.01}=0.1$. -$\blacksquare$ - -The reading is that Markov is **not** a conservative placeholder in the body; -it is the optimal first-moment statement for the guarantee Theorem 1 actually -asserts. Every sharper row in the menu prices a specific additional assumption -(loan independence, conditional variance, or a martingale protocol), and the -selected body point clears the stated Markov check with -`V = 0.035350 <= 0.100000`. - -Bennett is the closest match to the finite funded-set calculation because it -was designed for independent, non-identically distributed summands using only -the variance of the sum and component bounds. Freedman is included only as the -martingale analogue of Bernstein: it would become relevant under a pre-declared -sequential validation protocol with bounded increments and a -conditional-variance process, which is stronger than the frozen replay used -here. - -The table's role is assumption pricing. The sharper rows show what a reviewer -would gain by accepting independence, variance, or martingale structure; they -are not promoted because the body theorem asserts only Assumption 1. Chebyshev -is omitted because one-sided Cantelli dominates it for this event; Azuma is -omitted because it duplicates Hoeffding numerically while adding a sequential -protocol assumption; Chernoff is omitted because its sharp threshold requires -individual miss probabilities bounded by `alpha`; and a naive union-Markov -correction over the finite policy frontier is vacuous at the paper alphas. The -tables are regenerated by -`scripts/build_concentration_bound_table.py` and -`scripts/build_bound_tightening_audit.py` from frozen funded-set weights. - -## Cluster-Aware Conditional Tightening - -Let clusters $g = 1,\ldots,G$ represent period, grade, or period-grade cells, -and define +The optimizer constrains $\sum_iw_iq_i\le\tau$. It does not constrain $B_u$ +directly, which is why the residual premium remains visible in every audit. + +## A.3 Deterministic Identity + +**Proposition A.1 (funded-set accounting).** For any fixed allocation with +nonnegative normalized weights, outcomes $Y_i\in[0,1]$, upper endpoints +$u_i\in[0,1]$, and +$Z_i=\mathbf1\{Y_i>u_i\}$, $$ -Z_g(\alpha)=\sum_{i\in g} w_i\mathbf{1}\{Y_i>u_i(\alpha)\},\qquad -W_g=\sum_{i\in g}w_i . +\sum_iw_iY_i\le B_u+V, +\qquad +B_u=\sum_iw_iu_i, +\quad +V=\sum_iw_iZ_i. $$ -Within each cluster, defaults and conformal misses may be arbitrarily -dependent. The useful structure, if one is willing to assert it, is -cross-cluster independence after conditioning on the calibration sample and the -fixed funded allocation. Among the three reported partitions, period-grade is -the most defensible compromise for a temporal credit panel: it separates -calendar cohorts while conditioning on risk grade. Period alone ignores grade -mix, and grade alone cuts across calendar dependence. - -**Proposition A.2 (cluster-aware Hoeffding under cross-cluster independence).** -Let $\mathcal F$ contain the calibration sample, the frozen conformal recipe, -the declared cluster partition, and the selected funded allocation. Suppose -that, conditional on $\mathcal F$, the cluster aggregates -$Z_1(\alpha),\ldots,Z_G(\alpha)$ are independent, satisfy -$0\le Z_g(\alpha)\le W_g$, and obey conditional weighted validity -$\sum_g E[Z_g(\alpha)\mid\mathcal F]\le\alpha$ (for example, it is sufficient -that $E[Z_g(\alpha)\mid\mathcal F]\le\alpha W_g$ for every cluster). Then, for -every $\delta\in(0,1)$, +*Proof.* If $Y_i\le u_i$, then $Y_i\le u_i+Z_i=u_i$. If $Y_i>u_i$, then +$Z_i=1$ and $Y_i\le1\le u_i+1$. Multiply by $w_i$, sum, and use +$w_i\ge0$. $\square$ + +The proposition is deterministic and remains true after adaptive allocation. +It is deliberately weak: $V$ is observed only after outcomes become available. + +## A.4 Assumption-Conditional Markov Corollary + +**Assumption A.1 (weighted funded-set validity).** For the random +calibration/evaluation draw and frozen allocation rule, +$\mathbb E[V]\le\alpha$. + +This assumption is stronger than marginal or Mondrian coverage. The optimizer +uses $p_i$ and $u_i$ to choose $w_i$, so per-cell conformal validity does not +automatically transfer to funded-dollar weights. + +**Corollary A.1.** Under Assumption A.1, $$ -P\!\left( - V(\alpha)\ge - \alpha + \sqrt{\frac{1}{2}\left(\sum_g W_g^2\right)\log\frac{1}{\delta}} - \;\middle|\;\mathcal F -\right)\le\delta . +\Pr(V\ge t)\le\frac{\alpha}{t},\quad t>0, $$ -*Proof.* Let $\mu=\sum_g E[Z_g(\alpha)\mid\mathcal F]\le\alpha$ and -$S_2=\sum_g W_g^2$. Hoeffding's inequality for independent bounded summands -gives +and therefore $$ -P\{V(\alpha)-\mu\ge s\mid\mathcal F\}\le \exp(-2s^2/S_2). +\Pr\!\left(\sum_iw_iY_i\ge B_u+\sqrt\alpha\right) +\le\sqrt\alpha. $$ -Taking $s=\sqrt{S_2\log(1/\delta)/2}$ and using $\mu\le\alpha$ gives the -displayed bound. Integrating over $\mathcal F$ gives the same unconditional -statement. $\blacksquare$ - -Proposition A.2 is therefore the natural complement to Proposition A.1. Under -Assumption 1 alone, A.1 shows why Markov is the sharp distribution-free claim; -under an explicit cross-cluster structure, A.2 shows exactly when a -Hoeffding-style tightening becomes available [@hoeffding1963; -@boucheron2013concentration]. At the paper level $\alpha=0.01$ with matched -tail probability $\delta=\sqrt{\alpha}=0.10$, the cluster-aware threshold is -tighter than Markov only if $\sum_g W_g^2<0.0070$. The frozen funded set is much -more concentrated: period, grade, and period-grade partitions have -$\sum_g W_g^2=0.2407$, $0.3572$, and $0.0914$, respectively, so the corresponding -thresholds are `0.5365`, `0.6512`, and `0.3344`, all looser than Markov's -`0.1000`. This proposition does not replace the main theorem; it names the -extra structure a reviewer would have to accept and makes the empirical -concentration cost transparent in A21. - -# Appendix B: P1 Evidence - -The first evidence block asks whether the selected policy is a fragile point -estimate. Tables A3--A11 are generated from frozen or derived evidence files and -should be read as post-selection audit evidence rather than as a new selection -protocol. - -| Table | Role | Current-paper use | -|---|---|---| -| A3 nested holdout | Documents the 5K -> 25K -> 276K validation chain. | Appendix evidence against post-selection overclaiming. | -| A4 segment-period sensitivity | Checks coverage and funded-set quantities by period and grade. | Appendix robustness. | -| A5 decision-aware selector | Summarizes the CROMS-style screen across conformal candidates. | Method-defense appendix, not new training. | -| A6 synthetic shift | Stresses coverage under controlled covariate perturbations. | Robustness appendix. | -| A7 funded-set loan export | Provides row-level funded-set auditability. | Supplement only. | -| A8 funded-set composition | Summarizes grade, amount, and risk mix. | Appendix robustness. | -| A9 strict temporal holdout | Separates confirmation evidence from earlier selection decisions. | Strong appendix evidence. | -| A10 finalist exact bound evaluation | Shows exact alpha checks for conformal finalists. | Defends rank-1 selection. | -| A11 enhanced synthetic shift | Extends stress around the promoted policy. | Robustness appendix. | - -These tables support the current claim that the promoted decision is not a -fragile single point. They do not turn the current paper into a new -prospective selection protocol. A fully pre-declared prospective protocol is -future journal hardening. - -# Appendix C: Journal Robustness Package - -Tables A12--A40 answer likely reviewer questions that are too detailed for the -25-page body. The block structure is deliberate: A12--A18 report tail, margin -and policy-family diagnostics; A19 isolates regret-auditability; A20--A24 stress -tail, dependence, multi-distribution and online interpretations; A25--A34 test -external recipe transfer; and A35--A40 close the selected-policy frontier, -composition, tail, concentration, uncertainty profile, and matched baseline. A35 is the active -frontier consumed by the body, A36 is the selected-policy funded-set -grade audit, A37 is the selected-policy tail-risk repricing, A38 is the -selected-policy cluster-bound audit, A39 is the selected-policy -fixed-allocation bootstrap audit, and A40 is the matched point-PD decision audit. A12--A34 are diagnostic by design, except A19 -also supports the body framing around regret-auditability and A25--A34 support -the external-replication defense. -A20--A34 are journal-only add-ons: A20 audits tail-risk trade-offs as a -diagnostic package; A21 makes the dependence-aware caveat numerical; A22 turns -the tail-risk diagnostic into an *active CVaR/OCE selection constraint* (a -labeled challenger, not a new promoted policy); A23--A24 stress -multi-distribution coverage and online (ACI) stability on the frozen conformal -intervals; A25--A34 cover the external economic layer by testing the same frozen -recipe and adding the cross-dataset price-of-robustness scaling readout. Table -A12 follows the standard -definitions of Conditional -Value-at-Risk [@rockafellar2000cvar] and the Optimized Certainty Equivalent -[@bental2007oce], and the non-monotonic risk-control framing behind A22 follows -[@angelopoulos2026nonmonotonic]; all are reported as post-hoc summaries of the -frozen funded set or intervals, never as a re-promoted champion. - -| Evidence block | Paper status | Why it is here | -|---|---|---| -| A35 finite-grid frontier | Promoted body evidence | It is the active return-bound decision surface. | -| A36--A39 selected-policy audits | Support evidence | They describe the selected allocation without changing the selector. | -| A40 matched point-PD baseline | Support evidence | It quantifies the Lending Club return--risk trade-off with operating constraints fixed. | -| A19 regret-auditability | Support evidence | It answers the SPO+/DFL baseline objection. | -| A25--A34 external recipe transfer | Support evidence | It answers the single-dataset objection without new certificates. | -| A20--A24 tail/source/online diagnostics | Diagnostic evidence | They price assumptions and outside-claim lanes without promoting new guarantees. | +*Proof.* Apply Markov's inequality to nonnegative $V$, set +$t=\sqrt\alpha$, and combine with Proposition A.1. $\square$ -: Promoted, support, and diagnostic evidence hierarchy. +At active $\alpha=0.10$, the probability bound is `0.316228`. Because this is +loose and assumption-conditional, the paper treats it as sensitivity language, +not as the primary contribution or a deterministic default cap. Formal +selected-set validity would require a dedicated protocol +[@hegazy2025valid_selection_conformal_sets]. -| Table | Role | Scope caveat | -|---|---|---| -| A12 tail-risk OCE/CVaR diagnostics | Reprices the funded set under tail-risk summaries. | Diagnostic only; OCE/CVaR is not the optimized objective. | -| A13 satisficing margins | Expresses return, `V`, `Gamma_CP`, realized risk excess, and frontier pass as margins. | Thresholds are explanatory, not a new policy selector. | -| A14 dependency cluster diagnostics | Documents period/grade concentration for the tightening caveat. | Does not prove independence. | -| A15 leave-one-period stress | Reweights the funded set by leaving periods out. | Descriptive stress, not re-optimization. | -| A16 bootstrap funded-set metrics | Adds empirical intervals for return, defaults, `V`, and misses. | Bootstrap interval, not conformal guarantee. | -| A17 budget/LGD/cap sensitivity | Varies operating assumptions. | Segment caps are diagnostics, not solver constraints. | -| A18 policy-family robustness surface | Summarizes alpha-safe policies by bound-aware family. | Compatible leaderboard only inside the final family; A35 is the selected frontier. | -| A19 regret-auditability frontier | Compares two-stage, SPO+, and CRPTO robust on regret versus risk-control checks. | Comparator framing, not a new champion selector. | -| A20 tail-risk diagnostic audit | Ranks tail-risk alternatives by CVaR, OCE, return, and satisficing status on the legacy diagnostic surface. | Journal diagnostic machinery; A37 is the selected-allocation tail repricing. | -| A21 cluster-bound tightening | Reports cluster-aware Hoeffding thresholds by period, grade, and period-grade. | Transparent caveat; not tighter than Markov for the observed exposure concentration. | -| A22 tail-constrained re-optimization | Selects the max-return policy under a decision-time CVaR cap computed from conformal upper endpoints, tracing the return-vs-CVaR frontier. | CVaR/OCE as an active selection constraint; reports a tail-constrained challenger, not a new promoted policy. | -| A23 multi-distribution robustness | Worst-case 90% coverage by grade and grade x vintage cell on the frozen intervals. | Read-only diagnostic; the worst fine cell motivates MDCP/group-weighted only under a separately tagged calibration protocol. | -| A24 online conformal stability | Per-vintage and cumulative coverage plus the Gibbs-Candes ACI target trajectory over the OOT vintage sequence. | Static-OOT online-control diagnostic, not a streaming validation. | -| A25 external replication gate | Applies the frozen CRPTO scoring/conformal/LP recipe to Prosper final-status loans and Freddie/Mendeley FM48. | External economic replication; not a Lending Club champion rerun and not a new exact theorem. | -| A26 external candidate sensitivity | Checks whether the robust LP objective is stable as the OOT candidate pool grows. | Candidate-pool audit; supports the claim that reported objectives are not a tiny shortlist effect. | -| A27 Freddie horizon sensitivity | Audits Freddie/Mendeley default windows and red/green groups before selecting FM48 for the external table. | Dataset-selection audit; FM48 is promoted because it clears both coverage gates and keeps positive robust LP value. | -| A28 external LP exhaustiveness | Solves Prosper all-candidate LP and Freddie caps `500k`, `1M`, and `all`. | Exhaustiveness audit; the Freddie all-candidate optimum matches the screened optimum. | -| A29 Freddie sparse Mondrian audit | Splits Freddie coverage by all groups, eligible groups, and sparse fallback groups. | Documents sparse cells; does not claim perfect conditional coverage in every tiny group. | -| A30 external metric intervals | Adds uncertainty intervals for AUC, coverage, alpha coverage, and robust objective. | Bootstrap for funded-loan contribution only; it does not resample solver inputs. | -| A31 external OOT subperiod metrics | Breaks Prosper by OOT year and Freddie by OOT quarter. | Subperiod audit; Freddie 2015Q4 alpha coverage is just below 99%. | -| A32 Prosper default-definition sensitivity | Repeats Prosper under main, defaulted-only, and chargedoff-only labels. | Default semantics audit; all three variants pass the global gates. | -| A33 Freddie segment sensitivity | Repeats Freddie FM48 for red, green, and combined groups. | Segment audit; green and combined pass alpha01, red remains a documented caveat. | -| A34 cross-dataset price of robustness | Orders the frozen external applications (Freddie green/combined/red and Prosper) by panel default rate and reports the signed price of robustness. | Positive premium under blind external application, ordered by panel default risk across four frozen external applications. | -| A35 finite-grid frontier | Declares the promoted return-bound frontier and selected body point. | Exact finite-grid policy surface, not a continuous-region optimum. | -| A36 funded-set grade audit | Regenerates the selected body allocation at `alpha = 0.01` and summarizes funded exposure by grade bucket. | Composition evidence for the selected body point, not a fairness or protected-class audit. | -| A37 selected-policy tail-risk repricing | Reprices the selected body allocation under LGD alternatives, CVaR, OCE, and decision-time tail loss. | Diagnostic risk profile of the selected point; OCE/CVaR is not the optimized objective. | -| A38 selected-policy cluster-bound audit | Recomputes period, grade, period-grade, and score-vintage concentration thresholds from selected funded weights. | Sensitivity under extra cross-cluster assumptions; Markov remains the body-level bound. | -| A39 fixed-allocation bootstrap audit | Bootstraps funded-loan contributions under the selected body allocation. | Empirical contribution interval only; solver inputs, model, calibration, and policy search are not resampled. | -| A40 matched point-PD baseline | Solves a point-PD two-stage LP on the same candidates with the same budget, concentration cap, risk tolerance, LGD, and solver. | Frozen OOT trade-off audit; no causal, prospective, or universal-dominance claim. | - -## External Multi-Dataset Replication - -The external layer addresses the most natural single-dataset criticism without -changing the Lending Club frontier claim. Prosper final-status loans provide a -second P2P-style consumer-credit panel with direct investment fields -[@prosperLoanData]. Freddie/Mendeley FM48 provides a mortgage-credit panel built -from Freddie Mac loan-level performance data with train/OOS/OOT splits and -multiple default windows [@freddieMacSfLoanLevel; @mushava2023classimbalance]. -Home Credit was audited as a scoring/conformal source [@homeCreditDefaultRisk] -but is not promoted because it lacks a clean `exposure + return` investment -contract comparable to Lending Club, Prosper, or Freddie. - -| Dataset | Rows | Default | AUC | Cov. 90% | Cov. alpha01 | OOT cand. | Robust LP | -|---|---:|---:|---:|---:|---:|---:|---:| -| Prosper final-status | `54,807` | `30.92%` | `0.7074` | `0.9205` | `0.9943` | `10,531` | `$199,419` | -| Freddie/Mendeley FM48 | `3,173,355` | `1.45%` | `0.7839` | `0.9745` | `0.9907` | `1,396,053` | `$1,291,228` | - -: A25. External replication gate on the two reported economic datasets. The -source CSV is `reports/crpto/tables/crpto_tableA25_external_replication_gate.csv`. - -![A25 answers whether the recipe travels beyond Lending Club: Prosper and -Freddie/Mendeley both clear the global conformal and alpha = 0.01 coverage checks, with positive robust LP -objectives.](../reports/crpto/figures/crpto_fig22_external_replication.png){#fig-supp-external-replication width="88%" fig-alt="Bar chart comparing Prosper final-status and Freddie FM48 on coverage 90 percent, alpha 0.01 coverage, AUC and robust LP objective."} - -Table A26 checks whether the reported external robust objectives depend on an -artificially tiny candidate pool. Prosper reaches the same robust LP objective -using all `10,531` OOT candidates. Freddie remains stable at `$1,291,228` from -the top-`50,000` through top-`250,000` candidate pools, while random pools improve -monotonically with larger caps and stay below the top-return screen. A28 then -removes the remaining shortlist concern by solving Freddie on `500,000`, -`1,000,000`, and all `1,396,053` OOT candidates; the robust and nonrobust -objectives are identical across those three solves. - -![A26 answers the shortlist concern: robust LP value stabilizes for the -reported Prosper and Freddie external applications, and Freddie random pools improve -with candidate cap without overtaking the top-return screen.](../reports/crpto/figures/crpto_fig23_external_candidate_sensitivity.png){#fig-supp-external-candidate-sensitivity width="88%" fig-alt="Line chart of robust LP objective by candidate cap and sampling mode for Prosper and Freddie external replications."} - -![A28 answers the remaining Freddie cap concern: the full `1,396,053`-candidate LP gives the same objective as the large top screens, and the all-candidate optimum funds only loans with rank at most `551`.](../reports/crpto/figures/crpto_fig24_freddie_all_candidate_certificate.png){#fig-supp-freddie-all-candidate width="88%" fig-alt="Two-panel Freddie FM48 audit showing unchanged robust and nonrobust objectives across 500k, 1M, and all candidates, plus a log-scale comparison of all OOT candidates and worst funded rank."} - -| Dataset | Candidate cap | Candidates solved | Robust LP objective | Nonrobust LP objective | Funded loans | Worst funded rank | -|---|---:|---:|---:|---:|---:|---:| -| Prosper | all | `10,531` | `$199,419` | `$220,260` | `234` | `508` | -| Freddie FM48 | `500,000` | `500,000` | `$1,291,228` | `$1,305,409` | `143` | `551` | -| Freddie FM48 | `1,000,000` | `1,000,000` | `$1,291,228` | `$1,305,409` | `143` | `551` | -| Freddie FM48 | all | `1,396,053` | `$1,291,228` | `$1,305,409` | `143` | `551` | - -: A28. External LP exhaustiveness. The Freddie all-candidate run funds zero -loans outside the top-250,000 screen and therefore converts the former cap -caveat into an auditable dominance certificate. - -Table A27 documents why Freddie/Mendeley FM48 is the reported mortgage -replication. FM24 and FM36 are informative but miss one of the promoted gates; -FM60 keeps high 90% coverage but falls short at alpha = 0.01. FM48 is the only -Freddie horizon that clears the two conformal gates while preserving positive -economic robust value. This is a dataset-level selection audit, not a new search -over the Lending Club frontier. - -A29--A33 record the extended multidataset audit layer. The most important caveat -is Freddie's sparse Mondrian behavior: across all `29` Freddie groups, tiny -groups with only `43` OOT rows drive a minimum reported coverage of `0.5`. -After requiring at least `500` calibration+test rows per group, `25` eligible -groups cover `1,396,010` OOT rows and the minimum 90% coverage rises to -`0.8854`; this is close to, but still below, the nominal 90% threshold. The -paper therefore claims global external coverage and reports group diagnostics, -not perfect conditional validity in every sparse mortgage cell. - -| Audit | Main finding | Caveat | -|---|---|---| -| A29 sparse groups | Eligible Freddie groups cover `1,396,010 / 1,396,053` OOT rows; sparse groups contain only `43` OOT rows. | Minimum eligible 90% coverage is `0.8854`, so this remains diagnostic. | -| A30 intervals | Prosper AUC `0.7073` CI `[0.6956, 0.7190]`; Freddie AUC `0.7839` CI `[0.7799, 0.7878]`. | Robust-objective interval bootstraps funded-loan contributions only. | -| A31 subperiods | Prosper 2012 and 2013 both pass alpha coverage; Freddie quarters keep 90% coverage above target. | Freddie 2015Q4 alpha01 coverage is `0.9896`, just below 99%. | -| A32 Prosper defaults | Main, defaulted-only, and chargedoff-only definitions all pass 90% and alpha01 gates with positive all-candidate robust LP. | Default semantics change default rate and robust objective, so the main status definition remains declared. | -| A33 Freddie segments | Combined FM48 and green pass alpha01; all segment LPs are solved on all candidates with positive robust value. | Red passes 90% coverage but alpha01 is `0.9850`, so it is sensitivity evidence, not a promoted standalone claim. | - -## Cross-Dataset Price of Robustness - -A34 turns the external layer into a positive economic finding rather than a -defensive gate. Using the signed convention -$(\text{nonrobust}-\text{robust})/\text{nonrobust}$, the price of robustness is a -*positive* premium on every frozen external application and -increases with the panel default rate across the cases evaluated. This is a -pattern across two datasets (three Freddie default-window segments plus Prosper), -not a scaling law: four points are consistent with the mechanism but cannot -establish a general relationship. - -| Frozen application | Panel default rate | AUC | Price of robustness | -|---|---:|---:|---:| -| Freddie FM48 (green) | `0.58%` | `0.700` | `+1.00%` | -| Freddie FM48 (combined) | `1.45%` | `0.784` | `+1.09%` | -| Freddie FM48 (red) | `2.97%` | `0.700` | `+2.37%` | -| Prosper final-status | `30.92%` | `0.707` | `+9.46%` | - -: A34. Price of robustness by frozen application, ordered by panel default rate. -The source CSV is -`reports/crpto/tables/crpto_tableA34_price_of_robustness_cross_dataset.csv`. - -![A34 answers whether robustness has a stable interpretation outside Lending Club: under frozen external application the premium is positive and increases with the panel default rate.](../reports/crpto/figures/crpto_fig25_price_of_robustness_scaling.png){#fig-supp-price-scaling width="82%" fig-alt="Line chart on a log-scale x-axis showing the external price of robustness rising from +1.00 percent to +9.46 percent as the panel default rate increases."} - -Two readings matter. First, the premium tracks irreducible default risk, not -discrimination: the green and red Freddie segments have nearly identical AUC but -different premiums, while their default rates differ by roughly a factor of five. -Higher default risk widens the conformal intervals, so the robust worst case -discounts more economic return. Second, the Lending Club body claim is not read -through this external price table; it is read through the exact A35 -return-bound frontier. The measured headline is therefore narrow: in these -frozen external applications, the conformal robust layer costs at most a -low-double-digit premium, and CRPTO measures which regime a given panel is in. - -## Reviewer Claim Checks - -The table below links the paper's main claims to the evidence surface and -guardrails a reviewer can inspect. The point is not to add another result, but -to make the audit path explicit. - -| Claim | Evidence | Artifact | Test or guardrail | -|---|---|---|---| -| The predictive input is a frozen calibrated PD model, not a refreshed leaderboard model. | AUC, Brier, ECE, temporal backtesting, and calibration diagnostics. | `models/pd_canonical.cbm`, `models/pd_canonical_calibrator.pkl`, paper-facing metric tables. | `EXTRACTION_MANIFEST.json` and champion validation hashes. | -| The conformal layer gives conservative OOT uncertainty on the PD scale. | 90% and 95% coverage, minimum group coverage, and grade/decile audits. | `data/processed/conformal_intervals_mondrian.parquet`. | Conformal validation status and regression tests that check metric consistency across surfaces. | -| The promoted funded set passes the exact empirical safety screen `V <= sqrt(alpha)` (not nominal alpha-coverage). | Body point `V(alpha = 0.01) = 0.035350` (above alpha), `Gamma_CP = 0.162616`, `Gamma_res = 0.073584`, exact Markov threshold `0.345084`, zero realized risk-tolerance excess, `8/8` alpha pass. | A35, A36, exact bound-evaluation parquet, policy-aware frontier/governance JSON. | Exact-evaluation file and regression tests. | -| The result is not an isolated lucky policy. | The consolidated frontier has 50,010 deduplicated semantic policies and 27,508 all-alpha above-floor policies; terminal endpoint search has 37,068/37,068 all-alpha passers. | Table A35 and governance files. | Protected search/evaluation split; no continuous-region claim. | -| The conformal decision has a matched operating baseline. | A40 holds candidates, budget, concentration, risk tolerance, LGD, and solver fixed; CRPTO pays `5.875%` realized return for `8.305` percentage points less weighted default/miscoverage. | A40 CSV/TeX and point-baseline audit JSON. | One frozen OOT comparison; no causal or universal dominance claim. | -| The supplement strengthens interpretation without moving the body claim. | A20--A34 challenger, dependence, tail-risk, multi-distribution, online, and external-replication diagnostics; A35 is the active frontier; A36--A40 audit the selected decision and its matched baseline. | Journal robustness tables, Figures 15--25, A35--A40. | Scope caveats in each table; A37--A39 are risk-profile audits and A40 changes no selector. | -| The frozen PD binary is a faithful paper model. | E3/E4 T1 diagnostics show negligible seed-level discrimination movement and stable expanding-window validation. | `docs/refactor/SENSITIVITY_RUN_DESIGN_2026-06.md` and Appendix E summary. | Non-promoted diagnostics only; they do not replace the champion or become routine reproduction steps. | -| The manuscript is reproducible from frozen evidence. | Tables, figures, Quarto pages, and status reports regenerate from frozen inputs. | Repository code, DVC metadata, rendered book/paper outputs. | Pre-push test and lint hooks, DVC status checks, and manifest validation before release. | - -## Funded-Set Audit Card Status - -The selected body point has an exact aggregate funded-set audit at -`alpha = 0.01`: - -| Quantity | Value | -|---|---:| -| Funded rows at alpha01 | `314` | -| Allocated budget | `$1,000,000` | -| Realized return | `$184,832.48` | -| Weighted true default/loss proxy | `0.035350` | -| Weighted miscoverage `V` | `0.035350` | -| Funded empirical coverage | `0.9427` | -| `Gamma_CP` | `0.162616` | -| `Gamma_int` | `0.089032` | -| `Gamma_res` | `0.073584` | -| Exact endpoint budget $B_u$ | `0.245084` | -| Exact Markov loss threshold | `0.345084` | -| Realized risk-tolerance excess | `0.000000` | - -A36 regenerates the row-level funded-set audit from the selected body -allocation and closes the composition card for the live manuscript claim. The -source file is -`reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.csv`. - -| Grade bucket | Funded rows | Exposure share | Default rate | $V$ contribution | Mean $u_i(0.01)$ | -|---|---:|---:|---:|---:|---:| -| A-B | `2` | `0.51%` | `0.00%` | `0.00000` | `0.11070` | -| C | `85` | `28.71%` | `3.53%` | `0.00495` | `0.14456` | -| D | `174` | `59.43%` | `7.47%` | `0.02690` | `0.26042` | -| E-G | `53` | `11.35%` | `3.77%` | `0.00350` | `0.34866` | - -On a binary default outcome the metric has an exact reading worth stating. A -non-default cannot miss (`y_i = 0 <= u_i`), so `V` counts only funded defaults -whose conformal endpoint does not reach the PD ceiling `u_i = 1`. The conformal -layer's role at the tight `alpha` is to inflate the worst endpoints toward one, -and `V` measures the residual default mass the endpoints did not absorb. This is -why `V` tracks the -funded default rate and exceeds `alpha = 0.01` -- a return-seeking funded set has -a base default rate well above `1%` -- and why no reweighting of the calibration -(group-weighted, localized, or multi-distribution) can drive `V` to the nominal -`alpha` without capping most funded defaults at `u_i = 1`, which would void -`Gamma_CP` and the economics. The honest guarantee is therefore the -`sqrt(alpha)` level, with the deterministic identity holding exactly. - -A37 closes the tail-risk repricing caveat for the selected allocation. -At the baseline `LGD = 0.45`, the selected body point has realized return -`$184,832.48`, weighted default rate `0.035350`, realized CVaR95 loss rate -`0.276211`, and decision-time CVaR95 loss rate `0.218140`. Under the reported -LGD sensitivity grid, repriced return ranges from `$188,367.48` at `LGD = 0.35` -to `$179,529.98` at `LGD = 0.60`. These quantities profile the selected -policy's tail exposure; they do not change the body selector, which is still -the finite-grid return-bound point in A35. - -| LGD | Repriced return | Realized CVaR95 | Decision-time CVaR95 | OCE theta5 realized | Markov threshold | -|---:|---:|---:|---:|---:|---:| -| `0.35` | `$188,367.48` | `0.205511` | `0.149497` | `-0.118852` | `0.345084` | -| `0.45` | `$184,832.48` | `0.276211` | `0.218140` | `-0.075978` | `0.345084` | -| `0.60` | `$179,529.98` | `0.382261` | `0.321104` | `0.011384` | `0.345084` | - -: A37. Selected body-point tail-risk repricing from -`reports/crpto/tables/crpto_tableA37_pool93_body_tail_risk.csv`. - -A38 closes the corresponding cluster-bound caveat. At `alpha = 0.01` and -`delta = 0.10`, Markov's body threshold is `0.100000`. The tightest regenerated -cluster-aware Hoeffding threshold is period-grade at `0.281247`; period -(`0.395502`), score-vintage (`0.348546`), and grade bucket (`0.728588`) are also -looser than Markov. The empirical result supports the theorem boundary rather -than weakening it: sharper concentration language would require a less -concentrated funded set or extra assumptions not asserted by the body. - -| Cluster partition | Clusters | Max exposure share | Sum exposure squared | Hoeffding threshold | Tighter than Markov | -|---|---:|---:|---:|---:|:---:| -| period | `11` | `0.185295` | `0.129083` | `0.395502` | `False` | -| grade bucket | `4` | `0.594275` | `0.448512` | `0.728588` | `False` | -| period-grade | `27` | `0.119550` | `0.063907` | `0.281247` | `False` | -| score-vintage | `20` | `0.165295` | `0.099552` | `0.348546` | `False` | - -: A38. Selected body-point cluster-bound audit from -`reports/crpto/tables/crpto_tableA38_pool93_body_cluster_bound_audit.csv`. - -A39 closes the final bootstrap diagnostic under the selected allocation. -It resamples funded-loan contributions for the fixed body point (`5,000` draws, -seed `20260702`). The interval is deliberately narrower in scope than a new -statistical guarantee: it does not resample solver inputs, the PD model, -calibration data, conformal intervals, or the finite policy search. +# Appendix B: Active Evidence A35--A40 + +## A35. Exact Alpha Replay and Saturation + +| Selected | Target $\alpha$ | Used $\alpha$ | Coverage | Avg. width | Min partition coverage | Min grade coverage | $u=1$ rate | +|:---:|---:|---:|---:|---:|---:|---:|---:| +| no | `0.01` | `0.0095` | `0.996720` | `0.988215` | `0.989629` | `0.972270` | `0.935424` | +| no | `0.03` | `0.0285` | `0.988478` | `0.969779` | `0.969799` | `0.966493` | `0.832314` | +| no | `0.05` | `0.0475` | `0.972825` | `0.953580` | `0.950583` | `0.952575` | `0.718156` | +| no | `0.07` | `0.0665` | `0.955271` | `0.795666` | `0.934782` | `0.938443` | `0.613550` | +| yes | `0.10` | `0.0950` | `0.934836` | `0.788879` | `0.926310` | `0.926797` | `0.517873` | +| no | `0.12` | `0.1140` | `0.918265` | `0.783174` | `0.906875` | `0.905738` | `0.454511` | +| no | `0.15` | `0.1425` | `0.886098` | `0.646176` | `0.863562` | `0.863101` | `0.334859` | +| no | `0.20` | `0.1900` | `0.849380` | `0.636585` | `0.829376` | `0.828007` | `0.244668` | + +: A35 exact alpha sensitivity. Source: +`reports/crpto/tables/crpto_tableA35_exact_alpha_grid.csv`. + +The 99% row is nearly saturated and is not the active policy level. Rows other +than 90% apply the same frozen widening recipe as sensitivity; they are not +separately tuned conformal winners. + +## A36. Calibration Policy Selector + +| Selected | Eligible | Candidate | $\tau$ | $\gamma$ | Expected objective | Endpoint | Threshold | +|:---:|:---:|---|---:|---:|---:|---:|---:| +| yes | yes | `linear-005` | `0.17` | `0.50` | `$110,346.16` | `0.261047` | `0.577275` | +| no | yes | `linear-009` | `0.19` | `0.75` | `$107,122.88` | `0.228340` | `0.544567` | +| no | yes | `linear-002` | `0.15` | `0.50` | `$106,866.89` | `0.226670` | `0.542897` | +| no | yes | `linear-006` | `0.17` | `0.75` | `$104,272.78` | `0.203468` | `0.519696` | +| no | yes | `linear-003` | `0.15` | `0.75` | `$101,108.78` | `0.177940` | `0.494167` | +| no | no | `linear-007` | `0.19` | `0.25` | `$126,123.20` | `0.446819` | `0.763046` | +| no | no | `linear-004` | `0.17` | `0.25` | `$121,761.88` | `0.392607` | `0.708835` | +| no | no | `linear-001` | `0.15` | `0.25` | `$117,071.33` | `0.341219` | `0.657447` | +| no | no | `linear-008` | `0.19` | `0.50` | `$113,591.27` | `0.294789` | `0.611017` | + +: A36 complete selector. Source: +`reports/crpto/tables/crpto_tableA36_calibration_policy_selector.csv`. + +The selector uses `35,638` temporal calibration-holdout rows. Its output schema +contains zero forbidden outcome-derived fields. The selected row is the highest +expected-objective eligible candidate, not the highest objective overall. + +## A37. Temporal Fixed-Policy Evaluation + +| Period | Policy | Return | Default | Miscoverage | Endpoint | Threshold | +|---|---|---:|---:|---:|---:|---:| +| Full OOT | Selected 50/50 | `$179,327.59` | `0.039375` | `0.036875` | `0.258051` | `0.574279` | +| Full OOT | Conservative 75% | `$172,939.50` | `0.035875` | `0.035875` | `0.200396` | `0.516624` | +| Full OOT | Point PD | `$196,369.14` | `0.118400` | `0.041900` | `0.921317` | `1.237545` | +| 2018H1 | Selected 50/50 | `$92,530.73` | `0.106703` | `0.097428` | `0.259170` | `0.575398` | +| 2018H1 | Point PD | `$118,101.99` | `0.190825` | `0.076425` | `0.938912` | `1.255140` | +| 2018H2 | Selected 50/50 | `$156,185.51` | `0.026725` | `0.026725` | `0.262441` | `0.578669` | +| 2018H2 | Point PD | `$95,603.58` | `0.236728` | `0.113175` | `0.915546` | `1.231774` | +| 2019H1 | Selected 50/50 | `$123,590.69` | `0.077325` | `0.061700` | `0.261872` | `0.578100` | +| 2019H1 | Point PD | `$144,281.46` | `0.170275` | `0.038275` | `0.884838` | `1.201065` | +| 2019H2 | Selected 50/50 | `$110,251.95` | `0.103250` | `0.103250` | `0.261503` | `0.577731` | +| 2019H2 | Point PD | `$256,966.20` | `0.023775` | `0.007900` | `0.898805` | `1.215033` | +| 2020+ | Selected 50/50 | `$99,689.54` | `0.083775` | `0.083775` | `0.264462` | `0.580690` | +| 2020+ | Point PD | `$218,629.14` | `0.016900` | `0.000000` | `0.861425` | `1.177653` | + +: A37 selected temporal rows and matched point-PD rows. Source: +`reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.csv`. + +Each temporal policy receives a fresh `$1M` budget. The table is intentionally +unfavorable to a universal-dominance reading: point PD is much stronger in +2019H2 and 2020+, whereas CRPTO is much stronger in 2018H2. + +## A38. Funded Letter-Grade Audit + +| Grade | Funded | Exposure share | Default | Miscoverage | Point PD | Midpoint $q$ | Endpoint | Return | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| B | `1` | `0.0040` | `0.000000` | `0.000000` | `0.062075` | `0.101472` | `0.140869` | `$643.20` | +| C | `91` | `0.3136` | `0.028622` | `0.028622` | `0.067801` | `0.108598` | `0.149395` | `$47,303.75` | +| D | `170` | `0.5868` | `0.045839` | `0.045839` | `0.088328` | `0.194926` | `0.301523` | `$109,618.08` | +| E | `33` | `0.0806` | `0.000000` | `0.000000` | `0.085242` | `0.179551` | `0.273860` | `$19,879.71` | +| F | `11` | `0.0130` | `0.269231` | `0.076923` | `0.110970` | `0.425687` | `0.740404` | `$1,266.55` | +| G | `2` | `0.0020` | `0.000000` | `0.000000` | `0.146801` | `0.573401` | `1.000000` | `$616.30` | + +: A38 selected funded-set composition. Source: +`reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.csv`. + +The table uses letter grade recovered from `sub_grade`; the aligned conformal +group is stored separately. This prevents the score-quantile partition labels +from being misreported as credit grades. + +## A39. Fixed-Allocation Bootstrap | Metric | Observed | Bootstrap mean | 2.5% | Median | 97.5% | |---|---:|---:|---:|---:|---:| -| Return, LGD `0.45` | `$184,832.48` | `$184,623.11` | `$167,963.20` | `$185,098.41` | `$198,650.47` | -| Weighted default / `V` | `0.035350` | `0.035404` | `0.018157` | `0.034601` | `0.057193` | -| `Gamma_CP` | `0.162616` | `0.162317` | `0.137160` | `0.161384` | `0.193092` | -| Realized CVaR95 | `0.276211` | `0.270721` | `0.068928` | `0.266983` | `0.450000` | -| Decision-time CVaR95 | `0.218140` | `0.217210` | `0.205148` | `0.217709` | `0.226318` | - -: A39. Fixed-allocation funded-loan contribution bootstrap from -`reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.csv`. - -## Matched Point-PD Decision Audit (A40) - -A40 isolates the effect of carrying conformal uncertainty into the decision. -It solves a point-PD two-stage LP on all 276,869 OOT candidates with the same -`$1M` budget, maximum concentration, $\tau=0.1715$, LGD `0.45`, solver, and -minimum-utilization/slack settings as the selected-policy lineage. The point -baseline uses calibrated $\hat p_i$ in its objective and risk constraint; -selected CRPTO uses its declared $q_i$. Neither optimization receives OOT labels. - -| Policy | Realized return | Return cost vs point | Funded | Weighted default / $V$ | $\Gamma_{CP}$ | $B_u$ | Markov threshold | -|---|---:|---:|---:|---:|---:|---:|---:| -| Point-PD two-stage LP | `$196,369.14` | `0.000%` | `225` | `0.118400` | `0.526736` | `0.680579` | `0.780579` | -| Selected CRPTO | `$184,832.48` | `5.875%` | `314` | `0.035350` | `0.162616` | `0.245084` | `0.345084` | - -: A40. Matched Lending Club point-PD baseline from -`reports/crpto/tables/crpto_tableA40_pool93_point_baseline.csv`. - -The realized return difference is `$11,536.66`. CRPTO reduces weighted -default/miscoverage by `0.08305` (8.305 percentage points) and the exact Markov -threshold by `0.435495` (43.55 percentage points). -Both allocations have zero realized risk-tolerance excess, but the point-PD -funded set misses the tightest Markov screen because -$0.1184>\sqrt{0.01}=0.10$. The row-level point allocation is stored outside the -model directory as a processed experimental artifact, while the compact JSON -records inputs, solver status, both certificates, and the claim boundary. A40 -is a matched frozen OOT comparison, not a significance test, causal estimate, -prospective trial, or claim that CRPTO dominates every point-PD portfolio. - -## Why A21--A34 Do Not Strengthen the Main Claim - -A21--A34 are designed to make the paper harder to over-read. A21 shows that a -cluster-aware tightening is transparent but not sharper than Markov under the -observed exposure concentration. A23 shows where weighted, group-weighted, -non-exchangeable, or multi-distribution conformal methods would matter if CRPTO -were recalibrated under a new protocol [@farinhas2024nonexchangeable_crc]. A24 -shows that an online controller would have little to correct on the frozen OOT -vintages, but it is still a replay, not evidence from a live stream. A25--A34 -show that the same recipe clears useful gates on Prosper and Freddie/Mendeley, -that Freddie's full candidate universe has been solved, -that subperiod/definition/segment sensitivities are documented, and that the -economic cost of applying the frozen recipe is ordered by panel risk across the -four frozen external applications. They -still do not create a new exact funded-set certificate or theorem for every -external portfolio: they verify candidate-pool exhaustiveness and economic -viability under a frozen recipe, not a new post-selection conformal guarantee -[@hegazy2025valid_selection_conformal_sets]. -Together, these evidence files protect the IJDS claim: the submitted result is an -auditable post-hoc conformal robust credit-portfolio decision with an exact -frozen Lending Club funded-set certificate and external economic replication -evidence, not a universal conditional-coverage, cross-dataset, or -online-deployment guarantee. - -## Decision-Certificate Landscape - -The main manuscript positions CRPTO as an auditable post-hoc bridge, not as the -only possible conformal decision framework. The table below clarifies the -certificate landscape that motivated the journal package. - -| Family | Certificate object | What CRPTO uses now | Boundary for this submission | -|---|---|---|---| -| Contextual optimization / prescriptive analytics | A learned or estimated context-to-decision map [@sadana2025contextual]. | CRPTO is a post-hoc credit instance: frozen PD, conformal endpoint, robust LP, exact audit. | End-to-end policy learning or stochastic-programming consistency would be a different protocol. | -| Profit/risk credit scoring | Profit, rejection, or multiobjective classifier metrics under predictive or parameter uncertainty [@xu2025profit_uncertainty_credit; @xu2024profit_risk_credit]. | Economic return and conformal premium are evaluated after the funded set is chosen. | Classifier-level profit uncertainty is not a funded-portfolio certificate. | -| Split/Mondrian CP | Marginal or partitioned coverage of PD-scale intervals [@vovk2005; @bostrom2021; @gibbs2024]. | Upper conformal endpoint becomes the robust PD input. | Stronger conditional guarantees require new assumptions or diagnostics. | -| Data-driven robust optimization | Feasibility against an uncertainty set [@bertsimas2004; @goldfarb2003robustportfolio; @delage2010dro; @bertsimas2018datadriven]. | Budgeted robust portfolio with exact funded-set check. | New robust objectives, DRO ambiguity sets, or portfolio-selection variants would be new research lanes. | -| Conformal robustness control | Robustness probability or loss control in downstream decisions [@johnstone2021; @hu2026crc]. | Used as positioning language and audit inspiration. | Not re-promoted as a new CRPTO selector. | -| Non-exchangeable CRC | Expected monotone-loss control under relevance weights or source shift [@farinhas2024nonexchangeable_crc]. | A23--A24 diagnose where such methods would matter. | Requires a new weighted/non-exchangeable calibration run. | -| Valid selected conformal sets | Coverage after choosing among multiple conformal sets [@hegazy2025valid_selection_conformal_sets]. | The current frontier discloses finite-grid denominators and exact audit results. | Stability, nested, or held-out selection must be predeclared before promotion. | -| Decision-calibrated prediction sets | Calibrate the uncertainty set by downstream reliability or robust risk [@zhou2026creme; @stratigakos2026decision_calibrated_sets; @chen2026polyhedral_conformal_ro; @wang2026optimal_decision_prediction_sets]. | CRPTO reports one audited credit decision and the return-bound frontier. | Learning set geometry or robustness levels is outside the submitted evidence. | -| CROM/CREME/CREDO | Model or decision certificates for robust optimization [@bao2025croms; @zhou2025credo; @zhou2026creme]. | Used to motivate A20--A34 challenger and replication diagnostics. | The current paper uses them for positioning, not as an implemented certificate. | -| Decision-focused learning | Regret-aware training through the optimization loss [@elmachtoub2022; @liu2021riskbounds; @schutte2024robust]. | SPO+ is a comparator in A19. | End-to-end retraining would change the frozen predictive model. | -| OCE/CVaR risk-controlling prediction sets | Tail-oriented risk control for prediction sets [@huang2026oce_rcps]. | OCE/CVaR remain post-hoc tail summaries in A20--A22 and A37. | Promoting OCE/CVaR requires a new selector and exact funded-set audit. | - -## Single-Submission Boundary Map - -Several neighboring methods are scientifically attractive, but the current -submission is strongest when they are used as boundaries around one promoted -certificate. The distinction below is the operating rule for revision: use -frozen diagnostics to sharpen the submitted claim, and do not promote any -method-changing variant without separately tagged evidence. - -| Adjacent path | Paper use available now | Why it is not promoted now | Evidence required to enter this paper as a promoted claim | -|---|---|---|---| -| Tail-aware selector | Use A20--A22 and A37 to show the selected decision's tail profile and the available return-tail trade-off. | The body selector is return-bound, not CVaR/OCE. | Predeclare CVaR/OCE objective or constraint, rerun the finite policy search under a new tag, and exact-audit the selected funded set. | -| Prospective/nested selection | Use A3, A9, A35, and the declared finite-grid denominators to answer post-selection concerns. | The current frontier is a frozen retrospective audit, not a fully prospective clinical-trial-style protocol. | Freeze all selectors before a final untouched evaluation panel or a new dataset, then report the search/evaluation split as the main design. | -| Valid selected conformal sets | Cite the selection risk explicitly and report the finite-grid denominator. | Selecting the attractive policy among valid candidates is not itself a conformal theorem. | Add stability-based, nested, or independent recalibration for the selected set/policy. | -| Decision-calibrated robustness | Interpret A19/A35 as an empirical return-bound frontier. | The robustness level and uncertainty-set geometry were not learned against downstream regret/reliability. | Learn or inverse-calibrate the robustness level, then rerun the credit audit under a new tag. | -| Multi-distribution validity | Use A23 to show where grade, vintage, and fine-cell coverage remain strong or thin. | The intervals were not calibrated by a multi-source or group-weighted objective. | Fit a new conformal layer targeting multi-distribution or group-weighted coverage, then repeat the funded-set audit. | -| Online validity | Use A24 as an OOT vintage replay that documents whether ACI would have needed large corrections. | A replay over historical vintages is not a live sequential guarantee. | Run or simulate a predeclared sequential protocol with online alpha updates and decision-time logging. | -| Decision-focused conformal learner | Use A19 to state the regret-auditability trade-off: SPO+ owns low regret, CRPTO owns auditable funded-set controls. | The PD model is frozen and not trained through the optimizer. | Train an end-to-end learner, calibrate its decision uncertainty, and require the same funded-set certificate as CRPTO. | -| Causal decision layer | Use the discussion to note that observational credit panels support predictive/prescriptive certificates only. | No randomized or quasi-experimental assignment design is present. | Define a causal estimand, identification strategy, and policy-evaluation protocol before promotion. | - -## Coverage-Validity Ladder - -The table below records the validity ladder used to interpret A23--A33. The -purpose is to keep the strongest claims aligned with the available evidence. - -| Level | Claim form | Supporting CRPTO evidence | Boundary | -|---|---|---|---| -| Marginal split CP | Coverage over an exchangeable evaluation population. | OOT interval audit and paper-facing validation tables. | Does not imply profile-level conditional coverage. | -| Mondrian/group CP | Coverage within declared partitions such as score deciles or grades. | Frozen Mondrian intervals and grade diagnostics. | Small grade x vintage cells can remain weak. | -| Weighted / localized coverage | Coverage under known weights or local neighborhoods [@barber2023beyond; @guan2023localized; @jonkers2024wcps]. | A23 reports where reweighting/group focus would matter. | Not fitted as a new interval method. | -| Non-exchangeable CRC | Expected monotone-loss control under covariate/source relevance weights [@farinhas2024nonexchangeable_crc]. | A23--A24 reveal where source shift and temporal weighting would matter. | Not fitted as a new non-exchangeable calibration layer. | -| Post-selection conformal validity | Coverage preserved after selecting among multiple valid conformal sets [@hegazy2025valid_selection_conformal_sets]. | A35 reports the finite-grid frontier and exact checks transparently. | Audit evidence only; no stability-based selected-set theorem is claimed. | -| Multi-distribution validity | Coverage across multiple source distributions [@liu2024multisource; @yang2026multidistribution; @bhattacharyya2026groupweighted]. | A23 worst-cell table is a read-only stress test. | Full MDCP would need a new calibration protocol. | -| Online validity | Sequential alpha adaptation [@gibbs2021aci; @liu2026portfolio]. | A24 replays OOT vintages as a static online diagnostic. | Not evidence from a live stream. | -| External economic replication | Frozen recipe transfer to different credit products. | A25--A34 report Prosper and Freddie/Mendeley scoring, conformal, LP, exhaustiveness, sensitivity gates, and price-of-robustness scaling. | Replication evidence, not a new universal guarantee. | - -## Lending-Club And P2P Predecessors - -The table below anchors the empirical domain lineage. These papers narrow the novelty -claim: CRPTO is not the first Lending Club model, not the first P2P portfolio -optimizer, and not the first conformal credit-scoring application. Its claim is -the audited coupling of conformal PD uncertainty with a robust credit-portfolio -decision. - -| Paper family | Domain contribution | CRPTO distinction | -|---|---|---| -| IJDS credit-risk graph learning [@das2023creditgraph]. | Shows that richer financial data structures can improve credit-rating prediction in an IJDS setting. | CRPTO keeps prediction quality as an input and moves the contribution to auditable portfolio decision control. | -| Cost-aware classifier calibration [@yang2025costaware]. | Shows that miscalibration has asymmetric downstream decision costs. | CRPTO uses calibrated PD and conformal upper endpoints as a governance-visible decision input. | -| Profit/risk credit scoring under uncertainty [@xu2025profit_uncertainty_credit; @xu2024profit_risk_credit]. | Prices predictive and parameter uncertainty through profit, rejection, or multiobjective metrics. | CRPTO moves the uncertainty price to the funded portfolio through $\Gamma_{\mathrm{CP}}$ and exact $V(\alpha)$ audit. | -| Dynamic loan-portfolio profitability [@djeundje2025dynamic_loan_portfolio_profitability; @distaso2025business_cycle_losses]. | Models portfolio cash flows, LGD, and macroeconomic drivers in consumer credit. | CRPTO is static/OOT and certificate-focused; macro dynamics are outside this submitted claim. | -| Explanation stability in cost-sensitive credit scoring [@ballegeer2025explanation_stability]. | Cost-sensitive gains can reduce explanation stability under imbalance. | CRPTO keeps the predictive model frozen and makes the decision audit, not local explanations, the governance surface. | -| Lending Club / fintech credit scoring [@jagtiani2019altdata; @albanesi2024credit; @zheng2026twostage]. | Measures predictive and scorecard behavior on platform or fintech lending data. | Uses the PD model as an auditable input to a decision certificate. | -| P2P investment support [@guo2016p2p; @zhao2016p2pportfolio; @babaei2020p2p]. | Combines borrower-level prediction with portfolio-style investment recommendation. | Adds conformal uncertainty and exact alpha-safe funded-set validation. | -| Profit scoring in P2P lending [@serrano2016profitscoring]. | Reframes loan selection around economic return rather than classification accuracy alone. | Adds a post-allocation risk certificate and finite-grid frontier evidence around the economic objective. | -| Robust P2P credit portfolio optimization [@chi2019p2p]. | Brings data-driven robust optimization into P2P lending. | Makes the uncertainty set conformal and traceable. | -| AI/OR digital lending optimization [@aior2025lendingclub]. | Frames Lending Club funding as multi-objective OR. | Keeps risk controls and evidence governance as first-class outputs. | -| Ordinal conformal credit scoring [@kawasumi2026ordinal]. | Applies conformal prediction to credit-score intervals. | CRPTO moves from score uncertainty to a robust portfolio decision. | - -![A reviewer may ask whether another uncertainty baseline dominates; Figure 7 keeps those alternatives as supplement evidence and shows why conformal robust sets remain the promoted input without replacing the champion.](../reports/crpto/figures/crpto_fig7_uncertainty_baselines.png){#fig-supp-uncertainty-baselines width="90%" fig-alt="Three-panel comparison of uncertainty set methods by empirical coverage, mean interval width, and minimum grade coverage."} - -![A reviewer may ask whether regret training should replace CRPTO; Figure 9 keeps SPO+ as the low-regret comparator, not as a replacement champion.](../reports/crpto/figures/crpto_fig9_spo_regret.png){#fig-supp-spo width="90%" fig-alt="Decision regret comparison of two-stage, SPO+, and conformal robust methods, with SPO+ showing lower regret."} - -![A19 answers the regret objection by materializing the frontier: SPO+ is the low-regret corner, while CRPTO robust is the auditable-risk-control corner.](../reports/crpto/figures/crpto_fig15_regret_auditability_frontier.png){#fig-supp-regret-auditability width="82%" fig-alt="Scatter plot of regret versus verifiable risk controls, positioning SPO+ as low-regret and CRPTO robust as high-auditability."} - -![CQR is retained as conformal appendix evidence, not as the promoted method.](../reports/crpto/figures/crpto_fig10_cqr_per_grade.png){#fig-supp-cqr width="90%" fig-alt="Coverage by Lending Club grade for CQR and conformal variants, used as appendix evidence rather than promoted method."} - -![A20 answers whether a lower-tail policy was available: the diagnostic surface exposes the return/tail trade-off, while A37 remains the selected-allocation repricing.](../reports/crpto/figures/crpto_fig16_tail_risk_frontier.png){#fig-supp-tail-frontier width="90%" fig-alt="Scatter plot of tail risk versus realized return across a diagnostic policy surface, with lower-tail and higher-return policies marked."} - -![A12 answers how sensitive the frozen funded set is to LGD: CVaR grows with the assumption while OCE and mean loss stay milder, so OCE/CVaR enter as post-hoc stress summaries rather than as the optimized objective.](../reports/crpto/figures/crpto_fig17_tail_risk_lgd.png){#fig-supp-tail-lgd width="82%" fig-alt="Line chart of frozen funded-set loss rate versus LGD (0.35 to 0.60) for CVaR90, CVaR95, CVaR99, OCE and the mean; CVaR grows with LGD while OCE and the mean stay mild."} - -![A22 answers how a tail-constrained challenger would be documented: each point is the highest-return alpha01-safe policy admissible under a decision-time CVaR cap, but the active selector remains the return-bound point.](../reports/crpto/figures/crpto_fig18_tail_constrained_frontier.png){#fig-supp-tail-constrained width="88%" fig-alt="Upward line of realized return versus decision-time CVaR95 cap, with a high-return point and tightest tail cap marked."} - -![A24 answers the online-control objection on the frozen panel: per-vintage and cumulative coverage stay above the 90% target while the Gibbs-Candes ACI target alpha_t barely drifts, so live control remains outside the submitted claim.](../reports/crpto/figures/crpto_fig19_online_coverage_aci.png){#fig-supp-online-aci width="88%" fig-alt="Per-vintage and cumulative coverage lines above a 90% target line across eleven OOT quarters, with the ACI target alpha_t on a secondary axis rising slightly from 0.10 to 0.12."} - -A19--A40 should be read as literature-aligned stress evidence. A19 places CRPTO -against the regret-driven training tradition; A20--A22 translate tail risk and -satisficing into finite-grid and tail-constrained audits without changing the -body claim; A37--A39 regenerate the selected allocation's -tail-risk, concentration, and fixed-allocation bootstrap profile; A40 adds the -matched point-PD operating baseline; and A23--A24 show where multi-distribution and online conformal work -would enter if the project moved from a frozen historical panel to a new -protocol. A25--A34 add the -external economic replication layer on Prosper and Freddie/Mendeley, including -Freddie all-candidate exhaustiveness and negative/sparse-cell sensitivities, -while preserving the Lending Club certificate boundary. This keeps the -supplement ambitious without silently changing the submitted method. - -# Appendix D: Fair Lending, MRM, And Governance - -The fairness section is a model-risk diagnostic. The public Lending Club data do -not contain direct protected attributes, so the paper cannot claim statutory -fair-lending certification. Where protected attributes are unavailable, the -standard practice is to proxy them--for example via Bayesian Improved Surname -Geocoding [@cfpb2014bisg]--and to interpret machine-learning underwriting -fairness with care [@finreglab2023fairness]. The supplement reports proxy and -intersectional diagnostics to show that the selected funded set does not hide an -obvious weak segment under the available columns. - -The MRM material documents intended use, out-of-scope use, model assumptions, -calibration and conformal diagnostics, challenger criteria, evidence lineage, -and escalation triggers. In the CRPTO setting, a retraining trigger is not an -automatic production process. It is a research governance event that would -require a new, separately tagged training run and a fresh drift check against the -frozen champion. - -The governance boundary for the current submission is: - -| Topic | Current submission | Outside submitted claim | +| Realized return | `$179,327.59` | `$179,075.48` | `$162,706.17` | `$179,416.99` | `$193,924.74` | +| Weighted default | `0.039375` | `0.039638` | `0.020869` | `0.039043` | `0.061742` | +| Weighted miscoverage | `0.036875` | `0.037203` | `0.019358` | `0.036715` | `0.058812` | +| $\Gamma_{\mathrm{CP}}$ | `0.176102` | `0.176026` | `0.137159` | `0.174808` | `0.224308` | +| Endpoint budget | `0.258051` | `0.257942` | `0.217308` | `0.256762` | `0.308571` | + +: A39, 5,000 resamples with seed `20260709`. Source: +`reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.csv`. + +## A40. Matched Decision Audit + +| Policy | Expected objective | Realized return | Default | Miscoverage | Endpoint | Threshold | +|---|---:|---:|---:|---:|---:|---:| +| Selected 50/50 CRPTO | `$168,271.56` | `$179,327.59` | `0.039375` | `0.036875` | `0.258051` | `0.574279` | +| More-conservative 75% blend | `$160,690.13` | `$172,939.50` | `0.035875` | `0.035875` | `0.200396` | `0.516624` | +| Point-PD matched-$\tau$ | `$214,019.15` | `$196,369.14` | `0.118400` | `0.041900` | `0.921317` | `1.237545` | + +: A40 matched full-OOT audit. Source: +`reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.csv`. + +Selected CRPTO pays `$17,041.55` (`8.678%`) relative to point PD. It reduces +weighted default by `7.9025` percentage points, weighted miscoverage by +`0.5025` percentage points, and the threshold by `66.3266` percentage points. +Its exact active funded-set quantities are +$\Gamma_{\mathrm{CP}}=0.176102$, +$\Gamma_{\mathrm{res}}=0.088051$, +$B_u=0.258051$, +$B_u+V=0.294926$, and +$B_u+\sqrt{0.10}=0.574279$. + +# Appendix C: Supporting Diagnostics A1--A34 + +A1--A34 were generated before the final simplification. They remain useful for +reviewer questions but are not coequal methods or selectors. + +| Range | Contents | Current role | |---|---|---| -| Fairness | Proxy/intersectional audit on available data. | Direct protected-attribute validation if legally available. | -| Causal automated decisions | Observational credit-risk decisions are reported as predictive/prescriptive certificates only. | Experimental or causal policy evaluation would require a separate design [@fernandezloria2025observational]. | -| Monitoring | Evidence-backed guardrails and MRM triggers. | Live production dashboard. | -| Retraining | No automatic retraining; frozen paper champion. | New named run with drift report. | -| Companion | Quarto + DVC-backed evidence lineage after journal policy allows disclosure. | Streamlit/product showcase. | - -# Appendix E: Reproducibility - -The reviewer-facing reproduction path separates general checks from -evidence-aware validation. This structure mirrors the journal's data/code -expectation: the anonymous submission can describe a reproducible companion -without revealing author identity, and the accepted-paper package can disclose -the public repository, DVC pointers, raw-data instructions, and reproducibility -workflow. The aim is narrower than a replication market or incentive mechanism: -CRPTO uses reproducibility to make one decision certificate auditable, whereas -replication-robust analytics markets study how to allocate value across -strategic data contributors [@falconer2026replication]. - -**PD stability diagnostics.** Two isolated run-tag diagnostics -(`ijds-sensitivity-2026-06-14`, separate from the frozen champion) support -distributing a frozen PD binary rather than a "retrain it yourself" recipe. -Retraining the champion -configuration across three random seeds moves out-of-time AUC within a `0.0006` band, -with Brier stable to `+/- 0.0001` and ECE near `0.006`: the CatBoost -non-reproducibility is negligible at the discrimination level, so the frozen binary is -a faithful representative and the certified evidence bundle is the right reproduction -object. An expanding-window temporal walk-forward keeps internal validation AUC in -`[0.717, 0.733]`, dipping in the most recent window toward the harder post-2018 -out-of-time regime. Both support the stability interpretation rather than overturning -the headline, and are reported as non-promoted T1 diagnostics, not a new champion and -not routine reproduction steps. - -Minimal local checks are: +| A1--A2 | Predictive benchmarks and the historical robustness frontier. | Upstream model and provenance context. | +| A3--A11 | Nested/temporal holdouts, sensitivity, synthetic shift, funded-set loan audit, and exact finalist checks. | Robustness diagnostics; no active hyperparameter selection. | +| A12--A18 | OCE/CVaR, satisficing, dependence, leave-period-out, bootstrap, budget/LGD, and policy-family audits. | Historical mechanism checks. | +| A19 | Synthetic regret-auditability comparison with two-stage and SPO+. | Comparator only; not the `$1M` funded portfolio. | +| A20--A24 | Tail, concentration, multi-distribution, and online-style diagnostics. | Stress evidence; no new conformal guarantee. | +| A25--A34 | Prosper and Freddie/Mendeley external analyses. | Static transfer evidence; not active Lending Club certificates. | -```powershell -just setup-base -just smoke -just paper-submission -``` +The external applications are useful because they cover marketplace personal +loans and single-family mortgages. They use older frozen replication contracts, +not the final nine-cell selector. They therefore support plausibility of the +general PD-to-conformal-to-allocation workflow but cannot be quoted as direct +replications of the active midpoint policy. -Paper-facing output regeneration uses frozen inputs: +Similarly, the synthetic SPO+ experiment answers a different question. It +shows that decision-focused training reduces regret on its own synthetic +optimization task [@elmachtoub2022], whereas the active CRPTO result evaluates +an uncertainty-constrained real-dollar allocation. Combining those values into +one leaderboard would be misleading. -```powershell -just tables -just figures -just evidence -just journal-package -``` +# Appendix D: Reproducibility Protocol + +## D.1 Active Commands -The full release-facing checklist is: +The paper-facing evidence is regenerated from isolated experiment outputs: ```powershell -just lint -just smoke -just validate-champion +just ijds-evidence +uv run pytest tests/test_ijds_active_claim_sync.py -q just paper-submission -uv run pytest tests/test_publication_targets.py -q -uv run dvc status --no-updates +just paper-submission-official +just validate-champion ``` -The strongest check is the validation harness: it recomputes the -promoted Mondrian conformal intervals from the frozen PD binaries and the -recorded recipe (partition edges, calibration-split seed, score scaling, -floor multipliers) and asserts exact agreement with the published interval file -(zero maximum absolute difference per loan and per Mondrian cell under the -locked dependency stack). It is opt-in because it scores the full -calibration and OOT panels: +The stronger methodology replay is explicit and writes only to versioned +experiment paths: ```powershell -$env:CRPTO_RUN_CHAMPION_DRIFT = "1" -uv run pytest tests/test_models/test_conformal_mapie_drift.py -q +just ijds-active-replay ``` -The companion tool `scripts/rebuild_test_predictions_from_frozen.py` -regenerates the canonical test-prediction surface from the same frozen -bundle and refuses to write unless the calibrated scores match the frozen -intervals exactly, which keeps the predictive and decision lineages -permanently tied together. +It is not part of ordinary manuscript rendering because it solves portfolios +and recomputes exact interval grids. The frozen upstream model, calibrator, +historical intervals, and manifest are never overwritten. -Artifact-aware validation additionally requires credentials for the DVC remote: +## D.2 Artifact Lineage -```powershell -uv run dvc status -c -r -``` +| Object | Active source | +|---|---| +| Exact alpha grid | `data/processed/experiments/champion_reopen//conformal/exact_alpha_grid.parquet` | +| Selector grid | `data/processed/experiments/champion_reopen//portfolio/calibration_policy_selection_grid.parquet` | +| OOT evaluation | `data/processed/experiments/champion_reopen//portfolio/calibration_selected_policy_oot_evaluation.csv` | +| Funded allocations | `data/processed/experiments/champion_reopen//portfolio/calibration_selected_policy_full_oot_allocations.parquet` | +| Governance | `models/experiments/champion_reopen//portfolio/ijds_policy_governance.json` | +| Paper tables | `reports/crpto/tables/crpto_tableA35...A40_*` | -The following stages must not be rerun as routine reproduction steps: -`crpto.pd.champion`, `crpto.conformal.intervals`, -`crpto.conformal.validation`, `crpto.portfolio.optimization`, and especially -`crpto.portfolio.bound_exact_eval`. Paper-facing commands such as table, -figure, evidence, journal-package, manuscript, supplement, and book renders are -safe because they consume frozen inputs. +The active run is +`champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6`. +The exact-alpha run is +`champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1`. -The intended accepted-paper disclosure package has four tiers: +## D.3 Claim-Sync Contract -| Tier | Contents | Reviewer/reader value | -|---|---|---| -| Code and manuscript | Python source, scripts, tests, Quarto body, supplement, and book sources. | Rebuild paper outputs and inspect methodology. | -| Frozen metadata | `EXTRACTION_MANIFEST.json`, DVC metadata, lockfile, status reports, and table/figure provenance. | Verify that claims are tied to immutable files. | -| Data access path | Raw Lending Club source instructions plus external dataset source notes; processed files through the declared DVC remote when allowed. | Reproduce or audit without committing private credentials or raw CSVs to Git. | -| Guardrails | `just smoke`, `just validate-champion`, publication target tests, and DVC status checks. | Separate safe paper reruns from protected champion/search stages. | +`tests/test_ijds_active_claim_sync.py` verifies: -This disclosure plan deliberately avoids local paths, secrets, private tokens, -and author-identifying URLs in the double-anonymous packet. +1. policy mode, $\tau$, $\gamma$, alpha, selector size, and forbidden columns; +2. agreement between governance and A35/A40; +3. presence of every A35--A40 CSV and TeX file; +4. numeric anchors across body, supplement, and official submission TeX; +5. absence of retired headline values from active manuscript surfaces. -# Appendix F: Submission Files +Historical manifest tests remain separate. This separation lets the submitted +claim evolve without rewriting or silently reinterpreting frozen provenance. -The active IJDS submission surfaces are: +# Appendix E: Limitations and Claim Crosswalk -| Surface | Source | Role | +| Claim | Direct evidence | What is not claimed | |---|---|---| -| Anonymous body | `paper/CRPTO_ijds.qmd` | 25-page IJDS-style manuscript source. | -| Online supplement | `paper/supplement_ijds.qmd` | Proofs, A3--A40, MRM/fairness, reproducibility. | -| Long companion | `book/` | Public companion after acceptance or journal-approved disclosure. | -| Publication config | `configs/crpto_publication_targets.yaml` | Venue, template, anonymity, and pivot rules. | - -The active handoff body is mirrored in the official INFORMS IJDS LaTeX template -with double-anonymous settings under `paper/submission/CRPTO_ijds_submission.tex`. -The title page is submitted separately, and repository or remote-storage URLs are -disclosed only according to the journal's data/code and double-anonymous policies. - -# Appendix G: Body Claim Crosswalk - -This crosswalk turns the supplement into a map rather than a long appendix. Each -row links a body-level claim to the appendix evidence that defends it and to the -boundary that prevents overclaiming. - -| Body claim | Body location | Supplement defense | Boundary | -|---|---|---|---| -| CRPTO is a decision method, not a classifier leaderboard. | Abstract, Introduction, Closest Work. | Appendix B A3--A11, Appendix C A19, Figure 1 in the body. | Prediction quality is an input; the certified object is the funded-set decision. | -| The theorem is a Markov bound under weighted funded-set validity. | Theory. | Appendix A proof, Proposition A.1 sharpness, Proposition A.2 cluster sensitivity. | Assumption 1 is a modeling premise, not a universal conformal guarantee. | -| The promoted Lending Club point is exactly auditable. | Results, exact certificate table. | Appendix C A35--A39, funded-set audit card, governance files. | Exact means file-backed accounting on the frozen selected allocation. | -| The selected point is not a lucky singleton. | Finite-grid frontier table. | A35 finite-grid frontier plus terminal and consolidated governance denominators. | Finite declared grid, not continuous global optimality. | -| The return--risk trade-off has a matched baseline. | Matched point-PD baseline. | A40 point-PD audit with candidates and operating constraints held fixed. | Frozen OOT comparison, not causal or universal dominance evidence. | -| External credit panels support recipe transfer. | Multi-dataset replication protocol and results. | A25--A34 external replication, exhaustiveness, intervals, subperiod and segment sensitivity. | External evidence is economic replication, not new exact certificates. | -| Regret and auditability are different outputs. | Regret-auditability section. | A19 and Figure 15. | SPO+ owns the synthetic regret corner; CRPTO owns the audited funded-set corner. | -| Tail and concentration evidence interpret the selected point. | Tail risk and distribution robustness. | A20--A24 and A37--A39. | Tail-risk, bootstrap and online/multi-distribution rows are diagnostics, not hidden selectors. | -| Reproducibility is evidence-backed. | Reproducibility and companion. | Appendix E commands, validation harness, DVC/manifest boundary. | Routine reproduction excludes protected champion/search stages. | +| Exact 90% replay | Reference endpoints match to `6.67e-16`; A35. | Independent tuning of every alpha sensitivity row. | +| OOT-outcome-column-free policy ranking, conditional on the frozen recipe | A36 schema and selector audit. | A label-free conformal recipe, historically untouched OOT corpus, or preregistration. | +| Full-OOT return-risk trade-off | A37 and A40. | Causal effect or universal dominance. | +| Funded-set accounting | Proposition A.1 and exact row-level allocation. | Nominal selected-set conformal coverage. | +| Conditional Markov sensitivity | Corollary A.1 under weighted validity. | Deterministic risk cap or sharp tail guarantee. | +| Composition transparency | A38 letter-grade reconciliation. | Legal fair-lending certification. | +| Contribution stability | A39 fixed-allocation bootstrap. | Full pipeline uncertainty or model-selection confidence interval. | +| Reproducibility | Commands, run tags, hashes, and sync tests. | Cross-machine bit-identical model retraining. | + +The active result is intentionally narrow. Optimized OCE/CVaR, online or +non-exchangeable conformal recalibration [@farinhas2024nonexchangeable_crc], +formal valid selection among conformal sets +[@hegazy2025valid_selection_conformal_sets], causal allocation, and live +multi-period deployment would require new protocols and are not hidden +acceptance criteria for this paper. diff --git a/reports/crpto/tables/crpto_tableA35_exact_alpha_grid.csv b/reports/crpto/tables/crpto_tableA35_exact_alpha_grid.csv new file mode 100644 index 0000000..004b318 --- /dev/null +++ b/reports/crpto/tables/crpto_tableA35_exact_alpha_grid.csv @@ -0,0 +1,9 @@ +selected_for_policy,target_alpha,used_alpha,target_coverage,empirical_coverage,coverage_gap,avg_width,min_partition_coverage,min_grade_coverage,high_endpoint_at_one_rate +False,0.01,0.0095,0.99,0.9967204706919157,0.006720470691915725,0.9882149910656284,0.989628743645835,0.9722703639514731,0.9354243342519386 +False,0.03,0.028499999999999998,0.97,0.9884783056246818,0.01847830562468178,0.9697789837688854,0.9697990273996318,0.9664933564413634,0.8323141991338864 +False,0.05,0.0475,0.95,0.9728246932664907,0.022824693266490725,0.9535799768723978,0.9505830880848586,0.9525750124329999,0.7181555175913519 +False,0.07,0.0665,0.9299999999999999,0.9552712654721186,0.025271265472118665,0.795666199875876,0.9347822665680426,0.938442835829143,0.6135500904760013 +True,0.1,0.095,0.9,0.9348356081757077,0.03483560817570763,0.7888790790793794,0.9263099219620958,0.9267968822615554,0.5178730735474177 +False,0.12,0.11399999999999999,0.88,0.9182645944471934,0.038264594447193434,0.7831737841625059,0.906874767744333,0.9057375416605808,0.4545109781160043 +False,0.15,0.1425,0.85,0.8860977574231856,0.03609775742318566,0.6461762449744832,0.8635623982976592,0.8631010078209027,0.33485872380078663 +False,0.2,0.19,0.8,0.8493800317117481,0.049380031711748096,0.6365853175970958,0.8293756967670011,0.828006532910199,0.24466805601204902 diff --git a/reports/crpto/tables/crpto_tableA35_exact_alpha_grid.tex b/reports/crpto/tables/crpto_tableA35_exact_alpha_grid.tex new file mode 100644 index 0000000..8af704c --- /dev/null +++ b/reports/crpto/tables/crpto_tableA35_exact_alpha_grid.tex @@ -0,0 +1,14 @@ +\begin{tabular}{rrrrrrrrrr} +\toprule +selected\_for\_policy & target\_alpha & used\_alpha & target\_coverage & empirical\_coverage & coverage\_gap & avg\_width & min\_partition\_coverage & min\_grade\_coverage & high\_endpoint\_at\_one\_rate \\ +\midrule +False & 0.010000 & 0.009500 & 0.990000 & 0.996720 & 0.006720 & 0.988215 & 0.989629 & 0.972270 & 0.935424 \\ +False & 0.030000 & 0.028500 & 0.970000 & 0.988478 & 0.018478 & 0.969779 & 0.969799 & 0.966493 & 0.832314 \\ +False & 0.050000 & 0.047500 & 0.950000 & 0.972825 & 0.022825 & 0.953580 & 0.950583 & 0.952575 & 0.718156 \\ +False & 0.070000 & 0.066500 & 0.930000 & 0.955271 & 0.025271 & 0.795666 & 0.934782 & 0.938443 & 0.613550 \\ +True & 0.100000 & 0.095000 & 0.900000 & 0.934836 & 0.034836 & 0.788879 & 0.926310 & 0.926797 & 0.517873 \\ +False & 0.120000 & 0.114000 & 0.880000 & 0.918265 & 0.038265 & 0.783174 & 0.906875 & 0.905738 & 0.454511 \\ +False & 0.150000 & 0.142500 & 0.850000 & 0.886098 & 0.036098 & 0.646176 & 0.863562 & 0.863101 & 0.334859 \\ +False & 0.200000 & 0.190000 & 0.800000 & 0.849380 & 0.049380 & 0.636585 & 0.829376 & 0.828007 & 0.244668 \\ +\bottomrule +\end{tabular} diff --git a/reports/crpto/tables/crpto_tableA36_calibration_policy_selector.csv b/reports/crpto/tables/crpto_tableA36_calibration_policy_selector.csv new file mode 100644 index 0000000..030eaab --- /dev/null +++ b/reports/crpto/tables/crpto_tableA36_calibration_policy_selector.csv @@ -0,0 +1,10 @@ +selected,eligible,candidate_id,risk_tolerance,gamma,expected_objective,n_funded,weighted_pd_point,weighted_pd_effective,endpoint_budget,markov_loss_threshold +True,True,linear-005,0.17,0.5,110346.16233639097,211,0.07895300428885349,0.17,0.26104699571114653,0.5772747617279845 +False,True,linear-009,0.19,0.75,107122.87628227308,209,0.07498132983738741,0.19,0.22833955672087086,0.5445673227377088 +False,True,linear-002,0.15,0.5,106866.88915535323,206,0.07333026954904732,0.14999999999999997,0.22666973045095268,0.5428974964677906 +False,True,linear-006,0.17,0.75,104272.78444263109,207,0.06959450418765509,0.17,0.20346849860411498,0.5196962646209529 +False,True,linear-003,0.15,0.75,101108.78384345812,202,0.0661808100942338,0.15,0.1779397299685887,0.49416749598542664 +False,False,linear-007,0.19,0.25,126123.1973417947,207,0.10439377945504501,0.19,0.44681866163486494,0.7630464276517028 +False,False,linear-004,0.17,0.25,121761.8779833937,212,0.09579764669795818,0.17000000000000004,0.39260705990612554,0.7088348259229635 +False,False,linear-001,0.15,0.25,117071.33325909675,214,0.08626026685001403,0.15,0.34121919944995793,0.6574469654667958 +False,False,linear-008,0.19,0.5,113591.26651572464,211,0.0852111983407822,0.19,0.29478880165921784,0.6110165676760557 diff --git a/reports/crpto/tables/crpto_tableA36_calibration_policy_selector.tex b/reports/crpto/tables/crpto_tableA36_calibration_policy_selector.tex new file mode 100644 index 0000000..a22910e --- /dev/null +++ b/reports/crpto/tables/crpto_tableA36_calibration_policy_selector.tex @@ -0,0 +1,15 @@ +\begin{tabular}{rrlrrrrrrrr} +\toprule +selected & eligible & candidate\_id & risk\_tolerance & gamma & expected\_objective & n\_funded & weighted\_pd\_point & weighted\_pd\_effective & endpoint\_budget & markov\_loss\_threshold \\ +\midrule +True & True & linear-005 & 0.170000 & 0.500000 & 110346.162336 & 211 & 0.078953 & 0.170000 & 0.261047 & 0.577275 \\ +False & True & linear-009 & 0.190000 & 0.750000 & 107122.876282 & 209 & 0.074981 & 0.190000 & 0.228340 & 0.544567 \\ +False & True & linear-002 & 0.150000 & 0.500000 & 106866.889155 & 206 & 0.073330 & 0.150000 & 0.226670 & 0.542897 \\ +False & True & linear-006 & 0.170000 & 0.750000 & 104272.784443 & 207 & 0.069595 & 0.170000 & 0.203468 & 0.519696 \\ +False & True & linear-003 & 0.150000 & 0.750000 & 101108.783843 & 202 & 0.066181 & 0.150000 & 0.177940 & 0.494167 \\ +False & False & linear-007 & 0.190000 & 0.250000 & 126123.197342 & 207 & 0.104394 & 0.190000 & 0.446819 & 0.763046 \\ +False & False & linear-004 & 0.170000 & 0.250000 & 121761.877983 & 212 & 0.095798 & 0.170000 & 0.392607 & 0.708835 \\ +False & False & linear-001 & 0.150000 & 0.250000 & 117071.333259 & 214 & 0.086260 & 0.150000 & 0.341219 & 0.657447 \\ +False & False & linear-008 & 0.190000 & 0.500000 & 113591.266516 & 211 & 0.085211 & 0.190000 & 0.294789 & 0.611017 \\ +\bottomrule +\end{tabular} diff --git a/reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.csv b/reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.csv new file mode 100644 index 0000000..9ed9783 --- /dev/null +++ b/reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.csv @@ -0,0 +1,19 @@ +period,policy,risk_tolerance,gamma,n_funded,expected_objective,realized_return,weighted_outcome,weighted_miscoverage,weighted_pd_point,weighted_pd_effective,endpoint_budget,markov_loss_threshold +full_oot,Calibration-selected 50/50 CRPTO,0.17,0.5,308,168271.56287282018,179327.5851322598,0.039375,0.036875,0.0819489105765324,0.17,0.2580510894234676,0.5742788554403055 +full_oot,More-conservative 75% blend,0.17,0.75,312,160690.12704075055,172939.50485426714,0.035875,0.035875,0.0788113673633701,0.1699999999999999,0.2003962108788766,0.5166239768957146 +full_oot,Point-PD matched-tau,0.17,0.0,225,214019.1519816618,196369.14000000004,0.1184,0.0419,0.1538437011518627,0.1538437011518627,0.9213174385285344,1.2375452045453723 +2018H1,Calibration-selected 50/50 CRPTO,0.17,0.5,260,123713.651457411,92530.7342546382,0.106703492661675,0.0974284926616751,0.0808296789072766,0.17,0.2591703210927234,0.5753980871095614 +2018H1,More-conservative 75% blend,0.17,0.75,263,116631.43710632224,75935.74005281724,0.12065,0.11815,0.0758885509922111,0.17,0.2013704830025962,0.5175982490194342 +2018H1,Point-PD matched-tau,0.17,0.0,211,176503.44846593234,118101.99096593227,0.190825,0.076425,0.17,0.17,0.9389119545998964,1.2551397206167343 +2018H2,Calibration-selected 50/50 CRPTO,0.17,0.5,289,138315.09206362677,156185.50718903824,0.026725,0.026725,0.0775592058342477,0.1699999999999998,0.2624407941657518,0.5786685601825898 +2018H2,More-conservative 75% blend,0.17,0.75,286,131160.69145112435,145764.2646632018,0.031025,0.031025,0.0752711349157277,0.1699999999999999,0.2015762883614241,0.517804054378262 +2018H2,Point-PD matched-tau,0.17,0.0,256,192790.6360653248,95603.57631688572,0.2367281267296992,0.113175,0.17,0.17,0.9155458351509776,1.2317736011678155 +2019H1,Calibration-selected 50/50 CRPTO,0.17,0.5,244,137182.83555860247,123590.68683599537,0.077325,0.0617,0.0781281806164286,0.17,0.2618718193835714,0.5780995854004094 +2019H1,More-conservative 75% blend,0.17,0.75,241,129155.4740135068,117840.14770205291,0.0714,0.0608,0.0725032304189913,0.1699999999999999,0.2024989231936695,0.5187266892105076 +2019H1,Point-PD matched-tau,0.17,0.0,203,191975.2452370317,144281.46023703172,0.170275,0.038275,0.17,0.17,0.8848375856531457,1.2010653516699836 +2019H2,Calibration-selected 50/50 CRPTO,0.17,0.5,219,136249.87920464447,110251.95086330731,0.10325,0.10325,0.0784966036859173,0.1699999999999999,0.2615033963140826,0.5777311623309205 +2019H2,More-conservative 75% blend,0.17,0.75,220,127777.26882054852,99327.40175400236,0.10325,0.10325,0.0730478509632306,0.17,0.2023173830122564,0.5185451490290944 +2019H2,Point-PD matched-tau,0.17,0.0,129,197742.1451160193,256966.1951160193,0.023775,0.0079,0.1699999999999999,0.1699999999999999,0.8988048868151989,1.2150326528320368 +2020+,Calibration-selected 50/50 CRPTO,0.17,0.5,166,113753.93195239925,99689.53961659266,0.0837749999999999,0.0837749999999999,0.075537928142652,0.17,0.264462071857348,0.580689837874186 +2020+,More-conservative 75% blend,0.17,0.75,169,106744.29312891632,83162.7196944619,0.093775,0.093775,0.0673508590345457,0.17,0.2042163803218181,0.520444146338656 +2020+,Point-PD matched-tau,0.17,0.0,159,154068.98539948836,218629.1353994884,0.0169,0.0,0.17,0.17,0.861424853142836,1.177652619159674 diff --git a/reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.tex b/reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.tex new file mode 100644 index 0000000..29b5993 --- /dev/null +++ b/reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.tex @@ -0,0 +1,24 @@ +\begin{tabular}{llrrrrrrrrrrr} +\toprule +period & policy & risk\_tolerance & gamma & n\_funded & expected\_objective & realized\_return & weighted\_outcome & weighted\_miscoverage & weighted\_pd\_point & weighted\_pd\_effective & endpoint\_budget & markov\_loss\_threshold \\ +\midrule +full\_oot & Calibration-selected 50/50 CRPTO & 0.170000 & 0.500000 & 308 & 168271.562873 & 179327.585132 & 0.039375 & 0.036875 & 0.081949 & 0.170000 & 0.258051 & 0.574279 \\ +full\_oot & More-conservative 75\% blend & 0.170000 & 0.750000 & 312 & 160690.127041 & 172939.504854 & 0.035875 & 0.035875 & 0.078811 & 0.170000 & 0.200396 & 0.516624 \\ +full\_oot & Point-PD matched-tau & 0.170000 & 0.000000 & 225 & 214019.151982 & 196369.140000 & 0.118400 & 0.041900 & 0.153844 & 0.153844 & 0.921317 & 1.237545 \\ +2018H1 & Calibration-selected 50/50 CRPTO & 0.170000 & 0.500000 & 260 & 123713.651457 & 92530.734255 & 0.106703 & 0.097428 & 0.080830 & 0.170000 & 0.259170 & 0.575398 \\ +2018H1 & More-conservative 75\% blend & 0.170000 & 0.750000 & 263 & 116631.437106 & 75935.740053 & 0.120650 & 0.118150 & 0.075889 & 0.170000 & 0.201370 & 0.517598 \\ +2018H1 & Point-PD matched-tau & 0.170000 & 0.000000 & 211 & 176503.448466 & 118101.990966 & 0.190825 & 0.076425 & 0.170000 & 0.170000 & 0.938912 & 1.255140 \\ +2018H2 & Calibration-selected 50/50 CRPTO & 0.170000 & 0.500000 & 289 & 138315.092064 & 156185.507189 & 0.026725 & 0.026725 & 0.077559 & 0.170000 & 0.262441 & 0.578669 \\ +2018H2 & More-conservative 75\% blend & 0.170000 & 0.750000 & 286 & 131160.691451 & 145764.264663 & 0.031025 & 0.031025 & 0.075271 & 0.170000 & 0.201576 & 0.517804 \\ +2018H2 & Point-PD matched-tau & 0.170000 & 0.000000 & 256 & 192790.636065 & 95603.576317 & 0.236728 & 0.113175 & 0.170000 & 0.170000 & 0.915546 & 1.231774 \\ +2019H1 & Calibration-selected 50/50 CRPTO & 0.170000 & 0.500000 & 244 & 137182.835559 & 123590.686836 & 0.077325 & 0.061700 & 0.078128 & 0.170000 & 0.261872 & 0.578100 \\ +2019H1 & More-conservative 75\% blend & 0.170000 & 0.750000 & 241 & 129155.474014 & 117840.147702 & 0.071400 & 0.060800 & 0.072503 & 0.170000 & 0.202499 & 0.518727 \\ +2019H1 & Point-PD matched-tau & 0.170000 & 0.000000 & 203 & 191975.245237 & 144281.460237 & 0.170275 & 0.038275 & 0.170000 & 0.170000 & 0.884838 & 1.201065 \\ +2019H2 & Calibration-selected 50/50 CRPTO & 0.170000 & 0.500000 & 219 & 136249.879205 & 110251.950863 & 0.103250 & 0.103250 & 0.078497 & 0.170000 & 0.261503 & 0.577731 \\ +2019H2 & More-conservative 75\% blend & 0.170000 & 0.750000 & 220 & 127777.268821 & 99327.401754 & 0.103250 & 0.103250 & 0.073048 & 0.170000 & 0.202317 & 0.518545 \\ +2019H2 & Point-PD matched-tau & 0.170000 & 0.000000 & 129 & 197742.145116 & 256966.195116 & 0.023775 & 0.007900 & 0.170000 & 0.170000 & 0.898805 & 1.215033 \\ +2020+ & Calibration-selected 50/50 CRPTO & 0.170000 & 0.500000 & 166 & 113753.931952 & 99689.539617 & 0.083775 & 0.083775 & 0.075538 & 0.170000 & 0.264462 & 0.580690 \\ +2020+ & More-conservative 75\% blend & 0.170000 & 0.750000 & 169 & 106744.293129 & 83162.719694 & 0.093775 & 0.093775 & 0.067351 & 0.170000 & 0.204216 & 0.520444 \\ +2020+ & Point-PD matched-tau & 0.170000 & 0.000000 & 159 & 154068.985399 & 218629.135399 & 0.016900 & 0.000000 & 0.170000 & 0.170000 & 0.861425 & 1.177653 \\ +\bottomrule +\end{tabular} diff --git a/reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.csv b/reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.csv new file mode 100644 index 0000000..b3a4aa2 --- /dev/null +++ b/reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.csv @@ -0,0 +1,7 @@ +grade,n_funded,exposure,exposure_share,weighted_default_rate,weighted_miscoverage,weighted_pd_point,weighted_pd_effective,weighted_pd_high,realized_return +B,1,4000.0,0.004,0.0,0.0,0.06207486505052294,0.10147180683774551,0.14086874862496807,643.1999999999999 +C,91,313565.9904450649,0.31356599044506495,0.028622364266166715,0.028622364266166715,0.06780118581238306,0.10859786707930257,0.14939454834622204,47303.7528804385 +D,170,586834.0095549352,0.5868340095549351,0.045839197391441946,0.045839197391441946,0.08832773498776167,0.19492559567178316,0.30152345635580474,109618.0772518213 +E,33,80600.0,0.0806,0.0,0.0,0.08524228589130531,0.17955099586752693,0.27385970584374847,19879.705 +F,11,13000.0,0.013000000000000001,0.2692307692307692,0.07692307692307691,0.11097039235072648,0.4256872635040547,0.7404041346573831,1266.55 +G,2,2000.0,0.002,0.0,0.0,0.14680145627400495,0.5734007281370025,1.0,616.3000000000001 diff --git a/reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.tex b/reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.tex new file mode 100644 index 0000000..f08cf37 --- /dev/null +++ b/reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.tex @@ -0,0 +1,12 @@ +\begin{tabular}{lrrrrrrrrr} +\toprule +grade & n\_funded & exposure & exposure\_share & weighted\_default\_rate & weighted\_miscoverage & weighted\_pd\_point & weighted\_pd\_effective & weighted\_pd\_high & realized\_return \\ +\midrule +B & 1 & 4000.000000 & 0.004000 & 0.000000 & 0.000000 & 0.062075 & 0.101472 & 0.140869 & 643.200000 \\ +C & 91 & 313565.990445 & 0.313566 & 0.028622 & 0.028622 & 0.067801 & 0.108598 & 0.149395 & 47303.752880 \\ +D & 170 & 586834.009555 & 0.586834 & 0.045839 & 0.045839 & 0.088328 & 0.194926 & 0.301523 & 109618.077252 \\ +E & 33 & 80600.000000 & 0.080600 & 0.000000 & 0.000000 & 0.085242 & 0.179551 & 0.273860 & 19879.705000 \\ +F & 11 & 13000.000000 & 0.013000 & 0.269231 & 0.076923 & 0.110970 & 0.425687 & 0.740404 & 1266.550000 \\ +G & 2 & 2000.000000 & 0.002000 & 0.000000 & 0.000000 & 0.146801 & 0.573401 & 1.000000 & 616.300000 \\ +\bottomrule +\end{tabular} diff --git a/reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.csv b/reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.csv new file mode 100644 index 0000000..57fe384 --- /dev/null +++ b/reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.csv @@ -0,0 +1,6 @@ +metric,observed,boot_mean,boot_p025,boot_p50,boot_p975,n_draws,seed,note +realized_return,179327.5851322598,179075.4838443055,162706.17200230644,179416.98769601624,193924.73990027257,5000,20260709,"Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled." +weighted_default_rate,0.039375,0.03963778218343555,0.020869102395698755,0.039043420065914713,0.0617419907402632,5000,20260709,"Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled." +weighted_miscoverage,0.036875,0.03720263786105997,0.019357859801356386,0.03671487666893709,0.058812406466411934,5000,20260709,"Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled." +Gamma_CP,0.1761021788469351,0.17602573881357803,0.13715911012280738,0.17480824046170865,0.2243084301758857,5000,20260709,"Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled." +endpoint_budget,0.2580510894234676,0.25794196914296347,0.21730801774100114,0.2567618086877488,0.30857101650304447,5000,20260709,"Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled." diff --git a/reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.tex b/reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.tex new file mode 100644 index 0000000..673c60e --- /dev/null +++ b/reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.tex @@ -0,0 +1,11 @@ +\begin{tabular}{lrrrrrrrl} +\toprule +metric & observed & boot\_mean & boot\_p025 & boot\_p50 & boot\_p975 & n\_draws & seed & note \\ +\midrule +realized\_return & 179327.585132 & 179075.483844 & 162706.172002 & 179416.987696 & 193924.739900 & 5000 & 20260709 & Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled. \\ +weighted\_default\_rate & 0.039375 & 0.039638 & 0.020869 & 0.039043 & 0.061742 & 5000 & 20260709 & Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled. \\ +weighted\_miscoverage & 0.036875 & 0.037203 & 0.019358 & 0.036715 & 0.058812 & 5000 & 20260709 & Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled. \\ +Gamma\_CP & 0.176102 & 0.176026 & 0.137159 & 0.174808 & 0.224308 & 5000 & 20260709 & Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled. \\ +endpoint\_budget & 0.258051 & 0.257942 & 0.217308 & 0.256762 & 0.308571 & 5000 & 20260709 & Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled. \\ +\bottomrule +\end{tabular} diff --git a/reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.csv b/reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.csv new file mode 100644 index 0000000..e07feb7 --- /dev/null +++ b/reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.csv @@ -0,0 +1,4 @@ +policy,n_funded,expected_objective,realized_return,weighted_outcome,weighted_miscoverage,endpoint_budget,markov_loss_threshold,return_delta_vs_selected,default_delta_vs_selected,threshold_delta_vs_selected +Calibration-selected 50/50 CRPTO,308,168271.56287282018,179327.5851322598,0.039375,0.036875,0.2580510894234676,0.5742788554403055,0.0,0.0,0.0 +More-conservative 75% blend,312,160690.12704075055,172939.50485426714,0.035875,0.035875,0.2003962108788766,0.5166239768957146,-6388.080277992645,-0.003500000000000003,-0.05765487854459095 +Point-PD matched-tau,225,214019.1519816618,196369.14000000004,0.1184,0.0419,0.9213174385285344,1.2375452045453723,17041.554867740255,0.07902500000000001,0.6632663491050668 diff --git a/reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.tex b/reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.tex new file mode 100644 index 0000000..7653717 --- /dev/null +++ b/reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.tex @@ -0,0 +1,9 @@ +\begin{tabular}{lrrrrrrrrrr} +\toprule +policy & n\_funded & expected\_objective & realized\_return & weighted\_outcome & weighted\_miscoverage & endpoint\_budget & markov\_loss\_threshold & return\_delta\_vs\_selected & default\_delta\_vs\_selected & threshold\_delta\_vs\_selected \\ +\midrule +Calibration-selected 50/50 CRPTO & 308 & 168271.562873 & 179327.585132 & 0.039375 & 0.036875 & 0.258051 & 0.574279 & 0.000000 & 0.000000 & 0.000000 \\ +More-conservative 75\% blend & 312 & 160690.127041 & 172939.504854 & 0.035875 & 0.035875 & 0.200396 & 0.516624 & -6388.080278 & -0.003500 & -0.057655 \\ +Point-PD matched-tau & 225 & 214019.151982 & 196369.140000 & 0.118400 & 0.041900 & 0.921317 & 1.237545 & 17041.554868 & 0.079025 & 0.663266 \\ +\bottomrule +\end{tabular} diff --git a/scripts/README.md b/scripts/README.md index 74e3edd..4ba608f 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -9,7 +9,13 @@ purpose. Use these for the active submission workflow: - `check_publication_integrity.py` - checks that paper, supplement, README and - official-template docs agree on the active pool93 IJDS claim. + official-template docs agree on the active midpoint IJDS claim. +- `build_ijds_calibration_selected_evidence.py` - regenerates active A35--A40 + and governance from the exact-alpha and selected-policy outputs. +- `experiments/run_ijds_exact_alpha_grid_challenger.py` - exact replay into an + isolated experiment path. +- `experiments/run_ijds_calibration_selected_policy_challenger.py` - solves the + declared 3x3 calibration grid and fixed OOT evaluations. - `compile_ijds_submission.py` - compiles the official INFORMS/IJDS LaTeX handoff and scans `.log`/`.blg` for unresolved citations or references. - `run_ty_advisory.py` - runs pinned `ty` in a focused advisory scope for daily @@ -29,6 +35,7 @@ just smoke just type-advisory just hooks-check just complexity-report +just ijds-evidence just paper-submission just paper-submission-official just submission-check @@ -50,8 +57,9 @@ post-submission cleanup lane justifies touching them. ## Protected or historical search paths -The large scripts under `scripts/search/` and `scripts/experiments/` are mostly -historical or governed research surfaces. Do not run HPO, conformal interval +The large scripts under `scripts/search/` and most `scripts/experiments/` are +historical or governed research surfaces. The three active IJDS experiment +modules listed above are the narrow exception. Do not run HPO, conformal interval generation, champion search, or protected portfolio search unless the work has a fresh run tag, artifact sink, and drift/revalidation plan. @@ -64,8 +72,8 @@ TabPFN, SPO+/PyEPO/Torch and cuOpt remain optional experiment stacks. The scripts that need them use explicit optional imports so the base IJDS environment stays light and full-tree type checks still remain useful. -The active paper should cite the frozen pool93 finite-grid certificate, not a -new ad hoc rerun from these entry points. +The active paper should cite A35--A40 and `ijds_policy_governance.json`, not an +ad hoc rerun or the historical policy frontier. ## Refactor priority diff --git a/scripts/analyze_crpto_evidence.py b/scripts/analyze_crpto_evidence.py index 53b7c2e..17d65cb 100644 --- a/scripts/analyze_crpto_evidence.py +++ b/scripts/analyze_crpto_evidence.py @@ -19,6 +19,7 @@ import numpy as np import pandas as pd +from src.optimization.policy import policy_segment_labels from src.utils.script_helpers import load_json, policy_matches, write_json, write_table ROOT = Path(__file__).resolve().parents[1] @@ -590,23 +591,6 @@ def _interval_arrays_at_alpha( return pd_point, pd_low_90, pd_high_90 -def _segment_labels(frame: pd.DataFrame, policy_mode: str) -> np.ndarray | None: - if str(policy_mode).strip().lower() not in { - "segment_tail_blended_uncertainty", - "segment_relative_tail_blended_uncertainty", - }: - return None - grade = frame.get("original_grade", pd.Series(["unknown"] * len(frame))).fillna("unknown") - term = frame.get("term", pd.Series(["unknown"] * len(frame))).fillna("unknown") - verification = frame.get( - "verification_status", - pd.Series(["unknown"] * len(frame)), - ).fillna("unknown") - return (grade.astype(str) + "|" + term.astype(str) + "|" + verification.astype(str)).to_numpy( - dtype=object - ) - - def _realized_return( allocation: np.ndarray, loan_amounts: np.ndarray, @@ -641,7 +625,11 @@ def _solve_exact_policy( gamma=float(policy["gamma"]), delta_cap_quantile=float(policy["delta_cap_quantile"]), tail_focus_quantile=float(policy["tail_focus_quantile"]), - segment_labels=_segment_labels(aligned, str(policy["policy_mode"])), + segment_labels=policy_segment_labels( + aligned, + str(policy["policy_mode"]), + grade_column="original_grade", + ), ) loan_amounts = ( pd.to_numeric(aligned["loan_amnt"], errors="coerce").fillna(1.0).to_numpy(dtype=float) diff --git a/scripts/build_ijds_calibration_selected_evidence.py b/scripts/build_ijds_calibration_selected_evidence.py new file mode 100644 index 0000000..d49ced2 --- /dev/null +++ b/scripts/build_ijds_calibration_selected_evidence.py @@ -0,0 +1,398 @@ +"""Build paper-facing evidence for the calibration-selected IJDS policy.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from src.optimization.policy_selection import policy_eligibility_mask # noqa: E402 +from src.utils.script_helpers import load_json, write_json, write_table # noqa: E402 + +RUN_TAG = "champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6" +EXACT_ALPHA_TAG = "champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1" +MODEL_DIR = ROOT / "models/experiments/champion_reopen" / RUN_TAG / "portfolio" +DATA_DIR = ROOT / "data/processed/experiments/champion_reopen" / RUN_TAG / "portfolio" +EXACT_MODEL_DIR = ROOT / "models/experiments/champion_reopen" / EXACT_ALPHA_TAG / "conformal" +TABLE_DIR = ROOT / "reports/crpto/tables" + +TABLE_NAMES = { + "alpha": "crpto_tableA35_exact_alpha_grid", + "selector": "crpto_tableA36_calibration_policy_selector", + "temporal": "crpto_tableA37_calibration_selected_temporal_evaluation", + "grade": "crpto_tableA38_calibration_selected_grade_audit", + "bootstrap": "crpto_tableA39_calibration_selected_bootstrap", + "baseline": "crpto_tableA40_calibration_selected_point_baseline", +} +GOVERNANCE_NAME = "ijds_policy_governance.json" + + +def _weighted_average(frame: pd.DataFrame, column: str) -> float: + weights = frame["funded_weight"].to_numpy(dtype=float) + values = frame[column].to_numpy(dtype=float) + return float(np.sum(weights * values) / max(weights.sum(), 1e-12)) + + +def build_alpha_table(summary: dict[str, Any]) -> pd.DataFrame: + rows = pd.DataFrame(summary["alpha_summaries"]).copy() + rows.insert(0, "selected_for_policy", np.isclose(rows["target_alpha"], 0.10)) + return rows[ + [ + "selected_for_policy", + "target_alpha", + "used_alpha", + "target_coverage", + "empirical_coverage", + "coverage_gap", + "avg_width", + "min_partition_coverage", + "min_grade_coverage", + "high_endpoint_at_one_rate", + ] + ] + + +def build_selector_table(grid: pd.DataFrame, summary: dict[str, Any]) -> pd.DataFrame: + selected_id = str(summary["selected_policy"]["candidate_id"]) + cap = float(summary["design"]["markov_threshold_cap"]) + output = grid.copy() + output["eligible"] = policy_eligibility_mask( + output, + markov_threshold_cap=cap, + budget=float(summary["design"]["budget"]), + min_budget_utilization=float(summary["design"]["selection_min_budget_utilization"]), + ) + output["selected"] = output["candidate_id"].astype(str).eq(selected_id) + return output[ + [ + "selected", + "eligible", + "candidate_id", + "risk_tolerance", + "gamma", + "expected_objective", + "n_funded", + "weighted_pd_point", + "weighted_pd_effective", + "endpoint_budget", + "markov_loss_threshold", + ] + ].sort_values( + ["selected", "eligible", "expected_objective"], + ascending=[False, False, False], + kind="mergesort", + ) + + +def build_temporal_table(evaluation: pd.DataFrame) -> pd.DataFrame: + role_labels = { + "calibration_selected": "Calibration-selected 50/50 CRPTO", + "incumbent_linear": "More-conservative 75% blend", + "point_pd_matched_tau": "Point-PD matched-tau", + } + output = evaluation.copy() + output["policy"] = output["role"].map(role_labels) + return output[ + [ + "period", + "policy", + "risk_tolerance", + "gamma", + "n_funded", + "expected_objective", + "realized_return", + "weighted_outcome", + "weighted_miscoverage", + "weighted_pd_point", + "weighted_pd_effective", + "endpoint_budget", + "markov_loss_threshold", + ] + ] + + +def build_grade_table(allocations: pd.DataFrame) -> pd.DataFrame: + selected = allocations.loc[allocations["role"].eq("calibration_selected")].copy() + rows: list[dict[str, Any]] = [] + for grade, group in selected.groupby("grade", observed=True, sort=True): + rows.append( + { + "grade": str(grade), + "n_funded": int(len(group)), + "exposure": float(group["funded_exposure"].sum()), + "exposure_share": float(group["funded_weight"].sum()), + "weighted_default_rate": _weighted_average(group, "outcome"), + "weighted_miscoverage": _weighted_average(group, "miscoverage"), + "weighted_pd_point": _weighted_average(group, "pd_point"), + "weighted_pd_effective": _weighted_average(group, "pd_effective"), + "weighted_pd_high": _weighted_average(group, "pd_high"), + "realized_return": float(group["realized_return_contribution"].sum()), + } + ) + return pd.DataFrame(rows) + + +def _bootstrap_snapshot( + sample: pd.DataFrame, *, total_exposure: float, lgd: float +) -> dict[str, float]: + weights = sample["funded_exposure"].to_numpy(dtype=float) + weights = weights / max(float(weights.sum()), 1e-12) + outcome = sample["outcome"].to_numpy(dtype=float) + rates = sample["int_rate"].to_numpy(dtype=float) + point = sample["pd_point"].to_numpy(dtype=float) + high = sample["pd_high"].to_numpy(dtype=float) + realized_rate = np.where(outcome.astype(int) == 1, -float(lgd), rates) + return { + "realized_return": float(np.sum(weights * realized_rate) * total_exposure), + "weighted_default_rate": float(np.sum(weights * outcome)), + "weighted_miscoverage": float(np.sum(weights * sample["miscoverage"])), + "Gamma_CP": float(np.sum(weights * (high - point))), + "endpoint_budget": float(np.sum(weights * high)), + } + + +def build_bootstrap_table( + allocations: pd.DataFrame, + evaluation: pd.DataFrame, + *, + n_draws: int = 5000, + seed: int = 20260709, + lgd: float = 0.45, +) -> pd.DataFrame: + selected = allocations.loc[allocations["role"].eq("calibration_selected")].reset_index( + drop=True + ) + total_exposure = float(selected["funded_exposure"].sum()) + allocation_observed = _bootstrap_snapshot( + selected, + total_exposure=total_exposure, + lgd=lgd, + ) + official = evaluation.loc[ + evaluation["period"].eq("full_oot") & evaluation["role"].eq("calibration_selected") + ].iloc[0] + observed = { + "realized_return": float(official["realized_return"]), + "weighted_default_rate": float(official["weighted_outcome"]), + "weighted_miscoverage": float(official["weighted_miscoverage"]), + "Gamma_CP": float(official["gamma_cp"]), + "endpoint_budget": float(official["endpoint_budget"]), + } + mismatches = [ + metric + for metric, value in observed.items() + if not np.isclose(allocation_observed[metric], value, rtol=1e-10, atol=1e-8) + ] + if mismatches: + raise ValueError( + "Funded allocations do not reconcile to the full-OOT evaluation: " + + ", ".join(mismatches) + ) + rng = np.random.default_rng(seed) + draws = [ + _bootstrap_snapshot( + selected.iloc[rng.integers(0, len(selected), size=len(selected))].reset_index( + drop=True + ), + total_exposure=total_exposure, + lgd=lgd, + ) + for _ in range(n_draws) + ] + draw_frame = pd.DataFrame(draws) + note = "Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled." + return pd.DataFrame( + [ + { + "metric": metric, + "observed": observed[metric], + "boot_mean": float(draw_frame[metric].mean()), + "boot_p025": float(draw_frame[metric].quantile(0.025)), + "boot_p50": float(draw_frame[metric].quantile(0.50)), + "boot_p975": float(draw_frame[metric].quantile(0.975)), + "n_draws": n_draws, + "seed": seed, + "note": note, + } + for metric in draw_frame.columns + ] + ) + + +def build_baseline_table(evaluation: pd.DataFrame) -> pd.DataFrame: + full = evaluation.loc[evaluation["period"].eq("full_oot")].copy() + selected = full.loc[full["role"].eq("calibration_selected")].iloc[0] + labels = { + "calibration_selected": "Calibration-selected 50/50 CRPTO", + "incumbent_linear": "More-conservative 75% blend", + "point_pd_matched_tau": "Point-PD matched-tau", + } + full["policy"] = full["role"].map(labels) + full["return_delta_vs_selected"] = full["realized_return"] - float(selected["realized_return"]) + full["default_delta_vs_selected"] = full["weighted_outcome"] - float( + selected["weighted_outcome"] + ) + full["threshold_delta_vs_selected"] = full["markov_loss_threshold"] - float( + selected["markov_loss_threshold"] + ) + return full[ + [ + "policy", + "n_funded", + "expected_objective", + "realized_return", + "weighted_outcome", + "weighted_miscoverage", + "endpoint_budget", + "markov_loss_threshold", + "return_delta_vs_selected", + "default_delta_vs_selected", + "threshold_delta_vs_selected", + ] + ] + + +def build_governance( + summary: dict[str, Any], + exact_summary: dict[str, Any], + evaluation: pd.DataFrame, + bootstrap: pd.DataFrame, + table_paths: dict[str, list[Path]], +) -> dict[str, Any]: + full = evaluation.loc[ + evaluation["period"].eq("full_oot") & evaluation["role"].eq("calibration_selected") + ].iloc[0] + point = evaluation.loc[ + evaluation["period"].eq("full_oot") & evaluation["role"].eq("point_pd_matched_tau") + ].iloc[0] + return_boot = bootstrap.loc[bootstrap["metric"].eq("realized_return")].iloc[0] + return { + "schema_version": "2026-07-09.6", + "generated_at_utc": summary["generated_at_utc"], + "run_tag": RUN_TAG, + "status": "active_ijds_policy", + "selection_protocol": { + **summary["selection_audit"], + "calibration_metadata": summary["calibration_metadata"], + "selector_forbidden_columns_present": summary["selector_forbidden_columns_present"], + }, + "selected_policy": summary["selected_policy"], + "full_oot": { + "n_candidates": int(full["n_panel"]), + "n_funded": int(full["n_funded"]), + "total_allocated": float(full["total_allocated"]), + "expected_objective": float(full["expected_objective"]), + "realized_return": float(full["realized_return"]), + "weighted_default_rate": float(full["weighted_outcome"]), + "weighted_miscoverage": float(full["weighted_miscoverage"]), + "weighted_pd_point": float(full["weighted_pd_point"]), + "weighted_pd_effective": float(full["weighted_pd_effective"]), + "Gamma_CP": float(full["gamma_cp"]), + "Gamma_internalized": float(full["gamma_internalized"]), + "Gamma_residual": float(full["gamma_residual"]), + "endpoint_budget": float(full["endpoint_budget"]), + "markov_loss_threshold": float(full["markov_loss_threshold"]), + "observed_accounting_bound": float( + full["endpoint_budget"] + full["weighted_miscoverage"] + ), + "markov_tail_probability_bound": float(np.sqrt(summary["design"]["alpha"])), + }, + "point_pd_contrast": { + "realized_return": float(point["realized_return"]), + "weighted_default_rate": float(point["weighted_outcome"]), + "weighted_miscoverage": float(point["weighted_miscoverage"]), + "endpoint_budget": float(point["endpoint_budget"]), + "markov_loss_threshold": float(point["markov_loss_threshold"]), + "selected_return_cost": float(point["realized_return"] - full["realized_return"]), + "selected_return_cost_pct": float( + (point["realized_return"] - full["realized_return"]) / point["realized_return"] + ), + "selected_default_reduction": float( + point["weighted_outcome"] - full["weighted_outcome"] + ), + "selected_threshold_reduction": float( + point["markov_loss_threshold"] - full["markov_loss_threshold"] + ), + }, + "bootstrap_return_interval": { + "p025": float(return_boot["boot_p025"]), + "p975": float(return_boot["boot_p975"]), + "n_draws": int(return_boot["n_draws"]), + }, + "exact_alpha_reference_replay": exact_summary["reference_replay"], + "paper_tables": { + key: [str(path.relative_to(ROOT).as_posix()) for path in paths] + for key, paths in table_paths.items() + }, + "retired_active_claims": [ + "alpha01 intervals obtained by cross-family average-width scaling", + "8/8 approximate alpha-grid pass as a headline certificate", + "50,010-policy frontier as the active selector", + "0.345084 Markov threshold", + "capped_blended_uncertainty with delta_cap_quantile=0.975", + "OOT-outcome-selected portfolio hyperparameters", + "the exploratory 25-policy gamma=0.35, threshold-cap=0.65 challenger", + ], + "claim_boundary": summary["claim_boundary"], + } + + +def run(*, bootstrap_draws: int, bootstrap_seed: int) -> dict[str, Any]: + summary = load_json(MODEL_DIR / "calibration_selected_policy_summary.json") + exact_summary = load_json(EXACT_MODEL_DIR / "exact_alpha_grid_summary.json") + grid = pd.read_parquet(DATA_DIR / "calibration_policy_selection_grid.parquet") + evaluation = pd.read_csv(DATA_DIR / "calibration_selected_policy_oot_evaluation.csv") + allocations = pd.read_parquet( + DATA_DIR / "calibration_selected_policy_full_oot_allocations.parquet" + ) + tables = { + "alpha": build_alpha_table(exact_summary), + "selector": build_selector_table(grid, summary), + "temporal": build_temporal_table(evaluation), + "grade": build_grade_table(allocations), + "bootstrap": build_bootstrap_table( + allocations, + evaluation, + n_draws=bootstrap_draws, + seed=bootstrap_seed, + ), + "baseline": build_baseline_table(evaluation), + } + table_paths = { + key: write_table(TABLE_NAMES[key], frame, table_dir=TABLE_DIR, root=ROOT) + for key, frame in tables.items() + } + governance = build_governance( + summary, + exact_summary, + evaluation, + tables["bootstrap"], + table_paths, + ) + write_json(MODEL_DIR / GOVERNANCE_NAME, governance) + return governance + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--bootstrap-draws", type=int, default=5000) + parser.add_argument("--bootstrap-seed", type=int, default=20260709) + args = parser.parse_args() + payload = run( + bootstrap_draws=max(100, int(args.bootstrap_draws)), + bootstrap_seed=int(args.bootstrap_seed), + ) + print(json.dumps(payload["full_oot"], indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_publication_integrity.py b/scripts/check_publication_integrity.py index e04ff4d..2c64f17 100644 --- a/scripts/check_publication_integrity.py +++ b/scripts/check_publication_integrity.py @@ -22,40 +22,28 @@ class SurfaceCheck: COMMON_CLAIM_TOKENS = ( - "$184832.48", - "0.035350", - "0.162616", - "0.073584", - "0.245084", - "50010", - "27508", - "8/8", -) - -MAIN_SURFACE_REQUIRED = ( - *COMMON_CLAIM_TOKENS, - "0.345084", - "0.697056", + "$179327.59", + "0.039375", + "0.036875", + "0.176102", + "0.088051", + "0.258051", + "0.294926", + "0.574279", "196369.14", - "5.875%", - "8.305", - "43.55", - "decision certificate", - "single-submission boundary", + "8.678%", + "7.9025", ) ACTIVE_SURFACE_FORBIDDEN = ( - "five contributions", + "four contributions", "crpto v2", - "future work rather than", - "signed price is favorable", - "wins expected return", "-10.56%", "markov cap", - "0.510753", - "173314.04", - "zero deterministic violation", - "zero exact violation", + "0.345084", + "50010", + "27508", + "capped_blended_uncertainty", ) SURFACES = ( @@ -63,120 +51,96 @@ class SurfaceCheck: path=REPO / "README.md", required=( *COMMON_CLAIM_TOKENS, - "0.345084", "claim ijds activo", - "upstream congelado", - "no como el claim activo", + "q=(p+u)/2", + "grilla redonda 3x3", ), forbidden=("## champion congelado",), ), SurfaceCheck( - path=REPO / "paper" / "submission" / "README.md", + path=REPO / "paper/submission/README.md", required=( - "28-page official-template pdf", "pdflatex -> bibtex -> pdflatex -> pdflatex", "latexmk", - "body remains inside the ijds 25-page", - ), - forbidden=( - "26-page official pdf", - "26 pages total", - "27-page official-template pdf", - "27 pages total", + "official-template", ), + forbidden=ACTIVE_SURFACE_FORBIDDEN, ), SurfaceCheck( - path=REPO / "paper" / "CRPTO_ijds.qmd", + path=REPO / "paper/CRPTO_ijds.qmd", required=( - *MAIN_SURFACE_REQUIRED, - "the paper makes four contributions", - "one auditable post-hoc decision certificate", - "matched point-pd baseline", + *COMMON_CLAIM_TOKENS, + "the paper makes three contributions", + "retrospective lockbox replay", + "matched point-pd", + "q_i=(p_i+u_i)/2", ), forbidden=ACTIVE_SURFACE_FORBIDDEN, ), SurfaceCheck( - path=REPO / "paper" / "submission" / "CRPTO_ijds_submission.tex", + path=REPO / "paper/submission/CRPTO_ijds_submission.tex", required=( - *MAIN_SURFACE_REQUIRED, - "the paper makes four contributions", - "one auditable post-hoc decision certificate", - "matched point-pd baseline", + *COMMON_CLAIM_TOKENS, + "the paper makes three contributions", + "retrospective lockbox replay", + "matched point-pd", ), forbidden=ACTIVE_SURFACE_FORBIDDEN, ), SurfaceCheck( - path=REPO / "paper" / "supplement_ijds.qmd", + path=REPO / "paper/supplement_ijds.qmd", required=( *COMMON_CLAIM_TOKENS, - "decision certificate", - "single-submission boundary", - "outside the submitted claim", - "10423", - "2866", - "matched point-pd decision audit (a40)", + "a35. exact alpha replay", + "a36. calibration policy selector", + "a40. matched decision audit", + "retrospective lockbox replay", ), - forbidden=("crpto v2", "future work only", "markov cap", "0.510753"), + forbidden=ACTIVE_SURFACE_FORBIDDEN, ), SurfaceCheck( - path=REPO / "paper" / "submission" / "CLAIM_AUDIT_MATRIX.md", + path=REPO / "paper/submission/CLAIM_AUDIT_MATRIX.md", required=( - "exclude the historical lending club -10.56% field", - "a40 reports a matched lending club cost of 5.875%", - "gamma_cp = gamma_int + gamma_res", + "calibration-selected midpoint", + "a40", + "8.678%", + "7.9025", ), - forbidden=("lending club price -10.56%", "+27.03%", "markov cap"), + forbidden=ACTIVE_SURFACE_FORBIDDEN, ), SurfaceCheck( - path=REPO / "book" / "chapters" / "30-replicacion-multidataset.qmd", + path=REPO / "docs/research/active_claims_2026-07-04.md", required=( - "lending club no entra en esa serie", - "la auditoría point-pd corregida en tau=0.1715 es a40", - "frontera policy-aware a35", - "5.875%", - ), - forbidden=( - "lending club es la excepción informativa", - "la robustez nunca es económicamente catastrófica", - "+27.03%", + *COMMON_CLAIM_TOKENS, + "nine round-number candidates", + "retrospective lockbox replay", + "retired headline claims", ), + forbidden=("crpto v2", "markov cap", "+27.03%"), ), SurfaceCheck( - path=REPO / "scripts" / "generate_crpto_figures.py", - required=("stored nonrobust baseline was not a point-only comparator",), - forbidden=("lc_price", "sits below zero", "robustness adds value"), - ), - SurfaceCheck( - path=REPO / "docs" / "research" / "active_claims_2026-07-04.md", + path=REPO / "configs/crpto_publication_targets.yaml", required=( - *COMMON_CLAIM_TOKENS, - "0.345083866", - "decision certificate", + "exact 90% conformal replay", + "q=(p+u)/2", "outside the submitted claim", - "baseline semantics boundary", - "point-pd allocation earns $196369.14", - "maximum understatement was 0.241324", + "not acceptance criteria", ), - forbidden=("crpto v2", "future protocols", "markov cap", "+27.03%"), - ), - SurfaceCheck( - path=REPO / "configs" / "crpto_publication_targets.yaml", - required=("outside the submitted claim", "not acceptance criteria"), - forbidden=("future work and are not acceptance criteria", "crpto v2"), + forbidden=("crpto v2",), ), ) def _normalize(text: str) -> str: - """Normalize Markdown/LaTeX enough for robust manuscript-token checks.""" + """Normalize Markdown and LaTeX enough for robust token checks.""" lowered = text.lower() replacements = { - "\\$": "$", + r"\$": "$", "{,}": ",", - "\\_": "_", - "\\mathrm": "", - "\\gamma": "gamma", - "\\alpha": "alpha", + r"\_": "_", + r"\mathrm": "", + r"\gamma": "gamma", + r"\alpha": "alpha", "\\": "", "{": "", "}": "", diff --git a/scripts/experiments/ijds_policy_support.py b/scripts/experiments/ijds_policy_support.py new file mode 100644 index 0000000..627b7a4 --- /dev/null +++ b/scripts/experiments/ijds_policy_support.py @@ -0,0 +1,176 @@ +"""Shared exact-alpha panel loading and policy evaluation for IJDS experiments.""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) + +from scripts.validate_alpha_gamma_bound import _load_aligned_dataset # noqa: E402 +from src.models.conformal_alpha_grid import alpha_interval_columns # noqa: E402 +from src.optimization.certificate_semantics import ( # noqa: E402 + compute_funded_certificate_metrics, +) +from src.optimization.input_alignment import align_candidate_intervals # noqa: E402 +from src.optimization.policy_evaluation import ( # noqa: E402 + PolicyAllocationResult, + solve_policy_allocation, +) +from src.optimization.policy_selection import ( # noqa: E402 + LinearPolicyCandidate, + temporal_period_labels, +) +from src.utils.script_helpers import parse_percent_series, resolve_repo_artifact_path # noqa: E402 + + +def load_policy_panel(config: dict[str, Any], *, root: Path = ROOT) -> pd.DataFrame: + """Load aligned loans and the exact interval columns declared by config.""" + source = config["source"] + design = config["design"] + execution = config["execution"] + interval_path = resolve_repo_artifact_path(source["conformal_intervals_path"], root=root) + aligned = _load_aligned_dataset( + conformal_intervals_path=str(interval_path), + max_candidates=0, + random_state=int(execution.get("random_seed", 42)), + ) + exact_grid_value = str(source.get("exact_alpha_grid_path", "")).strip() + if not exact_grid_value: + raise ValueError("Active IJDS policy evaluation requires exact_alpha_grid_path.") + exact_grid_path = resolve_repo_artifact_path(exact_grid_value, root=root) + exact_alignment = align_candidate_intervals( + aligned, + pd.read_parquet(exact_grid_path), + max_candidates=0, + random_state=int(execution.get("random_seed", 42)), + ) + panel = exact_alignment.candidates.copy() + exact = exact_alignment.intervals + low_column, high_column = alpha_interval_columns(float(design["alpha"])) + required = {"y_pred", low_column, high_column} + missing = sorted(required.difference(exact.columns)) + if missing: + raise KeyError(f"Exact alpha grid is missing columns: {missing}") + + panel["_pd_point"] = exact["y_pred"].to_numpy(dtype=float) + panel["_pd_low"] = exact[low_column].to_numpy(dtype=float) + panel["_pd_high"] = exact[high_column].to_numpy(dtype=float) + panel["_outcome"] = pd.to_numeric(panel["y_true"], errors="raise").astype(float) + panel["_loan_amount"] = pd.to_numeric(panel["loan_amnt"], errors="coerce").fillna(1.0) + panel["_int_rate"] = parse_percent_series(panel["int_rate"]) + panel["_period"] = temporal_period_labels( + panel["issue_d"], + combine_years_from=int(design["combine_years_from"]), + ) + panel.attrs["exact_alpha_grid_path"] = str(exact_grid_path) + return panel + + +def solve_candidate( + frame: pd.DataFrame, + candidate: LinearPolicyCandidate, + *, + config: dict[str, Any], + robust: bool = True, +) -> PolicyAllocationResult: + """Solve one declared policy on one aligned panel.""" + design = config["design"] + execution = config["execution"] + return solve_policy_allocation( + loans=frame, + pd_point=frame["_pd_point"].to_numpy(dtype=float), + pd_low=frame["_pd_low"].to_numpy(dtype=float), + pd_high=frame["_pd_high"].to_numpy(dtype=float), + lgd=np.full(len(frame), float(design["lgd"]), dtype=float), + int_rates=frame["_int_rate"].to_numpy(dtype=float), + total_budget=float(design["budget"]), + max_concentration=float(design["max_concentration"]), + risk_tolerance=float(candidate.risk_tolerance), + robust=robust, + uncertainty_aversion=float(candidate.uncertainty_aversion) if robust else 0.0, + min_budget_utilization=float(candidate.min_budget_utilization), + pd_cap_slack_penalty=float(candidate.pd_cap_slack_penalty), + policy_mode=candidate.policy_mode, + gamma=float(candidate.gamma), + delta_cap_quantile=float(candidate.delta_cap_quantile), + tail_focus_quantile=float(candidate.tail_focus_quantile), + time_limit=int(execution["time_limit"]), + threads=int(execution["threads"]), + solver_backend=str(execution["solver_backend"]), + random_seed=int(execution.get("random_seed", 42)), + ) + + +def evaluate_candidate( + frame: pd.DataFrame, + candidate: LinearPolicyCandidate, + *, + config: dict[str, Any], + robust: bool, + period: str, +) -> tuple[dict[str, Any], PolicyAllocationResult]: + """Solve and score one policy on one evaluation period.""" + result = solve_candidate(frame, candidate, config=config, robust=robust) + exposure = result.allocation * frame["_loan_amount"].to_numpy(dtype=float) + total_allocated = float(exposure.sum()) + if total_allocated <= 0.0: + raise RuntimeError(f"Policy {candidate.candidate_id} allocated no capital in {period}.") + weights = exposure / total_allocated + outcomes = frame["_outcome"].to_numpy(dtype=float) + alpha = float(config["design"]["alpha"]) + certificate = compute_funded_certificate_metrics( + weights, + outcomes=outcomes, + pd_point=frame["_pd_point"].to_numpy(dtype=float), + pd_high=frame["_pd_high"].to_numpy(dtype=float), + pd_effective=result.effective_pd, + alpha=alpha, + risk_tolerance=float(candidate.risk_tolerance), + pd_cap_slack=float(result.solution.get("pd_cap_slack", 0.0)), + ) + funded = result.allocation > 0.01 + rates = frame["_int_rate"].to_numpy(dtype=float) + lgd = float(config["design"]["lgd"]) + realized_return = float( + np.sum( + np.where( + funded & (outcomes.astype(int) == 1), + -lgd * exposure, + np.where(funded, rates * exposure, 0.0), + ) + ) + ) + record: dict[str, Any] = { + "period": period, + **candidate.to_record(), + "solver_status": str(result.solution.get("solver_status", "unknown")), + "objective_risk_mode": result.objective_risk_mode, + "expected_objective": float(result.solution.get("objective_value", float("nan"))), + "n_panel": int(len(frame)), + "n_funded": int(certificate.n_funded), + "total_allocated": total_allocated, + "realized_return": realized_return, + "weighted_outcome": certificate.weighted_outcome, + "weighted_miscoverage": certificate.weighted_miscoverage, + "weighted_pd_point": certificate.weighted_pd_point, + "weighted_pd_effective": certificate.weighted_pd_effective, + "gamma_cp": certificate.gamma_cp, + "gamma_internalized": certificate.gamma_internalized, + "gamma_residual": certificate.gamma_residual, + "endpoint_budget": certificate.endpoint_budget, + "markov_loss_threshold": certificate.markov_loss_threshold, + "realized_risk_tolerance_excess": certificate.realized_risk_tolerance_excess, + "screen_V_leq_sqrt_alpha": bool( + certificate.weighted_miscoverage <= certificate.sqrt_alpha + 1e-12 + ), + "screen_risk_excess_leq_alpha": bool( + certificate.realized_risk_tolerance_excess <= alpha + 1e-12 + ), + } + return record, result diff --git a/scripts/experiments/run_ijds_calibration_selected_policy_challenger.py b/scripts/experiments/run_ijds_calibration_selected_policy_challenger.py new file mode 100644 index 0000000..5d8c845 --- /dev/null +++ b/scripts/experiments/run_ijds_calibration_selected_policy_challenger.py @@ -0,0 +1,440 @@ +"""Select a simple CRPTO policy without consulting OOT outcomes.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import pickle +import subprocess +import sys +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd +import yaml +from loguru import logger + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) + +from scripts.experiments.ijds_policy_support import ( # noqa: E402 + evaluate_candidate, + load_policy_panel, + solve_candidate, +) +from scripts.generate_conformal_intervals import ( # noqa: E402 + _build_probability_lookups, + _build_tuning_split, + _load_conformal_inputs, +) +from src.models.conformal_alpha_grid import ( # noqa: E402 + FrozenConformalRecipe, + compute_exact_alpha_intervals, +) +from src.optimization.policy_evaluation import PolicyAllocationResult # noqa: E402 +from src.optimization.policy_selection import ( # noqa: E402 + FORBIDDEN_POLICY_SELECTION_COLUMNS, + LinearPolicyCandidate, + build_linear_policy_grid, + select_policy_result_ex_ante, +) +from src.utils.script_helpers import ( # noqa: E402 + parse_percent_series, + resolve_repo_artifact_path, + write_json, +) + +DEFAULT_CONFIG = ( + ROOT / "configs/experiments/champion_reopen_ijds_calibration_selected_simple90_v6.yaml" +) +FORBIDDEN_SELECTOR_COLUMNS = FORBIDDEN_POLICY_SELECTION_COLUMNS + + +def _load_config(path: Path) -> dict[str, Any]: + payload = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise TypeError("Experiment config must contain a mapping.") + return payload + + +def _load_pickle_mapping(path: Path) -> dict[str, Any]: + with path.open("rb") as handle: + payload = pickle.load(handle) + if not isinstance(payload, dict): + raise TypeError("Frozen conformal results must contain a mapping.") + return payload + + +def _experiment_paths(run_tag: str) -> tuple[Path, Path]: + data_dir = ROOT / "data/processed/experiments/champion_reopen" / run_tag / "portfolio" + model_dir = ROOT / "models/experiments/champion_reopen" / run_tag / "portfolio" + data_dir.mkdir(parents=True, exist_ok=True) + model_dir.mkdir(parents=True, exist_ok=True) + return data_dir, model_dir + + +def _load_calibration_selection_panel( + config: dict[str, Any], +) -> tuple[pd.DataFrame, FrozenConformalRecipe, dict[str, Any]]: + source = config["source"] + design = config["design"] + os.environ["UPSTREAM_CANONICAL_RUN_TAG"] = str(source["upstream_canonical_run_tag"]) + results_path = resolve_repo_artifact_path(source["conformal_results_path"], root=ROOT) + payload = _load_pickle_mapping(results_path) + recipe = FrozenConformalRecipe.from_results_payload(payload) + inputs = _load_conformal_inputs( + calibration_fraction=recipe.calibration_fraction, + calibrator_override_path=(str(payload.get("calibrator_override_path", "")).strip() or None), + ) + split = _build_tuning_split( + cal_df=inputs.cal_df, + test_df=inputs.test_df, + X_cal=inputs.X_cal, + y_cal=inputs.y_cal, + group_cal_base=inputs.group_cal_base, + y_prob_cal_raw=inputs.y_prob_cal_raw, + tuning_holdout_ratio=recipe.tuning_holdout_ratio, + tuning_random_state=recipe.tuning_random_state, + ) + probability_fit, probability_tune, _probability_test = _build_probability_lookups( + inputs, + split, + ) + intervals = compute_exact_alpha_intervals( + recipe=recipe, + target_alpha=float(design["alpha"]), + y_cal=split.y_cal_fit, + interval_probability_cal=probability_fit["calibrated"], + interval_probability_eval=probability_tune["calibrated"], + partition_probability_cal=probability_fit[recipe.partition_probability_source], + partition_probability_eval=probability_tune[recipe.partition_probability_source], + base_groups_cal=split.group_cal_fit_base, + base_groups_eval=split.group_tune_base, + issue_dates_eval=split.issue_tune, + ) + panel = inputs.cal_df.iloc[split.idx_cal_tune].reset_index(drop=True).copy() + panel["_pd_point"] = intervals.point + panel["_pd_low"] = intervals.low + panel["_pd_high"] = intervals.high + panel["_loan_amount"] = pd.to_numeric(panel["loan_amnt"], errors="coerce").fillna(1.0) + panel["_int_rate"] = parse_percent_series(panel["int_rate"]) + metadata = { + "conformal_results_path": str(results_path.relative_to(ROOT)), + "calibration_fit_rows": int(len(split.idx_cal_fit)), + "calibration_selection_rows": int(len(split.idx_cal_tune)), + "calibration_selection_start": str(pd.to_datetime(split.issue_tune).min().date()), + "calibration_selection_end": str(pd.to_datetime(split.issue_tune).max().date()), + "target_alpha": intervals.target_alpha, + "used_alpha": intervals.used_alpha, + "partition": recipe.partition, + } + return panel, recipe, metadata + + +def _measure_ex_ante_solution( + frame: pd.DataFrame, + candidate: LinearPolicyCandidate, + result: PolicyAllocationResult, + *, + alpha: float, +) -> dict[str, Any]: + exposure = result.allocation * frame["_loan_amount"].to_numpy(dtype=float) + total_allocated = float(exposure.sum()) + if total_allocated <= 0.0: + raise RuntimeError(f"Policy {candidate.candidate_id} allocated no calibration capital.") + weights = exposure / total_allocated + point = frame["_pd_point"].to_numpy(dtype=float) + high = frame["_pd_high"].to_numpy(dtype=float) + weighted_point = float(np.sum(weights * point)) + weighted_effective = float(np.sum(weights * result.effective_pd)) + endpoint_budget = float(np.sum(weights * high)) + return { + **candidate.to_record(), + "solver_status": str(result.solution.get("solver_status", "unknown")), + "objective_risk_mode": result.objective_risk_mode, + "expected_objective": float(result.solution["objective_value"]), + "n_panel": int(len(frame)), + "n_funded": int(np.count_nonzero(result.allocation > 0.01)), + "total_allocated": total_allocated, + "weighted_pd_point": weighted_point, + "weighted_pd_effective": weighted_effective, + "gamma_cp": float(np.sum(weights * (high - point))), + "gamma_internalized": float(np.sum(weights * (result.effective_pd - point))), + "gamma_residual": float(np.sum(weights * (high - result.effective_pd))), + "endpoint_budget": endpoint_budget, + "markov_loss_threshold": endpoint_budget + float(np.sqrt(alpha)), + "effective_pd_cap_slack": float(candidate.risk_tolerance - weighted_effective), + } + + +def _run_calibration_grid( + panel: pd.DataFrame, + candidates: list[LinearPolicyCandidate], + *, + config: dict[str, Any], + output_path: Path, +) -> pd.DataFrame: + rows: list[dict[str, Any]] = [] + for candidate in candidates: + logger.info("Calibration selector evaluating {}", candidate.candidate_id) + result = solve_candidate(panel, candidate, config=config) + rows.append( + _measure_ex_ante_solution( + panel, + candidate, + result, + alpha=float(config["design"]["alpha"]), + ) + ) + output = pd.DataFrame(rows) + if FORBIDDEN_SELECTOR_COLUMNS.intersection(output.columns): + raise AssertionError("Calibration selector artifact contains outcome-derived columns.") + output.to_parquet(output_path, index=False) + return output + + +def _match_candidate( + candidates: list[LinearPolicyCandidate], + settings: dict[str, Any], +) -> LinearPolicyCandidate: + matches = [ + candidate + for candidate in candidates + if np.isclose(candidate.risk_tolerance, float(settings["risk_tolerance"])) + and np.isclose(candidate.gamma, float(settings["gamma"])) + and np.isclose(candidate.uncertainty_aversion, float(settings["uncertainty_aversion"])) + ] + if len(matches) != 1: + raise ValueError(f"Policy settings must match exactly one candidate, got {len(matches)}.") + return matches[0] + + +def _evaluate_fixed_policies( + panel: pd.DataFrame, + selected: LinearPolicyCandidate, + incumbent: LinearPolicyCandidate, + *, + config: dict[str, Any], +) -> tuple[pd.DataFrame, pd.DataFrame]: + periods = ["full_oot", *list(config["design"]["period_order"])] + rows: list[dict[str, Any]] = [] + allocation_frames: list[pd.DataFrame] = [] + for period in periods: + frame = ( + panel.reset_index(drop=True) + if period == "full_oot" + else panel.loc[panel["_period"].astype(str).eq(period)].reset_index(drop=True) + ) + point = LinearPolicyCandidate( + candidate_id="point-pd", + risk_tolerance=selected.risk_tolerance, + gamma=0.0, + uncertainty_aversion=0.0, + policy_mode="point_estimate", + ) + for role, candidate, robust in ( + ("calibration_selected", selected, True), + ("incumbent_linear", incumbent, True), + ("point_pd_matched_tau", point, False), + ): + record, result = evaluate_candidate( + frame, + candidate, + config=config, + robust=robust, + period=period, + ) + rows.append({"role": role, **record}) + if period == "full_oot": + allocation_frames.append( + _funded_allocation_frame( + frame, + result, + role=role, + lgd=float(config["design"]["lgd"]), + ) + ) + return pd.DataFrame(rows), pd.concat(allocation_frames, ignore_index=True) + + +def _funded_allocation_frame( + frame: pd.DataFrame, + result: PolicyAllocationResult, + *, + role: str, + lgd: float, +) -> pd.DataFrame: + allocation = result.allocation + funded = allocation > 0.01 + selected = frame.loc[funded].reset_index(drop=True).copy() + selected_allocation = allocation[funded] + exposure = selected_allocation * selected["_loan_amount"].to_numpy(dtype=float) + total_exposure = float(exposure.sum()) + outcome = selected["_outcome"].to_numpy(dtype=float) + rates = selected["_int_rate"].to_numpy(dtype=float) + point = selected["_pd_point"].to_numpy(dtype=float) + high = selected["_pd_high"].to_numpy(dtype=float) + effective = result.effective_pd[funded] + if "sub_grade" in selected.columns: + loan_grade = selected["sub_grade"].astype(str).str[:1] + elif "int_rate_bucket__grade" in selected.columns: + loan_grade = selected["int_rate_bucket__grade"].astype(str).str.rsplit("__").str[-1] + else: + loan_grade = selected["grade"].astype(str) + output = pd.DataFrame( + { + "role": role, + "id": selected["id"].astype(str), + "issue_d": selected["issue_d"], + "grade": loan_grade, + "conformal_group": selected["grade"].astype(str), + "loan_amnt": selected["_loan_amount"].to_numpy(dtype=float), + "int_rate": rates, + "outcome": outcome, + "pd_point": point, + "pd_low": selected["_pd_low"].to_numpy(dtype=float), + "pd_high": high, + "pd_effective": effective, + "allocation": selected_allocation, + "funded_exposure": exposure, + "funded_weight": exposure / total_exposure, + "miscoverage": (outcome > high).astype(int), + "expected_return_contribution": exposure * (rates - point * float(lgd)), + "realized_return_contribution": np.where( + outcome.astype(int) == 1, + -float(lgd) * exposure, + rates * exposure, + ), + } + ) + return output + + +def _contrast_payload(evaluation: pd.DataFrame) -> dict[str, Any]: + output: dict[str, Any] = {} + for period in ("full_oot", "2020+"): + period_rows = evaluation.loc[evaluation["period"].eq(period)].set_index("role") + selected = period_rows.loc["calibration_selected"] + point = period_rows.loc["point_pd_matched_tau"] + incumbent = period_rows.loc["incumbent_linear"] + output[period] = { + "selected_realized_return": float(selected["realized_return"]), + "selected_weighted_outcome": float(selected["weighted_outcome"]), + "selected_markov_threshold": float(selected["markov_loss_threshold"]), + "return_cost_vs_point": float(point["realized_return"] - selected["realized_return"]), + "default_delta_vs_point": float( + selected["weighted_outcome"] - point["weighted_outcome"] + ), + "threshold_delta_vs_point": float( + selected["markov_loss_threshold"] - point["markov_loss_threshold"] + ), + "return_delta_vs_incumbent": float( + selected["realized_return"] - incumbent["realized_return"] + ), + "default_delta_vs_incumbent": float( + selected["weighted_outcome"] - incumbent["weighted_outcome"] + ), + } + return output + + +def _git_commit() -> str: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=ROOT, + capture_output=True, + text=True, + check=False, + ) + return result.stdout.strip() if result.returncode == 0 else "unknown" + + +def run(config_path: Path) -> dict[str, Any]: + config = _load_config(config_path) + run_tag = str(config["run_tag"]) + data_dir, model_dir = _experiment_paths(run_tag) + calibration_panel, recipe, calibration_metadata = _load_calibration_selection_panel(config) + grid_config = config["policy_grid"] + candidates = build_linear_policy_grid( + risk_tolerances=[float(value) for value in grid_config["risk_tolerances"]], + gammas=[float(value) for value in grid_config["gammas"]], + uncertainty_aversions=[float(value) for value in grid_config["uncertainty_aversions"]], + ) + selection_results = _run_calibration_grid( + calibration_panel, + candidates, + config=config, + output_path=data_dir / "calibration_policy_selection_grid.parquet", + ) + selected_row, selection_audit = select_policy_result_ex_ante( + selection_results, + markov_threshold_cap=float(config["design"]["markov_threshold_cap"]), + budget=float(config["design"]["budget"]), + min_budget_utilization=float(config["design"]["selection_min_budget_utilization"]), + ) + candidate_lookup = {candidate.candidate_id: candidate for candidate in candidates} + selected = candidate_lookup[str(selected_row["candidate_id"])] + incumbent = _match_candidate(candidates, config["incumbent_policy"]) + oot_panel = load_policy_panel(config) + evaluation, allocations = _evaluate_fixed_policies( + oot_panel, + selected, + incumbent, + config=config, + ) + evaluation_path = data_dir / "calibration_selected_policy_oot_evaluation.csv" + allocation_path = data_dir / "calibration_selected_policy_full_oot_allocations.parquet" + evaluation.to_csv(evaluation_path, index=False) + allocations.to_parquet(allocation_path, index=False) + payload: dict[str, Any] = { + "schema_version": str(config["schema_version"]), + "generated_at_utc": datetime.now(tz=UTC).isoformat(), + "run_tag": run_tag, + "source_commit": _git_commit(), + "config_path": str(config_path.relative_to(ROOT)), + "config_sha256": hashlib.sha256(config_path.read_bytes()).hexdigest(), + "design": config["design"], + "calibration_metadata": calibration_metadata, + "recipe": { + "partition": recipe.partition, + "partition_probability_source": recipe.partition_probability_source, + "reference_target_alpha": recipe.reference_target_alpha, + "reference_used_alpha": recipe.reference_used_alpha, + }, + "grid_size": int(len(candidates)), + "selector_columns": list(selection_results.columns), + "selector_forbidden_columns_present": sorted( + FORBIDDEN_SELECTOR_COLUMNS.intersection(selection_results.columns) + ), + "selection_audit": selection_audit, + "selected_policy": selected.to_record(), + "selected_calibration_metrics": selected_row.to_dict(), + "incumbent_policy": incumbent.to_record(), + "evaluation_path": str(evaluation_path.relative_to(ROOT)), + "allocation_path": str(allocation_path.relative_to(ROOT)), + "contrasts": _contrast_payload(evaluation), + "claim_boundary": str(config["claim_boundary"]), + } + summary_path = model_dir / "calibration_selected_policy_summary.json" + write_json(summary_path, payload) + logger.info("Selected calibration-only policy: {}", selected.candidate_id) + logger.info("Wrote calibration-only policy summary to {}", summary_path) + return payload + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG) + args = parser.parse_args() + config_path = args.config if args.config.is_absolute() else ROOT / args.config + payload = run(config_path.resolve()) + print(json.dumps(payload["contrasts"], indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/experiments/run_ijds_exact_alpha_grid_challenger.py b/scripts/experiments/run_ijds_exact_alpha_grid_challenger.py new file mode 100644 index 0000000..5b71902 --- /dev/null +++ b/scripts/experiments/run_ijds_exact_alpha_grid_challenger.py @@ -0,0 +1,291 @@ +"""Recompute the IJDS alpha grid from the frozen conformal recipe.""" + +from __future__ import annotations + +import argparse +import hashlib +import os +import pickle +import sys +from dataclasses import asdict +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd +import yaml +from loguru import logger + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) + +from scripts.generate_conformal_intervals import ( # noqa: E402 + _build_probability_lookups, + _build_tuning_split, + _load_conformal_inputs, +) +from src.models.conformal import conditional_coverage_by_group # noqa: E402 +from src.models.conformal_alpha_grid import ( # noqa: E402 + ExactAlphaIntervals, + FrozenConformalRecipe, + alpha_interval_columns, + compute_exact_alpha_intervals, +) +from src.utils.script_helpers import resolve_repo_artifact_path, write_json # noqa: E402 + +DEFAULT_CONFIG = ROOT / "configs/experiments/champion_reopen_ijds_exact_alpha_grid_v1.yaml" + + +def _utc_now() -> str: + return datetime.now(tz=UTC).isoformat() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _load_config(path: Path) -> dict[str, Any]: + payload = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise TypeError("Experiment config must contain a mapping.") + return payload + + +def _experiment_paths(run_tag: str) -> tuple[Path, Path]: + data_dir = ROOT / "data/processed/experiments/champion_reopen" / run_tag / "conformal" + model_dir = ROOT / "models/experiments/champion_reopen" / run_tag / "conformal" + data_dir.mkdir(parents=True, exist_ok=True) + model_dir.mkdir(parents=True, exist_ok=True) + return data_dir, model_dir + + +def _load_results_payload(path: Path) -> dict[str, Any]: + with path.open("rb") as handle: + payload = pickle.load(handle) + if not isinstance(payload, dict): + raise TypeError("Frozen conformal results must contain a mapping.") + return payload + + +def _coverage_summary( + result: ExactAlphaIntervals, + *, + y_true: np.ndarray, + grades: pd.Series, +) -> dict[str, Any]: + covered = (y_true >= result.low) & (y_true <= result.high) + partition_metrics = conditional_coverage_by_group( + y_true, + np.column_stack([result.low, result.high]), + result.partition_labels, + ) + grade_metrics = conditional_coverage_by_group( + y_true, + np.column_stack([result.low, result.high]), + grades, + ) + return { + "target_alpha": result.target_alpha, + "used_alpha": result.used_alpha, + "target_coverage": 1.0 - result.target_alpha, + "empirical_coverage": float(covered.mean()), + "coverage_gap": float(covered.mean() - (1.0 - result.target_alpha)), + "avg_width": float(np.mean(result.high - result.low)), + "median_width": float(np.median(result.high - result.low)), + "min_partition_coverage": float(partition_metrics["coverage"].min()), + "min_grade_coverage": float(grade_metrics["coverage"].min()), + "high_endpoint_mean": float(result.high.mean()), + "high_endpoint_min": float(result.high.min()), + "high_endpoint_p01": float(np.quantile(result.high, 0.01)), + "high_endpoint_p10": float(np.quantile(result.high, 0.10)), + "high_endpoint_at_one_rate": float(np.mean(result.high >= 1.0 - 1e-12)), + "partition_count": int(result.partition_labels.nunique()), + "group_quantiles": { + str(key): float(value) + for key, value in result.diagnostics.get("group_quantiles", {}).items() + }, + } + + +def _base_grid_frame(source_intervals: pd.DataFrame) -> pd.DataFrame: + preferred = [ + "_row_number", + "id", + "y_true", + "grade", + "loan_amnt", + "temporal_segment", + ] + columns = [column for column in preferred if column in source_intervals.columns] + return source_intervals.loc[:, columns].copy() + + +def _add_alpha_result(frame: pd.DataFrame, result: ExactAlphaIntervals) -> None: + low_column, high_column = alpha_interval_columns(result.target_alpha) + frame[low_column] = result.low + frame[high_column] = result.high + + +def _replay_differences( + result: ExactAlphaIntervals, + source_intervals: pd.DataFrame, +) -> dict[str, float]: + return { + "point_max_abs": float( + np.max(np.abs(result.point - source_intervals["y_pred"].to_numpy(dtype=float))) + ), + "low_max_abs": float( + np.max(np.abs(result.low - source_intervals["pd_low_90"].to_numpy(dtype=float))) + ), + "high_max_abs": float( + np.max(np.abs(result.high - source_intervals["pd_high_90"].to_numpy(dtype=float))) + ), + } + + +def run(config_path: Path) -> dict[str, Any]: + config = _load_config(config_path) + source = config["source"] + design = config["design"] + run_tag = str(config["run_tag"]) + results_path = resolve_repo_artifact_path(source["conformal_results_path"], root=ROOT) + intervals_path = resolve_repo_artifact_path(source["conformal_intervals_path"], root=ROOT) + os.environ["UPSTREAM_CANONICAL_RUN_TAG"] = str(source["upstream_canonical_run_tag"]) + + results_payload = _load_results_payload(results_path) + recipe = FrozenConformalRecipe.from_results_payload(results_payload) + inputs = _load_conformal_inputs( + calibration_fraction=recipe.calibration_fraction, + calibrator_override_path=( + str(results_payload.get("calibrator_override_path", "")).strip() or None + ), + ) + split = _build_tuning_split( + cal_df=inputs.cal_df, + test_df=inputs.test_df, + X_cal=inputs.X_cal, + y_cal=inputs.y_cal, + group_cal_base=inputs.group_cal_base, + y_prob_cal_raw=inputs.y_prob_cal_raw, + tuning_holdout_ratio=recipe.tuning_holdout_ratio, + tuning_random_state=recipe.tuning_random_state, + ) + probability_fit, _probability_tune, probability_test = _build_probability_lookups( + inputs, + split, + ) + source_intervals = pd.read_parquet(intervals_path) + if len(source_intervals) != len(inputs.y_test): + raise ValueError("Source conformal intervals and evaluation rows differ in length.") + + grid = _base_grid_frame(source_intervals) + grid["y_pred"] = np.nan + alpha_summaries: list[dict[str, Any]] = [] + results: dict[float, ExactAlphaIntervals] = {} + for alpha in [float(value) for value in design["alpha_grid"]]: + result = compute_exact_alpha_intervals( + recipe=recipe, + target_alpha=alpha, + y_cal=split.y_cal_fit, + interval_probability_cal=probability_fit["calibrated"], + interval_probability_eval=probability_test["calibrated"], + partition_probability_cal=probability_fit[recipe.partition_probability_source], + partition_probability_eval=probability_test[recipe.partition_probability_source], + base_groups_cal=split.group_cal_fit_base, + base_groups_eval=inputs.group_test_base, + issue_dates_eval=split.issue_test, + ) + results[alpha] = result + if grid["y_pred"].isna().all(): + grid["y_pred"] = result.point + elif not np.array_equal(grid["y_pred"].to_numpy(dtype=float), result.point): + raise AssertionError("Point predictions changed across alpha levels.") + _add_alpha_result(grid, result) + summary = _coverage_summary( + result, + y_true=inputs.y_test.to_numpy(dtype=float), + grades=inputs.group_test_base.reset_index(drop=True), + ) + alpha_summaries.append(summary) + logger.info( + "Exact alpha={:.3f} (used={:.4f}): coverage={:.4f}, width={:.4f}, high=1 rate={:.2%}", + alpha, + result.used_alpha, + summary["empirical_coverage"], + summary["avg_width"], + summary["high_endpoint_at_one_rate"], + ) + + reference_alpha = recipe.reference_target_alpha + reference_result = next( + (result for alpha, result in results.items() if np.isclose(alpha, reference_alpha)), + None, + ) + if reference_result is None: + raise ValueError("Alpha grid must contain the recipe reference target alpha.") + replay = _replay_differences(reference_result, source_intervals) + tolerance = float(design["replay_tolerance"]) + replay["tolerance"] = tolerance + replay["pass"] = bool(max(replay.values()) <= tolerance) + if not replay["pass"]: + raise AssertionError(f"Frozen 90% interval replay drifted: {replay}") + + data_dir, model_dir = _experiment_paths(run_tag) + grid_path = data_dir / "exact_alpha_grid.parquet" + summary_path = model_dir / "exact_alpha_grid_summary.json" + grid.to_parquet(grid_path, index=False) + summary_payload: dict[str, Any] = { + "schema_version": str(config["schema_version"]), + "generated_at_utc": _utc_now(), + "run_tag": run_tag, + "source_commit": _git_commit(), + "config_path": str(config_path.relative_to(ROOT)), + "config_sha256": _sha256(config_path), + "source": { + **source, + "conformal_results_sha256": _sha256(results_path), + "conformal_intervals_sha256": _sha256(intervals_path), + }, + "recipe": asdict(recipe), + "alpha_mapping": str(design["alpha_mapping"]), + "reference_replay": replay, + "alpha_summaries": alpha_summaries, + "grid_path": str(grid_path.relative_to(ROOT)), + "grid_rows": int(len(grid)), + "claim_boundary": str(config["claim_boundary"]), + } + write_json(summary_path, summary_payload) + logger.info("Wrote exact alpha grid to {}", grid_path) + logger.info("Wrote exact alpha summary to {}", summary_path) + return summary_payload + + +def _git_commit() -> str: + import subprocess + + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=ROOT, + capture_output=True, + text=True, + check=False, + ) + return result.stdout.strip() if result.returncode == 0 else "unknown" + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG) + args = parser.parse_args() + config_path = args.config if args.config.is_absolute() else ROOT / args.config + run(config_path.resolve()) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/optimize_portfolio_tradeoff.py b/scripts/optimize_portfolio_tradeoff.py index 7ca951c..8a2df6b 100644 --- a/scripts/optimize_portfolio_tradeoff.py +++ b/scripts/optimize_portfolio_tradeoff.py @@ -23,11 +23,7 @@ from src.models.conformal_artifacts import load_conformal_intervals from src.optimization.input_alignment import align_candidate_intervals -from src.optimization.portfolio_model import ( - compute_effective_pd, - optimize_portfolio_allocation, - solution_allocation_vector, -) +from src.optimization.policy_evaluation import solve_policy_allocation from src.utils.artifact_metadata import resolve_run_tag from src.utils.pipeline_runtime import ( atomic_write_json, @@ -287,41 +283,7 @@ def _solve_single( cuopt_presolve: int | None = 1, cuopt_parameters: dict[str, Any] | None = None, ) -> tuple[dict[str, float | int | str], np.ndarray]: - effective_policy_mode = str(policy_mode) if robust else "point_estimate" - effective_gamma = float(gamma) if robust else 0.0 - effective_delta_cap = float(delta_cap_quantile) if robust else 1.0 - effective_tail_focus = float(tail_focus_quantile) if robust else 1.0 - segment_labels: np.ndarray | None = None - if effective_policy_mode in { - "segment_tail_blended_uncertainty", - "segment_relative_tail_blended_uncertainty", - }: - grade = ( - loans["grade"].fillna("unknown").astype(str) - if "grade" in loans.columns - else pd.Series(["unknown"] * len(loans)) - ) - term = ( - loans["term"].fillna("unknown").astype(str) - if "term" in loans.columns - else pd.Series(["unknown"] * len(loans)) - ) - verification = ( - loans["verification_status"].fillna("unknown").astype(str) - if "verification_status" in loans.columns - else pd.Series(["unknown"] * len(loans)) - ) - segment_labels = (grade + "|" + term + "|" + verification).to_numpy(dtype=object) - pd_constraint = compute_effective_pd( - pd_point=pd_point, - pd_high=pd_high, - policy_mode=effective_policy_mode, - gamma=effective_gamma, - delta_cap_quantile=effective_delta_cap, - tail_focus_quantile=effective_tail_focus, - segment_labels=segment_labels, - ) - solution = optimize_portfolio_allocation( + result = solve_policy_allocation( loans=loans, pd_point=pd_point, pd_low=pd_low, @@ -330,12 +292,15 @@ def _solve_single( int_rates=int_rates, total_budget=total_budget, max_concentration=max_concentration, - max_portfolio_pd=risk_tolerance, + risk_tolerance=risk_tolerance, robust=robust, uncertainty_aversion=uncertainty_aversion, min_budget_utilization=min_budget_utilization, pd_cap_slack_penalty=pd_cap_slack_penalty, - pd_constraint_override=pd_constraint, + policy_mode=policy_mode, + gamma=gamma, + delta_cap_quantile=delta_cap_quantile, + tail_focus_quantile=tail_focus_quantile, time_limit=time_limit, threads=threads, solver_backend=solver_backend, @@ -343,9 +308,9 @@ def _solve_single( cuopt_presolve=cuopt_presolve, cuopt_parameters=cuopt_parameters, ) - + solution = result.solution n = len(loans) - allocation = solution_allocation_vector(solution, n) + allocation = result.allocation loan_amounts = ( loans["loan_amnt"].to_numpy(dtype=float) if "loan_amnt" in loans.columns @@ -374,10 +339,10 @@ def _solve_single( return { "solver_status": str(solution["solver_status"]), "solver_backend": str(solver_backend), - "policy_mode": effective_policy_mode, - "gamma": effective_gamma, - "delta_cap_quantile": effective_delta_cap, - "tail_focus_quantile": effective_tail_focus, + "policy_mode": result.policy_mode.value, + "gamma": result.gamma, + "delta_cap_quantile": result.delta_cap_quantile, + "tail_focus_quantile": result.tail_focus_quantile, "objective_value": float(solution["objective_value"]), "n_funded": int(solution["n_funded"]), "total_allocated": total_allocated, diff --git a/scripts/run_ty_advisory.py b/scripts/run_ty_advisory.py index 546c592..ab6736c 100644 --- a/scripts/run_ty_advisory.py +++ b/scripts/run_ty_advisory.py @@ -1,4 +1,4 @@ -"""Run ty as a fast, non-blocking advisory checker for CRPTO.""" +"""Run pinned ty as an advisory or blocking checker for CRPTO.""" from __future__ import annotations @@ -23,6 +23,11 @@ "scripts/train_pd_model.py", "src/optimization/cuopt_adapter.py", } +ACTIVE_EXPERIMENT_FILES = { + "scripts/experiments/ijds_policy_support.py", + "scripts/experiments/run_ijds_calibration_selected_policy_challenger.py", + "scripts/experiments/run_ijds_exact_alpha_grid_challenger.py", +} SUMMARY_RE = re.compile(r"^Found \d+ diagnostics", flags=re.MULTILINE) @@ -38,7 +43,9 @@ def iter_python_files(*, scope: str) -> list[str]: rel = _relative_posix(path) parts = rel.split("/") if scope == "active": - if parts[:2] in (["scripts", "archive"], ["scripts", "experiments"]): + if parts[:2] == ["scripts", "archive"]: + continue + if parts[:2] == ["scripts", "experiments"] and rel not in (ACTIVE_EXPERIMENT_FILES): continue if parts[:2] == ["scripts", "search"] and path.name.startswith("run_"): continue diff --git a/scripts/simulate_ab_test.py b/scripts/simulate_ab_test.py index ff1aa0e..7ed3a8f 100644 --- a/scripts/simulate_ab_test.py +++ b/scripts/simulate_ab_test.py @@ -30,10 +30,7 @@ from loguru import logger from src.evaluation.ab_testing import ab_summary, compare_strategies -from src.optimization.portfolio_model import ( - compute_effective_pd, - optimize_portfolio_allocation, -) +from src.optimization.policy_evaluation import solve_policy_allocation from src.utils.artifact_metadata import build_artifact_metadata, resolve_run_tag from src.utils.script_helpers import artifact_path as _artifact_path @@ -510,60 +507,27 @@ def _run_strategy( robust_policy: dict[str, Any] | None = None, ) -> tuple[dict, np.ndarray]: pd_point = np.asarray(common["pd_point"], dtype=float) - pd_high = np.asarray(common["pd_high"], dtype=float) - if robust: - policy = robust_policy or {} - loans = cast(pd.DataFrame, common["loans"]) - segment_labels: np.ndarray | None = None - if str(policy.get("policy_mode", "hard_worst_case")) in { - "segment_tail_blended_uncertainty", - "segment_relative_tail_blended_uncertainty", - }: - grade = ( - loans["grade"].fillna("unknown").astype(str) - if "grade" in loans.columns - else pd.Series(["unknown"] * len(loans)) - ) - term = ( - loans["term"].fillna("unknown").astype(str) - if "term" in loans.columns - else pd.Series(["unknown"] * len(loans)) - ) - verification = ( - loans["verification_status"].fillna("unknown").astype(str) - if "verification_status" in loans.columns - else pd.Series(["unknown"] * len(loans)) - ) - segment_labels = (grade + "|" + term + "|" + verification).to_numpy(dtype=object) - effective_pd = compute_effective_pd( - pd_point=pd_point, - pd_high=pd_high, - policy_mode=str(policy.get("policy_mode", "hard_worst_case")), - gamma=float(policy.get("gamma", 1.0)), - delta_cap_quantile=float(policy.get("delta_cap_quantile", 1.0)), - tail_focus_quantile=float(policy.get("tail_focus_quantile", 1.0)), - segment_labels=segment_labels, - ) - solution = optimize_portfolio_allocation( - robust=True, - uncertainty_aversion=float(policy.get("uncertainty_aversion", 0.0)), - min_budget_utilization=float(policy.get("min_budget_utilization", 0.0)), - pd_cap_slack_penalty=float(policy.get("pd_cap_slack_penalty", 0.0)), - pd_constraint_override=effective_pd, - total_budget=total_budget, - max_portfolio_pd=max_portfolio_pd, - solver_backend=solver_backend, - **common, - ) - else: - solution = optimize_portfolio_allocation( - robust=False, - total_budget=total_budget, - max_portfolio_pd=max_portfolio_pd, - solver_backend=solver_backend, - **common, - ) - return solution, pd_point + policy = robust_policy or {} + result = solve_policy_allocation( + loans=cast(pd.DataFrame, common["loans"]), + pd_point=pd_point, + pd_low=np.asarray(common["pd_low"], dtype=float), + pd_high=np.asarray(common["pd_high"], dtype=float), + lgd=np.asarray(common["lgd"], dtype=float), + int_rates=np.asarray(common["int_rates"], dtype=float), + total_budget=total_budget, + risk_tolerance=max_portfolio_pd, + robust=robust, + uncertainty_aversion=float(policy.get("uncertainty_aversion", 0.0)), + min_budget_utilization=float(policy.get("min_budget_utilization", 0.0)), + pd_cap_slack_penalty=float(policy.get("pd_cap_slack_penalty", 0.0)), + policy_mode=str(policy.get("policy_mode", "hard_worst_case")), + gamma=float(policy.get("gamma", 1.0)), + delta_cap_quantile=float(policy.get("delta_cap_quantile", 1.0)), + tail_focus_quantile=float(policy.get("tail_focus_quantile", 1.0)), + solver_backend=solver_backend, + ) + return result.solution, result.effective_pd def _candidate_metrics( diff --git a/scripts/validate_alpha_gamma_bound.py b/scripts/validate_alpha_gamma_bound.py index 079ed9b..4f93269 100644 --- a/scripts/validate_alpha_gamma_bound.py +++ b/scripts/validate_alpha_gamma_bound.py @@ -26,6 +26,7 @@ from src.optimization.certificate_semantics import ( # noqa: E402 compute_funded_certificate_metrics, ) +from src.optimization.policy import policy_segment_labels # noqa: E402 from src.optimization.portfolio_model import ( # noqa: E402 compute_effective_pd, optimize_portfolio_allocation, @@ -120,30 +121,6 @@ def _load_aligned_dataset( return aligned -def _policy_segment_labels(loans: pd.DataFrame, policy_mode: str) -> np.ndarray | None: - if str(policy_mode).strip().lower() not in { - "segment_tail_blended_uncertainty", - "segment_relative_tail_blended_uncertainty", - }: - return None - grade = ( - loans["grade"].fillna("unknown").astype(str) - if "grade" in loans.columns - else pd.Series(["unknown"] * len(loans)) - ) - term = ( - loans["term"].fillna("unknown").astype(str) - if "term" in loans.columns - else pd.Series(["unknown"] * len(loans)) - ) - verification = ( - loans["verification_status"].fillna("unknown").astype(str) - if "verification_status" in loans.columns - else pd.Series(["unknown"] * len(loans)) - ) - return (grade + "|" + term + "|" + verification).to_numpy(dtype=object) - - def _compute_intervals_at_alpha( frame: pd.DataFrame, alpha: float, @@ -183,7 +160,7 @@ def _compute_effective_pd_vector( gamma=float(policy["gamma"]), delta_cap_quantile=float(policy["delta_cap_quantile"]), tail_focus_quantile=float(policy["tail_focus_quantile"]), - segment_labels=_policy_segment_labels(loans, str(policy["policy_mode"])), + segment_labels=policy_segment_labels(loans, str(policy["policy_mode"])), ) diff --git a/src/models/conformal_alpha_grid.py b/src/models/conformal_alpha_grid.py new file mode 100644 index 0000000..cad3e6a --- /dev/null +++ b/src/models/conformal_alpha_grid.py @@ -0,0 +1,211 @@ +"""Exact alpha-grid replay for a frozen Mondrian conformal recipe.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import numpy as np +import pandas as pd + +from src.models.conformal import ( + build_mondrian_partition_labels, + create_pd_intervals_mondrian_from_predictions, +) +from src.models.conformal_tuning import apply_group_multipliers, build_group_temporal_segments + + +@dataclass(frozen=True) +class FrozenConformalRecipe: + """Selected interval design and holdout-learned widening policy.""" + + partition: str + partition_probability_source: str + n_score_bins: int + fallback_mode: str + score_scale_family: str + min_group_size: int + reference_target_alpha: float + reference_used_alpha: float + calibration_fraction: float + tuning_holdout_ratio: float + tuning_random_state: int + group_multipliers: dict[str, float] + temporal_segment_multipliers: dict[str, float] + temporal_segment_freq: str + global_rebalance_factor: float + + @classmethod + def from_results_payload(cls, payload: dict[str, Any]) -> FrozenConformalRecipe: + """Build a replay recipe from ``conformal_results_mondrian.pkl``.""" + selected = payload["tuning_90_best"] + split = payload["calibration_split"] + global_rebalance = payload.get("global_rebalance", {}) or {} + factor = ( + float(global_rebalance.get("factor", 1.0)) + if bool(global_rebalance.get("applied", False)) + else 1.0 + ) + recipe = cls( + partition=str(selected["partition"]), + partition_probability_source=str(selected["partition_probability_source"]), + n_score_bins=int(selected["n_score_bins"]), + fallback_mode=str(selected["fallback_mode"]), + score_scale_family=str(selected["score_scale_family"]), + min_group_size=int(selected["min_group_size"]), + reference_target_alpha=float(selected["alpha_target_90"]), + reference_used_alpha=float(selected["alpha_used_90"]), + calibration_fraction=float(split["calibration_fraction"]), + tuning_holdout_ratio=float(split["holdout_ratio"]), + tuning_random_state=int(split["random_state"]), + group_multipliers={ + str(key): float(value) + for key, value in (payload.get("group_coverage_multipliers", {}) or {}).items() + }, + temporal_segment_multipliers={ + str(key): float(value) + for key, value in (payload.get("temporal_segment_multipliers", {}) or {}).items() + }, + temporal_segment_freq=str(payload.get("temporal_segment_freq", "Q")), + global_rebalance_factor=factor, + ) + recipe.validate() + return recipe + + def validate(self) -> None: + """Reject recipe settings that could narrow a nominal interval silently.""" + if not 0.0 < self.reference_target_alpha < 1.0: + raise ValueError("reference_target_alpha must lie in (0, 1).") + if not 0.0 < self.reference_used_alpha <= self.reference_target_alpha: + raise ValueError( + "reference_used_alpha must be positive and no larger than the target alpha." + ) + multipliers = [ + *self.group_multipliers.values(), + *self.temporal_segment_multipliers.values(), + self.global_rebalance_factor, + ] + if any(value < 1.0 for value in multipliers): + raise ValueError("Frozen alpha-grid replay only supports widening adjustments.") + + def used_alpha(self, target_alpha: float) -> float: + """Apply the frozen conservative alpha ratio selected at the reference level.""" + target = float(target_alpha) + if not 0.0 < target < 1.0: + raise ValueError("target_alpha must lie in (0, 1).") + ratio = self.reference_used_alpha / self.reference_target_alpha + return target * ratio + + +@dataclass(frozen=True) +class ExactAlphaIntervals: + """One exact conformal interval vector and its replay metadata.""" + + target_alpha: float + used_alpha: float + point: np.ndarray + low: np.ndarray + high: np.ndarray + partition_labels: pd.Series + partition_metadata: dict[str, Any] + diagnostics: dict[str, Any] + + +def alpha_column_token(alpha: float) -> str: + """Return a stable column-safe token such as ``0p010``.""" + return f"{float(alpha):.3f}".replace(".", "p") + + +def alpha_interval_columns(alpha: float) -> tuple[str, str]: + """Return the low/high column names for an exact alpha-grid artifact.""" + token = alpha_column_token(alpha) + return f"pd_low_alpha_{token}", f"pd_high_alpha_{token}" + + +def _scale_around_prediction( + point: np.ndarray, + intervals: np.ndarray, + factor: float, +) -> np.ndarray: + radius = np.maximum(point - intervals[:, 0], intervals[:, 1] - point) + return np.column_stack( + [ + np.clip(point - radius * factor, 0.0, 1.0), + np.clip(point + radius * factor, 0.0, 1.0), + ] + ) + + +def compute_exact_alpha_intervals( + *, + recipe: FrozenConformalRecipe, + target_alpha: float, + y_cal: pd.Series | np.ndarray, + interval_probability_cal: np.ndarray, + interval_probability_eval: np.ndarray, + partition_probability_cal: np.ndarray, + partition_probability_eval: np.ndarray, + base_groups_cal: pd.Series | np.ndarray, + base_groups_eval: pd.Series | np.ndarray, + issue_dates_eval: pd.Series | np.ndarray | None = None, +) -> ExactAlphaIntervals: + """Recompute one alpha exactly under a frozen partition and widening recipe.""" + group_cal, group_eval, partition_metadata = build_mondrian_partition_labels( + y_prob_cal=partition_probability_cal, + y_prob_eval=partition_probability_eval, + partition=recipe.partition, + base_groups_cal=base_groups_cal, + base_groups_eval=base_groups_eval, + n_score_bins=recipe.n_score_bins, + min_group_size=recipe.min_group_size, + fallback_mode=recipe.fallback_mode, + ) + used_alpha = recipe.used_alpha(target_alpha) + point, intervals, diagnostics = create_pd_intervals_mondrian_from_predictions( + y_cal_pred=interval_probability_cal, + y_test_pred=interval_probability_eval, + y_cal=y_cal, + group_cal=group_cal, + group_test=group_eval, + alpha=used_alpha, + min_group_size=recipe.min_group_size, + score_scale_family=recipe.score_scale_family, + log_summary=False, + ) + if recipe.group_multipliers: + intervals = apply_group_multipliers( + point, + intervals, + group_eval, + recipe.group_multipliers, + ) + if recipe.temporal_segment_multipliers: + if issue_dates_eval is None: + raise ValueError("issue_dates_eval is required by the frozen temporal multipliers.") + temporal_segments = build_group_temporal_segments( + group_eval, + issue_dates_eval, + freq=recipe.temporal_segment_freq, + ) + intervals = apply_group_multipliers( + point, + intervals, + temporal_segments, + recipe.temporal_segment_multipliers, + ) + if not np.isclose(recipe.global_rebalance_factor, 1.0): + intervals = _scale_around_prediction( + point, + intervals, + recipe.global_rebalance_factor, + ) + return ExactAlphaIntervals( + target_alpha=float(target_alpha), + used_alpha=used_alpha, + point=point, + low=intervals[:, 0], + high=intervals[:, 1], + partition_labels=group_eval, + partition_metadata=partition_metadata, + diagnostics=diagnostics, + ) diff --git a/src/optimization/policy.py b/src/optimization/policy.py index 89303ea..408f5c6 100644 --- a/src/optimization/policy.py +++ b/src/optimization/policy.py @@ -10,6 +10,10 @@ from __future__ import annotations from enum import StrEnum +from typing import cast + +import numpy as np +import pandas as pd class PolicyMode(StrEnum): @@ -79,3 +83,44 @@ def resolve_policy_mode(value: str | PolicyMode | None) -> PolicyMode: def all_policy_modes() -> tuple[PolicyMode, ...]: """Tuple of all canonical policy modes — useful for parametrised tests and sweeps.""" return tuple(PolicyMode) + + +_SEGMENT_POLICY_MODES = frozenset( + { + PolicyMode.SEGMENT_TAIL_BLENDED_UNCERTAINTY, + PolicyMode.SEGMENT_RELATIVE_TAIL_BLENDED_UNCERTAINTY, + } +) + + +def policy_uses_segment_labels(value: str | PolicyMode | None) -> bool: + """Return whether a policy requires contextual segment labels.""" + return resolve_policy_mode(value) in _SEGMENT_POLICY_MODES + + +def policy_segment_labels( + loans: pd.DataFrame, + policy_mode: str | PolicyMode | None, + *, + grade_column: str = "grade", +) -> np.ndarray | None: + """Build the canonical grade/term/verification labels for segment policies. + + Non-segment policies return ``None``. Missing context columns are represented + by ``"unknown"`` so all optimization entrypoints use the same fallback. + """ + if not policy_uses_segment_labels(policy_mode): + return None + + def _labels(column: str) -> pd.Series: + if column not in loans.columns: + return pd.Series("unknown", index=loans.index, dtype="string") + return loans[column].fillna("unknown").astype(str) + + grade = _labels(grade_column) + term = _labels("term") + verification = _labels("verification_status") + return cast( + np.ndarray, + (grade + "|" + term + "|" + verification).to_numpy(dtype=object), + ) diff --git a/src/optimization/policy_evaluation.py b/src/optimization/policy_evaluation.py new file mode 100644 index 0000000..43e69e2 --- /dev/null +++ b/src/optimization/policy_evaluation.py @@ -0,0 +1,109 @@ +"""Shared execution contract for portfolio uncertainty policies.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import numpy as np +import pandas as pd + +from src.optimization.policy import PolicyMode, policy_segment_labels, resolve_policy_mode +from src.optimization.portfolio_model import ( + compute_effective_pd, + optimize_portfolio_allocation, + solution_allocation_vector, +) + + +@dataclass(frozen=True) +class PolicyAllocationResult: + """Solver payload, dense allocation, and effective PD for one policy.""" + + solution: dict[str, Any] + allocation: np.ndarray + effective_pd: np.ndarray + policy_mode: PolicyMode + gamma: float + delta_cap_quantile: float + tail_focus_quantile: float + objective_risk_mode: str + + +def solve_policy_allocation( + *, + loans: pd.DataFrame, + pd_point: np.ndarray, + pd_low: np.ndarray, + pd_high: np.ndarray, + lgd: np.ndarray, + int_rates: np.ndarray, + total_budget: float = 1_000_000.0, + max_concentration: float = 0.25, + risk_tolerance: float = 0.10, + robust: bool = True, + uncertainty_aversion: float = 0.0, + min_budget_utilization: float = 0.0, + pd_cap_slack_penalty: float = 0.0, + policy_mode: str | PolicyMode = PolicyMode.HARD_WORST_CASE, + gamma: float = 1.0, + delta_cap_quantile: float = 1.0, + tail_focus_quantile: float = 1.0, + time_limit: int = 300, + threads: int = 4, + solver_backend: str = "highs", + random_seed: int | None = None, + cuopt_presolve: int | None = 1, + cuopt_parameters: dict[str, Any] | None = None, +) -> PolicyAllocationResult: + """Resolve policy semantics once and solve the corresponding portfolio. + + ``robust=False`` always means a point-PD constraint, regardless of any + uncertainty-policy arguments supplied by a legacy caller. + """ + effective_mode = resolve_policy_mode(policy_mode) if robust else PolicyMode.POINT_ESTIMATE + effective_gamma = float(gamma) if robust else 0.0 + effective_delta_cap = float(delta_cap_quantile) if robust else 1.0 + effective_tail_focus = float(tail_focus_quantile) if robust else 1.0 + effective_aversion = float(uncertainty_aversion) if robust else 0.0 + effective_pd = compute_effective_pd( + pd_point=pd_point, + pd_high=pd_high, + policy_mode=effective_mode, + gamma=effective_gamma, + delta_cap_quantile=effective_delta_cap, + tail_focus_quantile=effective_tail_focus, + segment_labels=policy_segment_labels(loans, effective_mode), + ) + solution = optimize_portfolio_allocation( + loans=loans, + pd_point=pd_point, + pd_low=pd_low, + pd_high=pd_high, + lgd=lgd, + int_rates=int_rates, + total_budget=total_budget, + max_concentration=max_concentration, + max_portfolio_pd=risk_tolerance, + robust=robust, + uncertainty_aversion=effective_aversion, + min_budget_utilization=min_budget_utilization, + pd_cap_slack_penalty=pd_cap_slack_penalty, + pd_constraint_override=effective_pd, + time_limit=time_limit, + threads=threads, + solver_backend=solver_backend, + random_seed=random_seed, + cuopt_presolve=cuopt_presolve, + cuopt_parameters=cuopt_parameters, + ) + return PolicyAllocationResult( + solution=solution, + allocation=solution_allocation_vector(solution, len(loans)), + effective_pd=effective_pd, + policy_mode=effective_mode, + gamma=effective_gamma, + delta_cap_quantile=effective_delta_cap, + tail_focus_quantile=effective_tail_focus, + objective_risk_mode="point_pd_plus_aversion", + ) diff --git a/src/optimization/policy_selection.py b/src/optimization/policy_selection.py new file mode 100644 index 0000000..1c087c2 --- /dev/null +++ b/src/optimization/policy_selection.py @@ -0,0 +1,166 @@ +"""Simple, outcome-free selection primitives for the active IJDS policy.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass +from itertools import product + +import numpy as np +import pandas as pd + +from src.optimization.policy import PolicyMode + +FORBIDDEN_POLICY_SELECTION_COLUMNS = frozenset( + { + "default_flag", + "y_true", + "outcome", + "weighted_outcome", + "weighted_miscoverage", + "realized_return", + "realized_risk_tolerance_excess", + } +) +REQUIRED_POLICY_SELECTION_COLUMNS = frozenset( + { + "candidate_id", + "solver_status", + "expected_objective", + "markov_loss_threshold", + "weighted_pd_effective", + "risk_tolerance", + "total_allocated", + } +) + + +@dataclass(frozen=True) +class LinearPolicyCandidate: + """One linear conformal-guardrail policy.""" + + candidate_id: str + risk_tolerance: float + gamma: float + uncertainty_aversion: float + policy_mode: str = PolicyMode.BLENDED_UNCERTAINTY.value + delta_cap_quantile: float = 1.0 + tail_focus_quantile: float = 1.0 + min_budget_utilization: float = 0.0 + pd_cap_slack_penalty: float = 0.0 + + def to_record(self) -> dict[str, float | str]: + """Return a JSON/table-friendly record.""" + return asdict(self) + + +def build_linear_policy_grid( + *, + risk_tolerances: list[float], + gammas: list[float], + uncertainty_aversions: list[float], +) -> list[LinearPolicyCandidate]: + """Build a deterministic Cartesian grid of linear policies.""" + return [ + LinearPolicyCandidate( + candidate_id=f"linear-{index:03d}", + risk_tolerance=float(tau), + gamma=float(gamma), + uncertainty_aversion=float(aversion), + ) + for index, (tau, gamma, aversion) in enumerate( + product(risk_tolerances, gammas, uncertainty_aversions), + start=1, + ) + ] + + +def temporal_period_labels( + issue_dates: pd.Series, + *, + combine_years_from: int | None = 2020, +) -> pd.Series: + """Map issue dates to half-year periods, optionally pooling late years.""" + dates = pd.to_datetime(issue_dates, errors="coerce") + if dates.isna().any(): + raise ValueError(f"issue_dates contains {int(dates.isna().sum())} invalid values.") + years = dates.dt.year.astype(int) + halves = np.where(dates.dt.month.to_numpy() <= 6, "H1", "H2") + labels = pd.Series(years.astype(str) + halves, index=issue_dates.index, dtype="string") + if combine_years_from is not None: + labels.loc[years >= int(combine_years_from)] = f"{int(combine_years_from)}+" + return labels + + +def policy_eligibility_mask( + results: pd.DataFrame, + *, + markov_threshold_cap: float, + budget: float, + min_budget_utilization: float = 0.999, +) -> pd.Series: + """Return the canonical ex-ante feasibility screen for policy rows.""" + missing = sorted(REQUIRED_POLICY_SELECTION_COLUMNS.difference(results.columns)) + if missing: + raise ValueError(f"Policy results are missing required columns: {missing}") + if float(budget) <= 0.0: + raise ValueError("budget must be positive.") + if not 0.0 <= float(min_budget_utilization) <= 1.0: + raise ValueError("min_budget_utilization must lie in [0, 1].") + + solver_ok = results["solver_status"].astype(str).str.strip().str.casefold().eq("optimal") + budget_ok = pd.to_numeric(results["total_allocated"], errors="raise") >= ( + float(budget) * float(min_budget_utilization) + ) + threshold_ok = pd.to_numeric(results["markov_loss_threshold"], errors="raise") <= ( + float(markov_threshold_cap) + 1e-12 + ) + cap_ok = pd.to_numeric(results["weighted_pd_effective"], errors="raise") <= ( + pd.to_numeric(results["risk_tolerance"], errors="raise") + 1e-12 + ) + objective_ok = np.isfinite( + pd.to_numeric(results["expected_objective"], errors="raise").to_numpy(dtype=float) + ) + return solver_ok & budget_ok & threshold_ok & cap_ok & objective_ok + + +def select_policy_result_ex_ante( + results: pd.DataFrame, + *, + markov_threshold_cap: float, + budget: float, + min_budget_utilization: float = 0.999, +) -> tuple[pd.Series, dict[str, int | float | str]]: + """Maximize expected objective under outcome-free feasibility screens.""" + forbidden = sorted(FORBIDDEN_POLICY_SELECTION_COLUMNS.intersection(results.columns)) + if forbidden: + raise ValueError(f"Ex-ante selector received outcome-derived columns: {forbidden}") + if results.empty: + raise ValueError("Ex-ante selection results are empty.") + if results["candidate_id"].astype(str).duplicated().any(): + raise ValueError("Ex-ante selection results contain duplicate candidate_id values.") + eligible_mask = policy_eligibility_mask( + results, + markov_threshold_cap=markov_threshold_cap, + budget=budget, + min_budget_utilization=min_budget_utilization, + ) + eligible = results.loc[eligible_mask].copy() + if eligible.empty: + raise RuntimeError( + "No policy satisfies the ex-ante endpoint, effective-PD, and budget screens." + ) + selected = eligible.sort_values( + ["expected_objective", "markov_loss_threshold", "candidate_id"], + ascending=[False, True, True], + kind="mergesort", + ).iloc[0] + audit: dict[str, int | float | str] = { + "selection_rule": "max_expected_objective_under_ex_ante_screen", + "n_total": int(len(results)), + "n_eligible": int(len(eligible)), + "selected_candidate_id": str(selected["candidate_id"]), + "markov_threshold_cap": float(markov_threshold_cap), + "min_budget_utilization": float(min_budget_utilization), + "outcome_columns_used": 0, + } + return selected, audit diff --git a/tests/test_crpto_final_sync.py b/tests/test_crpto_final_sync.py index d240d69..254194c 100644 --- a/tests/test_crpto_final_sync.py +++ b/tests/test_crpto_final_sync.py @@ -32,9 +32,8 @@ def load_key_metrics() -> dict[str, str]: def test_crpto_champion_artifacts_agree() -> None: """Validate the historical frozen rebaseline chain (two-tag scheme). - Under the pool93 promotion this run tag remains the frozen upstream - chain and declared return floor; the paper body claim is guarded by - tests/test_pool93_body_claim_sync.py. + This run tag remains the frozen upstream provenance chain. The active + manuscript policy is guarded by tests/test_ijds_active_claim_sync.py. """ assert Path("data/processed/final_project_summary.parquet.dvc").exists() promotion = load_json("models/final_project_promotion.json") @@ -98,11 +97,12 @@ def test_crpto_tables_and_figures_exist() -> None: "reports/crpto/tables/crpto_tableA22_tail_constrained_reoptimization.csv", "reports/crpto/tables/crpto_tableA23_multidistribution_robustness.csv", "reports/crpto/tables/crpto_tableA24_online_conformal_stability.csv", - "reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv", - "reports/crpto/tables/crpto_tableA36_pool93_body_funded_grade_audit.csv", - "reports/crpto/tables/crpto_tableA37_pool93_body_tail_risk.csv", - "reports/crpto/tables/crpto_tableA38_pool93_body_cluster_bound_audit.csv", - "reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.csv", + "reports/crpto/tables/crpto_tableA35_exact_alpha_grid.csv", + "reports/crpto/tables/crpto_tableA36_calibration_policy_selector.csv", + "reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.csv", + "reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.csv", + "reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.csv", + "reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.csv", "reports/crpto/figures/crpto_fig1_journal_pipeline.png", "reports/crpto/figures/crpto_fig1_journal_pipeline.pdf", "reports/crpto/figures/crpto_fig1_journal_pipeline.svg", @@ -110,8 +110,6 @@ def test_crpto_tables_and_figures_exist() -> None: "book/assets/figures/publication/crpto_fig1_journal_pipeline.pdf", "book/assets/figures/publication/crpto_fig1_journal_pipeline.svg", "reports/crpto/figures/crpto_fig12_crpto_conceptual_pipeline.png", - "reports/crpto/figures/crpto_fig13_alpha_gamma_funded_set.png", - "reports/crpto/figures/crpto_fig14_robust_region_heatmap.png", "reports/crpto/figures/crpto_fig15_regret_auditability_frontier.png", "reports/crpto/figures/crpto_fig20_bound_claim_layers.png", ] diff --git a/tests/test_experiments/test_ijds_calibration_selected_policy_challenger.py b/tests/test_experiments/test_ijds_calibration_selected_policy_challenger.py new file mode 100644 index 0000000..4699895 --- /dev/null +++ b/tests/test_experiments/test_ijds_calibration_selected_policy_challenger.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import numpy as np +import pandas as pd + +from scripts.experiments.run_ijds_calibration_selected_policy_challenger import ( + FORBIDDEN_SELECTOR_COLUMNS, + _funded_allocation_frame, + _measure_ex_ante_solution, +) +from src.optimization.policy import PolicyMode +from src.optimization.policy_evaluation import PolicyAllocationResult +from src.optimization.policy_selection import LinearPolicyCandidate + + +def test_ex_ante_measurement_contains_no_outcome_fields() -> None: + frame = pd.DataFrame( + { + "_loan_amount": [100.0, 200.0], + "_pd_point": [0.10, 0.20], + "_pd_high": [0.30, 0.50], + } + ) + candidate = LinearPolicyCandidate( + candidate_id="linear-001", + risk_tolerance=0.20, + gamma=0.50, + uncertainty_aversion=0.0, + ) + result = PolicyAllocationResult( + solution={ + "solver_status": "Optimal", + "objective_value": 10.0, + }, + allocation=np.array([1.0, 0.5]), + effective_pd=np.array([0.20, 0.35]), + policy_mode=PolicyMode.BLENDED_UNCERTAINTY, + gamma=0.50, + delta_cap_quantile=1.0, + tail_focus_quantile=1.0, + objective_risk_mode="point_pd_plus_aversion", + ) + + record = _measure_ex_ante_solution(frame, candidate, result, alpha=0.10) + + assert not FORBIDDEN_SELECTOR_COLUMNS.intersection(record) + assert record["expected_objective"] == 10.0 + assert record["weighted_pd_effective"] == 0.275 + + +def test_funded_allocation_frame_reconciles_exposure_and_return() -> None: + frame = pd.DataFrame( + { + "id": ["a", "b"], + "issue_d": ["2020-01-01", "2020-02-01"], + "grade": ["A", "B"], + "_loan_amount": [100.0, 200.0], + "_int_rate": [0.10, 0.20], + "_outcome": [0.0, 1.0], + "_pd_point": [0.10, 0.20], + "_pd_low": [0.0, 0.0], + "_pd_high": [0.30, 0.50], + } + ) + result = PolicyAllocationResult( + solution={"solver_status": "Optimal", "objective_value": 10.0}, + allocation=np.array([1.0, 0.5]), + effective_pd=np.array([0.20, 0.35]), + policy_mode=PolicyMode.BLENDED_UNCERTAINTY, + gamma=0.50, + delta_cap_quantile=1.0, + tail_focus_quantile=1.0, + objective_risk_mode="point_pd_plus_aversion", + ) + + funded = _funded_allocation_frame(frame, result, role="selected", lgd=0.45) + + assert funded["funded_exposure"].sum() == 200.0 + assert funded["funded_weight"].sum() == 1.0 + assert funded["realized_return_contribution"].sum() == -35.0 diff --git a/tests/test_experiments/test_ijds_exact_alpha_grid_challenger.py b/tests/test_experiments/test_ijds_exact_alpha_grid_challenger.py new file mode 100644 index 0000000..4ab34c0 --- /dev/null +++ b/tests/test_experiments/test_ijds_exact_alpha_grid_challenger.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import numpy as np +import pandas as pd + +from scripts.experiments.run_ijds_exact_alpha_grid_challenger import ( + _base_grid_frame, + _replay_differences, +) +from src.models.conformal_alpha_grid import ExactAlphaIntervals + + +def _result() -> ExactAlphaIntervals: + return ExactAlphaIntervals( + target_alpha=0.10, + used_alpha=0.095, + point=np.array([0.2, 0.4]), + low=np.array([0.0, 0.1]), + high=np.array([0.5, 0.8]), + partition_labels=pd.Series(["a", "b"]), + partition_metadata={}, + diagnostics={}, + ) + + +def test_base_grid_keeps_only_traceability_columns() -> None: + source = pd.DataFrame( + { + "_row_number": [0], + "id": ["x"], + "y_true": [0.0], + "grade": ["A"], + "pd_high_90": [0.5], + } + ) + + assert _base_grid_frame(source).columns.tolist() == [ + "_row_number", + "id", + "y_true", + "grade", + ] + + +def test_reference_replay_reports_exact_match() -> None: + source = pd.DataFrame( + { + "y_pred": [0.2, 0.4], + "pd_low_90": [0.0, 0.1], + "pd_high_90": [0.5, 0.8], + } + ) + + assert _replay_differences(_result(), source) == { + "point_max_abs": 0.0, + "low_max_abs": 0.0, + "high_max_abs": 0.0, + } diff --git a/tests/test_ijds_active_claim_sync.py b/tests/test_ijds_active_claim_sync.py new file mode 100644 index 0000000..e07e91a --- /dev/null +++ b/tests/test_ijds_active_claim_sync.py @@ -0,0 +1,132 @@ +"""Drift guard for the active calibration-selected IJDS policy.""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +import pytest + +REPO = Path(__file__).resolve().parents[1] +TABLES = REPO / "reports/crpto/tables" +RUN_TAG = "champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6" +GOVERNANCE = ( + REPO / "models/experiments/champion_reopen" / RUN_TAG / "portfolio/ijds_policy_governance.json" +) +SUMMARY = GOVERNANCE.with_name("calibration_selected_policy_summary.json") +SURFACES = ( + REPO / "paper/CRPTO_ijds.qmd", + REPO / "paper/supplement_ijds.qmd", + REPO / "paper/submission/CRPTO_ijds_submission.tex", +) +TABLE_STEMS = ( + "crpto_tableA35_exact_alpha_grid", + "crpto_tableA36_calibration_policy_selector", + "crpto_tableA37_calibration_selected_temporal_evaluation", + "crpto_tableA38_calibration_selected_grade_audit", + "crpto_tableA39_calibration_selected_bootstrap", + "crpto_tableA40_calibration_selected_point_baseline", +) + + +def _json(path: Path) -> dict[str, Any]: + assert path.is_file(), f"Missing active governance: {path}" + return json.loads(path.read_text(encoding="utf-8")) + + +def _rows(stem: str) -> list[dict[str, str]]: + with (TABLES / f"{stem}.csv").open(encoding="utf-8") as handle: + return list(csv.DictReader(handle)) + + +def _surface_text(path: Path) -> str: + return path.read_text(encoding="utf-8").replace(r"\$", "$").replace("{,}", ",") + + +def test_active_governance_locks_simple_policy_and_selector() -> None: + payload = _json(GOVERNANCE) + summary = _json(SUMMARY) + policy = payload["selected_policy"] + selector = payload["selection_protocol"] + + assert payload["status"] == "active_ijds_policy" + assert payload["run_tag"] == RUN_TAG + assert payload["generated_at_utc"] == summary["generated_at_utc"] + assert policy["policy_mode"] == "blended_uncertainty" + assert policy["risk_tolerance"] == pytest.approx(0.17) + assert policy["gamma"] == pytest.approx(0.50) + assert policy["uncertainty_aversion"] == pytest.approx(0.0) + assert policy["min_budget_utilization"] == pytest.approx(0.0) + assert selector["min_budget_utilization"] == pytest.approx(0.999) + assert selector["n_total"] == 9 + assert selector["n_eligible"] == 5 + assert selector["outcome_columns_used"] == 0 + assert selector["selector_forbidden_columns_present"] == [] + assert selector["calibration_metadata"]["target_alpha"] == pytest.approx(0.10) + assert payload["exact_alpha_reference_replay"]["pass"] is True + + +def test_active_tables_agree_with_governance() -> None: + payload = _json(GOVERNANCE) + full = payload["full_oot"] + selected = next( + row + for row in _rows("crpto_tableA40_calibration_selected_point_baseline") + if row["policy"] == "Calibration-selected 50/50 CRPTO" + ) + alpha = next( + row + for row in _rows("crpto_tableA35_exact_alpha_grid") + if row["selected_for_policy"] == "True" + ) + + assert float(selected["realized_return"]) == pytest.approx(full["realized_return"]) + assert float(selected["weighted_outcome"]) == pytest.approx(full["weighted_default_rate"]) + assert float(selected["weighted_miscoverage"]) == pytest.approx(full["weighted_miscoverage"]) + assert float(selected["endpoint_budget"]) == pytest.approx(full["endpoint_budget"]) + assert float(selected["markov_loss_threshold"]) == pytest.approx(full["markov_loss_threshold"]) + assert float(alpha["target_alpha"]) == pytest.approx(0.10) + assert float(alpha["empirical_coverage"]) == pytest.approx(0.9348356081757077) + + +def test_active_a35_to_a40_exist_in_csv_and_tex() -> None: + missing = [ + f"{stem}.{suffix}" + for stem in TABLE_STEMS + for suffix in ("csv", "tex") + if not (TABLES / f"{stem}.{suffix}").is_file() + ] + assert not missing, "Missing active IJDS evidence: " + ", ".join(missing) + + +def test_active_manuscript_surfaces_share_numeric_anchors() -> None: + payload = _json(GOVERNANCE) + full = payload["full_oot"] + contrast = payload["point_pd_contrast"] + anchors = ( + f"${full['realized_return']:,.2f}", + f"{full['weighted_default_rate']:.6f}", + f"{full['weighted_miscoverage']:.6f}", + f"{full['Gamma_CP']:.6f}", + f"{full['Gamma_residual']:.6f}", + f"{full['endpoint_budget']:.6f}", + f"{full['observed_accounting_bound']:.6f}", + f"{full['markov_loss_threshold']:.6f}", + f"{contrast['realized_return']:,.2f}", + f"{100 * contrast['selected_return_cost_pct']:.3f}", + f"{100 * contrast['selected_default_reduction']:.4f}", + ) + for surface in SURFACES: + text = _surface_text(surface) + missing = [anchor for anchor in anchors if anchor not in text] + assert not missing, f"{surface.name} missing active anchors: {missing}" + + +def test_retired_headline_numbers_do_not_appear_in_active_surfaces() -> None: + retired = ("0.345084", "50,010", "27,508", "capped_blended_uncertainty") + for surface in SURFACES: + text = _surface_text(surface) + present = [token for token in retired if token in text] + assert not present, f"{surface.name} retains retired claims: {present}" diff --git a/tests/test_models/test_conformal_alpha_grid.py b/tests/test_models/test_conformal_alpha_grid.py new file mode 100644 index 0000000..8864a2e --- /dev/null +++ b/tests/test_models/test_conformal_alpha_grid.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from src.models.conformal_alpha_grid import ( + FrozenConformalRecipe, + alpha_interval_columns, + compute_exact_alpha_intervals, +) + + +def _payload() -> dict: + return { + "tuning_90_best": { + "partition": "grade", + "partition_probability_source": "calibrated", + "n_score_bins": 5, + "fallback_mode": "global_only", + "score_scale_family": "none", + "min_group_size": 1, + "alpha_target_90": 0.10, + "alpha_used_90": 0.095, + }, + "calibration_split": { + "calibration_fraction": 0.75, + "holdout_ratio": 0.20, + "random_state": 42, + }, + "group_coverage_multipliers": {"A": 1.05}, + "temporal_segment_multipliers": {}, + "global_rebalance": {"enabled": False, "applied": False}, + } + + +def test_recipe_scales_alpha_by_frozen_conservative_ratio() -> None: + recipe = FrozenConformalRecipe.from_results_payload(_payload()) + + assert recipe.used_alpha(0.01) == pytest.approx(0.0095) + assert alpha_interval_columns(0.01) == ( + "pd_low_alpha_0p010", + "pd_high_alpha_0p010", + ) + + +def test_recipe_rejects_narrowing_adjustments() -> None: + payload = _payload() + payload["global_rebalance"] = {"applied": True, "factor": 0.95} + + with pytest.raises(ValueError, match="widening"): + FrozenConformalRecipe.from_results_payload(payload) + + +def test_exact_alpha_intervals_use_frozen_partition_and_multiplier() -> None: + recipe = FrozenConformalRecipe.from_results_payload(_payload()) + y_cal = np.array([0.0, 1.0, 0.0, 1.0]) + p_cal = np.array([0.1, 0.8, 0.2, 0.7]) + p_eval = np.array([0.15, 0.75]) + groups_cal = pd.Series(["A", "A", "B", "B"]) + groups_eval = pd.Series(["A", "B"]) + + result = compute_exact_alpha_intervals( + recipe=recipe, + target_alpha=0.10, + y_cal=y_cal, + interval_probability_cal=p_cal, + interval_probability_eval=p_eval, + partition_probability_cal=p_cal, + partition_probability_eval=p_eval, + base_groups_cal=groups_cal, + base_groups_eval=groups_eval, + ) + + assert result.used_alpha == pytest.approx(0.095) + assert result.partition_labels.tolist() == ["A", "B"] + assert np.all(result.low <= result.point) + assert np.all(result.high >= result.point) + assert result.high[0] == pytest.approx(0.36) diff --git a/tests/test_optimization/test_policy.py b/tests/test_optimization/test_policy.py index 63a05eb..c7eb5cb 100644 --- a/tests/test_optimization/test_policy.py +++ b/tests/test_optimization/test_policy.py @@ -9,11 +9,18 @@ from __future__ import annotations import numpy as np +import pandas as pd import pytest from hypothesis import HealthCheck, given, settings, strategies as st from hypothesis.extra.numpy import arrays -from src.optimization.policy import PolicyMode, all_policy_modes, resolve_policy_mode +from src.optimization.policy import ( + PolicyMode, + all_policy_modes, + policy_segment_labels, + policy_uses_segment_labels, + resolve_policy_mode, +) from src.optimization.portfolio_model import compute_effective_pd # --------------------------------------------------------------------------- @@ -64,6 +71,31 @@ def test_resolve_policy_mode_defaults_for_none_or_empty() -> None: assert resolve_policy_mode("") is PolicyMode.HARD_WORST_CASE +def test_only_segment_policies_request_segment_labels() -> None: + assert policy_uses_segment_labels(PolicyMode.SEGMENT_TAIL_BLENDED_UNCERTAINTY) + assert policy_uses_segment_labels(PolicyMode.SEGMENT_RELATIVE_TAIL_BLENDED_UNCERTAINTY) + assert not policy_uses_segment_labels(PolicyMode.BLENDED_UNCERTAINTY) + + +def test_policy_segment_labels_use_one_canonical_fallback_contract() -> None: + loans = pd.DataFrame( + { + "original_grade": ["A", None], + "term": [36, 60], + } + ) + + labels = policy_segment_labels( + loans, + PolicyMode.SEGMENT_TAIL_BLENDED_UNCERTAINTY, + grade_column="original_grade", + ) + + assert labels is not None + assert labels.tolist() == ["A|36|unknown", "unknown|60|unknown"] + assert policy_segment_labels(loans, PolicyMode.BLENDED_UNCERTAINTY) is None + + # --------------------------------------------------------------------------- # compute_effective_pd — invariants per policy # --------------------------------------------------------------------------- diff --git a/tests/test_optimization/test_policy_evaluation.py b/tests/test_optimization/test_policy_evaluation.py new file mode 100644 index 0000000..45a5866 --- /dev/null +++ b/tests/test_optimization/test_policy_evaluation.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +from typing import Any + +import numpy as np +import pandas as pd + +from src.optimization import policy_evaluation +from src.optimization.policy import PolicyMode + + +def _inputs() -> dict[str, Any]: + loans = pd.DataFrame( + { + "loan_amnt": [100.0, 200.0], + "grade": ["A", "B"], + "term": [36, 60], + "verification_status": ["Verified", "Not Verified"], + } + ) + return { + "loans": loans, + "pd_point": np.array([0.10, 0.20]), + "pd_low": np.array([0.05, 0.10]), + "pd_high": np.array([0.30, 0.50]), + "lgd": np.array([0.45, 0.45]), + "int_rates": np.array([0.12, 0.15]), + } + + +def test_solve_policy_allocation_resolves_effective_pd_once(monkeypatch) -> None: + captured: dict[str, Any] = {} + + def fake_optimize(**kwargs): + captured.update(kwargs) + return { + "allocation_vector": np.array([1.0, 0.5]), + "objective_value": 1.0, + "n_funded": 2, + "total_allocated": 200.0, + } + + monkeypatch.setattr(policy_evaluation, "optimize_portfolio_allocation", fake_optimize) + result = policy_evaluation.solve_policy_allocation( + **_inputs(), + policy_mode=PolicyMode.BLENDED_UNCERTAINTY, + gamma=0.5, + ) + + assert result.policy_mode is PolicyMode.BLENDED_UNCERTAINTY + assert np.allclose(result.effective_pd, np.array([0.20, 0.35])) + assert np.allclose(captured["pd_constraint_override"], result.effective_pd) + assert np.allclose(captured["pd_point"], _inputs()["pd_point"]) + assert result.objective_risk_mode == "point_pd_plus_aversion" + assert np.allclose(result.allocation, np.array([1.0, 0.5])) + + +def test_nonrobust_policy_always_uses_point_pd(monkeypatch) -> None: + captured: dict[str, Any] = {} + + def fake_optimize(**kwargs): + captured.update(kwargs) + return { + "allocation_vector": np.array([0.0, 1.0]), + "objective_value": 1.0, + "n_funded": 1, + "total_allocated": 200.0, + } + + monkeypatch.setattr(policy_evaluation, "optimize_portfolio_allocation", fake_optimize) + result = policy_evaluation.solve_policy_allocation( + **_inputs(), + robust=False, + policy_mode=PolicyMode.SEGMENT_TAIL_BLENDED_UNCERTAINTY, + gamma=0.9, + uncertainty_aversion=0.5, + ) + + assert result.policy_mode is PolicyMode.POINT_ESTIMATE + assert result.gamma == 0.0 + assert np.allclose(result.effective_pd, _inputs()["pd_point"]) + assert np.allclose(captured["pd_constraint_override"], _inputs()["pd_point"]) + assert captured["uncertainty_aversion"] == 0.0 + + +def test_uncertainty_aversion_keeps_point_pd_objective(monkeypatch) -> None: + captured: dict[str, Any] = {} + + def fake_optimize(**kwargs): + captured.update(kwargs) + return { + "allocation_vector": np.array([1.0, 0.0]), + "objective_value": 1.0, + "n_funded": 1, + "total_allocated": 100.0, + } + + monkeypatch.setattr(policy_evaluation, "optimize_portfolio_allocation", fake_optimize) + result = policy_evaluation.solve_policy_allocation( + **_inputs(), + policy_mode=PolicyMode.BLENDED_UNCERTAINTY, + gamma=0.5, + uncertainty_aversion=0.05, + ) + + assert np.allclose(captured["pd_point"], _inputs()["pd_point"]) + assert np.allclose(captured["pd_constraint_override"], result.effective_pd) + assert captured["uncertainty_aversion"] == 0.05 + assert result.objective_risk_mode == "point_pd_plus_aversion" diff --git a/tests/test_optimization/test_policy_selection.py b/tests/test_optimization/test_policy_selection.py new file mode 100644 index 0000000..f8204af --- /dev/null +++ b/tests/test_optimization/test_policy_selection.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import pandas as pd +import pytest + +from src.optimization.policy_selection import ( + build_linear_policy_grid, + select_policy_result_ex_ante, + temporal_period_labels, +) + + +def _selection_results() -> pd.DataFrame: + return pd.DataFrame( + { + "candidate_id": ["linear-001", "linear-002", "linear-003"], + "solver_status": ["Optimal", "Optimal", "Optimal"], + "expected_objective": [120.0, 110.0, 100.0], + "markov_loss_threshold": [0.70, 0.58, 0.50], + "weighted_pd_effective": [0.17, 0.17, 0.17], + "risk_tolerance": [0.17, 0.17, 0.17], + "total_allocated": [1_000.0, 1_000.0, 1_000.0], + } + ) + + +def test_round_grid_is_deterministic() -> None: + grid = build_linear_policy_grid( + risk_tolerances=[0.15, 0.17, 0.19], + gammas=[0.25, 0.50, 0.75], + uncertainty_aversions=[0.0], + ) + + assert len(grid) == 9 + assert grid[4].candidate_id == "linear-005" + assert grid[4].risk_tolerance == 0.17 + assert grid[4].gamma == 0.50 + + +def test_selector_uses_expected_objective_inside_screen() -> None: + selected, audit = select_policy_result_ex_ante( + _selection_results(), + markov_threshold_cap=0.60, + budget=1_000.0, + ) + + assert selected["candidate_id"] == "linear-002" + assert audit["n_eligible"] == 2 + assert audit["outcome_columns_used"] == 0 + + +def test_selector_does_not_accept_suboptimal_status() -> None: + results = _selection_results() + results.loc[0, "solver_status"] = "Suboptimal" + results.loc[0, "markov_loss_threshold"] = 0.50 + + selected, _ = select_policy_result_ex_ante( + results, + markov_threshold_cap=0.60, + budget=1_000.0, + ) + + assert selected["candidate_id"] == "linear-002" + + +def test_selector_rejects_duplicate_candidates() -> None: + results = pd.concat([_selection_results(), _selection_results().iloc[[0]]]) + + with pytest.raises(ValueError, match="duplicate candidate_id"): + select_policy_result_ex_ante( + results, + markov_threshold_cap=0.60, + budget=1_000.0, + ) + + +def test_selector_rejects_outcome_derived_columns() -> None: + results = _selection_results().assign(realized_return=[1.0, 2.0, 3.0]) + + with pytest.raises(ValueError, match="outcome-derived"): + select_policy_result_ex_ante( + results, + markov_threshold_cap=0.60, + budget=1_000.0, + ) + + +def test_selector_rejects_invalid_budget_utilization() -> None: + with pytest.raises(ValueError, match="min_budget_utilization"): + select_policy_result_ex_ante( + _selection_results(), + markov_threshold_cap=0.60, + budget=1_000.0, + min_budget_utilization=1.01, + ) + + +def test_temporal_labels_pool_late_years() -> None: + dates = pd.Series(["2018-01-01", "2018-08-01", "2020-03-01", "2021-09-01"]) + + assert temporal_period_labels(dates).tolist() == [ + "2018H1", + "2018H2", + "2020+", + "2020+", + ] diff --git a/tests/test_pool93_body_claim_sync.py b/tests/test_pool93_body_claim_sync.py index 39b8bce..50054de 100644 --- a/tests/test_pool93_body_claim_sync.py +++ b/tests/test_pool93_body_claim_sync.py @@ -1,21 +1,7 @@ -"""Drift guard for the promoted pool93 IJDS body claim. - -The paper body point (A35 "Body/default balanced point") lives in the pool93 -governance sidecars and the A35-A40 tables, all generated outside the DVC DAG -by the champion-reopen experiment scripts. The IJDS manuscript embeds those -numbers as hand-written prose/Markdown, so a regenerated CSV or a retyped -figure can silently desync the submission from its evidence. These tests lock -the two-tag scheme together: the frozen rebaseline chain stays the declared -return floor, and the pool93 governance JSONs stay the authoritative source -for every body-claim number printed in the paper surfaces. - -Only anchor values are checked -- enough to catch a stale copy/paste, without -re-implementing a Markdown table parser. -""" +"""Historical integrity checks for the manifest-protected pool93 bundle.""" from __future__ import annotations -import csv import json from pathlib import Path from typing import Any @@ -24,33 +10,20 @@ REPO = Path(__file__).resolve().parents[1] TABLES = REPO / "reports" / "crpto" / "tables" - -PAPER = REPO / "paper" / "CRPTO_ijds.qmd" -SUPPLEMENT = REPO / "paper" / "supplement_ijds.qmd" -SUBMISSION = REPO / "paper" / "submission" / "CRPTO_ijds_submission.tex" - +MANIFEST = REPO / "EXTRACTION_MANIFEST.json" +PROMOTION = REPO / "models" / "final_project_promotion.json" TERMINAL_GOVERNANCE = ( REPO - / "models" - / "experiments" - / "champion_reopen" + / "models/experiments/champion_reopen" / "champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal" - / "portfolio" - / "pool93_ijds_claim_governance.json" + / "portfolio/pool93_ijds_claim_governance.json" ) CONSOLIDATED_GOVERNANCE = ( REPO - / "models" - / "experiments" - / "champion_reopen" + / "models/experiments/champion_reopen" / "champion-reopen-2026-06-19__pool93__ijds-certificate-semantics-v2" - / "portfolio" - / "pool93_ijds_consolidated_governance.json" + / "portfolio/pool93_ijds_consolidated_governance.json" ) -POINT_BASELINE_AUDIT = CONSOLIDATED_GOVERNANCE.with_name("pool93_point_pd_baseline_audit.json") -PROMOTION = REPO / "models" / "final_project_promotion.json" -MANIFEST = REPO / "EXTRACTION_MANIFEST.json" - POOL93_TABLE_STEMS = ( "crpto_tableA35_pool93_ijds_frontier", "crpto_tableA36_pool93_body_funded_grade_audit", @@ -60,230 +33,34 @@ "crpto_tableA40_pool93_point_baseline", ) -TERMINAL_RUN_TAG = "champion-reopen-2026-06-19__pool93__ijds-claim-bound-terminal" -REBASELINE_RUN_TAG = "ijds-rebaseline-2026-06-07" -EXPECTED_BODY = { - "return": 184832.475845, - "Gamma_CP": 0.162616, - "Gamma_internalized": 0.089032, - "Gamma_residual": 0.073584, - "V": 0.03535, - "Markov_threshold": 0.345084, - "endpoint_budget": 0.245084, - "risk_tolerance": 0.1715, - "gamma": 0.5475, - "uncertainty_aversion": 0.05, - "alpha_pass": "8/8", -} -EXPECTED_FLOOR = 170464.54 -EXPECTED_FROZEN_RETURN = 170464.5429284627 -EXPECTED_FRONTIER_COUNTS = { - "raw_rows": 51678, - "deduped_semantic_policies": 50010, - "eligible_all_alpha_return_floor_policies": 27508, -} -EXPECTED_TERMINAL_COUNTS = { - "n_policies": 37068, - "n_all_alpha_passers": 37068, - "n_all_alpha_passers_above_return_floor": 14814, -} - - -def _load_json(path: Path) -> dict[str, Any]: +def _load(path: Path) -> dict[str, Any]: if not path.is_file(): - pytest.skip(f"{path.name} not present locally.") + pytest.skip(f"Historical artifact unavailable: {path}") return json.loads(path.read_text(encoding="utf-8")) -def _read_rows(name: str) -> list[dict[str, str]]: - path = TABLES / name - if not path.is_file(): - pytest.skip(f"{name} not present locally.") - with path.open(encoding="utf-8") as fh: - return list(csv.DictReader(fh)) - - -def _text(path: Path) -> str: - if not path.is_file(): - pytest.skip(f"{path} not present locally.") - return path.read_text(encoding="utf-8") - - -def _normalized_manuscript_text(path: Path) -> str: - return _text(path).replace("{,}", ",").replace(r"\$", "$") - - -def _body_row_a35() -> dict[str, str]: - rows = _read_rows("crpto_tableA35_pool93_ijds_frontier.csv") - matches = [r for r in rows if r["role"] == "Body/default balanced point"] - assert len(matches) == 1, "A35 must expose exactly one body/default balanced row" - return matches[0] - - -def test_pool93_claim_governance_matches_expected_body_point() -> None: - """Consolidated governance body point and A35 body row agree with the claim.""" - consolidated = _load_json(CONSOLIDATED_GOVERNANCE) - body = consolidated["selected_candidates"]["paper_body"] - assert body["return"] == pytest.approx(EXPECTED_BODY["return"], abs=1e-6) - assert body["Gamma_CP"] == pytest.approx(EXPECTED_BODY["Gamma_CP"], abs=1e-9) - assert body["Gamma_internalized"] == pytest.approx( - EXPECTED_BODY["Gamma_internalized"], abs=1e-9 - ) - assert body["Gamma_residual"] == pytest.approx(EXPECTED_BODY["Gamma_residual"], abs=1e-9) - assert body["V"] == pytest.approx(EXPECTED_BODY["V"], abs=1e-9) - assert body["Markov_threshold"] == pytest.approx(EXPECTED_BODY["Markov_threshold"], abs=1e-9) - assert body["endpoint_budget"] == pytest.approx(EXPECTED_BODY["endpoint_budget"], abs=1e-9) - assert body["risk_tolerance"] == EXPECTED_BODY["risk_tolerance"] - assert body["gamma"] == EXPECTED_BODY["gamma"] - assert body["uncertainty_aversion"] == EXPECTED_BODY["uncertainty_aversion"] - assert body["alpha_pass"] == EXPECTED_BODY["alpha_pass"] - - terminal = _load_json(TERMINAL_GOVERNANCE) - assert terminal["run_tag"] == TERMINAL_RUN_TAG - summary = terminal["claim_summary"] - assert summary["declared_return_floor"] == EXPECTED_FLOOR - assert summary["finite_grid_policy"]["alpha_grid_size"] == 8 - - row = _body_row_a35() - assert float(row["realized_return"]) == pytest.approx(body["return"], abs=1e-6) - assert float(row["Gamma_CP_alpha01"]) == pytest.approx(body["Gamma_CP"], abs=1e-9) - assert float(row["V_alpha01"]) == pytest.approx(body["V"], abs=1e-9) - assert float(row["Gamma_residual_alpha01"]) == pytest.approx(body["Gamma_residual"], abs=1e-9) - assert float(row["Markov_threshold_alpha01"]) == pytest.approx( - body["Markov_threshold"], abs=1e-9 - ) - assert row["alpha_grid_pass"] == body["alpha_pass"] - - -def test_pool93_consolidated_governance_frontier_counts() -> None: - """Frontier and terminal-search counts quoted in the paper match governance.""" - consolidated = _load_json(CONSOLIDATED_GOVERNANCE) - for key, expected in EXPECTED_FRONTIER_COUNTS.items(): - assert consolidated["counts"][key] == expected, key - - terminal = _load_json(TERMINAL_GOVERNANCE) - counts = terminal["claim_hierarchy"]["current_counts"] - for key, expected in EXPECTED_TERMINAL_COUNTS.items(): - assert counts[key] == expected, key - assert terminal["runtime_status"]["total_checks"] == 296544 - - -def test_pool93_tables_exist() -> None: - """A35-A40 evidence tables exist in both CSV and TEX form.""" - missing = [ - f"{stem}.{ext}" - for stem in POOL93_TABLE_STEMS - for ext in ("csv", "tex") - if not (TABLES / f"{stem}.{ext}").is_file() - ] - assert not missing, "pool93 evidence tables missing: " + ", ".join(missing) - - -def test_pool93_paper_anchors_match_csvs() -> None: - """Body-claim numbers printed in the paper surfaces derive from A35/A39.""" - row = _body_row_a35() - budget = float(row["endpoint_budget_alpha01"]) - deterministic_bound = budget + float(row["V_alpha01"]) - paper_anchors = [ - f"${float(row['realized_return']):,.2f}", - f"{float(row['V_alpha01']):.6f}", - f"{float(row['Gamma_CP_alpha01']):.6f}", - f"{float(row['Gamma_residual_alpha01']):.6f}", - f"{float(row['Markov_threshold_alpha01']):.6f}", - f"{budget:.6f}", - f"{deterministic_bound:.6f}", - ] - - boot = {r["metric"]: r for r in _read_rows("crpto_tableA39_pool93_body_bootstrap_metrics.csv")} - return_boot = boot["funded_set_repriced_return_lgd45"] - supplement_anchors = [ - *paper_anchors, - f"${float(return_boot['boot_p025']):,.2f}", - f"${float(return_boot['boot_p975']):,.2f}", - ] - - missing: list[str] = [] - paper_text = _text(PAPER) - missing.extend(f"{a} missing in {PAPER.name}" for a in paper_anchors if a not in paper_text) - supplement_text = _text(SUPPLEMENT) - missing.extend( - f"{a} missing in {SUPPLEMENT.name}" for a in supplement_anchors if a not in supplement_text - ) - submission_text = _normalized_manuscript_text(SUBMISSION) - missing.extend( - f"{a} missing in {SUBMISSION.name}" for a in paper_anchors if a not in submission_text - ) - assert not missing, "pool93 body-claim drift:\n" + "\n".join(missing) - - -def test_pool93_matched_point_baseline_agrees_across_surfaces() -> None: - """A40 audit, table, body, supplement, and submission share one contrast.""" - audit = _load_json(POINT_BASELINE_AUDIT) - table = {row["policy"]: row for row in _read_rows("crpto_tableA40_pool93_point_baseline.csv")} - point = audit["point_pd_baseline"] - selected = audit["selected_crpto"] - contrasts = audit["contrasts"] +def test_historical_pool93_bundle_remains_internally_coherent() -> None: + consolidated = _load(CONSOLIDATED_GOVERNANCE) + terminal = _load(TERMINAL_GOVERNANCE) + promotion = _load(PROMOTION) - assert float(table["Point-PD two-stage LP"]["realized_return"]) == pytest.approx( - point["realized_return"], abs=1e-6 + assert consolidated["counts"]["deduped_semantic_policies"] == 50_010 + assert consolidated["counts"]["eligible_all_alpha_return_floor_policies"] == 27_508 + assert terminal["runtime_status"]["total_checks"] == 296_544 + assert promotion["run_tag"] == "ijds-rebaseline-2026-06-07" + assert terminal["claim_summary"]["declared_return_floor"] == pytest.approx( + round(promotion["final_champion"]["realized_total_return"], 2) ) - assert float(table["Selected CRPTO"]["realized_return"]) == pytest.approx( - selected["realized_return"], abs=1e-6 - ) - assert contrasts["realized_return_cost_pct"] == pytest.approx(5.8749883793) - assert contrasts["weighted_default_rate_reduction"] == pytest.approx(0.08305) - assert contrasts["markov_threshold_reduction"] == pytest.approx(0.4354954304) - - anchors = ("196,369.14", "5.875", "8.305", "43.55") - for surface in (PAPER, SUPPLEMENT, SUBMISSION): - text = _normalized_manuscript_text(surface) - assert all(anchor in text for anchor in anchors), surface - -def test_pool93_two_tag_scheme_is_coherent() -> None: - """The frozen rebaseline chain stays the declared return floor for pool93.""" - promotion = _load_json(PROMOTION) - assert promotion["run_tag"] == REBASELINE_RUN_TAG - frozen_return = promotion["final_champion"]["realized_total_return"] - assert frozen_return == pytest.approx(EXPECTED_FROZEN_RETURN, abs=1e-6) - - terminal = _load_json(TERMINAL_GOVERNANCE) - floor = terminal["claim_summary"]["declared_return_floor"] - assert floor == pytest.approx(round(frozen_return, 2), abs=1e-9) - - -def test_pool93_manifest_block_agrees() -> None: - """EXTRACTION_MANIFEST pool93 block mirrors the governance sidecars.""" - manifest = _load_json(MANIFEST) - block = manifest.get("pool93_ijds_promotion") - if block is None: - pytest.skip("pool93_ijds_promotion block not yet added to EXTRACTION_MANIFEST.json.") - - consolidated = _load_json(CONSOLIDATED_GOVERNANCE) - body = consolidated["selected_candidates"]["paper_body"] - point = block["paper_body_point"] - assert block["terminal_run_tag"] == TERMINAL_RUN_TAG - assert point["realized_total_return"] == pytest.approx(body["return"], abs=1e-6) - assert point["alpha01_gamma_cp"] == pytest.approx(body["Gamma_CP"], abs=1e-9) - assert point["alpha01_weighted_miscoverage_V"] == pytest.approx(body["V"], abs=1e-9) - assert point["alpha01_gamma_internalized"] == pytest.approx( - body["Gamma_internalized"], abs=1e-9 - ) - assert point["alpha01_gamma_residual"] == pytest.approx(body["Gamma_residual"], abs=1e-9) - assert point["endpoint_budget_alpha01"] == pytest.approx(body["endpoint_budget"], abs=1e-9) - assert point["markov_threshold_alpha01"] == pytest.approx(body["Markov_threshold"], abs=1e-9) - assert point["declared_return_floor"] == EXPECTED_FLOOR - assert ( - block["frontier_counts"]["deduped_semantic_policies"] - == (EXPECTED_FRONTIER_COUNTS["deduped_semantic_policies"]) - ) +def test_historical_pool93_tables_remain_hash_protected() -> None: + manifest = _load(MANIFEST) hashed = set(manifest["critical_hashes"]) - missing = [ - f"reports/crpto/tables/{stem}.{ext}" + expected = { + f"reports/crpto/tables/{stem}.{suffix}" for stem in POOL93_TABLE_STEMS - for ext in ("csv", "tex") - if f"reports/crpto/tables/{stem}.{ext}" not in hashed - ] - assert not missing, "pool93 tables not hash-protected: " + ", ".join(missing) + for suffix in ("csv", "tex") + } + assert expected.issubset(hashed) + assert all((TABLES / path.split("/")[-1]).is_file() for path in expected) diff --git a/tests/test_publication_targets.py b/tests/test_publication_targets.py index 7f886ba..2cb08e8 100644 --- a/tests/test_publication_targets.py +++ b/tests/test_publication_targets.py @@ -51,7 +51,7 @@ def test_journal_strengthening_pack_classifies_current_and_backlog_items() -> No assert set(included) == { "regret_auditability_frontier", "tail_risk_oce_cvar_diagnostic", - "pool93_frontier_and_selected_allocation", + "exact_alpha_calibration_selected_policy", "matched_point_pd_baseline", "robust_satisficing_margins", "dependence_aware_bound", @@ -62,7 +62,7 @@ def test_journal_strengthening_pack_classifies_current_and_backlog_items() -> No } assert included["regret_auditability_frontier"]["status"] == "include_body" assert included["tail_risk_oce_cvar_diagnostic"]["status"] == "include_supplement" - assert included["pool93_frontier_and_selected_allocation"]["status"] == ( + assert included["exact_alpha_calibration_selected_policy"]["status"] == ( "include_body_and_supplement" ) assert included["matched_point_pd_baseline"]["status"] == ("include_body_and_supplement") @@ -74,13 +74,15 @@ def test_journal_strengthening_pack_classifies_current_and_backlog_items() -> No assert included["multidataset_external_replication"]["status"] == ( "include_supplement_or_short_body" ) - pool93_artifacts = included["pool93_frontier_and_selected_allocation"]["artifacts"] - assert "reports/crpto/tables/crpto_tableA35_pool93_ijds_frontier.csv" in pool93_artifacts - assert "reports/crpto/tables/crpto_tableA39_pool93_body_bootstrap_metrics.csv" in ( - pool93_artifacts + active_artifacts = included["exact_alpha_calibration_selected_policy"]["artifacts"] + assert "reports/crpto/tables/crpto_tableA35_exact_alpha_grid.csv" in active_artifacts + assert "reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.csv" in ( + active_artifacts ) - assert "reports/crpto/tables/crpto_tableA40_pool93_point_baseline.csv" in (pool93_artifacts) - for artifact in pool93_artifacts: + assert "reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.csv" in ( + active_artifacts + ) + for artifact in active_artifacts: assert Path(artifact).exists(), artifact multidataset_artifacts = included["multidataset_external_replication"]["artifacts"] assert "reports/crpto/tables/crpto_tableA28_external_lp_exhaustiveness.csv" in ( @@ -103,7 +105,9 @@ def test_journal_strengthening_pack_classifies_current_and_backlog_items() -> No paper_readme = Path("paper/README.md").read_text(encoding="utf-8") for text in (body, supplement, paper_readme): - assert "regret-auditability" in text.lower() - assert "OCE/CVaR" in text - assert "satisficing" in text.lower() - assert "multi-dataset" in text.lower() + assert "midpoint" in text.lower() + assert "calibration" in text.lower() + assert "point-PD" in text + + for diagnostic in ("OCE/CVaR", "SPO+", "Prosper"): + assert diagnostic in supplement diff --git a/tests/test_scripts/test_build_ijds_calibration_selected_evidence.py b/tests/test_scripts/test_build_ijds_calibration_selected_evidence.py new file mode 100644 index 0000000..2f3b250 --- /dev/null +++ b/tests/test_scripts/test_build_ijds_calibration_selected_evidence.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from scripts.build_ijds_calibration_selected_evidence import ( + build_baseline_table, + build_bootstrap_table, + build_grade_table, +) + + +def _allocations() -> pd.DataFrame: + return pd.DataFrame( + { + "role": ["calibration_selected", "calibration_selected", "point_pd_matched_tau"], + "grade": ["A", "B", "A"], + "funded_exposure": [100.0, 100.0, 200.0], + "funded_weight": [0.5, 0.5, 1.0], + "outcome": [0.0, 1.0, 0.0], + "miscoverage": [0.0, 1.0, 0.0], + "pd_point": [0.1, 0.2, 0.2], + "pd_effective": [0.2, 0.3, 0.2], + "pd_high": [0.4, 0.6, 0.8], + "int_rate": [0.1, 0.2, 0.1], + "realized_return_contribution": [10.0, -45.0, 20.0], + } + ) + + +def _evaluation() -> pd.DataFrame: + return pd.DataFrame( + [ + { + "period": "full_oot", + "role": "calibration_selected", + "n_panel": 10, + "n_funded": 2, + "total_allocated": 200.0, + "expected_objective": 5.0, + "realized_return": -35.0, + "weighted_outcome": 0.5, + "weighted_miscoverage": 0.5, + "weighted_pd_point": 0.15, + "weighted_pd_effective": 0.25, + "gamma_cp": 0.35, + "gamma_internalized": 0.10, + "gamma_residual": 0.25, + "endpoint_budget": 0.50, + "markov_loss_threshold": 0.80, + }, + { + "period": "full_oot", + "role": "point_pd_matched_tau", + "n_panel": 10, + "n_funded": 1, + "total_allocated": 200.0, + "expected_objective": 10.0, + "realized_return": 20.0, + "weighted_outcome": 0.0, + "weighted_miscoverage": 0.0, + "weighted_pd_point": 0.20, + "weighted_pd_effective": 0.20, + "gamma_cp": 0.60, + "gamma_internalized": 0.0, + "gamma_residual": 0.60, + "endpoint_budget": 0.80, + "markov_loss_threshold": 1.10, + }, + ] + ) + + +def test_grade_table_preserves_selected_exposure() -> None: + table = build_grade_table(_allocations()) + + assert table["exposure"].sum() == 200.0 + assert table["exposure_share"].sum() == 1.0 + + +def test_bootstrap_is_deterministic_and_uses_official_observed_values() -> None: + first = build_bootstrap_table(_allocations(), _evaluation(), n_draws=100, seed=7) + second = build_bootstrap_table(_allocations(), _evaluation(), n_draws=100, seed=7) + + pd.testing.assert_frame_equal(first, second) + observed = first.set_index("metric")["observed"] + assert observed["realized_return"] == -35.0 + assert observed["Gamma_CP"] == 0.35 + + +def test_bootstrap_rejects_allocation_evaluation_drift() -> None: + evaluation = _evaluation() + evaluation.loc[evaluation["role"].eq("calibration_selected"), "realized_return"] = 1.0 + + with pytest.raises(ValueError, match="realized_return"): + build_bootstrap_table(_allocations(), evaluation, n_draws=10, seed=7) + + +def test_baseline_table_uses_selected_policy_as_zero_delta() -> None: + table = build_baseline_table(_evaluation()) + selected = table.loc[table["policy"].eq("Calibration-selected 50/50 CRPTO")].iloc[0] + + assert np.isclose(selected["return_delta_vs_selected"], 0.0) + assert np.isclose(selected["default_delta_vs_selected"], 0.0) diff --git a/tests/test_scripts/test_optimize_portfolio_tradeoff.py b/tests/test_scripts/test_optimize_portfolio_tradeoff.py index 3ed4526..688add0 100644 --- a/tests/test_scripts/test_optimize_portfolio_tradeoff.py +++ b/tests/test_scripts/test_optimize_portfolio_tradeoff.py @@ -4,7 +4,6 @@ import pandas as pd import pytest -import scripts.optimize_portfolio_tradeoff as tradeoff_module from scripts.optimize_portfolio import _align_candidates_and_intervals from scripts.optimize_portfolio_tradeoff import ( _align_loans_and_intervals, @@ -13,6 +12,7 @@ _select_champion_policy, _solve_single, ) +from src.optimization import policy_evaluation def test_portfolio_alignment_wrappers_share_strict_id_contract() -> None: @@ -72,7 +72,7 @@ def _fake_optimize_portfolio_allocation(**kwargs: object) -> dict[str, object]: } monkeypatch.setattr( - tradeoff_module, + policy_evaluation, "optimize_portfolio_allocation", _fake_optimize_portfolio_allocation, ) diff --git a/tests/test_scripts/test_run_ty_advisory.py b/tests/test_scripts/test_run_ty_advisory.py index 71171e9..43db7b3 100644 --- a/tests/test_scripts/test_run_ty_advisory.py +++ b/tests/test_scripts/test_run_ty_advisory.py @@ -11,7 +11,9 @@ def test_active_ty_scope_excludes_archived_optional_and_protected_paths() -> Non assert "scripts/run_spo_real.py" not in files assert "src/optimization/cuopt_adapter.py" not in files assert all(not path.startswith("scripts/archive/") for path in files) - assert all(not path.startswith("scripts/experiments/") for path in files) + assert "scripts/experiments/ijds_policy_support.py" in files + assert "scripts/experiments/run_ijds_calibration_selected_policy_challenger.py" in files + assert "scripts/experiments/run_ijds_exact_alpha_grid_challenger.py" in files assert all( not (path.startswith("scripts/search/run_") and path.endswith(".py")) for path in files ) @@ -21,7 +23,10 @@ def test_active_ty_scope_keeps_live_ijds_helpers() -> None: files = set(iter_python_files(scope="active")) assert "scripts/compile_ijds_submission.py" in files + assert "scripts/build_ijds_calibration_selected_evidence.py" in files assert "scripts/search/build_pool93_body_allocation_audit.py" in files + assert "src/models/conformal_alpha_grid.py" in files + assert "src/optimization/policy_selection.py" in files assert "src/optimization/portfolio_model.py" in files diff --git a/tests/test_supplement_table_sync.py b/tests/test_supplement_table_sync.py index d2fa0de..a425dcd 100644 --- a/tests/test_supplement_table_sync.py +++ b/tests/test_supplement_table_sync.py @@ -1,13 +1,9 @@ -"""Drift guard for hand-authored multidataset tables in the paper surfaces. +"""Drift guards for the diagnostic multidataset evidence. -The online supplement, the IJDS body, and book chapter 30 embed the external -replication tables (A25, A34) as hand-written Markdown rather than reading the -CSV at render time. That is convenient but can silently drift if a CSV is -regenerated without updating the prose. These tests assert the headline numbers -shown in those surfaces still match the source CSVs. - -Only a few anchor values are checked -- enough to catch a stale copy/paste, -without re-implementing a full Markdown table parser. +Chapter 30 retains the detailed A25/A34 values. The submitted body and +supplement deliberately keep only the scope boundary: these older replications +are static transfer evidence, not active-policy certificates. Tests preserve +both contracts without forcing retired detail back into the IJDS narrative. """ from __future__ import annotations @@ -39,27 +35,37 @@ def _text(path: Path) -> str: return path.read_text(encoding="utf-8") -def test_a25_robust_objectives_match_surfaces() -> None: - """A25 robust LP objectives (as `$N`) appear in supplement, paper, and book.""" +def test_a25_robust_objectives_match_book_and_diagnostic_status() -> None: + """A25 values remain in the book while IJDS surfaces retain their boundary.""" rows = _read_rows("crpto_tableA25_external_replication_gate.csv") - surfaces = {p: _text(p) for p in (SUPPLEMENT, PAPER, BOOK_CH30)} + book = _text(BOOK_CH30) missing: list[str] = [] for row in rows: dollars = f"${round(float(row['robust_objective'])):,}" - for path, body in surfaces.items(): - if dollars not in body: - missing.append(f"{row['dataset']} {dollars} missing in {path.name}") + if dollars not in book: + missing.append(f"{row['dataset']} {dollars} missing in {BOOK_CH30.name}") assert not missing, "A25 robust objective drift:\n" + "\n".join(missing) + supplement = _text(SUPPLEMENT) + paper = _text(PAPER) + assert "A25--A34" in supplement + assert "Static transfer evidence; not active Lending Club certificates." in supplement + assert "Prosper" in paper and "Freddie/Mendeley" in paper + assert "retained as diagnostics or external context" in paper + -def test_a34_price_of_robustness_match_surfaces() -> None: - """A34 signed price-of-robustness (as `+X.XX%`) appears in supplement and book.""" +def test_a34_price_of_robustness_matches_historical_book_surface() -> None: + """A34 signed price-of-robustness values remain traceable in chapter 30.""" rows = _read_rows("crpto_tableA34_price_of_robustness_cross_dataset.csv") - surfaces = {p: _text(p) for p in (SUPPLEMENT, BOOK_CH30)} + book = _text(BOOK_CH30) missing: list[str] = [] for row in rows: pct = f"+{float(row['price_of_robustness_pct']) * 100:.2f}%" - for path, body in surfaces.items(): - if pct not in body: - missing.append(f"{row['application']} {pct} missing in {path.name}") + if pct not in book: + missing.append(f"{row['application']} {pct} missing in {BOOK_CH30.name}") assert not missing, "A34 price-of-robustness drift:\n" + "\n".join(missing) + + supplement = _text(SUPPLEMENT) + assert "older frozen replication contracts" in supplement + assert "cannot be quoted as direct" in supplement + assert "replications of the active midpoint policy" in supplement From de9a5d6b1d4b8097e84560daa1f8958275deea77 Mon Sep 17 00:00:00 2001 From: Carlos Alfredo Vergara Rojas Date: Thu, 9 Jul 2026 21:33:54 -0500 Subject: [PATCH 4/7] chore: record final IJDS replay provenance --- .../calibration_selected_policy_summary.json | 4 +- .../portfolio/ijds_policy_governance.json | 2 +- .../conformal/exact_alpha_grid_summary.json | 330 +++++++++--------- 3 files changed, 168 insertions(+), 168 deletions(-) diff --git a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6/portfolio/calibration_selected_policy_summary.json b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6/portfolio/calibration_selected_policy_summary.json index fe63910..f388671 100644 --- a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6/portfolio/calibration_selected_policy_summary.json +++ b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6/portfolio/calibration_selected_policy_summary.json @@ -53,7 +53,7 @@ "selection_rule": "maximize expected point-PD objective on the calibration holdout under a 0.60 endpoint-plus-Markov screen, the effective-PD cap, and full budget use" }, "evaluation_path": "data\\processed\\experiments\\champion_reopen\\champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6\\portfolio\\calibration_selected_policy_oot_evaluation.csv", - "generated_at_utc": "2026-07-10T02:25:53.745446+00:00", + "generated_at_utc": "2026-07-10T02:32:57.168718+00:00", "grid_size": 9, "incumbent_policy": { "candidate_id": "linear-006", @@ -145,5 +145,5 @@ "effective_pd_cap_slack" ], "selector_forbidden_columns_present": [], - "source_commit": "17811d8fb4c5dfc0035f86ac7095088533bfec5b" + "source_commit": "4134a6d32156fb02d7c7a2787383f20caea315eb" } diff --git a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6/portfolio/ijds_policy_governance.json b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6/portfolio/ijds_policy_governance.json index a012946..9d3491b 100644 --- a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6/portfolio/ijds_policy_governance.json +++ b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6/portfolio/ijds_policy_governance.json @@ -30,7 +30,7 @@ "weighted_pd_effective": 0.17, "weighted_pd_point": 0.0819489105765324 }, - "generated_at_utc": "2026-07-10T02:25:53.745446+00:00", + "generated_at_utc": "2026-07-10T02:32:57.168718+00:00", "paper_tables": { "alpha": [ "reports/crpto/tables/crpto_tableA35_exact_alpha_grid.csv", diff --git a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1/conformal/exact_alpha_grid_summary.json b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1/conformal/exact_alpha_grid_summary.json index 70b3255..ef2eb7a 100644 --- a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1/conformal/exact_alpha_grid_summary.json +++ b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1/conformal/exact_alpha_grid_summary.json @@ -1,244 +1,244 @@ { - "schema_version": "2026-07-09.1", - "generated_at_utc": "2026-07-10T00:28:51.509823+00:00", - "run_tag": "champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1", - "source_commit": "17811d8fb4c5dfc0035f86ac7095088533bfec5b", - "config_path": "configs\\experiments\\champion_reopen_ijds_exact_alpha_grid_v1.yaml", - "config_sha256": "c2d2af0d1c0e0f83ef8612888f9b12d423063a601f04c8c51c9680e92327ef3f", - "source": { - "upstream_canonical_run_tag": "champion-reopen-2026-06-19__hpo-wave1__pool93__seed42", - "conformal_namespace": "champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1", - "conformal_results_path": "models/conformal_gap/champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1/conformal_results_mondrian.pkl", - "conformal_intervals_path": "data/processed/conformal_gap/champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1/conformal_intervals_mondrian.parquet", - "conformal_results_sha256": "cb8db92908739c37a11f373b9802f6b74831d8d9c26c4824dae586423d3cb07b", - "conformal_intervals_sha256": "3784cc68a5b72cf290c9678e6cd5d30f7b794dac4c1eedaf393fe9f72cc0c3a1" - }, - "recipe": { - "partition": "score_decile_mondrian", - "partition_probability_source": "calibrated", - "n_score_bins": 5, - "fallback_mode": "score_only", - "score_scale_family": "bernoulli_sqrt", - "min_group_size": 100, - "reference_target_alpha": 0.1, - "reference_used_alpha": 0.095, - "calibration_fraction": 0.75, - "tuning_holdout_ratio": 0.2, - "tuning_random_state": 42, - "group_multipliers": { - "score_q00": 1.02, - "score_q01": 1.05, - "score_q02": 1.05, - "score_q03": 1.02, - "score_q04": 1.02 - }, - "temporal_segment_multipliers": {}, - "temporal_segment_freq": "Q", - "global_rebalance_factor": 1.0 - }, "alpha_mapping": "proportional frozen conservative ratio alpha_used/alpha_target from the selected 90% recipe", - "reference_replay": { - "point_max_abs": 4.440892098500626e-16, - "low_max_abs": 3.3306690738754696e-16, - "high_max_abs": 6.661338147750939e-16, - "tolerance": 1e-12, - "pass": true - }, "alpha_summaries": [ { - "target_alpha": 0.01, - "used_alpha": 0.0095, - "target_coverage": 0.99, - "empirical_coverage": 0.9967204706919157, - "coverage_gap": 0.006720470691915725, "avg_width": 0.9882149910656284, - "median_width": 1.0, - "min_partition_coverage": 0.989628743645835, - "min_grade_coverage": 0.9722703639514731, - "high_endpoint_mean": 0.9883685381679577, - "high_endpoint_min": 0.28857911767290634, - "high_endpoint_p01": 0.6715297528015821, - "high_endpoint_p10": 1.0, - "high_endpoint_at_one_rate": 0.9354243342519386, - "partition_count": 5, + "coverage_gap": 0.006720470691915725, + "empirical_coverage": 0.9967204706919157, "group_quantiles": { "score_q00": 4.620892417510161, "score_q01": 2.9770143672127003, "score_q02": 2.2280905155147157, "score_q03": 1.7679793270133157, "score_q04": 1.379561550348839 - } + }, + "high_endpoint_at_one_rate": 0.9354243342519386, + "high_endpoint_mean": 0.9883685381679577, + "high_endpoint_min": 0.28857911767290634, + "high_endpoint_p01": 0.6715297528015821, + "high_endpoint_p10": 1.0, + "median_width": 1.0, + "min_grade_coverage": 0.9722703639514731, + "min_partition_coverage": 0.989628743645835, + "partition_count": 5, + "target_alpha": 0.01, + "target_coverage": 0.99, + "used_alpha": 0.0095 }, { - "target_alpha": 0.03, - "used_alpha": 0.028499999999999998, - "target_coverage": 0.97, - "empirical_coverage": 0.9884783056246818, - "coverage_gap": 0.01847830562468178, "avg_width": 0.9697789837688854, - "median_width": 1.0, - "min_partition_coverage": 0.9697990273996318, - "min_grade_coverage": 0.9664933564413634, - "high_endpoint_mean": 0.9699657056860221, - "high_endpoint_min": 0.22825533153683755, - "high_endpoint_p01": 0.5334804414885259, - "high_endpoint_p10": 0.9018799702618802, - "high_endpoint_at_one_rate": 0.8323141991338864, - "partition_count": 5, + "coverage_gap": 0.01847830562468178, + "empirical_coverage": 0.9884783056246818, "group_quantiles": { "score_q00": 3.642519971959776, "score_q01": 2.8072631501367455, "score_q02": 2.1737923639215784, "score_q03": 1.7387907144323151, "score_q04": 1.3580168082764283 - } + }, + "high_endpoint_at_one_rate": 0.8323141991338864, + "high_endpoint_mean": 0.9699657056860221, + "high_endpoint_min": 0.22825533153683755, + "high_endpoint_p01": 0.5334804414885259, + "high_endpoint_p10": 0.9018799702618802, + "median_width": 1.0, + "min_grade_coverage": 0.9664933564413634, + "min_partition_coverage": 0.9697990273996318, + "partition_count": 5, + "target_alpha": 0.03, + "target_coverage": 0.97, + "used_alpha": 0.028499999999999998 }, { - "target_alpha": 0.05, - "used_alpha": 0.0475, - "target_coverage": 0.95, - "empirical_coverage": 0.9728246932664907, - "coverage_gap": 0.022824693266490725, "avg_width": 0.9535799768723978, - "median_width": 1.0, - "min_partition_coverage": 0.9505830880848586, - "min_grade_coverage": 0.9525750124329999, - "high_endpoint_mean": 0.953801858012357, - "high_endpoint_min": 0.20305073151598757, - "high_endpoint_p01": 0.47580041346523627, - "high_endpoint_p10": 0.806835962194862, - "high_endpoint_at_one_rate": 0.7181555175913519, - "partition_count": 5, + "coverage_gap": 0.022824693266490725, + "empirical_coverage": 0.9728246932664907, "group_quantiles": { "score_q00": 3.233734520299044, "score_q01": 2.6752015014256125, "score_q02": 2.1212263259384927, "score_q03": 1.7077473640552945, "score_q04": 1.3388102247632694 - } + }, + "high_endpoint_at_one_rate": 0.7181555175913519, + "high_endpoint_mean": 0.953801858012357, + "high_endpoint_min": 0.20305073151598757, + "high_endpoint_p01": 0.47580041346523627, + "high_endpoint_p10": 0.806835962194862, + "median_width": 1.0, + "min_grade_coverage": 0.9525750124329999, + "min_partition_coverage": 0.9505830880848586, + "partition_count": 5, + "target_alpha": 0.05, + "target_coverage": 0.95, + "used_alpha": 0.0475 }, { - "target_alpha": 0.07, - "used_alpha": 0.0665, - "target_coverage": 0.9299999999999999, - "empirical_coverage": 0.9552712654721186, - "coverage_gap": 0.025271265472118665, "avg_width": 0.795666199875876, - "median_width": 1.0, - "min_partition_coverage": 0.9347822665680426, - "min_grade_coverage": 0.938442835829143, - "high_endpoint_mean": 0.7959333620592705, - "high_endpoint_min": 0.023668097607277484, - "high_endpoint_p01": 0.06528823147517372, - "high_endpoint_p10": 0.13040211717455735, - "high_endpoint_at_one_rate": 0.6135500904760013, - "partition_count": 5, + "coverage_gap": 0.025271265472118665, + "empirical_coverage": 0.9552712654721186, "group_quantiles": { "score_q00": 0.32438420480886127, "score_q01": 2.567436984095836, "score_q02": 2.0727963588460026, "score_q03": 1.6764580225891925, "score_q04": 1.3186561259152434 - } + }, + "high_endpoint_at_one_rate": 0.6135500904760013, + "high_endpoint_mean": 0.7959333620592705, + "high_endpoint_min": 0.023668097607277484, + "high_endpoint_p01": 0.06528823147517372, + "high_endpoint_p10": 0.13040211717455735, + "median_width": 1.0, + "min_grade_coverage": 0.938442835829143, + "min_partition_coverage": 0.9347822665680426, + "partition_count": 5, + "target_alpha": 0.07, + "target_coverage": 0.9299999999999999, + "used_alpha": 0.0665 }, { - "target_alpha": 0.1, - "used_alpha": 0.095, - "target_coverage": 0.9, - "empirical_coverage": 0.9348356081757077, - "coverage_gap": 0.03483560817570763, "avg_width": 0.7888790790793794, - "median_width": 1.0, - "min_partition_coverage": 0.9263099219620958, - "min_grade_coverage": 0.9267968822615554, - "high_endpoint_mean": 0.7892193805747533, - "high_endpoint_min": 0.023406884837288408, - "high_endpoint_p01": 0.06469045329656556, - "high_endpoint_p10": 0.12941711012832322, - "high_endpoint_at_one_rate": 0.5178730735474177, - "partition_count": 5, + "coverage_gap": 0.03483560817570763, + "empirical_coverage": 0.9348356081757077, "group_quantiles": { "score_q00": 0.3201476773467629, "score_q01": 2.4331722444504513, "score_q02": 2.007709882293012, "score_q03": 1.633544476383572, "score_q04": 1.2899294761760491 - } + }, + "high_endpoint_at_one_rate": 0.5178730735474177, + "high_endpoint_mean": 0.7892193805747533, + "high_endpoint_min": 0.023406884837288408, + "high_endpoint_p01": 0.06469045329656556, + "high_endpoint_p10": 0.12941711012832322, + "median_width": 1.0, + "min_grade_coverage": 0.9267968822615554, + "min_partition_coverage": 0.9263099219620958, + "partition_count": 5, + "target_alpha": 0.1, + "target_coverage": 0.9, + "used_alpha": 0.095 }, { - "target_alpha": 0.12, - "used_alpha": 0.11399999999999999, - "target_coverage": 0.88, - "empirical_coverage": 0.9182645944471934, - "coverage_gap": 0.038264594447193434, "avg_width": 0.7831737841625059, - "median_width": 0.9867966540367542, - "min_partition_coverage": 0.906874767744333, - "min_grade_coverage": 0.9057375416605808, - "high_endpoint_mean": 0.7835760369425744, - "high_endpoint_min": 0.023236857746810564, - "high_endpoint_p01": 0.06430135101621326, - "high_endpoint_p10": 0.12877595509528877, - "high_endpoint_at_one_rate": 0.4545109781160043, - "partition_count": 5, + "coverage_gap": 0.038264594447193434, + "empirical_coverage": 0.9182645944471934, "group_quantiles": { "score_q00": 0.31739006163700956, "score_q01": 2.3534507550187485, "score_q02": 1.9655367760210574, "score_q03": 1.6066047959404717, "score_q04": 1.269964951004726 - } + }, + "high_endpoint_at_one_rate": 0.4545109781160043, + "high_endpoint_mean": 0.7835760369425744, + "high_endpoint_min": 0.023236857746810564, + "high_endpoint_p01": 0.06430135101621326, + "high_endpoint_p10": 0.12877595509528877, + "median_width": 0.9867966540367542, + "min_grade_coverage": 0.9057375416605808, + "min_partition_coverage": 0.906874767744333, + "partition_count": 5, + "target_alpha": 0.12, + "target_coverage": 0.88, + "used_alpha": 0.11399999999999999 }, { - "target_alpha": 0.15, - "used_alpha": 0.1425, - "target_coverage": 0.85, - "empirical_coverage": 0.8860977574231856, - "coverage_gap": 0.03609775742318566, "avg_width": 0.6461762449744832, - "median_width": 0.94425532567159, - "min_partition_coverage": 0.8635623982976592, - "min_grade_coverage": 0.8631010078209027, - "high_endpoint_mean": 0.6466929852254412, - "high_endpoint_min": 0.02298905825369661, - "high_endpoint_p01": 0.06373426874967675, - "high_endpoint_p10": 0.1278415281651632, - "high_endpoint_at_one_rate": 0.33485872380078663, - "partition_count": 5, + "coverage_gap": 0.03609775742318566, + "empirical_coverage": 0.8860977574231856, "group_quantiles": { "score_q00": 0.31337107987851737, "score_q01": 0.4420899636645897, "score_q02": 1.9079870902279796, "score_q03": 1.5675398154360425, "score_q04": 1.2412696239135141 - } + }, + "high_endpoint_at_one_rate": 0.33485872380078663, + "high_endpoint_mean": 0.6466929852254412, + "high_endpoint_min": 0.02298905825369661, + "high_endpoint_p01": 0.06373426874967675, + "high_endpoint_p10": 0.1278415281651632, + "median_width": 0.94425532567159, + "min_grade_coverage": 0.8631010078209027, + "min_partition_coverage": 0.8635623982976592, + "partition_count": 5, + "target_alpha": 0.15, + "target_coverage": 0.85, + "used_alpha": 0.1425 }, { - "target_alpha": 0.2, - "used_alpha": 0.19, - "target_coverage": 0.8, - "empirical_coverage": 0.8493800317117481, - "coverage_gap": 0.049380031711748096, "avg_width": 0.6365853175970958, - "median_width": 0.9128005695874017, - "min_partition_coverage": 0.8293756967670011, - "min_grade_coverage": 0.828006532910199, - "high_endpoint_mean": 0.6373641323567814, - "high_endpoint_min": 0.022569416686283303, - "high_endpoint_p01": 0.06277393066265638, - "high_endpoint_p10": 0.1262591020808178, - "high_endpoint_at_one_rate": 0.24466805601204902, - "partition_count": 5, + "coverage_gap": 0.049380031711748096, + "empirical_coverage": 0.8493800317117481, "group_quantiles": { "score_q00": 0.30656504576058813, "score_q01": 0.43573433937656825, "score_q02": 1.8175729021078393, "score_q03": 1.5052283270500615, "score_q04": 1.194771966347316 - } + }, + "high_endpoint_at_one_rate": 0.24466805601204902, + "high_endpoint_mean": 0.6373641323567814, + "high_endpoint_min": 0.022569416686283303, + "high_endpoint_p01": 0.06277393066265638, + "high_endpoint_p10": 0.1262591020808178, + "median_width": 0.9128005695874017, + "min_grade_coverage": 0.828006532910199, + "min_partition_coverage": 0.8293756967670011, + "partition_count": 5, + "target_alpha": 0.2, + "target_coverage": 0.8, + "used_alpha": 0.19 } ], + "claim_boundary": "Recomputes finite-sample Mondrian quantiles for every declared alpha while freezing the selected partition, calibration fit/holdout split, score scale, and holdout-learned widening factors. It is a retrospective OOT audit, not a post-selection or live-deployment guarantee.", + "config_path": "configs\\experiments\\champion_reopen_ijds_exact_alpha_grid_v1.yaml", + "config_sha256": "c2d2af0d1c0e0f83ef8612888f9b12d423063a601f04c8c51c9680e92327ef3f", + "generated_at_utc": "2026-07-10T02:32:16.941448+00:00", "grid_path": "data\\processed\\experiments\\champion_reopen\\champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1\\conformal\\exact_alpha_grid.parquet", "grid_rows": 276869, - "claim_boundary": "Recomputes finite-sample Mondrian quantiles for every declared alpha while freezing the selected partition, calibration fit/holdout split, score scale, and holdout-learned widening factors. It is a retrospective OOT audit, not a post-selection or live-deployment guarantee." + "recipe": { + "calibration_fraction": 0.75, + "fallback_mode": "score_only", + "global_rebalance_factor": 1.0, + "group_multipliers": { + "score_q00": 1.02, + "score_q01": 1.05, + "score_q02": 1.05, + "score_q03": 1.02, + "score_q04": 1.02 + }, + "min_group_size": 100, + "n_score_bins": 5, + "partition": "score_decile_mondrian", + "partition_probability_source": "calibrated", + "reference_target_alpha": 0.1, + "reference_used_alpha": 0.095, + "score_scale_family": "bernoulli_sqrt", + "temporal_segment_freq": "Q", + "temporal_segment_multipliers": {}, + "tuning_holdout_ratio": 0.2, + "tuning_random_state": 42 + }, + "reference_replay": { + "high_max_abs": 6.661338147750939e-16, + "low_max_abs": 3.3306690738754696e-16, + "pass": true, + "point_max_abs": 4.440892098500626e-16, + "tolerance": 1e-12 + }, + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1", + "schema_version": "2026-07-09.1", + "source": { + "conformal_intervals_path": "data/processed/conformal_gap/champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1/conformal_intervals_mondrian.parquet", + "conformal_intervals_sha256": "3784cc68a5b72cf290c9678e6cd5d30f7b794dac4c1eedaf393fe9f72cc0c3a1", + "conformal_namespace": "champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1", + "conformal_results_path": "models/conformal_gap/champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1/conformal_results_mondrian.pkl", + "conformal_results_sha256": "cb8db92908739c37a11f373b9802f6b74831d8d9c26c4824dae586423d3cb07b", + "upstream_canonical_run_tag": "champion-reopen-2026-06-19__hpo-wave1__pool93__seed42" + }, + "source_commit": "4134a6d32156fb02d7c7a2787383f20caea315eb" } From e0daf55685988408bc68f5b65df0298174f1a516 Mon Sep 17 00:00:00 2001 From: Carlos Alfredo Vergara Rojas Date: Thu, 9 Jul 2026 23:23:50 -0500 Subject: [PATCH 5/7] Promote deterministic IJDS policy selector v7 --- .codex/skills/crpto/SKILL.md | 11 +- CLAUDE.md | 14 +- README.md | 5 +- configs/crpto_publication_targets.yaml | 9 +- ...ds_calibration_selected_endpoint28_v7.yaml | 52 +++++ docs/SCOPE_AND_GOVERNANCE.md | 17 +- .../ijds_tooling_decisions_2026-07-09.md | 21 +- docs/research/active_claims_2026-07-04.md | 42 ++-- ..._alpha_calibration_selection_2026-07-09.md | 53 ++++- justfile | 2 +- paper/CRPTO.qmd | 9 +- paper/CRPTO_ijds.qmd | 164 +++++++++----- paper/README.md | 9 +- paper/submission/CLAIM_AUDIT_MATRIX.md | 10 +- .../submission/COVER_LETTER_AND_DISCLOSURE.md | 14 +- paper/submission/CRPTO_ijds_submission.tex | 159 +++++++++----- .../IJDS_SUBMISSION_ROADMAP_2026-08-10.md | 15 +- paper/submission/README.md | 19 +- paper/submission/REPRODUCIBILITY_PACKAGE.md | 13 +- .../submission/SCHOLARONE_FINAL_CHECKLIST.md | 2 +- paper/supplement_ijds.qmd | 94 ++++---- reports/crpto/tables/README.md | 19 +- ...o_tableA36_calibration_policy_selector.csv | 20 +- ...o_tableA36_calibration_policy_selector.tex | 22 +- ...ableA39_calibration_selected_bootstrap.csv | 17 +- ...ableA39_calibration_selected_bootstrap.tex | 19 +- ...uild_ijds_calibration_selected_evidence.py | 190 +++++++++++++--- scripts/check_publication_integrity.py | 2 +- scripts/compile_ijds_submission.py | 29 ++- scripts/experiments/ijds_policy_support.py | 34 ++- ..._calibration_selected_policy_challenger.py | 207 +++++++++++++++--- src/optimization/policy_selection.py | 86 +++++++- ..._calibration_selected_policy_challenger.py | 25 +++ tests/test_ijds_active_claim_sync.py | 23 +- .../test_policy_selection.py | 37 +++- ...uild_ijds_calibration_selected_evidence.py | 6 +- .../test_compile_ijds_submission.py | 13 +- 37 files changed, 1109 insertions(+), 374 deletions(-) create mode 100644 configs/experiments/champion_reopen_ijds_calibration_selected_endpoint28_v7.yaml diff --git a/.codex/skills/crpto/SKILL.md b/.codex/skills/crpto/SKILL.md index 3394e9e..c03cfe4 100644 --- a/.codex/skills/crpto/SKILL.md +++ b/.codex/skills/crpto/SKILL.md @@ -19,14 +19,17 @@ assumptions. ## Active IJDS Policy - Run tag: - `champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6` + `champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7` - Exact conformal replay: target `alpha=0.10`, frozen used alpha `0.095`. - Decision score: `q=(p+u)/2`. - Risk tolerance: `tau=0.17`. - Objective: point-PD expected net return; conformal `q` is the risk guardrail. -- Selector: nine round-number policies on the temporal calibration holdout; - five satisfy full budget, effective-PD, and `0.60` threshold screens; no - outcome-derived selector columns. +- Selector: nine round-number policies on November 2017; five satisfy full + budget, effective-PD, and deterministic `B_u<=0.28` screens. Outcomes are + stored separately from the 12-column selector frame. +- Audit: an outcome-free December replay selects the same policy; opening + outcomes afterward gives weighted default `0.145650` and miscoverage + `0.124925`, so stability is not reported as selected-set validity. - Full OOT: 276,869 candidates, 308 funded, `$179,327.59` realized return, `0.039375` weighted default, `0.036875` weighted miscoverage. - `Gamma_CP=0.176102`, `Gamma_residual=0.088051`, endpoint `0.258051`. diff --git a/CLAUDE.md b/CLAUDE.md index 4845186..219e7db 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -41,22 +41,24 @@ tag aislado; **no regenera ni sobreescribe ningún artefacto upstream**. | Campo | Valor | | --- | --- | -| Run tag | `champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6` | +| Run tag | `champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7` | | Conformal | exact replay at target `alpha=0.10` (frozen used alpha `0.095`) | | Policy | `q=(p+u)/2`, `tau=0.17`, point-PD economic objective | -| Selector | 9 round-number policies on the temporal calibration holdout; 5 eligible; no outcome-derived selector columns | +| Selector | 9 policies on Nov 2017; deterministic `B_u<=0.28`; 5 eligible; same policy in outcome-free Dec replay | | Realized return | `$179,327.59` on a `$1M` budget | | Weighted default / miscoverage | `0.039375 / 0.036875` | | Gamma_CP / Gamma_residual | `0.176102 / 0.088051` | | Endpoint / Markov threshold | `0.258051 / 0.574279` | | Matched point-PD A40 | return cost `8.678%`; default reduction `7.9025` pp; threshold reduction `66.3266` pp | -| Evidence | exact alpha A35 + calibration selector A36 + temporal/funded-set/baseline A37--A40 | +| Evidence | exact alpha A35 + split selector/audit A36 + temporal/funded-set/baseline A37--A40 | The exact policy-facing quantities come from `models/experiments/champion_reopen//portfolio/ijds_policy_governance.json`. -The primary claim is the simple calibration-selected guardrail and its exact -funded-set audit. Markov remains an assumption-conditional sensitivity, not the -headline novelty. +The primary claim is the simple calibration-selected guardrail, deterministic +endpoint screen, independent December audit, and exact funded-set accounting. +December miscoverage `0.124925` documents that stable selection is not +selected-set validity. Markov remains an assumption-conditional sensitivity, +not a selector or headline novelty. **Cadena upstream congelada (histórica; su retorno es el return floor declarado del pool93):** diff --git a/README.md b/README.md index ed16e40..4c61475 100644 --- a/README.md +++ b/README.md @@ -8,15 +8,16 @@ Pipeline de investigación y libro Quarto que acompañan el paper **CRPTO**, una | Campo | Valor | | --- | --- | -| Run tag | `champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6` | +| Run tag | `champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7` | | Conformal | replay exacto al `90%` (`alpha=0.10`, used `0.095`) | | Política | `q=(p+u)/2`, `tau=0.17`; PD puntual en el objetivo y `q` en el guardrail | -| Selector | grilla redonda `3x3` en calibración; `5/9` elegibles; sin outcomes OOT | +| Selector | grilla `3x3` en noviembre; cap determinista `B_u<=0.28`; misma política en auditoría de diciembre | | Retorno realizado | **$179,327.59** | | Default / miscoverage ponderados | `0.039375 / 0.036875` | | `Gamma_CP / Gamma_residual` | `0.176102 / 0.088051` | | Endpoint / contabilidad observada / umbral condicional | `0.258051 / 0.294926 / 0.574279` | | Baseline A40 | `$196,369.14`; costo de retorno `8.678%`; reducción de default `7.9025` pp | +| Auditoría pre-OOT | diciembre: default `0.145650`, miscoverage `0.124925`; estabilidad no implica cobertura seleccionada | Hashes SHA256 de los artefactos críticos están en [`EXTRACTION_MANIFEST.json`](EXTRACTION_MANIFEST.json). Verifica con `just validate-champion` o el skill `/crpto-validate-champion`. El rebaseline y la frontera pool93 anterior se conservan como procedencia diff --git a/configs/crpto_publication_targets.yaml b/configs/crpto_publication_targets.yaml index 76f3c7c..0a9632d 100644 --- a/configs/crpto_publication_targets.yaml +++ b/configs/crpto_publication_targets.yaml @@ -36,7 +36,8 @@ primary_target: - "Freeze a release tag and reproducibility bundle after anonymity decision." current_paper_scope: include: - - "Exact 90% conformal replay plus a calibration-selected 3x3 linear policy grid as the active IJDS body claim." + - "Exact 90% conformal replay plus a November-selected 3x3 linear policy grid under deterministic endpoint cap B_u<=0.28." + - "Outcome-free December selector replay and independent decision audit showing that policy stability is not selected-set coverage validity." - "One interpretable policy, q=(p+u)/2 with tau=0.17, and a matched point-PD comparator." - "Frozen upstream PD/calibration/conformal chain and historical pool93 frontier retained only as provenance." - "A3--A34 as supporting diagnostics and A35--A40 as the active exact-alpha/selector/evaluation bundle." @@ -70,7 +71,7 @@ journal_strengthening_pack: requires_new_run: false exact_alpha_calibration_selected_policy: status: include_body_and_supplement - role: "Active IJDS evidence: exact alpha replay, nine-policy calibration selector, temporal/funded-set audits, bootstrap, and matched comparisons." + role: "Active IJDS evidence: exact alpha replay, split nine-policy selector/audit, temporal/funded-set audits, month-cluster bootstrap, and matched comparisons." artifacts: - reports/crpto/tables/crpto_tableA35_exact_alpha_grid.csv - reports/crpto/tables/crpto_tableA36_calibration_policy_selector.csv @@ -78,7 +79,7 @@ journal_strengthening_pack: - reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.csv - reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.csv - reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.csv - - models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6/portfolio/ijds_policy_governance.json + - models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7/portfolio/ijds_policy_governance.json requires_new_run: false matched_point_pd_baseline: status: include_body_and_supplement @@ -86,7 +87,7 @@ journal_strengthening_pack: artifacts: - reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.csv - reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.tex - - models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6/portfolio/ijds_policy_governance.json + - models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7/portfolio/ijds_policy_governance.json requires_new_run: false robust_satisficing_margins: status: include_supplement_or_short_body diff --git a/configs/experiments/champion_reopen_ijds_calibration_selected_endpoint28_v7.yaml b/configs/experiments/champion_reopen_ijds_calibration_selected_endpoint28_v7.yaml new file mode 100644 index 0000000..d1fe691 --- /dev/null +++ b/configs/experiments/champion_reopen_ijds_calibration_selected_endpoint28_v7.yaml @@ -0,0 +1,52 @@ +schema_version: "2026-07-09.7" +run_tag: "champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7" + +source: + upstream_canonical_run_tag: "champion-reopen-2026-06-19__hpo-wave1__pool93__seed42" + conformal_results_path: "models/conformal_gap/champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1/conformal_results_mondrian.pkl" + candidate_path: "data/processed/test_fe.parquet" + exact_alpha_grid_path: "data/processed/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1/conformal/exact_alpha_grid.parquet" + +design: + alpha: 0.10 + budget: 1000000.0 + max_concentration: 0.25 + lgd: 0.45 + period_order: ["2018H1", "2018H2", "2019H1", "2019H2", "2020+"] + combine_years_from: 2020 + selection_period: "2017-11" + audit_period: "2017-12" + endpoint_budget_cap: 0.28 + selection_min_budget_utilization: 0.999 + selection_rule: >- + maximize expected point-PD objective on November 2017 under a deterministic + endpoint-budget cap of 0.28, the effective-PD cap, and full budget use; + reserve December 2017 for an outcome-free selector stability replay and a + post-selection decision audit + +policy_grid: + family: "simple_linear_conformal_guardrail" + risk_tolerances: [0.15, 0.17, 0.19] + gammas: [0.25, 0.50, 0.75] + uncertainty_aversions: [0.0] + +incumbent_policy: + risk_tolerance: 0.17 + gamma: 0.75 + uncertainty_aversion: 0.0 + +execution: + solver_backend: "highspy" + time_limit: 300 + threads: 1 + random_seed: 42 + +claim_boundary: >- + The final tagged rule selects among nine round-number policies on November + 2017 without exposing the selector to outcomes or assumption-conditional + statistics. December 2017 independently replays the outcome-free selector + and audits the already-fixed decision. The audit is diagnostic rather than a + selected-set coverage theorem. Earlier development inspected the static OOT + corpus, so the January 2018--September 2020 evaluation remains a transparent + retrospective lockbox replay, not a pristine prospective trial, causal + estimate, or live-deployment guarantee. diff --git a/docs/SCOPE_AND_GOVERNANCE.md b/docs/SCOPE_AND_GOVERNANCE.md index 349e6a6..5f0d253 100644 --- a/docs/SCOPE_AND_GOVERNANCE.md +++ b/docs/SCOPE_AND_GOVERNANCE.md @@ -30,21 +30,24 @@ CRPTO does not cover: The current IJDS body point is the simple calibration-selected 90% guardrail: - run tag: - `champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6` + `champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7` - exact conformal replay: target `alpha=0.10`, frozen used alpha `0.095`; - policy: `q=(p+u)/2`, `tau=0.17`, with point PD in the economic objective and conformal `q` in the risk constraint; -- selector: nine round-number policies on the temporal calibration holdout, - five eligible under the `0.60` endpoint-plus-Markov screen, no - outcome-derived selector columns; +- selector: nine round-number policies on November 2017, five eligible under + deterministic full-budget, effective-PD, and `B_u<=0.28` screens; no outcome + or assumption-conditional selector columns; +- independent audit: the outcome-free December replay selects the same policy; + after outcomes are opened, weighted default is `0.145650` and miscoverage is + `0.124925`, explicitly ruling out a selected-set validity claim; - realized return: `$179,327.59`; - weighted default and miscoverage: `0.039375` and `0.036875`; - `Gamma_CP=0.176102`, `Gamma_residual=0.088051`; - endpoint budget `0.258051`; assumption-conditional Markov threshold `0.574279`; -- paper artifacts: A35 exact-alpha audit, A36 calibration selector, A37 - temporal evaluation, A38 grade audit, A39 bootstrap, and A40 matched - comparisons. +- paper artifacts: A35 exact-alpha audit, A36 split selector/audit, A37 + temporal evaluation, A38 grade audit, A39 month-cluster bootstrap with + loan-level sensitivity, and A40 matched comparisons. The selector is outcome-free with respect to OOT policy ranking, but earlier project development inspected the static OOT corpus. The paper must describe diff --git a/docs/refactor/ijds_tooling_decisions_2026-07-09.md b/docs/refactor/ijds_tooling_decisions_2026-07-09.md index c0038b5..c703407 100644 --- a/docs/refactor/ijds_tooling_decisions_2026-07-09.md +++ b/docs/refactor/ijds_tooling_decisions_2026-07-09.md @@ -47,15 +47,15 @@ stable repository contract. 2. `src/optimization/policy_evaluation.py` uses point PD in the economic objective and an effective PD only in the risk constraint. 3. `src/optimization/policy_selection.py` defines the nine-cell round-number - grid and rejects selectors containing outcome-derived columns. -4. `scripts/experiments/ijds_policy_support.py` owns shared alignment, solving, - and evaluation for the active challengers. + grid, deterministic endpoint screen, and cap-stability interval. +4. `scripts/experiments/ijds_policy_support.py` aligns candidate and exact-alpha + rows directly by ID and owns shared solving/evaluation. 5. `scripts/build_ijds_calibration_selected_evidence.py` materializes A35-A40 from versioned experiment outputs. Manuscript rendering does not solve or retune portfolios. The active run is -`champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6`. +`champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7`. The exact-alpha run is `champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1`. @@ -64,7 +64,11 @@ The exact-alpha run is - One active policy family: `q=(p+u)/2`, `tau=0.17`, `gamma=0.50`. - Point PD remains the economic objective; uncertainty is a feasibility guardrail. -- One deterministic 3x3 calibration selector, with an outcome-column denylist. +- One deterministic 3x3 November selector, with outcomes isolated from a + 12-column input frame, `B_u<=0.28`, and an outcome-free December replay. +- One independent December decision audit that records the funded-set coverage + miss instead of converting it into an unsupported guarantee. +- Month-cluster bootstrap is primary; loan-level resampling is a sensitivity. - One A35-A40 active evidence bundle. - One body, one supplement, and one official submission TeX source. - No nested temporal selector, effective-PD objective branch, active cap/tail @@ -94,9 +98,10 @@ interval grids and solves experiment portfolios. ## Compilation contract -The official source first attempts `latexmk`. On the current Windows TinyTeX -installation its `runscript.tlu` wrapper may fail, so the documented robust -fallback is: +The official build uses `latexmk`. On Windows it resolves TinyTeX's +`latexmk.pl` and launches it with Perl, bypassing the defective +`runscript.tlu` executable wrapper. If that payload is unavailable or fails, +the robust fallback is: ```text pdflatex -> bibtex -> pdflatex -> pdflatex diff --git a/docs/research/active_claims_2026-07-04.md b/docs/research/active_claims_2026-07-04.md index 2107f46..f765e9f 100644 --- a/docs/research/active_claims_2026-07-04.md +++ b/docs/research/active_claims_2026-07-04.md @@ -8,7 +8,7 @@ longer evidence for the manuscript's main claim. ## Active Decision - Run tag: - `champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6` + `champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7` - Conformal target: `alpha = 0.10`; frozen conservative alpha used by the recipe: `0.095`. - Partition: five score-quantile Mondrian cells on calibrated PD (the frozen @@ -21,17 +21,23 @@ longer evidence for the manuscript's main claim. The policy is selected from nine round-number candidates: `tau in {0.15, 0.17, 0.19}` crossed with -`gamma in {0.25, 0.50, 0.75}`. The final tagged selector uses the temporal -calibration holdout only, requires full budget use, enforces the effective-PD -cap and `B_u + sqrt(0.10) <= 0.60`, and maximizes expected point-PD objective. -Five of nine candidates are eligible; the selected policy is -`tau = 0.17, gamma = 0.50`. +`gamma in {0.25, 0.50, 0.75}`. The final tagged selector uses November 2017, +requires full budget use, enforces the effective-PD cap and deterministic +`B_u <= 0.28`, and maximizes expected point-PD objective. Five of nine +candidates are eligible; the selected policy is `tau = 0.17, gamma = 0.50`. +The selected row remains optimal for endpoint caps in +`[0.259036, 0.290491)`. The conformal recipe uses `142,550` calibration-fit rows. Policy selection is -performed on a later `35,638`-row calibration holdout covering November and -December 2017. The policy-ranking artifact contains no defaults, realized -returns, miscoverage, or other outcome-derived selector columns. Conformal -endpoints themselves use calibration labels, as required. +performed on `14,943` November rows. Outcomes are stored separately from the +12-column selector frame, which contains no defaults, realized returns, +miscoverage, or assumption-conditional fields. An outcome-free replay on +`20,695` December rows independently selects the same policy. Outcomes joined +afterward give weighted default `0.145650`, weighted +miscoverage `0.124925`, endpoint budget `0.262082`, and accounting bound +`0.387007`. This audit is diagnostic evidence that stable policy selection is +not selected-set coverage validity. Conformal endpoints themselves use +calibration labels, as required. ## Full OOT Result @@ -55,10 +61,11 @@ September 2020: | Observed accounting bound `B_u + V` | `0.294926` | | Markov event threshold `B_u + sqrt(alpha)` | `0.574279` | -The fixed-allocation bootstrap return interval is -`$162,706.17`--`$193,924.74` (`5,000` draws). It resamples funded-loan -contributions only; it does not resample the model, conformal recipe, selector, -or optimizer. +The primary fixed-allocation bootstrap return interval is +`$163,421.14`--`$193,551.65` (`5,000` draws over `31` origination-month +clusters). A funded-loan sensitivity gives +`$162,706.17`--`$193,924.74`. Neither resamples the model, conformal recipe, +selector, or optimizer. ## Matched Baseline @@ -120,10 +127,11 @@ The active evidence bundle is: - `models/experiments/champion_reopen//portfolio/ijds_policy_governance.json` - A35: exact alpha replay and saturation audit. -- A36: nine-policy calibration selector. +- A36: November selector, endpoint-cap stability interval, December replay, + and independent decision audit. - A37: full-OOT and temporal fixed-policy evaluation. - A38: selected funded-set grade composition. -- A39: fixed-allocation bootstrap. +- A39: fixed-allocation month-cluster bootstrap plus funded-loan sensitivity. - A40: selected, more-conservative, and matched point-PD comparison. The manuscript must say explicitly that earlier project development inspected @@ -150,7 +158,7 @@ tables, and `EXTRACTION_MANIFEST.json` remain untouched. Reopen the active method only for one of four reasons: -1. a calibration-only rule materially improves return at the same `0.60` +1. a calibration-only rule materially improves return at the same `B_u<=0.28` screen; 2. a simpler rule matches the selected policy within prespecified tolerances; 3. a valid selected-set or prospective protocol materially strengthens the diff --git a/docs/research/ijds_exact_alpha_calibration_selection_2026-07-09.md b/docs/research/ijds_exact_alpha_calibration_selection_2026-07-09.md index 082c656..2f62787 100644 --- a/docs/research/ijds_exact_alpha_calibration_selection_2026-07-09.md +++ b/docs/research/ijds_exact_alpha_calibration_selection_2026-07-09.md @@ -23,7 +23,7 @@ and `93.54%` of upper endpoints equal one. ## Final policy protocol The final run is -`champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6`. +`champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7`. It uses the exactly replayed 90% interval recipe and a nine-policy round-number grid: @@ -34,12 +34,20 @@ grid: - point-PD expected net return in the objective; - conformal `q` only in the portfolio-risk constraint. -The policy-ranking code reads no default, realized-return, or miscoverage -columns. It requires at least 99.9% budget use, feasibility of the effective-PD -cap, and `B_u + sqrt(0.10) <= 0.60` on the calibration holdout. Five candidates -are eligible. Maximizing expected point-PD objective selects -`tau = 0.17, gamma = 0.50`, the interpretable midpoint -`q = (p+u)/2`. +The loader keeps outcomes physically separate from the 12-column policy frame, +which has no default, realized-return, miscoverage, or assumption-conditional +quantity. It requires +at least 99.9% budget use, feasibility of the effective-PD cap, and +`B_u <= 0.28` on November 2017. Five candidates are eligible. Maximizing +expected point-PD objective selects `tau = 0.17, gamma = 0.50`, the +interpretable midpoint `q = (p+u)/2`. The selected row is invariant for caps in +`[0.259036, 0.290491)`. + +An outcome-free December replay selects the same policy. Outcomes are opened +only afterward: weighted default is `0.145650`, weighted miscoverage is +`0.124925`, and endpoint budget is `0.262082`. This independent audit is not a +coverage theorem; it is direct evidence that policy stability does not imply +selected-set conformal validity. ## What was learned from the challengers @@ -55,6 +63,30 @@ are eligible. Maximizing expected point-PD objective selects rate and endpoint audit are substantially worse. It also beats CRPTO in some temporal slices, so no universal dominance claim is supportable. +## Endpoint-screen v7 promotion + +The v6 selector used `B_u + sqrt(0.10) <= 0.60`. Because alpha was fixed, that +screen was algebraically an endpoint cap but was described with the same +Markov expression whose interpretation requires weighted funded-set validity. +The v7 challenger removes that coupling: + +- selection uses the round deterministic cap `B_u<=0.28`; +- November selects and December replays the same nine-row grid; +- the winner is unchanged for caps in `[0.259036, 0.290491)`; +- the loader separates outcomes from a 12-column selector frame with no + statistical-bound columns; +- the OOT loader aligns candidates directly to exact-alpha rows by unique ID, + removing a private script import and redundant intermediate alignment; +- v6 and v7 produce identical 845 funded comparator rows, IDs, allocations, + exposures, interval values, and full/temporal metrics (`max abs diff = 0`); +- A39 now treats 31 origination months as the primary resampling units and + retains funded-loan resampling as a sensitivity. + +No protected DVC stage was rerun because the challenger changes only policy +selection semantics, panel alignment, and downstream evidence. The exact-alpha +artifact and every manifest-protected upstream byte remain the comparison +baseline. + ## Interpretation The scientific upgrade is not a larger search. It is a smaller and auditable @@ -63,7 +95,6 @@ of point-PD economics from conformal feasibility, and a policy selector whose inputs can be inspected for outcome leakage. This is the active IJDS narrative. The historical OOT panel was inspected during earlier project development. -Accordingly, v6 is described as a retrospective lockbox replay with an -OOT-outcome-column-free final selector conditional on the frozen conformal -recipe, not as a pristine prospective holdout or -preregistered trial. +Accordingly, v7 is described as a retrospective lockbox replay with an +outcome-free, assumption-free final selector conditional on the frozen +conformal recipe, not as a pristine prospective holdout or preregistered trial. diff --git a/justfile b/justfile index 1337709..87b41d8 100644 --- a/justfile +++ b/justfile @@ -85,7 +85,7 @@ ijds-exact-alpha: uv run python scripts/experiments/run_ijds_exact_alpha_grid_challenger.py --config configs/experiments/champion_reopen_ijds_exact_alpha_grid_v1.yaml ijds-policy-challenger: - uv run python scripts/experiments/run_ijds_calibration_selected_policy_challenger.py --config configs/experiments/champion_reopen_ijds_calibration_selected_simple90_v6.yaml + uv run python scripts/experiments/run_ijds_calibration_selected_policy_challenger.py --config configs/experiments/champion_reopen_ijds_calibration_selected_endpoint28_v7.yaml ijds-active-replay: ijds-exact-alpha ijds-policy-challenger ijds-evidence diff --git a/paper/CRPTO.qmd b/paper/CRPTO.qmd index 7274b92..29ca30e 100644 --- a/paper/CRPTO.qmd +++ b/paper/CRPTO.qmd @@ -17,8 +17,9 @@ execute: CRPTO convierte una PD calibrada en una decision de portafolio auditable. La version IJDS activa reproduce exactamente el endpoint conformal al 90%, usa la regla simple `q=(p+u)/2` con `tau=0.17`, y selecciona esa politica dentro de una -grilla redonda `3x3` usando el holdout temporal de calibracion. El ranking final -no lee outcomes OOT. +grilla `3x3` en noviembre bajo `B_u<=0.28`. Un replay outcome-free en diciembre +elige la misma politica; al abrir outcomes, la miscoverage `0.124925` muestra +que estabilidad de seleccion no equivale a cobertura del funded set. En 276,869 prestamos OOT, la politica financia 308 prestamos y obtiene `$179,327.59` sobre `$1M`, con default ponderado `0.039375`, miscoverage @@ -38,10 +39,10 @@ los periodos donde point-PD domina; no afirma superioridad universal. # Evidencia activa - A35: replay exacto y saturacion por alpha. -- A36: selector de calibracion de nueve politicas. +- A36: selector de noviembre, estabilidad del cap y auditoria de diciembre. - A37: evaluacion OOT total y temporal. - A38: composicion por grado de credito. -- A39: bootstrap de asignacion fija. +- A39: bootstrap por mes de originacion y sensibilidad por prestamo. - A40: politica seleccionada, blend conservador y baseline point-PD. La gobernanza vive en diff --git a/paper/CRPTO_ijds.qmd b/paper/CRPTO_ijds.qmd index 5bcb9fd..e1dd100 100644 --- a/paper/CRPTO_ijds.qmd +++ b/paper/CRPTO_ijds.qmd @@ -1,5 +1,5 @@ --- -title: "CRPTO: A Calibration-Selected Conformal Guardrail for Auditable Credit Portfolio Decisions" +title: "CRPTO: A Calibration-Selected Conformal Guardrail for Credit Portfolios" author: "Anonymous" date: today lang: en @@ -31,9 +31,12 @@ probability-of-default (PD) model has been frozen. Conformal Robust Predict-Then-Optimize (CRPTO) recomputes a 90% Mondrian conformal upper endpoint exactly, forms the transparent decision score $q_i=(p_i+u_i)/2$, and places $q_i$ in a portfolio-risk constraint while retaining point PD in the economic -objective. Nine round-number policies are ranked on a temporal calibration -holdout without default, realized-return, or miscoverage columns; the selected -policy is then frozen and replayed on 276,869 out-of-time Lending Club loans. +objective. Nine round-number policies are ranked on November 2017 without +default, realized-return, miscoverage, or assumption-conditional columns; an +outcome-free December replay selects the same rule before its outcomes are +opened. That audit misses nominal funded-set coverage, making the statistical +boundary observable rather than rhetorical. The policy is then replayed on +276,869 out-of-time Lending Club loans. It funds 308 loans and earns `$179,327.59` on a `$1M` budget, with weighted default `0.039375`, weighted miscoverage `0.036875`, and conformal endpoint budget `0.258051`. A matched point-PD allocation earns `$196,369.14` but has @@ -84,25 +87,27 @@ removes the capped, tail-focused, and uncertainty-penalty branches that made an earlier research frontier difficult to explain and easy to misread. The empirical design uses a temporal Lending Club panel. The conformal recipe -is fit inside the calibration period, and a later calibration holdout ranks a -declared $3\times3$ grid of round-number risk tolerances and conformal weights. -The final ranking artifact contains no defaults, realized returns, or -miscoverage. The fixed decision rule is then evaluated on loans originated from -January 2018 through September 2020. Earlier project development did inspect -this static OOT corpus, so we describe the final run as a transparent -retrospective lockbox replay, not as a pristine prospective or preregistered -trial. +is fit inside the calibration period, November 2017 ranks a declared +$3\times3$ grid of round-number risk tolerances and conformal weights, and +December replays the outcome-free selector before auditing the already-fixed +decision. The ranking artifact contains no defaults, realized returns, +miscoverage, or assumption-conditional statistics. The fixed rule is then +evaluated on loans originated from January 2018 through September 2020. Earlier +project development did inspect this static OOT corpus, so we describe the +final run as a transparent retrospective lockbox replay, not as a pristine +prospective or preregistered trial. The paper makes three contributions. First, it gives an auditable prediction-to-decision construction in which the economic objective and the conformal guardrail have separate, inspectable roles. Second, it replaces -approximate cross-alpha scaling with an exact replay of the frozen conformal -recipe and a calibration-only final policy selector. Third, it reports the -price and limits of that guardrail against matched point-PD and more-conservative -comparators, including temporal slices where CRPTO wins and slices where it -does not. The novelty is the closed, inspectable decision protocol for a frozen -credit model, not a claim that conformal prediction, robust optimization, or -credit scoring is individually new. +approximate cross-alpha scaling with an exact replay, a deterministic endpoint +screen, and a temporally separated selector audit. Third, it reports the price +and limits of that guardrail against matched point-PD and more-conservative +comparators, including a pre-OOT funded-set coverage miss and temporal slices +where CRPTO wins and slices where it does not. The novelty is the closed, +inspectable decision protocol for a frozen credit model, not a claim that +conformal prediction, robust optimization, or credit scoring is individually +new. ![CRPTO carries a frozen calibrated PD through an exact conformal replay, a simple portfolio guardrail, and a funded-set audit.](../reports/crpto/figures/crpto_fig1_journal_pipeline.png){#fig-crpto-pipeline width="92%" fig-alt="Four-stage CRPTO pipeline from calibrated PD to conformal intervals, portfolio allocation, and funded-set audit."} @@ -126,6 +131,15 @@ sets directly in robust optimization [@johnstone2021; @patel2024; credit PD model, exposes the exact funded rows and uncertainty premium, and compares the resulting allocation with a matched point-PD decision. +Decision-calibrated prediction sets and inverse conformal risk control go +further by calibrating downstream violation or regret and, after choosing a +robustness level, using a separate split to restore risk-estimation validity +[@zhou2026creme; @stratigakos2026decision_calibrated_sets]. CRPTO does not +import that guarantee into a batch portfolio where the optimizer chooses the +funded weights. Instead, it separates a deterministic endpoint-budget screen +from an independent post-selection audit and reports when selected-set +miscoverage misses its nominal target. + Decision-focused learning and SPO+ train predictions against downstream regret [@donti2017; @elmachtoub2022; @mandi2024]. That is a different institutional choice. CRPTO asks what can be done after a calibrated model already exists and @@ -150,7 +164,7 @@ that change costs. | P2P and robust credit portfolios | Economic loan selection under risk and uncertainty. | Uses an exact conformal endpoint as a simple portfolio guardrail. | | Conformal robust optimization | Coverage-backed uncertainty sets for downstream decisions. | Adds a frozen credit stack, funded-set audit, and matched economic comparator. | | Decision-focused learning | Training-time reduction of decision regret. | Keeps the predictive model frozen and emphasizes post-hoc auditability. | -| Valid conformal selection | Corrects validity after selecting among sets or models [@hegazy2025valid_selection_conformal_sets]. | Motivates our narrow claim; formal selected-set validity is not asserted. | +| Decision-risk calibration and valid selection | Calibrates decision losses or corrects validity after choosing among sets [@zhou2026creme; @hegazy2025valid_selection_conformal_sets]. | Uses a deterministic selector plus an independent diagnostic audit; formal selected-set validity is not asserted. | : Closest-work boundary for the submitted CRPTO claim. @@ -163,19 +177,23 @@ and evaluation pipeline use temporal rather than random splits. | Split | Period | Loans | Role | |---|---|---:|---| | Train | Jun 2007--Mar 2017 | `1,346,311` | Fit the PD model and calibrator. | -| Calibration pool | Mar--Dec 2017 | `237,584` | Develop and freeze conformal and policy rules. | +| Conformal fit | Mar--Oct 2017 | `142,550` | Estimate the frozen conformal recipe. | +| Policy selection | Nov 2017 | `14,943` | Rank the nine outcome-free policies. | +| Calibration audit | Dec 2017 | `20,695` | Replay the selector, then open outcomes. | | OOT evaluation | Jan 2018--Sep 2020 | `276,869` | Freeze-then-evaluate portfolio decisions. | : Temporal Lending Club design. -The frozen conformal recipe uses the most recent 75% of the calibration pool -(`178,188` rows). Within that subset, `142,550` rows estimate conformal -quantiles and `35,638` later rows, from November and December 2017, form the -temporal development holdout. The conformal recipe uses calibration labels, as -conformal prediction requires. Policy ranking on the development holdout uses -only candidate parameters, solver status, expected point-PD objective, budget -use, effective-PD exposure, and conformal endpoint summaries. A schema guard -rejects outcome, default, realized-return, and miscoverage columns. +The frozen conformal recipe uses the most recent 75% of the original +calibration pool (`178,188` rows). Within that subset, `142,550` rows estimate +conformal quantiles, `14,943` November rows select the policy, and `20,695` +December rows form a pre-OOT audit. The conformal recipe uses calibration +labels, as conformal prediction requires. The loader stores holdout outcomes +separately from the 12-column policy frame, which contains identifiers, +origination context, PD endpoints, amounts, and rates. Ranking cannot access +outcomes, realized returns, miscoverage, or the assumption-conditional Markov +quantity. December first reruns that same outcome-free selector and only then +joins outcomes for the decision audit. The final OOT panel covers several regimes, including the 2020 disruption. We report the full panel and five temporal slices (`2018H1`, `2018H2`, `2019H1`, @@ -250,7 +268,7 @@ B_u &= \sum_iw_i u_i. $$ For the midpoint policy, -$\Gamma_{\mathrm{int}}=\Gamma_{\mathrm{res}}=Gamma_{\mathrm{CP}}/2$. +$\Gamma_{\mathrm{int}}=\Gamma_{\mathrm{res}}=\Gamma_{\mathrm{CP}}/2$. This identity is one reason to prefer the midpoint over nonlinear caps or tail rules: every quantity has a direct interpretation. @@ -263,23 +281,26 @@ $q_i=p_i+\gamma(u_i-p_i)$. A candidate is eligible when the solver is optimal, at least 99.9% of the budget is allocated, the effective-PD cap holds, and $$ -B_u+\sqrt{0.10}\le 0.60 +B_u\le 0.28 $$ -on the calibration development holdout. Among eligible candidates, the rule -maximizes expected point-PD objective. Five of nine candidates pass, and the -selected candidate is $\tau=0.17,\gamma=0.50$. The screen value `0.60` is a -declared decision threshold for the final protocol, not an estimated property -of OOT outcomes. +on the November block. Among eligible candidates, the rule maximizes expected +point-PD objective. Five of nine candidates pass, and the selected candidate is +$\tau=0.17,\gamma=0.50$. The endpoint cap is deterministic; it does not use +the weighted-validity assumption introduced below. The selected row remains +optimal for every cap in $[0.259036,0.290491)$, so the round `0.28` value has +positive margin to both policy-change boundaries. An outcome-free replay on +December independently selects the same candidate. -| Candidate | $\tau$ | $\gamma$ | Calibration expected objective | $B_u+\sqrt{0.10}$ | Status | -|---|---:|---:|---:|---:|---| -| Higher-return, low guardrail | `0.17` | `0.25` | `$121,761.88` | `0.708835` | Ineligible | -| Selected midpoint | `0.17` | `0.50` | `$110,346.16` | `0.577275` | Selected | -| More-conservative blend | `0.17` | `0.75` | `$104,272.78` | `0.519696` | Eligible | -| Higher risk tolerance | `0.19` | `0.50` | `$113,591.27` | `0.611017` | Ineligible | +| Candidate | $\tau$ | $\gamma$ | Nov. expected objective | Nov. $B_u$ | Dec. $B_u$ | Status | +|---|---:|---:|---:|---:|---:|---| +| Higher-return, low guardrail | `0.17` | `0.25` | `$109,885.56` | `0.375105` | `0.396751` | Ineligible | +| Selected midpoint | `0.17` | `0.50` | `$99,387.12` | `0.259036` | `0.262082` | Selected twice | +| More-conservative blend | `0.17` | `0.75` | `$93,760.13` | `0.202259` | `0.203504` | Eligible | +| Higher risk tolerance | `0.19` | `0.50` | `$102,671.87` | `0.290491` | `0.294861` | Ineligible | -: Calibration selector examples. A36 reports all nine candidates. +: Temporally separated selector examples. A36 reports all nine candidates on +both months. ## Accounting and Statistical Boundary @@ -325,6 +346,30 @@ carry almost no ranking information for a portfolio. The selected 90% level is the frozen recipe's reference level and preserves materially more decision resolution. A35 reports the complete sensitivity from alpha `0.01` to `0.20`. +## Pre-OOT Selector and Decision Audit + +November selects the midpoint policy from five eligible rows. Applying the +same outcome-free rule to December again selects `linear-005`; policy identity +is therefore stable across the two calibration months. Outcomes are opened +only after that replay. + +| December policy | Funded | Realized return | Weighted default | Miscoverage | $B_u$ | +|---|---:|---:|---:|---:|---:| +| Selected 50/50 CRPTO | `193` | `$53,313.05` | `0.145650` | `0.124925` | `0.262082` | +| More-conservative 75% blend | `191` | `$38,379.50` | `0.155250` | `0.134525` | `0.203504` | +| Point-PD matched-$\tau$ | `169` | `$89,732.35` | `0.185650` | `0.058300` | `0.888071` | + +: Independent December 2017 post-selection decision audit. + +The selected policy remains below its operational tolerance +(`0.145650 < 0.17`) but misses nominal funded-set coverage +(`V=0.124925 > 0.10`). Its deterministic accounting right-hand side is +`B_u+V=0.387007`. Thus an outcome-free, month-stable selector does not create +selected-set conformal validity. This negative audit is part of the result: it +motivates the two-layer reporting of deterministic endpoint exposure and +observed miscoverage instead of relabeling marginal coverage as a portfolio +guarantee. + ## Full OOT Funded-Set Audit The fixed midpoint policy allocates the full `$1M` budget across 308 loans. Its @@ -351,11 +396,14 @@ $\sqrt{0.10}=0.316228$. This loose probability statement is not interpreted as a direct default cap. The operational controls are $\tau=0.17$, the midpoint score, and the exact funded-set diagnostics. -The fixed-allocation bootstrap gives a 95% return interval of -`$162,706.17`--`$193,924.74` from 5,000 funded-loan resamples. It does not -resample the model, conformal recipe, policy selector, or optimizer and is -therefore a contribution-level stability diagnostic rather than a full -pipeline confidence interval. +The primary fixed-allocation bootstrap resamples 31 origination-month clusters +and gives a 95% return interval of `$163,421.14`--`$193,551.65` from 5,000 +draws. A loan-level sensitivity gives +`$162,706.17`--`$193,924.74`. Neither scheme resamples the model, conformal +recipe, policy selector, or optimizer; both are contribution-level stability +diagnostics rather than full-pipeline confidence intervals. The month-level +scheme is primary because the temporal results reject a naive independence +story across loans. ## Matched Point-PD and More-Conservative Comparators @@ -455,16 +503,19 @@ static OOT corpus. The evaluation is therefore retrospective, not a new prospective holdout. Third, the conformal intervals are broad because the outcome is binary; at 90%, more than half of OOT upper endpoints equal one. Fourth, marginal or Mondrian coverage does not imply validity under -optimizer-selected funded weights. The Markov result is conditional on an -explicit weighted-validity assumption. Fifth, temporal slices show that the -point-PD comparator can dominate both return and default. Finally, public data -do not support a legal fair-lending certification or a causal interpretation -of policy contrasts. +optimizer-selected funded weights. The independent December audit makes this +visible: the same policy is reselected without outcomes, yet funded-set +miscoverage is `0.124925`. The Markov result is therefore conditional on an +explicit weighted-validity assumption and is absent from policy selection. +Fifth, temporal slices show that the point-PD comparator can dominate both +return and default. Finally, public data do not support a legal fair-lending +certification or a causal interpretation of policy contrasts. These limitations suggest a focused next step rather than a larger current paper: a genuinely prospective or formally selection-valid protocol that freezes conformal and policy choices before a new evaluation period -[@farinhas2024nonexchangeable_crc; @hegazy2025valid_selection_conformal_sets]. +[@farinhas2024nonexchangeable_crc; @hegazy2025valid_selection_conformal_sets; +@zhou2026creme]. It is not required to interpret the present retrospective decision audit. For double-anonymous review, author-identifying repository URLs are omitted @@ -477,7 +528,10 @@ identity in the anonymous PDF. CRPTO shows how a frozen credit model can become an auditable portfolio decision without adding a maze of policy variants. An exact 90% conformal replay produces $u_i$; the midpoint $q_i=(p_i+u_i)/2$ constrains risk; and a -nine-cell calibration selector fixes $\tau=0.17$ without reading OOT outcomes. +nine-cell November selector fixes $\tau=0.17$ under the deterministic +$B_u\le0.28$ screen. December independently selects the same rule, then misses +nominal funded-set coverage; this is why CRPTO reports observed miscoverage +rather than claiming that policy stability implies conformal validity. On the full OOT panel, the policy earns `$179,327.59`, with weighted default `0.039375`, miscoverage `0.036875`, $\Gamma_{\mathrm{CP}}=0.176102$, $\Gamma_{\mathrm{res}}=0.088051$, endpoint `0.258051`, observed accounting diff --git a/paper/README.md b/paper/README.md index f594e16..a494001 100644 --- a/paper/README.md +++ b/paper/README.md @@ -35,10 +35,11 @@ the submission-shaped versions are written. - `submission/SCHOLARONE_FINAL_CHECKLIST.md`: final upload/proof checklist. The active paper has one method: exact 90% conformal replay, the midpoint -guardrail `q=(p+u)/2`, `tau=0.17`, and a nine-cell calibration selector. A35 is -the exact-alpha audit, A36 is the selector, A37 is temporal evaluation, A38 is -letter-grade composition, A39 is the fixed-allocation bootstrap, and A40 is the -matched point-PD comparison. OCE/CVaR, SPO+, satisficing, online-style checks, +guardrail `q=(p+u)/2`, `tau=0.17`, and a nine-cell November selector under +`B_u<=0.28`. A35 is the exact-alpha audit, A36 is the split selector/December +audit, A37 is temporal evaluation, A38 is letter-grade composition, A39 is the +month-cluster bootstrap with loan-level sensitivity, and A40 is the matched +point-PD comparison. OCE/CVaR, SPO+, satisficing, online-style checks, and Prosper/Freddie replications remain supplement diagnostics; they do not select or redefine the midpoint policy. Prospective validation, causal variants, live recalibration, production, and package tracks remain outside the claim. diff --git a/paper/submission/CLAIM_AUDIT_MATRIX.md b/paper/submission/CLAIM_AUDIT_MATRIX.md index 7aa5e42..fc792fd 100644 --- a/paper/submission/CLAIM_AUDIT_MATRIX.md +++ b/paper/submission/CLAIM_AUDIT_MATRIX.md @@ -9,8 +9,9 @@ midpoint policy. Numeric authority is | The 90% conformal recipe is replayed exactly. | A35; stored endpoint replay max error `6.67e-16`. | "Exact" is mistaken for universal conditional validity. | Exact refers to numerical reconstruction of the frozen finite-sample recipe. Coverage remains marginal/Mondrian under its assumptions. | | The 99% setting is not decision-useful here. | A35: width `0.988215`; `93.5424%` of upper endpoints equal one. | Reviewer thinks 90% was chosen only to improve economics. | The 90% level is the frozen recipe's reference level and preserves materially more ranking resolution; alpha sensitivity is fully reported. | | The final policy is simple. | `q=(p+u)/2`, `tau=0.17`; one linear policy family. | Complexity or hidden nonlinear logic. | Point PD prices expected loss; the midpoint is used only in the risk constraint. | -| Final ranking does not use OOT outcomes. | A36; nine candidates, five eligible, zero forbidden selector columns. | Historical OOT inspection makes "untouched holdout" false. | Call it a retrospective lockbox replay with an outcome-free final ranking code path, not preregistration or a pristine prospective trial. | -| The selector has a declared rule. | A36: full budget, effective-PD feasibility, threshold `<=0.60`, then maximum expected objective. | The `0.60` screen appears arbitrary. | Present it as a committee risk preference in the tagged final protocol and report the complete 3x3 grid. | +| Final ranking does not use OOT outcomes or statistical assumptions. | A36; outcomes stored separately from a 12-column frame, nine candidates, five eligible, zero outcome/Markov fields. | Historical OOT inspection makes "untouched holdout" false. | Call it a retrospective lockbox replay with an outcome-free final ranking code path, not preregistration or a pristine prospective trial. | +| The selector has a declared stable rule. | A36: full budget, effective-PD feasibility, deterministic `B_u<=0.28`, then maximum expected objective; winner stable on `[0.259036,0.290491)`. | The endpoint cap appears chosen to force the midpoint. | Report the complete 3x3 grid, cap-change boundaries, and the independent December replay. | +| Selector stability is not selected-set validity. | December reselects `linear-005` before outcomes are opened; default `0.145650`, miscoverage `0.124925`. | The 90% conformal label is read as 90% funded-set coverage. | Lead with the observed miss: stable outcome-free selection does not transfer marginal conformal validity to optimizer-selected weights. | | The selected funded set has an exact accounting audit. | Full OOT: return `$179,327.59`, default `0.039375`, miscoverage `0.036875`, endpoint `0.258051`, `B_u+V=0.294926`. | Accounting is confused with a coverage theorem. | The identity is deterministic after outcomes; it is not nominal selected-set coverage. | | The Markov statement is secondary and conditional. | Threshold `0.574279`; tail-probability bound `0.316228` under weighted validity. | Bound is too loose or presented as a hard risk cap. | Use it as sensitivity only. Operational controls are `tau`, midpoint exposure, and observed funded-set metrics. | | A40 is a matched point-PD comparison. | Same candidates, budget, concentration, LGD, solver, and `tau=0.17`. | Comparator changes multiple semantics or sees labels. | Only the risk score changes; neither optimization reads OOT outcomes. | @@ -18,7 +19,7 @@ midpoint policy. Numeric authority is | The midpoint is not the safest CRPTO policy. | A40: 75% blend return `$172,939.50`, default `0.035875`, threshold `0.516624`. | Selected point is sold as dominant. | It is the highest calibration expected-objective candidate under the declared screen. | | Performance is temporally heterogeneous. | A37: CRPTO wins strongly in 2018H2; point PD dominates 2019H2 and 2020+. | Full-panel result is overgeneralized. | State the reversals in body and abstract; no universal dominance claim. | | Funded-set composition is correctly labeled. | A38 uses letter grade recovered from `sub_grade` and stores conformal group separately. | Score-quantile groups are mistaken for loan grades. | Call A38 a business composition audit, not fairness certification. | -| Bootstrap uncertainty is bounded. | A39 return interval `$162,706.17`--`$193,924.74`, 5,000 draws. | Interval is read as full model/selection uncertainty. | It is a fixed-allocation funded-loan contribution bootstrap only. | +| Bootstrap uncertainty is bounded. | A39 primary month-cluster return interval `$163,421.14`--`$193,551.65`; funded-loan sensitivity `$162,706.17`--`$193,924.74`. | Interval is read as full model/selection uncertainty. | Both are fixed-allocation diagnostics; model, conformal recipe, selector, and optimizer remain fixed. | | Earlier methods do not multiply the contribution. | Supplement A1--A34. | Paper reads as several papers or an uncontrolled tournament. | OCE/CVaR, SPO+, online-style checks, and external replications are diagnostics or context, not active selectors. | | Reproducibility is substantive evidence quality. | Run tags, configs, A35--A40 builder, claim-sync tests, manifest validation. | Tooling is presented as the only novelty. | Lead with decision method and empirical implication; reproducibility makes them auditable. | @@ -43,3 +44,6 @@ midpoint policy. Numeric authority is - point-PD return: `$196,369.14`; - return cost / default reduction: `8.678% / 7.9025` pp; - selector: `5/9` eligible, selected `tau=0.17`, `gamma=0.50`. +- deterministic selector cap / stability interval: `0.28 / [0.259036, 0.290491)`; +- December audit default / miscoverage: `0.145650 / 0.124925`; +- primary month-cluster return interval: `$163,421.14`--`$193,551.65`. diff --git a/paper/submission/COVER_LETTER_AND_DISCLOSURE.md b/paper/submission/COVER_LETTER_AND_DISCLOSURE.md index 540762a..5945a2a 100644 --- a/paper/submission/COVER_LETTER_AND_DISCLOSURE.md +++ b/paper/submission/COVER_LETTER_AND_DISCLOSURE.md @@ -15,10 +15,14 @@ with an exactly replayed 90% Mondrian conformal endpoint. The resulting midpoint score, `q=(p+u)/2`, constrains a `$1M` portfolio while point PD remains in the expected-return objective. -The final policy is selected from nine round-number candidates on a temporal -calibration holdout. Its ranking artifact contains no OOT default, -realized-return, or miscoverage fields. On 276,869 out-of-time Lending Club -loans, the fixed policy earns `$179,327.59`, with weighted default `0.039375`. +The final policy is selected from nine round-number candidates on November +2017 using a deterministic endpoint cap. Outcomes are stored separately from +its 12-column ranking frame, which contains no assumption-conditional +statistics. An outcome-free +December replay selects the same rule; opening outcomes afterward reveals +miscoverage `0.124925`, so the paper explicitly does not infer selected-set +validity from policy stability. On 276,869 out-of-time Lending Club loans, the +fixed policy earns `$179,327.59`, with weighted default `0.039375`. A matched point-PD allocation earns `$196,369.14` with weighted default `0.118400`. The paper reports both the `8.678%` return cost and the `7.9025` percentage-point default reduction, together with temporal periods in which @@ -27,7 +31,7 @@ auditable retrospective return-risk guardrail, not as a universal winner or prospective deployment guarantee. The submission contributes an explicit prediction-to-decision contract, an -exact conformal replay, an inspectable calibration selector, matched economic +exact conformal replay, a temporally separated selector/audit, matched economic comparisons, and a file-backed reproducibility package. These features align with IJDS's emphasis on data, innovative methodology, decision relevance, and reproducible evidence. diff --git a/paper/submission/CRPTO_ijds_submission.tex b/paper/submission/CRPTO_ijds_submission.tex index 30b9e55..1696887 100644 --- a/paper/submission/CRPTO_ijds_submission.tex +++ b/paper/submission/CRPTO_ijds_submission.tex @@ -44,8 +44,7 @@ \RUNAUTHOR{Anonymous} \RUNTITLE{A Calibration-Selected Conformal Guardrail for Credit Portfolios} -\TITLE{CRPTO: A Calibration-Selected Conformal Guardrail for Auditable Credit -Portfolio Decisions} +\TITLE{CRPTO: A Calibration-Selected Conformal Guardrail for Credit Portfolios} \ARTICLEAUTHORS{% \AUTHOR{} @@ -59,9 +58,12 @@ Predict-Then-Optimize (CRPTO) recomputes a 90\% Mondrian conformal upper endpoint exactly, forms the transparent decision score $q_i=(p_i+u_i)/2$, and places $q_i$ in a portfolio-risk constraint while retaining point PD in the economic -objective. Nine round-number policies are ranked on a temporal calibration -holdout without default, realized-return, or miscoverage columns; the selected -policy is then frozen and replayed on 276{,}869 out-of-time Lending Club loans. +objective. Nine round-number policies are ranked on November 2017 without +default, realized-return, miscoverage, or assumption-conditional columns; an +outcome-free December replay selects the same rule before its outcomes are +opened. That audit misses nominal funded-set coverage, making the statistical +boundary observable rather than rhetorical. The policy is then replayed on +276{,}869 out-of-time Lending Club loans. It funds 308 loans and earns \$179{,}327.59 on a \$1M budget, with weighted default 0.039375, weighted miscoverage 0.036875, and conformal endpoint budget 0.258051. A matched point-PD allocation earns \$196{,}369.14 but has weighted @@ -69,7 +71,7 @@ realized return for a 7.9025 percentage-point default reduction; the advantage reverses in some temporal slices, so we do not claim universal dominance. The contribution is a small, auditable prediction-to-decision guardrail with an -exact replay, an inspectable selector, and explicit statistical boundaries.% +exact replay, a deterministic selector, and explicit statistical boundaries.% } \KEYWORDS{conformal prediction; predict-then-optimize; credit risk; portfolio @@ -112,24 +114,25 @@ \section{Introduction}\label{sec:intro} research frontier difficult to explain. The empirical design uses a temporal Lending Club panel. The conformal recipe -is fit inside the calibration period, and a later calibration holdout ranks a -declared $3\times3$ grid of round-number risk tolerances and conformal weights. -The final ranking artifact contains no defaults, realized returns, or -miscoverage. The fixed rule is then evaluated on January 2018 through September -2020 originations. Earlier project development inspected this static OOT -corpus, so the final run is a transparent retrospective lockbox replay, not a -pristine prospective or preregistered trial. +is fit inside the calibration period, November 2017 ranks a declared +$3\times3$ grid, and December replays the outcome-free selector before auditing +the fixed decision. The ranking artifact contains no defaults, realized +returns, miscoverage, or assumption-conditional statistics. The fixed rule is +then evaluated on January 2018 through September 2020 originations. Earlier +project development inspected this static OOT corpus, so the final run is a +transparent retrospective lockbox replay, not a pristine prospective or +preregistered trial. The paper makes three contributions. First, it gives an auditable prediction-to-decision construction in which the economic objective and the conformal guardrail have separate roles. Second, it replaces approximate -cross-alpha scaling with an exact replay of the frozen conformal recipe and a -calibration-only final policy selector. Third, it reports the price and limits +cross-alpha scaling with an exact replay, a deterministic endpoint screen, and +a temporally separated selector audit. Third, it reports the price and limits of that guardrail against matched point-PD and more-conservative comparators, -including temporal slices where CRPTO wins and slices where it does not. The -novelty is the closed, inspectable decision protocol for a frozen credit model, -not a claim that conformal prediction, robust optimization, or credit scoring -is individually new. +including a pre-OOT funded-set coverage miss and temporal slices where CRPTO +wins and slices where it does not. The novelty is the closed, inspectable +decision protocol for a frozen credit model, not a claim that conformal +prediction, robust optimization, or credit scoring is individually new. \begin{figure}[t] \centering @@ -160,6 +163,15 @@ \section{Related Work}\label{sec:related} uncertainty premium, and compares the resulting allocation with a matched point-PD decision. +Decision-calibrated prediction sets and inverse conformal risk control go +further by calibrating downstream violation or regret and, after robustness +selection, using a separate split to restore risk-estimation validity +\citep{zhou2026creme,stratigakos2026decision_calibrated_sets}. CRPTO does not +import that guarantee into a batch portfolio where the optimizer chooses the +funded weights. It separates a deterministic endpoint screen from an +independent post-selection audit and reports when funded-set miscoverage misses +its nominal target. + Decision-focused learning and SPO+ train predictions against downstream regret \citep{donti2017,elmachtoub2022,mandi2024}. That is a different institutional choice. CRPTO asks what can be done after a calibrated model already exists and @@ -188,7 +200,7 @@ \section{Related Work}\label{sec:related} Robust credit portfolios & Economic selection under uncertainty & Exact conformal endpoint becomes a simple guardrail. \\ Conformal robust optimization & Coverage-backed uncertainty sets & Frozen credit stack, funded-set audit, matched economics. \\ Decision-focused learning & Training-time regret reduction & Frozen predictor and post-hoc auditability. \\ -Valid conformal selection & Validity after model or set selection & Motivates the boundary; selected-set validity is not claimed. \\ +Decision-risk calibration and valid selection & Calibrated decision losses or validity after set selection & Deterministic selector plus diagnostic audit; selected-set validity is not claimed. \\ \bottomrule \end{tabular} \end{table} @@ -209,20 +221,23 @@ \section{Data and Evaluation Design}\label{sec:data} Split & Period & Loans & Role \\ \midrule Train & Jun 2007--Mar 2017 & 1{,}346{,}311 & Fit PD model and calibrator. \\ -Calibration pool & Mar--Dec 2017 & 237{,}584 & Develop and freeze conformal and policy rules. \\ +Conformal fit & Mar--Oct 2017 & 142{,}550 & Estimate frozen conformal recipe. \\ +Policy selection & Nov 2017 & 14{,}943 & Rank nine outcome-free policies. \\ +Calibration audit & Dec 2017 & 20{,}695 & Replay selector, then open outcomes. \\ OOT evaluation & Jan 2018--Sep 2020 & 276{,}869 & Freeze-then-evaluate portfolio decisions. \\ \bottomrule \end{tabular} \end{table} -The frozen conformal recipe uses the most recent 75\% of the calibration pool -(178{,}188 rows). Within that subset, 142{,}550 rows estimate conformal -quantiles and 35{,}638 later rows, from November and December 2017, form the -temporal development holdout. Conformal endpoints use calibration labels, as -conformal prediction requires. Policy ranking uses candidate settings, solver -status, expected point-PD objective, budget use, effective-PD exposure, and -endpoint summaries. A schema guard rejects default, outcome, realized-return, -and miscoverage fields. +The frozen recipe uses the most recent 75\% of the original calibration pool +(178{,}188 rows). Within it, 142{,}550 rows estimate conformal quantiles, +14{,}943 November rows select the policy, and 20{,}695 December rows form a +pre-OOT audit. The loader stores holdout outcomes separately from the 12-column +policy frame, which contains identifiers, origination context, PD endpoints, +amounts, and rates. Ranking cannot access outcomes, realized returns, +miscoverage, or the assumption-conditional Markov quantity. December first +reruns that outcome-free selector and only then joins outcomes for the decision +audit. The final OOT panel covers several regimes, including the 2020 disruption. We report the full panel and five temporal slices. Each slice solves the same fixed @@ -293,33 +308,36 @@ \subsection{Calibration-Only Final Selector} $q_i=p_i+\gamma(u_i-p_i)$. A candidate is eligible when the solver is optimal, at least 99.9\% of budget is allocated, the effective-PD cap holds, and \begin{equation}\label{eq:screen} -B_u+\sqrt{0.10}\le0.60 +B_u\le0.28 \end{equation} -on the development holdout. Among eligible candidates, the rule maximizes -expected point-PD objective. Five of nine pass; the selected row is -$\tau=0.17,\gamma=0.50$. +on November 2017. Among eligible candidates, the rule maximizes expected +point-PD objective. Five of nine pass; the selected row is +$\tau=0.17,\gamma=0.50$. The endpoint cap is deterministic and does not use +the weighted-validity assumption below. The midpoint remains selected for +every cap in $[0.259036,0.290491)$, and an outcome-free December replay selects +the same row. \begin{table}[t] \centering \small -\caption{Calibration selector examples; the supplement reports all nine rows.} +\caption{Temporally separated selector examples; the supplement reports all nine rows.} \label{tab:selector} \resizebox{\textwidth}{!}{% -\begin{tabular}{lrrrrl} +\begin{tabular}{lrrrrrl} \toprule -Candidate & $\tau$ & $\gamma$ & Expected objective & Screen value & Status \\ +Candidate & $\tau$ & $\gamma$ & Nov. objective & Nov. $B_u$ & Dec. $B_u$ & Status \\ \midrule -Low guardrail & 0.17 & 0.25 & \$121{,}761.88 & 0.708835 & Ineligible \\ -Selected midpoint & 0.17 & 0.50 & \$110{,}346.16 & 0.577275 & Selected \\ -Conservative blend & 0.17 & 0.75 & \$104{,}272.78 & 0.519696 & Eligible \\ -Higher tolerance & 0.19 & 0.50 & \$113{,}591.27 & 0.611017 & Ineligible \\ +Low guardrail & 0.17 & 0.25 & \$109{,}885.56 & 0.375105 & 0.396751 & Ineligible \\ +Selected midpoint & 0.17 & 0.50 & \$99{,}387.12 & 0.259036 & 0.262082 & Selected twice \\ +Conservative blend & 0.17 & 0.75 & \$93{,}760.13 & 0.202259 & 0.203504 & Eligible \\ +Higher tolerance & 0.19 & 0.50 & \$102{,}671.87 & 0.290491 & 0.294861 & Ineligible \\ \bottomrule \end{tabular}} \end{table} -The 0.60 value is a declared ex-ante screen in the final tagged protocol, not -an estimate from OOT outcomes. Earlier research iterations used the OOT corpus; -the narrow claim is that this final ranking code path does not. +The selector's input whitelist contains no outcomes or +assumption-conditional quantities. Earlier research iterations used the OOT +corpus; the narrow claim is that this final ranking code path does not. \subsection{Accounting and Statistical Boundary} @@ -357,6 +375,36 @@ \subsection{Exact 90\% Conformal Evidence} almost no ranking information for a portfolio. The selected 90\% level is the frozen recipe's reference level and preserves more decision resolution. +\subsection{Pre-OOT Selector and Decision Audit} + +November selects the midpoint policy from five eligible rows. Applying the +same outcome-free rule to December again selects \texttt{linear-005}; outcomes +are opened only after that replay. + +\begin{table}[t] +\centering +\small +\caption{Independent December 2017 post-selection decision audit.} +\label{tab:calibration-audit} +\resizebox{\textwidth}{!}{% +\begin{tabular}{lrrrrr} +\toprule +Policy & Funded & Realized return & Weighted default & Miscoverage & $B_u$ \\ +\midrule +Selected 50/50 CRPTO & 193 & \$53{,}313.05 & 0.145650 & 0.124925 & 0.262082 \\ +Conservative 75\% blend & 191 & \$38{,}379.50 & 0.155250 & 0.134525 & 0.203504 \\ +Point-PD matched $\tau$ & 169 & \$89{,}732.35 & 0.185650 & 0.058300 & 0.888071 \\ +\bottomrule +\end{tabular}} +\end{table} + +The selected policy remains below its operational tolerance +($0.145650<0.17$) but misses nominal funded-set coverage +($V=0.124925>0.10$). Its deterministic accounting right-hand side is +$B_u+V=0.387007$. Thus outcome-free, month-stable selection does not create +selected-set conformal validity. This negative audit motivates the two-layer +reporting of deterministic endpoint exposure and observed miscoverage. + \subsection{Full OOT Funded-Set Audit} The fixed midpoint policy allocates the full \$1M budget across 308 loans. Its @@ -391,9 +439,11 @@ \subsection{Full OOT Funded-Set Audit} statement is not a direct default cap; the operational controls are $\tau$, $q_i$, and the funded-set diagnostics. -A fixed-allocation bootstrap gives a 95\% return interval of -\$162{,}706.17--\$193{,}924.74 from 5{,}000 funded-loan resamples. It does not -resample the model, conformal recipe, selector, or optimizer. +A fixed-allocation bootstrap over 31 origination-month clusters gives a 95\% +return interval of \$163{,}421.14--\$193{,}551.65 from 5{,}000 draws. A +funded-loan sensitivity gives \$162{,}706.17--\$193{,}924.74. Neither resamples +the model, conformal recipe, selector, or optimizer. The month-level scheme is +primary because the temporal results reject a naive independence story. \subsection{Matched Comparators} @@ -500,14 +550,17 @@ \section{Reproducibility and Limitations}\label{sec:limitations} development inspected the same OOT corpus. The evaluation is retrospective, not prospective. The conformal intervals are broad because the outcome is binary; more than half of 90\% OOT upper endpoints equal one. Mondrian coverage -does not imply validity under optimizer-selected funded weights, so the Markov -statement requires an explicit assumption. Temporal slices show that point PD -can dominate both return and default. Public data also do not support a legal -fair-lending certification or a causal interpretation. +does not imply validity under optimizer-selected funded weights. The December +audit makes this visible: the same policy is reselected without outcomes, yet +funded-set miscoverage is 0.124925. The Markov statement therefore requires an +explicit assumption and is absent from policy selection. Temporal slices show +that point PD can dominate both return and default. Public data also do not +support a legal fair-lending certification or a causal interpretation. A focused next step is a genuinely prospective or formally selection-valid protocol frozen before a new evaluation period -\citep{farinhas2024nonexchangeable_crc,hegazy2025valid_selection_conformal_sets}. +\citep{farinhas2024nonexchangeable_crc,hegazy2025valid_selection_conformal_sets, +zhou2026creme}. That extension is not hidden as an acceptance criterion for this retrospective decision audit. @@ -520,7 +573,9 @@ \section{Conclusion}\label{sec:conclusion} CRPTO shows how a frozen credit model can become an auditable portfolio decision without a maze of policy variants. An exact 90\% conformal replay produces $u_i$; the midpoint $q_i=(p_i+u_i)/2$ constrains risk; and a nine-cell -calibration selector fixes $\tau=0.17$ without reading OOT outcomes. On the +November selector fixes $\tau=0.17$ under $B_u\le0.28$. December independently +selects the same rule and then misses nominal funded-set coverage; policy +stability is therefore not relabeled as conformal validity. On the full panel, the policy earns \$179{,}327.59, with weighted default 0.039375, miscoverage 0.036875, $\Gamma_{\mathrm{CP}}=0.176102$, $\Gamma_{\mathrm{res}}=0.088051$, endpoint 0.258051, observed accounting bound diff --git a/paper/submission/IJDS_SUBMISSION_ROADMAP_2026-08-10.md b/paper/submission/IJDS_SUBMISSION_ROADMAP_2026-08-10.md index 060d592..9e77443 100644 --- a/paper/submission/IJDS_SUBMISSION_ROADMAP_2026-08-10.md +++ b/paper/submission/IJDS_SUBMISSION_ROADMAP_2026-08-10.md @@ -13,10 +13,10 @@ Official sources to recheck in the submission week: | IJDS dimension | CRPTO answer | |---|---| -| Data | Temporal Lending Club panel with a calibration development block and OOT evaluation. | -| Method | Exact 90% conformal replay and one midpoint portfolio guardrail. | +| Data | Temporal Lending Club panel with conformal fit, November selection, December audit, and OOT evaluation. | +| Method | Exact 90% conformal replay, deterministic endpoint cap, and one midpoint portfolio guardrail. | | Decision | Allocate `$1M` under capital, concentration, and effective-PD constraints. | -| Evidence | Nine-cell calibration selector, matched point-PD decision, temporal reversals, and funded-set audit. | +| Evidence | Split nine-cell selector/audit, matched point-PD decision, temporal reversals, and month-cluster bootstrap. | | Implication | An inspectable price of uncertainty, including cases where the static guardrail should be rejected. | ## Completed Scientific Refactor @@ -28,7 +28,12 @@ Official sources to recheck in the submission week: - Replaced nonlinear/tail policy families with `q=(p+u)/2`. - Separated point-PD economics from conformal feasibility. - Reduced policy selection to a round-number `3x3` calibration grid. -- Added schema guards against outcome-derived selector columns. +- Replaced the Markov-based selector screen with deterministic `B_u<=0.28` and + documented the exact cap-stability interval. +- Isolated outcomes from a 12-column selector frame; November selects and an + outcome-free December replay checks policy identity. +- Added the independent December decision audit, including the funded-set + coverage miss, and a 31-month cluster bootstrap. - Added matched point-PD and 75% blend comparators. - Promoted temporal reversals and limitations to the body. - Rebuilt A35--A40 and active claim-sync tests. @@ -50,7 +55,7 @@ Official sources to recheck in the submission week: |---|---| | Applied pipeline rather than method | One explicit objective/constraint contract and exact selector protocol. | | Broad binary conformal intervals | A35 reports width and endpoint saturation; no 99% headline. | -| Adaptive funded-set validity | Deterministic accounting is separated from assumption-conditional Markov language. | +| Adaptive funded-set validity | December directly demonstrates the coverage miss; deterministic accounting is separated from conditional Markov language. | | Historical OOT reuse | "Retrospective lockbox replay" stated in abstract, design, limitations, supplement, and cover letter. | | Baseline cherry-picking | Same candidates, budget, concentration, LGD, solver, and `tau`; temporal failures are shown. | | Too many methods | A1--A34 demoted to diagnostics; A35--A40 support one midpoint policy. | diff --git a/paper/submission/README.md b/paper/submission/README.md index 6a084ad..5f4b0b6 100644 --- a/paper/submission/README.md +++ b/paper/submission/README.md @@ -11,9 +11,12 @@ submission materials. The synchronized scientific sources are: - `paper/submission/REPRODUCIBILITY_PACKAGE.md`: data/code package plan. The active manuscript has one policy: exact 90% conformal replay, -`q=(p+u)/2`, `tau=0.17`, and a nine-cell calibration selector. A35--A40 are the -active evidence bundle. Keep body, supplement, TeX, and governance numerically -aligned with `tests/test_ijds_active_claim_sync.py`. +`q=(p+u)/2`, `tau=0.17`, and a nine-cell November selector under +`B_u<=0.28`. An outcome-free December replay and post-selection audit are part +of A36; the audit deliberately records that stable policy identity does not +imply selected-set coverage. A35--A40 are the active evidence bundle. Keep +body, supplement, TeX, and governance numerically aligned with +`tests/test_ijds_active_claim_sync.py`. ## Preview @@ -48,9 +51,10 @@ if (-not $env:WINDIR) { $env:WINDIR = $env:SystemRoot } latexmk -pdf -gg -interaction=nonstopmode CRPTO_ijds_submission.tex ``` -`latexmk` is preferred because it automates convergence. Some Windows TinyTeX -installations fail in the wrapper even when `pdflatex`, BibTeX, and TeX Live are -healthy. The robust fallback is: +`latexmk` is preferred because it automates convergence. The repository build +resolves TinyTeX's `latexmk.pl` and launches it with Perl on Windows, bypassing +the defective `runscript.tlu` executable wrapper. If that payload is unavailable +or fails, the robust fallback is: ```text pdflatex -> bibtex -> pdflatex -> pdflatex @@ -78,7 +82,8 @@ if (-not $env:WINDIR) { $env:WINDIR = $env:SystemRoot } fmtutil-sys --byfmt pdflatex ``` -The repository wrapper runs `latexmk` first and falls back automatically: +The repository wrapper runs the working `latexmk` payload first and falls back +automatically: ```powershell just paper-submission-official diff --git a/paper/submission/REPRODUCIBILITY_PACKAGE.md b/paper/submission/REPRODUCIBILITY_PACKAGE.md index 45b7654..b3570b0 100644 --- a/paper/submission/REPRODUCIBILITY_PACKAGE.md +++ b/paper/submission/REPRODUCIBILITY_PACKAGE.md @@ -19,7 +19,7 @@ local paths, or author identity during double-anonymous review. |---|---|---| | Environment | `pyproject.toml`, `uv.lock`, `justfile`. | Recreate the Windows-first toolchain. | | Method source | `src/models/conformal_alpha_grid.py`, `src/optimization/`, active experiment scripts. | Replay exact intervals and solve declared policies. | -| Active config | `configs/experiments/champion_reopen_ijds_calibration_selected_simple90_v6.yaml`. | Fix alpha, 3x3 grid, selector, and solver settings. | +| Active config | `configs/experiments/champion_reopen_ijds_calibration_selected_endpoint28_v7.yaml`. | Fix alpha, split 3x3 selector/audit, endpoint cap, and solver settings. | | Active evidence | A35--A40 CSV/TeX files and `ijds_policy_governance.json`. | Tie every paper claim to generated evidence. | | Manuscript | body QMD, supplement QMD, official TeX. | Reproduce reviewer-facing surfaces. | | Data pointers | `dvc.yaml`, `dvc.lock`, `.dvc/`, raw-data notes. | Retrieve large artifacts where terms permit. | @@ -31,13 +31,15 @@ local paths, or author identity during double-anonymous review. |---|---| | Exact alpha grid | `data/processed/experiments/champion_reopen//conformal/exact_alpha_grid.parquet` | | Calibration selector | `data/processed/experiments/champion_reopen//portfolio/calibration_policy_selection_grid.parquet` | +| Calibration audit grid | `data/processed/experiments/champion_reopen//portfolio/calibration_policy_audit_grid.parquet` | +| Calibration decision audit | `data/processed/experiments/champion_reopen//portfolio/calibration_policy_holdout_audit.csv` | | OOT evaluation | `data/processed/experiments/champion_reopen//portfolio/calibration_selected_policy_oot_evaluation.csv` | | Funded rows | `data/processed/experiments/champion_reopen//portfolio/calibration_selected_policy_full_oot_allocations.parquet` | | Governance | `models/experiments/champion_reopen//portfolio/ijds_policy_governance.json` | | Paper tables | `reports/crpto/tables/crpto_tableA35...A40_*` | Active run: -`champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6`. +`champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7`. Exact-alpha run: `champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1`. @@ -66,9 +68,10 @@ Full isolated methodology replay: just ijds-active-replay ``` -The full replay recomputes the exact alpha grid, solves the nine calibration -policies, evaluates the frozen selected policy, and rebuilds A35--A40. It writes -only to versioned experiment paths and does not overwrite the frozen PD model, +The full replay recomputes the exact alpha grid, solves the nine policies on +November and December, opens December outcomes only for the independent audit, +evaluates the frozen selected policy, and rebuilds A35--A40. It writes only to +versioned experiment paths and does not overwrite the frozen PD model, calibrator, historical intervals, or manifest. Official-template compilation is automated by: diff --git a/paper/submission/SCHOLARONE_FINAL_CHECKLIST.md b/paper/submission/SCHOLARONE_FINAL_CHECKLIST.md index 3c00d4d..0e1c35e 100644 --- a/paper/submission/SCHOLARONE_FINAL_CHECKLIST.md +++ b/paper/submission/SCHOLARONE_FINAL_CHECKLIST.md @@ -19,7 +19,7 @@ Use only after the scientific content and official PDFs are frozen. just paper-submission-official ``` -The wrapper tries `latexmk` and falls back to the verified +The wrapper uses the direct `latexmk.pl` payload on Windows and falls back to the verified `pdflatex -> bibtex -> pdflatex -> pdflatex` loop. Accept only when: - `.blg` has zero warnings; diff --git a/paper/supplement_ijds.qmd b/paper/supplement_ijds.qmd index 7a978f4..35aebb3 100644 --- a/paper/supplement_ijds.qmd +++ b/paper/supplement_ijds.qmd @@ -1,5 +1,5 @@ --- -title: "Online Supplement for CRPTO: A Calibration-Selected Conformal Guardrail for Auditable Credit Portfolio Decisions" +title: "Online Supplement for CRPTO: A Calibration-Selected Conformal Guardrail for Credit Portfolios" author: "Anonymous" date: today lang: en @@ -37,7 +37,7 @@ policy. | A36 | All nine calibration selector cells. | No OOT outcome-derived ranking columns. | | A37 | Full-OOT and fixed-policy temporal evaluation. | Retrospective stress evidence, not prospective performance. | | A38 | Funded exposure and outcomes by letter grade. | Business composition, not legal fair-lending certification. | -| A39 | Fixed-allocation funded-loan bootstrap. | Does not resample model, conformal recipe, selector, or solver. | +| A39 | Fixed-allocation month-cluster bootstrap plus loan-level sensitivity. | Does not resample model, conformal recipe, selector, or solver. | | A40 | Selected, more-conservative, and matched point-PD decisions. | Retrospective contrasts, not causal effects or universal dominance. | The active governance file is @@ -181,8 +181,7 @@ selected-set validity would require a dedicated protocol | no | `0.15` | `0.1425` | `0.886098` | `0.646176` | `0.863562` | `0.863101` | `0.334859` | | no | `0.20` | `0.1900` | `0.849380` | `0.636585` | `0.829376` | `0.828007` | `0.244668` | -: A35 exact alpha sensitivity. Source: -`reports/crpto/tables/crpto_tableA35_exact_alpha_grid.csv`. +: A35 exact alpha sensitivity. Source: `crpto_tableA35_exact_alpha_grid.csv`. The 99% row is nearly saturated and is not the active policy level. Rows other than 90% apply the same frozen widening recipe as sensitivity; they are not @@ -190,24 +189,30 @@ separately tuned conformal winners. ## A36. Calibration Policy Selector -| Selected | Eligible | Candidate | $\tau$ | $\gamma$ | Expected objective | Endpoint | Threshold | -|:---:|:---:|---|---:|---:|---:|---:|---:| -| yes | yes | `linear-005` | `0.17` | `0.50` | `$110,346.16` | `0.261047` | `0.577275` | -| no | yes | `linear-009` | `0.19` | `0.75` | `$107,122.88` | `0.228340` | `0.544567` | -| no | yes | `linear-002` | `0.15` | `0.50` | `$106,866.89` | `0.226670` | `0.542897` | -| no | yes | `linear-006` | `0.17` | `0.75` | `$104,272.78` | `0.203468` | `0.519696` | -| no | yes | `linear-003` | `0.15` | `0.75` | `$101,108.78` | `0.177940` | `0.494167` | -| no | no | `linear-007` | `0.19` | `0.25` | `$126,123.20` | `0.446819` | `0.763046` | -| no | no | `linear-004` | `0.17` | `0.25` | `$121,761.88` | `0.392607` | `0.708835` | -| no | no | `linear-001` | `0.15` | `0.25` | `$117,071.33` | `0.341219` | `0.657447` | -| no | no | `linear-008` | `0.19` | `0.50` | `$113,591.27` | `0.294789` | `0.611017` | - -: A36 complete selector. Source: -`reports/crpto/tables/crpto_tableA36_calibration_policy_selector.csv`. - -The selector uses `35,638` temporal calibration-holdout rows. Its output schema -contains zero forbidden outcome-derived fields. The selected row is the highest -expected-objective eligible candidate, not the highest objective overall. +| Nov. status | Candidate | $\tau$ | $\gamma$ | Nov. objective | Nov. $B_u$ | Dec. $B_u$ | +|---|---|---:|---:|---:|---:|---:| +| **Selected** | `linear-005` | `0.17` | `0.50` | `$99,387.12` | `0.259036` | `0.262082` | +| Eligible | `linear-009` | `0.19` | `0.75` | `$96,446.16` | `0.226723` | `0.228577` | +| Eligible | `linear-002` | `0.15` | `0.50` | `$95,995.22` | `0.226088` | `0.227695` | +| Eligible | `linear-006` | `0.17` | `0.75` | `$93,760.13` | `0.202259` | `0.203504` | +| Eligible | `linear-003` | `0.15` | `0.75` | `$90,893.16` | `0.177614` | `0.178203` | +| Ineligible | `linear-007` | `0.19` | `0.25` | `$113,986.45` | `0.422203` | `0.446378` | +| Ineligible | `linear-004` | `0.17` | `0.25` | `$109,885.56` | `0.375105` | `0.396751` | +| Ineligible | `linear-001` | `0.15` | `0.25` | `$105,398.23` | `0.328413` | `0.343502` | +| Ineligible | `linear-008` | `0.19` | `0.50` | `$102,671.87` | `0.290491` | `0.294861` | + +: A36 complete selector. Source: `crpto_tableA36_calibration_policy_selector.csv`. + +The selector uses `14,943` November rows and the deterministic screen +$B_u\le0.28$. The loader keeps outcomes separate from its 12-column input +frame, which contains no outcome or assumption-conditional quantity. The +selected row remains optimal for caps in +`[0.259036, 0.290491)`. Replaying the same outcome-free rule on `20,695` +December rows selects the same policy. Only then are December outcomes opened: +the midpoint funds 193 loans, returns `$53,313.05`, has weighted default +`0.145650`, weighted miscoverage `0.124925`, and endpoint `0.262082`. The +miscoverage miss is direct evidence that selector stability is not selected-set +coverage validity. ## A37. Temporal Fixed-Policy Evaluation @@ -228,7 +233,7 @@ expected-objective eligible candidate, not the highest objective overall. | 2020+ | Point PD | `$218,629.14` | `0.016900` | `0.000000` | `0.861425` | `1.177653` | : A37 selected temporal rows and matched point-PD rows. Source: -`reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.csv`. +`crpto_tableA37_calibration_selected_temporal_evaluation.csv`. Each temporal policy receives a fresh `$1M` budget. The table is intentionally unfavorable to a universal-dominance reading: point PD is much stronger in @@ -246,7 +251,7 @@ unfavorable to a universal-dominance reading: point PD is much stronger in | G | `2` | `0.0020` | `0.000000` | `0.000000` | `0.146801` | `0.573401` | `1.000000` | `$616.30` | : A38 selected funded-set composition. Source: -`reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.csv`. +`crpto_tableA38_calibration_selected_grade_audit.csv`. The table uses letter grade recovered from `sub_grade`; the aligned conformal group is stored separately. This prevents the score-quantile partition labels @@ -254,31 +259,39 @@ from being misreported as credit grades. ## A39. Fixed-Allocation Bootstrap -| Metric | Observed | Bootstrap mean | 2.5% | Median | 97.5% | -|---|---:|---:|---:|---:|---:| -| Realized return | `$179,327.59` | `$179,075.48` | `$162,706.17` | `$179,416.99` | `$193,924.74` | -| Weighted default | `0.039375` | `0.039638` | `0.020869` | `0.039043` | `0.061742` | -| Weighted miscoverage | `0.036875` | `0.037203` | `0.019358` | `0.036715` | `0.058812` | -| $\Gamma_{\mathrm{CP}}$ | `0.176102` | `0.176026` | `0.137159` | `0.174808` | `0.224308` | -| Endpoint budget | `0.258051` | `0.257942` | `0.217308` | `0.256762` | `0.308571` | - -: A39, 5,000 resamples with seed `20260709`. Source: -`reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.csv`. +| Resampling unit | Metric | Observed | 2.5% | 97.5% | +|---|---|---:|---:|---:| +| Month clusters | Realized return | `$179,327.59` | `$163,421.14` | `$193,551.65` | +| Month clusters | Weighted default | `0.039375` | `0.020517` | `0.062622` | +| Month clusters | Weighted miscoverage | `0.036875` | `0.018338` | `0.060084` | +| Month clusters | $\Gamma_{\mathrm{CP}}$ | `0.176102` | `0.132145` | `0.230815` | +| Month clusters | Endpoint budget | `0.258051` | `0.211996` | `0.314870` | +| Funded loans | Realized return | `$179,327.59` | `$162,706.17` | `$193,924.74` | +| Funded loans | Weighted default | `0.039375` | `0.020869` | `0.061742` | +| Funded loans | Weighted miscoverage | `0.036875` | `0.019358` | `0.058812` | +| Funded loans | $\Gamma_{\mathrm{CP}}$ | `0.176102` | `0.137159` | `0.224308` | +| Funded loans | Endpoint budget | `0.258051` | `0.217308` | `0.308571` | + +: A39, 5,000 fixed-allocation resamples per scheme with seed `20260709`. +Month clusters are primary; funded-loan resampling is an independence +sensitivity. Source: `crpto_tableA39_calibration_selected_bootstrap.csv`. The +CSV also records bootstrap means and medians. ## A40. Matched Decision Audit -| Policy | Expected objective | Realized return | Default | Miscoverage | Endpoint | Threshold | +| Policy | Expected objective | Realized return | Default | Miscoverage | Endpoint | Cond. threshold | |---|---:|---:|---:|---:|---:|---:| | Selected 50/50 CRPTO | `$168,271.56` | `$179,327.59` | `0.039375` | `0.036875` | `0.258051` | `0.574279` | | More-conservative 75% blend | `$160,690.13` | `$172,939.50` | `0.035875` | `0.035875` | `0.200396` | `0.516624` | | Point-PD matched-$\tau$ | `$214,019.15` | `$196,369.14` | `0.118400` | `0.041900` | `0.921317` | `1.237545` | : A40 matched full-OOT audit. Source: -`reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.csv`. +`crpto_tableA40_calibration_selected_point_baseline.csv`. Selected CRPTO pays `$17,041.55` (`8.678%`) relative to point PD. It reduces weighted default by `7.9025` percentage points, weighted miscoverage by -`0.5025` percentage points, and the threshold by `66.3266` percentage points. +`0.5025` percentage points, and the conditional threshold by `66.3266` +percentage points. Its exact active funded-set quantities are $\Gamma_{\mathrm{CP}}=0.176102$, $\Gamma_{\mathrm{res}}=0.088051$, @@ -349,7 +362,7 @@ historical intervals, and manifest are never overwritten. | Paper tables | `reports/crpto/tables/crpto_tableA35...A40_*` | The active run is -`champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6`. +`champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7`. The exact-alpha run is `champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1`. @@ -371,12 +384,13 @@ claim evolve without rewriting or silently reinterpreting frozen provenance. | Claim | Direct evidence | What is not claimed | |---|---|---| | Exact 90% replay | Reference endpoints match to `6.67e-16`; A35. | Independent tuning of every alpha sensitivity row. | -| OOT-outcome-column-free policy ranking, conditional on the frozen recipe | A36 schema and selector audit. | A label-free conformal recipe, historically untouched OOT corpus, or preregistration. | +| Outcome-free, assumption-free policy ranking, conditional on the frozen recipe | A36 physically separated outcomes, deterministic endpoint cap, cap-stability interval, and December replay. | A label-free conformal recipe, historically untouched OOT corpus, or preregistration. | +| Independent calibration decision audit | December reselects the midpoint before outcomes are opened; default `0.145650`, miscoverage `0.124925`. | Selected-set conformal validity or a prospective deployment guarantee. | | Full-OOT return-risk trade-off | A37 and A40. | Causal effect or universal dominance. | | Funded-set accounting | Proposition A.1 and exact row-level allocation. | Nominal selected-set conformal coverage. | | Conditional Markov sensitivity | Corollary A.1 under weighted validity. | Deterministic risk cap or sharp tail guarantee. | | Composition transparency | A38 letter-grade reconciliation. | Legal fair-lending certification. | -| Contribution stability | A39 fixed-allocation bootstrap. | Full pipeline uncertainty or model-selection confidence interval. | +| Contribution stability | A39 month-cluster bootstrap with funded-loan sensitivity. | Full pipeline uncertainty or model-selection confidence interval. | | Reproducibility | Commands, run tags, hashes, and sync tests. | Cross-machine bit-identical model retraining. | The active result is intentionally narrow. Optimized OCE/CVaR, online or diff --git a/reports/crpto/tables/README.md b/reports/crpto/tables/README.md index 27b1f44..a0a07fa 100644 --- a/reports/crpto/tables/README.md +++ b/reports/crpto/tables/README.md @@ -15,12 +15,12 @@ those is cited in the paper, the supplement, or a book chapter (see | `A12`–`A21`, `A21b` | Tail risk, satisficing, dependence, regret-auditability, cluster/concentration bounds. | | `A22`–`A24` | Tail-constrained re-opt, multi-distribution, online ACI diagnostics. | | `A25`–`A34` | External multidataset replication (Prosper/Freddie) and cross-dataset price of robustness. | -| `A35` | Pool93 IJDS finite-grid return-bound frontier and final claim endpoints. | -| `A36` | Pool93 body-point funded-set grade audit regenerated from the selected allocation. | -| `A37` | Pool93 body-point tail-risk repricing regenerated from the selected allocation. | -| `A38` | Pool93 body-point cluster-bound audit regenerated from the selected allocation. | -| `A39` | Pool93 body-point fixed-allocation bootstrap metrics regenerated from the selected allocation. | -| `A40` | Matched point-PD baseline on the same candidate universe and operating constraints. | +| `A35` | Active exact-alpha replay, coverage, width, and endpoint-saturation audit. | +| `A36` | Active November selector, endpoint-cap stability, and December outcome-free replay. | +| `A37` | Active full-OOT and temporal fixed-policy evaluation. | +| `A38` | Active selected funded-set letter-grade composition. | +| `A39` | Active month-cluster bootstrap with funded-loan sensitivity. | +| `A40` | Active selected, conservative, and matched point-PD decision comparison. | ## Legacy / superseded (retained for provenance, not paper-facing) @@ -31,9 +31,10 @@ Do not cite them in new prose; prefer the active set above. The frozen Lending Club `price_of_robustness=-10.56%` fields in table 0, table 1 and A2 are retained only for provenance. Their historical "nonrobust" baseline inherited a conformal endpoint constraint and was not a point-PD -comparator. The active IJDS claim uses A35's policy-aware frontier together with -the matched A40 decision audit; see -`docs/research/pool93_certificate_semantics_v2_2026-07-09.md`. +comparator. Protected historical pool93 exports retain their explicit +`*_pool93_*` filenames; they are provenance, not the active A35--A40 bundle. +See `docs/research/active_claims_2026-07-04.md` for the active contract and +`docs/research/pool93_certificate_semantics_v2_2026-07-09.md` for history. - `crpto_table1_robustness_summary` - `crpto_table2_conformal_variant_benchmark` diff --git a/reports/crpto/tables/crpto_tableA36_calibration_policy_selector.csv b/reports/crpto/tables/crpto_tableA36_calibration_policy_selector.csv index 030eaab..1e270cf 100644 --- a/reports/crpto/tables/crpto_tableA36_calibration_policy_selector.csv +++ b/reports/crpto/tables/crpto_tableA36_calibration_policy_selector.csv @@ -1,10 +1,10 @@ -selected,eligible,candidate_id,risk_tolerance,gamma,expected_objective,n_funded,weighted_pd_point,weighted_pd_effective,endpoint_budget,markov_loss_threshold -True,True,linear-005,0.17,0.5,110346.16233639097,211,0.07895300428885349,0.17,0.26104699571114653,0.5772747617279845 -False,True,linear-009,0.19,0.75,107122.87628227308,209,0.07498132983738741,0.19,0.22833955672087086,0.5445673227377088 -False,True,linear-002,0.15,0.5,106866.88915535323,206,0.07333026954904732,0.14999999999999997,0.22666973045095268,0.5428974964677906 -False,True,linear-006,0.17,0.75,104272.78444263109,207,0.06959450418765509,0.17,0.20346849860411498,0.5196962646209529 -False,True,linear-003,0.15,0.75,101108.78384345812,202,0.0661808100942338,0.15,0.1779397299685887,0.49416749598542664 -False,False,linear-007,0.19,0.25,126123.1973417947,207,0.10439377945504501,0.19,0.44681866163486494,0.7630464276517028 -False,False,linear-004,0.17,0.25,121761.8779833937,212,0.09579764669795818,0.17000000000000004,0.39260705990612554,0.7088348259229635 -False,False,linear-001,0.15,0.25,117071.33325909675,214,0.08626026685001403,0.15,0.34121919944995793,0.6574469654667958 -False,False,linear-008,0.19,0.5,113591.26651572464,211,0.0852111983407822,0.19,0.29478880165921784,0.6110165676760557 +selected,eligible,audit_selected,audit_eligible,candidate_id,risk_tolerance,gamma,expected_objective,n_funded,weighted_pd_point,weighted_pd_effective,endpoint_budget,audit_expected_objective,audit_n_funded,audit_endpoint_budget +True,True,True,True,linear-005,0.17,0.5,99387.12330098984,162,0.08096395060564901,0.17,0.25903604939435104,103567.0099590662,193,0.2620820323280323 +False,True,False,True,linear-009,0.19,0.75,96446.15634571345,165,0.07983135268116329,0.19,0.22672288243961225,100439.75691860713,192,0.22857742256378005 +False,True,False,True,linear-002,0.15,0.5,95995.21888305439,166,0.07391185908208518,0.15000000000000002,0.2260881409179148,100253.3313725306,190,0.22769467893384143 +False,True,False,True,linear-006,0.17,0.75,93760.12764708955,168,0.0732219020269643,0.17,0.20225936599101194,97897.74942687502,191,0.2035038230017554 +False,True,False,True,linear-003,0.15,0.75,90893.16015692815,167,0.06715742012587558,0.15,0.17761419329137482,95103.61442940257,186,0.17820330775871246 +False,False,False,False,linear-007,0.19,0.25,113986.45107415256,170,0.11259889470623649,0.19,0.42220331588129056,118537.51039714873,194,0.4463778904768255 +False,False,False,False,linear-004,0.17,0.25,109885.56215750676,166,0.10163175926165047,0.16999999999999998,0.3751047222150486,114373.21276113224,196,0.396751177625259 +False,False,False,False,linear-001,0.15,0.25,105398.22903742193,166,0.09052889811149456,0.15,0.3284133056655163,109967.55921988936,194,0.34350244765822646 +False,False,False,False,linear-008,0.19,0.5,102671.8741997059,163,0.08950921111283665,0.19,0.29049078888716334,106734.80147640246,191,0.29486067740758526 diff --git a/reports/crpto/tables/crpto_tableA36_calibration_policy_selector.tex b/reports/crpto/tables/crpto_tableA36_calibration_policy_selector.tex index a22910e..1146224 100644 --- a/reports/crpto/tables/crpto_tableA36_calibration_policy_selector.tex +++ b/reports/crpto/tables/crpto_tableA36_calibration_policy_selector.tex @@ -1,15 +1,15 @@ -\begin{tabular}{rrlrrrrrrrr} +\begin{tabular}{rrrrlrrrrrrrrrr} \toprule -selected & eligible & candidate\_id & risk\_tolerance & gamma & expected\_objective & n\_funded & weighted\_pd\_point & weighted\_pd\_effective & endpoint\_budget & markov\_loss\_threshold \\ +selected & eligible & audit\_selected & audit\_eligible & candidate\_id & risk\_tolerance & gamma & expected\_objective & n\_funded & weighted\_pd\_point & weighted\_pd\_effective & endpoint\_budget & audit\_expected\_objective & audit\_n\_funded & audit\_endpoint\_budget \\ \midrule -True & True & linear-005 & 0.170000 & 0.500000 & 110346.162336 & 211 & 0.078953 & 0.170000 & 0.261047 & 0.577275 \\ -False & True & linear-009 & 0.190000 & 0.750000 & 107122.876282 & 209 & 0.074981 & 0.190000 & 0.228340 & 0.544567 \\ -False & True & linear-002 & 0.150000 & 0.500000 & 106866.889155 & 206 & 0.073330 & 0.150000 & 0.226670 & 0.542897 \\ -False & True & linear-006 & 0.170000 & 0.750000 & 104272.784443 & 207 & 0.069595 & 0.170000 & 0.203468 & 0.519696 \\ -False & True & linear-003 & 0.150000 & 0.750000 & 101108.783843 & 202 & 0.066181 & 0.150000 & 0.177940 & 0.494167 \\ -False & False & linear-007 & 0.190000 & 0.250000 & 126123.197342 & 207 & 0.104394 & 0.190000 & 0.446819 & 0.763046 \\ -False & False & linear-004 & 0.170000 & 0.250000 & 121761.877983 & 212 & 0.095798 & 0.170000 & 0.392607 & 0.708835 \\ -False & False & linear-001 & 0.150000 & 0.250000 & 117071.333259 & 214 & 0.086260 & 0.150000 & 0.341219 & 0.657447 \\ -False & False & linear-008 & 0.190000 & 0.500000 & 113591.266516 & 211 & 0.085211 & 0.190000 & 0.294789 & 0.611017 \\ +True & True & True & True & linear-005 & 0.170000 & 0.500000 & 99387.123301 & 162 & 0.080964 & 0.170000 & 0.259036 & 103567.009959 & 193 & 0.262082 \\ +False & True & False & True & linear-009 & 0.190000 & 0.750000 & 96446.156346 & 165 & 0.079831 & 0.190000 & 0.226723 & 100439.756919 & 192 & 0.228577 \\ +False & True & False & True & linear-002 & 0.150000 & 0.500000 & 95995.218883 & 166 & 0.073912 & 0.150000 & 0.226088 & 100253.331373 & 190 & 0.227695 \\ +False & True & False & True & linear-006 & 0.170000 & 0.750000 & 93760.127647 & 168 & 0.073222 & 0.170000 & 0.202259 & 97897.749427 & 191 & 0.203504 \\ +False & True & False & True & linear-003 & 0.150000 & 0.750000 & 90893.160157 & 167 & 0.067157 & 0.150000 & 0.177614 & 95103.614429 & 186 & 0.178203 \\ +False & False & False & False & linear-007 & 0.190000 & 0.250000 & 113986.451074 & 170 & 0.112599 & 0.190000 & 0.422203 & 118537.510397 & 194 & 0.446378 \\ +False & False & False & False & linear-004 & 0.170000 & 0.250000 & 109885.562158 & 166 & 0.101632 & 0.170000 & 0.375105 & 114373.212761 & 196 & 0.396751 \\ +False & False & False & False & linear-001 & 0.150000 & 0.250000 & 105398.229037 & 166 & 0.090529 & 0.150000 & 0.328413 & 109967.559220 & 194 & 0.343502 \\ +False & False & False & False & linear-008 & 0.190000 & 0.500000 & 102671.874200 & 163 & 0.089509 & 0.190000 & 0.290491 & 106734.801476 & 191 & 0.294861 \\ \bottomrule \end{tabular} diff --git a/reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.csv b/reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.csv index 57fe384..ebd91b3 100644 --- a/reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.csv +++ b/reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.csv @@ -1,6 +1,11 @@ -metric,observed,boot_mean,boot_p025,boot_p50,boot_p975,n_draws,seed,note -realized_return,179327.5851322598,179075.4838443055,162706.17200230644,179416.98769601624,193924.73990027257,5000,20260709,"Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled." -weighted_default_rate,0.039375,0.03963778218343555,0.020869102395698755,0.039043420065914713,0.0617419907402632,5000,20260709,"Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled." -weighted_miscoverage,0.036875,0.03720263786105997,0.019357859801356386,0.03671487666893709,0.058812406466411934,5000,20260709,"Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled." -Gamma_CP,0.1761021788469351,0.17602573881357803,0.13715911012280738,0.17480824046170865,0.2243084301758857,5000,20260709,"Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled." -endpoint_budget,0.2580510894234676,0.25794196914296347,0.21730801774100114,0.2567618086877488,0.30857101650304447,5000,20260709,"Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled." +bootstrap_unit,metric,observed,boot_mean,boot_p025,boot_p50,boot_p975,n_units,n_draws,seed,note +origination_month,realized_return,179327.5851322598,179293.01466566528,163421.1394838545,179487.08382041534,193551.6460089551,31,5000,20260709,"Fixed-allocation origination-month cluster bootstrap, renormalized to the $1M budget; model, intervals, selector, and solver are not resampled." +origination_month,weighted_default_rate,0.039375,0.03943546399994348,0.02051693505224659,0.03875339983492982,0.06262154165393485,31,5000,20260709,"Fixed-allocation origination-month cluster bootstrap, renormalized to the $1M budget; model, intervals, selector, and solver are not resampled." +origination_month,weighted_miscoverage,0.036875,0.03691656808682405,0.01833810380778745,0.03613937487855996,0.060083696168230914,31,5000,20260709,"Fixed-allocation origination-month cluster bootstrap, renormalized to the $1M budget; model, intervals, selector, and solver are not resampled." +origination_month,Gamma_CP,0.1761021788469351,0.17637453569205386,0.1321447744854007,0.1746349832245928,0.2308148609432896,31,5000,20260709,"Fixed-allocation origination-month cluster bootstrap, renormalized to the $1M budget; model, intervals, selector, and solver are not resampled." +origination_month,endpoint_budget,0.2580510894234676,0.2583077550716801,0.21199608169774178,0.25666011731200084,0.3148701678446982,31,5000,20260709,"Fixed-allocation origination-month cluster bootstrap, renormalized to the $1M budget; model, intervals, selector, and solver are not resampled." +funded_loan,realized_return,179327.5851322598,179075.4838443055,162706.17200230644,179416.98769601624,193924.73990027257,308,5000,20260709,"Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled." +funded_loan,weighted_default_rate,0.039375,0.03963778218343555,0.020869102395698755,0.039043420065914713,0.0617419907402632,308,5000,20260709,"Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled." +funded_loan,weighted_miscoverage,0.036875,0.03720263786105997,0.019357859801356386,0.03671487666893709,0.058812406466411934,308,5000,20260709,"Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled." +funded_loan,Gamma_CP,0.1761021788469351,0.17602573881357803,0.13715911012280738,0.17480824046170865,0.2243084301758857,308,5000,20260709,"Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled." +funded_loan,endpoint_budget,0.2580510894234676,0.25794196914296347,0.21730801774100114,0.2567618086877488,0.30857101650304447,308,5000,20260709,"Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled." diff --git a/reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.tex b/reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.tex index 673c60e..dda735b 100644 --- a/reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.tex +++ b/reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.tex @@ -1,11 +1,16 @@ -\begin{tabular}{lrrrrrrrl} +\begin{tabular}{llrrrrrrrrl} \toprule -metric & observed & boot\_mean & boot\_p025 & boot\_p50 & boot\_p975 & n\_draws & seed & note \\ +bootstrap\_unit & metric & observed & boot\_mean & boot\_p025 & boot\_p50 & boot\_p975 & n\_units & n\_draws & seed & note \\ \midrule -realized\_return & 179327.585132 & 179075.483844 & 162706.172002 & 179416.987696 & 193924.739900 & 5000 & 20260709 & Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled. \\ -weighted\_default\_rate & 0.039375 & 0.039638 & 0.020869 & 0.039043 & 0.061742 & 5000 & 20260709 & Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled. \\ -weighted\_miscoverage & 0.036875 & 0.037203 & 0.019358 & 0.036715 & 0.058812 & 5000 & 20260709 & Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled. \\ -Gamma\_CP & 0.176102 & 0.176026 & 0.137159 & 0.174808 & 0.224308 & 5000 & 20260709 & Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled. \\ -endpoint\_budget & 0.258051 & 0.257942 & 0.217308 & 0.256762 & 0.308571 & 5000 & 20260709 & Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled. \\ +origination\_month & realized\_return & 179327.585132 & 179293.014666 & 163421.139484 & 179487.083820 & 193551.646009 & 31 & 5000 & 20260709 & Fixed-allocation origination-month cluster bootstrap, renormalized to the \$1M budget; model, intervals, selector, and solver are not resampled. \\ +origination\_month & weighted\_default\_rate & 0.039375 & 0.039435 & 0.020517 & 0.038753 & 0.062622 & 31 & 5000 & 20260709 & Fixed-allocation origination-month cluster bootstrap, renormalized to the \$1M budget; model, intervals, selector, and solver are not resampled. \\ +origination\_month & weighted\_miscoverage & 0.036875 & 0.036917 & 0.018338 & 0.036139 & 0.060084 & 31 & 5000 & 20260709 & Fixed-allocation origination-month cluster bootstrap, renormalized to the \$1M budget; model, intervals, selector, and solver are not resampled. \\ +origination\_month & Gamma\_CP & 0.176102 & 0.176375 & 0.132145 & 0.174635 & 0.230815 & 31 & 5000 & 20260709 & Fixed-allocation origination-month cluster bootstrap, renormalized to the \$1M budget; model, intervals, selector, and solver are not resampled. \\ +origination\_month & endpoint\_budget & 0.258051 & 0.258308 & 0.211996 & 0.256660 & 0.314870 & 31 & 5000 & 20260709 & Fixed-allocation origination-month cluster bootstrap, renormalized to the \$1M budget; model, intervals, selector, and solver are not resampled. \\ +funded\_loan & realized\_return & 179327.585132 & 179075.483844 & 162706.172002 & 179416.987696 & 193924.739900 & 308 & 5000 & 20260709 & Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled. \\ +funded\_loan & weighted\_default\_rate & 0.039375 & 0.039638 & 0.020869 & 0.039043 & 0.061742 & 308 & 5000 & 20260709 & Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled. \\ +funded\_loan & weighted\_miscoverage & 0.036875 & 0.037203 & 0.019358 & 0.036715 & 0.058812 & 308 & 5000 & 20260709 & Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled. \\ +funded\_loan & Gamma\_CP & 0.176102 & 0.176026 & 0.137159 & 0.174808 & 0.224308 & 308 & 5000 & 20260709 & Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled. \\ +funded\_loan & endpoint\_budget & 0.258051 & 0.257942 & 0.217308 & 0.256762 & 0.308571 & 308 & 5000 & 20260709 & Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled. \\ \bottomrule \end{tabular} diff --git a/scripts/build_ijds_calibration_selected_evidence.py b/scripts/build_ijds_calibration_selected_evidence.py index d49ced2..685ef0e 100644 --- a/scripts/build_ijds_calibration_selected_evidence.py +++ b/scripts/build_ijds_calibration_selected_evidence.py @@ -17,7 +17,7 @@ from src.optimization.policy_selection import policy_eligibility_mask # noqa: E402 from src.utils.script_helpers import load_json, write_json, write_table # noqa: E402 -RUN_TAG = "champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6" +RUN_TAG = "champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7" EXACT_ALPHA_TAG = "champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1" MODEL_DIR = ROOT / "models/experiments/champion_reopen" / RUN_TAG / "portfolio" DATA_DIR = ROOT / "data/processed/experiments/champion_reopen" / RUN_TAG / "portfolio" @@ -60,21 +60,58 @@ def build_alpha_table(summary: dict[str, Any]) -> pd.DataFrame: ] -def build_selector_table(grid: pd.DataFrame, summary: dict[str, Any]) -> pd.DataFrame: +def build_selector_table( + grid: pd.DataFrame, + audit_grid: pd.DataFrame, + summary: dict[str, Any], +) -> pd.DataFrame: selected_id = str(summary["selected_policy"]["candidate_id"]) - cap = float(summary["design"]["markov_threshold_cap"]) + audit_selected_id = str(summary["calibration_audit"]["outcome_free_selected_candidate_id"]) + cap = float(summary["design"]["endpoint_budget_cap"]) output = grid.copy() output["eligible"] = policy_eligibility_mask( output, - markov_threshold_cap=cap, + endpoint_budget_cap=cap, budget=float(summary["design"]["budget"]), min_budget_utilization=float(summary["design"]["selection_min_budget_utilization"]), ) output["selected"] = output["candidate_id"].astype(str).eq(selected_id) + audit = audit_grid.copy() + audit["audit_eligible"] = policy_eligibility_mask( + audit, + endpoint_budget_cap=cap, + budget=float(summary["design"]["budget"]), + min_budget_utilization=float(summary["design"]["selection_min_budget_utilization"]), + ) + audit["audit_selected"] = audit["candidate_id"].astype(str).eq(audit_selected_id) + audit = audit.rename( + columns={ + "expected_objective": "audit_expected_objective", + "n_funded": "audit_n_funded", + "endpoint_budget": "audit_endpoint_budget", + } + ) + output = output.merge( + audit[ + [ + "candidate_id", + "audit_selected", + "audit_eligible", + "audit_expected_objective", + "audit_n_funded", + "audit_endpoint_budget", + ] + ], + on="candidate_id", + how="left", + validate="one_to_one", + ) return output[ [ "selected", "eligible", + "audit_selected", + "audit_eligible", "candidate_id", "risk_tolerance", "gamma", @@ -83,7 +120,9 @@ def build_selector_table(grid: pd.DataFrame, summary: dict[str, Any]) -> pd.Data "weighted_pd_point", "weighted_pd_effective", "endpoint_budget", - "markov_loss_threshold", + "audit_expected_objective", + "audit_n_funded", + "audit_endpoint_budget", ] ].sort_values( ["selected", "eligible", "expected_objective"], @@ -159,6 +198,34 @@ def _bootstrap_snapshot( } +def _bootstrap_summary_rows( + draw_frame: pd.DataFrame, + observed: dict[str, float], + *, + bootstrap_unit: str, + n_units: int, + n_draws: int, + seed: int, + note: str, +) -> list[dict[str, Any]]: + return [ + { + "bootstrap_unit": bootstrap_unit, + "metric": metric, + "observed": observed[metric], + "boot_mean": float(draw_frame[metric].mean()), + "boot_p025": float(draw_frame[metric].quantile(0.025)), + "boot_p50": float(draw_frame[metric].quantile(0.50)), + "boot_p975": float(draw_frame[metric].quantile(0.975)), + "n_units": n_units, + "n_draws": n_draws, + "seed": seed, + "note": note, + } + for metric in draw_frame.columns + ] + + def build_bootstrap_table( allocations: pd.DataFrame, evaluation: pd.DataFrame, @@ -196,10 +263,10 @@ def build_bootstrap_table( "Funded allocations do not reconcile to the full-OOT evaluation: " + ", ".join(mismatches) ) - rng = np.random.default_rng(seed) - draws = [ + loan_rng = np.random.default_rng(seed) + loan_draws = [ _bootstrap_snapshot( - selected.iloc[rng.integers(0, len(selected), size=len(selected))].reset_index( + selected.iloc[loan_rng.integers(0, len(selected), size=len(selected))].reset_index( drop=True ), total_exposure=total_exposure, @@ -207,24 +274,53 @@ def build_bootstrap_table( ) for _ in range(n_draws) ] - draw_frame = pd.DataFrame(draws) - note = "Fixed funded-loan contribution bootstrap; model, intervals, selector, and solver are not resampled." - return pd.DataFrame( - [ - { - "metric": metric, - "observed": observed[metric], - "boot_mean": float(draw_frame[metric].mean()), - "boot_p025": float(draw_frame[metric].quantile(0.025)), - "boot_p50": float(draw_frame[metric].quantile(0.50)), - "boot_p975": float(draw_frame[metric].quantile(0.975)), - "n_draws": n_draws, - "seed": seed, - "note": note, - } - for metric in draw_frame.columns - ] + issue_month = pd.to_datetime(selected["issue_d"], errors="raise").dt.to_period("M") + month_groups = [ + selected.loc[issue_month.eq(month)].reset_index(drop=True) + for month in sorted(issue_month.unique()) + ] + month_rng = np.random.default_rng(seed) + month_draws = [ + _bootstrap_snapshot( + pd.concat( + [ + month_groups[index] + for index in month_rng.integers(0, len(month_groups), len(month_groups)) + ], + ignore_index=True, + ), + total_exposure=total_exposure, + lgd=lgd, + ) + for _ in range(n_draws) + ] + rows = _bootstrap_summary_rows( + pd.DataFrame(month_draws), + observed, + bootstrap_unit="origination_month", + n_units=len(month_groups), + n_draws=n_draws, + seed=seed, + note=( + "Fixed-allocation origination-month cluster bootstrap, renormalized to the " + "$1M budget; model, intervals, selector, and solver are not resampled." + ), ) + rows.extend( + _bootstrap_summary_rows( + pd.DataFrame(loan_draws), + observed, + bootstrap_unit="funded_loan", + n_units=len(selected), + n_draws=n_draws, + seed=seed, + note=( + "Fixed funded-loan contribution bootstrap; model, intervals, selector, " + "and solver are not resampled." + ), + ) + ) + return pd.DataFrame(rows) def build_baseline_table(evaluation: pd.DataFrame) -> pd.DataFrame: @@ -260,6 +356,35 @@ def build_baseline_table(evaluation: pd.DataFrame) -> pd.DataFrame: ] +def _compact_calibration_audit(summary: dict[str, Any]) -> dict[str, Any]: + audit = summary["calibration_audit"] + rows = {str(row["role"]): row for row in audit["policy_evaluations"]} + + def _policy(role: str) -> dict[str, Any]: + row = rows[role] + return { + "n_funded": int(row["n_funded"]), + "realized_return": float(row["realized_return"]), + "weighted_default_rate": float(row["weighted_outcome"]), + "weighted_miscoverage": float(row["weighted_miscoverage"]), + "endpoint_budget": float(row["endpoint_budget"]), + "observed_accounting_bound": float( + row["endpoint_budget"] + row["weighted_miscoverage"] + ), + } + + return { + "period": audit["period"], + "n_candidates": int(summary["calibration_metadata"]["audit_rows"]), + "outcome_free_selected_candidate_id": audit["outcome_free_selected_candidate_id"], + "same_policy_selected": bool(audit["same_policy_selected"]), + "selected_policy": _policy("calibration_selected"), + "more_conservative_policy": _policy("incumbent_linear"), + "matched_point_pd": _policy("point_pd_matched_tau"), + "claim_boundary": audit["claim_boundary"], + } + + def build_governance( summary: dict[str, Any], exact_summary: dict[str, Any], @@ -273,9 +398,12 @@ def build_governance( point = evaluation.loc[ evaluation["period"].eq("full_oot") & evaluation["role"].eq("point_pd_matched_tau") ].iloc[0] - return_boot = bootstrap.loc[bootstrap["metric"].eq("realized_return")].iloc[0] + return_boot = bootstrap.loc[ + bootstrap["metric"].eq("realized_return") + & bootstrap["bootstrap_unit"].eq("origination_month") + ].iloc[0] return { - "schema_version": "2026-07-09.6", + "schema_version": "2026-07-09.7", "generated_at_utc": summary["generated_at_utc"], "run_tag": RUN_TAG, "status": "active_ijds_policy", @@ -283,6 +411,9 @@ def build_governance( **summary["selection_audit"], "calibration_metadata": summary["calibration_metadata"], "selector_forbidden_columns_present": summary["selector_forbidden_columns_present"], + "selector_input_columns": summary["selector_input_columns"], + "endpoint_cap_stability": summary["endpoint_cap_stability"], + "calibration_audit": _compact_calibration_audit(summary), }, "selected_policy": summary["selected_policy"], "full_oot": { @@ -323,8 +454,10 @@ def build_governance( ), }, "bootstrap_return_interval": { + "bootstrap_unit": "origination_month", "p025": float(return_boot["boot_p025"]), "p975": float(return_boot["boot_p975"]), + "n_units": int(return_boot["n_units"]), "n_draws": int(return_boot["n_draws"]), }, "exact_alpha_reference_replay": exact_summary["reference_replay"], @@ -349,13 +482,14 @@ def run(*, bootstrap_draws: int, bootstrap_seed: int) -> dict[str, Any]: summary = load_json(MODEL_DIR / "calibration_selected_policy_summary.json") exact_summary = load_json(EXACT_MODEL_DIR / "exact_alpha_grid_summary.json") grid = pd.read_parquet(DATA_DIR / "calibration_policy_selection_grid.parquet") + audit_grid = pd.read_parquet(DATA_DIR / "calibration_policy_audit_grid.parquet") evaluation = pd.read_csv(DATA_DIR / "calibration_selected_policy_oot_evaluation.csv") allocations = pd.read_parquet( DATA_DIR / "calibration_selected_policy_full_oot_allocations.parquet" ) tables = { "alpha": build_alpha_table(exact_summary), - "selector": build_selector_table(grid, summary), + "selector": build_selector_table(grid, audit_grid, summary), "temporal": build_temporal_table(evaluation), "grade": build_grade_table(allocations), "bootstrap": build_bootstrap_table( diff --git a/scripts/check_publication_integrity.py b/scripts/check_publication_integrity.py index 2c64f17..3579b06 100644 --- a/scripts/check_publication_integrity.py +++ b/scripts/check_publication_integrity.py @@ -53,7 +53,7 @@ class SurfaceCheck: *COMMON_CLAIM_TOKENS, "claim ijds activo", "q=(p+u)/2", - "grilla redonda 3x3", + "cap determinista", ), forbidden=("## champion congelado",), ), diff --git a/scripts/compile_ijds_submission.py b/scripts/compile_ijds_submission.py index 24589d0..9cd7eba 100644 --- a/scripts/compile_ijds_submission.py +++ b/scripts/compile_ijds_submission.py @@ -60,6 +60,28 @@ def _submission_env() -> dict[str, str]: return env +def _windows_latexmk_script(latexmk_executable: str | Path) -> Path | None: + """Locate ``latexmk.pl`` beside a TeX Live/TinyTeX Windows wrapper.""" + executable = Path(latexmk_executable).resolve() + for root in executable.parents: + script = root / "texmf-dist" / "scripts" / "latexmk" / "latexmk.pl" + if script.is_file(): + return script + return None + + +def _latexmk_command() -> list[str] | None: + """Return a working latexmk launcher, bypassing fragile TinyTeX wrappers.""" + executable = shutil.which("latexmk") + if executable is None: + return None + if os.name == "nt" and (perl := shutil.which("perl")) is not None: + script = _windows_latexmk_script(executable) + if script is not None: + return [perl, str(script)] + return [executable] + + def _manual_pdflatex_bibtex(cwd: Path, env: dict[str, str], transcript: Path) -> int: sequence = [ ["pdflatex", "-interaction=nonstopmode", "-halt-on-error", TEX_NAME], @@ -84,13 +106,14 @@ def compile_submission(*, prefer_manual: bool = False) -> int: transcript = REPORT_DIR / "ijds-latex-build.txt" if transcript.exists(): transcript.unlink() - if prefer_manual or shutil.which("latexmk") is None: - if shutil.which("latexmk") is None: + latexmk_command = _latexmk_command() + if prefer_manual or latexmk_command is None: + if latexmk_command is None: logger.warning("latexmk unavailable; using manual pdflatex/BibTeX fallback.") return _manual_pdflatex_bibtex(SUBMISSION_DIR, env, transcript) latexmk_code = _run( - ["latexmk", "-pdf", "-gg", "-interaction=nonstopmode", TEX_NAME], + [*latexmk_command, "-pdf", "-gg", "-interaction=nonstopmode", TEX_NAME], cwd=SUBMISSION_DIR, env=env, transcript=transcript, diff --git a/scripts/experiments/ijds_policy_support.py b/scripts/experiments/ijds_policy_support.py index 627b7a4..10ab494 100644 --- a/scripts/experiments/ijds_policy_support.py +++ b/scripts/experiments/ijds_policy_support.py @@ -12,7 +12,6 @@ ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT)) -from scripts.validate_alpha_gamma_bound import _load_aligned_dataset # noqa: E402 from src.models.conformal_alpha_grid import alpha_interval_columns # noqa: E402 from src.optimization.certificate_semantics import ( # noqa: E402 compute_funded_certificate_metrics, @@ -30,26 +29,26 @@ def load_policy_panel(config: dict[str, Any], *, root: Path = ROOT) -> pd.DataFrame: - """Load aligned loans and the exact interval columns declared by config.""" + """Load candidates and exact-alpha intervals under a strict ID contract.""" source = config["source"] design = config["design"] execution = config["execution"] - interval_path = resolve_repo_artifact_path(source["conformal_intervals_path"], root=root) - aligned = _load_aligned_dataset( - conformal_intervals_path=str(interval_path), - max_candidates=0, - random_state=int(execution.get("random_seed", 42)), - ) + candidate_value = str(source.get("candidate_path", "data/processed/test_fe.parquet")) + candidate_path = resolve_repo_artifact_path(candidate_value, root=root) exact_grid_value = str(source.get("exact_alpha_grid_path", "")).strip() if not exact_grid_value: raise ValueError("Active IJDS policy evaluation requires exact_alpha_grid_path.") exact_grid_path = resolve_repo_artifact_path(exact_grid_value, root=root) exact_alignment = align_candidate_intervals( - aligned, + pd.read_parquet(candidate_path), pd.read_parquet(exact_grid_path), max_candidates=0, random_state=int(execution.get("random_seed", 42)), ) + if exact_alignment.mode != "id": + raise RuntimeError( + "Active IJDS policy evaluation requires one-to-one candidate/exact-grid ID alignment." + ) panel = exact_alignment.candidates.copy() exact = exact_alignment.intervals low_column, high_column = alpha_interval_columns(float(design["alpha"])) @@ -61,14 +60,27 @@ def load_policy_panel(config: dict[str, Any], *, root: Path = ROOT) -> pd.DataFr panel["_pd_point"] = exact["y_pred"].to_numpy(dtype=float) panel["_pd_low"] = exact[low_column].to_numpy(dtype=float) panel["_pd_high"] = exact[high_column].to_numpy(dtype=float) - panel["_outcome"] = pd.to_numeric(panel["y_true"], errors="raise").astype(float) + exact_outcome = pd.to_numeric(exact["y_true"], errors="raise").to_numpy(dtype=float) + if "default_flag" in panel.columns: + candidate_outcome = pd.to_numeric(panel["default_flag"], errors="raise").to_numpy( + dtype=float + ) + if not np.array_equal(candidate_outcome, exact_outcome): + raise ValueError("Candidate default_flag does not match exact-grid y_true by ID.") + panel["_outcome"] = exact_outcome panel["_loan_amount"] = pd.to_numeric(panel["loan_amnt"], errors="coerce").fillna(1.0) panel["_int_rate"] = parse_percent_series(panel["int_rate"]) panel["_period"] = temporal_period_labels( panel["issue_d"], combine_years_from=int(design["combine_years_from"]), ) - panel.attrs["exact_alpha_grid_path"] = str(exact_grid_path) + panel.attrs.update( + { + "candidate_path": str(candidate_path), + "exact_alpha_grid_path": str(exact_grid_path), + "alignment_mode": exact_alignment.mode, + } + ) return panel diff --git a/scripts/experiments/run_ijds_calibration_selected_policy_challenger.py b/scripts/experiments/run_ijds_calibration_selected_policy_challenger.py index 5d8c845..21f86e7 100644 --- a/scripts/experiments/run_ijds_calibration_selected_policy_challenger.py +++ b/scripts/experiments/run_ijds_calibration_selected_policy_challenger.py @@ -40,6 +40,7 @@ FORBIDDEN_POLICY_SELECTION_COLUMNS, LinearPolicyCandidate, build_linear_policy_grid, + endpoint_cap_stability, select_policy_result_ex_ante, ) from src.utils.script_helpers import ( # noqa: E402 @@ -49,9 +50,23 @@ ) DEFAULT_CONFIG = ( - ROOT / "configs/experiments/champion_reopen_ijds_calibration_selected_simple90_v6.yaml" + ROOT / "configs/experiments/champion_reopen_ijds_calibration_selected_endpoint28_v7.yaml" ) FORBIDDEN_SELECTOR_COLUMNS = FORBIDDEN_POLICY_SELECTION_COLUMNS +SELECTOR_INPUT_COLUMNS = ( + "id", + "loan_amnt", + "purpose", + "grade", + "term", + "verification_status", + "issue_d", + "_pd_point", + "_pd_low", + "_pd_high", + "_loan_amount", + "_int_rate", +) def _load_config(path: Path) -> dict[str, Any]: @@ -77,9 +92,27 @@ def _experiment_paths(run_tag: str) -> tuple[Path, Path]: return data_dir, model_dir +def _endpoint_budget_cap(config: dict[str, Any]) -> float: + design = config["design"] + if "endpoint_budget_cap" in design: + return float(design["endpoint_budget_cap"]) + if "markov_threshold_cap" in design: + return float(design["markov_threshold_cap"]) - float(np.sqrt(design["alpha"])) + raise KeyError("Experiment design requires endpoint_budget_cap.") + + +def _selector_input_frame(frame: pd.DataFrame) -> pd.DataFrame: + """Expose only columns required to solve the outcome-free policy grid.""" + required = {"loan_amnt", "purpose", "_pd_point", "_pd_low", "_pd_high", "_int_rate"} + missing = sorted(required.difference(frame.columns)) + if missing: + raise KeyError(f"Calibration selector input is missing columns: {missing}") + return frame.loc[:, [column for column in SELECTOR_INPUT_COLUMNS if column in frame]].copy() + + def _load_calibration_selection_panel( config: dict[str, Any], -) -> tuple[pd.DataFrame, FrozenConformalRecipe, dict[str, Any]]: +) -> tuple[pd.DataFrame, pd.Series, FrozenConformalRecipe, dict[str, Any]]: source = config["source"] design = config["design"] os.environ["UPSTREAM_CANONICAL_RUN_TAG"] = str(source["upstream_canonical_run_tag"]) @@ -122,17 +155,21 @@ def _load_calibration_selection_panel( panel["_pd_high"] = intervals.high panel["_loan_amount"] = pd.to_numeric(panel["loan_amnt"], errors="coerce").fillna(1.0) panel["_int_rate"] = parse_percent_series(panel["int_rate"]) + outcomes = pd.Series(split.y_tune.to_numpy(dtype=float), name="_outcome") + months = pd.to_datetime(panel["issue_d"], errors="raise").dt.to_period("M").astype(str) + panel = _selector_input_frame(panel) + panel["_month"] = months.to_numpy(dtype=str) metadata = { "conformal_results_path": str(results_path.relative_to(ROOT)), "calibration_fit_rows": int(len(split.idx_cal_fit)), - "calibration_selection_rows": int(len(split.idx_cal_tune)), - "calibration_selection_start": str(pd.to_datetime(split.issue_tune).min().date()), - "calibration_selection_end": str(pd.to_datetime(split.issue_tune).max().date()), + "calibration_holdout_rows": int(len(split.idx_cal_tune)), + "calibration_holdout_start": str(pd.to_datetime(split.issue_tune).min().date()), + "calibration_holdout_end": str(pd.to_datetime(split.issue_tune).max().date()), "target_alpha": intervals.target_alpha, "used_alpha": intervals.used_alpha, "partition": recipe.partition, } - return panel, recipe, metadata + return panel, outcomes, recipe, metadata def _measure_ex_ante_solution( @@ -178,13 +215,14 @@ def _run_calibration_grid( config: dict[str, Any], output_path: Path, ) -> pd.DataFrame: + selector_panel = _selector_input_frame(panel) rows: list[dict[str, Any]] = [] for candidate in candidates: logger.info("Calibration selector evaluating {}", candidate.candidate_id) - result = solve_candidate(panel, candidate, config=config) + result = solve_candidate(selector_panel, candidate, config=config) rows.append( _measure_ex_ante_solution( - panel, + selector_panel, candidate, result, alpha=float(config["design"]["alpha"]), @@ -213,6 +251,24 @@ def _match_candidate( return matches[0] +def _comparison_policies( + selected: LinearPolicyCandidate, + incumbent: LinearPolicyCandidate, +) -> tuple[tuple[str, LinearPolicyCandidate, bool], ...]: + point = LinearPolicyCandidate( + candidate_id="point-pd", + risk_tolerance=selected.risk_tolerance, + gamma=0.0, + uncertainty_aversion=0.0, + policy_mode="point_estimate", + ) + return ( + ("calibration_selected", selected, True), + ("incumbent_linear", incumbent, True), + ("point_pd_matched_tau", point, False), + ) + + def _evaluate_fixed_policies( panel: pd.DataFrame, selected: LinearPolicyCandidate, @@ -229,18 +285,7 @@ def _evaluate_fixed_policies( if period == "full_oot" else panel.loc[panel["_period"].astype(str).eq(period)].reset_index(drop=True) ) - point = LinearPolicyCandidate( - candidate_id="point-pd", - risk_tolerance=selected.risk_tolerance, - gamma=0.0, - uncertainty_aversion=0.0, - policy_mode="point_estimate", - ) - for role, candidate, robust in ( - ("calibration_selected", selected, True), - ("incumbent_linear", incumbent, True), - ("point_pd_matched_tau", point, False), - ): + for role, candidate, robust in _comparison_policies(selected, incumbent): record, result = evaluate_candidate( frame, candidate, @@ -261,6 +306,28 @@ def _evaluate_fixed_policies( return pd.DataFrame(rows), pd.concat(allocation_frames, ignore_index=True) +def _evaluate_calibration_audit( + panel: pd.DataFrame, + selected: LinearPolicyCandidate, + incumbent: LinearPolicyCandidate, + *, + config: dict[str, Any], + period: str, +) -> pd.DataFrame: + rows: list[dict[str, Any]] = [] + frame = panel.reset_index(drop=True) + for role, candidate, robust in _comparison_policies(selected, incumbent): + record, _result = evaluate_candidate( + frame, + candidate, + config=config, + robust=robust, + period=period, + ) + rows.append({"role": role, **record}) + return pd.DataFrame(rows) + + def _funded_allocation_frame( frame: pd.DataFrame, result: PolicyAllocationResult, @@ -357,7 +424,38 @@ def run(config_path: Path) -> dict[str, Any]: config = _load_config(config_path) run_tag = str(config["run_tag"]) data_dir, model_dir = _experiment_paths(run_tag) - calibration_panel, recipe, calibration_metadata = _load_calibration_selection_panel(config) + calibration_holdout, calibration_outcomes, recipe, calibration_metadata = ( + _load_calibration_selection_panel(config) + ) + design = config["design"] + selection_period = str(design.get("selection_period", "")).strip() + audit_period = str(design.get("audit_period", "")).strip() + selection_mask = ( + calibration_holdout["_month"].eq(selection_period) + if selection_period + else pd.Series(True, index=calibration_holdout.index) + ) + selection_panel = calibration_holdout.loc[selection_mask].reset_index(drop=True).copy() + if selection_panel.empty: + raise ValueError(f"Calibration selection period has no rows: {selection_period!r}") + audit_mask = ( + calibration_holdout["_month"].eq(audit_period) + if audit_period + else pd.Series(False, index=calibration_holdout.index) + ) + audit_panel = calibration_holdout.loc[audit_mask].reset_index(drop=True).copy() + if audit_period and audit_panel.empty: + raise ValueError(f"Calibration audit period has no rows: {audit_period!r}") + audit_outcomes = calibration_outcomes.loc[audit_mask.to_numpy()].reset_index(drop=True) + calibration_metadata.update( + { + "selection_period": selection_period or "full_calibration_holdout", + "selection_rows": int(len(selection_panel)), + "audit_period": audit_period or None, + "audit_rows": int(len(audit_panel)), + "outcomes_isolated_until_post_selection_audit": True, + } + ) grid_config = config["policy_grid"] candidates = build_linear_policy_grid( risk_tolerances=[float(value) for value in grid_config["risk_tolerances"]], @@ -365,20 +463,59 @@ def run(config_path: Path) -> dict[str, Any]: uncertainty_aversions=[float(value) for value in grid_config["uncertainty_aversions"]], ) selection_results = _run_calibration_grid( - calibration_panel, + selection_panel, candidates, config=config, output_path=data_dir / "calibration_policy_selection_grid.parquet", ) + endpoint_cap = _endpoint_budget_cap(config) selected_row, selection_audit = select_policy_result_ex_ante( selection_results, - markov_threshold_cap=float(config["design"]["markov_threshold_cap"]), - budget=float(config["design"]["budget"]), - min_budget_utilization=float(config["design"]["selection_min_budget_utilization"]), + endpoint_budget_cap=endpoint_cap, + budget=float(design["budget"]), + min_budget_utilization=float(design["selection_min_budget_utilization"]), ) candidate_lookup = {candidate.candidate_id: candidate for candidate in candidates} selected = candidate_lookup[str(selected_row["candidate_id"])] incumbent = _match_candidate(candidates, config["incumbent_policy"]) + cap_stability = endpoint_cap_stability( + selection_results, + selected_candidate_id=selected.candidate_id, + endpoint_budget_cap=endpoint_cap, + budget=float(design["budget"]), + min_budget_utilization=float(design["selection_min_budget_utilization"]), + ) + + audit_grid_path: Path | None = None + audit_evaluation_path: Path | None = None + audit_selected_id: str | None = None + audit_evaluation = pd.DataFrame() + if not audit_panel.empty: + audit_grid_path = data_dir / "calibration_policy_audit_grid.parquet" + audit_results = _run_calibration_grid( + audit_panel, + candidates, + config=config, + output_path=audit_grid_path, + ) + audit_selected_row, _audit_selection = select_policy_result_ex_ante( + audit_results, + endpoint_budget_cap=endpoint_cap, + budget=float(design["budget"]), + min_budget_utilization=float(design["selection_min_budget_utilization"]), + ) + audit_selected_id = str(audit_selected_row["candidate_id"]) + audited_decisions = audit_panel.assign(_outcome=audit_outcomes.to_numpy(dtype=float)) + audit_evaluation = _evaluate_calibration_audit( + audited_decisions, + selected, + incumbent, + config=config, + period=audit_period, + ) + audit_evaluation_path = data_dir / "calibration_policy_holdout_audit.csv" + audit_evaluation.to_csv(audit_evaluation_path, index=False) + oot_panel = load_policy_panel(config) evaluation, allocations = _evaluate_fixed_policies( oot_panel, @@ -406,16 +543,36 @@ def run(config_path: Path) -> dict[str, Any]: "reference_used_alpha": recipe.reference_used_alpha, }, "grid_size": int(len(candidates)), + "selector_input_columns": list(_selector_input_frame(selection_panel).columns), "selector_columns": list(selection_results.columns), "selector_forbidden_columns_present": sorted( FORBIDDEN_SELECTOR_COLUMNS.intersection(selection_results.columns) ), "selection_audit": selection_audit, + "endpoint_cap_stability": cap_stability, + "calibration_audit": { + "period": audit_period or None, + "outcome_free_selected_candidate_id": audit_selected_id, + "same_policy_selected": bool(audit_selected_id == selected.candidate_id), + "grid_path": ( + None if audit_grid_path is None else str(audit_grid_path.relative_to(ROOT)) + ), + "evaluation_path": ( + None + if audit_evaluation_path is None + else str(audit_evaluation_path.relative_to(ROOT)) + ), + "policy_evaluations": audit_evaluation.to_dict(orient="records"), + "claim_boundary": ( + "Independent post-selection decision audit; not a selected-set coverage theorem." + ), + }, "selected_policy": selected.to_record(), "selected_calibration_metrics": selected_row.to_dict(), "incumbent_policy": incumbent.to_record(), "evaluation_path": str(evaluation_path.relative_to(ROOT)), "allocation_path": str(allocation_path.relative_to(ROOT)), + "oot_alignment": dict(oot_panel.attrs), "contrasts": _contrast_payload(evaluation), "claim_boundary": str(config["claim_boundary"]), } diff --git a/src/optimization/policy_selection.py b/src/optimization/policy_selection.py index 1c087c2..a8b3d7f 100644 --- a/src/optimization/policy_selection.py +++ b/src/optimization/policy_selection.py @@ -26,7 +26,7 @@ "candidate_id", "solver_status", "expected_objective", - "markov_loss_threshold", + "endpoint_budget", "weighted_pd_effective", "risk_tolerance", "total_allocated", @@ -94,11 +94,11 @@ def temporal_period_labels( def policy_eligibility_mask( results: pd.DataFrame, *, - markov_threshold_cap: float, + endpoint_budget_cap: float, budget: float, min_budget_utilization: float = 0.999, ) -> pd.Series: - """Return the canonical ex-ante feasibility screen for policy rows.""" + """Return the deterministic ex-ante feasibility screen for policy rows.""" missing = sorted(REQUIRED_POLICY_SELECTION_COLUMNS.difference(results.columns)) if missing: raise ValueError(f"Policy results are missing required columns: {missing}") @@ -111,8 +111,8 @@ def policy_eligibility_mask( budget_ok = pd.to_numeric(results["total_allocated"], errors="raise") >= ( float(budget) * float(min_budget_utilization) ) - threshold_ok = pd.to_numeric(results["markov_loss_threshold"], errors="raise") <= ( - float(markov_threshold_cap) + 1e-12 + endpoint_ok = pd.to_numeric(results["endpoint_budget"], errors="raise") <= ( + float(endpoint_budget_cap) + 1e-12 ) cap_ok = pd.to_numeric(results["weighted_pd_effective"], errors="raise") <= ( pd.to_numeric(results["risk_tolerance"], errors="raise") + 1e-12 @@ -120,13 +120,74 @@ def policy_eligibility_mask( objective_ok = np.isfinite( pd.to_numeric(results["expected_objective"], errors="raise").to_numpy(dtype=float) ) - return solver_ok & budget_ok & threshold_ok & cap_ok & objective_ok + return solver_ok & budget_ok & endpoint_ok & cap_ok & objective_ok + + +def endpoint_cap_stability( + results: pd.DataFrame, + *, + selected_candidate_id: str, + endpoint_budget_cap: float, + budget: float, + min_budget_utilization: float = 0.999, +) -> dict[str, float | str | None]: + """Return the cap interval over which the selected candidate stays optimal. + + Solver, budget, effective-PD, and finite-objective screens are held fixed. + Eligibility is monotone in the endpoint cap, so a higher-ranked candidate + can displace the selected policy only when its endpoint first becomes + feasible. + """ + base_feasible = results.loc[ + policy_eligibility_mask( + results, + endpoint_budget_cap=float("inf"), + budget=budget, + min_budget_utilization=min_budget_utilization, + ) + ].copy() + selected_rows = base_feasible.loc[ + base_feasible["candidate_id"].astype(str).eq(str(selected_candidate_id)) + ] + if len(selected_rows) != 1: + raise ValueError( + "selected_candidate_id must identify exactly one base-feasible policy row." + ) + ranked = base_feasible.sort_values( + ["expected_objective", "endpoint_budget", "candidate_id"], + ascending=[False, True, True], + kind="mergesort", + ).reset_index(drop=True) + selected_position = int( + ranked.index[ranked["candidate_id"].astype(str).eq(str(selected_candidate_id))][0] + ) + selected_endpoint = float(selected_rows.iloc[0]["endpoint_budget"]) + superior = ranked.iloc[:selected_position] + upper_exclusive = ( + float(pd.to_numeric(superior["endpoint_budget"], errors="raise").min()) + if not superior.empty + else None + ) + cap = float(endpoint_budget_cap) + if cap + 1e-12 < selected_endpoint or ( + upper_exclusive is not None and cap + 1e-12 >= upper_exclusive + ): + raise ValueError("Selected candidate is not optimal at endpoint_budget_cap.") + return { + "selected_candidate_id": str(selected_candidate_id), + "selected_endpoint_budget": selected_endpoint, + "cap_lower_inclusive": selected_endpoint, + "cap_upper_exclusive": upper_exclusive, + "declared_endpoint_budget_cap": cap, + "margin_to_lower_boundary": cap - selected_endpoint, + "margin_to_upper_boundary": (None if upper_exclusive is None else upper_exclusive - cap), + } def select_policy_result_ex_ante( results: pd.DataFrame, *, - markov_threshold_cap: float, + endpoint_budget_cap: float, budget: float, min_budget_utilization: float = 0.999, ) -> tuple[pd.Series, dict[str, int | float | str]]: @@ -140,27 +201,28 @@ def select_policy_result_ex_ante( raise ValueError("Ex-ante selection results contain duplicate candidate_id values.") eligible_mask = policy_eligibility_mask( results, - markov_threshold_cap=markov_threshold_cap, + endpoint_budget_cap=endpoint_budget_cap, budget=budget, min_budget_utilization=min_budget_utilization, ) eligible = results.loc[eligible_mask].copy() if eligible.empty: raise RuntimeError( - "No policy satisfies the ex-ante endpoint, effective-PD, and budget screens." + "No policy satisfies the deterministic endpoint, effective-PD, and budget screens." ) selected = eligible.sort_values( - ["expected_objective", "markov_loss_threshold", "candidate_id"], + ["expected_objective", "endpoint_budget", "candidate_id"], ascending=[False, True, True], kind="mergesort", ).iloc[0] audit: dict[str, int | float | str] = { - "selection_rule": "max_expected_objective_under_ex_ante_screen", + "selection_rule": "max_expected_objective_under_deterministic_endpoint_screen", "n_total": int(len(results)), "n_eligible": int(len(eligible)), "selected_candidate_id": str(selected["candidate_id"]), - "markov_threshold_cap": float(markov_threshold_cap), + "endpoint_budget_cap": float(endpoint_budget_cap), "min_budget_utilization": float(min_budget_utilization), "outcome_columns_used": 0, + "statistical_assumption_columns_used": 0, } return selected, audit diff --git a/tests/test_experiments/test_ijds_calibration_selected_policy_challenger.py b/tests/test_experiments/test_ijds_calibration_selected_policy_challenger.py index 4699895..db4dd2d 100644 --- a/tests/test_experiments/test_ijds_calibration_selected_policy_challenger.py +++ b/tests/test_experiments/test_ijds_calibration_selected_policy_challenger.py @@ -7,6 +7,7 @@ FORBIDDEN_SELECTOR_COLUMNS, _funded_allocation_frame, _measure_ex_ante_solution, + _selector_input_frame, ) from src.optimization.policy import PolicyMode from src.optimization.policy_evaluation import PolicyAllocationResult @@ -48,6 +49,30 @@ def test_ex_ante_measurement_contains_no_outcome_fields() -> None: assert record["weighted_pd_effective"] == 0.275 +def test_selector_input_frame_drops_all_outcomes() -> None: + frame = pd.DataFrame( + { + "id": ["a"], + "loan_amnt": [100.0], + "purpose": ["credit_card"], + "issue_d": ["2017-11-01"], + "default_flag": [1], + "y_true": [1], + "loan_status": ["Charged Off"], + "_outcome": [1.0], + "_pd_point": [0.1], + "_pd_low": [0.0], + "_pd_high": [0.3], + "_loan_amount": [100.0], + "_int_rate": [0.2], + } + ) + + selector = _selector_input_frame(frame) + + assert not {"default_flag", "y_true", "loan_status", "_outcome"}.intersection(selector.columns) + + def test_funded_allocation_frame_reconciles_exposure_and_return() -> None: frame = pd.DataFrame( { diff --git a/tests/test_ijds_active_claim_sync.py b/tests/test_ijds_active_claim_sync.py index e07e91a..29c95c0 100644 --- a/tests/test_ijds_active_claim_sync.py +++ b/tests/test_ijds_active_claim_sync.py @@ -11,7 +11,7 @@ REPO = Path(__file__).resolve().parents[1] TABLES = REPO / "reports/crpto/tables" -RUN_TAG = "champion-reopen-2026-06-19__pool93__ijds-calibration-selected-simple90-v6" +RUN_TAG = "champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7" GOVERNANCE = ( REPO / "models/experiments/champion_reopen" / RUN_TAG / "portfolio/ijds_policy_governance.json" ) @@ -63,8 +63,26 @@ def test_active_governance_locks_simple_policy_and_selector() -> None: assert selector["n_total"] == 9 assert selector["n_eligible"] == 5 assert selector["outcome_columns_used"] == 0 + assert selector["statistical_assumption_columns_used"] == 0 + assert selector["endpoint_budget_cap"] == pytest.approx(0.28) assert selector["selector_forbidden_columns_present"] == [] assert selector["calibration_metadata"]["target_alpha"] == pytest.approx(0.10) + assert selector["calibration_metadata"]["selection_period"] == "2017-11" + assert selector["calibration_metadata"]["selection_rows"] == 14943 + assert selector["calibration_metadata"]["audit_period"] == "2017-12" + assert selector["calibration_metadata"]["audit_rows"] == 20695 + assert selector["calibration_metadata"]["outcomes_isolated_until_post_selection_audit"] is True + assert "_outcome" not in selector["selector_input_columns"] + assert selector["endpoint_cap_stability"]["cap_lower_inclusive"] == pytest.approx( + 0.25903604939435104 + ) + assert selector["endpoint_cap_stability"]["cap_upper_exclusive"] == pytest.approx( + 0.29049078888716334 + ) + assert selector["calibration_audit"]["same_policy_selected"] is True + assert selector["calibration_audit"]["selected_policy"][ + "weighted_miscoverage" + ] == pytest.approx(0.124925) assert payload["exact_alpha_reference_replay"]["pass"] is True @@ -117,6 +135,9 @@ def test_active_manuscript_surfaces_share_numeric_anchors() -> None: f"{contrast['realized_return']:,.2f}", f"{100 * contrast['selected_return_cost_pct']:.3f}", f"{100 * contrast['selected_default_reduction']:.4f}", + "0.28", + "0.124925", + "$163,421.14", ) for surface in SURFACES: text = _surface_text(surface) diff --git a/tests/test_optimization/test_policy_selection.py b/tests/test_optimization/test_policy_selection.py index f8204af..d485ff6 100644 --- a/tests/test_optimization/test_policy_selection.py +++ b/tests/test_optimization/test_policy_selection.py @@ -5,6 +5,7 @@ from src.optimization.policy_selection import ( build_linear_policy_grid, + endpoint_cap_stability, select_policy_result_ex_ante, temporal_period_labels, ) @@ -16,7 +17,7 @@ def _selection_results() -> pd.DataFrame: "candidate_id": ["linear-001", "linear-002", "linear-003"], "solver_status": ["Optimal", "Optimal", "Optimal"], "expected_objective": [120.0, 110.0, 100.0], - "markov_loss_threshold": [0.70, 0.58, 0.50], + "endpoint_budget": [0.40, 0.27, 0.20], "weighted_pd_effective": [0.17, 0.17, 0.17], "risk_tolerance": [0.17, 0.17, 0.17], "total_allocated": [1_000.0, 1_000.0, 1_000.0], @@ -40,23 +41,24 @@ def test_round_grid_is_deterministic() -> None: def test_selector_uses_expected_objective_inside_screen() -> None: selected, audit = select_policy_result_ex_ante( _selection_results(), - markov_threshold_cap=0.60, + endpoint_budget_cap=0.28, budget=1_000.0, ) assert selected["candidate_id"] == "linear-002" assert audit["n_eligible"] == 2 assert audit["outcome_columns_used"] == 0 + assert audit["statistical_assumption_columns_used"] == 0 def test_selector_does_not_accept_suboptimal_status() -> None: results = _selection_results() results.loc[0, "solver_status"] = "Suboptimal" - results.loc[0, "markov_loss_threshold"] = 0.50 + results.loc[0, "endpoint_budget"] = 0.20 selected, _ = select_policy_result_ex_ante( results, - markov_threshold_cap=0.60, + endpoint_budget_cap=0.28, budget=1_000.0, ) @@ -69,7 +71,7 @@ def test_selector_rejects_duplicate_candidates() -> None: with pytest.raises(ValueError, match="duplicate candidate_id"): select_policy_result_ex_ante( results, - markov_threshold_cap=0.60, + endpoint_budget_cap=0.28, budget=1_000.0, ) @@ -80,7 +82,7 @@ def test_selector_rejects_outcome_derived_columns() -> None: with pytest.raises(ValueError, match="outcome-derived"): select_policy_result_ex_ante( results, - markov_threshold_cap=0.60, + endpoint_budget_cap=0.28, budget=1_000.0, ) @@ -89,12 +91,33 @@ def test_selector_rejects_invalid_budget_utilization() -> None: with pytest.raises(ValueError, match="min_budget_utilization"): select_policy_result_ex_ante( _selection_results(), - markov_threshold_cap=0.60, + endpoint_budget_cap=0.28, budget=1_000.0, min_budget_utilization=1.01, ) +def test_endpoint_cap_stability_finds_next_higher_objective_boundary() -> None: + results = _selection_results() + selected, _ = select_policy_result_ex_ante( + results, + endpoint_budget_cap=0.28, + budget=1_000.0, + ) + + stability = endpoint_cap_stability( + results, + selected_candidate_id=str(selected["candidate_id"]), + endpoint_budget_cap=0.28, + budget=1_000.0, + ) + + assert stability["cap_lower_inclusive"] == pytest.approx(0.27) + assert stability["cap_upper_exclusive"] == pytest.approx(0.40) + assert stability["margin_to_lower_boundary"] == pytest.approx(0.01) + assert stability["margin_to_upper_boundary"] == pytest.approx(0.12) + + def test_temporal_labels_pool_late_years() -> None: dates = pd.Series(["2018-01-01", "2018-08-01", "2020-03-01", "2021-09-01"]) diff --git a/tests/test_scripts/test_build_ijds_calibration_selected_evidence.py b/tests/test_scripts/test_build_ijds_calibration_selected_evidence.py index 2f3b250..9443fa4 100644 --- a/tests/test_scripts/test_build_ijds_calibration_selected_evidence.py +++ b/tests/test_scripts/test_build_ijds_calibration_selected_evidence.py @@ -15,6 +15,7 @@ def _allocations() -> pd.DataFrame: return pd.DataFrame( { "role": ["calibration_selected", "calibration_selected", "point_pd_matched_tau"], + "issue_d": ["2020-01-01", "2020-02-01", "2020-01-01"], "grade": ["A", "B", "A"], "funded_exposure": [100.0, 100.0, 200.0], "funded_weight": [0.5, 0.5, 1.0], @@ -84,7 +85,10 @@ def test_bootstrap_is_deterministic_and_uses_official_observed_values() -> None: second = build_bootstrap_table(_allocations(), _evaluation(), n_draws=100, seed=7) pd.testing.assert_frame_equal(first, second) - observed = first.set_index("metric")["observed"] + assert set(first["bootstrap_unit"]) == {"origination_month", "funded_loan"} + observed = first.loc[first["bootstrap_unit"].eq("origination_month")].set_index("metric")[ + "observed" + ] assert observed["realized_return"] == -35.0 assert observed["Gamma_CP"] == 0.35 diff --git a/tests/test_scripts/test_compile_ijds_submission.py b/tests/test_scripts/test_compile_ijds_submission.py index a096ab9..13c5480 100644 --- a/tests/test_scripts/test_compile_ijds_submission.py +++ b/tests/test_scripts/test_compile_ijds_submission.py @@ -1,6 +1,6 @@ from __future__ import annotations -from scripts.compile_ijds_submission import LatexScan +from scripts.compile_ijds_submission import LatexScan, _windows_latexmk_script def test_latex_scan_ok_property_flags_clean_build() -> None: @@ -12,3 +12,14 @@ def test_latex_scan_ok_property_flags_clean_build() -> None: def test_latex_scan_ok_property_rejects_warnings_or_log_failures() -> None: assert not LatexScan(pages=27, blg_warnings=("Warning--empty journal",), log_failures=()).ok assert not LatexScan(pages=27, blg_warnings=(), log_failures=("undefined references",)).ok + + +def test_windows_latexmk_script_finds_tinytex_payload(tmp_path) -> None: + wrapper = tmp_path / "TinyTeX" / "bin" / "windows" / "latexmk.exe" + script = tmp_path / "TinyTeX" / "texmf-dist" / "scripts" / "latexmk" / "latexmk.pl" + wrapper.parent.mkdir(parents=True) + wrapper.touch() + script.parent.mkdir(parents=True) + script.touch() + + assert _windows_latexmk_script(wrapper) == script From f83a0b84c052116c108e522f85dba393802528c4 Mon Sep 17 00:00:00 2001 From: Carlos Alfredo Vergara Rojas Date: Thu, 9 Jul 2026 23:26:06 -0500 Subject: [PATCH 6/7] Record IJDS selector v7 evidence --- .../calibration_selected_policy_summary.json | 291 ++++++++++++++++++ .../portfolio/ijds_policy_governance.json | 176 +++++++++++ 2 files changed, 467 insertions(+) create mode 100644 models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7/portfolio/calibration_selected_policy_summary.json create mode 100644 models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7/portfolio/ijds_policy_governance.json diff --git a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7/portfolio/calibration_selected_policy_summary.json b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7/portfolio/calibration_selected_policy_summary.json new file mode 100644 index 0000000..d5f5bcb --- /dev/null +++ b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7/portfolio/calibration_selected_policy_summary.json @@ -0,0 +1,291 @@ +{ + "allocation_path": "data\\processed\\experiments\\champion_reopen\\champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7\\portfolio\\calibration_selected_policy_full_oot_allocations.parquet", + "calibration_audit": { + "claim_boundary": "Independent post-selection decision audit; not a selected-set coverage theorem.", + "evaluation_path": "data\\processed\\experiments\\champion_reopen\\champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7\\portfolio\\calibration_policy_holdout_audit.csv", + "grid_path": "data\\processed\\experiments\\champion_reopen\\champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7\\portfolio\\calibration_policy_audit_grid.parquet", + "outcome_free_selected_candidate_id": "linear-005", + "period": "2017-12", + "policy_evaluations": [ + { + "candidate_id": "linear-005", + "delta_cap_quantile": 1.0, + "endpoint_budget": 0.2620820323280323, + "expected_objective": 103567.0099590662, + "gamma": 0.5, + "gamma_cp": 0.1841640646560647, + "gamma_internalized": 0.09208203232803233, + "gamma_residual": 0.09208203232803232, + "markov_loss_threshold": 0.5783097983448702, + "min_budget_utilization": 0.0, + "n_funded": 193, + "n_panel": 20695, + "objective_risk_mode": "point_pd_plus_aversion", + "pd_cap_slack_penalty": 0.0, + "period": "2017-12", + "policy_mode": "blended_uncertainty", + "realized_return": 53313.04791145165, + "realized_risk_tolerance_excess": 0.0, + "risk_tolerance": 0.17, + "role": "calibration_selected", + "screen_V_leq_sqrt_alpha": true, + "screen_risk_excess_leq_alpha": true, + "solver_status": "Optimal", + "tail_focus_quantile": 1.0, + "total_allocated": 999999.9999999999, + "uncertainty_aversion": 0.0, + "weighted_miscoverage": 0.12492500000000001, + "weighted_outcome": 0.14565, + "weighted_pd_effective": 0.16999999999999998, + "weighted_pd_point": 0.07791796767196764 + }, + { + "candidate_id": "linear-006", + "delta_cap_quantile": 1.0, + "endpoint_budget": 0.20350382300175537, + "expected_objective": 97897.74942687502, + "gamma": 0.75, + "gamma_cp": 0.13401529200702159, + "gamma_internalized": 0.10051146900526618, + "gamma_residual": 0.033503823001755396, + "markov_loss_threshold": 0.5197315890185933, + "min_budget_utilization": 0.0, + "n_funded": 191, + "n_panel": 20695, + "objective_risk_mode": "point_pd_plus_aversion", + "pd_cap_slack_penalty": 0.0, + "period": "2017-12", + "policy_mode": "blended_uncertainty", + "realized_return": 38379.50087450525, + "realized_risk_tolerance_excess": 0.0, + "risk_tolerance": 0.17, + "role": "incumbent_linear", + "screen_V_leq_sqrt_alpha": true, + "screen_risk_excess_leq_alpha": true, + "solver_status": "Optimal", + "tail_focus_quantile": 1.0, + "total_allocated": 1000000.0, + "uncertainty_aversion": 0.0, + "weighted_miscoverage": 0.134525, + "weighted_outcome": 0.15525, + "weighted_pd_effective": 0.17000000000000004, + "weighted_pd_point": 0.06948853099473384 + }, + { + "candidate_id": "point-pd", + "delta_cap_quantile": 1.0, + "endpoint_budget": 0.8880705762941006, + "expected_objective": 141162.2113059011, + "gamma": 0.0, + "gamma_cp": 0.7180705762941006, + "gamma_internalized": 0.0, + "gamma_residual": 0.7180705762941006, + "markov_loss_threshold": 1.2042983423109386, + "min_budget_utilization": 0.0, + "n_funded": 169, + "n_panel": 20695, + "objective_risk_mode": "point_pd_plus_aversion", + "pd_cap_slack_penalty": 0.0, + "period": "2017-12", + "policy_mode": "point_estimate", + "realized_return": 89732.35130590109, + "realized_risk_tolerance_excess": 0.015650000000000025, + "risk_tolerance": 0.17, + "role": "point_pd_matched_tau", + "screen_V_leq_sqrt_alpha": true, + "screen_risk_excess_leq_alpha": true, + "solver_status": "Optimal", + "tail_focus_quantile": 1.0, + "total_allocated": 1000000.0, + "uncertainty_aversion": 0.0, + "weighted_miscoverage": 0.058300000000000005, + "weighted_outcome": 0.18565000000000004, + "weighted_pd_effective": 0.16999999999999996, + "weighted_pd_point": 0.16999999999999996 + } + ], + "same_policy_selected": true + }, + "calibration_metadata": { + "audit_period": "2017-12", + "audit_rows": 20695, + "calibration_fit_rows": 142550, + "calibration_holdout_end": "2017-12-01", + "calibration_holdout_rows": 35638, + "calibration_holdout_start": "2017-11-01", + "conformal_results_path": "models\\conformal_gap\\champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1\\conformal_results_mondrian.pkl", + "outcomes_isolated_until_post_selection_audit": true, + "partition": "score_decile_mondrian", + "selection_period": "2017-11", + "selection_rows": 14943, + "target_alpha": 0.1, + "used_alpha": 0.095 + }, + "claim_boundary": "The final tagged rule selects among nine round-number policies on November 2017 without exposing the selector to outcomes or assumption-conditional statistics. December 2017 independently replays the outcome-free selector and audits the already-fixed decision. The audit is diagnostic rather than a selected-set coverage theorem. Earlier development inspected the static OOT corpus, so the January 2018--September 2020 evaluation remains a transparent retrospective lockbox replay, not a pristine prospective trial, causal estimate, or live-deployment guarantee.", + "config_path": "configs\\experiments\\champion_reopen_ijds_calibration_selected_endpoint28_v7.yaml", + "config_sha256": "9df5445defe22044b235dcc0f21bc8f5b734d35c73f916a09e1fe60060f84cc0", + "contrasts": { + "2020+": { + "default_delta_vs_incumbent": -0.010000000000000009, + "default_delta_vs_point": 0.06687499999999999, + "return_cost_vs_point": 118939.59578289572, + "return_delta_vs_incumbent": 16526.81992213076, + "selected_markov_threshold": 0.580689837874186, + "selected_realized_return": 99689.53961659266, + "selected_weighted_outcome": 0.08377499999999999, + "threshold_delta_vs_point": -0.5969627812854879 + }, + "full_oot": { + "default_delta_vs_incumbent": 0.003500000000000003, + "default_delta_vs_point": -0.07902500000000001, + "return_cost_vs_point": 17041.554867740255, + "return_delta_vs_incumbent": 6388.080277992645, + "selected_markov_threshold": 0.5742788554403055, + "selected_realized_return": 179327.5851322598, + "selected_weighted_outcome": 0.03937500000000001, + "threshold_delta_vs_point": -0.663266349105067 + } + }, + "design": { + "alpha": 0.1, + "audit_period": "2017-12", + "budget": 1000000.0, + "combine_years_from": 2020, + "endpoint_budget_cap": 0.28, + "lgd": 0.45, + "max_concentration": 0.25, + "period_order": [ + "2018H1", + "2018H2", + "2019H1", + "2019H2", + "2020+" + ], + "selection_min_budget_utilization": 0.999, + "selection_period": "2017-11", + "selection_rule": "maximize expected point-PD objective on November 2017 under a deterministic endpoint-budget cap of 0.28, the effective-PD cap, and full budget use; reserve December 2017 for an outcome-free selector stability replay and a post-selection decision audit" + }, + "endpoint_cap_stability": { + "cap_lower_inclusive": 0.25903604939435104, + "cap_upper_exclusive": 0.29049078888716334, + "declared_endpoint_budget_cap": 0.28, + "margin_to_lower_boundary": 0.020963950605648984, + "margin_to_upper_boundary": 0.010490788887163316, + "selected_candidate_id": "linear-005", + "selected_endpoint_budget": 0.25903604939435104 + }, + "evaluation_path": "data\\processed\\experiments\\champion_reopen\\champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7\\portfolio\\calibration_selected_policy_oot_evaluation.csv", + "generated_at_utc": "2026-07-10T04:24:39.860761+00:00", + "grid_size": 9, + "incumbent_policy": { + "candidate_id": "linear-006", + "delta_cap_quantile": 1.0, + "gamma": 0.75, + "min_budget_utilization": 0.0, + "pd_cap_slack_penalty": 0.0, + "policy_mode": "blended_uncertainty", + "risk_tolerance": 0.17, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.0 + }, + "oot_alignment": { + "alignment_mode": "id", + "candidate_path": "C:\\Users\\carlos\\Documents\\Paper_CRPTO\\data\\processed\\test_fe.parquet", + "exact_alpha_grid_path": "C:\\Users\\carlos\\Documents\\Paper_CRPTO\\data\\processed\\experiments\\champion_reopen\\champion-reopen-2026-06-19__pool93__ijds-exact-alpha-grid-v1\\conformal\\exact_alpha_grid.parquet" + }, + "recipe": { + "partition": "score_decile_mondrian", + "partition_probability_source": "calibrated", + "reference_target_alpha": 0.1, + "reference_used_alpha": 0.095 + }, + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7", + "schema_version": "2026-07-09.7", + "selected_calibration_metrics": { + "candidate_id": "linear-005", + "delta_cap_quantile": 1.0, + "effective_pd_cap_slack": 0.0, + "endpoint_budget": 0.25903604939435104, + "expected_objective": 99387.12330098984, + "gamma": 0.5, + "gamma_cp": 0.17807209878870203, + "gamma_internalized": 0.08903604939435102, + "gamma_residual": 0.089036049394351, + "markov_loss_threshold": 0.575263815411189, + "min_budget_utilization": 0.0, + "n_funded": 162, + "n_panel": 14943, + "objective_risk_mode": "point_pd_plus_aversion", + "pd_cap_slack_penalty": 0.0, + "policy_mode": "blended_uncertainty", + "risk_tolerance": 0.17, + "solver_status": "Optimal", + "tail_focus_quantile": 1.0, + "total_allocated": 1000000.0, + "uncertainty_aversion": 0.0, + "weighted_pd_effective": 0.17, + "weighted_pd_point": 0.08096395060564901 + }, + "selected_policy": { + "candidate_id": "linear-005", + "delta_cap_quantile": 1.0, + "gamma": 0.5, + "min_budget_utilization": 0.0, + "pd_cap_slack_penalty": 0.0, + "policy_mode": "blended_uncertainty", + "risk_tolerance": 0.17, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.0 + }, + "selection_audit": { + "endpoint_budget_cap": 0.28, + "min_budget_utilization": 0.999, + "n_eligible": 5, + "n_total": 9, + "outcome_columns_used": 0, + "selected_candidate_id": "linear-005", + "selection_rule": "max_expected_objective_under_deterministic_endpoint_screen", + "statistical_assumption_columns_used": 0 + }, + "selector_columns": [ + "candidate_id", + "risk_tolerance", + "gamma", + "uncertainty_aversion", + "policy_mode", + "delta_cap_quantile", + "tail_focus_quantile", + "min_budget_utilization", + "pd_cap_slack_penalty", + "solver_status", + "objective_risk_mode", + "expected_objective", + "n_panel", + "n_funded", + "total_allocated", + "weighted_pd_point", + "weighted_pd_effective", + "gamma_cp", + "gamma_internalized", + "gamma_residual", + "endpoint_budget", + "markov_loss_threshold", + "effective_pd_cap_slack" + ], + "selector_forbidden_columns_present": [], + "selector_input_columns": [ + "id", + "loan_amnt", + "purpose", + "grade", + "term", + "verification_status", + "issue_d", + "_pd_point", + "_pd_low", + "_pd_high", + "_loan_amount", + "_int_rate" + ], + "source_commit": "e0daf55685988408bc68f5b65df0298174f1a516" +} diff --git a/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7/portfolio/ijds_policy_governance.json b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7/portfolio/ijds_policy_governance.json new file mode 100644 index 0000000..08f6f0b --- /dev/null +++ b/models/experiments/champion_reopen/champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7/portfolio/ijds_policy_governance.json @@ -0,0 +1,176 @@ +{ + "bootstrap_return_interval": { + "bootstrap_unit": "origination_month", + "n_draws": 5000, + "n_units": 31, + "p025": 163421.1394838545, + "p975": 193551.6460089551 + }, + "claim_boundary": "The final tagged rule selects among nine round-number policies on November 2017 without exposing the selector to outcomes or assumption-conditional statistics. December 2017 independently replays the outcome-free selector and audits the already-fixed decision. The audit is diagnostic rather than a selected-set coverage theorem. Earlier development inspected the static OOT corpus, so the January 2018--September 2020 evaluation remains a transparent retrospective lockbox replay, not a pristine prospective trial, causal estimate, or live-deployment guarantee.", + "exact_alpha_reference_replay": { + "high_max_abs": 6.661338147750939e-16, + "low_max_abs": 3.3306690738754696e-16, + "pass": true, + "point_max_abs": 4.440892098500626e-16, + "tolerance": 1e-12 + }, + "full_oot": { + "Gamma_CP": 0.1761021788469351, + "Gamma_internalized": 0.0880510894234675, + "Gamma_residual": 0.0880510894234675, + "endpoint_budget": 0.2580510894234676, + "expected_objective": 168271.56287282018, + "markov_loss_threshold": 0.5742788554403055, + "markov_tail_probability_bound": 0.31622776601683794, + "n_candidates": 276869, + "n_funded": 308, + "observed_accounting_bound": 0.2949260894234676, + "realized_return": 179327.5851322598, + "total_allocated": 1000000.0, + "weighted_default_rate": 0.039375, + "weighted_miscoverage": 0.036875, + "weighted_pd_effective": 0.17, + "weighted_pd_point": 0.0819489105765324 + }, + "generated_at_utc": "2026-07-10T04:24:39.860761+00:00", + "paper_tables": { + "alpha": [ + "reports/crpto/tables/crpto_tableA35_exact_alpha_grid.csv", + "reports/crpto/tables/crpto_tableA35_exact_alpha_grid.tex" + ], + "baseline": [ + "reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.csv", + "reports/crpto/tables/crpto_tableA40_calibration_selected_point_baseline.tex" + ], + "bootstrap": [ + "reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.csv", + "reports/crpto/tables/crpto_tableA39_calibration_selected_bootstrap.tex" + ], + "grade": [ + "reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.csv", + "reports/crpto/tables/crpto_tableA38_calibration_selected_grade_audit.tex" + ], + "selector": [ + "reports/crpto/tables/crpto_tableA36_calibration_policy_selector.csv", + "reports/crpto/tables/crpto_tableA36_calibration_policy_selector.tex" + ], + "temporal": [ + "reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.csv", + "reports/crpto/tables/crpto_tableA37_calibration_selected_temporal_evaluation.tex" + ] + }, + "point_pd_contrast": { + "endpoint_budget": 0.9213174385285344, + "markov_loss_threshold": 1.2375452045453723, + "realized_return": 196369.14000000004, + "selected_default_reduction": 0.07902500000000001, + "selected_return_cost": 17041.554867740255, + "selected_return_cost_pct": 0.0867832637436832, + "selected_threshold_reduction": 0.6632663491050668, + "weighted_default_rate": 0.1184, + "weighted_miscoverage": 0.0419 + }, + "retired_active_claims": [ + "alpha01 intervals obtained by cross-family average-width scaling", + "8/8 approximate alpha-grid pass as a headline certificate", + "50,010-policy frontier as the active selector", + "0.345084 Markov threshold", + "capped_blended_uncertainty with delta_cap_quantile=0.975", + "OOT-outcome-selected portfolio hyperparameters", + "the exploratory 25-policy gamma=0.35, threshold-cap=0.65 challenger" + ], + "run_tag": "champion-reopen-2026-06-19__pool93__ijds-calibration-selected-endpoint28-v7", + "schema_version": "2026-07-09.7", + "selected_policy": { + "candidate_id": "linear-005", + "delta_cap_quantile": 1.0, + "gamma": 0.5, + "min_budget_utilization": 0.0, + "pd_cap_slack_penalty": 0.0, + "policy_mode": "blended_uncertainty", + "risk_tolerance": 0.17, + "tail_focus_quantile": 1.0, + "uncertainty_aversion": 0.0 + }, + "selection_protocol": { + "calibration_audit": { + "claim_boundary": "Independent post-selection decision audit; not a selected-set coverage theorem.", + "matched_point_pd": { + "endpoint_budget": 0.8880705762941006, + "n_funded": 169, + "observed_accounting_bound": 0.9463705762941006, + "realized_return": 89732.35130590109, + "weighted_default_rate": 0.18565000000000004, + "weighted_miscoverage": 0.058300000000000005 + }, + "more_conservative_policy": { + "endpoint_budget": 0.20350382300175537, + "n_funded": 191, + "observed_accounting_bound": 0.3380288230017554, + "realized_return": 38379.50087450525, + "weighted_default_rate": 0.15525, + "weighted_miscoverage": 0.134525 + }, + "n_candidates": 20695, + "outcome_free_selected_candidate_id": "linear-005", + "period": "2017-12", + "same_policy_selected": true, + "selected_policy": { + "endpoint_budget": 0.2620820323280323, + "n_funded": 193, + "observed_accounting_bound": 0.3870070323280323, + "realized_return": 53313.04791145165, + "weighted_default_rate": 0.14565, + "weighted_miscoverage": 0.12492500000000001 + } + }, + "calibration_metadata": { + "audit_period": "2017-12", + "audit_rows": 20695, + "calibration_fit_rows": 142550, + "calibration_holdout_end": "2017-12-01", + "calibration_holdout_rows": 35638, + "calibration_holdout_start": "2017-11-01", + "conformal_results_path": "models\\conformal_gap\\champion-reopen-2026-06-19__hpo-wave1__claim-max-incremental-conformal__pool93__conformal__phase1__final__rank-1\\conformal_results_mondrian.pkl", + "outcomes_isolated_until_post_selection_audit": true, + "partition": "score_decile_mondrian", + "selection_period": "2017-11", + "selection_rows": 14943, + "target_alpha": 0.1, + "used_alpha": 0.095 + }, + "endpoint_budget_cap": 0.28, + "endpoint_cap_stability": { + "cap_lower_inclusive": 0.25903604939435104, + "cap_upper_exclusive": 0.29049078888716334, + "declared_endpoint_budget_cap": 0.28, + "margin_to_lower_boundary": 0.020963950605648984, + "margin_to_upper_boundary": 0.010490788887163316, + "selected_candidate_id": "linear-005", + "selected_endpoint_budget": 0.25903604939435104 + }, + "min_budget_utilization": 0.999, + "n_eligible": 5, + "n_total": 9, + "outcome_columns_used": 0, + "selected_candidate_id": "linear-005", + "selection_rule": "max_expected_objective_under_deterministic_endpoint_screen", + "selector_forbidden_columns_present": [], + "selector_input_columns": [ + "id", + "loan_amnt", + "purpose", + "grade", + "term", + "verification_status", + "issue_d", + "_pd_point", + "_pd_low", + "_pd_high", + "_loan_amount", + "_int_rate" + ], + "statistical_assumption_columns_used": 0 + }, + "status": "active_ijds_policy" +} From 63f9a9bf0ef0492c4f2eb2b3d7a990e52775c2d1 Mon Sep 17 00:00:00 2001 From: Carlos Alfredo Vergara Rojas Date: Thu, 9 Jul 2026 23:38:31 -0500 Subject: [PATCH 7/7] Synchronize IJDS submission metadata --- paper/submission/COVER_LETTER_AND_DISCLOSURE.md | 10 +++++----- paper/submission/README.md | 8 ++++---- paper/submission/SCHOLARONE_FINAL_CHECKLIST.md | 4 ++-- paper/submission/TITLE_PAGE_DRAFT.md | 7 +++---- 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/paper/submission/COVER_LETTER_AND_DISCLOSURE.md b/paper/submission/COVER_LETTER_AND_DISCLOSURE.md index 5945a2a..cf80bbd 100644 --- a/paper/submission/COVER_LETTER_AND_DISCLOSURE.md +++ b/paper/submission/COVER_LETTER_AND_DISCLOSURE.md @@ -7,9 +7,9 @@ packet unless ScholarOne requests the corresponding disclosure text. Dear Editors, -We submit "CRPTO: A Calibration-Selected Conformal Guardrail for Auditable -Credit Portfolio Decisions" for consideration at the *INFORMS Journal on Data -Science*. The paper treats credit allocation as data science for decisions, +We submit "CRPTO: A Calibration-Selected Conformal Guardrail for Credit +Portfolios" for consideration at the *INFORMS Journal on Data Science*. The +paper treats credit allocation as data science for decisions, not as a credit-scoring leaderboard. A frozen calibrated PD model is combined with an exactly replayed 90% Mondrian conformal endpoint. The resulting midpoint score, `q=(p+u)/2`, constrains a `$1M` portfolio while point PD remains @@ -18,8 +18,8 @@ in the expected-return objective. The final policy is selected from nine round-number candidates on November 2017 using a deterministic endpoint cap. Outcomes are stored separately from its 12-column ranking frame, which contains no assumption-conditional -statistics. An outcome-free -December replay selects the same rule; opening outcomes afterward reveals +statistics. An outcome-free December replay selects the same rule; opening +outcomes afterward reveals miscoverage `0.124925`, so the paper explicitly does not infer selected-set validity from policy stability. On 276,869 out-of-time Lending Club loans, the fixed policy earns `$179,327.59`, with weighted default `0.039375`. diff --git a/paper/submission/README.md b/paper/submission/README.md index 5f4b0b6..7a9d1e5 100644 --- a/paper/submission/README.md +++ b/paper/submission/README.md @@ -100,10 +100,10 @@ The official-template PDF is acceptable only when: - the PDF remains double-anonymous; - visual inspection confirms that figures, equations, and references render. -Current verified build (2026-07-09): 12 pages total; References begin on page -10, so the body is 9 pages under the template's pagination. The bibliography is -clean, and visual inspection of all 12 rendered pages found no clipping, -overlap, missing glyphs, or unreadable tables. +Current verified build (2026-07-09): 13 pages total; References begin on page +11, so the main text occupies pages 1--10 and remains well within the 25-page +limit. The bibliography is clean, and visual inspection of all 13 pages found +no clipping, overlap, missing glyphs, or unreadable tables. Do not keep a page-count statement in this README without rebuilding the current TeX. The final compile wrapper records the current count and warning diff --git a/paper/submission/SCHOLARONE_FINAL_CHECKLIST.md b/paper/submission/SCHOLARONE_FINAL_CHECKLIST.md index 0e1c35e..d257269 100644 --- a/paper/submission/SCHOLARONE_FINAL_CHECKLIST.md +++ b/paper/submission/SCHOLARONE_FINAL_CHECKLIST.md @@ -28,8 +28,8 @@ The wrapper uses the direct `latexmk.pl` payload on Windows and falls back to th - figures and tables fit; - PDF metadata and visible content remain anonymous. -Current local build (2026-07-09): 12 pages total, with References beginning on -page 10; citation/reference scans are clean. Recount after every substantive +Current local build (2026-07-09): 13 pages total, with References beginning on +page 11; citation/reference scans are clean. Recount after every substantive TeX edit. ## Local Gates diff --git a/paper/submission/TITLE_PAGE_DRAFT.md b/paper/submission/TITLE_PAGE_DRAFT.md index 60f530b..eacede6 100644 --- a/paper/submission/TITLE_PAGE_DRAFT.md +++ b/paper/submission/TITLE_PAGE_DRAFT.md @@ -8,13 +8,12 @@ Upload it only as the separate title page requested by ScholarOne/INFORMS. ## Manuscript Title -CRPTO: Conformal Robust Predict-Then-Optimize for Auditable Credit Portfolio -Decisions +CRPTO: A Calibration-Selected Conformal Guardrail for Credit Portfolios ## Keywords -conformal prediction; robust optimization; credit risk; predict-then-optimize; -portfolio decisions; reproducibility; model risk governance +conformal prediction; predict-then-optimize; credit risk; portfolio +optimization; calibration; reproducible data science ## Author