From 0ee3446d9d426875ce51d7b3e3d0406fbd5ae827 Mon Sep 17 00:00:00 2001 From: Mateo Bodon Date: Tue, 14 Jul 2026 19:32:22 -0400 Subject: [PATCH] fix: restore CI and safe docs deployment Tests: 128 passed, 1 deselected; coverage 77%; ruff; black; isort; detect-secrets baseline; mkdocs strict --- .github/workflows/docs.yml | 12 ++- README.md | 14 ++- benchmarks/bench_multi_stream.py | 10 ++- docs/index.md | 17 +++- pyproject.toml | 7 ++ reports/factors_ff.py | 4 +- reports/html_report.py | 73 ++++++++++++---- reports/wfv_report.py | 10 ++- scripts/__init__.py | 1 - scripts/build_flagship_universe.py | 6 +- scripts/build_runs_index.py | 10 ++- scripts/build_wrds_signals.py | 18 ++-- scripts/diagnose_artifact_integrity.py | 12 ++- scripts/export_wrds_flagship.py | 33 +++++-- scripts/validate_run_logs.py | 5 +- src/microalpha/allocators.py | 21 ++++- src/microalpha/cli.py | 6 +- src/microalpha/config.py | 6 +- src/microalpha/engine.py | 8 +- src/microalpha/execution.py | 12 +-- src/microalpha/integrity.py | 8 +- src/microalpha/manifest.py | 12 ++- src/microalpha/market_metadata.py | 4 +- src/microalpha/metrics.py | 8 +- src/microalpha/order_flow.py | 34 ++++++-- src/microalpha/portfolio.py | 45 +++++++--- src/microalpha/reporting/analytics.py | 100 +++++++++++++++++----- src/microalpha/reporting/baselines.py | 89 ++++++++++++++----- src/microalpha/reporting/factors.py | 24 ++++-- src/microalpha/reporting/robustness.py | 29 +++++-- src/microalpha/reporting/spa.py | 30 +++++-- src/microalpha/reporting/summary.py | 4 +- src/microalpha/reporting/tearsheet.py | 9 +- src/microalpha/reporting/wrds_summary.py | 96 ++++++++++++++++----- src/microalpha/runner.py | 19 ++-- src/microalpha/slippage.py | 5 +- src/microalpha/strategies/flagship_mom.py | 14 +-- src/microalpha/walkforward.py | 75 +++++++--------- src/microalpha/wrds/__init__.py | 4 +- tests/test_build_wrds_signals.py | 82 +++++++++++++++--- tests/test_degeneracy_constraints.py | 16 +++- tests/test_docs_links.py | 9 +- tests/test_flagship_filter_diagnostics.py | 5 +- tests/test_flagship_momentum.py | 18 +++- tests/test_gpt_bundle_dirty.py | 4 +- tests/test_metrics_hac.py | 12 +-- tests/test_multiasset_data_handler.py | 4 +- tests/test_order_flow_diagnostics.py | 2 +- tests/test_portfolio_risk_caps.py | 4 +- tests/test_portfolio_weight_sizing.py | 4 +- tests/test_reporting_analytics.py | 18 ++-- tests/test_reporting_robustness.py | 53 ++++++++++-- tests/test_reporting_spa.py | 20 ++++- tests/test_time_ordering.py | 3 +- tests/test_walkforward.py | 14 +-- tests/test_wrds_flagship_spec.py | 8 +- tests/test_wrds_markers.py | 4 +- tests/test_wrds_summary_render.py | 20 +++-- tools/agentic/gpt_bundle.py | 66 ++++++++++---- tools/agentic/project_state_refresh.py | 43 ++++++++-- tools/agentic/repo_snapshot.py | 22 +++-- tools/build_project_state.py | 35 +++++--- tools/gpt_bundle.py | 8 +- tools/render_project_state_docs.py | 72 +++++++++++----- 64 files changed, 1046 insertions(+), 394 deletions(-) mode change 100755 => 100644 scripts/build_flagship_universe.py mode change 100755 => 100644 scripts/export_wrds_flagship.py mode change 100755 => 100644 tools/agentic/gpt_bundle.py mode change 100755 => 100644 tools/agentic/project_state_refresh.py mode change 100755 => 100644 tools/agentic/repo_snapshot.py diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 144106be..29a4fae1 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -4,6 +4,13 @@ on: push: branches: [main] +permissions: + contents: write + +concurrency: + group: microalpha-docs + cancel-in-progress: true + jobs: deploy: runs-on: ubuntu-latest @@ -22,7 +29,7 @@ jobs: pip install mkdocs mkdocs-material - name: Build documentation - run: mkdocs build + run: mkdocs build --strict - name: Deploy documentation env: @@ -30,4 +37,5 @@ jobs: run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - mkdocs gh-deploy --force --remote-name origin --config-file mkdocs.yml + mkdocs gh-deploy --force --remote-name origin --config-file mkdocs.yml \ + --message "docs: deploy ${GITHUB_SHA}" diff --git a/README.md b/README.md index a6f2175e..2b414c5d 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,12 @@ that is actually ready to publish. > licensed-data campaign remains pre-holdout: its 2023–2025 final holdout is > sealed, and no alpha or live-performance claim is made. +| Completed evidence | Scope | Claim boundary | +| --- | --- | --- | +| Six frozen mechanisms | 2017–2022 validation | Every candidate was rejected by at least one preregistered gate | +| Immutable run manifests | Config, data identity, code state, outputs | Aggregate public receipts; licensed rows remain local | +| Final holdout | 2023–2025 | Sealed and not used in the reported economic evidence | + ## What it makes auditable | Research risk | microalpha control | @@ -69,9 +75,11 @@ execution as separate steps so their timing assumptions can be tested directly. ## Evidence, including negative results -The latest aggregate-only research ledger is intentionally more useful than a -single best backtest. Six frozen mechanisms were evaluated on a 2017–2022 -validation window while the 2023–2025 final holdout remained sealed. +The latest completed economic ledger (as of **2026-07-11**) is intentionally +more useful than a single best backtest. Six frozen mechanisms were evaluated +on a 2017–2022 validation window while the 2023–2025 final holdout remained +sealed. Newer SEC 13F pipeline work is infrastructure progress, not newer +economic evidence. ![Validation HAC Sharpe for six preregistered mechanisms; only the SEC cash-earnings candidate approaches the 0.50 promotion gate and it still fails the full gate set](docs/assets/portfolio/validation_frontier.svg) diff --git a/benchmarks/bench_multi_stream.py b/benchmarks/bench_multi_stream.py index aa7ae7bf..070cbb7f 100644 --- a/benchmarks/bench_multi_stream.py +++ b/benchmarks/bench_multi_stream.py @@ -13,7 +13,9 @@ from microalpha.events import MarketEvent -def _write_panel(csv_dir: Path, symbols: List[str], base_dates: pd.DatetimeIndex) -> None: +def _write_panel( + csv_dir: Path, symbols: List[str], base_dates: pd.DatetimeIndex +) -> None: rng = np.random.default_rng(2025) csv_dir.mkdir(parents=True, exist_ok=True) for symbol in symbols: @@ -48,7 +50,11 @@ def _baseline_stream(handler: MultiCsvDataHandler) -> Iterator[MarketEvent]: value = df.loc[ts, "close"] # type: ignore[index] except KeyError: continue - price = float(value.iloc[0]) if isinstance(value, pd.Series) else float(value) + price = ( + float(value.iloc[0]) + if isinstance(value, pd.Series) + else float(value) + ) else: idx = df.index.searchsorted(ts, side="right") - 1 if idx < 0: diff --git a/docs/index.md b/docs/index.md index da9c11ac..bd982b25 100644 --- a/docs/index.md +++ b/docs/index.md @@ -12,14 +12,20 @@ Microalpha is an event-driven research platform for reproducible quantitative st ## Quickstart -1. **Install the package** +1. **Install this repository from source** ```bash - pip install microalpha - # or, for local development + git clone https://github.com/MateoBodon/microalpha.git + cd microalpha + python -m venv .venv + source .venv/bin/activate pip install -e ".[dev]" ``` + > Do not use `pip install microalpha`: that PyPI name belongs to an + > unrelated third-party project. This repository has no public package + > release; the supported installation path is the source checkout above. + 2. **Run the bundled mean-reversion backtest** ```bash @@ -36,3 +42,8 @@ Microalpha is an event-driven research platform for reproducible quantitative st - Try the scenarios in [Examples](examples.md). Use the navigation to dive into leakage guarantees, reproducibility tooling, API surfaces, and runnable examples. + +--- + +These docs are deployed from the public `main` branch. The deployment commit is +recorded in the repository's [Docs workflow](https://github.com/MateoBodon/microalpha/actions/workflows/docs.yml). diff --git a/pyproject.toml b/pyproject.toml index a8b4a5d4..704fe01a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,10 @@ dev = [ [tool.black] line-length = 88 +[tool.isort] +profile = "black" +combine_as_imports = true + [tool.ruff] line-length = 88 exclude = [ @@ -51,6 +55,9 @@ ignore = [ "E501", ] +[tool.ruff.lint.isort] +combine-as-imports = true + [tool.mypy] python_version = "3.12" packages = ["microalpha"] diff --git a/reports/factors_ff.py b/reports/factors_ff.py index 677c5d60..3c196aec 100644 --- a/reports/factors_ff.py +++ b/reports/factors_ff.py @@ -30,7 +30,9 @@ def _format(results, meta) -> str: def main() -> None: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("artifact_dir", type=Path, help="Artifact directory with equity_curve.csv") + parser.add_argument( + "artifact_dir", type=Path, help="Artifact directory with equity_curve.csv" + ) parser.add_argument( "--factors", type=Path, diff --git a/reports/html_report.py b/reports/html_report.py index a4c8f9cc..6d048a6b 100644 --- a/reports/html_report.py +++ b/reports/html_report.py @@ -40,15 +40,32 @@ def main() -> None: args = ap.parse_args() eq = pd.read_csv(args.equity_csv) - eq_ts = pd.to_datetime(eq["timestamp"]) if "timestamp" in eq else pd.RangeIndex(len(eq)) + eq_ts = ( + pd.to_datetime(eq["timestamp"]) if "timestamp" in eq else pd.RangeIndex(len(eq)) + ) # Figure with subplots: equity + rolling Sharpe + PnL hist (if available) from plotly.subplots import make_subplots - fig = make_subplots(rows=3, cols=1, shared_xaxes=False, specs=[[{"secondary_y": True}], [{}], [{}]], - row_heights=[0.6, 0.2, 0.2], vertical_spacing=0.06) + + fig = make_subplots( + rows=3, + cols=1, + shared_xaxes=False, + specs=[[{"secondary_y": True}], [{}], [{}]], + row_heights=[0.6, 0.2, 0.2], + vertical_spacing=0.06, + ) fig.add_trace( - go.Scatter(x=eq_ts, y=eq["equity"], mode="lines", name="Equity", line=dict(color="#1f77b4")), - row=1, col=1, secondary_y=False + go.Scatter( + x=eq_ts, + y=eq["equity"], + mode="lines", + name="Equity", + line=dict(color="#1f77b4"), + ), + row=1, + col=1, + secondary_y=False, ) trades = read_trades_jsonl(args.trades) @@ -64,7 +81,10 @@ def main() -> None: mode="markers", name="Buys", marker=dict(symbol="triangle-up", color="#2ca02c"), - ), row=1, col=1, secondary_y=True + ), + row=1, + col=1, + secondary_y=True, ) fig.add_trace( go.Scatter( @@ -73,25 +93,48 @@ def main() -> None: mode="markers", name="Sells", marker=dict(symbol="triangle-down", color="#d62728"), - ), row=1, col=1, secondary_y=True + ), + row=1, + col=1, + secondary_y=True, ) # Rolling Sharpe on equity returns if len(eq) > 2: ret = pd.Series(eq["equity"]).pct_change().fillna(0.0) - window = min(63, max(2, len(ret)//5)) + window = min(63, max(2, len(ret) // 5)) rolling_mean = ret.rolling(window).mean() rolling_std = ret.rolling(window).std(ddof=0) - sharpe = (rolling_mean / (rolling_std.replace(0, pd.NA))).fillna(0.0) * (252 ** 0.5) - fig.add_trace(go.Scatter(x=eq_ts, y=sharpe, mode="lines", name="Rolling Sharpe (63d)", line=dict(color="#9467bd")), - row=2, col=1) + sharpe = (rolling_mean / (rolling_std.replace(0, pd.NA))).fillna( + 0.0 + ) * (252**0.5) + fig.add_trace( + go.Scatter( + x=eq_ts, + y=sharpe, + mode="lines", + name="Rolling Sharpe (63d)", + line=dict(color="#9467bd"), + ), + row=2, + col=1, + ) # Per-trade realized PnL histogram if "realized_pnl" in trades: - fig.add_trace(go.Histogram(x=trades["realized_pnl"], name="Trade PnL", marker_color="#8c564b"), - row=3, col=1) - - fig.update_layout(title="Microalpha Report", legend=dict(orientation="h"), template="plotly_white") + fig.add_trace( + go.Histogram( + x=trades["realized_pnl"], + name="Trade PnL", + marker_color="#8c564b", + ), + row=3, + col=1, + ) + + fig.update_layout( + title="Microalpha Report", legend=dict(orientation="h"), template="plotly_white" + ) fig.update_yaxes(title_text="Equity", row=1, col=1, secondary_y=False) fig.update_yaxes(title_text="Price", row=1, col=1, secondary_y=True) fig.update_xaxes(title_text="Time", row=1, col=1) diff --git a/reports/wfv_report.py b/reports/wfv_report.py index 2c4983a3..59f630bb 100644 --- a/reports/wfv_report.py +++ b/reports/wfv_report.py @@ -29,8 +29,12 @@ def build_wfv_report(folds_path: str) -> plt.Figure: # Bar plot of Sharpe per fold (train vs test) x = np.arange(len(folds)) width = 0.35 - axes[0].bar(x - width / 2, train_sharpes, width, label="Train", color="#1f77b4", alpha=0.7) - axes[0].bar(x + width / 2, test_sharpes, width, label="Test", color="#d62728", alpha=0.7) + axes[0].bar( + x - width / 2, train_sharpes, width, label="Train", color="#1f77b4", alpha=0.7 + ) + axes[0].bar( + x + width / 2, test_sharpes, width, label="Test", color="#d62728", alpha=0.7 + ) axes[0].set_xticks(x, labels) axes[0].set_ylabel("Sharpe") axes[0].set_title("Per-Fold Sharpe (Train vs Test)") @@ -72,5 +76,3 @@ def main() -> None: if __name__ == "__main__": main() - - diff --git a/scripts/__init__.py b/scripts/__init__.py index 73dc645e..196418fb 100644 --- a/scripts/__init__.py +++ b/scripts/__init__.py @@ -1,2 +1 @@ """Utility scripts for Microalpha.""" - diff --git a/scripts/build_flagship_universe.py b/scripts/build_flagship_universe.py old mode 100755 new mode 100644 index 7e5b243f..25f28a2c --- a/scripts/build_flagship_universe.py +++ b/scripts/build_flagship_universe.py @@ -225,9 +225,9 @@ def main() -> None: summary = { "rebalance_dates": len(universe_sizes), - "average_size": float(np.mean(list(universe_sizes.values()))) - if universe_sizes - else 0, + "average_size": ( + float(np.mean(list(universe_sizes.values()))) if universe_sizes else 0 + ), "min_size": min(universe_sizes.values()) if universe_sizes else 0, "max_size": max(universe_sizes.values()) if universe_sizes else 0, "parameters": { diff --git a/scripts/build_runs_index.py b/scripts/build_runs_index.py index 29e2d9a8..ec37fe9d 100644 --- a/scripts/build_runs_index.py +++ b/scripts/build_runs_index.py @@ -15,7 +15,7 @@ if str(SRC_ROOT) not in sys.path: sys.path.insert(0, str(SRC_ROOT)) -from microalpha.manifest import ManifestLoadError, load_manifest_path +from microalpha.manifest import ManifestLoadError, load_manifest_path # noqa: E402 DEFAULT_ARTIFACTS_ROOT = Path("artifacts") DEFAULT_OUTPUT = Path("reports/summaries/runs_index.csv") @@ -133,7 +133,9 @@ def _collect_manifests(artifacts_root: Path) -> list[Path]: for path in candidates: key = path.resolve().as_posix() unique[key] = path - return sorted(unique.values(), key=lambda p: _relative_to(p, artifacts_root).as_posix()) + return sorted( + unique.values(), key=lambda p: _relative_to(p, artifacts_root).as_posix() + ) def _extract_walkforward(payload: Mapping[str, Any]) -> Mapping[str, Any]: @@ -200,7 +202,9 @@ def build_runs_index(artifacts_root: Path, repo_root: Path) -> list[dict[str, st def write_csv(rows: list[dict[str, str]], output_path: Path) -> None: output_path.parent.mkdir(parents=True, exist_ok=True) with output_path.open("w", encoding="utf-8", newline="") as handle: - writer = csv.DictWriter(handle, fieldnames=RUNS_INDEX_COLUMNS, lineterminator="\n") + writer = csv.DictWriter( + handle, fieldnames=RUNS_INDEX_COLUMNS, lineterminator="\n" + ) writer.writeheader() writer.writerows(rows) diff --git a/scripts/build_wrds_signals.py b/scripts/build_wrds_signals.py index 4f03b193..60d19b80 100644 --- a/scripts/build_wrds_signals.py +++ b/scripts/build_wrds_signals.py @@ -29,7 +29,11 @@ def _wrds_universe_path(path: str | None) -> Path: root = os.environ.get("WRDS_DATA_ROOT") if not root: raise SystemExit("Set WRDS_DATA_ROOT or pass --universe explicitly") - candidate = Path(root).expanduser().resolve() / "universes" / "flagship_sector_neutral.csv" + candidate = ( + Path(root).expanduser().resolve() + / "universes" + / "flagship_sector_neutral.csv" + ) if not candidate.exists(): raise SystemExit(f"Universe CSV not found: {candidate}") return candidate @@ -92,16 +96,20 @@ def _build_signals( df["adv"] = float("nan") mask = df["score"].notna() & df["forward_return"].notna() - mask &= (~df["score"].isin([float("inf"), float("-inf")])) - mask &= (~df["forward_return"].isin([float("inf"), float("-inf")])) + mask &= ~df["score"].isin([float("inf"), float("-inf")]) + mask &= ~df["forward_return"].isin([float("inf"), float("-inf")]) adv_filter = df["adv"].fillna(min_adv) >= min_adv mask &= adv_filter - signals = df.loc[mask, ["date", "symbol", "score", "forward_return", "adv", "sector"]].copy() + signals = df.loc[ + mask, ["date", "symbol", "score", "forward_return", "adv", "sector"] + ].copy() signals = signals.rename(columns={"date": "as_of"}) signals["as_of"] = signals["as_of"].dt.strftime("%Y-%m-%d") if signals.empty: - raise SystemExit("No signals survived filtering; check lookback/min-adv parameters") + raise SystemExit( + "No signals survived filtering; check lookback/min-adv parameters" + ) output_path = output_path.expanduser().resolve() output_path.parent.mkdir(parents=True, exist_ok=True) diff --git a/scripts/diagnose_artifact_integrity.py b/scripts/diagnose_artifact_integrity.py index da0ed6a0..25a048c7 100644 --- a/scripts/diagnose_artifact_integrity.py +++ b/scripts/diagnose_artifact_integrity.py @@ -19,7 +19,9 @@ def _count_trades(path: Path | None) -> int: if path is None or not path.exists(): return 0 if path.suffix.lower() == ".jsonl": - return sum(1 for line in path.read_text(encoding="utf-8").splitlines() if line.strip()) + return sum( + 1 for line in path.read_text(encoding="utf-8").splitlines() if line.strip() + ) if path.suffix.lower() == ".csv": df = pd.read_csv(path) return int(len(df)) @@ -89,7 +91,13 @@ def main() -> int: print("Returns std:", stats["returns_std"]) print("Trades (metrics/trades file):", num_trades_metric, "/", trades_count) print("Turnover:", turnover) - print("Costs (commission/slippage/borrow/total):", commission, slippage, borrow, total_costs) + print( + "Costs (commission/slippage/borrow/total):", + commission, + slippage, + borrow, + total_costs, + ) checks = [] if turnover > 0 and trades_count == 0: diff --git a/scripts/export_wrds_flagship.py b/scripts/export_wrds_flagship.py old mode 100755 new mode 100644 index e5241205..8ff7f150 --- a/scripts/export_wrds_flagship.py +++ b/scripts/export_wrds_flagship.py @@ -78,7 +78,9 @@ def _ensure_pgpass() -> None: mode = path.stat().st_mode & 0o777 except OSError as exc: # pragma: no cover - filesystem edge case raise SystemExit(f"Unable to stat {path}: {exc}") from exc - raise SystemExit(f"{path} must contain WRDS entry with 600 perms (found {oct(mode)})") + raise SystemExit( + f"{path} must contain WRDS entry with 600 perms (found {oct(mode)})" + ) def _resolve_wrds_username() -> str | None: @@ -97,7 +99,9 @@ def _resolve_wrds_username() -> str | None: if len(fields) < 4: continue host_field = fields[0] - if host_field in {"*", WRDS_HOST} or host_field.endswith("wharton.upenn.edu"): + if host_field in {"*", WRDS_HOST} or host_field.endswith( + "wharton.upenn.edu" + ): return fields[3] except OSError: # pragma: no cover - filesystem edge cases return None @@ -109,7 +113,12 @@ def _load_universe(root: Path) -> list[str]: if not universe_csv.exists(): raise SystemExit(f"Universe file missing: {universe_csv}") symbols = ( - pd.read_csv(universe_csv)["symbol"].astype(str).str.upper().dropna().unique().tolist() + pd.read_csv(universe_csv)["symbol"] + .astype(str) + .str.upper() + .dropna() + .unique() + .tolist() ) if not symbols: raise SystemExit("Universe file has no symbols") @@ -248,10 +257,16 @@ def _map_gics(code: float | int | str | None) -> str: return GICS_SECTORS.get(text[:2], "UNKNOWN") -def _write_metadata(dsf: pd.DataFrame, gics_df: pd.DataFrame, meta_path: Path) -> pd.DataFrame: +def _write_metadata( + dsf: pd.DataFrame, gics_df: pd.DataFrame, meta_path: Path +) -> pd.DataFrame: latest = dsf.sort_values("date").groupby("ticker").tail(1).copy() latest["market_cap"] = latest["close"].abs() * latest["shares_out"].abs() - gics_df = gics_df.rename(columns={"gsector": "gics_sector"}) if not gics_df.empty else gics_df + gics_df = ( + gics_df.rename(columns={"gsector": "gics_sector"}) + if not gics_df.empty + else gics_df + ) latest = latest.merge(gics_df[["permno", "gics_sector"]], on="permno", how="left") latest["sector"] = latest["gics_sector"].apply(_map_gics) meta_path.parent.mkdir(parents=True, exist_ok=True) @@ -281,7 +296,13 @@ def _write_metadata(dsf: pd.DataFrame, gics_df: pd.DataFrame, meta_path: Path) - return payload -def _write_manifest(stats: ExportStats, num_symbols: int, csv_dir: Path, parquet_dir: Path, meta_path: Path) -> None: +def _write_manifest( + stats: ExportStats, + num_symbols: int, + csv_dir: Path, + parquet_dir: Path, + meta_path: Path, +) -> None: MANIFEST_PATH.parent.mkdir(parents=True, exist_ok=True) payload = { "exported_at": datetime.now(timezone.utc).isoformat(), diff --git a/scripts/validate_run_logs.py b/scripts/validate_run_logs.py index 906e17d3..2c0310dc 100644 --- a/scripts/validate_run_logs.py +++ b/scripts/validate_run_logs.py @@ -8,7 +8,6 @@ import sys from pathlib import Path - RUNS_ROOT = Path("docs/agent_runs") TICKETS_FILE = Path("docs/CODEX_SPRINT_TICKETS.md") @@ -58,7 +57,9 @@ def require(condition: bool, errors: list[str], message: str) -> None: errors.append(message) -def validate_meta(meta_path: Path, run_dir: Path, ticket_ids: set[str], errors: list[str]) -> None: +def validate_meta( + meta_path: Path, run_dir: Path, ticket_ids: set[str], errors: list[str] +) -> None: try: data = json.loads(meta_path.read_text()) except Exception as exc: # pragma: no cover - generic parse guard diff --git a/src/microalpha/allocators.py b/src/microalpha/allocators.py index 3dc39d30..af6de11d 100644 --- a/src/microalpha/allocators.py +++ b/src/microalpha/allocators.py @@ -66,8 +66,12 @@ def lw_min_var( return (weights, returns_df) if return_cov else weights cov_shrink = _ledoit_wolf_cov(returns_df.to_numpy(dtype=float)) - cov_df = pd.DataFrame(cov_shrink, index=returns_df.columns, columns=returns_df.columns) - weights = _min_var_weights(cov_df.to_numpy(), allow_short=allow_short, epsilon=epsilon) + cov_df = pd.DataFrame( + cov_shrink, index=returns_df.columns, columns=returns_df.columns + ) + weights = _min_var_weights( + cov_df.to_numpy(), allow_short=allow_short, epsilon=epsilon + ) weights_series = pd.Series(weights, index=cov_df.index, name="weight") if return_cov: @@ -98,8 +102,16 @@ def budgeted_allocator( total_signal = float(long_signals.abs().sum() + short_signals.abs().sum()) total_signal = max(total_signal, 1e-12) - long_budget = total_budget * float(long_signals.sum()) / total_signal if not long_signals.empty else 0.0 - short_budget = total_budget * float(short_signals.abs().sum()) / total_signal if not short_signals.empty else 0.0 + long_budget = ( + total_budget * float(long_signals.sum()) / total_signal + if not long_signals.empty + else 0.0 + ) + short_budget = ( + total_budget * float(short_signals.abs().sum()) / total_signal + if not short_signals.empty + else 0.0 + ) weights = pd.Series(0.0, index=cov_df.index, name="weight") @@ -131,6 +143,7 @@ def budgeted_allocator( # --------------------------------------------------------------------------- # Helpers + def _as_dataframe( data: pd.DataFrame | np.ndarray | Mapping[str, Mapping[str, float]], *, diff --git a/src/microalpha/cli.py b/src/microalpha/cli.py index 0af1a63e..7d8faf3f 100644 --- a/src/microalpha/cli.py +++ b/src/microalpha/cli.py @@ -158,7 +158,11 @@ def main() -> None: Path(args.bootstrap_out).resolve() if args.bootstrap_out else ( - (equity_plot_path.parent if equity_plot_path.suffix else equity_plot_path) + ( + equity_plot_path.parent + if equity_plot_path.suffix + else equity_plot_path + ) / DEFAULT_BOOTSTRAP_NAME ) ) diff --git a/src/microalpha/config.py b/src/microalpha/config.py index e477b793..0f100789 100644 --- a/src/microalpha/config.py +++ b/src/microalpha/config.py @@ -115,10 +115,12 @@ class CapitalPolicyCfg(BaseModel): class BorrowCfg(BaseModel): annual_fee_bps: float | None = Field( - default=None, description="Fallback annualized borrow fee in bps if metadata is missing." + default=None, + description="Fallback annualized borrow fee in bps if metadata is missing.", ) floor_bps: float | None = Field( - default=None, description="Minimum borrow fee in bps (applied after multiplier)." + default=None, + description="Minimum borrow fee in bps (applied after multiplier).", ) multiplier: float = Field( default=1.0, description="Multiplier applied to metadata borrow fees." diff --git a/src/microalpha/engine.py b/src/microalpha/engine.py index 04b0a415..74cc4ccc 100644 --- a/src/microalpha/engine.py +++ b/src/microalpha/engine.py @@ -65,7 +65,9 @@ def _on_market(self, market_event: MarketEvent) -> None: if order_flow and signals: try: order_flow.begin_rebalance(signals, market_event.timestamp) - except Exception as exc: # pragma: no cover - diagnostics should not fail run + except ( + Exception + ) as exc: # pragma: no cover - diagnostics should not fail run order_flow.record_error( f"begin_rebalance_error: {type(exc).__name__}: {exc}" ) @@ -103,7 +105,9 @@ def _on_market(self, market_event: MarketEvent) -> None: if order_flow and signals: try: order_flow.end_rebalance() - except Exception as exc: # pragma: no cover - diagnostics should not fail run + except ( + Exception + ) as exc: # pragma: no cover - diagnostics should not fail run order_flow.record_error( f"end_rebalance_error: {type(exc).__name__}: {exc}" ) diff --git a/src/microalpha/execution.py b/src/microalpha/execution.py index 8ca010d6..d77c7ae0 100644 --- a/src/microalpha/execution.py +++ b/src/microalpha/execution.py @@ -120,7 +120,9 @@ def _build_fill( limit_price = order.price if order.order_type == "LIMIT" else None if self.limit_mode is not None: - spread_bps = meta.spread_bps if meta.spread_bps and meta.spread_bps > 0 else 10.0 + spread_bps = ( + meta.spread_bps if meta.spread_bps and meta.spread_bps > 0 else 10.0 + ) half_spread_px = (spread_bps / 20_000.0) * market_price if self.limit_mode == "IOC": limit_price = market_price if limit_price is None else limit_price @@ -202,9 +204,7 @@ def _resolve_fill_quantity( if not limit_context: return qty - fraction = self._queue_fill_fraction( - order, market_price, timestamp, qty, meta - ) + fraction = self._queue_fill_fraction(order, market_price, timestamp, qty, meta) if fraction <= 0.0: return 0 @@ -223,7 +223,9 @@ def _queue_fill_fraction( ) -> float: abs_qty = max(abs(qty), 1) adv = meta.adv if meta.adv and meta.adv > 0 else float(abs_qty) * 20.0 - spread_bps = meta.spread_bps if meta.spread_bps and meta.spread_bps > 0 else 10.0 + spread_bps = ( + meta.spread_bps if meta.spread_bps and meta.spread_bps > 0 else 10.0 + ) vol_bps = self._resolve_volatility_bps(order.symbol, timestamp, meta) if vol_bps <= 0: diff --git a/src/microalpha/integrity.py b/src/microalpha/integrity.py index 4536f028..47146b60 100644 --- a/src/microalpha/integrity.py +++ b/src/microalpha/integrity.py @@ -63,10 +63,14 @@ def evaluate_portfolio_integrity( expected_equity = initial_equity + realized_pnl + unrealized_pnl - commission_total recon_error = float(final_equity - expected_equity) - recon_tol = max(tol_abs, tol_rel * max(abs(final_equity), abs(expected_equity), 1.0)) + recon_tol = max( + tol_abs, tol_rel * max(abs(final_equity), abs(expected_equity), 1.0) + ) total_costs = commission_total + borrow_cost_total + float(slippage_total) - equity_constant = _equity_is_constant(equity_series, tol_abs=tol_abs, tol_rel=tol_rel) + equity_constant = _equity_is_constant( + equity_series, tol_abs=tol_abs, tol_rel=tol_rel + ) reasons: list[str] = [] if abs(recon_error) > recon_tol: diff --git a/src/microalpha/manifest.py b/src/microalpha/manifest.py index 8e9c5ba0..ca132954 100644 --- a/src/microalpha/manifest.py +++ b/src/microalpha/manifest.py @@ -67,7 +67,9 @@ def load_manifest_path(path: str | Path) -> dict[str, Any]: return payload -def load_manifest(artifact_dir: str | Path, *, required: bool = True) -> dict[str, Any] | None: +def load_manifest( + artifact_dir: str | Path, *, required: bool = True +) -> dict[str, Any] | None: """Load a manifest.json from an artifact directory.""" manifest_path = Path(artifact_dir) / "manifest.json" @@ -178,9 +180,11 @@ def _as_mapping(value: Any) -> Mapping[str, Any]: borrow_cfg = _as_mapping(template.get("borrow")) risk_caps = { - "max_gross_leverage": template.get("max_gross_leverage") - if template.get("max_gross_leverage") is not None - else template.get("max_portfolio_heat"), + "max_gross_leverage": ( + template.get("max_gross_leverage") + if template.get("max_gross_leverage") is not None + else template.get("max_portfolio_heat") + ), "max_portfolio_heat": template.get("max_portfolio_heat"), "max_net_leverage": template.get("max_exposure"), "max_single_name_weight": template.get("max_single_name_weight"), diff --git a/src/microalpha/market_metadata.py b/src/microalpha/market_metadata.py index 6495ae0e..4224b759 100644 --- a/src/microalpha/market_metadata.py +++ b/src/microalpha/market_metadata.py @@ -39,9 +39,7 @@ def with_overrides( else self.borrow_fee_annual_bps ), volatility_bps=( - volatility_bps - if volatility_bps is not None - else self.volatility_bps + volatility_bps if volatility_bps is not None else self.volatility_bps ), ) diff --git a/src/microalpha/metrics.py b/src/microalpha/metrics.py index fe756423..2d7666a3 100644 --- a/src/microalpha/metrics.py +++ b/src/microalpha/metrics.py @@ -85,9 +85,7 @@ def compute_metrics( max_gross_exposure = ( float(df["gross_exposure"].max()) if "gross_exposure" in df else None ) - max_net_exposure = ( - float(df["exposure"].abs().max()) if "exposure" in df else None - ) + max_net_exposure = float(df["exposure"].abs().max()) if "exposure" in df else None # Annualized volatility and CAGR ann_vol = float(returns.std(ddof=0) * (periods**0.5)) if len(returns) > 1 else 0.0 @@ -174,9 +172,7 @@ def compute_metrics( "max_net_exposure": max_net_exposure, "exposure_std": float(df["exposure"].std(ddof=0)) if "exposure" in df else 0.0, "gross_exposure_std": ( - float(df["gross_exposure"].std(ddof=0)) - if "gross_exposure" in df - else None + float(df["gross_exposure"].std(ddof=0)) if "gross_exposure" in df else None ), "total_turnover": float(turnover), "turnover_per_day": float(turnover / max(len(df), 1)), diff --git a/src/microalpha/order_flow.py b/src/microalpha/order_flow.py index 51f10825..e33defb2 100644 --- a/src/microalpha/order_flow.py +++ b/src/microalpha/order_flow.py @@ -5,7 +5,7 @@ import math from collections import Counter from dataclasses import dataclass, field -from typing import Any, Dict, Iterable, List, Mapping, Optional +from typing import Any, Dict, Iterable, List, Mapping import pandas as pd @@ -90,7 +90,9 @@ def begin_rebalance( try: signals_list = list(signals) except Exception as exc: # pragma: no cover - defensive - self.record_error(f"begin_rebalance signals error: {type(exc).__name__}: {exc}") + self.record_error( + f"begin_rebalance signals error: {type(exc).__name__}: {exc}" + ) key = _timestamp_to_date(timestamp) self._active_key = key self._entry_for_key(key) @@ -128,7 +130,9 @@ def begin_rebalance( nonzero_weights = [w for w in weights if abs(w) > 0] entry["target_weights_nonzero_count"] = int(len(nonzero_weights)) - entry["sum_abs_weights"] = float(sum(abs(w) for w in weights)) if weights else 0.0 + entry["sum_abs_weights"] = ( + float(sum(abs(w) for w in weights)) if weights else 0.0 + ) entry["min_weight"] = float(min(weights)) if weights else None entry["max_weight"] = float(max(weights)) if weights else None except Exception as exc: # pragma: no cover - diagnostics should not fail run @@ -140,12 +144,16 @@ def begin_rebalance( def end_rebalance(self) -> None: self._active_key = None - def record_order_created(self, order: OrderEvent, signal: SignalEvent | None = None) -> None: + def record_order_created( + self, order: OrderEvent, signal: SignalEvent | None = None + ) -> None: key = self._resolve_key(signal=signal, order=order) entry = self._entry_for_key(key) entry["orders_created_count"] = int(entry["orders_created_count"]) + 1 if abs(int(getattr(order, "qty", 0) or 0)) > 0: - entry["orders_nonzero_qty_count"] = int(entry["orders_nonzero_qty_count"]) + 1 + entry["orders_nonzero_qty_count"] = ( + int(entry["orders_nonzero_qty_count"]) + 1 + ) def record_order_drop( self, @@ -199,7 +207,9 @@ def record_fill(self, fill: FillEvent) -> None: except (TypeError, ValueError): self.record_error("fill_notional_cast_error") - def merge_filter_diagnostics(self, filter_diagnostics: Mapping[str, Any] | None) -> None: + def merge_filter_diagnostics( + self, filter_diagnostics: Mapping[str, Any] | None + ) -> None: if filter_diagnostics is None: return if not isinstance(filter_diagnostics, Mapping): @@ -278,7 +288,9 @@ def summary(self) -> Dict[str, Any]: return summary def payload(self) -> Dict[str, Any]: - entries = sorted(self._entries.values(), key=lambda e: str(e.get("rebalance_date"))) + entries = sorted( + self._entries.values(), key=lambda e: str(e.get("rebalance_date")) + ) payload: Dict[str, Any] = {"entries": entries, "summary": self.summary()} if self._errors: payload["errors"] = list(self._errors) @@ -304,9 +316,13 @@ def infer_non_degenerate_reason(payload: Mapping[str, Any] | None) -> str | None for entry in entries: if not isinstance(entry, Mapping): continue - totals["targets_nonzero"] += int(entry.get("target_weights_nonzero_count", 0) or 0) + totals["targets_nonzero"] += int( + entry.get("target_weights_nonzero_count", 0) or 0 + ) totals["orders_created"] += int(entry.get("orders_created_count", 0) or 0) - totals["orders_nonzero_qty"] += int(entry.get("orders_nonzero_qty_count", 0) or 0) + totals["orders_nonzero_qty"] += int( + entry.get("orders_nonzero_qty_count", 0) or 0 + ) totals["orders_accepted"] += int(entry.get("orders_accepted_count", 0) or 0) totals["orders_rejected"] += int(entry.get("orders_rejected_count", 0) or 0) totals["fills"] += int(entry.get("fills_count", 0) or 0) diff --git a/src/microalpha/portfolio.py b/src/microalpha/portfolio.py index 29671677..08859fdb 100644 --- a/src/microalpha/portfolio.py +++ b/src/microalpha/portfolio.py @@ -3,8 +3,7 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Dict, Iterable, List, Literal, Mapping, cast -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Dict, Iterable, List, Literal, Mapping, cast from .events import FillEvent, LookaheadError, MarketEvent, OrderEvent, SignalEvent from .logging import JsonlWriter @@ -99,7 +98,9 @@ def __init__( else None ) if borrow_cfg.get("multiplier") is not None: - self.borrow_fee_multiplier = float(borrow_cfg.get("multiplier", 1.0)) + self.borrow_fee_multiplier = float( + borrow_cfg.get("multiplier", 1.0) + ) else: self.borrow_fee_bps = ( float(getattr(borrow_cfg, "annual_fee_bps")) @@ -112,10 +113,14 @@ def __init__( else None ) if getattr(borrow_cfg, "multiplier", None) is not None: - self.borrow_fee_multiplier = float(getattr(borrow_cfg, "multiplier")) + self.borrow_fee_multiplier = float( + getattr(borrow_cfg, "multiplier") + ) def on_market(self, event: MarketEvent) -> None: - self._record_equity(event.timestamp, apply_borrow_costs=True, overwrite_last=True) + self._record_equity( + event.timestamp, apply_borrow_costs=True, overwrite_last=True + ) def refresh_equity_after_fills(self, timestamp: int) -> None: """Refresh the latest equity snapshot after same-day fills.""" @@ -179,7 +184,9 @@ def _record_equity( else: self.equity_curve.append(record) - def valuation_snapshot(self, timestamp: int | None = None) -> tuple[float, float, float]: + def valuation_snapshot( + self, timestamp: int | None = None + ) -> tuple[float, float, float]: """Return (market_value, gross_market_value, unrealized_pnl) at timestamp.""" ts = self.current_time if timestamp is None else timestamp if ts is None: @@ -209,9 +216,7 @@ def on_signal(self, signal: SignalEvent) -> Iterable[OrderEvent]: position = self.positions.get(signal.symbol) if not position or position.qty == 0: if self.order_flow: - self.order_flow.record_order_drop( - "exit_no_position", signal=signal - ) + self.order_flow.record_order_drop("exit_no_position", signal=signal) return [] side = cast(Literal["BUY", "SELL"], "SELL" if position.qty > 0 else "BUY") order = OrderEvent(signal.timestamp, signal.symbol, abs(position.qty), side) @@ -257,7 +262,9 @@ def on_signal(self, signal: SignalEvent) -> Iterable[OrderEvent]: self.market_value + (qty if side == "BUY" else -qty) * price ) projected_exposure = ( - abs(anticipated_market_value) / projected_equity if projected_equity else 0.0 + abs(anticipated_market_value) / projected_equity + if projected_equity + else 0.0 ) if self.max_exposure is not None and projected_exposure > self.max_exposure: @@ -267,7 +274,9 @@ def on_signal(self, signal: SignalEvent) -> Iterable[OrderEvent]: max_additional = max_abs_mv - self.market_value else: max_additional = max_abs_mv + self.market_value - max_qty_exposure = int(max_additional / price) if max_additional > 0 else 0 + max_qty_exposure = ( + int(max_additional / price) if max_additional > 0 else 0 + ) if max_qty_exposure <= 0: if self.order_flow: self.order_flow.record_order_drop( @@ -466,7 +475,9 @@ def _borrow_cost_for( return 0.0 meta = self._symbol_meta.get(key) - raw_bps = meta.borrow_fee_annual_bps if meta and meta.borrow_fee_annual_bps else None + raw_bps = ( + meta.borrow_fee_annual_bps if meta and meta.borrow_fee_annual_bps else None + ) if raw_bps is None and self.borrow_fee_bps is not None: raw_bps = self.borrow_fee_bps if raw_bps is None and self.borrow_fee_floor_bps is not None: @@ -517,9 +528,15 @@ def _signal_quantity(self, signal: SignalEvent) -> int: return int(signal.meta["qty"]) if "weight" in signal.meta: target_weight = float(signal.meta["weight"]) - equity = self.last_equity if self.last_equity is not None else self.initial_cash + equity = ( + self.last_equity + if self.last_equity is not None + else self.initial_cash + ) if equity and equity > 0: - price = self.data_handler.get_latest_price(signal.symbol, signal.timestamp) + price = self.data_handler.get_latest_price( + signal.symbol, signal.timestamp + ) if price is None and self.current_time is not None: price = self.data_handler.get_latest_price( signal.symbol, self.current_time diff --git a/src/microalpha/reporting/analytics.py b/src/microalpha/reporting/analytics.py index bea210d8..3c2a8950 100644 --- a/src/microalpha/reporting/analytics.py +++ b/src/microalpha/reporting/analytics.py @@ -97,7 +97,9 @@ def compute_rolling_ir(ic_series: pd.Series, window: int = 63) -> pd.Series: rolling_mean = ic_series.rolling(window).mean() rolling_std = ic_series.rolling(window).std(ddof=0) with np.errstate(divide="ignore", invalid="ignore"): - ir = np.where(rolling_std > 0, np.sqrt(window) * (rolling_mean / rolling_std), np.nan) + ir = np.where( + rolling_std > 0, np.sqrt(window) * (rolling_mean / rolling_std), np.nan + ) result = pd.Series(ir, index=ic_series.index, name="rolling_ir") return result @@ -126,12 +128,7 @@ def compute_decile_table(signals: pd.DataFrame, deciles: int = 10) -> pd.DataFra ) if not records: return pd.DataFrame(columns=["decile", "mean_return"]) - summary = ( - pd.DataFrame(records) - .groupby("decile")["mean_return"] - .mean() - .sort_index() - ) + summary = pd.DataFrame(records).groupby("decile")["mean_return"].mean().sort_index() summary.index = [f"P{int(idx)}" for idx in summary.index] table = summary.to_frame().reset_index().rename(columns={"index": "decile"}) tail_label = f"P{deciles}" @@ -216,16 +213,32 @@ def plot_ic_series(ic_series: pd.Series, ir_series: pd.Series, output: Path) -> output.parent.mkdir(parents=True, exist_ok=True) fig, axes = plt.subplots(2, 1, figsize=(10, 6), sharex=True) if ic_series.empty: - axes[0].text(0.5, 0.5, "IC unavailable", ha="center", va="center", transform=axes[0].transAxes) + axes[0].text( + 0.5, + 0.5, + "IC unavailable", + ha="center", + va="center", + transform=axes[0].transAxes, + ) else: axes[0].plot(ic_series.index, ic_series.values, label="IC", color="#1f77b4") axes[0].axhline(0.0, color="black", linewidth=0.8, linestyle=":") axes[0].set_ylabel("IC") axes[0].grid(True, linestyle=":", alpha=0.3) if ir_series.empty: - axes[1].text(0.5, 0.5, "Rolling IR unavailable", ha="center", va="center", transform=axes[1].transAxes) + axes[1].text( + 0.5, + 0.5, + "Rolling IR unavailable", + ha="center", + va="center", + transform=axes[1].transAxes, + ) else: - axes[1].plot(ir_series.index, ir_series.values, label="Rolling IR", color="#d62728") + axes[1].plot( + ir_series.index, ir_series.values, label="Rolling IR", color="#d62728" + ) axes[1].axhline(0.0, color="black", linewidth=0.8, linestyle=":") axes[1].set_ylabel("IR") axes[1].grid(True, linestyle=":", alpha=0.3) @@ -239,7 +252,14 @@ def plot_deciles(table: pd.DataFrame, output: Path) -> None: output.parent.mkdir(parents=True, exist_ok=True) fig, ax = plt.subplots(figsize=(9, 4)) if table.empty: - ax.text(0.5, 0.5, "Deciles unavailable", ha="center", va="center", transform=ax.transAxes) + ax.text( + 0.5, + 0.5, + "Deciles unavailable", + ha="center", + va="center", + transform=ax.transAxes, + ) else: labels = table["decile"].astype(str) mask = labels.str.match(r"^P\d+$") @@ -266,7 +286,14 @@ def plot_rolling_betas(betas: pd.DataFrame, output: Path) -> None: output.parent.mkdir(parents=True, exist_ok=True) fig, ax = plt.subplots(figsize=(10, 5)) if betas.empty: - ax.text(0.5, 0.5, "Rolling betas unavailable", ha="center", va="center", transform=ax.transAxes) + ax.text( + 0.5, + 0.5, + "Rolling betas unavailable", + ha="center", + va="center", + transform=ax.transAxes, + ) else: for column in betas.columns: ax.plot(betas.index, betas[column], label=column) @@ -320,7 +347,9 @@ def generate_analytics( rolling_betas: pd.DataFrame | None = None if factors_df is not None: factor_cols = [col for col in factors_df.columns if col != "RF"] - rolling_betas = compute_rolling_betas(returns, factors_df, factor_cols=factor_cols, window=window) + rolling_betas = compute_rolling_betas( + returns, factors_df, factor_cols=factor_cols, window=window + ) analytics_dir.mkdir(parents=True, exist_ok=True) ic_path = analytics_dir / f"{artifact_dir.name}_ic_series.csv" @@ -354,13 +383,44 @@ def generate_analytics( def _build_arg_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("artifact_dir", type=Path, help="Artifact directory containing equity_curve.csv and signals.csv") - parser.add_argument("--signals", type=Path, default=None, help="Override path to signals CSV") - parser.add_argument("--factors", type=Path, default=None, help="Optional factor CSV (FF5+MOM compatible)") - parser.add_argument("--plots-dir", type=Path, default=PLOTS_DIR, help="Directory for saving plots (default: artifacts/plots)") - parser.add_argument("--analytics-dir", type=Path, default=ANALYTICS_DIR, help="Directory for saving CSV analytics (default: artifacts/analytics)") - parser.add_argument("--window", type=int, default=63, help="Rolling window (trading days) for IR and betas") - parser.add_argument("--deciles", type=int, default=10, help="Number of buckets for decile aggregation") + parser.add_argument( + "artifact_dir", + type=Path, + help="Artifact directory containing equity_curve.csv and signals.csv", + ) + parser.add_argument( + "--signals", type=Path, default=None, help="Override path to signals CSV" + ) + parser.add_argument( + "--factors", + type=Path, + default=None, + help="Optional factor CSV (FF5+MOM compatible)", + ) + parser.add_argument( + "--plots-dir", + type=Path, + default=PLOTS_DIR, + help="Directory for saving plots (default: artifacts/plots)", + ) + parser.add_argument( + "--analytics-dir", + type=Path, + default=ANALYTICS_DIR, + help="Directory for saving CSV analytics (default: artifacts/analytics)", + ) + parser.add_argument( + "--window", + type=int, + default=63, + help="Rolling window (trading days) for IR and betas", + ) + parser.add_argument( + "--deciles", + type=int, + default=10, + help="Number of buckets for decile aggregation", + ) return parser diff --git a/src/microalpha/reporting/baselines.py b/src/microalpha/reporting/baselines.py index a1dc366a..197f58ce 100644 --- a/src/microalpha/reporting/baselines.py +++ b/src/microalpha/reporting/baselines.py @@ -85,16 +85,24 @@ def compute_baselines( if "timestamp" not in equity_df.columns: baseline_df = pd.DataFrame(columns=BASELINE_COLUMNS) _write_baselines(baseline_df, baselines_path, status, status_path) - _write_missing_flagship_readme(artifact_dir, "equity_curve.csv missing timestamp") + _write_missing_flagship_readme( + artifact_dir, "equity_curve.csv missing timestamp" + ) return baseline_df - equity_df["date"] = pd.to_datetime(equity_df["timestamp"], unit="ns", errors="coerce") + equity_df["date"] = pd.to_datetime( + equity_df["timestamp"], unit="ns", errors="coerce" + ) equity_df["date"] = equity_df["date"].dt.normalize() - equity_df = equity_df.dropna(subset=["date"]).drop_duplicates("date").sort_values("date") + equity_df = ( + equity_df.dropna(subset=["date"]).drop_duplicates("date").sort_values("date") + ) if equity_df.empty: baseline_df = pd.DataFrame(columns=BASELINE_COLUMNS) _write_baselines(baseline_df, baselines_path, status, status_path) - _write_missing_flagship_readme(artifact_dir, "equity_curve.csv has no usable dates") + _write_missing_flagship_readme( + artifact_dir, "equity_curve.csv has no usable dates" + ) return baseline_df if "returns" in equity_df.columns: @@ -117,8 +125,13 @@ def compute_baselines( if flagship_returns.notna().any(): status["flagship_net"] = {"status": "ok", "reason": "equity_curve.csv returns"} else: - status["flagship_net"] = {"status": "missing", "reason": "flagship returns unavailable"} - _write_missing_flagship_readme(artifact_dir, "flagship returns unavailable in equity_curve.csv") + status["flagship_net"] = { + "status": "missing", + "reason": "flagship returns unavailable", + } + _write_missing_flagship_readme( + artifact_dir, "flagship returns unavailable in equity_curve.csv" + ) config_path = _resolve_config_path(artifact_dir) config_payload = _load_config_payload(config_path) @@ -254,7 +267,9 @@ def _turnover_for(label: str) -> float | None: { "Series": BASELINE_LABELS["flagship_net"], **_metrics_from_returns( - baselines["flagship_net"], periods_per_year=periods_per_year, hac_lags=hac_lags + baselines["flagship_net"], + periods_per_year=periods_per_year, + hac_lags=hac_lags, ), "Turnover": _turnover_for("flagship_net"), } @@ -277,11 +292,16 @@ def _turnover_for(label: str) -> float | None: } ) - return pd.DataFrame(rows, columns=["Series", "Sharpe_HAC", "MaxDD", "CAGR", "Turnover"]) + return pd.DataFrame( + rows, columns=["Series", "Sharpe_HAC", "MaxDD", "CAGR", "Turnover"] + ) def render_baseline_table(metrics_df: pd.DataFrame) -> str: - lines = ["| Series | Sharpe_HAC | MaxDD | CAGR | Turnover |", "| --- | ---:| ---:| ---:| ---:|"] + lines = [ + "| Series | Sharpe_HAC | MaxDD | CAGR | Turnover |", + "| --- | ---:| ---:| ---:| ---:|", + ] for row in metrics_df.itertuples(index=False): lines.append( "| {series} | {sharpe} | {maxdd} | {cagr} | {turnover} |".format( @@ -374,7 +394,9 @@ def _resolve_config_path(artifact_dir: Path) -> Path | None: return artifact_candidate except Exception: pass - yaml_files = sorted(list(artifact_dir.glob("*.yaml")) + list(artifact_dir.glob("*.yml"))) + yaml_files = sorted( + list(artifact_dir.glob("*.yaml")) + list(artifact_dir.glob("*.yml")) + ) return yaml_files[0] if yaml_files else None @@ -387,8 +409,14 @@ def _load_config_payload(config_path: Path | None) -> dict[str, Any]: return {} -def _resolve_data_path(config: Mapping[str, Any], config_path: Path | None) -> Path | None: - base = config.get("template") if isinstance(config.get("template"), Mapping) else config +def _resolve_data_path( + config: Mapping[str, Any], config_path: Path | None +) -> Path | None: + base = ( + config.get("template") + if isinstance(config.get("template"), Mapping) + else config + ) if not isinstance(base, Mapping): return None data_path = base.get("data_path") or base.get("data_dir") or base.get("data") @@ -397,8 +425,14 @@ def _resolve_data_path(config: Mapping[str, Any], config_path: Path | None) -> P return _resolve_path(str(data_path), config_path) -def _resolve_universe_path(config: Mapping[str, Any], config_path: Path | None) -> Path | None: - base = config.get("template") if isinstance(config.get("template"), Mapping) else config +def _resolve_universe_path( + config: Mapping[str, Any], config_path: Path | None +) -> Path | None: + base = ( + config.get("template") + if isinstance(config.get("template"), Mapping) + else config + ) if not isinstance(base, Mapping): return None strategy = base.get("strategy") @@ -441,7 +475,9 @@ def _load_universe(path: Path) -> dict[pd.Timestamp, pd.DataFrame]: return universe -def _collect_universe_symbols(universe: Mapping[pd.Timestamp, pd.DataFrame]) -> list[str]: +def _collect_universe_symbols( + universe: Mapping[pd.Timestamp, pd.DataFrame], +) -> list[str]: symbols: set[str] = set() for snapshot in universe.values(): symbols.update([str(sym).upper() for sym in snapshot.index]) @@ -644,7 +680,10 @@ def _attach_market_proxy( ) -> pd.DataFrame: path, reason = _resolve_market_proxy_path(data_path) if path is None or not path.exists(): - status["market_proxy"] = {"status": "missing", "reason": reason or "market proxy not found"} + status["market_proxy"] = { + "status": "missing", + "reason": reason or "market proxy not found", + } baselines["market_proxy"] = np.nan return baselines @@ -658,7 +697,9 @@ def _attach_market_proxy( return baselines -def _resolve_market_proxy_path(data_path: Path | None) -> tuple[Path | None, str | None]: +def _resolve_market_proxy_path( + data_path: Path | None, +) -> tuple[Path | None, str | None]: candidates: list[tuple[Path, str]] = [] if data_path: candidates.extend( @@ -695,12 +736,18 @@ def _load_proxy_returns(path: Path, calendar: pd.DatetimeIndex) -> pd.Series: df = df.dropna(subset=[date_col]) df = df.set_index(date_col).sort_index() - return_cols = [col for col in df.columns if col.lower() in {"vwretd", "return", "returns", "ret"}] + return_cols = [ + col + for col in df.columns + if col.lower() in {"vwretd", "return", "returns", "ret"} + ] if return_cols: series = df[return_cols[0]].astype(float) return series.reindex(calendar, fill_value=0.0) - price_cols = [col for col in df.columns if col.lower() in {"close", "price", "adj_close"}] + price_cols = [ + col for col in df.columns if col.lower() in {"close", "price", "adj_close"} + ] if price_cols: prices = df[price_cols[0]].astype(float) prices = prices.reindex(calendar, method="ffill") @@ -793,7 +840,9 @@ def _write_baselines( ) -> None: output = df.copy() if "date" in output.columns: - output["date"] = pd.to_datetime(output["date"], errors="coerce").dt.strftime("%Y-%m-%d") + output["date"] = pd.to_datetime(output["date"], errors="coerce").dt.strftime( + "%Y-%m-%d" + ) baselines_path.write_text(output.to_csv(index=False), encoding="utf-8") status_path.write_text(json.dumps(status, indent=2), encoding="utf-8") diff --git a/src/microalpha/reporting/factors.py b/src/microalpha/reporting/factors.py index 68259310..17b25457 100644 --- a/src/microalpha/reporting/factors.py +++ b/src/microalpha/reporting/factors.py @@ -231,7 +231,9 @@ def _prepare_factors(factor_csv: Path, required: Sequence[str]) -> pd.DataFrame: def _design_matrix( factors: pd.DataFrame, factor_names: Sequence[str], excess_returns: pd.Series ) -> tuple[np.ndarray, np.ndarray]: - aligned = factors[list(factor_names)].join(excess_returns.rename("excess"), how="inner") + aligned = factors[list(factor_names)].join( + excess_returns.rename("excess"), how="inner" + ) aligned = aligned.dropna() if aligned.empty: raise ValueError("No overlapping dates between factors and returns") @@ -275,7 +277,9 @@ def compute_factor_regression( model_key = model.lower() if model_key not in MODEL_FACTORS: - raise ValueError(f"Unknown factor model '{model}'. Valid options: {sorted(MODEL_FACTORS)}") + raise ValueError( + f"Unknown factor model '{model}'. Valid options: {sorted(MODEL_FACTORS)}" + ) factor_names = MODEL_FACTORS[model_key] returns = _prepare_returns(equity_csv) @@ -326,8 +330,12 @@ def _format_meta_line(meta: FactorRegressionMeta) -> str: def main() -> None: - parser = argparse.ArgumentParser(description="Run factor regressions on Microalpha artifacts") - parser.add_argument("artifact_dir", type=Path, help="Artifact directory containing equity_curve.csv") + parser = argparse.ArgumentParser( + description="Run factor regressions on Microalpha artifacts" + ) + parser.add_argument( + "artifact_dir", type=Path, help="Artifact directory containing equity_curve.csv" + ) parser.add_argument( "--factors", type=Path, @@ -340,7 +348,9 @@ def main() -> None: default=None, help="Optional markdown file to write the regression table to", ) - parser.add_argument("--hac-lags", type=int, default=5, help="Newey-West lag length (default: 5)") + parser.add_argument( + "--hac-lags", type=int, default=5, help="Newey-West lag length (default: 5)" + ) parser.add_argument( "--model", choices=sorted(MODEL_FACTORS.keys()), @@ -378,7 +388,9 @@ def main() -> None: print(_format_meta_line(output.meta)) if args.output: args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(table + "\n" + _format_meta_line(output.meta) + "\n", encoding="utf-8") + args.output.write_text( + table + "\n" + _format_meta_line(output.meta) + "\n", encoding="utf-8" + ) if __name__ == "__main__": diff --git a/src/microalpha/reporting/robustness.py b/src/microalpha/reporting/robustness.py index 617db007..f1f4a7f8 100644 --- a/src/microalpha/reporting/robustness.py +++ b/src/microalpha/reporting/robustness.py @@ -46,6 +46,7 @@ def write_robustness_artifacts( # Cost sensitivity # --------------------------------------------------------------------------- + def compute_cost_sensitivity( artifact_dir: Path | str, *, @@ -88,14 +89,18 @@ def compute_cost_sensitivity( ) commission_total = float(trades_df["commission"].sum()) slippage_total = float(trades_df["slippage_cost"].sum()) - cost_by_day = trades_df.groupby("date")[["commission", "slippage_cost"]].sum().sum(axis=1) + cost_by_day = ( + trades_df.groupby("date")[["commission", "slippage_cost"]].sum().sum(axis=1) + ) borrow_total = None metrics_path = artifact_dir / "metrics.json" if metrics_path.exists(): try: metrics_payload = json.loads(metrics_path.read_text(encoding="utf-8")) - borrow_total = metrics_payload.get("borrow_cost_total") or metrics_payload.get("borrow_total") + borrow_total = metrics_payload.get( + "borrow_cost_total" + ) or metrics_payload.get("borrow_total") if borrow_total is not None: borrow_total = float(borrow_total) except (OSError, ValueError, TypeError): @@ -108,7 +113,9 @@ def compute_cost_sensitivity( for multiplier in multipliers: adjustment = (multiplier - 1.0) * (per_day_cost / prev_equity.to_numpy()) adjusted_returns = returns - adjustment - metrics = _metrics_from_returns(adjusted_returns, periods_per_year=periods_per_year) + metrics = _metrics_from_returns( + adjusted_returns, periods_per_year=periods_per_year + ) cost_drag = (base_metrics["cagr"] - metrics["cagr"]) * 10_000.0 grid.append( { @@ -176,6 +183,7 @@ def _metrics_from_returns( # Metadata coverage # --------------------------------------------------------------------------- + def compute_metadata_coverage(artifact_dir: Path | str) -> Mapping[str, object]: """Compute liquidity/financing metadata coverage for executed trades.""" @@ -245,7 +253,9 @@ def _pct(mask: pd.Series) -> float | None: missing_spread=~frame["has_spread"], missing_borrow=frame["short_side"] & ~frame["has_borrow"], ) - .groupby("symbol")[["notional", "missing_adv", "missing_spread", "missing_borrow"]] + .groupby("symbol")[ + ["notional", "missing_adv", "missing_spread", "missing_borrow"] + ] .agg( notional_missing_adv=("missing_adv", "sum"), notional_missing_spread=("missing_spread", "sum"), @@ -291,6 +301,7 @@ def _pct(mask: pd.Series) -> float | None: # Helpers # --------------------------------------------------------------------------- + def _load_trades(trades_path: Path) -> pd.DataFrame: if trades_path.suffix == ".jsonl": records = [] @@ -327,7 +338,9 @@ def _load_config(config_path: Path) -> Mapping[str, object]: return yaml.safe_load(handle) or {} -def _resolve_meta_path(config_path: Path | None, config: Mapping[str, object]) -> Path | None: +def _resolve_meta_path( + config_path: Path | None, config: Mapping[str, object] +) -> Path | None: meta_value = None if isinstance(config, Mapping): meta_value = config.get("meta_path") or config.get("meta") @@ -359,7 +372,11 @@ def _from_exec(key: str, default: float | None = None) -> float | None: if isinstance(exec_block, Mapping): slippage = exec_block.get("slippage") if isinstance(slippage, Mapping): - return float(slippage.get(key)) if slippage.get(key) is not None else default + return ( + float(slippage.get(key)) + if slippage.get(key) is not None + else default + ) return default return { diff --git a/src/microalpha/reporting/spa.py b/src/microalpha/reporting/spa.py index 25a9b690..fd999460 100644 --- a/src/microalpha/reporting/spa.py +++ b/src/microalpha/reporting/spa.py @@ -118,17 +118,23 @@ def load_grid_returns(grid_path: Path) -> pd.DataFrame: raise ValueError("Grid returns data must include 'model' and 'value' columns") if "panel_id" not in frame.columns: if {"fold", "timestamp"}.issubset(frame.columns): - frame["panel_id"] = frame["fold"].astype(str) + ":" + frame["timestamp"].astype(str) + frame["panel_id"] = ( + frame["fold"].astype(str) + ":" + frame["timestamp"].astype(str) + ) else: frame["panel_id"] = frame.index.astype(str) frame["_order"] = np.arange(len(frame)) - pivot = frame.pivot_table(index="panel_id", columns="model", values="value", aggfunc="first") + pivot = frame.pivot_table( + index="panel_id", columns="model", values="value", aggfunc="first" + ) order = frame.groupby("panel_id")["_order"].min().sort_values() pivot = pivot.reindex(order.index) return pivot -def _stationary_bootstrap_indices(n: int, avg_block: int, rng: np.random.Generator) -> np.ndarray: +def _stationary_bootstrap_indices( + n: int, avg_block: int, rng: np.random.Generator +) -> np.ndarray: p = 1.0 / max(1, avg_block) indices = np.empty(n, dtype=int) current = int(rng.integers(0, n)) @@ -362,7 +368,9 @@ def write_outputs(summary: SpaSummary, json_path: Path, markdown_path: Path) -> if summary.status != "ok": lines.append(f"- **Status:** {summary.status}") if summary.status == "error": - lines.append(f"- **Error:** {summary.error or summary.reason or 'unknown error'}") + lines.append( + f"- **Error:** {summary.error or summary.reason or 'unknown error'}" + ) else: lines.append(f"- **Reason:** {summary.reason or 'invalid inputs'}") lines.append(f"- **Observations:** {summary.n_obs}") @@ -408,8 +416,18 @@ def _build_parser() -> argparse.ArgumentParser: default=Path("artifacts/analytics/spa.md"), help="Path to write SPA markdown summary", ) - parser.add_argument("--bootstrap", type=int, default=2000, help="Number of stationary bootstrap draws") - parser.add_argument("--avg-block", type=int, default=63, help="Average block length for stationary bootstrap") + parser.add_argument( + "--bootstrap", + type=int, + default=2000, + help="Number of stationary bootstrap draws", + ) + parser.add_argument( + "--avg-block", + type=int, + default=63, + help="Average block length for stationary bootstrap", + ) parser.add_argument("--seed", type=int, default=0, help="Seed for reproducibility") return parser diff --git a/src/microalpha/reporting/summary.py b/src/microalpha/reporting/summary.py index 164d3373..072b4bf6 100644 --- a/src/microalpha/reporting/summary.py +++ b/src/microalpha/reporting/summary.py @@ -662,9 +662,7 @@ def _render_cost_section(cost_path: Path) -> str | None: return "_Cost sensitivity unavailable._" lines = ["**Cost sensitivity (ex-post scaling of recorded costs)**", ""] - lines.append( - "| Multiplier | Sharpe | MaxDD | CAGR | MAR | Cost drag (bps/yr) |" - ) + lines.append("| Multiplier | Sharpe | MaxDD | CAGR | MAR | Cost drag (bps/yr) |") lines.append("| --- | ---:| ---:| ---:| ---:| ---:|") for row in grid: lines.append( diff --git a/src/microalpha/reporting/tearsheet.py b/src/microalpha/reporting/tearsheet.py index dd89f588..1272a601 100644 --- a/src/microalpha/reporting/tearsheet.py +++ b/src/microalpha/reporting/tearsheet.py @@ -185,7 +185,9 @@ def render_tearsheet( ) ax_hist.set_xlabel("Bootstrapped Sharpe") ax_hist.set_ylabel("Frequency") - p_value = metrics.get("reality_check_p_value") or metrics.get("bootstrap_p_value") + p_value = metrics.get("reality_check_p_value") or metrics.get( + "bootstrap_p_value" + ) mean = float(np.mean(bootstrap_samples)) std = float(np.std(bootstrap_samples)) textbox = [ @@ -220,7 +222,10 @@ def render_tearsheet( fig_hist.savefig(bootstrap_path, dpi=200) plt.close(fig_hist) - return {"equity_curve": equity_path.resolve(), "bootstrap_hist": bootstrap_path.resolve()} + return { + "equity_curve": equity_path.resolve(), + "bootstrap_hist": bootstrap_path.resolve(), + } def _parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: diff --git a/src/microalpha/reporting/wrds_summary.py b/src/microalpha/reporting/wrds_summary.py index d776f5a0..58ea2444 100644 --- a/src/microalpha/reporting/wrds_summary.py +++ b/src/microalpha/reporting/wrds_summary.py @@ -27,6 +27,7 @@ from microalpha.reporting.spa import SpaSummary, load_grid_returns, write_outputs from microalpha.wrds import guard_no_wrds_copy + @dataclass(frozen=True) class HeadlineMetrics: sharpe_hac: float @@ -151,7 +152,12 @@ def _format_human_currency(value: float | None) -> str: if value is None or not math.isfinite(value): return "$0" abs_val = abs(value) - suffixes = ((1_000_000_000_000, "T"), (1_000_000_000, "B"), (1_000_000, "MM"), (1_000, "K")) + suffixes = ( + (1_000_000_000_000, "T"), + (1_000_000_000, "B"), + (1_000_000, "MM"), + (1_000, "K"), + ) for threshold, suffix in suffixes: if abs_val >= threshold: return f"${value / threshold:.2f}{suffix}" @@ -323,7 +329,9 @@ def _render_cost_breakdown(cost_payload: dict | None) -> list[str] | None: def _has_trade_log(artifact_dir: Path) -> bool: - return any((artifact_dir / name).exists() for name in ("trades.jsonl", "trades.csv")) + return any( + (artifact_dir / name).exists() for name in ("trades.jsonl", "trades.csv") + ) def _load_cost_payload(artifact_dir: Path) -> dict | None: @@ -360,13 +368,17 @@ def _extract_headline( return HeadlineMetrics(sharpe, mar, max_dd, turnover, rc_p, spa_p) -def _parse_factor_table(markdown: str) -> tuple[list[dict[str, float | str]], str | None]: +def _parse_factor_table( + markdown: str, +) -> tuple[list[dict[str, float | str]], str | None]: rows: list[dict[str, float | str]] = [] for raw in markdown.splitlines(): line = raw.strip() if not line.startswith("|"): continue - if line.startswith("| ---") or ("Factor" in line and "Beta" in line and "t-stat" in line): + if line.startswith("| ---") or ( + "Factor" in line and "Beta" in line and "t-stat" in line + ): continue cells = [cell.strip() for cell in line.strip("|").split("|")] if len(cells) < 3: @@ -381,7 +393,10 @@ def _parse_factor_table(markdown: str) -> tuple[list[dict[str, float | str]], st if not rows: return [], "Factor regression table is empty; run reports/factors.py first." if all(abs(row["beta"]) < 1e-9 and abs(row["t_stat"]) < 1e-9 for row in rows): - return rows, "Factor regression table contains only zeros; rerun the regression." + return ( + rows, + "Factor regression table contains only zeros; rerun the regression.", + ) return rows, None @@ -449,7 +464,9 @@ def _coerce_float(value: object) -> float | None: return candidate if math.isfinite(candidate) else None -def _infer_spa_dimensions(artifact_dir: Path, diagnostics: list[str]) -> tuple[int, int]: +def _infer_spa_dimensions( + artifact_dir: Path, diagnostics: list[str] +) -> tuple[int, int]: grid_path = artifact_dir / "grid_returns.csv" if not grid_path.exists(): return 0, 0 @@ -651,7 +668,9 @@ def _render_spa_plot( _render_spa_placeholder(destination, f"SPA error: {reason or 'unknown error'}") return SpaRenderResult(destination, "error", reason) if status != "ok": - _render_spa_placeholder(destination, f"SPA degenerate: {reason or 'invalid inputs'}") + _render_spa_placeholder( + destination, f"SPA degenerate: {reason or 'invalid inputs'}" + ) return SpaRenderResult(destination, "degenerate", reason) candidates = spa_payload.get("candidate_stats") or [] @@ -937,10 +956,16 @@ def _write_docs_results( ) lines.append( "- Target turnover ≈ " - + (f"{turnover_target:.2%}" if isinstance(turnover_target, (int, float)) else "N/A") + + ( + f"{turnover_target:.2%}" + if isinstance(turnover_target, (int, float)) + else "N/A" + ) + f" of ADV with max {max_sector or 'N/A'} positions per sector." ) - lines.append("- Execution assumes TWAP slicing with linear+sqrt impact, 5 bps commissions, and borrow spread floor of 8 bps.") + lines.append( + "- Execution assumes TWAP slicing with linear+sqrt impact, 5 bps commissions, and borrow spread floor of 8 bps." + ) lines.append("") docs_artifacts_root = None @@ -977,10 +1002,14 @@ def render_wrds_summary( artifact_dir = artifact_dir.resolve() output_path = output_path.resolve() if docs_results and not docs_image_root: - raise SystemExit("--docs-image-root must be provided when --docs-results is set") + raise SystemExit( + "--docs-image-root must be provided when --docs-results is set" + ) metrics_path = _require_file(artifact_dir / "metrics.json", "metrics.json") - equity_png = _require_file(equity_image or (artifact_dir / "equity_curve.png"), "equity_curve.png") + equity_png = _require_file( + equity_image or (artifact_dir / "equity_curve.png"), "equity_curve.png" + ) bootstrap_png = _require_file( bootstrap_image or (artifact_dir / "bootstrap_hist.png"), "bootstrap_hist.png", @@ -1021,15 +1050,23 @@ def render_wrds_summary( run_id = manifest_payload.get("run_id") or artifact_dir.name or "wrds_run" config_path_value = manifest_payload.get("config_path") config_path = Path(config_path_value).expanduser() if config_path_value else None - config_meta = _load_config_metadata(config_path if config_path and config_path.exists() else None) - config_label = _relative_to_repo(config_path) if config_path else (config_path_value or "unknown") + config_meta = _load_config_metadata( + config_path if config_path and config_path.exists() else None + ) + config_label = ( + _relative_to_repo(config_path) + if config_path + else (config_path_value or "unknown") + ) train_start, test_end, fold_count = _load_folds_metadata(folds_path) unsafe_lines = _unsafe_banner(manifest_payload) analytics_dir = (analytics_plots or Path("artifacts/plots")).expanduser().resolve() ic_plot = _require_file(analytics_dir / f"{run_id}_ic_ir.png", "IC/IR plot") decile_plot = _require_file(analytics_dir / f"{run_id}_deciles.png", "deciles plot") - beta_plot = _require_file(analytics_dir / f"{run_id}_rolling_betas.png", "rolling betas plot") + beta_plot = _require_file( + analytics_dir / f"{run_id}_rolling_betas.png", "rolling betas plot" + ) spa_plot = spa_result.path if metrics_json_out: @@ -1040,7 +1077,9 @@ def render_wrds_summary( metrics_copy["spa_status"] = spa_status if spa_skip_reason: metrics_copy["spa_skip_reason"] = spa_skip_reason - metrics_json_out.write_text(json.dumps(metrics_copy, indent=2) + "\n", encoding="utf-8") + metrics_json_out.write_text( + json.dumps(metrics_copy, indent=2) + "\n", encoding="utf-8" + ) if spa_json_out: spa_json_out = spa_json_out.expanduser().resolve() spa_json_out.parent.mkdir(parents=True, exist_ok=True) @@ -1081,7 +1120,9 @@ def render_wrds_summary( lines.extend(_render_non_degenerate(manifest_payload)) if spa_status != "ok": lines.extend(_spa_failure_banner(spa_status, spa_skip_reason)) - headline_title = "## Headline Metrics" if spa_status == "ok" else "## Headline Metrics (blocked)" + headline_title = ( + "## Headline Metrics" if spa_status == "ok" else "## Headline Metrics (blocked)" + ) lines.extend([headline_title, ""]) lines.extend(_render_table(headline)) lines.append("") @@ -1126,11 +1167,17 @@ def render_wrds_summary( lines.append("## Visuals") lines.append("") - lines.append(f"![Equity Curve]({_relpath(image_for_summary['equity'], output_path)})") + lines.append( + f"![Equity Curve]({_relpath(image_for_summary['equity'], output_path)})" + ) lines.append("") - lines.append(f"![Bootstrap Sharpe Histogram]({_relpath(image_for_summary['bootstrap'], output_path)})") + lines.append( + f"![Bootstrap Sharpe Histogram]({_relpath(image_for_summary['bootstrap'], output_path)})" + ) lines.append("") - lines.append(f"![SPA Comparator t-stats]({_relpath(image_for_summary['spa'], output_path)})") + lines.append( + f"![SPA Comparator t-stats]({_relpath(image_for_summary['spa'], output_path)})" + ) lines.append("") lines.append("## Hansen SPA Summary") @@ -1186,7 +1233,9 @@ def render_wrds_summary( def _build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("artifact_dir", type=Path, help="Artifact directory for WRDS run") + parser.add_argument( + "artifact_dir", type=Path, help="Artifact directory for WRDS run" + ) parser.add_argument( "--output", type=Path, @@ -1199,7 +1248,12 @@ def _build_parser() -> argparse.ArgumentParser: default=None, help="Path to the FF5+MOM markdown table output by reports/factors.py", ) - parser.add_argument("--docs-results", type=Path, default=None, help="Optional docs/results_wrds.md destination") + parser.add_argument( + "--docs-results", + type=Path, + default=None, + help="Optional docs/results_wrds.md destination", + ) parser.add_argument( "--docs-image-root", type=Path, diff --git a/src/microalpha/runner.py b/src/microalpha/runner.py index 38278065..ec9b7fb1 100644 --- a/src/microalpha/runner.py +++ b/src/microalpha/runner.py @@ -26,8 +26,6 @@ ImplementationShortfall, KyleLambda, LOBExecution, -) -from .execution import ( SquareRootImpact as SquareRootImpactExecutor, ) from .execution_safety import evaluate_execution_safety @@ -35,13 +33,9 @@ from .logging import JsonlWriter from .manifest import ( build as build_manifest, -) -from .manifest import ( extract_config_summary, generate_run_id, resolve_git_sha, -) -from .manifest import ( write as write_manifest, ) from .market_metadata import load_symbol_meta @@ -52,10 +46,8 @@ from .slippage import ( LinearImpact, LinearPlusSqrtImpact, - VolumeSlippageModel, -) -from .slippage import ( SquareRootImpact as SquareRootImpactSlippage, + VolumeSlippageModel, ) from .strategies.breakout import BreakoutStrategy from .strategies.cs_momentum import CrossSectionalMomentum @@ -177,7 +169,10 @@ def run_from_config( ) strategy_params["symbols"] = universe_symbols strategy_params.setdefault("warmup_history", None) - if strategy_name == "CrossSectionalMomentum" and "symbols" not in strategy_params: + if ( + strategy_name == "CrossSectionalMomentum" + and "symbols" not in strategy_params + ): strategy_params["symbols"] = config.get("symbols") or [symbol] symbols = strategy_params.get("symbols") or config.get("symbols") or [symbol] @@ -487,7 +482,9 @@ def _persist_order_flow_diagnostics( try: order_flow.merge_filter_diagnostics(filter_diagnostics) except Exception as exc: # pragma: no cover - diagnostics should not fail run - order_flow.record_error(f"merge_filter_diagnostics_error: {type(exc).__name__}: {exc}") + order_flow.record_error( + f"merge_filter_diagnostics_error: {type(exc).__name__}: {exc}" + ) payload = order_flow.payload() path = artifacts_dir / "order_flow_diagnostics.json" with path.open("w", encoding="utf-8") as handle: diff --git a/src/microalpha/slippage.py b/src/microalpha/slippage.py index e59708a4..2a01c32f 100644 --- a/src/microalpha/slippage.py +++ b/src/microalpha/slippage.py @@ -50,7 +50,10 @@ class VolumeSlippageModel(SlippageModel): """Legacy quadratic volume model retained for backward compatibility.""" def __init__( - self, price_impact: float = 0.0001, *, metadata: Mapping[str, SymbolMeta] | None = None + self, + price_impact: float = 0.0001, + *, + metadata: Mapping[str, SymbolMeta] | None = None, ) -> None: super().__init__(metadata=metadata) self.price_impact = float(price_impact) diff --git a/src/microalpha/strategies/flagship_mom.py b/src/microalpha/strategies/flagship_mom.py index 62307c80..66a9b6dd 100644 --- a/src/microalpha/strategies/flagship_mom.py +++ b/src/microalpha/strategies/flagship_mom.py @@ -73,8 +73,8 @@ def __init__( self.universe = self._load_universe(universe_path) self.universe_dates = sorted(self.universe.keys()) self.max_history = ( - (self.lookback_months + self.skip_months + 2) * TRADING_DAYS_MONTH - ) + self.lookback_months + self.skip_months + 2 + ) * TRADING_DAYS_MONTH self._filter_diagnostics: List[Dict[str, float | int | str]] = [] # ------------------------------------------------------------------ @@ -177,7 +177,7 @@ def on_market(self, event) -> List[SignalEvent]: history = self.price_history.setdefault(event.symbol, []) history.append(float(event.price)) if len(history) > self.max_history: - del history[:-self.max_history] + del history[: -self.max_history] timestamp = pd.to_datetime(event.timestamp) period_end = ( @@ -217,9 +217,7 @@ def _rebalance( return self._build_signals(long_sel, short_sel, event_timestamp, period_end) # ------------------------------------------------------------------ - def _universe_snapshot( - self, period_end: pd.Timestamp - ) -> Optional[pd.DataFrame]: + def _universe_snapshot(self, period_end: pd.Timestamp) -> Optional[pd.DataFrame]: eligible = [date for date in self.universe_dates if date <= period_end] if not eligible: return None @@ -273,7 +271,9 @@ def _build_score_frame( records.append( { "symbol": symbol_str, - "sector": str(row.get("sector", self.sector_map.get(symbol_str, "UNKNOWN"))), + "sector": str( + row.get("sector", self.sector_map.get(symbol_str, "UNKNOWN")) + ), "momentum": float(momentum), "adv": adv, "close": float(row.get("close", prices[-1])), diff --git a/src/microalpha/walkforward.py b/src/microalpha/walkforward.py index 157dc0e4..5784056f 100644 --- a/src/microalpha/walkforward.py +++ b/src/microalpha/walkforward.py @@ -29,23 +29,19 @@ SquareRootImpact, ) from .execution_safety import evaluate_execution_safety +from .integrity import evaluate_portfolio_integrity from .logging import JsonlWriter from .manifest import ( build as build_manifest, -) -from .manifest import ( extract_config_summary, generate_run_id, resolve_git_sha, -) -from .manifest import ( write as write_manifest, ) from .market_metadata import load_symbol_meta from .metrics import compute_metrics from .order_flow import OrderFlowDiagnostics, infer_non_degenerate_reason from .portfolio import Portfolio -from .integrity import evaluate_portfolio_integrity from .risk_stats import block_bootstrap from .runner import ( persist_config, @@ -97,9 +93,7 @@ def _non_degenerate_reasons( if cfg is None: return reasons if cfg.min_trades is not None and num_trades < cfg.min_trades: - reasons.append( - f"num_trades {num_trades} < min_trades {int(cfg.min_trades)}" - ) + reasons.append(f"num_trades {num_trades} < min_trades {int(cfg.min_trades)}") if cfg.min_turnover is not None and turnover < cfg.min_turnover: reasons.append( f"total_turnover {turnover:.4f} < min_turnover {float(cfg.min_turnover):.4f}" @@ -277,9 +271,9 @@ def run_walk_forward( cs_symbols = [str(sym).upper() for sym in raw_symbols] base_params.setdefault("symbols", cs_symbols) else: - universe_path = base_params.get("universe_path") or cfg.template.strategy.params.get( + universe_path = base_params.get( "universe_path" - ) + ) or cfg.template.strategy.params.get("universe_path") if universe_path is None: raise ValueError( "FlagshipMomentumStrategy requires 'universe_path' defined in template params" @@ -335,8 +329,7 @@ def run_walk_forward( try: while ( - current_date - + pd.Timedelta(days=training_days + testing_days + 1) + current_date + pd.Timedelta(days=training_days + testing_days + 1) <= selection_end ): train_start = current_date @@ -418,9 +411,7 @@ def run_walk_forward( symbol=symbol, **_strategy_kwargs(best_params, warmup_prices) ) order_flow = ( - OrderFlowDiagnostics() - if cfg.template.order_flow_diagnostics - else None + OrderFlowDiagnostics() if cfg.template.order_flow_diagnostics else None ) portfolio = _build_portfolio( data_handler, @@ -458,7 +449,9 @@ def run_walk_forward( if hasattr(strategy, "get_filter_diagnostics"): try: filter_diagnostics = strategy.get_filter_diagnostics() - except Exception as exc: # pragma: no cover - diagnostics should not fail the run + except ( + Exception + ) as exc: # pragma: no cover - diagnostics should not fail the run filter_diagnostics = { "error": f"{type(exc).__name__}: {exc}", } @@ -635,9 +628,7 @@ def run_walk_forward( non_degenerate_cfg is not None and non_degenerate_cfg.min_turnover is not None ): - criteria.append( - f"min_turnover={float(non_degenerate_cfg.min_turnover)}" - ) + criteria.append(f"min_turnover={float(non_degenerate_cfg.min_turnover)}") criteria_text = ", ".join(criteria) if criteria else "unspecified" selection_failure_reason = ( "Non-degenerate constraints rejected all candidates " @@ -716,9 +707,7 @@ def run_walk_forward( str(artifacts_dir / "holdout_trades.jsonl") ) holdout_order_flow = ( - OrderFlowDiagnostics() - if cfg.template.order_flow_diagnostics - else None + OrderFlowDiagnostics() if cfg.template.order_flow_diagnostics else None ) holdout_portfolio = _build_portfolio( data_handler, @@ -757,8 +746,12 @@ def run_walk_forward( holdout_filter_diagnostics: Dict[str, Any] | None = None if hasattr(holdout_strategy, "get_filter_diagnostics"): try: - holdout_filter_diagnostics = holdout_strategy.get_filter_diagnostics() - except Exception as exc: # pragma: no cover - diagnostics should not fail the run + holdout_filter_diagnostics = ( + holdout_strategy.get_filter_diagnostics() + ) + except ( + Exception + ) as exc: # pragma: no cover - diagnostics should not fail the run holdout_filter_diagnostics = { "error": f"{type(exc).__name__}: {exc}", } @@ -872,7 +865,9 @@ def run_walk_forward( holdout_loss_trades = 0 for trade in holdout_trades: try: - holdout_total_commission += float(trade.get("commission", 0.0) or 0.0) + holdout_total_commission += float( + trade.get("commission", 0.0) or 0.0 + ) holdout_total_slippage += abs( float(trade.get("slippage", 0.0) or 0.0) ) * abs(float(trade.get("qty", 0.0) or 0.0)) @@ -901,7 +896,9 @@ def run_walk_forward( holdout_win_trades / holdout_win_denom if holdout_win_denom > 0 else 0.0 ) holdout_avg_trade_notional = ( - holdout_trade_notional / holdout_num_trades if holdout_num_trades > 0 else 0.0 + holdout_trade_notional / holdout_num_trades + if holdout_num_trades > 0 + else 0.0 ) holdout_metrics.update( { @@ -1165,7 +1162,9 @@ def _optimise_parameters( if hasattr(strategy, "get_filter_diagnostics"): try: filter_diagnostics = strategy.get_filter_diagnostics() - except Exception as exc: # pragma: no cover - diagnostics should not fail tuning + except ( + Exception + ) as exc: # pragma: no cover - diagnostics should not fail tuning filter_diagnostics = { "error": f"{type(exc).__name__}: {exc}", } @@ -1177,9 +1176,7 @@ def _optimise_parameters( "model": _format_param_label(params), "params": dict(params), "num_trades": len(getattr(portfolio, "trades", None) or []), - "turnover": float( - getattr(portfolio, "total_turnover", 0.0) or 0.0 - ), + "turnover": float(getattr(portfolio, "total_turnover", 0.0) or 0.0), "reasons": ["empty_equity_curve"], "filter_diagnostics": filter_diagnostics, "order_flow_diagnostics": order_flow_payload, @@ -1201,9 +1198,7 @@ def _optimise_parameters( "model": _format_param_label(params), "params": dict(params), "num_trades": len(getattr(portfolio, "trades", None) or []), - "turnover": float( - getattr(portfolio, "total_turnover", 0.0) or 0.0 - ), + "turnover": float(getattr(portfolio, "total_turnover", 0.0) or 0.0), "reasons": exclusion_reasons, "filter_diagnostics": filter_diagnostics, "order_flow_diagnostics": order_flow_payload, @@ -1338,9 +1333,7 @@ def _build_executor( if exec_cfg.queue_coefficient is not None: kwargs["queue_coefficient"] = float(exec_cfg.queue_coefficient) if exec_cfg.queue_passive_multiplier is not None: - kwargs["queue_passive_multiplier"] = float( - exec_cfg.queue_passive_multiplier - ) + kwargs["queue_passive_multiplier"] = float(exec_cfg.queue_passive_multiplier) if exec_cfg.queue_seed is not None: kwargs["queue_seed"] = int(exec_cfg.queue_seed) if exec_cfg.queue_randomize is not None: @@ -1524,9 +1517,7 @@ def _aggregate_selection_summary( aggregated: Dict[str, Dict[str, Any]] = {} for fold_index, summary in enumerate(grid_summaries): for entry in summary: - model = entry.get("model") or _format_param_label( - entry.get("params", {}) - ) + model = entry.get("model") or _format_param_label(entry.get("params", {})) params = dict(entry.get("params") or {}) sharpe = float(entry.get("sharpe_ratio", 0.0) or 0.0) cagr = float(entry.get("cagr", 0.0) or 0.0) @@ -1556,9 +1547,9 @@ def _aggregate_selection_summary( { "model": model, "params": record.get("params") or {}, - "mean_sharpe": float(np.mean(sharpe_vals)) - if sharpe_vals - else float("-inf"), + "mean_sharpe": ( + float(np.mean(sharpe_vals)) if sharpe_vals else float("-inf") + ), "mean_cagr": float(np.mean(cagr_vals)) if cagr_vals else 0.0, "mean_ann_vol": float(np.mean(ann_vol_vals)) if ann_vol_vals else 0.0, "num_folds": int(len(sharpe_vals)), diff --git a/src/microalpha/wrds/__init__.py b/src/microalpha/wrds/__init__.py index 681ee164..e17e6362 100644 --- a/src/microalpha/wrds/__init__.py +++ b/src/microalpha/wrds/__init__.py @@ -154,9 +154,7 @@ def guard_no_wrds_copy(path: Path, *, operation: str = "copy") -> None: """Raise if attempting to copy data directly from WRDS_DATA_ROOT.""" if is_wrds_path(path): - raise ValueError( - f"Refusing to {operation} file from WRDS_DATA_ROOT: {path}" - ) + raise ValueError(f"Refusing to {operation} file from WRDS_DATA_ROOT: {path}") __all__ = [ diff --git a/tests/test_build_wrds_signals.py b/tests/test_build_wrds_signals.py index a86938db..7db81b51 100644 --- a/tests/test_build_wrds_signals.py +++ b/tests/test_build_wrds_signals.py @@ -8,17 +8,77 @@ def test_build_signals_filters_and_computes_scores(tmp_path) -> None: universe = tmp_path / "universe.csv" rows = [ - {"symbol": "AAA", "date": "2020-01-31", "close": 100.0, "adv_20": 5_000_000, "sector": "Tech"}, - {"symbol": "AAA", "date": "2020-02-29", "close": 110.0, "adv_20": 5_500_000, "sector": "Tech"}, - {"symbol": "AAA", "date": "2020-03-31", "close": 120.0, "adv_20": 6_000_000, "sector": "Tech"}, - {"symbol": "AAA", "date": "2020-04-30", "close": 130.0, "adv_20": 6_500_000, "sector": "Tech"}, - {"symbol": "AAA", "date": "2020-05-31", "close": 135.0, "adv_20": 6_700_000, "sector": "Tech"}, - {"symbol": "BBB", "date": "2020-01-31", "close": 50.0, "adv_20": 1_000_000, "sector": "Energy"}, - {"symbol": "BBB", "date": "2020-02-29", "close": 55.0, "adv_20": 1_200_000, "sector": "Energy"}, - {"symbol": "BBB", "date": "2020-03-31", "close": 60.0, "adv_20": 1_300_000, "sector": "Energy"}, - {"symbol": "BBB", "date": "2020-04-30", "close": 65.0, "adv_20": 1_400_000, "sector": "Energy"}, - {"symbol": "BBB", "date": "2020-05-31", "close": 70.0, "adv_20": 1_500_000, "sector": "Energy"}, - ] + { + "symbol": "AAA", + "date": "2020-01-31", + "close": 100.0, + "adv_20": 5_000_000, + "sector": "Tech", + }, + { + "symbol": "AAA", + "date": "2020-02-29", + "close": 110.0, + "adv_20": 5_500_000, + "sector": "Tech", + }, + { + "symbol": "AAA", + "date": "2020-03-31", + "close": 120.0, + "adv_20": 6_000_000, + "sector": "Tech", + }, + { + "symbol": "AAA", + "date": "2020-04-30", + "close": 130.0, + "adv_20": 6_500_000, + "sector": "Tech", + }, + { + "symbol": "AAA", + "date": "2020-05-31", + "close": 135.0, + "adv_20": 6_700_000, + "sector": "Tech", + }, + { + "symbol": "BBB", + "date": "2020-01-31", + "close": 50.0, + "adv_20": 1_000_000, + "sector": "Energy", + }, + { + "symbol": "BBB", + "date": "2020-02-29", + "close": 55.0, + "adv_20": 1_200_000, + "sector": "Energy", + }, + { + "symbol": "BBB", + "date": "2020-03-31", + "close": 60.0, + "adv_20": 1_300_000, + "sector": "Energy", + }, + { + "symbol": "BBB", + "date": "2020-04-30", + "close": 65.0, + "adv_20": 1_400_000, + "sector": "Energy", + }, + { + "symbol": "BBB", + "date": "2020-05-31", + "close": 70.0, + "adv_20": 1_500_000, + "sector": "Energy", + }, + ] pd.DataFrame(rows).to_csv(universe, index=False) output = tmp_path / "signals.csv" diff --git a/tests/test_degeneracy_constraints.py b/tests/test_degeneracy_constraints.py index e24bccab..670a005b 100644 --- a/tests/test_degeneracy_constraints.py +++ b/tests/test_degeneracy_constraints.py @@ -28,8 +28,12 @@ def _write_prices(tmp_path: Path) -> Path: return data_dir -def test_non_degenerate_rejects_zero_trade_selection(tmp_path: Path, monkeypatch) -> None: - monkeypatch.setitem(walkforward.STRATEGY_MAPPING, "NoTradeStrategy", NoTradeStrategy) +def test_non_degenerate_rejects_zero_trade_selection( + tmp_path: Path, monkeypatch +) -> None: + monkeypatch.setitem( + walkforward.STRATEGY_MAPPING, "NoTradeStrategy", NoTradeStrategy + ) data_dir = _write_prices(tmp_path) config = { @@ -54,5 +58,9 @@ def test_non_degenerate_rejects_zero_trade_selection(tmp_path: Path, monkeypatch cfg_path = tmp_path / "wfv_no_trade.yaml" cfg_path.write_text(yaml.safe_dump(config), encoding="utf-8") - with pytest.raises(ValueError, match="Non-degenerate constraints rejected all candidates"): - run_walk_forward(str(cfg_path), override_artifacts_dir=str(tmp_path / "artifacts")) + with pytest.raises( + ValueError, match="Non-degenerate constraints rejected all candidates" + ): + run_walk_forward( + str(cfg_path), override_artifacts_dir=str(tmp_path / "artifacts") + ) diff --git a/tests/test_docs_links.py b/tests/test_docs_links.py index 62152f87..28ee5725 100644 --- a/tests/test_docs_links.py +++ b/tests/test_docs_links.py @@ -3,11 +3,14 @@ from pathlib import Path -def test_readme_references_docs_and_artifacts() -> None: +def test_public_install_and_artifact_links_are_safe() -> None: readme = Path("README.md").read_text() + docs_home = Path("docs/index.md").read_text() - assert "mateobodon.github.io/microalpha" in readme - assert "[![Docs]" in readme + assert "git clone https://github.com/MateoBodon/microalpha.git" in readme + assert "git clone https://github.com/MateoBodon/microalpha.git" in docs_home + assert "Do not use `pip install microalpha`" in docs_home + assert "\n pip install microalpha\n" not in docs_home for rel_path in ("artifacts/sample_flagship", "artifacts/sample_wfv"): assert rel_path in readme diff --git a/tests/test_flagship_filter_diagnostics.py b/tests/test_flagship_filter_diagnostics.py index bae5f993..a77b67ef 100644 --- a/tests/test_flagship_filter_diagnostics.py +++ b/tests/test_flagship_filter_diagnostics.py @@ -1,6 +1,9 @@ import pandas as pd -from microalpha.strategies.flagship_mom import FlagshipMomentumStrategy, TRADING_DAYS_MONTH +from microalpha.strategies.flagship_mom import ( + TRADING_DAYS_MONTH, + FlagshipMomentumStrategy, +) def test_flagship_filter_diagnostics_counts(tmp_path): diff --git a/tests/test_flagship_momentum.py b/tests/test_flagship_momentum.py index d701b368..c2709814 100644 --- a/tests/test_flagship_momentum.py +++ b/tests/test_flagship_momentum.py @@ -80,19 +80,29 @@ def test_flagship_sector_normalised_signals(tmp_path: Path) -> None: signals = [] for sym in symbols: price = history[sym][-1] + (0.3 if sym in {"AAA", "BBB"} else -0.3) - signals.extend(strategy.on_market(MarketEvent(int(ts_mar.value), sym, price, 1_000_000.0))) + signals.extend( + strategy.on_market(MarketEvent(int(ts_mar.value), sym, price, 1_000_000.0)) + ) assert {s.symbol for s in signals if s.side == "LONG"} == {"AAA", "BBB"} assert {s.symbol for s in signals if s.side == "SHORT"} == {"AAB", "BBC"} assert all(s.meta and "sector_z" in s.meta for s in signals if s.side != "EXIT") assert all(s.meta and s.meta.get("turnover_heat") == 0.08 for s in signals) - assert all(s.meta and s.meta.get("sleeve") in {"long", "short"} for s in signals if s.side != "EXIT") - assert all("weight" in (s.meta or {}) for s in signals if s.side in {"LONG", "SHORT"}) + assert all( + s.meta and s.meta.get("sleeve") in {"long", "short"} + for s in signals + if s.side != "EXIT" + ) + assert all( + "weight" in (s.meta or {}) for s in signals if s.side in {"LONG", "SHORT"} + ) long_budget = sum(s.meta["weight"] for s in signals if s.side == "LONG") short_budget = sum(abs(s.meta["weight"]) for s in signals if s.side == "SHORT") assert long_budget > 0 assert short_budget > 0 - assert math.isclose(long_budget + short_budget, strategy.total_risk_budget, rel_tol=1e-6) + assert math.isclose( + long_budget + short_budget, strategy.total_risk_budget, rel_tol=1e-6 + ) def test_flagship_requires_full_warmup_before_signalling(tmp_path: Path) -> None: diff --git a/tests/test_gpt_bundle_dirty.py b/tests/test_gpt_bundle_dirty.py index 45096284..17845563 100644 --- a/tests/test_gpt_bundle_dirty.py +++ b/tests/test_gpt_bundle_dirty.py @@ -19,7 +19,9 @@ def fake_stash_push(label: str) -> str: mod._git_status_porcelain = fake_status mod._stash_push = fake_stash_push - status_before, stash_ref, dirty = mod._prepare_worktree("temp: gpt_bundle", no_stash=False) + status_before, stash_ref, dirty = mod._prepare_worktree( + "temp: gpt_bundle", no_stash=False + ) assert dirty is True assert status_before.strip() == "M dirty.txt" assert stash_ref == "stash@{0}" diff --git a/tests/test_metrics_hac.py b/tests/test_metrics_hac.py index be2c7431..1a63d523 100644 --- a/tests/test_metrics_hac.py +++ b/tests/test_metrics_hac.py @@ -5,20 +5,20 @@ def test_compute_metrics_includes_hac_fields(monkeypatch) -> None: equity_records = [ - {"timestamp": i, "equity": 100.0 + i, "exposure": 0.5} - for i in range(40) + {"timestamp": i, "equity": 100.0 + i, "exposure": 0.5} for i in range(40) ] monkeypatch.setenv("METRICS_HAC_LAGS", "5") metrics = compute_metrics(equity_records, turnover=123.0, rf=0.02) assert metrics["sharpe_ratio_se"] >= 0.0 assert metrics["sharpe_ratio_ci_low"] <= metrics["sharpe_ratio_ci_high"] - assert metrics["sharpe_ratio_tstat"] == metrics["sharpe_ratio"] / metrics["sharpe_ratio_se"] + assert ( + metrics["sharpe_ratio_tstat"] + == metrics["sharpe_ratio"] / metrics["sharpe_ratio_se"] + ) assert metrics["sharpe_hac_lags"] == 5.0 monkeypatch.delenv("METRICS_HAC_LAGS") - metrics_override = compute_metrics( - equity_records, turnover=123.0, hac_lags=7 - ) + metrics_override = compute_metrics(equity_records, turnover=123.0, hac_lags=7) assert metrics_override["sharpe_hac_lags"] == 7.0 crash_records = [ diff --git a/tests/test_multiasset_data_handler.py b/tests/test_multiasset_data_handler.py index 2fde1d9a..aceff423 100644 --- a/tests/test_multiasset_data_handler.py +++ b/tests/test_multiasset_data_handler.py @@ -112,7 +112,9 @@ def test_stream_matches_baseline_logic(tmp_path: Path) -> None: _write_csv(data_dir / f"{sym}.csv", list(dates), prices) for mode in ("ffill", "exact"): - handler = MultiCsvDataHandler(csv_dir=data_dir, symbols=list(symbol_data), mode=mode) + handler = MultiCsvDataHandler( + csv_dir=data_dir, symbols=list(symbol_data), mode=mode + ) handler.set_date_range(idx[0], idx[-1]) expected = _baseline_events(handler) observed = _collect_events(handler) diff --git a/tests/test_order_flow_diagnostics.py b/tests/test_order_flow_diagnostics.py index cb4c80d0..0ab5a043 100644 --- a/tests/test_order_flow_diagnostics.py +++ b/tests/test_order_flow_diagnostics.py @@ -8,8 +8,8 @@ from microalpha.order_flow import OrderFlowDiagnostics from microalpha.portfolio import Portfolio from microalpha.strategies.flagship_mom import ( - FlagshipMomentumStrategy, TRADING_DAYS_MONTH, + FlagshipMomentumStrategy, ) diff --git a/tests/test_portfolio_risk_caps.py b/tests/test_portfolio_risk_caps.py index 905fb181..a2f1ff45 100644 --- a/tests/test_portfolio_risk_caps.py +++ b/tests/test_portfolio_risk_caps.py @@ -9,7 +9,9 @@ class _StubDataHandler: def __init__(self, price: float = 100.0) -> None: self.price = price - def get_latest_price(self, symbol: str, timestamp: int): # pragma: no cover - trivial + def get_latest_price( + self, symbol: str, timestamp: int + ): # pragma: no cover - trivial return self.price diff --git a/tests/test_portfolio_weight_sizing.py b/tests/test_portfolio_weight_sizing.py index c7dab272..354ee86d 100644 --- a/tests/test_portfolio_weight_sizing.py +++ b/tests/test_portfolio_weight_sizing.py @@ -19,9 +19,7 @@ def test_portfolio_uses_signal_weight_for_sizing(): portfolio = Portfolio(data_handler=_StubData(), initial_cash=1000.0) portfolio.on_market(MarketEvent(1, "AAA", 50.0, 1_000)) - long_signal = SignalEvent( - 1, "AAA", "LONG", meta={"weight": 0.2, "reason": "test"} - ) + long_signal = SignalEvent(1, "AAA", "LONG", meta={"weight": 0.2, "reason": "test"}) orders = list(portfolio.on_signal(long_signal)) assert orders and orders[0].qty == 4 # 0.2 * 1000 / 50 assert orders[0].side == "BUY" diff --git a/tests/test_reporting_analytics.py b/tests/test_reporting_analytics.py index c2d55a25..ef9112cc 100644 --- a/tests/test_reporting_analytics.py +++ b/tests/test_reporting_analytics.py @@ -17,14 +17,16 @@ def test_compute_ic_series_matches_spearman() -> None: signals = pd.DataFrame( { - "as_of": pd.to_datetime([ - "2024-01-01", - "2024-01-01", - "2024-01-01", - "2024-01-02", - "2024-01-02", - "2024-01-02", - ]), + "as_of": pd.to_datetime( + [ + "2024-01-01", + "2024-01-01", + "2024-01-01", + "2024-01-02", + "2024-01-02", + "2024-01-02", + ] + ), "symbol": ["A", "B", "C", "A", "B", "C"], "score": [1.0, 2.0, 3.0, 3.0, 2.0, 1.0], "forward_return": [0.1, 0.2, 0.3, 0.1, 0.2, 0.4], diff --git a/tests/test_reporting_robustness.py b/tests/test_reporting_robustness.py index a38853d1..0aa55385 100644 --- a/tests/test_reporting_robustness.py +++ b/tests/test_reporting_robustness.py @@ -16,7 +16,9 @@ def _write_equity(tmp_path: Path) -> Path: dates = pd.date_range("2025-01-01", periods=5, freq="D") - equity = pd.Series([1_000_000, 1_010_000, 1_005_000, 1_020_000, 1_025_000], index=dates) + equity = pd.Series( + [1_000_000, 1_010_000, 1_005_000, 1_020_000, 1_025_000], index=dates + ) df = pd.DataFrame( { "timestamp": dates.view("int64"), @@ -52,8 +54,22 @@ def test_cost_sensitivity_generates_grid(tmp_path: Path) -> None: _write_equity(artifact_dir) trades = [ - {"timestamp": pd.Timestamp("2025-01-02").value, "symbol": "AAA", "qty": 100, "price": 10.0, "commission": 5.0, "slippage": 0.02}, - {"timestamp": pd.Timestamp("2025-01-03").value, "symbol": "AAA", "qty": -50, "price": 11.0, "commission": 2.5, "slippage": 0.01}, + { + "timestamp": pd.Timestamp("2025-01-02").value, + "symbol": "AAA", + "qty": 100, + "price": 10.0, + "commission": 5.0, + "slippage": 0.02, + }, + { + "timestamp": pd.Timestamp("2025-01-03").value, + "symbol": "AAA", + "qty": -50, + "price": 11.0, + "commission": 2.5, + "slippage": 0.01, + }, ] trades_path = artifact_dir / "trades.jsonl" trades_path.write_text("\n".join(json.dumps(t) for t in trades), encoding="utf-8") @@ -92,15 +108,33 @@ def test_metadata_coverage_uses_meta_csv(tmp_path: Path) -> None: cfg_path.write_text(yaml.safe_dump(config), encoding="utf-8") trades = [ - {"timestamp": pd.Timestamp("2025-01-02").value, "symbol": "AAA", "qty": 100, "price": 10.0, "commission": 0.0, "slippage": 0.0}, - {"timestamp": pd.Timestamp("2025-01-03").value, "symbol": "BBB", "qty": -50, "price": 11.0, "commission": 0.0, "slippage": 0.0}, + { + "timestamp": pd.Timestamp("2025-01-02").value, + "symbol": "AAA", + "qty": 100, + "price": 10.0, + "commission": 0.0, + "slippage": 0.0, + }, + { + "timestamp": pd.Timestamp("2025-01-03").value, + "symbol": "BBB", + "qty": -50, + "price": 11.0, + "commission": 0.0, + "slippage": 0.0, + }, ] - (artifact_dir / "trades.jsonl").write_text("\n".join(json.dumps(t) for t in trades), encoding="utf-8") + (artifact_dir / "trades.jsonl").write_text( + "\n".join(json.dumps(t) for t in trades), encoding="utf-8" + ) coverage = compute_metadata_coverage(artifact_dir) assert coverage["meta_source"].endswith("meta.csv") cov = coverage["coverage"] - assert 0.6 <= cov["pct_notional_with_adv"] <= 0.7 # majority of notional has metadata + assert ( + 0.6 <= cov["pct_notional_with_adv"] <= 0.7 + ) # majority of notional has metadata assert cov["pct_short_notional_with_borrow_fee"] == 0.0 # BBB missing borrow meta assert coverage["defaults"]["default_adv"] == 2_000_000.0 @@ -110,7 +144,10 @@ def test_summary_includes_robustness_section(tmp_path: Path) -> None: artifact_dir.mkdir() _write_equity(artifact_dir) (artifact_dir / "trades.jsonl").write_text("", encoding="utf-8") - (artifact_dir / "metrics.json").write_text(json.dumps({"sharpe_ratio": 1.0, "max_drawdown": 0.1, "total_turnover": 0.0}), encoding="utf-8") + (artifact_dir / "metrics.json").write_text( + json.dumps({"sharpe_ratio": 1.0, "max_drawdown": 0.1, "total_turnover": 0.0}), + encoding="utf-8", + ) (artifact_dir / "bootstrap.json").write_text("[]", encoding="utf-8") write_robustness_artifacts(artifact_dir) diff --git a/tests/test_reporting_spa.py b/tests/test_reporting_spa.py index bceb7488..58e641aa 100644 --- a/tests/test_reporting_spa.py +++ b/tests/test_reporting_spa.py @@ -14,8 +14,24 @@ def test_load_grid_returns_pivots_panel(tmp_path) -> None: for fold in range(2): for t in range(3): panel_id = f"{fold}:{t}" - rows.append({"fold": fold, "timestamp": t, "model": "A", "value": 0.01 * (t + fold), "panel_id": panel_id}) - rows.append({"fold": fold, "timestamp": t, "model": "B", "value": 0.008 * (t + fold), "panel_id": panel_id}) + rows.append( + { + "fold": fold, + "timestamp": t, + "model": "A", + "value": 0.01 * (t + fold), + "panel_id": panel_id, + } + ) + rows.append( + { + "fold": fold, + "timestamp": t, + "model": "B", + "value": 0.008 * (t + fold), + "panel_id": panel_id, + } + ) pd.DataFrame(rows).to_csv(path, index=False) pivot = load_grid_returns(path) assert list(pivot.columns) == ["A", "B"] diff --git a/tests/test_time_ordering.py b/tests/test_time_ordering.py index f1fa5320..d0c4b34b 100644 --- a/tests/test_time_ordering.py +++ b/tests/test_time_ordering.py @@ -1,7 +1,6 @@ import numpy as np import pytest -from hypothesis import HealthCheck, given, settings -from hypothesis import strategies as st +from hypothesis import HealthCheck, given, settings, strategies as st from microalpha.engine import Engine from microalpha.events import LookaheadError, MarketEvent diff --git a/tests/test_walkforward.py b/tests/test_walkforward.py index 56f4041e..b33792ab 100644 --- a/tests/test_walkforward.py +++ b/tests/test_walkforward.py @@ -7,9 +7,9 @@ import pytest import yaml +import microalpha.walkforward as walkforward from microalpha.events import SignalEvent from microalpha.walkforward import run_walk_forward -import microalpha.walkforward as walkforward def test_sample_walkforward_produces_folds(tmp_path: Path) -> None: @@ -142,14 +142,10 @@ def test_holdout_selection_excludes_holdout_data(tmp_path: Path, monkeypatch) -> HoldoutDirectionalStrategy, ) cfg_path = _write_holdout_fixture(tmp_path, include_holdout=True) - no_holdout_cfg_path = _write_holdout_fixture( - tmp_path, include_holdout=False - ) + no_holdout_cfg_path = _write_holdout_fixture(tmp_path, include_holdout=False) artifacts_dir = tmp_path / "artifacts" - result = run_walk_forward( - str(cfg_path), override_artifacts_dir=str(artifacts_dir) - ) + result = run_walk_forward(str(cfg_path), override_artifacts_dir=str(artifacts_dir)) no_holdout_dir = tmp_path / "artifacts_no_holdout" no_holdout_result = run_walk_forward( str(no_holdout_cfg_path), @@ -179,9 +175,7 @@ def test_holdout_window_does_not_overlap_selection(tmp_path: Path, monkeypatch) cfg_path = _write_holdout_fixture(tmp_path, include_holdout=True) artifacts_dir = tmp_path / "artifacts" - result = run_walk_forward( - str(cfg_path), override_artifacts_dir=str(artifacts_dir) - ) + result = run_walk_forward(str(cfg_path), override_artifacts_dir=str(artifacts_dir)) holdout_start = pd.Timestamp(result["walkforward"]["holdout_start"]) for fold in result["folds"]: diff --git a/tests/test_wrds_flagship_spec.py b/tests/test_wrds_flagship_spec.py index 27effe9c..e4e67318 100644 --- a/tests/test_wrds_flagship_spec.py +++ b/tests/test_wrds_flagship_spec.py @@ -43,6 +43,10 @@ def test_wrds_flagship_risk_limits_match_spec() -> None: def test_wrds_smoke_config_keeps_same_limits() -> None: - base = _extract_risk(yaml.safe_load(_BASE_CFG.read_text(encoding="utf-8"))["template"]) - smoke = _extract_risk(yaml.safe_load(_SMOKE_CFG.read_text(encoding="utf-8"))["template"]) + base = _extract_risk( + yaml.safe_load(_BASE_CFG.read_text(encoding="utf-8"))["template"] + ) + smoke = _extract_risk( + yaml.safe_load(_SMOKE_CFG.read_text(encoding="utf-8"))["template"] + ) assert base == smoke diff --git a/tests/test_wrds_markers.py b/tests/test_wrds_markers.py index 52b78244..766a9839 100644 --- a/tests/test_wrds_markers.py +++ b/tests/test_wrds_markers.py @@ -10,7 +10,9 @@ def __init__(self, *, wrds: bool = False) -> None: self.keywords = {"wrds": wrds} self._markers: list[pytest.Mark] = [] - def add_marker(self, marker: pytest.Mark) -> None: # pragma: no cover - pytest interface shim + def add_marker( + self, marker: pytest.Mark + ) -> None: # pragma: no cover - pytest interface shim self._markers.append(marker) diff --git a/tests/test_wrds_summary_render.py b/tests/test_wrds_summary_render.py index 69b01f46..94904f65 100644 --- a/tests/test_wrds_summary_render.py +++ b/tests/test_wrds_summary_render.py @@ -104,7 +104,9 @@ def test_wrds_summary_renders_markdown_and_docs(tmp_path: Path) -> None: "test_end": "2013-04-01", } ] - (artifact_dir / "folds.json").write_text(json.dumps(folds_payload), encoding="utf-8") + (artifact_dir / "folds.json").write_text( + json.dumps(folds_payload), encoding="utf-8" + ) plots_dir = tmp_path / "plots" plots_dir.mkdir() @@ -143,7 +145,9 @@ def test_wrds_summary_renders_markdown_and_docs(tmp_path: Path) -> None: def test_wrds_summary_missing_equity(tmp_path: Path) -> None: artifact_dir = tmp_path / "missing" artifact_dir.mkdir() - (artifact_dir / "metrics.json").write_text(json.dumps({"sharpe_ratio": 1.0}), encoding="utf-8") + (artifact_dir / "metrics.json").write_text( + json.dumps({"sharpe_ratio": 1.0}), encoding="utf-8" + ) _write_png(artifact_dir / "bootstrap_hist.png") (artifact_dir / "spa.json").write_text( json.dumps( @@ -169,7 +173,9 @@ def test_wrds_summary_missing_equity(tmp_path: Path) -> None: json.dumps({"run_id": "run", "config_path": str(tmp_path / "cfg.yaml")}), encoding="utf-8", ) - (tmp_path / "cfg.yaml").write_text("walkforward: {testing_days: 10}\n", encoding="utf-8") + (tmp_path / "cfg.yaml").write_text( + "walkforward: {testing_days: 10}\n", encoding="utf-8" + ) (artifact_dir / "folds.json").write_text( json.dumps( [ @@ -241,7 +247,9 @@ def test_wrds_summary_allows_zero_spa(tmp_path: Path) -> None: ) manifest = {"run_id": "run", "config_path": str(tmp_path / "cfg.yaml")} (artifact_dir / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8") - (tmp_path / "cfg.yaml").write_text("walkforward: {testing_days: 10}\n", encoding="utf-8") + (tmp_path / "cfg.yaml").write_text( + "walkforward: {testing_days: 10}\n", encoding="utf-8" + ) (artifact_dir / "folds.json").write_text( json.dumps( [ @@ -307,7 +315,9 @@ def test_wrds_summary_creates_degenerate_spa_when_missing(tmp_path: Path) -> Non ) manifest = {"run_id": "run", "config_path": str(tmp_path / "cfg.yaml")} (artifact_dir / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8") - (tmp_path / "cfg.yaml").write_text("walkforward: {testing_days: 10}\n", encoding="utf-8") + (tmp_path / "cfg.yaml").write_text( + "walkforward: {testing_days: 10}\n", encoding="utf-8" + ) (artifact_dir / "folds.json").write_text( json.dumps( [ diff --git a/tools/agentic/gpt_bundle.py b/tools/agentic/gpt_bundle.py old mode 100755 new mode 100644 index 0b524f06..aff5eee7 --- a/tools/agentic/gpt_bundle.py +++ b/tools/agentic/gpt_bundle.py @@ -13,10 +13,10 @@ - ticket file (if present) - selected small changed files (best-effort) """ + from __future__ import annotations import argparse -import os import subprocess import sys import zipfile @@ -27,7 +27,9 @@ def run(cmd: list[str], cwd: Optional[Path] = None) -> Tuple[int, str]: try: - out = subprocess.check_output(cmd, cwd=str(cwd) if cwd else None, stderr=subprocess.STDOUT) + out = subprocess.check_output( + cmd, cwd=str(cwd) if cwd else None, stderr=subprocess.STDOUT + ) return 0, out.decode("utf-8", errors="replace") except subprocess.CalledProcessError as e: return e.returncode, e.output.decode("utf-8", errors="replace") @@ -48,7 +50,9 @@ def ensure_repo_snapshot(repo: Path, scratch_dir: Path) -> Optional[Path]: if tool.exists(): scratch_dir.mkdir(parents=True, exist_ok=True) scratch_snap = scratch_dir / "repo_snapshot.md" - code, out = run([sys.executable, str(tool), "--out", str(scratch_snap)], cwd=repo) + code, out = run( + [sys.executable, str(tool), "--out", str(scratch_snap)], cwd=repo + ) if code == 0: p = Path(out.strip().splitlines()[-1]) if p.exists(): @@ -109,7 +113,9 @@ def restore_stash(repo: Path, stash_ref: str, status_before: str) -> None: raise SystemExit(f"Failed to drop stash {stash_ref}:\n{out}") -def prepare_worktree(repo: Path, label: str, no_stash: bool) -> tuple[str, str | None, bool]: +def prepare_worktree( + repo: Path, label: str, no_stash: bool +) -> tuple[str, str | None, bool]: status_before = git_status_porcelain(repo) dirty = bool(status_before.strip()) stash_ref = None @@ -135,24 +141,28 @@ def bundle_root(repo: Path) -> Path: def list_changed_files(repo: Path, stash_ref: Optional[str] = None) -> list[str]: if stash_ref: - _, out = run(["git", "-C", str(repo), "stash", "show", "--name-only", stash_ref]) - return [l.strip() for l in out.splitlines() if l.strip()] + _, out = run( + ["git", "-C", str(repo), "stash", "show", "--name-only", stash_ref] + ) + return [line.strip() for line in out.splitlines() if line.strip()] # Prefer git diff names for working tree _, out = run(["git", "-C", str(repo), "diff", "--name-only"]) - changed = [l.strip() for l in out.splitlines() if l.strip()] + changed = [line.strip() for line in out.splitlines() if line.strip()] # Include staged _, out2 = run(["git", "-C", str(repo), "diff", "--cached", "--name-only"]) - for l in out2.splitlines(): - l = l.strip() - if l and l not in changed: - changed.append(l) + for line in out2.splitlines(): + line = line.strip() + if line and line not in changed: + changed.append(line) return changed def collect_diffs(repo: Path, stash_ref: Optional[str]) -> tuple[str, str, str]: if stash_ref: _, diff = run(["git", "-C", str(repo), "stash", "show", "-p", stash_ref]) - _, diff_stat = run(["git", "-C", str(repo), "stash", "show", "--stat", stash_ref]) + _, diff_stat = run( + ["git", "-C", str(repo), "stash", "show", "--stat", stash_ref] + ) return diff, "", diff_stat _, diff = run(["git", "-C", str(repo), "diff"]) _, diff_cached = run(["git", "-C", str(repo), "diff", "--cached"]) @@ -160,7 +170,9 @@ def collect_diffs(repo: Path, stash_ref: Optional[str]) -> tuple[str, str, str]: return diff, diff_cached, diff_stat -def add_file_if_small(z: zipfile.ZipFile, repo: Path, rel_path: str, max_bytes: int = 120_000) -> None: +def add_file_if_small( + z: zipfile.ZipFile, repo: Path, rel_path: str, max_bytes: int = 120_000 +) -> None: p = repo / rel_path if not p.exists() or not p.is_file(): return @@ -174,10 +186,18 @@ def add_file_if_small(z: zipfile.ZipFile, repo: Path, rel_path: str, max_bytes: def main() -> int: ap = argparse.ArgumentParser() - ap.add_argument("--zip", action="store_true", help="Create zip bundle (default behavior).") - ap.add_argument("--ticket", type=str, default=None, help="Ticket id to include (optional).") + ap.add_argument( + "--zip", action="store_true", help="Create zip bundle (default behavior)." + ) + ap.add_argument( + "--ticket", type=str, default=None, help="Ticket id to include (optional)." + ) ap.add_argument("--out", type=str, default=None, help="Output zip path (optional).") - ap.add_argument("--include-files", action="store_true", help="Include small changed files in addition to diffs.") + ap.add_argument( + "--include-files", + action="store_true", + help="Include small changed files in addition to diffs.", + ) ap.add_argument( "--no-stash", action="store_true", @@ -204,14 +224,20 @@ def main() -> int: snap = ensure_repo_snapshot(repo, bundle_dir) # Collect git info - _, log = run(["git", "-C", str(repo), "log", "-n", "50", "--oneline", "--decorate"]) + _, log = run( + ["git", "-C", str(repo), "log", "-n", "50", "--oneline", "--decorate"] + ) diff, diff_cached, diff_stat = collect_diffs(repo, stash_ref) changed = list_changed_files(repo, stash_ref) ts = datetime.utcnow().strftime("%Y%m%d_%H%M%S") ticket = (args.ticket or "").strip() suffix = f"_{ticket}" if ticket else "" - out_zip = Path(args.out) if args.out else (bundle_dir / f"gpt_bundle_{ts}{suffix}.zip") + out_zip = ( + Path(args.out) + if args.out + else (bundle_dir / f"gpt_bundle_{ts}{suffix}.zip") + ) readme = f"""GPT Bundle @@ -238,7 +264,9 @@ def main() -> int: z.writestr("git_diff.patch", diff) z.writestr("git_diff_cached.patch", diff_cached) z.writestr("git_diff_stat.txt", diff_stat) - z.writestr("changed_files.txt", "\n".join(changed) + ("\n" if changed else "")) + z.writestr( + "changed_files.txt", "\n".join(changed) + ("\n" if changed else "") + ) if snap and snap.exists(): if snap.is_relative_to(repo): diff --git a/tools/agentic/project_state_refresh.py b/tools/agentic/project_state_refresh.py old mode 100755 new mode 100644 index 62ee56e3..e9d9a075 --- a/tools/agentic/project_state_refresh.py +++ b/tools/agentic/project_state_refresh.py @@ -11,12 +11,11 @@ This script does NOT attempt to "understand" the repo. It creates the stable raw materials that an AI agent can summarize accurately. """ + from __future__ import annotations import argparse -import os import subprocess -import sys import zipfile from datetime import datetime from pathlib import Path @@ -25,7 +24,9 @@ def run(cmd: list[str], cwd: Optional[Path] = None) -> Tuple[int, str]: try: - out = subprocess.check_output(cmd, cwd=str(cwd) if cwd else None, stderr=subprocess.STDOUT) + out = subprocess.check_output( + cmd, cwd=str(cwd) if cwd else None, stderr=subprocess.STDOUT + ) return 0, out.decode("utf-8", errors="replace") except subprocess.CalledProcessError as e: return e.returncode, e.output.decode("utf-8", errors="replace") @@ -120,10 +121,19 @@ def write_generated(repo: Path, project_state_dir: Path) -> None: # simple dependency hints dep = [] - for fname in ["Cargo.toml", "package.json", "pyproject.toml", "requirements.txt", "CMakeLists.txt", "Makefile"]: + for fname in [ + "Cargo.toml", + "package.json", + "pyproject.toml", + "requirements.txt", + "CMakeLists.txt", + "Makefile", + ]: if (repo / fname).exists(): dep.append(fname) - (gen / "dependency_hints.txt").write_text("\n".join(dep) + ("\n" if dep else ""), encoding="utf-8") + (gen / "dependency_hints.txt").write_text( + "\n".join(dep) + ("\n" if dep else ""), encoding="utf-8" + ) def zip_project_state(repo: Path, project_state_dir: Path, out_zip: Path) -> Path: @@ -137,12 +147,21 @@ def zip_project_state(repo: Path, project_state_dir: Path, out_zip: Path) -> Pat z.write(p, arcname=str(rel)) # include key root docs if present - for p in [repo/"PROJECT.md", repo/"PROGRESS.md", repo/"AGENTS.md", repo/"README.md"]: + for p in [ + repo / "PROJECT.md", + repo / "PROGRESS.md", + repo / "AGENTS.md", + repo / "README.md", + ]: if p.exists() and p.is_file(): z.write(p, arcname=str(p.relative_to(repo))) # include key docs if present - for p in [repo/"docs"/"RUNBOOK.md", repo/"docs"/"DECISIONS.md", repo/"docs"/"PLAN_OF_RECORD.md"]: + for p in [ + repo / "docs" / "RUNBOOK.md", + repo / "docs" / "DECISIONS.md", + repo / "docs" / "PLAN_OF_RECORD.md", + ]: if p.exists() and p.is_file(): z.write(p, arcname=str(p.relative_to(repo))) @@ -151,7 +170,9 @@ def zip_project_state(repo: Path, project_state_dir: Path, out_zip: Path) -> Pat def main() -> int: ap = argparse.ArgumentParser() - ap.add_argument("--zip", action="store_true", help="Create project_state zip in docs/_bundles/") + ap.add_argument( + "--zip", action="store_true", help="Create project_state zip in docs/_bundles/" + ) ap.add_argument("--out", type=str, default=None, help="Zip output path (optional)") args = ap.parse_args() @@ -166,7 +187,11 @@ def main() -> int: if args.zip: ts = datetime.utcnow().strftime("%Y%m%d_%H%M%S") - out_zip = Path(args.out) if args.out else (repo / "docs" / "_bundles" / f"project_state_{ts}.zip") + out_zip = ( + Path(args.out) + if args.out + else (repo / "docs" / "_bundles" / f"project_state_{ts}.zip") + ) out = zip_project_state(repo, project_state_dir, out_zip) print(str(out)) diff --git a/tools/agentic/repo_snapshot.py b/tools/agentic/repo_snapshot.py old mode 100755 new mode 100644 index f47ec6e9..37ae7392 --- a/tools/agentic/repo_snapshot.py +++ b/tools/agentic/repo_snapshot.py @@ -7,14 +7,13 @@ This is intentionally non-AI: it is cheap, fast, and stable. """ + from __future__ import annotations import argparse import collections import os import subprocess -import sys -import textwrap from datetime import datetime from pathlib import Path from typing import Iterable, Optional, Tuple @@ -22,7 +21,9 @@ def run(cmd: list[str], cwd: Optional[Path] = None) -> Tuple[int, str]: try: - out = subprocess.check_output(cmd, cwd=str(cwd) if cwd else None, stderr=subprocess.STDOUT) + out = subprocess.check_output( + cmd, cwd=str(cwd) if cwd else None, stderr=subprocess.STDOUT + ) return 0, out.decode("utf-8", errors="replace") except subprocess.CalledProcessError as e: return e.returncode, e.output.decode("utf-8", errors="replace") @@ -52,7 +53,12 @@ def guess_language_counts(paths: Iterable[str]) -> dict[str, int]: def main() -> int: ap = argparse.ArgumentParser() - ap.add_argument("--out", type=str, default=None, help="Output path (default: docs/_generated/repo_snapshot.md)") + ap.add_argument( + "--out", + type=str, + default=None, + help="Output path (default: docs/_generated/repo_snapshot.md)", + ) args = ap.parse_args() start = Path.cwd() @@ -75,14 +81,18 @@ def main() -> int: if len(tracked) > 1200: tree_lines.append(f"... ({len(tracked) - 1200} more tracked files)") - out_path = Path(args.out) if args.out else (repo / "docs" / "_generated" / "repo_snapshot.md") + out_path = ( + Path(args.out) + if args.out + else (repo / "docs" / "_generated" / "repo_snapshot.md") + ) out_path.parent.mkdir(parents=True, exist_ok=True) now = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%SZ") md = f"""# Repo Snapshot -Generated: **{now}** +Generated: **{now}** Repo root: `{repo}` ## Git diff --git a/tools/build_project_state.py b/tools/build_project_state.py index 8357d97e..17d54f7e 100644 --- a/tools/build_project_state.py +++ b/tools/build_project_state.py @@ -3,6 +3,7 @@ Stdlib only. Writes JSON outputs to project_state/_generated. """ + from __future__ import annotations import ast @@ -31,7 +32,9 @@ def rg_files(root: Path) -> list[str]: return files -def add_explicit_dirs(files: list[str], root: Path, extra_dirs: list[Path]) -> list[str]: +def add_explicit_dirs( + files: list[str], root: Path, extra_dirs: list[Path] +) -> list[str]: seen = set(files) for directory in extra_dirs: if not directory.exists(): @@ -60,7 +63,11 @@ def classify_role(path: str) -> str: return "report" if path.startswith("artifacts/"): return "artifact" - if path.startswith("data/") or path.startswith("data_sp500/") or path.startswith("data_sp500_enriched/"): + if ( + path.startswith("data/") + or path.startswith("data_sp500/") + or path.startswith("data_sp500_enriched/") + ): return "data" if path.startswith("scripts/"): return "script" @@ -281,7 +288,7 @@ def import_graph(py_files: list[Path]) -> dict[str, list[str]]: elif isinstance(node, ast.ImportFrom): if node.level and current_module: parts = current_module.split(".") - base_parts = parts[:-node.level] + base_parts = parts[: -node.level] if node.module: base_parts += node.module.split(".") if base_parts: @@ -322,10 +329,7 @@ def main() -> None: files, ROOT, [ - ROOT - / "artifacts" - / "sample_flagship" - / "2025-10-30T18-39-31Z-a4ab8e7", + ROOT / "artifacts" / "sample_flagship" / "2025-10-30T18-39-31Z-a4ab8e7", ROOT / "artifacts" / "sample_wfv" / "2025-10-30T18-39-47Z-a4ab8e7", ], ) @@ -335,9 +339,16 @@ def main() -> None: json.dumps(inventory, indent=2, sort_keys=True), encoding="utf-8" ) - py_files = [ROOT / f for f in files if f.endswith(".py") and ( - f.startswith("src/") or f.startswith("experiments/") or f.startswith("tools/") - )] + py_files = [ + ROOT / f + for f in files + if f.endswith(".py") + and ( + f.startswith("src/") + or f.startswith("experiments/") + or f.startswith("tools/") + ) + ] sym_index = symbol_index(py_files) (GENERATED_DIR / "symbol_index.json").write_text( @@ -350,7 +361,9 @@ def main() -> None: ) targets = make_targets(ROOT / "Makefile") - (GENERATED_DIR / "make_targets.txt").write_text("\n".join(targets) + "\n", encoding="utf-8") + (GENERATED_DIR / "make_targets.txt").write_text( + "\n".join(targets) + "\n", encoding="utf-8" + ) if __name__ == "__main__": diff --git a/tools/gpt_bundle.py b/tools/gpt_bundle.py index 0e24a114..81831e39 100644 --- a/tools/gpt_bundle.py +++ b/tools/gpt_bundle.py @@ -33,7 +33,9 @@ def _git_status_porcelain() -> str: def _stash_push(label: str) -> str: try: subprocess.check_output( - ["git", "stash", "push", "-u", "-m", label], text=True, stderr=subprocess.STDOUT + ["git", "stash", "push", "-u", "-m", label], + text=True, + stderr=subprocess.STDOUT, ) except subprocess.CalledProcessError as exc: raise SystemExit(f"Failed to stash worktree: {exc.output}") from exc @@ -392,7 +394,9 @@ def main() -> None: base, head, source = _derive_diff_range(meta_path) _write_commits(stage, base, head, source) diff_cmd = ["git", "diff", f"{base}..{head}"] - diff_text = subprocess.check_output(diff_cmd, text=True, stderr=subprocess.STDOUT) + diff_text = subprocess.check_output( + diff_cmd, text=True, stderr=subprocess.STDOUT + ) diff_path.write_text(diff_text, encoding="utf-8") _verify_patch_matches(diff_path, stage, base, head, run_name) diff --git a/tools/render_project_state_docs.py b/tools/render_project_state_docs.py index 9928c7d9..0eb7ffca 100644 --- a/tools/render_project_state_docs.py +++ b/tools/render_project_state_docs.py @@ -3,6 +3,7 @@ Stdlib only. Writes project_state/*.md with consistent metadata headers. """ + from __future__ import annotations import json @@ -41,7 +42,9 @@ def utc_now() -> str: def git_sha() -> str: - return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() + return subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=ROOT, text=True + ).strip() def git_branch() -> str: @@ -235,9 +238,11 @@ def recent_run_summaries(root: Path, limit: int = 3) -> list[dict[str, str]]: { "run": run_dir.name, "summary": summary, - "results_path": results_path.relative_to(ROOT).as_posix() - if results_path.exists() - else "", + "results_path": ( + results_path.relative_to(ROOT).as_posix() + if results_path.exists() + else "" + ), } ) return summaries @@ -267,7 +272,9 @@ def wrds_caveat(results_text: str) -> str: return "" -def render_architecture(symbol_index: dict[str, Any], inventory: list[dict[str, Any]]) -> str: +def render_architecture( + symbol_index: dict[str, Any], inventory: list[dict[str, Any]] +) -> str: core_modules = [ "src/microalpha/engine.py", "src/microalpha/data.py", @@ -336,14 +343,19 @@ def render_architecture(symbol_index: dict[str, Any], inventory: list[dict[str, - WRDS helpers: `src/microalpha/wrds/` and `scripts/export_wrds_flagship.py`. """.format( module_notes="\n".join(module_notes), - artifact_lines="\n".join(artifact_lines) if artifact_lines else "- (no artifacts indexed)", + artifact_lines=( + "\n".join(artifact_lines) if artifact_lines else "- (no artifacts indexed)" + ), ) def render_module_summaries(symbol_index: dict[str, Any]) -> str: rows = summarize_module_symbols(symbol_index) - table_lines = ["| Module | Docstring | Classes | Functions |", "| --- | --- | ---: | ---: |"] + table_lines = [ + "| Module | Docstring | Classes | Functions |", + "| --- | --- | ---: | ---: |", + ] for row in rows: doc = row["module_doc"].replace("|", "\\|") table_lines.append( @@ -381,7 +393,12 @@ def render_function_index(symbol_index: dict[str, Any]) -> str: def render_dependency_graph(import_graph: dict[str, list[str]]) -> str: total_edges = sum(len(v) for v in import_graph.values()) - lines = ["# Dependency Graph", "", f"Internal import edges (microalpha.*): {total_edges}", ""] + lines = [ + "# Dependency Graph", + "", + f"Internal import edges (microalpha.*): {total_edges}", + "", + ] lines.append("## Adjacency list (file -> internal imports)") for path in sorted(import_graph.keys()): imports = import_graph[path] @@ -588,9 +605,7 @@ def fmt_metric(metrics: dict[str, Any], key: str, fmt: str = "{:.2f}") -> str: for key, value in smoke_metrics.items(): smoke_block += f" - {key}: {value}\n" smoke_block += "- Report: `reports/summaries/wrds_flagship_smoke.md`\n" - smoke_block += ( - "- Note: Smoke run validates WRDS pipeline wiring; metrics are not interpretable for performance.\n" - ) + smoke_block += "- Note: Smoke run validates WRDS pipeline wiring; metrics are not interpretable for performance.\n" progress_date, progress_entries = latest_progress_section(progress_text) progress_block = "" @@ -669,7 +684,9 @@ def render_known_issues(progress_text: str, wrds_text: str) -> str: issues.append(f"From `PROGRESS.md`: {entry}") if not issues: - issues.append("No known issues recorded in `PROGRESS.md` or `docs/results_wrds.md`.") + issues.append( + "No known issues recorded in `PROGRESS.md` or `docs/results_wrds.md`." + ) return "\n".join(["# Known Issues", ""] + [f"- {issue}" for issue in issues]) + "\n" @@ -687,7 +704,12 @@ def render_roadmap() -> str: def render_config_reference(config_paths: list[Path]) -> str: - lines = ["# Config Reference", "", "| Config | Top-level keys | Notes |", "| --- | --- | --- |"] + lines = [ + "# Config Reference", + "", + "| Config | Top-level keys | Notes |", + "| --- | --- | --- |", + ] for path in config_paths: keys = top_level_keys_from_yaml(path) note = "" @@ -836,14 +858,14 @@ def main() -> None: / "2025-10-30T18-39-47Z-a4ab8e7" / "metrics.json" ) - sample_metrics = read_json(sample_metrics_path) if sample_metrics_path.exists() else {} + sample_metrics = ( + read_json(sample_metrics_path) if sample_metrics_path.exists() else {} + ) wfv_metrics = read_json(wfv_metrics_path) if wfv_metrics_path.exists() else {} holdout_root = ROOT / "artifacts" / "sample_wfv_holdout" holdout_dir = latest_run_dir(holdout_root) holdout_run = holdout_dir.name if holdout_dir else None - holdout_metrics_path = ( - holdout_dir / "holdout_metrics.json" if holdout_dir else None - ) + holdout_metrics_path = holdout_dir / "holdout_metrics.json" if holdout_dir else None holdout_metrics = ( read_json(holdout_metrics_path) if holdout_metrics_path and holdout_metrics_path.exists() @@ -856,9 +878,15 @@ def main() -> None: deps = [] dep_match = re.search(r"dependencies\s*=\s*\[(.*?)\]", pyproject_text, re.S) if dep_match: - deps = [d.strip().strip('"') for d in dep_match.group(1).split(",") if d.strip()] - - test_files = [item["path"] for item in inventory if item.get("role") == "test" and item["path"].endswith('.py')] + deps = [ + d.strip().strip('"') for d in dep_match.group(1).split(",") if d.strip() + ] + + test_files = [ + item["path"] + for item in inventory + if item.get("role") == "test" and item["path"].endswith(".py") + ] changelog_text = read_text(ROOT / "CHANGELOG.md") @@ -886,7 +914,9 @@ def main() -> None: "KNOWN_ISSUES.md": render_known_issues(progress_text, wrds_text), "ROADMAP.md": render_roadmap(), "CONFIG_REFERENCE.md": render_config_reference(config_paths), - "SERVER_ENVIRONMENT.md": render_server_environment(platform.python_version(), deps), + "SERVER_ENVIRONMENT.md": render_server_environment( + platform.python_version(), deps + ), "TEST_COVERAGE.md": render_test_coverage(test_files), "STYLE_GUIDE.md": render_style_guide(), "CHANGELOG.md": render_changelog(changelog_text),