diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..82a2837 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,25 @@ +# Keep Docker build context lean — exclude dev/test artifacts +# NOTE: requirements-deploy.txt MUST be present — do not add *.txt here +.git +.gitignore +__pycache__ +*.py[cod] +*.egg-info +.eggs +venv/ +ENV/ +env/ +.env +.env.local +node_modules/ +*.log +*.db +*.sqlite +tests/ +docs/ +.pytest_cache/ +.mypy_cache/ +cursorReview/ +research/ +demos/ +examples/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..9568c53 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,57 @@ +FROM python:3.11-slim + +# System deps +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc \ + g++ \ + libffi-dev \ + libssl-dev \ + curl \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Install production deps inline — no separate requirements file to copy. +# Stripped of: streamlit, redis, alpaca-py, statsmodels, discord.py, matplotlib, +# seaborn, pytest, black, flake6 — saves ~140MB vs full requirements.txt +RUN pip install --no-cache-dir --upgrade pip && pip install --no-cache-dir \ + requests>=2.31.0 \ + numpy>=1.24.0 \ + "yfinance>=0.2.18" \ + pandas>=2.0.0 \ + python-dateutil>=2.8.0 \ + pytz>=2023.3 \ + python-dotenv>=1.0.0 \ + beautifulsoup4>=4.12.0 \ + feedparser>=6.0.10 \ + pyfedwatch>=1.2.0 \ + psutil>=5.9.0 \ + "fastapi>=0.104.0" \ + "uvicorn[standard]>=0.24.0" \ + "httpx>=0.27.0" \ + "google-generativeai>=0.3.0" \ + "cohere>=5.0.0" \ + "groq>=0.9.0" \ + "finnhub-python>=2.4.0" \ + "cot_reports>=0.1.0" \ + "supabase>=2.0.0" \ + "ta>=0.11.0" \ + "scikit-learn>=1.3.0" \ + "langgraph>=0.3.0" \ + "langchain-groq>=0.3.0" + +# Copy application code +COPY . . + +# Railway injects PORT at runtime +ENV PORT=8000 +ENV PYTHONUNBUFFERED=1 +ENV PYTHONDONTWRITEBYTECODE=1 + +# API_LIGHT_MODE=1 skips UnifiedAlphaMonitor (~300-400MB startup bomb). +# All API endpoints remain functional via compute_kill_chain() in the API layer. +ENV API_LIGHT_MODE=1 + +EXPOSE $PORT + +CMD ["uvicorn", "backend.app.main:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "1"] diff --git a/backend/app/api/v1/memstats.py b/backend/app/api/v1/memstats.py new file mode 100644 index 0000000..bbd30c2 --- /dev/null +++ b/backend/app/api/v1/memstats.py @@ -0,0 +1,70 @@ +""" +/debug/memory — in-process RSS burn monitor endpoints. + +Reads memory directly from the running process via psutil (or /proc/self/status +as fallback). No external API, no auth, ground truth from inside the container. + +Endpoints: + GET /debug/memory — current snapshot (rss_mb, vms_mb, percent) + GET /debug/memory/history — full burn curve deque from startup to now +""" + +from fastapi import APIRouter +from fastapi.responses import JSONResponse + +router = APIRouter(tags=["memstats"]) + + +def _read_rss(): + """Return (rss_bytes, vms_bytes) using psutil or /proc fallback.""" + try: + import psutil, os + p = psutil.Process(os.getpid()) + mi = p.memory_info() + return mi.rss, mi.vms + except Exception: + pass + # /proc/self/status fallback (Linux only) + try: + rss_kb = vms_kb = 0 + with open("/proc/self/status") as f: + for line in f: + if line.startswith("VmRSS:"): + rss_kb = int(line.split()[1]) + elif line.startswith("VmSize:"): + vms_kb = int(line.split()[1]) + return rss_kb * 1024, vms_kb * 1024 + except Exception: + return 0, 0 + + +@router.get("/debug/memory") +async def memory_snapshot(): + """Current RSS + VMS in MB, read directly from the process.""" + rss, vms = _read_rss() + return { + "rss_mb": round(rss / 1024 / 1024, 2), + "vms_mb": round(vms / 1024 / 1024, 2), + "rss_bytes": rss, + "note": "psutil or /proc/self/status — in-process, ground truth", + } + + +@router.get("/debug/memory/history") +async def memory_history(): + """Full RSS burn curve from startup to now (1-min samples, max 200 entries ~3.3h).""" + from backend.app.main import _rss_history # imported at call time to avoid circular + rows = list(_rss_history) + if not rows: + return JSONResponse({"error": "no data yet — logger starts 10s after startup", "rows": []}) + + first_rss = rows[0]["rss_mb"] + last_rss = rows[-1]["rss_mb"] + return { + "count": len(rows), + "first_rss_mb": first_rss, + "last_rss_mb": last_rss, + "growth_mb": round(last_rss - first_rss, 2), + "elapsed_min": rows[-1]["elapsed_min"], + "rows": rows, + } diff --git a/backend/app/core/dependencies.py b/backend/app/core/dependencies.py index 6b3df80..3f3657f 100644 --- a/backend/app/core/dependencies.py +++ b/backend/app/core/dependencies.py @@ -5,7 +5,7 @@ import os import logging from typing import Optional -import redis +# redis is optional — only imported when REDIS_URL is set (not needed on Railway without Redis) logger = logging.getLogger(__name__) @@ -22,6 +22,7 @@ def get_redis(): redis_url = os.getenv('REDIS_URL') if redis_url: try: + import redis # lazy import — only when REDIS_URL is configured _redis_client = redis.from_url(redis_url, decode_responses=True) logger.info("✅ Redis client connected") except Exception as e: diff --git a/backend/app/main.py b/backend/app/main.py index e84ef75..b1fe77f 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -24,7 +24,7 @@ import uvicorn from backend.app.api import llm_routes -from backend.app.api.v1 import agents, websocket, dp, health, market, killchain, signals, darkpool, gamma, options, squeeze, charts, agentx, calendar, enrichment, economic, pivots, cot, ta, axlfi, gate, intraday, brief, oracle, morningstar, training +from backend.app.api.v1 import agents, websocket, dp, health, market, killchain, signals, darkpool, gamma, options, squeeze, charts, agentx, calendar, enrichment, economic, pivots, cot, ta, axlfi, gate, intraday, brief, oracle, morningstar, training, memstats from backend.app.core.dependencies import set_monitor_bridge logging.basicConfig(level=logging.INFO) @@ -83,6 +83,7 @@ app.include_router(morningstar.router, prefix="/api/v1", tags=["morningstar"]) app.include_router(training.router, prefix="/api/v1", tags=["training"]) app.include_router(llm_routes.router, prefix="/api", tags=["llm-aliases"]) +app.include_router(memstats.router, tags=["memstats"]) # /debug/memory + /debug/memory/history @app.get("/debug/git") @@ -220,6 +221,13 @@ async def debug_supabase(): _pipe_instances = {} _startup_errors = {} # Captures init failures at startup for /startup-errors +# ── In-process RSS burn monitor ── +# Populated by _rss_burn_logger() which runs every 60s regardless of API_LIGHT_MODE. +# Exposed via GET /debug/memory/history — full burn curve from startup to now. +from collections import deque as _deque +_rss_history: _deque = _deque(maxlen=200) # 200 × 60s = 3.3 hours of history +_rss_start_time: float = 0.0 # set at startup + def _run_pipe(name, instance, method_name, interval, first_capture_method=None): """Wrapper that tracks thread status and does immediate first capture.""" import traceback as tb @@ -250,33 +258,42 @@ async def startup(): import asyncio import threading - # Production guard: Render must never run light mode (skips all background tasks). - if os.getenv("RENDER") and os.getenv("API_LIGHT_MODE", "0") == "1": - logger.warning( - "⚠️ API_LIGHT_MODE=1 ignored on Render — forcing full startup " - "(light mode skips brain/alpha-graph/staggered threads)" - ) - os.environ["API_LIGHT_MODE"] = "0" + # NOTE: API_LIGHT_MODE=1 is intentionally honoured on Railway (and Render). + # Setting API_LIGHT_MODE=1 skips UnifiedAlphaMonitor (~300-400MB startup bomb) + # while keeping all API endpoints functional — kill-chain data is served by + # compute_kill_chain() in the API layer, not the monitor. + # The old Render guard that forced API_LIGHT_MODE=0 has been removed (2026-05-29). + # To re-enable the monitor on a specific platform, unset API_LIGHT_MODE or set it to 0. # Lightweight API mode for local diagnostics: skip UnifiedAlphaMonitor only. if os.getenv("API_LIGHT_MODE", "0") == "1": + # 🔥 OOM FIX (2026-05-29): Full light mode — ALL background threads disabled. + # Previously this block still launched 4 staggered threads + brain + alpha-graph + # + auto-snapshot, causing RSS to grow from 110MB → 400MB+ in 35 minutes. + # In true light mode, ONLY the FastAPI request handlers run. + # No background data fetches, no polling loops, no thread memory accumulation. _thread_status['monitor_run_loop'] = {'status': 'disabled (API_LIGHT_MODE=1)'} _thread_status['paper_trade_scheduler'] = {'status': 'disabled (API_LIGHT_MODE=1)'} _thread_status['econ_release_capture'] = {'status': 'disabled (API_LIGHT_MODE=1)'} + _thread_status['dp_recorder'] = {'status': 'disabled (API_LIGHT_MODE=1)'} + _thread_status['signal_differ'] = {'status': 'disabled (API_LIGHT_MODE=1)'} + _thread_status['volume_spikes'] = {'status': 'disabled (API_LIGHT_MODE=1)'} + _thread_status['premarket_scheduler'] = {'status': 'disabled (API_LIGHT_MODE=1)'} + _thread_status['brain_polling'] = {'status': 'disabled (API_LIGHT_MODE=1)'} + _thread_status['alpha_graph_polling'] = {'status': 'disabled (API_LIGHT_MODE=1)'} + _thread_status['auto_snapshot'] = {'status': 'disabled (API_LIGHT_MODE=1)'} logger.info( - "⚡ API_LIGHT_MODE=1 — skipping UnifiedAlphaMonitor; " - "still starting brain/alpha-graph/staggered threads" + "⚡ API_LIGHT_MODE=1 — ALL background threads disabled. " + "Only FastAPI request handlers are active. True idle baseline mode." ) - asyncio.create_task(_staggered_thread_launcher()) - asyncio.create_task(_brain_polling_loop()) - asyncio.create_task(_alpha_graph_polling_loop()) - asyncio.create_task(_auto_snapshot_loop()) _port = os.getenv("PORT", "8000") logger.info( "📡 Local smoke: curl -sS -m 90 http://127.0.0.1:%s/api/v1/health && " "scripts/smoke_signals.sh (first /signals can take 10–25s; not a 3s endpoint)", _port, ) + # Always start the RSS burn logger — zero overhead, needed for burn curve + asyncio.create_task(_rss_burn_logger()) return if MONITOR_AVAILABLE: @@ -371,6 +388,9 @@ def _monitor_run_wrapper(): # Autonomous training snapshot capture — saves kill-shots result every 30min during market hours asyncio.create_task(_auto_snapshot_loop()) + # Always start the RSS burn logger — zero overhead, needed for burn curve + asyncio.create_task(_rss_burn_logger()) + _port = os.getenv("PORT", "8000") logger.info( "📡 Signals smoke: curl -sS -m 90 http://127.0.0.1:%s/api/v1/signals | " @@ -380,6 +400,52 @@ def _monitor_run_wrapper(): ) +async def _rss_burn_logger(): + """Always-on in-process RSS logger. Runs every 60s regardless of API_LIGHT_MODE. + Appends to _rss_history deque (maxlen=200, ~3.3h at 1-min intervals). + Also emits RSS_BURN log lines captured by Railway log stream. + Zero meaningful memory overhead — one psutil call per minute. + """ + import asyncio, os, time + global _rss_start_time + _rss_start_time = time.time() + await asyncio.sleep(10) # Let uvicorn finish binding before first read + while True: + try: + rss_b, vms_b = 0, 0 + try: + import psutil + mi = psutil.Process(os.getpid()).memory_info() + rss_b, vms_b = mi.rss, mi.vms + except Exception: + try: + with open("/proc/self/status") as _f: + for _line in _f: + if _line.startswith("VmRSS:"): + rss_b = int(_line.split()[1]) * 1024 + elif _line.startswith("VmSize:"): + vms_b = int(_line.split()[1]) * 1024 + except Exception: + pass + + rss_mb = round(rss_b / 1024 / 1024, 2) + vms_mb = round(vms_b / 1024 / 1024, 2) + elapsed = round((time.time() - _rss_start_time) / 60, 1) + + entry = { + "ts": datetime.utcnow().isoformat() + "Z", + "elapsed_min": elapsed, + "rss_mb": rss_mb, + "vms_mb": vms_mb, + } + _rss_history.append(entry) + # Emit structured log line — captured by Railway log stream + logger.info("RSS_BURN elapsed=%.1fmin rss_mb=%.2f vms_mb=%.2f", elapsed, rss_mb, vms_mb) + except Exception as _e: + logger.warning("RSS logger error: %s", _e) + await asyncio.sleep(60) + + async def _staggered_thread_launcher(): """🔥 OOM FIX: Launch background threads one-by-one with 30s gaps. Prevents concurrent memory spikes from all threads downloading data at once. diff --git a/docs/architecture/mdc/00-master-index.mdc b/docs/architecture/mdc/00-master-index.mdc new file mode 100644 index 0000000..a5cc86d --- /dev/null +++ b/docs/architecture/mdc/00-master-index.mdc @@ -0,0 +1,197 @@ +--- +description: Master Data Control — lotto-machine squad directory, capability map, and inter-agent contracts +globs: ["docs/architecture/mdc/**/*.mdc"] +alwaysApply: true +--- + +# lotto-machine.mdc — Master Data Control Directory + +> **Version:** 1.0 | **Date:** 2026-05-31 | **Principal Dispatcher:** Biomni +> **Status:** AUTHORITATIVE — all squad agents read this before touching code + +--- + +## Why This Exists + +We spent months acting like a solo rogue hacker patching 262 leaks while simultaneously building a macro prediction engine. That ends here. This MDC directory is the single source of truth for what each squad owns, what they must NOT touch, and how they pass data to each other. + +**The application has 5 distinct capability domains. Each has one squad. Squads do not cross lanes.** + +--- + +## Application Capability Map + +``` +┌─────────────────────────────────────────────────────────────────────┐ +│ lotto-machine │ +│ │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────────────┐ │ +│ │ SQUAD 1 │ │ SQUAD 2 │ │ SQUAD 3 │ │ +│ │ Core Infra │───▶│ Execution │◀───│ Macro & Data │ │ +│ │ & State │ │ & Risk │ │ Prediction │ │ +│ │ │ │ │ │ │ │ +│ │ /specs/01 │ │ /specs/02 │ │ /specs/03 │ │ +│ └──────┬───────┘ └──────┬───────┘ └──────────────────────┘ │ +│ │ │ │ │ +│ │ SnapshotState │ GateResult │ DTE output │ +│ │ (shared memory) │ (typed) │ (cached) │ +│ ▼ ▼ ▼ │ +│ ┌──────────────────────────────────────────────────────────────┐ │ +│ │ SQUAD 4: ML Flywheel │ │ +│ │ /specs/04 — Training & Win Rate │ │ +│ └──────────────────────────────────────────────────────────────┘ │ +│ │ │ +│ │ Verified signals │ +│ ▼ │ +│ ┌──────────────────────────────────────────────────────────────┐ │ +│ │ SQUAD 5: Command Center │ │ +│ │ /specs/05 — Dashboard & Signal Lineage │ │ +│ └──────────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## Squad Directory + +| Squad | File | Domain | Phase 2 Priority | +|-------|------|---------|-----------------| +| 1 — Core Infra & State | `01-core-infra.mdc` | Memory, SQLite, async I/O, WebSockets | 🔴 CRITICAL — blocks all others | +| 2 — Execution & Risk | `02-execution-risk.mdc` | Gate logic, DTE, regime, slippage | 🔴 CRITICAL — Monday readiness | +| 3 — Macro & Data | `03-macro-data.mdc` | Narrative divergence, econ prediction | 🟡 HIGH — signal quality | +| 4 — Training & ML | `04-training-ml.mdc` | Win rate, LLM snapshots, flywheel | 🟡 HIGH — self-improvement | +| 5 — Command Center | `05-command-center.mdc` | Dashboard, lineage, tooltips | 🟢 MEDIUM — visibility | + +--- + +## Inter-Agent Contracts (Data Sharing Rules) + +### Contract 1: Squad 1 → Squad 2 (State → Gate) + +**What:** The intraday snapshot (spy_price, walls, thesis_valid, regime) must reach the gate without blocking the FastAPI event loop. + +**Current state (broken):** `confluence_gate.py` reads `/tmp/intraday_snapshot.json` synchronously inside a FastAPI route handler. This is a blocking file I/O call on the async event loop. + +**Contract:** +```python +# Squad 1 owns: SnapshotState singleton (in-memory, thread-safe) +# Interface Squad 2 consumes: +class SnapshotState: + def get_current() -> dict: # non-blocking, returns last written snapshot + def get_age_seconds() -> float: # how stale is the snapshot + def is_stale(threshold_s: int = 120) -> bool: +``` + +**Rule:** Squad 2 (gate) calls `SnapshotState.get_current()` — never opens `/tmp/intraday_snapshot.json` directly. Squad 1 writes the file AND updates the singleton. The singleton is the fast path; the file is the persistence path. + +--- + +### Contract 2: Squad 3 → Squad 2 (Macro → Gate) + +**What:** DTE threshold modifiers (hawkish_mult, dovish_mult) must reach `confluence_gate.py` without a blocking FRED API call on every gate evaluation. + +**Current state (broken):** DTE is completely disconnected from the gate. Gate uses hardcoded 90%/70% thresholds. + +**Contract:** +```python +# Squad 3 owns: DTECache singleton +# Interface Squad 2 consumes: +class DTECache: + def get_threshold_modifier() -> float: # 0.85–1.15, cached 1h, default 1.0 + def get_regime_label() -> str: # "HOLD" | "HIKE_RISK" | "CUT_CYCLE" + def last_updated() -> datetime +``` + +**Rule:** Squad 3 refreshes DTECache on a background schedule (every 1h). Squad 2 reads it synchronously — never calls FRED directly from inside `should_fire()`. + +--- + +### Contract 3: Squad 2 → Squad 4 (Gate → Training) + +**What:** Every gate decision (pass or block) must be logged for training. The training pipeline needs the full GateResult, not just the final verdict. + +**Current state (partial):** `GateOutcomeTracker` exists but only logs blocked/passed + reason. Missing: regime, DTE modifier used, synthesis_score, sizing_multiplier. + +**Contract:** +```python +# Squad 2 emits on every should_fire() call: +GateDecisionEvent = { + "ts": str, # ISO UTC + "symbol": str, + "direction": str, # LONG | SHORT + "regime": str, # BULLISH | CHOPPY | BREAKDOWN | TREND_EXTENDED | PRE_MARKET + "blocked": bool, + "reason": str, + "raw_confidence": float, + "adjusted_confidence": float, + "sizing_multiplier": float, + "dte_modifier": float, # NEW — what DTE modifier was active + "synthesis_bias": str, + "synthesis_score": float, + "gates_passed": list[str], + "gates_failed": list[str], +} +# Squad 4 consumes this via: training.py POST /training/gate-decision +``` + +--- + +### Contract 4: Squad 3 → Squad 5 (Macro → Dashboard) + +**What:** DTE thresholds, regime label, and VIX proxy must be surfaced on the UI so operators can see WHY a trade fired or blocked. + +**Contract:** +```python +# Squad 3 exposes via: GET /api/v1/economic/dte-status +{ + "threshold_modifier": float, + "regime_label": str, + "choppy_threshold": int, # e.g. 81 (90 * 0.9 in CUT_CYCLE) + "trend_threshold": int, # e.g. 63 (70 * 0.9 in CUT_CYCLE) + "last_updated": str, + "p_hike": float, + "p_cut": float, +} +# Squad 5 reads this endpoint and displays it in the gate health bar +``` + +--- + +### Contract 5: Squad 4 → Squad 5 (ML → Dashboard) + +**What:** Win rate stats and similar-setup lookups must be available to the signal chain lineage view. + +**Contract:** +```python +# Squad 4 exposes via: GET /api/v1/training/win-rate-summary +{ + "total_signals": int, + "win_rate_pct": float, + "by_regime": { "BULLISH": float, "CHOPPY": float, ... }, + "by_direction": { "LONG": float, "SHORT": float }, + "pending_outcomes": int, + "last_snapshot_ts": str, +} +``` + +--- + +## Phase Status + +| Phase | Status | Description | +|-------|--------|-------------| +| Phase 1 | ✅ COMPLETE | 5 root-cause memory leaks fixed (commit d11a26f). RSS stable at 133 MB in API_LIGHT_MODE=1. | +| Phase 1.5 | ✅ COMPLETE | Railway deploy, OOM fix, in-process RSS burn monitor (commit bc9924f). | +| Phase 2 | 🔴 IN PROGRESS | Squad 1: SnapshotState singleton + async SQLite. Squad 2: RegimeResult typed, AAPL desync, DTE wired. | +| Phase 3 | ⏳ BLOCKED on Phase 2 | Squad 3: Narrative divergence rebuild. Squad 4: Training flywheel. Squad 5: Signal lineage UI. | + +--- + +## Hard Rules (All Squads) + +1. **No squad touches another squad's owned files without a contract change.** If Squad 2 needs something from Squad 1, it goes through the defined interface — not a direct import. +2. **No blocking I/O in FastAPI route handlers.** All file reads, SQLite queries, and external API calls in route handlers must use `asyncio.to_thread()` or be pre-cached. +3. **No silent exception swallowing.** `except: pass` and `except Exception as e: pass` (without logging) are banned. Every exception must be logged at WARNING or higher. +4. **No hardcoded thresholds in gate logic.** All confidence thresholds must read from DTECache (Squad 3) with a hardcoded fallback only as a last resort. +5. **No raw market orders in paper_trader.** All orders must use limit orders with a configurable slippage tolerance. +6. **The `/tmp/intraday_snapshot.json` file is Squad 1's write path only.** All other squads read via `SnapshotState.get_current()`. diff --git a/docs/architecture/mdc/01-core-infra.mdc b/docs/architecture/mdc/01-core-infra.mdc new file mode 100644 index 0000000..227e22f --- /dev/null +++ b/docs/architecture/mdc/01-core-infra.mdc @@ -0,0 +1,209 @@ +--- +description: Squad 1 — Core Infra & State. Memory management, SQLite lifecycle, async I/O, WebSocket stability. +globs: ["backend/app/core/**/*.py", "backend/app/main.py", "live_monitoring/core/**/*.py", "live_monitoring/orchestrator/confluence_gate.py"] +alwaysApply: false +--- + +# Squad 1: Core Infra & State — The Plumbers + +> **Agent Profile:** Low-level Systems Engineer. Strict, boring, obsessive about memory. +> **Lane:** System survival. You do not touch signal logic, gate thresholds, or UI. +> **Owns:** Memory lifecycle, SQLite connections, async I/O wrappers, WebSocket manager, SnapshotState singleton. + +--- + +## Inherited Gaps (From Prior Audits) + +### From FORENSIC AUDIT — Railway Deployment + End-to-End Memory Fix Plan + +| Gap ID | Description | Severity | Status | +|--------|-------------|----------|--------| +| INF-01 | `requests.Session` created per-call in FinnhubClient and StockgridClient | 🔴 CRITICAL | ✅ FIXED (commit d11a26f, Fix 1.1) | +| INF-02 | `brain.py` self._conn persistent SQLite connection never closed | 🔴 CRITICAL | ✅ FIXED (commit d11a26f, Fix 1.2a) | +| INF-03 | `canonical_state.py` sqlite3 context manager commits but does NOT close — 2880 leaked FDs/day | 🔴 CRITICAL | ✅ FIXED (commit d11a26f, Fix 1.2b) | +| INF-04 | `_signals_cache` unbounded growth — ~78MB/day from unique oracle_signal_cache_bump() keys | 🔴 CRITICAL | ✅ FIXED (commit d11a26f, Fix 1.3) | +| INF-05 | `_seen_urls` in brain.py unbounded set | 🔴 CRITICAL | ✅ FIXED (commit d11a26f, Fix 1.4) | +| INF-06 | `AXLFISignalDiffer._history` unbounded list | 🔴 CRITICAL | ✅ FIXED (commit d11a26f, Fix 1.5) | +| INF-07 | `UnifiedAlphaMonitor.initialize_all()` loads 8+ heavy objects at boot (~300-400MB startup bomb) | 🔴 CRITICAL | ✅ FIXED (API_LIGHT_MODE=1 gate, commit 99c4d48) | +| INF-08 | `_staggered_thread_launcher` ran in API_LIGHT_MODE=1, causing 110→402MB RSS growth in 35 min | 🔴 CRITICAL | ✅ FIXED (commit 99c4d48) | +| INF-09 | `import redis` at module level crashed startup when redis not installed | 🔴 CRITICAL | ✅ FIXED (lazy import, commit 1a194f4) | + +### From AUDIT_ITERATION_2 — Data Layer (262 Silent Errors) + +| Gap ID | Description | Severity | Status | +|--------|-------------|----------|--------| +| INF-10 | `/tmp/intraday_snapshot.json` read synchronously inside FastAPI route handlers (blocking event loop) | 🔴 CRITICAL | ❌ OPEN | +| INF-11 | `/tmp/intraday_snapshot.json` has no write lock — concurrent writes from `intraday_guardian.py` and `guardian_replay.py` can corrupt the file | 🔴 CRITICAL | ❌ OPEN | +| INF-12 | SQLite calls in `training.py`, `gate_outcome_tracker.py`, `dp_learning/database.py` are synchronous inside async FastAPI routes | 🟡 HIGH | ❌ OPEN | +| INF-13 | WebSocket manager has no reconnection logic, no heartbeat, no exponential backoff | 🟡 HIGH | ❌ OPEN | +| INF-14 | `premarket_scheduler.py` writes stage files to `/tmp/premarket_stages/` — lost on container restart | 🟢 MEDIUM | ❌ OPEN | +| INF-15 | Thread pool in `brief/router.py` (ThreadPoolExecutor, 3 workers) has no timeout enforcement per fetcher — one slow fetcher blocks the wave | 🟡 HIGH | ❌ OPEN | + +--- + +## Phase 2 Scope + +### P2-INF-01: SnapshotState In-Memory Singleton + +**Problem:** `confluence_gate.py`, `unified_monitor.py`, and 4 other files all open `/tmp/intraday_snapshot.json` independently. No locking. Blocking I/O in async context. + +**Solution:** Create `live_monitoring/core/snapshot_state.py`: + +```python +# live_monitoring/core/snapshot_state.py +import threading, json, time +from pathlib import Path +from typing import Optional + +_SNAPSHOT_PATH = Path("/tmp/intraday_snapshot.json") +_lock = threading.RLock() +_current: dict = {} +_written_at: float = 0.0 + +def write(snapshot: dict) -> None: + """Write snapshot to memory AND file atomically.""" + global _current, _written_at + with _lock: + _current = snapshot.copy() + _written_at = time.time() + # Atomic file write: write to .tmp then rename + tmp = _SNAPSHOT_PATH.with_suffix(".tmp") + tmp.write_text(json.dumps(snapshot, default=str)) + tmp.replace(_SNAPSHOT_PATH) + +def get_current() -> dict: + """Non-blocking read. Returns last written snapshot or {}.""" + with _lock: + return _current.copy() + +def get_age_seconds() -> float: + return time.time() - _written_at if _written_at else float("inf") + +def is_stale(threshold_s: int = 120) -> bool: + return get_age_seconds() > threshold_s +``` + +**Migration:** Replace all `open("/tmp/intraday_snapshot.json")` reads with `snapshot_state.get_current()`. Replace all `json.dump(snapshot, f)` writes with `snapshot_state.write(snapshot)`. + +**Files to migrate:** +- `live_monitoring/intraday_guardian.py` (writer) +- `live_monitoring/guardian_replay.py` (writer + reader) +- `live_monitoring/orchestrator/confluence_gate.py` (reader) +- `live_monitoring/orchestrator/unified_monitor.py` (reader) +- `live_monitoring/enrichment/apis/level_watcher.py` (reader) +- `live_monitoring/core/risk_manager.py` (reader) + +--- + +### P2-INF-02: Async SQLite Wrappers for FastAPI Routes + +**Problem:** `training.py`, `gate_outcome_tracker.py`, and `dp_learning/database.py` call `sqlite3.connect()` synchronously inside `async def` route handlers. This blocks the uvicorn event loop. + +**Solution:** Wrap all SQLite calls with `asyncio.to_thread()`: + +```python +# Pattern for all async route handlers: +import asyncio + +async def save_snapshot(req: SnapshotRequest): + def _write(): + # all synchronous sqlite3 code here + conn = sqlite3.connect(str(DB_PATH)) + ... + conn.close() + await asyncio.to_thread(_write) +``` + +**Files to migrate:** +- `backend/app/api/v1/training.py` — all 5 endpoints +- `live_monitoring/orchestrator/gate_outcome_tracker.py` — `log_signal()`, `get_recent()` +- `live_monitoring/agents/dp_learning/database.py` — `save_interaction()`, `get_interactions()` + +**Do NOT use aiosqlite** — it adds a dependency and the existing code is already structured as synchronous blocks. `asyncio.to_thread()` is the minimal-change fix. + +--- + +### P2-INF-03: WebSocket Exponential Backoff + +**Problem:** `backend/app/core/websocket_manager.py` has no reconnection logic. When a client disconnects, the server just removes it. The client has no guidance on when to reconnect. + +**Solution:** Add a `ConnectionPolicy` to the WebSocket manager: + +```python +# backend/app/core/websocket_manager.py additions: +RECONNECT_POLICY = { + "initial_delay_s": 1, + "max_delay_s": 30, + "backoff_factor": 2, + "max_attempts": 10, + "heartbeat_interval_s": 30, +} + +async def heartbeat_loop(self, websocket: WebSocket, channel: str): + """Send ping every 30s. Disconnect on failure.""" + while True: + await asyncio.sleep(RECONNECT_POLICY["heartbeat_interval_s"]) + try: + await websocket.send_json({"type": "ping", "ts": time.time()}) + except Exception: + self.disconnect(websocket) + break +``` + +**Frontend contract:** The frontend WebSocket client must implement the matching backoff policy. See Squad 5 spec for frontend implementation. + +--- + +### P2-INF-04: Thread Re-enable (Monday Readiness) + +**Problem:** All 10 background threads are currently disabled in `API_LIGHT_MODE=1`. The system is a comatose patient. + +**Solution:** Re-enable 3 core threads in `API_LIGHT_MODE=1` block in `main.py`: + +```python +# In API_LIGHT_MODE=1 block, before return: +asyncio.create_task(_brain_polling_loop()) # core signal — 40-80MB +asyncio.create_task(_staggered_thread_launcher( + enabled=["dp_recorder", "volume_spikes"] # 30-60MB combined +)) +asyncio.create_task(_rss_burn_logger()) # always-on, ~0MB +``` + +**Modify `_staggered_thread_launcher()`** to accept `enabled: list[str] | None = None` parameter. If provided, only launch threads whose key is in the list. + +**Pass criteria:** `/debug/memory/history` shows RSS < 420MB after 30 min with these 3 threads active. + +**Leave disabled:** `signal_differ`, `premarket_scheduler`, `alpha_graph_polling`, `auto_snapshot`, `monitor_run_loop`, `paper_trade_scheduler`, `econ_release_capture`. + +--- + +## Files Owned by Squad 1 + +``` +backend/app/main.py (startup sequence, thread management) +backend/app/core/websocket_manager.py (WebSocket lifecycle) +backend/app/core/dependencies.py (lazy imports, shared deps) +live_monitoring/core/snapshot_state.py (NEW — SnapshotState singleton) +live_monitoring/intraday_guardian.py (snapshot writer) +live_monitoring/enrichment/apis/_http.py (requests.Session singleton) +``` + +## Files Squad 1 Must NOT Touch + +``` +live_monitoring/orchestrator/confluence_gate.py (Squad 2) +live_monitoring/enrichment/apis/dynamic_threshold_engine.py (Squad 3) +backend/app/api/v1/training.py (Squad 4) +frontend/ (Squad 5) +``` + +--- + +## Acceptance Criteria + +- [ ] `SnapshotState.get_current()` returns last snapshot in < 1ms (no file I/O) +- [ ] No `open("/tmp/intraday_snapshot.json")` calls outside `snapshot_state.py` +- [ ] All SQLite calls in async route handlers wrapped with `asyncio.to_thread()` +- [ ] WebSocket heartbeat fires every 30s, disconnects dead clients +- [ ] RSS with 3 threads re-enabled: < 420MB at 30 min (verified via `/debug/memory/history`) +- [ ] Zero `except: pass` or `except Exception as e: pass` (without logging) in owned files diff --git a/docs/architecture/mdc/02-execution-risk.mdc b/docs/architecture/mdc/02-execution-risk.mdc new file mode 100644 index 0000000..8be6588 --- /dev/null +++ b/docs/architecture/mdc/02-execution-risk.mdc @@ -0,0 +1,247 @@ +--- +description: Squad 2 — Execution & Risk. Gate logic, regime detection, DTE wiring, slippage control, circuit breakers. +globs: ["live_monitoring/orchestrator/confluence_gate.py", "live_monitoring/orchestrator/checkers/**/*.py", "live_monitoring/trading/paper_trader.py", "backend/app/api/v1/gate.py"] +alwaysApply: false +--- + +# Squad 2: Execution & Risk — The Snipers + +> **Agent Profile:** Ruthless Quant Developer. Obsessed with execution speed and capital preservation. +> **Lane:** Signal filtering, gate logic, order execution. You do not touch data fetching, UI, or training pipelines. +> **Owns:** `confluence_gate.py`, all checkers, `paper_trader.py`, `gate_outcome_tracker.py`, `intraday_guardian.py` (gate-side reads only). + +--- + +## Inherited Gaps (From Prior Audits) + +### From FULL FINDINGS — Chapters 3 & 4 + +| Gap ID | Description | Severity | Status | +|--------|-------------|----------|--------| +| EX-01 | `_get_market_regime()` returns bare `str` — no typed contract, no auditable price source | 🔴 CRITICAL | ❌ OPEN | +| EX-02 | `regime == "UNKNOWN"` blocks ALL signals when `spy_price=0` — fires during pre-market when market is closed and snapshot has no price | 🔴 CRITICAL | ❌ OPEN | +| EX-03 | AAPL/SPY desync: `_evaluate_long_proposal()` Gate 1 calls `regime_detector.detect(current_price, symbol)` but `current_price` is `None` for non-SPY symbols, causing silent fallback to `"BULLISH"` | 🔴 CRITICAL | ❌ OPEN | +| EX-04 | Hardcoded confidence thresholds: CHOPPY=90%, TREND_EXTENDED=70%, DOWNTREND counter-trend=90% — static regardless of macro regime | 🔴 CRITICAL | ❌ OPEN | +| EX-05 | DTE (`DynamicThresholdEngine`) exists and works but is completely disconnected from `confluence_gate.py` | 🔴 CRITICAL | ❌ OPEN | +| EX-06 | `paper_trader.py` uses raw market orders (`submit_order` with no limit price) — subject to full slippage on volatile opens | 🟡 HIGH | ❌ OPEN | +| EX-07 | No pre-market volume gate — signals can fire at 9:30 open when spread is 10x normal | 🟡 HIGH | ❌ OPEN | +| EX-08 | `_regime_evaluators` dict has no explicit handler for `"UNKNOWN"` — falls through to `_evaluate_long_proposal` silently | 🟡 HIGH | ❌ OPEN | +| EX-09 | VIX is not in the intraday snapshot schema — gate cannot use live VIX for threshold adjustment | 🟡 HIGH | ❌ OPEN | +| EX-10 | `synthesis_bias` defaults to `None` when no synthesis data available — gate passes with 15% confidence penalty but no explicit warning | 🟢 MEDIUM | ❌ OPEN | + +### From CAN_WE_MAKE_MONEY.md + +| Gap ID | Description | Severity | Status | +|--------|-------------|----------|--------| +| EX-11 | Signal thresholds too high — system generates 0 signals in normal market conditions | 🔴 CRITICAL | ❌ OPEN (related to EX-04) | +| EX-12 | No circuit breaker for consecutive losses — system can keep firing after 3 losing trades | 🟡 HIGH | ❌ OPEN | + +--- + +## Phase 2 Scope + +### P2-EX-01: Typed RegimeResult Dataclass + +**Problem:** `_get_market_regime()` returns a bare `str`. No audit trail of what price was used, what walls were compared, or why UNKNOWN was returned. + +**Solution:** Replace the return type with a typed dataclass: + +```python +# In confluence_gate.py, add at top: +@dataclass +class RegimeResult: + regime: str # "BULLISH" | "CHOPPY" | "BREAKDOWN" | "TREND_EXTENDED" | "PRE_MARKET" | "UNKNOWN" + spy_price: float # price used for regime determination + price_source: str # "snapshot" | "alternate" | "yfinance_fallback" | "none" + call_wall: float # 0.0 if unavailable + put_wall: float # 0.0 if unavailable + reason: str # human-readable explanation of why this regime was chosen +``` + +**Update `_get_market_regime()`** to return `RegimeResult` instead of `str`. + +**Update `should_fire()`** to unpack: `regime_result = self._get_market_regime(...)` then `regime = regime_result.regime`. + +**Log the full RegimeResult** at DEBUG level on every gate call — this is the audit trail. + +--- + +### P2-EX-02: Pre-Market Bypass (Fix UNKNOWN Block) + +**Problem:** When market is closed (`market_open=False` in snapshot), `spy_price=0`, so `_get_market_regime()` returns `"UNKNOWN"`, which blocks ALL signals. Pre-market signals (valid use case) are silently killed. + +**Solution:** In `_get_market_regime()`, add pre-market check before the price check: + +```python +if not snapshot.get("market_open", True): + return RegimeResult( + regime="PRE_MARKET", + spy_price=0.0, + price_source="none", + call_wall=0.0, + put_wall=0.0, + reason="Market closed — pre-market permissive regime" + ) +``` + +**Add `_evaluate_premarket_proposal()`** to `_regime_evaluators`: +- LONG: requires 75%+ confidence, sizing 0.75x +- SHORT: blocked (no pre-market shorts) + +**Add explicit `_evaluate_unknown_proposal()`** to `_regime_evaluators`: +- Blocks all signals with clear reason: `"⛔ UNKNOWN REGIME: spy_price=0 during market hours — data feed failure"` +- This replaces the silent fallback to `_evaluate_long_proposal` + +--- + +### P2-EX-03: AAPL/SPY Desync Fix + +**Problem:** In `_evaluate_long_proposal()` Gate 1, `current_price` is `None` for non-SPY symbols when callers don't pass it. The gate silently skips regime detection and defaults to `"BULLISH"`. + +**Root cause confirmed:** `dp_divergence_checker.py` calls `gate.should_fire(symbol=signal.symbol, raw_confidence=signal.confidence)` without `current_price`. For AAPL, `current_price=None`, so Gate 1 is skipped. + +**Solution:** + +```python +# In _evaluate_long_proposal(), Gate 1: +if current_price is None: + # Fetch symbol-specific price — not SPY's price + current_price = self._get_price(symbol) + if current_price: + logger.debug(f"Gate: fetched {symbol} price ${current_price:.2f} for regime check") + +if self.regime_detector and current_price: + try: + regime = self.regime_detector.detect(current_price, symbol) + except Exception as e: + logger.warning(f"⚠️ Gate: regime detection failed for {symbol}: {e}") + regime = "BULLISH" # explicit fallback, not silent +``` + +**Note:** The wall-relative regime (BULLISH/CHOPPY/TREND_EXTENDED) in `_get_market_regime()` correctly uses SPY's walls — this is intentional. SPY walls are market-level constructs. The fix is only in Gate 1's `regime_detector.detect()` call, which should use the symbol's own price. + +--- + +### P2-EX-04: DTE Wired into Confluence Gate + +**Problem:** Gate uses hardcoded thresholds. DTE exists but is disconnected. + +**Solution:** Add `DTECache` consumer to `ConfluenceGate.__init__()`: + +```python +def __init__(self, regime_detector=None, kill_chain_logger=None): + ... + self._dte_modifier_cache: tuple[float, float] = (1.0, 0.0) # (modifier, timestamp) + self._dte_cache_ttl = 3600 # 1 hour + +def _get_dte_threshold_modifier(self) -> float: + """Get DTE modifier. Cached 1h. Returns 1.0 on any failure.""" + modifier, ts = self._dte_modifier_cache + if time.time() - ts < self._dte_cache_ttl: + return modifier + try: + # Read from DTECache singleton (Squad 3 owns this) + from live_monitoring.enrichment.apis.dte_cache import DTECache + modifier = DTECache.get_threshold_modifier() + self._dte_modifier_cache = (modifier, time.time()) + return modifier + except Exception as e: + logger.debug(f"DTE modifier fetch failed: {e} — using 1.0") + return 1.0 +``` + +**Apply modifier in `_evaluate_choppy_proposal()`:** +```python +dte_mod = self._get_dte_threshold_modifier() +choppy_threshold = int(min(95, max(75, round(90 * dte_mod)))) +# HOLD regime: 90%, CUT_CYCLE: ~81%, HIKE_RISK: ~99% (clamped to 95%) +``` + +**Apply modifier in `_evaluate_trend_extended_proposal()`:** +```python +trend_threshold = int(min(85, max(55, round(70 * dte_mod)))) +``` + +**Apply modifier in `_evaluate_long_proposal()` DOWNTREND soft block:** +```python +downtrend_threshold = int(min(95, max(80, round(90 * dte_mod)))) +``` + +--- + +### P2-EX-05: Limit Orders in paper_trader.py + +**Problem:** `paper_trader.py` line 171: `"Use market order for simplicity"` — raw market orders on volatile opens bleed slippage. + +**Solution:** Replace market orders with limit orders using a configurable slippage tolerance: + +```python +# paper_trader.py +LIMIT_SLIPPAGE_PCT = float(os.getenv("LIMIT_SLIPPAGE_PCT", "0.05")) # 0.05% default + +def _build_order(self, symbol, qty, side, current_price): + if side == "buy": + limit_price = round(current_price * (1 + LIMIT_SLIPPAGE_PCT / 100), 2) + else: + limit_price = round(current_price * (1 - LIMIT_SLIPPAGE_PCT / 100), 2) + + return { + "symbol": symbol, + "qty": qty, + "side": side, + "type": "limit", + "time_in_force": "day", + "limit_price": str(limit_price), + "extended_hours": False, + } +``` + +**Add pre-market volume gate:** +```python +def _passes_volume_gate(self, symbol: str) -> bool: + """Block orders in first 5 minutes of market open (9:30-9:35 ET).""" + et = datetime.now(pytz.timezone("America/New_York")) + if et.hour == 9 and et.minute < 35: + logger.info(f"Volume gate: blocking {symbol} order — first 5 min of open") + return False + return True +``` + +--- + +## Files Owned by Squad 2 + +``` +live_monitoring/orchestrator/confluence_gate.py +live_monitoring/orchestrator/checkers/dp_bearish_checker.py +live_monitoring/orchestrator/checkers/dp_divergence_checker.py +live_monitoring/orchestrator/checkers/gamma_checker.py +live_monitoring/orchestrator/checkers/dark_pool_checker.py +live_monitoring/orchestrator/checkers/options_flow_checker.py +live_monitoring/orchestrator/checkers/squeeze_checker.py +live_monitoring/orchestrator/gate_outcome_tracker.py +live_monitoring/trading/paper_trader.py +backend/app/api/v1/gate.py +``` + +## Files Squad 2 Must NOT Touch + +``` +live_monitoring/core/snapshot_state.py (Squad 1 — read via interface only) +live_monitoring/enrichment/apis/dynamic_threshold_engine.py (Squad 3 — read via DTECache only) +backend/app/api/v1/training.py (Squad 4) +frontend/ (Squad 5) +``` + +--- + +## Acceptance Criteria + +- [ ] `_get_market_regime()` returns `RegimeResult` dataclass — never a bare `str` +- [ ] Pre-market signals (market_open=False) route to `_evaluate_premarket_proposal()` — not blocked by UNKNOWN +- [ ] AAPL signal with `current_price=None` fetches AAPL's own price before Gate 1 regime check +- [ ] `_evaluate_unknown_proposal()` is an explicit handler — no silent fallback to `_evaluate_long_proposal` +- [ ] CHOPPY threshold adjusts with DTE modifier: 81% in CUT_CYCLE, 90% in HOLD, 95% in HIKE_RISK +- [ ] `paper_trader.py` uses limit orders with `LIMIT_SLIPPAGE_PCT` env var +- [ ] Pre-market volume gate blocks orders before 9:35 ET +- [ ] Every gate decision logs `RegimeResult` at DEBUG level diff --git a/docs/architecture/mdc/03-macro-data.mdc b/docs/architecture/mdc/03-macro-data.mdc new file mode 100644 index 0000000..f0377e8 --- /dev/null +++ b/docs/architecture/mdc/03-macro-data.mdc @@ -0,0 +1,248 @@ +--- +description: Squad 3 — Macro & Data Prediction. Narrative divergence, economic event parsing, DTE cache, IV skew ingest. +globs: ["live_monitoring/agents/economic/**/*.py", "live_monitoring/enrichment/apis/dynamic_threshold_engine.py", "live_monitoring/pipeline/components/trump_monitor.py", "live_monitoring/agents/narrative_brain/**/*.py"] +alwaysApply: false +--- + +# Squad 3: Macro & Data Prediction — The Oracles + +> **Agent Profile:** Macro-Economist Data Scientist. Deeply analytical, connects news to price action. +> **Lane:** Economic data, narrative divergence, DTE cache. You do not touch gate logic or UI rendering. +> **Owns:** `dynamic_threshold_engine.py`, `macro_regime_detector.py`, `narrative_brain/`, `trump_monitor.py`, `economic/` agents, `DTECache` singleton (new). + +--- + +## Inherited Gaps (From Prior Audits) + +### From Gap Audit Fix Plan — Narrative/Divergence (2026-05-23) + +| Gap ID | Description | Severity | Status | +|--------|-------------|----------|--------| +| MAC-01 | `trump_monitor.py` swallows all exceptions silently — `except Exception as e: pass` with no logging | 🔴 CRITICAL | ❌ OPEN | +| MAC-02 | `economic/calendar.py` line 80: bare `except:` — swallows ALL exceptions including KeyboardInterrupt | 🔴 CRITICAL | ❌ OPEN | +| MAC-03 | `economic/pre_event_analyzer.py` has 6 bare `except:` blocks — silent failures mean the pre-event signal never fires | 🔴 CRITICAL | ❌ OPEN | +| MAC-04 | Narrative divergence detection exists in `narrative_brain.py` but output is never routed to the gate — `divergence_detected` flag is computed but discarded | 🔴 CRITICAL | ❌ OPEN | +| MAC-05 | `DynamicThresholdEngine` is completely disconnected from `confluence_gate.py` — Squad 2 cannot consume it | 🔴 CRITICAL | ❌ OPEN (Squad 3 must build DTECache; Squad 2 consumes it) | +| MAC-06 | No VIX synthetic fallback — when `^VIX` yfinance call fails, DTE returns `regime_label: "UNKNOWN"` and all modifiers default to 1.0 | 🟡 HIGH | ❌ OPEN | +| MAC-07 | `economic/predictor.py` line 266: bare `except:` — prediction failures are silent | 🟡 HIGH | ❌ OPEN | +| MAC-08 | AAPL IV skew not ingested — system uses lagging price action for AAPL signals instead of forward-looking options IV | 🟡 HIGH | ❌ OPEN | + +### From Economic Data Prediction Framework — Multi-Phase Audit + +| Gap ID | Description | Severity | Status | +|--------|-------------|----------|--------| +| MAC-09 | `MacroRegimeDetector.get_regime()` returns a dict but callers sometimes treat it as a string — no typed contract | 🟡 HIGH | ❌ OPEN | +| MAC-10 | FRED API key not set on Railway — `DynamicThresholdEngine` degrades silently, no alert to operator | 🟡 HIGH | ❌ OPEN | +| MAC-11 | Cleveland Fed nowcast scraper has no retry logic — single network failure kills the nowcast signal | 🟢 MEDIUM | ❌ OPEN | +| MAC-12 | `economic/data_collector.py` has 4 separate `except Exception as e: logger.error(...)` blocks that log but return `None` — callers don't check for None | 🟢 MEDIUM | ❌ OPEN | + +### From Macro Prediction Framework — Production Integration Plan + +| Gap ID | Description | Severity | Status | +|--------|-------------|----------|--------| +| MAC-13 | `FedShiftPredictor` uses static coefficients — not calibrated to current FedWatch probabilities | 🟡 HIGH | ❌ OPEN | +| MAC-14 | No forward-looking IV skew ingest for SPY/QQQ — system is reactive, not predictive | 🟡 HIGH | ❌ OPEN | +| MAC-15 | `narrative_brain.py` divergence output has no structured schema — it's a free-text string, not a typed signal | 🟢 MEDIUM | ❌ OPEN | + +--- + +## Phase 2 Scope + +### P2-MAC-01: DTECache Singleton (Critical — Squad 2 Dependency) + +**Problem:** Squad 2 needs DTE threshold modifiers without blocking the gate on a FRED API call. Squad 3 must build the cache layer. + +**Solution:** Create `live_monitoring/enrichment/apis/dte_cache.py`: + +```python +# live_monitoring/enrichment/apis/dte_cache.py +import threading, time, logging +from typing import Optional + +logger = logging.getLogger(__name__) + +_lock = threading.RLock() +_modifier: float = 1.0 +_regime_label: str = "HOLD" +_p_hike: float = 0.0 +_p_cut: float = 0.0 +_last_updated: float = 0.0 +_TTL = 3600 # 1 hour + +def refresh() -> None: + """Fetch fresh DTE data. Called by background scheduler every 1h.""" + global _modifier, _regime_label, _p_hike, _p_cut, _last_updated + try: + from live_monitoring.enrichment.apis.dynamic_threshold_engine import DynamicThresholdEngine + dte = DynamicThresholdEngine() + shifts = dte.get_regime_adjusted_shifts("INFLATION") + + hawkish = shifts.get("hawkish_mult", 1.0) + dovish = shifts.get("dovish_mult", 1.0) + + # Modifier: hawkish → tighten thresholds (>1.0), dovish → relax (<1.0) + if hawkish > 1.0: + raw_modifier = 1.0 + (hawkish - 1.0) * 0.15 # 1.6x hawkish → 1.09 modifier + elif dovish > 1.0: + raw_modifier = 1.0 - (dovish - 1.0) * 0.15 # 1.6x dovish → 0.91 modifier + else: + raw_modifier = 1.0 + + with _lock: + _modifier = max(0.85, min(1.15, raw_modifier)) # clamp ±15% + _regime_label = shifts.get("regime", "HOLD") + _p_hike = shifts.get("p_hike", 0.0) + _p_cut = shifts.get("p_cut", 0.0) + _last_updated = time.time() + + logger.info(f"DTECache refreshed: modifier={_modifier:.3f} regime={_regime_label} p_hike={_p_hike:.1f}% p_cut={_p_cut:.1f}%") + except Exception as e: + logger.warning(f"DTECache refresh failed: {e} — keeping modifier={_modifier:.3f}") + +def get_threshold_modifier() -> float: + with _lock: + return _modifier + +def get_regime_label() -> str: + with _lock: + return _regime_label + +def get_status() -> dict: + with _lock: + return { + "threshold_modifier": _modifier, + "regime_label": _regime_label, + "p_hike": _p_hike, + "p_cut": _p_cut, + "last_updated": _last_updated, + "age_seconds": time.time() - _last_updated if _last_updated else None, + "choppy_threshold": int(min(95, max(75, round(90 * _modifier)))), + "trend_threshold": int(min(85, max(55, round(70 * _modifier)))), + } +``` + +**Wire into `main.py`:** Add `DTECache.refresh()` call to the background scheduler (every 1h). Also call once at startup (non-blocking, in a thread). + +--- + +### P2-MAC-02: Fix Silent Exception Swallowing + +**Problem:** 6+ bare `except:` and `except Exception as e: pass` blocks in economic agents mean failures are invisible. + +**Rule:** Every `except` block must either: +1. Log at `WARNING` or higher with the exception message, OR +2. Re-raise the exception + +**Files to fix:** +- `live_monitoring/agents/economic/calendar.py` line 80: `except:` → `except Exception as e: logger.warning(f"Calendar parse error: {e}")` +- `live_monitoring/agents/economic/pre_event_analyzer.py` lines 112, 120, 147, 163: all bare `except:` → log + continue +- `live_monitoring/agents/economic/predictor.py` line 266: bare `except:` → log +- `live_monitoring/pipeline/components/trump_monitor.py` line 138: `except Exception as e: pass` → log + +**Pattern to enforce:** +```python +# BANNED: +except: + pass +except Exception as e: + pass + +# REQUIRED: +except Exception as e: + logger.warning(f"[module_name] operation failed: {e}") + # optionally: return None or default value +``` + +--- + +### P2-MAC-03: VIX Synthetic Fallback + +**Problem:** When `^VIX` yfinance call fails, DTE returns `regime_label: "UNKNOWN"` and all modifiers default to 1.0. No fallback. + +**Solution:** Add VIX fallback chain in `DynamicThresholdEngine._get_regime()`: + +```python +def _get_vix_fallback(self) -> float: + """VIX fallback chain: yfinance → CBOE API → last known → 20.0 (neutral).""" + # 1. Try yfinance + try: + import yfinance as yf + vix = yf.Ticker("^VIX").fast_info.get("lastPrice") + if vix and vix > 0: + return float(vix) + except Exception: + pass + + # 2. Try CBOE direct + try: + import requests + r = requests.get("https://cdn.cboe.com/api/global/delayed_quotes/charts/historical/_VIX.json", timeout=5) + data = r.json() + if data and "data" in data: + return float(data["data"][-1][4]) # last close + except Exception: + pass + + # 3. Return neutral fallback + logger.warning("VIX fetch failed on all paths — using neutral fallback 20.0") + return 20.0 +``` + +--- + +### P2-MAC-04: Narrative Divergence → Typed Signal + +**Problem:** `narrative_brain.py` computes `divergence_detected` but it's a free-text string. No structured output that the gate can consume. + +**Solution:** Add a typed `NarrativeDivergenceSignal` to the narrative brain output: + +```python +@dataclass +class NarrativeDivergenceSignal: + detected: bool + direction: str # "BULLISH_DIVERGENCE" | "BEARISH_DIVERGENCE" | "NONE" + confidence: float # 0.0–1.0 + summary: str # 1-sentence human-readable + sources: list[str] # ["finnhub", "trump_monitor", "fed_tone"] + ts: str # ISO UTC +``` + +**Expose via:** `GET /api/v1/enrichment/narrative-divergence` — Squad 5 displays this on the dashboard. + +--- + +## Files Owned by Squad 3 + +``` +live_monitoring/enrichment/apis/dynamic_threshold_engine.py +live_monitoring/enrichment/apis/dte_cache.py (NEW) +live_monitoring/agents/economic/macro_regime_detector.py +live_monitoring/agents/economic/calendar.py +live_monitoring/agents/economic/pre_event_analyzer.py +live_monitoring/agents/economic/predictor.py +live_monitoring/agents/economic/data_collector.py +live_monitoring/agents/economic/fed_shift_predictor.py +live_monitoring/agents/narrative_brain/narrative_brain.py +live_monitoring/pipeline/components/trump_monitor.py +``` + +## Files Squad 3 Must NOT Touch + +``` +live_monitoring/orchestrator/confluence_gate.py (Squad 2 — expose via DTECache interface only) +backend/app/api/v1/training.py (Squad 4) +frontend/ (Squad 5) +live_monitoring/core/snapshot_state.py (Squad 1) +``` + +--- + +## Acceptance Criteria + +- [ ] `DTECache.get_threshold_modifier()` returns a float in [0.85, 1.15] within 1ms (no blocking I/O) +- [ ] `DTECache.refresh()` runs every 1h in background — failure logs at WARNING, does not crash +- [ ] Zero bare `except:` or `except Exception as e: pass` (without logging) in owned files +- [ ] VIX fallback chain: yfinance → CBOE → 20.0 neutral — never returns None +- [ ] `GET /api/v1/economic/dte-status` returns `threshold_modifier`, `regime_label`, `choppy_threshold`, `trend_threshold` +- [ ] `NarrativeDivergenceSignal` is a typed dataclass — not a free-text string +- [ ] FRED_API_KEY absence logs a WARNING at startup — not a silent degradation diff --git a/docs/architecture/mdc/04-training-ml.mdc b/docs/architecture/mdc/04-training-ml.mdc new file mode 100644 index 0000000..c95daa0 --- /dev/null +++ b/docs/architecture/mdc/04-training-ml.mdc @@ -0,0 +1,216 @@ +--- +description: Squad 4 — Training & ML Flywheel. Win rate validation, LLM snapshot training, quality gates, autonomous distillation. +globs: ["backend/app/api/v1/training.py", "live_monitoring/orchestrator/gate_outcome_tracker.py", "live_monitoring/data/training/**/*"] +alwaysApply: false +--- + +# Squad 4: Training & ML Flywheel — The Forge + +> **Agent Profile:** MLOps Architect. Builds self-improving loops. Obsessed with verified win rates. +> **Lane:** Training data collection, outcome labeling, win rate analysis, LLM snapshot evaluation. You do not touch gate logic or UI. +> **Owns:** `training.py`, `gate_outcome_tracker.py`, training JSONL pipeline, win rate analytics. + +--- + +## Inherited Gaps (From Prior Audits) + +### From Win Rate Validation & Training Pipeline Plan + +| Gap ID | Description | Severity | Status | +|--------|-------------|----------|--------| +| ML-01 | Training snapshots are saved but outcomes are never automatically recorded — `pending_outcomes.json` grows indefinitely with no resolution | 🔴 CRITICAL | ❌ OPEN | +| ML-02 | Win rate calculation has no regime breakdown — can't tell if BULLISH signals win at 70% but CHOPPY signals win at 30% | 🔴 CRITICAL | ❌ OPEN | +| ML-03 | `gate_outcome_tracker.py` logs gate decisions but missing: DTE modifier used, synthesis_score, sizing_multiplier — incomplete training record | 🔴 CRITICAL | ❌ OPEN | +| ML-04 | No quality gate on training data — snapshots with `confidence < 50` or `regime = UNKNOWN` are included in fine-tune export | 🟡 HIGH | ❌ OPEN | +| ML-05 | Training JSONL export has no versioning — overwriting the file loses historical data | 🟡 HIGH | ❌ OPEN | +| ML-06 | No similar-setup lookup — when a new signal fires, there's no way to find historical setups with matching regime + bias + confidence range | 🟡 HIGH | ❌ OPEN | + +### From Training Pipeline — Autonomous + Quality Gates + Similar Setups + +| Gap ID | Description | Severity | Status | +|--------|-------------|----------|--------| +| ML-07 | Auto-snapshot loop (`_auto_snapshot_loop` in main.py) is disabled in API_LIGHT_MODE=1 — no autonomous training data collection on Railway | 🟡 HIGH | ❌ OPEN | +| ML-08 | No outcome window enforcement — snapshots older than `OUTCOME_WINDOW_DAYS` (default 3) with no outcome are never flagged or auto-resolved | 🟡 HIGH | ❌ OPEN | +| ML-09 | Training export endpoint (`GET /training/export`) returns raw JSONL with no filtering — includes incomplete records (no outcome) | 🟡 HIGH | ❌ OPEN | +| ML-10 | No fine-tune trigger — even when training data is ready, there's no mechanism to kick off retraining | 🟢 MEDIUM | ❌ OPEN | + +### From Signal Chain: LLM Snapshot Training + +| Gap ID | Description | Severity | Status | +|--------|-------------|----------|--------| +| ML-11 | LLM snapshot format is not standardized — different callers save different fields, making fine-tuning inconsistent | 🟡 HIGH | ❌ OPEN | +| ML-12 | No deduplication — the same market setup can be saved multiple times if the auto-snapshot loop fires twice in 30 min | 🟢 MEDIUM | ❌ OPEN | + +--- + +## Phase 2 Scope + +### P2-ML-01: Extended GateDecisionEvent Schema + +**Problem:** `gate_outcome_tracker.py` logs gate decisions but is missing the fields Squad 2 now emits (DTE modifier, synthesis_score, sizing_multiplier). + +**Solution:** Update `GateOutcomeTracker.log_signal()` to accept and store the full `GateDecisionEvent` from the inter-agent contract: + +```python +# gate_outcome_tracker.py — extended schema +def log_signal( + self, + ticker: str, + direction: str, + entry_price: float, + confidence: float, + blocked: bool, + reason: str, + regime: str, + bias: str, + source: str = "", + # NEW fields from GateDecisionEvent contract: + dte_modifier: float = 1.0, + synthesis_score: float = 50.0, + sizing_multiplier: float = 1.0, + raw_confidence: float = 0.0, + gates_passed: list = None, + gates_failed: list = None, +) -> None: +``` + +**SQLite schema migration:** Add columns `dte_modifier REAL`, `synthesis_score REAL`, `sizing_multiplier REAL`, `raw_confidence REAL`, `gates_passed TEXT`, `gates_failed TEXT` to `gate_decisions` table. + +--- + +### P2-ML-02: Win Rate Analytics Endpoint + +**Problem:** No regime-breakdown win rate. Can't tell which regimes are profitable. + +**Solution:** Add `GET /api/v1/training/win-rate-summary` endpoint: + +```python +# Returns: +{ + "total_signals": int, + "total_with_outcomes": int, + "overall_win_rate_pct": float, + "by_regime": { + "BULLISH": {"signals": int, "wins": int, "win_rate_pct": float}, + "CHOPPY": {"signals": int, "wins": int, "win_rate_pct": float}, + "TREND_EXTENDED": {"signals": int, "wins": int, "win_rate_pct": float}, + "BREAKDOWN": {"signals": int, "wins": int, "win_rate_pct": float}, + }, + "by_direction": { + "LONG": {"signals": int, "wins": int, "win_rate_pct": float}, + "SHORT": {"signals": int, "wins": int, "win_rate_pct": float}, + }, + "by_dte_regime": { + "HOLD": {"signals": int, "wins": int, "win_rate_pct": float}, + "HIKE_RISK": {"signals": int, "wins": int, "win_rate_pct": float}, + "CUT_CYCLE": {"signals": int, "wins": int, "win_rate_pct": float}, + }, + "pending_outcomes": int, + "last_snapshot_ts": str, +} +``` + +--- + +### P2-ML-03: Quality Gate on Training Export + +**Problem:** Fine-tune export includes incomplete and low-quality records. + +**Solution:** Add quality filter to `GET /training/export`: + +```python +QUALITY_GATES = { + "min_confidence": 50, # exclude low-confidence signals + "exclude_regimes": ["UNKNOWN"], # exclude data-failure signals + "require_outcome": True, # exclude pending outcomes + "min_sizing_multiplier": 0.5, # exclude 0x-sized (blocked) signals +} +``` + +Add `?quality=strict` query param to apply gates. Default export includes all records (backward compat). + +--- + +### P2-ML-04: Standardized LLM Snapshot Schema + +**Problem:** Different callers save different fields. Fine-tuning is inconsistent. + +**Solution:** Define a `SnapshotSchema` Pydantic model that all callers must use: + +```python +class SnapshotSchema(BaseModel): + # Required + ts: str # ISO UTC + symbol: str + direction: str # LONG | SHORT | WATCH + regime: str # from RegimeResult.regime + label: str # WIN | LOSS | PENDING + confidence: float # adjusted_confidence from GateResult + raw_confidence: float # before gate adjustments + + # Gate context + gates_passed: list[str] + gates_failed: list[str] + sizing_multiplier: float + dte_modifier: float + synthesis_bias: str + synthesis_score: float + + # Market context + spy_price: float + vix: Optional[float] + kill_chain_score: Optional[float] + + # LLM training fields + oracle_summary: Optional[str] # NYX oracle brief at time of signal + narrative_summary: Optional[str] # narrative brain summary + + # Outcome (filled later) + outcome: Optional[str] # WIN | LOSS + outcome_pct: Optional[float] # actual P&L % + outcome_recorded_at: Optional[str] +``` + +--- + +### P2-ML-05: Outcome Auto-Resolution + +**Problem:** Snapshots older than `OUTCOME_WINDOW_DAYS` with no outcome are never resolved. + +**Solution:** Add `POST /training/auto-resolve-outcomes` endpoint that: +1. Finds all pending snapshots older than `OUTCOME_WINDOW_DAYS` +2. Fetches the price at `entry_price` + `OUTCOME_WINDOW_DAYS` days later via yfinance +3. Computes P&L and labels WIN (>0.5%) or LOSS (<-0.5%) or NEUTRAL +4. Records the outcome automatically + +**Run:** Via cron or manual trigger. Not automatic on every request. + +--- + +## Files Owned by Squad 4 + +``` +backend/app/api/v1/training.py +live_monitoring/orchestrator/gate_outcome_tracker.py +live_monitoring/data/training/ (data directory) +``` + +## Files Squad 4 Must NOT Touch + +``` +live_monitoring/orchestrator/confluence_gate.py (Squad 2) +live_monitoring/enrichment/apis/dte_cache.py (Squad 3) +frontend/ (Squad 5) +live_monitoring/core/snapshot_state.py (Squad 1) +``` + +--- + +## Acceptance Criteria + +- [ ] `GateOutcomeTracker.log_signal()` accepts and stores `dte_modifier`, `synthesis_score`, `sizing_multiplier` +- [ ] `GET /training/win-rate-summary` returns regime-breakdown win rates +- [ ] `GET /training/export?quality=strict` excludes UNKNOWN regime, confidence < 50, no outcome +- [ ] `SnapshotSchema` Pydantic model is used by all callers of `POST /training/snapshot` +- [ ] `POST /training/auto-resolve-outcomes` resolves pending snapshots older than `OUTCOME_WINDOW_DAYS` +- [ ] Training JSONL is versioned: `kill_chain_snapshots_YYYY-MM.jsonl` (monthly rotation) diff --git a/docs/architecture/mdc/05-command-center.mdc b/docs/architecture/mdc/05-command-center.mdc new file mode 100644 index 0000000..12526b8 --- /dev/null +++ b/docs/architecture/mdc/05-command-center.mdc @@ -0,0 +1,225 @@ +--- +description: Squad 5 — Command Center UI. Exploitation dashboard, signal lineage, DTE visibility, tooltips, WebSocket client. +globs: ["frontend/src/**/*.tsx", "frontend/src/**/*.ts", "frontend/src/components/brief/**", "frontend/src/components/signal-chain/**"] +alwaysApply: false +--- + +# Squad 5: Command Center UI — The Dashboard + +> **Agent Profile:** React/Vite Frontend Ninja. Obsessed with data visibility and UX. +> **Lane:** Frontend only. You consume backend contracts — you do not write backend logic. +> **Owns:** All `frontend/src/` files, `docs/scaffolding/` reference components. + +--- + +## Inherited Gaps (From Prior Audits) + +### From Exploitation Dashboard — Today's Brief Overhaul + +| Gap ID | Description | Severity | Status | +|--------|-------------|----------|--------| +| UI-01 | `ExploitationCommandCenter.tsx` does not show WHY a signal was blocked — gate reason is not surfaced | 🔴 CRITICAL | ❌ OPEN | +| UI-02 | `GateHealthBar.tsx` shows pass/fail count but not the active regime or DTE modifier | 🔴 CRITICAL | ❌ OPEN | +| UI-03 | `VerdictBanner.tsx` shows final verdict but not the confidence adjustment path (raw → adjusted) | 🟡 HIGH | ❌ OPEN | +| UI-04 | `MasterBriefPanels.tsx` has no staleness indicator — user cannot tell if data is 2 min or 2 hours old | 🟡 HIGH | ❌ OPEN | +| UI-05 | `MacroEdgeStrip.tsx` does not show DTE threshold modifiers — operator cannot see if thresholds are tightened/relaxed | 🔴 CRITICAL | ❌ OPEN | + +### From Signal Chain — Full Lineage View + +| Gap ID | Description | Severity | Status | +|--------|-------------|----------|--------| +| UI-06 | `SignalChainView.tsx` exists but shows a flat list — no tree structure showing which gates passed/failed | 🔴 CRITICAL | ❌ OPEN | +| UI-07 | No lineage tooltip on signal cards — clicking a signal shows no explanation of why it fired | 🟡 HIGH | ❌ OPEN | +| UI-08 | `PillarCardCombined.tsx` does not show the regime that was active when the signal fired | 🟡 HIGH | ❌ OPEN | +| UI-09 | Signal chain has no "similar setups" panel — no historical context for current signal | 🟢 MEDIUM | ❌ OPEN | + +### From Signal Chain: Consolidation + Tooltips + LLM Snapshot Training + +| Gap ID | Description | Severity | Status | +|--------|-------------|----------|--------| +| UI-10 | No tooltip on confidence score — user doesn't know if 85% means "strong" or "barely passed" | 🟡 HIGH | ❌ OPEN | +| UI-11 | No tooltip on sizing multiplier — user doesn't know what 1.5x means in dollar terms | 🟡 HIGH | ❌ OPEN | +| UI-12 | WebSocket client has no reconnection logic — connection drops silently, UI shows stale data | 🔴 CRITICAL | ❌ OPEN | + +### From Production Handoff Plan — Today's Brief + Signal Chain + +| Gap ID | Description | Severity | Status | +|--------|-------------|----------|--------| +| UI-13 | `AiBriefingPanel.tsx` oracle path badge (UNIFIED/KC/DEV/OFFLINE) exists but is not visible enough — operators miss it | 🟢 MEDIUM | ❌ OPEN | +| UI-14 | No AAPL/SPY regime state display — when AAPL signal fires, UI doesn't show which regime was used | 🟡 HIGH | ❌ OPEN | + +--- + +## Phase 2 Scope + +### P2-UI-01: Gate Reason Transparency + +**Problem:** Operators cannot see why a signal was blocked. The gate reason string exists in the API response but is not surfaced. + +**Solution:** In `ExploitationCommandCenter.tsx` and `SignalChainView.tsx`, add a `GateReasonBadge` component: + +```tsx +// GateReasonBadge.tsx +interface GateReasonBadgeProps { + blocked: boolean; + reason: string; + gatesPassed: string[]; + gatesFailed: string[]; + regime: string; + sizingMultiplier: number; +} + +// Renders: +// ✅ PASS: LONG SPY (4 gates passed | 1.5x size) +// ⛔ BLOCK: CHOPPY CONFIDENCE BLOCK: Need 90%+ (got 72%) +// Gates passed: THESIS ✓ | DIRECTION ✓ +// Gates failed: CONFIDENCE ✗ +``` + +**Wire into:** `SignalFeedGrid.tsx` — show `GateReasonBadge` on each signal card. + +--- + +### P2-UI-02: DTE Modifier Display in GateHealthBar + +**Problem:** `GateHealthBar.tsx` shows pass/fail count but not the active DTE regime or threshold adjustments. + +**Solution:** Extend `GateHealthBar.tsx` to consume `GET /api/v1/economic/dte-status` (Squad 3 contract): + +```tsx +// GateHealthBar additions: + +``` + +**Visual:** Show a colored badge: +- `HOLD` → gray, "Thresholds: 90% / 70%" +- `CUT_CYCLE` → green, "Thresholds: 81% / 63% (relaxed)" +- `HIKE_RISK` → red, "Thresholds: 95% / 77% (tightened)" + +--- + +### P2-UI-03: Signal Chain Lineage Tree + +**Problem:** `SignalChainView.tsx` is a flat list. No tree showing the gate evaluation path. + +**Solution:** Replace flat list with a `SignalLineageTree` component: + +``` +Signal: LONG SPY @ $585.20 +├── Gate 0: THESIS ✅ (valid) +├── Gate 1: REGIME ✅ (BULLISH — SPY $585.20 between walls $580/$590) +├── Gate 2: SYNTHESIS ✅ (BULLISH bias, 72% score) +├── Gate 2.5: DP PROXIMITY ✅ (support @ $584.50, 89% WR) +└── Gate 3: KILL CHAIN ✅ (2/3 layers active, 1.0x size) + └── Confidence: 85% → 93.5% (DP boost +10%) + └── Sizing: 1.0x +``` + +**Data source:** `gates_passed` and `gates_failed` arrays from `GateResult` — already in the API response. + +--- + +### P2-UI-04: WebSocket Reconnection (Client-Side) + +**Problem:** WebSocket client drops silently. Squad 1 is adding server-side heartbeat — Squad 5 must implement the matching client-side reconnection. + +**Solution:** Create `frontend/src/hooks/useReconnectingWebSocket.ts`: + +```typescript +const RECONNECT_POLICY = { + initialDelayMs: 1000, + maxDelayMs: 30000, + backoffFactor: 2, + maxAttempts: 10, +}; + +export function useReconnectingWebSocket(url: string) { + const [status, setStatus] = useState<"connecting" | "open" | "closed" | "error">("connecting"); + const [lastMessage, setLastMessage] = useState(null); + const attemptRef = useRef(0); + + // Exponential backoff reconnection logic + // Responds to server ping with pong + // Shows connection status badge in UI +} +``` + +**Replace** all direct `new WebSocket(url)` calls in `frontend/src/` with `useReconnectingWebSocket`. + +--- + +### P2-UI-05: Confidence Score Tooltips + +**Problem:** Confidence scores have no explanation. 85% means nothing to an operator without context. + +**Solution:** Add `ConfidenceTooltip` component: + +```tsx +// ConfidenceTooltip.tsx +// On hover over confidence score, shows: +// Raw: 85% +// After regime penalty: 85% (no penalty — BULLISH regime) +// After DP boost: 93.5% (+10% — near support @ $584.50) +// DTE modifier: 0.91 (CUT_CYCLE — thresholds relaxed) +// Sizing: 1.0x (2/3 Kill Chain layers active) +``` + +**Data source:** `raw_confidence`, `adjusted_confidence`, `gates_passed` from `GateResult`. + +--- + +### P2-UI-06: Staleness Indicators + +**Problem:** `MasterBriefPanels.tsx` shows data with no age indicator. + +**Solution:** Add `DataFreshnessBar` component that shows: +- Green: data < 2 min old +- Yellow: data 2-10 min old +- Red: data > 10 min old (with "STALE" badge) + +**Data source:** `scan_time` and `as_of` fields already in `/api/v1/brief/master` response. + +--- + +## Files Owned by Squad 5 + +``` +frontend/src/ (all frontend files) +docs/scaffolding/ (reference components) +``` + +## Files Squad 5 Must NOT Touch + +``` +backend/ (all backend files) +live_monitoring/ (all live monitoring files) +``` + +## Backend Endpoints Squad 5 Consumes (Read-Only) + +| Endpoint | Owner | Purpose | +|----------|-------|---------| +| `GET /api/v1/economic/dte-status` | Squad 3 | DTE modifier + regime label | +| `GET /api/v1/training/win-rate-summary` | Squad 4 | Win rate by regime | +| `GET /api/v1/enrichment/narrative-divergence` | Squad 3 | Narrative divergence signal | +| `GET /debug/memory/history` | Squad 1 | RSS burn curve (ops panel) | +| `GET /api/v1/brief/master` | Squad 1/2/3 | Full brief payload | +| `GET /kill-shots-live` | Squad 2 | Gate result + verdict | + +--- + +## Acceptance Criteria + +- [ ] `GateReasonBadge` shows blocked reason and gates passed/failed on every signal card +- [ ] `GateHealthBar` shows DTE modifier, regime label, and adjusted thresholds +- [ ] `SignalLineageTree` renders gate evaluation path as a tree (not flat list) +- [ ] `useReconnectingWebSocket` implements exponential backoff, responds to server ping +- [ ] `ConfidenceTooltip` shows raw → adjusted confidence path on hover +- [ ] `DataFreshnessBar` shows data age with green/yellow/red indicator +- [ ] AAPL signals show AAPL's regime state (not SPY's) in the signal card diff --git a/railway.toml b/railway.toml new file mode 100644 index 0000000..4f3e5ec --- /dev/null +++ b/railway.toml @@ -0,0 +1,9 @@ +[build] +builder = "DOCKERFILE" +dockerfilePath = "Dockerfile" + +[deploy] +healthcheckPath = "/health" +healthcheckTimeout = 300 +restartPolicyType = "ON_FAILURE" +restartPolicyMaxRetries = 3 diff --git a/requirements-deploy.txt b/requirements-deploy.txt new file mode 100644 index 0000000..1a29ee4 --- /dev/null +++ b/requirements-deploy.txt @@ -0,0 +1,55 @@ +# ============================================================ +# PRODUCTION DEPLOY REQUIREMENTS (Railway / Render) +# Stripped of dev tools, streamlit, paper-trade-only deps. +# Heavy deps (langgraph, langchain-groq) are lazy-imported +# inside their route handlers — not loaded at startup. +# ============================================================ + +# Core +requests>=2.31.0 +numpy>=1.24.0 +yfinance>=0.2.18 +pandas>=2.0.0 +python-dateutil>=2.8.0 +pytz>=2023.3 +python-dotenv>=1.0.0 +beautifulsoup4>=4.12.0 +feedparser>=6.0.10 +pyfedwatch>=1.2.0 +psutil>=5.9.0 + +# FastAPI +fastapi>=0.104.0 +uvicorn[standard]>=0.24.0 +httpx>=0.27.0 + +# LLM / AI +google-generativeai>=0.3.0 +cohere>=5.0.0 +groq>=0.9.0 + +# Kill Chain (Finnhub insider + CFTC COT) +finnhub-python>=2.4.0 +cot_reports>=0.1.0 +supabase>=2.0.0 + +# Technical Analysis +ta>=0.11.0 + +# Analog scorer (MinMaxScaler) +scikit-learn>=1.3.0 + +# LangGraph — lazy-imported in /kill-shots-live route only +# NOT imported at startup — saves ~80MB baseline +langgraph>=0.3.0 +langchain-groq>=0.3.0 + +# NOTE: Intentionally excluded from deploy: +# streamlit>=1.28.0 — demo only, ~60MB baseline +# redis>=5.0.0 — not used in production path +# alpaca-py>=0.21.0 — paper trade only, not needed on Railway +# statsmodels>=0.14.0 — paper trade only +# matplotlib>=3.7.0 — not needed for API +# seaborn>=0.12.0 — not needed for API +# discord.py==2.3.2 — run as separate worker if needed +# pytest, black, flake8 — dev only