From d24bb1f975a66eb413622f75c3dc4a331e2c6a4b Mon Sep 17 00:00:00 2001 From: mrcha033 Date: Thu, 30 Jul 2026 11:54:40 +0900 Subject: [PATCH] chore: move QTA out of the skill marketplace --- .agents/plugins/marketplace.json | 24 - .claude-plugin/marketplace.json | 38 +- .github/workflows/validate.yml | 52 - README.md | 116 +- .../.claude-plugin/plugin.json | 20 - .../.codex-plugin/plugin.json | 35 - .../quant-stock-polling-trader/SKILL.md | 192 -- .../agents/openai.yaml | 4 - .../references/account-snapshot-contract.md | 117 - .../references/broker-contracts.md | 209 -- .../references/daily-pipeline-contract.md | 155 - .../references/execution-contract.md | 279 -- .../fixtures/aggressive-2m-risk-policy.json | 16 - .../kr-market-session-2026-07-27.json | 13 - .../us-market-session-2026-07-26-invalid.json | 13 - .../us-market-session-2026-07-27.json | 13 - .../us-market-session-2026-12-07.json | 13 - .../references/policy-examples.md | 192 -- .../references/systemd-contract.md | 105 - .../scripts/account_snapshot.py | 1199 ------- .../scripts/broker_adapters.py | 2379 -------------- .../scripts/broker_credentials.py | 469 --- .../scripts/daily_pipeline.py | 1565 --------- .../scripts/execution_core.py | 2838 ----------------- .../scripts/freeze_market_session.py | 68 - .../scripts/market_calendar.py | 670 ---- .../scripts/plan_orders.py | 217 -- .../scripts/position_lifecycle.py | 1236 ------- .../scripts/reconcile.py | 390 --- .../scripts/run_session.py | 2655 --------------- .../scripts/systemd_units.py | 1026 ------ .../.claude-plugin/plugin.json | 19 - .../.codex-plugin/plugin.json | 34 - .../skills/quant-stock-technical/SKILL.md | 131 - .../quant-stock-technical/agents/openai.yaml | 4 - .../references/eod-provider-contract.md | 139 - .../references/methodology.md | 162 - .../references/screen-contract.md | 163 - .../references/universe-contract.md | 252 -- .../references/us-eod-provider-contract.md | 60 - .../scripts/analyze_stock.py | 693 ---- .../scripts/analyze_stock_v2.py | 221 -- .../scripts/build_universe_manifest.py | 2194 ------------- .../scripts/collect_open1h_research.py | 920 ------ .../scripts/evaluate_open1h_research.py | 721 ----- .../scripts/fetch_kis_kr_eod.py | 1253 -------- .../scripts/fetch_kis_us_eod.py | 997 ------ .../scripts/screen_universe.py | 1951 ----------- .../scripts/selected_open1h_outcomes.py | 535 ---- .../scripts/walk_forward_open1h.py | 854 ----- .../.claude-plugin/plugin.json | 2 +- .../.codex-plugin/plugin.json | 2 +- .../skills/stock-scenario-story/SKILL.md | 11 +- .../stock-scenario-story/agents/openai.yaml | 2 +- release/catalog.json | 44 +- release/release-notes.md | 7 + skills/quant-stock-polling-trader/SKILL.md | 192 -- .../agents/openai.yaml | 4 - .../references/account-snapshot-contract.md | 117 - .../references/broker-contracts.md | 209 -- .../references/daily-pipeline-contract.md | 155 - .../references/execution-contract.md | 279 -- .../fixtures/aggressive-2m-risk-policy.json | 16 - .../kr-market-session-2026-07-27.json | 13 - .../us-market-session-2026-07-26-invalid.json | 13 - .../us-market-session-2026-07-27.json | 13 - .../us-market-session-2026-12-07.json | 13 - .../references/policy-examples.md | 192 -- .../references/systemd-contract.md | 105 - .../scripts/account_snapshot.py | 1199 ------- .../scripts/broker_adapters.py | 2379 -------------- .../scripts/broker_credentials.py | 469 --- .../scripts/daily_pipeline.py | 1565 --------- .../scripts/execution_core.py | 2838 ----------------- .../scripts/freeze_market_session.py | 68 - .../scripts/market_calendar.py | 670 ---- .../scripts/plan_orders.py | 217 -- .../scripts/position_lifecycle.py | 1236 ------- .../scripts/reconcile.py | 390 --- .../scripts/run_session.py | 2655 --------------- .../scripts/systemd_units.py | 1026 ------ skills/quant-stock-technical/SKILL.md | 131 - .../quant-stock-technical/agents/openai.yaml | 4 - .../references/eod-provider-contract.md | 139 - .../references/methodology.md | 162 - .../references/screen-contract.md | 163 - .../references/universe-contract.md | 252 -- .../references/us-eod-provider-contract.md | 60 - .../scripts/analyze_stock.py | 693 ---- .../scripts/analyze_stock_v2.py | 221 -- .../scripts/build_universe_manifest.py | 2194 ------------- .../scripts/collect_open1h_research.py | 920 ------ .../scripts/evaluate_open1h_research.py | 721 ----- .../scripts/fetch_kis_kr_eod.py | 1253 -------- .../scripts/fetch_kis_us_eod.py | 997 ------ .../scripts/screen_universe.py | 1951 ----------- .../scripts/selected_open1h_outcomes.py | 535 ---- .../scripts/walk_forward_open1h.py | 854 ----- skills/stock-scenario-story/SKILL.md | 11 +- .../stock-scenario-story/agents/openai.yaml | 2 +- tests/test_daily_pipeline.py | 401 --- tests/test_handoff_integration.py | 9 +- tests/test_kis_kr_eod.py | 283 -- tests/test_kis_us_eod.py | 372 --- tests/test_market_calendar.py | 143 - tests/test_marketplace_packaging.py | 6 +- tests/test_position_lifecycle.py | 565 ---- tests/test_qta2_execution.py | 143 - tests/test_qta2_market_regime.py | 174 - tests/test_quant_execution.py | 1576 --------- tests/test_screen_universe.py | 106 - tests/test_systemd_units.py | 438 --- tests/test_universe_builder.py | 299 -- 113 files changed, 53 insertions(+), 59447 deletions(-) delete mode 100644 plugins/quant-stock-polling-trader/.claude-plugin/plugin.json delete mode 100644 plugins/quant-stock-polling-trader/.codex-plugin/plugin.json delete mode 100644 plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/SKILL.md delete mode 100644 plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/agents/openai.yaml delete mode 100644 plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/account-snapshot-contract.md delete mode 100644 plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/broker-contracts.md delete mode 100644 plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/daily-pipeline-contract.md delete mode 100644 plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/execution-contract.md delete mode 100644 plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/fixtures/aggressive-2m-risk-policy.json delete mode 100644 plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/fixtures/kr-market-session-2026-07-27.json delete mode 100644 plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/fixtures/us-market-session-2026-07-26-invalid.json delete mode 100644 plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/fixtures/us-market-session-2026-07-27.json delete mode 100644 plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/fixtures/us-market-session-2026-12-07.json delete mode 100644 plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/policy-examples.md delete mode 100644 plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/systemd-contract.md delete mode 100644 plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/account_snapshot.py delete mode 100755 plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/broker_adapters.py delete mode 100644 plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/broker_credentials.py delete mode 100644 plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/daily_pipeline.py delete mode 100755 plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/execution_core.py delete mode 100644 plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/freeze_market_session.py delete mode 100644 plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/market_calendar.py delete mode 100755 plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/plan_orders.py delete mode 100644 plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/position_lifecycle.py delete mode 100755 plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/reconcile.py delete mode 100755 plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/run_session.py delete mode 100644 plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/systemd_units.py delete mode 100644 plugins/quant-stock-technical/.claude-plugin/plugin.json delete mode 100644 plugins/quant-stock-technical/.codex-plugin/plugin.json delete mode 100644 plugins/quant-stock-technical/skills/quant-stock-technical/SKILL.md delete mode 100644 plugins/quant-stock-technical/skills/quant-stock-technical/agents/openai.yaml delete mode 100644 plugins/quant-stock-technical/skills/quant-stock-technical/references/eod-provider-contract.md delete mode 100644 plugins/quant-stock-technical/skills/quant-stock-technical/references/methodology.md delete mode 100644 plugins/quant-stock-technical/skills/quant-stock-technical/references/screen-contract.md delete mode 100644 plugins/quant-stock-technical/skills/quant-stock-technical/references/universe-contract.md delete mode 100644 plugins/quant-stock-technical/skills/quant-stock-technical/references/us-eod-provider-contract.md delete mode 100644 plugins/quant-stock-technical/skills/quant-stock-technical/scripts/analyze_stock.py delete mode 100644 plugins/quant-stock-technical/skills/quant-stock-technical/scripts/analyze_stock_v2.py delete mode 100755 plugins/quant-stock-technical/skills/quant-stock-technical/scripts/build_universe_manifest.py delete mode 100644 plugins/quant-stock-technical/skills/quant-stock-technical/scripts/collect_open1h_research.py delete mode 100644 plugins/quant-stock-technical/skills/quant-stock-technical/scripts/evaluate_open1h_research.py delete mode 100644 plugins/quant-stock-technical/skills/quant-stock-technical/scripts/fetch_kis_kr_eod.py delete mode 100644 plugins/quant-stock-technical/skills/quant-stock-technical/scripts/fetch_kis_us_eod.py delete mode 100755 plugins/quant-stock-technical/skills/quant-stock-technical/scripts/screen_universe.py delete mode 100644 plugins/quant-stock-technical/skills/quant-stock-technical/scripts/selected_open1h_outcomes.py delete mode 100644 plugins/quant-stock-technical/skills/quant-stock-technical/scripts/walk_forward_open1h.py delete mode 100644 skills/quant-stock-polling-trader/SKILL.md delete mode 100644 skills/quant-stock-polling-trader/agents/openai.yaml delete mode 100644 skills/quant-stock-polling-trader/references/account-snapshot-contract.md delete mode 100644 skills/quant-stock-polling-trader/references/broker-contracts.md delete mode 100644 skills/quant-stock-polling-trader/references/daily-pipeline-contract.md delete mode 100644 skills/quant-stock-polling-trader/references/execution-contract.md delete mode 100644 skills/quant-stock-polling-trader/references/fixtures/aggressive-2m-risk-policy.json delete mode 100644 skills/quant-stock-polling-trader/references/fixtures/kr-market-session-2026-07-27.json delete mode 100644 skills/quant-stock-polling-trader/references/fixtures/us-market-session-2026-07-26-invalid.json delete mode 100644 skills/quant-stock-polling-trader/references/fixtures/us-market-session-2026-07-27.json delete mode 100644 skills/quant-stock-polling-trader/references/fixtures/us-market-session-2026-12-07.json delete mode 100644 skills/quant-stock-polling-trader/references/policy-examples.md delete mode 100644 skills/quant-stock-polling-trader/references/systemd-contract.md delete mode 100644 skills/quant-stock-polling-trader/scripts/account_snapshot.py delete mode 100755 skills/quant-stock-polling-trader/scripts/broker_adapters.py delete mode 100644 skills/quant-stock-polling-trader/scripts/broker_credentials.py delete mode 100644 skills/quant-stock-polling-trader/scripts/daily_pipeline.py delete mode 100755 skills/quant-stock-polling-trader/scripts/execution_core.py delete mode 100644 skills/quant-stock-polling-trader/scripts/freeze_market_session.py delete mode 100644 skills/quant-stock-polling-trader/scripts/market_calendar.py delete mode 100755 skills/quant-stock-polling-trader/scripts/plan_orders.py delete mode 100644 skills/quant-stock-polling-trader/scripts/position_lifecycle.py delete mode 100755 skills/quant-stock-polling-trader/scripts/reconcile.py delete mode 100755 skills/quant-stock-polling-trader/scripts/run_session.py delete mode 100644 skills/quant-stock-polling-trader/scripts/systemd_units.py delete mode 100644 skills/quant-stock-technical/SKILL.md delete mode 100644 skills/quant-stock-technical/agents/openai.yaml delete mode 100644 skills/quant-stock-technical/references/eod-provider-contract.md delete mode 100644 skills/quant-stock-technical/references/methodology.md delete mode 100644 skills/quant-stock-technical/references/screen-contract.md delete mode 100644 skills/quant-stock-technical/references/universe-contract.md delete mode 100644 skills/quant-stock-technical/references/us-eod-provider-contract.md delete mode 100644 skills/quant-stock-technical/scripts/analyze_stock.py delete mode 100644 skills/quant-stock-technical/scripts/analyze_stock_v2.py delete mode 100755 skills/quant-stock-technical/scripts/build_universe_manifest.py delete mode 100644 skills/quant-stock-technical/scripts/collect_open1h_research.py delete mode 100644 skills/quant-stock-technical/scripts/evaluate_open1h_research.py delete mode 100644 skills/quant-stock-technical/scripts/fetch_kis_kr_eod.py delete mode 100644 skills/quant-stock-technical/scripts/fetch_kis_us_eod.py delete mode 100755 skills/quant-stock-technical/scripts/screen_universe.py delete mode 100644 skills/quant-stock-technical/scripts/selected_open1h_outcomes.py delete mode 100644 skills/quant-stock-technical/scripts/walk_forward_open1h.py delete mode 100644 tests/test_daily_pipeline.py delete mode 100644 tests/test_kis_kr_eod.py delete mode 100644 tests/test_kis_us_eod.py delete mode 100644 tests/test_market_calendar.py delete mode 100644 tests/test_position_lifecycle.py delete mode 100644 tests/test_qta2_execution.py delete mode 100644 tests/test_qta2_market_regime.py delete mode 100755 tests/test_quant_execution.py delete mode 100644 tests/test_screen_universe.py delete mode 100644 tests/test_systemd_units.py delete mode 100755 tests/test_universe_builder.py diff --git a/.agents/plugins/marketplace.json b/.agents/plugins/marketplace.json index e6dff3e..e934641 100644 --- a/.agents/plugins/marketplace.json +++ b/.agents/plugins/marketplace.json @@ -16,18 +16,6 @@ }, "category": "Productivity" }, - { - "name": "quant-stock-technical", - "source": { - "source": "local", - "path": "./plugins/quant-stock-technical" - }, - "policy": { - "installation": "AVAILABLE", - "authentication": "ON_INSTALL" - }, - "category": "Finance" - }, { "name": "stock-scenario-story", "source": { @@ -87,18 +75,6 @@ "authentication": "ON_INSTALL" }, "category": "Productivity" - }, - { - "name": "quant-stock-polling-trader", - "source": { - "source": "local", - "path": "./plugins/quant-stock-polling-trader" - }, - "policy": { - "installation": "AVAILABLE", - "authentication": "ON_INSTALL" - }, - "category": "Finance" } ] } diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 81b744a..3237160 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -24,30 +24,12 @@ "codex" ] }, - { - "name": "quant-stock-technical", - "displayName": "Quant Stock Technical", - "source": "./plugins/quant-stock-technical", - "description": "Acquire audited KIS EOD data, screen four exchanges, and evaluate selected opening-hour outcomes.", - "version": "0.4.6", - "author": { - "name": "mrcha033" - }, - "homepage": "https://github.com/mrcha033/skills", - "repository": "https://github.com/mrcha033/skills", - "category": "finance", - "tags": [ - "stocks", - "technical-analysis", - "quantitative" - ] - }, { "name": "stock-scenario-story", "displayName": "Stock Scenario Story", "source": "./plugins/stock-scenario-story", "description": "Research and narrate a sourced, numbers-free market story after a validated quant handoff.", - "version": "0.1.0", + "version": "0.1.1", "author": { "name": "mrcha033" }, @@ -131,24 +113,6 @@ "reply", "retrieval" ] - }, - { - "name": "quant-stock-polling-trader", - "displayName": "Quant Stock Polling Trader", - "source": "./plugins/quant-stock-polling-trader", - "description": "Build and supervise deterministic daily KIS shadow sessions from official calendars through reconciliation.", - "version": "0.5.4", - "author": { - "name": "mrcha033" - }, - "homepage": "https://github.com/mrcha033/skills", - "repository": "https://github.com/mrcha033/skills", - "category": "finance", - "tags": [ - "stocks", - "broker-api", - "polling" - ] } ] } diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 30568e8..5e29110 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -26,36 +26,6 @@ jobs: PY - name: Validate distribution catalog synchronization run: python -B scripts/sync_distribution.py --check - - name: Run deterministic calculator self-test - run: python -B skills/quant-stock-technical/scripts/analyze_stock.py --self-test - - name: Run first-hour research calculator self-test - run: python -B skills/quant-stock-technical/scripts/analyze_stock_v2.py --self-test - - name: Run read-only opening-hour collector self-test - run: python -B skills/quant-stock-technical/scripts/collect_open1h_research.py --self-test - - name: Run opening-hour evaluator self-test - run: python -B skills/quant-stock-technical/scripts/evaluate_open1h_research.py --self-test - - name: Run selected opening-hour outcomes self-test - run: python -B skills/quant-stock-technical/scripts/selected_open1h_outcomes.py --self-test - - name: Run selected opening-hour walk-forward self-test - run: python -B skills/quant-stock-technical/scripts/walk_forward_open1h.py --self-test - - name: Run four-exchange universe builder self-test - run: python -B skills/quant-stock-technical/scripts/build_universe_manifest.py --self-test - - name: Run deterministic universe screen self-test - run: python -B skills/quant-stock-technical/scripts/screen_universe.py --self-test - - name: Run KIS Korea EOD collector self-test - run: python -B skills/quant-stock-technical/scripts/fetch_kis_kr_eod.py --self-test - - name: Run KIS U.S. EOD collector self-test - run: python -B skills/quant-stock-technical/scripts/fetch_kis_us_eod.py --self-test - - name: Run account snapshot collector self-test - run: python -B skills/quant-stock-polling-trader/scripts/account_snapshot.py self-test - - name: Run official market calendar self-test - run: python -B skills/quant-stock-polling-trader/scripts/market_calendar.py self-test - - name: Run recurring daily pipeline self-test - run: python -B skills/quant-stock-polling-trader/scripts/daily_pipeline.py self-test - - name: Run strategy-position lifecycle self-test - run: python -B skills/quant-stock-polling-trader/scripts/position_lifecycle.py --self-test - - name: Run systemd generator self-test - run: python -B skills/quant-stock-polling-trader/scripts/systemd_units.py self-test - name: Run mandatory handoff-gate self-test run: python -B skills/stock-scenario-story/scripts/validate_quant_handoff.py --self-test - name: Run numbers-free story gate self-test @@ -82,28 +52,6 @@ jobs: run: python -B tests/test_handoff_integration.py - name: Run advisor integration test run: python -B tests/test_advisor_review.py - - name: Run universe builder regression - run: python -B tests/test_universe_builder.py - - name: Run four-exchange screen contract regression - run: python -B tests/test_screen_universe.py - - name: Run screening and execution regression - run: python -B tests/test_quant_execution.py - - name: Run QTA2 execution and market-regime regression - run: | - python -B tests/test_qta2_execution.py - python -B tests/test_qta2_market_regime.py - - name: Run strategy-position ledger regression - run: python -B tests/test_position_lifecycle.py - - name: Run systemd unit generator regression - run: python -B tests/test_systemd_units.py - - name: Run KIS U.S. EOD regression - run: python -B tests/test_kis_us_eod.py - - name: Run KIS Korea EOD regression - run: python -B tests/test_kis_kr_eod.py - - name: Run official market calendar regression - run: python -B tests/test_market_calendar.py - - name: Run recurring daily pipeline regression - run: python -B tests/test_daily_pipeline.py - name: Validate marketplace packaging run: python -B tests/test_marketplace_packaging.py - name: Validate deterministic release packaging diff --git a/README.md b/README.md index e163098..ae9ac60 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # MrCha Skills -A public, multi-topic collection of portable Agent Skills by `mrcha033`, packaged as eight independently installable marketplace plugins for Codex and Claude Code. +A public, multi-topic collection of portable Agent Skills by `mrcha033`, packaged as six independently installable marketplace plugins for Codex and Claude Code. Each directory under `skills/` remains a self-contained skill built around `SKILL.md`. Each marketplace plugin contains exactly one matching skill; none adds MCP servers or hooks. Broker credentials stay local and are never packaged. @@ -15,8 +15,6 @@ codex plugin add agent-finish-line@mrcha-skills codex plugin add yonsei-central-student-governance-counsel@mrcha-skills codex plugin add learnus-course-copilot@mrcha-skills codex plugin add katok-reply-reuse@mrcha-skills -codex plugin add quant-stock-technical@mrcha-skills -codex plugin add quant-stock-polling-trader@mrcha-skills codex plugin add stock-scenario-story@mrcha-skills ``` @@ -31,8 +29,6 @@ claude plugin install agent-finish-line@mrcha-skills claude plugin install yonsei-central-student-governance-counsel@mrcha-skills claude plugin install learnus-course-copilot@mrcha-skills claude plugin install katok-reply-reuse@mrcha-skills -claude plugin install quant-stock-technical@mrcha-skills -claude plugin install quant-stock-polling-trader@mrcha-skills claude plugin install stock-scenario-story@mrcha-skills ``` @@ -111,60 +107,21 @@ The General Student Council corpus is sourced from the public Drive linked by th ## Finance skills -### Quant Stock Technical - -`quant-stock-technical` calculates reproducible technical opinions, risk, entry, stop, target, and a historical technical-strength score from completed daily adjusted OHLCV data. It can also build and screen a deterministic, source-hashed intersection of KOSPI, KOSDAQ, NYSE, and NASDAQ listings without changing `qta-1.0.0`. It excludes news, fundamentals, sentiment, analyst opinion, discretionary model judgment, and broker mutations. - -`quant-stock-polling-trader` is a separately installable execution plugin. Its -daily path is a direct `prepare → snapshot/plan → shadow entry → reconcile` -workflow for KIS. It does not require approved source digests, provenance -signatures, account-identity HMACs, trading-arm files, or Toss/NH exports. -Additional Toss/NH exposure files may be supplied when the user wants them -included, but they are not prerequisites. -Its first-hour runner precomputes the slow path before the open, warms -authentication 30 seconds early, admits only broker-rate-feasible candidate -counts, polls on absolute rank-ordered cycles, and records cycle/quote/submit -latency. Live mutation remains disabled. -Its systemd generator emits an execution-readable strict bundle and hardened -units into an empty directory. The command itself is non-activating, but an -explicit request for actual automation is not complete until the reviewed -units are installed, enabled or started as requested, and their user-systemd -state is verified. -The four-exchange v2 universe is currently qualified from KIS stock masters, so -v2 planning is KIS-only; Toss remains available only through the legacy -contract until a Toss-qualified universe snapshot is implemented and verified. -V2 planning also requires a local SHA-256-bound market-session snapshot, -the latest completed session date, and fresh same-instant account/exposure -snapshots. - -For KIS, keep paper and live app keys separate. Prefer process environment or -an OS secret store; use `~/.config/mrcha-skills/secrets.env` with permission -`0600` only as a fallback. Check status first: +### QTA migration -```bash -python3 skills/quant-stock-polling-trader/scripts/broker_credentials.py status --environment paper -``` - -If it reports `BLOCKED`, run `configure` yourself in an interactive local -terminal so credentials never enter chat or an agent-owned PTY: - -```bash -python3 skills/quant-stock-polling-trader/scripts/broker_credentials.py configure --environment paper -``` - -After configuration, let the agent recheck presence and verify token issuance -without printing or persisting the token: - -```bash -python3 skills/quant-stock-polling-trader/scripts/broker_credentials.py status --environment paper -python3 skills/quant-stock-polling-trader/scripts/broker_credentials.py auth-check --environment paper -``` - -Authentication does not enable an order, and live mutation remains disabled. +`quant-stock-technical` and `quant-stock-polling-trader` are no longer +distributed as marketplace skills. The QTA implementation and its automation +now live in a dedicated standalone project, where long-running market-data, +execution, ledger, and calibration jobs can be managed as one application. +Existing users should remove those two marketplace plugins after migrating +their local runtime; this repository does not publish a replacement plugin. ### Stock Scenario Story -`stock-scenario-story` is its qualitative antithesis. It runs only after validating the complete result from `quant-stock-technical`, hides every upstream number, researches the surrounding company and market context, and tells a sourced, theatrical, numbers-free story purely for entertainment. +`stock-scenario-story` runs only after validating a complete external QTA +result matching the legacy `quant-stock-technical/v1` contract, hides every +upstream number, researches the surrounding company and market context, and +tells a sourced, theatrical, numbers-free story purely for entertainment. ## Versioned release packages @@ -218,8 +175,6 @@ If marketplace installation is unavailable, clone the repository and copy only t ```bash git clone https://github.com/mrcha033/skills.git mrcha-skills -cp -R mrcha-skills/skills/quant-stock-technical ~/.codex/skills/ -cp -R mrcha-skills/skills/quant-stock-polling-trader ~/.codex/skills/ cp -R mrcha-skills/skills/stock-scenario-story ~/.codex/skills/ cp -R mrcha-skills/skills/advisor-review ~/.codex/skills/ cp -R mrcha-skills/skills/agent-finish-line ~/.codex/skills/ @@ -228,7 +183,10 @@ cp -R mrcha-skills/skills/learnus-course-copilot ~/.codex/skills/ cp -R mrcha-skills/skills/katok-reply-reuse ~/.codex/skills/ ``` -Start a new Codex task after copying. Invoke `$advisor-review` explicitly when you want an independent review. For the paired finance workflow, invoke `$quant-stock-technical` first and pass its unmodified JSON result to `$stock-scenario-story`. +Start a new Codex task after copying. Invoke `$advisor-review` explicitly when +you want an independent review. For the finance story workflow, pass an +unmodified external QTA result matching `quant-stock-technical/v1` to +`$stock-scenario-story`. ## Standalone Claude Code skills @@ -257,42 +215,9 @@ marketplace and update normally. ChatGPT and Claude web uploads remain copied artifacts, so those users must upload the newly versioned file; the platform does not currently turn a local upload into a repository subscription. -## Input - -The deterministic calculator requires finalized ticker and same-exchange -benchmark CSV files with: - -```text -date,open,high,low,close,adjusted_close,volume -``` - -Also provide the market, ticker, and valid effective tick size. The calculator -requires at least 756 shared completed sessions and recommends 1,260. - -The four-exchange builder additionally requires dated official listing and -broker-master snapshots, explicit adjusted-EOD file mappings, benchmark -mappings, effective-dated tick contracts, and a hashed per-exchange catalog -coverage minimum. Missing data is recorded as an exclusion; it is never -fetched or guessed during the entry window. - ## Validation ```bash -python3 -B skills/quant-stock-technical/scripts/analyze_stock.py --self-test -python3 -B skills/quant-stock-technical/scripts/build_universe_manifest.py --self-test -python3 -B skills/quant-stock-technical/scripts/screen_universe.py --self-test -python3 -B skills/quant-stock-technical/scripts/fetch_kis_kr_eod.py --self-test -python3 -B skills/quant-stock-technical/scripts/fetch_kis_us_eod.py --self-test -python3 -B skills/quant-stock-polling-trader/scripts/execution_core.py --self-test -python3 -B skills/quant-stock-polling-trader/scripts/broker_adapters.py --self-test -python3 -B skills/quant-stock-polling-trader/scripts/broker_credentials.py self-test -python3 -B skills/quant-stock-polling-trader/scripts/account_snapshot.py self-test -python3 -B skills/quant-stock-polling-trader/scripts/market_calendar.py self-test -python3 -B skills/quant-stock-polling-trader/scripts/daily_pipeline.py self-test -python3 -B skills/quant-stock-polling-trader/scripts/systemd_units.py self-test -python3 -B skills/quant-stock-polling-trader/scripts/plan_orders.py --self-test -python3 -B skills/quant-stock-polling-trader/scripts/run_session.py self-test -python3 -B skills/quant-stock-polling-trader/scripts/reconcile.py --self-test python3 -B skills/stock-scenario-story/scripts/validate_quant_handoff.py --self-test python3 -B skills/stock-scenario-story/scripts/validate_story_text.py --self-test python3 -B skills/advisor-review/scripts/build_context_packet.py --self-test @@ -306,19 +231,10 @@ python3 -B skills/katok-reply-reuse/scripts/rank_replies.py --self-test python3 -B skills/yonsei-central-student-governance-counsel/scripts/self_test.py python3 -B tests/test_handoff_integration.py python3 -B tests/test_advisor_review.py -python3 -B tests/test_universe_builder.py -python3 -B tests/test_screen_universe.py -python3 -B tests/test_quant_execution.py -python3 -B tests/test_kis_us_eod.py -python3 -B tests/test_market_calendar.py -python3 -B tests/test_daily_pipeline.py -python3 -B tests/test_systemd_units.py python3 -B scripts/sync_distribution.py --check python3 -B tests/test_marketplace_packaging.py python3 -B tests/test_release_packaging.py claude plugin validate . --strict ``` -The score is historical technical strength, not a probability of profit. Calculated price levels do not account for suitability, position sizing, fees, taxes, FX, slippage, broker rules, or fill probability. - No software license has been granted yet. Public availability does not grant permission to copy, modify, or redistribute the contents. diff --git a/plugins/quant-stock-polling-trader/.claude-plugin/plugin.json b/plugins/quant-stock-polling-trader/.claude-plugin/plugin.json deleted file mode 100644 index bec07ec..0000000 --- a/plugins/quant-stock-polling-trader/.claude-plugin/plugin.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", - "name": "quant-stock-polling-trader", - "displayName": "Quant Stock Polling Trader", - "version": "0.5.4", - "description": "Run a direct deterministic daily KIS shadow workflow from official calendars through reconciliation.", - "author": { - "name": "mrcha033", - "url": "https://github.com/mrcha033" - }, - "homepage": "https://github.com/mrcha033/skills", - "repository": "https://github.com/mrcha033/skills", - "keywords": [ - "stocks", - "broker-api", - "polling", - "execution" - ], - "skills": "./skills/" -} diff --git a/plugins/quant-stock-polling-trader/.codex-plugin/plugin.json b/plugins/quant-stock-polling-trader/.codex-plugin/plugin.json deleted file mode 100644 index a4c5cdf..0000000 --- a/plugins/quant-stock-polling-trader/.codex-plugin/plugin.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "name": "quant-stock-polling-trader", - "version": "0.5.4", - "description": "Run a direct deterministic daily KIS shadow workflow.", - "author": { - "name": "mrcha033", - "url": "https://github.com/mrcha033" - }, - "homepage": "https://github.com/mrcha033/skills", - "repository": "https://github.com/mrcha033/skills", - "keywords": [ - "stocks", - "broker-api", - "polling", - "execution" - ], - "skills": "./skills/", - "interface": { - "displayName": "Quant Stock Polling Trader", - "shortDescription": "Simple daily KIS shadow automation", - "longDescription": "Freeze official calendars and four-exchange inputs, create date-bound KIS snapshots and plans, run deterministic first-hour shadow polling, reconcile, and supervise with systemd without approval-digest or trading-arm gates.", - "developerName": "mrcha033", - "category": "Finance", - "capabilities": [ - "Interactive", - "Deterministic execution" - ], - "websiteURL": "https://github.com/mrcha033/skills", - "defaultPrompt": [ - "Securely guide KIS credential setup and verify authentication.", - "Prepare a direct deterministic KIS live/shadow polling session.", - "Generate and explicitly activate verified daily KR and US shadow systemd jobs." - ] - } -} diff --git a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/SKILL.md b/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/SKILL.md deleted file mode 100644 index 9674a9c..0000000 --- a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/SKILL.md +++ /dev/null @@ -1,192 +0,0 @@ ---- -name: quant-stock-polling-trader -description: Prepare and supervise deterministic Korean and U.S. KIS shadow sessions from a frozen quant-stock-technical screen. Use when the user wants secure KIS credential setup, pre-open screening, settled-cash sizing, first-hour quote polling, reconciliation, or recurring systemd automation. The daily path is shadow-only and sends no order, correction, or cancellation mutations. ---- - -# Quant Stock Polling Trader - -Run a small, direct pipeline: - -```text -prepare -> snapshot and plan -> first-hour shadow session -> reconcile -``` - -The default daily workflow does not use approved source digests, provenance -signatures, account-identity HMACs, trading-arm files, or mandatory Toss/NH -exports. Do not add those gates back unless the user explicitly asks for a -separate audited deployment profile. - -## Credentials - -Keep credentials out of chat, repositories, command arguments, plans, and -status files. Resolve them from the process environment first and -`~/.config/mrcha-skills/secrets.env` second. The fallback file must be a -regular user-owned `0600` file. - -Use separate KIS paper and live values: - -```text -QTA_KIS_PAPER_APP_KEY -QTA_KIS_PAPER_APP_SECRET -QTA_KIS_PAPER_ACCOUNT_PREFIX -QTA_KIS_PAPER_ACCOUNT_PRODUCT -QTA_KIS_LIVE_APP_KEY -QTA_KIS_LIVE_APP_SECRET -QTA_KIS_LIVE_ACCOUNT_PREFIX -QTA_KIS_LIVE_ACCOUNT_PRODUCT -``` - -For the daily workflow use `live` credentials only to read the live account -and market data while keeping `mode=shadow`. - -Check presence first: - -```bash -python3 scripts/broker_credentials.py status --environment live -``` - -If configuration is missing, ask the user to run this in their own terminal: - -```bash -python3 scripts/broker_credentials.py configure --environment live -``` - -Never ask the user to paste a credential into chat. Report missing variable -names only. Wait for the user to confirm that interactive configuration has -finished, then check presence again and authenticate: - -```bash -python3 scripts/broker_credentials.py status --environment live -python3 scripts/broker_credentials.py auth-check --environment live -``` - -Authentication is proven only when the result says `AUTHENTICATED`. An -authenticated token does not enable live trading. - -## Daily workflow - -Read `references/daily-pipeline-contract.md`, then: - -1. Use one stable `qta-daily-shadow-config/v2` file. -2. Run `scripts/daily_pipeline.py prepare` before the selected market opens. - It freezes the official calendar, stops normally on a closed market, - refreshes only that market's exchange pair and adjusted EOD cache, and - creates the deterministic screen. -3. Run `scripts/daily_pipeline.py snapshot` shortly before the open. It reads - KIS settled cash, positions, and open orders, optionally merges any - explicitly configured external exposure files, and creates the plan. -4. Run `scripts/daily_pipeline.py entry` before the 30-second authentication - warm-up. The Python runner polls the first hour mechanically and then - reconciles the local intent ledger. -5. Use `scripts/systemd_units.py generate` for recurring user services and - timers. When the user asks to automate, generation is intermediate work: - verify, install, enable, start, and inspect the requested units. - -Daily preparation starts at `01:00` in each market timezone. This leaves eight -hours before the Korean open and eight and a half hours before the U.S. open, -without making a Korean session wait for U.S. EOD or vice versa. -Finish an initial multi-year bootstrap before relying on a daily timer; the -timer is intended to resume and increment an existing cache. - -Set optional `qta_method_version` to `qta-2.0.0` for the research-only -first-hour model. The runner then performs one read-only benchmark-regime call -per active exchange and cycle before stock quotes. A negative, stale, -wrong-session, or unavailable regime observation blocks new entries for that -cycle. QTA2 cannot run in paper or live mode. - -To receive optional ntfy completion/block notifications, place these values in -the same mode-`0600` environment file used by the generated services: - -```text -QTA_NTFY_TOPIC_URL=https://ntfy.example/a-long-private-topic -QTA_NTFY_TOKEN=optional-access-token -``` - -The full topic URL and token are never written to a bundle, status file, or -journal. Notification delivery is best-effort and cannot turn a completed -shadow stage into a trading failure. A journal receipt records only -`SENT`, `FAILED`, or `DISABLED`. - -The state path is: - -```text -RUNTIME_ROOT/state/// -``` - -The account alias is nonsecret and configured by the user. A current session -is never entered twice. An unresolved broker mutation from a paper-mode run -still requires reconciliation before another entry. - -Filled strategy quantities are stored in -`RUNTIME_ROOT/state///positions.sqlite3`, outside the -date directory. Before a later session reads quotes, the runner reconciles -that ledger against the frozen KIS account snapshot. Matching positions roll -forward as `CARRY_EXIT_ONLY`; quantity differences, working orders, duplicate -ownership, or a prior `MANUAL_BLOCK` stop new entries. Holdings not created by -this strategy remain visible as unmanaged broker positions and are never -silently adopted or sold. - -## Direct planning and polling - -For a non-recurring session, read `references/execution-contract.md`, -`references/account-snapshot-contract.md`, and -`references/broker-contracts.md`. - -```bash -python3 scripts/account_snapshot.py collect --job /absolute/account-job.json -python3 scripts/plan_orders.py \ - --screen /absolute/screen.json \ - --account /absolute/account.json \ - --exposure /absolute/exposure.json \ - --risk /absolute/risk.json \ - --execution /absolute/execution.json \ - --output /absolute/order-plan.json -python3 scripts/run_session.py run \ - --plan /absolute/order-plan.json \ - --broker kis-live \ - --mode shadow \ - --state-dir /absolute/state \ - --output /absolute/session-status.json -python3 scripts/reconcile.py \ - --plan /absolute/order-plan.json \ - --broker kis-live \ - --state-dir /absolute/state \ - --output /absolute/reconciliation-status.json -``` - -Additional Toss or NH exposure files are optional. If the user configures one, -validate and merge it; if none is configured, continue with the KIS account -view without fabricating an empty external-broker file. - -## Fixed safety boundary - -- The daily path accepts only `broker=kis-live`, `mode=shadow`. -- `live_enabled` stays `false` and `api_mutation_count` stays `0`. -- Do not submit, amend, or cancel an order in shadow mode. -- Use settled cash only. Exclude borrowed buying power. -- Keep margin, shorting, automatic FX, and additions to existing positions - disabled. -- Use the official calendar snapshot; do not guess holidays, early closes, - timezones, or DST. -- Use complete prior-session data; do not fill gaps with internet quotes or - estimates. -- Keep ranking, sizing, quote checks, and cycle timing deterministic. -- Do not put an LLM in the polling loop. -- Never claim profit, guaranteed returns, or a recommendation. - -Run all bundled self-tests after changing the implementation: - -```bash -python3 scripts/execution_core.py --self-test -python3 scripts/freeze_market_session.py --self-test -python3 scripts/broker_adapters.py --self-test -python3 scripts/broker_credentials.py self-test -python3 scripts/account_snapshot.py self-test -python3 scripts/market_calendar.py self-test -python3 scripts/daily_pipeline.py self-test -python3 scripts/plan_orders.py --self-test -python3 scripts/position_lifecycle.py --self-test -python3 scripts/run_session.py self-test -python3 scripts/reconcile.py --self-test -python3 scripts/systemd_units.py self-test -``` diff --git a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/agents/openai.yaml b/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/agents/openai.yaml deleted file mode 100644 index f3a6248..0000000 --- a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/agents/openai.yaml +++ /dev/null @@ -1,4 +0,0 @@ -interface: - display_name: "Quant Stock Polling Trader" - short_description: "Simple KIS shadow polling and systemd automation" - default_prompt: "Use $quant-stock-polling-trader to configure a direct KIS shadow workflow, prepare the market, snapshot and plan, run first-hour polling, reconcile, and automate it with systemd without approval-digest or trading-arm gates." diff --git a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/account-snapshot-contract.md b/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/account-snapshot-contract.md deleted file mode 100644 index d9e9973..0000000 --- a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/account-snapshot-contract.md +++ /dev/null @@ -1,117 +0,0 @@ -# Account snapshot collector contract - -Use `scripts/account_snapshot.py` only for KIS live credentials in shadow mode. -It sends read-only requests and emits one `qta-account-snapshot/v2`, one -`qta-exposure-snapshot/v2`, and one readable status with -`api_mutation_count: 0`. - -## Inputs - -The job is nonsecret JSON. Every path must be absolute: - -```json -{ - "schema": "qta-account-snapshot-job/v2", - "broker": "kis-live", - "mode": "shadow", - "market": "KR", - "account_alias": "kis-live-kr", - "universe_manifest": "/secure/runtime/universe-manifest.json", - "fx_to_krw": "1", - "manual_exposure_components": [ - "/secure/runtime/exposure/toss.json", - "/secure/runtime/exposure/nh.json" - ], - "manual_component_max_age_seconds": 1800, - "output_account_path": "/secure/runtime/account-kr.json", - "output_exposure_path": "/secure/runtime/exposure-kr.json", - "output_status_path": "/secure/runtime/account-kr-status.json" -} -``` - -For a U.S. job, set `fx_to_krw` to `KIS_PRESENT_BALANCE` so the same -`CTRP6504R` snapshot supplies its positive `frst_bltn_exrt` value. A caller may -instead bind an exact observed positive numeric value, in which case the -receipt records `fx_source: JOB_INPUT`. KIS legitimately omits the USD cash -row when the account has neither settled USD nor a U.S. position. In that -case the collector freezes `settled_cash: "0"` and `fx_to_krw: null`, records -`fx_source: NOT_APPLICABLE:NO_SETTLED_USD_OR_POSITION`, and the planner -deterministically produces no positive quantity. It does not use -auto-exchange buying power or invent an exchange rate. If a position exists -without a USD cash row, the collector uses the position's positive -`bass_exrt`; conflicting rates or cash rows are `BLOCKED`. - -Never insert a placeholder or silently reuse an old rate. The collector -validates the manifest hash and uses its `broker_symbol -> exchange` mapping. -Anything held or working outside KOSPI, KOSDAQ, NYSE, and NASDAQ is `BLOCKED`; -extend the exposure schema instead of assigning a false exchange. - -KIS credentials follow the resolution rules in `SKILL.md`. Account routing -values and secrets are never written. The nonsecret `account_alias` identifies -the local account view. - -## Optional additional exposure - -Toss and NH files are optional. Use them only when the user wants those -holdings included and has a fresh export or verified account view: - -```bash -python3 scripts/account_snapshot.py manual-template \ - --broker toss \ - --source-as-of 2026-07-27T08:45:00+09:00 \ - --output /secure/runtime/exposure/toss.json -``` - -Each configured component has exactly: - -```json -{ - "schema": "qta-exposure-component/v1", - "broker": "toss", - "source_kind": "verified-manual", - "source_as_of": "2026-07-27T08:45:00+09:00", - "positions": [ - { - "market": "US", - "exchange": "NASDAQ", - "symbol": "AAPL", - "quantity": "0.182408", - "market_value_krw": "89096" - } - ] -} -``` - -`quantity` may be `null` only when the source genuinely omits it. Include -fractional holdings. The collector rejects invalid configured components, but -an empty `manual_exposure_components` list is valid and means “use KIS only.” -It does not scrape a browser, infer a symbol from a Korean display name, or -fabricate a source time. - -## Cash derivation - -The account contract requires settled, unborrowed cash: - -- KR uses the minimum nonnegative value among KIS `dnca_tot_amt`, - `nxdy_excc_amt`, and `prvs_rcdl_excc_amt`. -- US uses the minimum nonnegative USD value among `frcr_dncl_amt_2`, - `frcr_drwg_psbl_amt_1`, and `nxdy_frcr_drwg_psbl_amt`. -- An absent USD row means zero settled USD; it never authorizes KRW - auto-exchange buying power. -- Loan amounts are preserved separately as `borrowed_buying_power` and never - added to `settled_cash`. - -This is a settlement-safe derivation. A new broker field or account product -must be contract-probed before changing it. - -## Collection - -```bash -python3 scripts/account_snapshot.py collect \ - --job /secure/runtime/account-snapshot-kr-job.json -``` - -The collector freezes one common `as_of` after the KIS reads and any configured -component validation complete. The account and exposure objects therefore -meet the planner's exact timestamp equality requirement. Missing or stale -configured inputs return `BLOCKED`; unconfigured external brokers do not. diff --git a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/broker-contracts.md b/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/broker-contracts.md deleted file mode 100644 index 98723fe..0000000 --- a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/broker-contracts.md +++ /dev/null @@ -1,209 +0,0 @@ -# Broker contracts - -Checked against official sources on 2026-07-26. Re-run contract probes whenever -the broker changes an API version or order rule. - -## Toss Securities Open API 1.2.4 - -Source of truth: -`https://openapi.tossinvest.com/openapi-docs/latest/openapi.json`. - -Toss currently exposes one production REST server and no documented paper -server. Keep mutations disabled until the account's staged Open API access is -active. - -Core paths: - -```text -POST /oauth2/token -GET /api/v1/accounts -GET /api/v1/prices -GET /api/v1/orderbook -GET /api/v1/stocks -GET /api/v1/holdings -GET /api/v1/buying-power -GET /api/v1/orders -POST /api/v1/orders -GET /api/v1/orders/{orderId} -POST /api/v1/orders/{orderId}/modify -POST /api/v1/orders/{orderId}/cancel -POST /api/v1/conditional-orders -``` - -`GET /api/v1/stocks` accepts one to 200 requested symbols and returns market, -security type, common-share flag, listing status, currency, and Korean -liquidation/suspension details. It does not enumerate the full catalog. Once -this account has API access, query the frozen official-universe symbols in -deterministic batches, persist the complete responses as a dated snapshot, and -add a separate Toss-qualified universe contract. Until that snapshot path is -implemented and verified, never reuse the KIS-qualified v2 `broker_symbol` for -Toss. - -Send `Authorization: Bearer ...`; account operations also require -`X-Tossinvest-Account: accountSeq`. Centralize token refresh because issuing a -new token invalidates the prior token for that client. If -`QTA_TOSS_ACCESS_TOKEN` supplies an external token, require its timezone-aware -ISO-8601 expiry in `QTA_TOSS_ACCESS_TOKEN_EXPIRES_AT`; block when exactly one -of the pair is present. - -Keep `accountSeq` in the runtime credential environment. Do not copy it into a -plan, status file, or error message. - -Bind every Toss mutation request hash to the uppercase HTTP method, exact API -path, account sequence header value, and canonical request body. An identical -body for another account or path must never share the same hash. - -Pace `MARKET_DATA`, `ORDER`, and `ORDER_HISTORY` independently. The initial -client limits are 10, 3, and 5 TPS respectively; the conservative 3 TPS order -limit remains safe during the documented 09:00–09:10 KST peak. Only lower a -client limit when `X-RateLimit-Limit` reports a smaller current value. Do not -blindly retry an order mutation after a 429 or timeout. - -After a mutation has been sent, an HTTP 5xx, malformed success body, or -success body without an order ID is ambiguous and must enter `UNKNOWN`. -Only an explicit structured Toss 4xx rejection is terminal `REJECTED`. - -Both price and order-book source timestamps are nullable in the schema. Require -both during Toss polling, parse each as timezone-aware ISO-8601, and block the -entry when either is missing, invalid, future-dated beyond tolerance, or older -than the frozen quote-age limit. Select the lowest positive ask and highest -positive bid rather than trusting array position. - -`clientOrderId` is optional on creates, limited to 36 safe characters, and -idempotent for only 10 minutes. A same-key/same-body retry returns the original -result; a changed body conflicts. Order queries cannot search or return that -client ID. Modify and cancel return new operation order IDs and have no -idempotency key. Never blindly retry them after a timeout. - -Use decimal strings. KR orders are whole-share. U.S. fractional buys are -amount-based market orders and fractional sells are quantity-based market -orders during regular hours. Keep the deterministic price-capped path -whole-share and limit-only. - -The current schema has a documented contradiction around -`GET /orders?status=CLOSED`. Treat both a successful page and -`closed-not-supported` as valid contract-probe outcomes; do not depend on -CLOSED history until the user's account confirms it. - -Toss live status remains `BLOCKED`. Do not promote it until all three -mutation-lifecycle gaps are closed and fault-injected: - -- recover an accepted-then-timeout create with the same `clientOrderId` and - byte-equivalent body inside the 10-minute window, then reconcile it; -- verify the original order reaches a terminal state after a cancel operation, - including cancel/fill races and rate limiting; -- register and continuously supervise the intended stop/take-profit protection - after an entry fill. - -## Korea Investment & Securities Open API - -Official examples: -`https://github.com/koreainvestment/open-trading-api`. - -```text -live https://openapi.koreainvestment.com:9443 -paper https://openapivts.koreainvestment.com:29443 -POST /oauth2/tokenP -``` - -KIS issues separate app keys and secrets for live and paper use. Resolve -process environment first, then an OS secret-store injection, then the local -`0600` `~/.config/mrcha-skills/secrets.env` fallback. Use the scoped -`QTA_KIS_PAPER_*` and `QTA_KIS_LIVE_*` names documented in `SKILL.md`; never -persist access tokens. The credential helper may prove token issuance, but it -must not submit an order or claim account/order readiness from OAuth alone. - -Use `authorization`, `appkey`, `appsecret`, `tr_id`, and `custtype: P`. -Account requests contain `CANO` and `ACNT_PRDT_CD`. - -Keep `CANO` and `ACNT_PRDT_CD` in the runtime credential environment. Do not -copy them into a plan, status file, or error message. - -Core paths and current official TR IDs: - -```text -KR quote GET /uapi/domestic-stock/v1/quotations/inquire-price - FHKST01010100 -KR order book GET /uapi/domestic-stock/v1/quotations/inquire-asking-price-exp-ccn - FHKST01010200 -KR balance GET /uapi/domestic-stock/v1/trading/inquire-balance - live TTTC8434R / paper VTTC8434R -KR open orders GET /uapi/domestic-stock/v1/trading/inquire-psbl-rvsecncl - live TTTC0084R -KR fills GET /uapi/domestic-stock/v1/trading/inquire-daily-ccld - live TTTC0081R / paper VTTC0081R -KR buy POST /uapi/domestic-stock/v1/trading/order-cash - live TTTC0012U / paper VTTC0012U -KR sell POST same path - live TTTC0011U / paper VTTC0011U -KR modify/cancel POST /uapi/domestic-stock/v1/trading/order-rvsecncl - live TTTC0013U / paper VTTC0013U - -US quote GET /uapi/overseas-price/v1/quotations/price - HHDFS00000300 -US order book GET /uapi/overseas-price/v1/quotations/inquire-asking-price - live HHDFS76200100 / paper unsupported -US balance GET /uapi/overseas-stock/v1/trading/inquire-balance - live TTTS3012R / paper VTTS3012R -US present balance GET /uapi/overseas-stock/v1/trading/inquire-present-balance - live CTRP6504R / paper VTRP6504R -US fills GET /uapi/overseas-stock/v1/trading/inquire-ccnl - live TTTS3035R / paper VTTS3035R -US open orders GET /uapi/overseas-stock/v1/trading/inquire-nccs - live TTTS3018R -US buy POST /uapi/overseas-stock/v1/trading/order - live TTTT1002U / paper VTTT1002U -US sell POST same path - live TTTT1006U / paper VTTT1001U -US modify/cancel POST /uapi/overseas-stock/v1/trading/order-rvsecncl - live TTTT1004U / paper VTTT1004U -``` - -KIS does not expose the local deterministic intent ID as a broker search key in -these order calls. Persist the request first. After an ambiguous submit, -reconcile orders, fills, and balances; do not automatically resubmit. -An HTTP 5xx, malformed success body, or successful response without `ODNO` is -ambiguous. Only a structured nonzero `rt_cd` with a broker `msg_cd` is an -authoritative rejection. - -The two documented KIS domestic REST quote responses do not currently provide -a verified independent trading date, trade timestamp, and order-book -timestamp contract. The adapter refuses to synthesize the date from local -receipt time or reuse `aspr_acpt_hour` for both observations. Domestic KIS -polling therefore remains fail-closed unless a contract-probed response -supplies a consistent `stck_bsop_date`, `stck_cntg_hour`, and -`aspr_acpt_hour`, or a separately verified timestamp-bearing source is added. - -Paper REST limits are lower than production limits and supported order types -can be narrower. The official U.S. paper examples support limit orders; use -whole-share limit orders for the initial contract. - -The U.S. order-book TR `HHDFS76200100` is production-only. Do not combine a -production quote with a paper mutation in one automated session. Validate U.S. -trigger decisions through a mutation-free production shadow session, and test -the paper order request and response serialization separately with explicit -fixtures or a bounded contract probe. A U.S. KIS paper polling session must -return `BLOCKED` before fetching a quote. - -For live U.S. shadow polling, use the one `HHDFS76200100` response only when it -contains last, bid, ask, `dymd`, and `dhms`. Assign its one source timestamp to -those values because they come from that same response. Do not fall back to the -separate price endpoint and label its value with the order-book timestamp. - -Keep KIS live promotion disabled until all of these blockers are implemented -and fault-injected: - -- paginate domestic and overseas order history using `tr_cont` and the - `CTX_AREA_FK/NK100|200` continuation keys; -- recover accepted-then-timeout mutations by reconciling uniquely matched - orders, fills, open orders, and balances without automatic resubmission; -- normalize domestic and overseas accepted, partial, filled, cancelled, and - rejected records using the broker's full status fields; -- enforce a process-shared app-key/account rate limiter, including the - temporary lower limit for newly registered production applications. - -The read-only account collector currently supports only KIS live credentials -with shadow execution. It uses `TTTC8434R + TTTC0084R` for Korea and -`CTRP6504R + TTTS3018R` for the U.S. It follows documented continuation -headers and keys, freezes only after all reads complete, and never sends an -order, modify, or cancel request. diff --git a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/daily-pipeline-contract.md b/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/daily-pipeline-contract.md deleted file mode 100644 index a8638d6..0000000 --- a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/daily-pipeline-contract.md +++ /dev/null @@ -1,155 +0,0 @@ -# Daily shadow pipeline - -`daily_pipeline.py` turns one stable configuration into the current market -date's artifacts. It supports only: - -```text -broker=kis-live -mode=shadow -live_enabled=false -api_mutation_count=0 -``` - -The stages are: - -1. `prepare`: freeze the official market calendar, stop normally on - `MARKET_CLOSED`, refresh official membership/KIS masters/adjusted EOD and - benchmarks for KOSPI/KOSDAQ or NYSE/NASDAQ, build that market's universe, - and screen it. -2. `snapshot`: read KIS settled cash, positions, and open orders; merge any - explicitly configured additional exposure files; create the execution - policy and deterministic plan; and preview request serialization. -3. `entry`: start the deterministic first-hour shadow runner, then perform - bounded read-only reconciliation. - -There is no source-approval digest, provenance receipt, account-identity HMAC, -or trading-arm artifact in this workflow. Those artifacts are not needed for -a no-mutation shadow session and must not be made prerequisites. - -## Stable configuration - -Use `qta-daily-shadow-config/v2`: - -```json -{ - "schema": "qta-daily-shadow-config/v2", - "runtime_root": "/secure/qta", - "broker": "kis-live", - "mode": "shadow", - "history_start_date": "2021-01-01", - "minimum_sessions": 756, - "qta_method_version": "qta-2.0.0", - "request_interval_ms": 120, - "catalog_coverage_contract": { - "schema": "qta-catalog-coverage-contract/v1", - "minimum_ratio_by_exchange": { - "KOSPI": "1", - "KOSDAQ": "1", - "NYSE": "1", - "NASDAQ": "1" - }, - "minimum_screenable_ratio_by_exchange": { - "KOSPI": "0.5", - "KOSDAQ": "0.5", - "NYSE": "0.5", - "NASDAQ": "0.5" - } - }, - "selector": { - "selector_version": "qta-screen-1.1.0", - "min_total_score": "60", - "eligible_setup_statuses": ["READY"], - "top_k_by_exchange": { - "KOSPI": 2, - "KOSDAQ": 2, - "NYSE": 2, - "NASDAQ": 2 - }, - "min_selected_by_exchange": { - "KOSPI": 0, - "KOSDAQ": 0, - "NYSE": 0, - "NASDAQ": 0 - }, - "max_blocked_fraction": "0.02" - }, - "risk_policy_path": "/secure/qta/config/risk-policy.json", - "execution_policy": { - "snapshot_max_age_seconds": 1800, - "poll_interval_seconds": 2, - "quote_max_age_seconds": 5, - "max_spread_bps": "25", - "max_gap_bps": "20", - "trigger_mode": "CROSS_FROM_BELOW", - "order_ttl_seconds": 30, - "order_type": "LIMIT", - "time_in_force": "DAY", - "allow_partial_fill": true, - "cancel_remainder_at_window_end": true - }, - "account_aliases": { - "KR": "kis-live-kr", - "US": "kis-live-us" - }, - "prepare_complete_seconds_before_open": 600 -} -``` - -`qta_method_version` is optional and defaults to `qta-1.0.0`. Select -`qta-2.0.0` only for shadow research; the screen and runner then enforce its -liquidity and same-session market-regime contracts. - -Three other optional fields may be added: - -```json -{ - "exposure_component_paths": [ - "/secure/qta/manual/{session_date}/toss.json", - "/secure/qta/manual/{session_date}/nh.json" - ], - "exposure_component_max_age_seconds": 86400 -} -``` - -If a path is configured, it must exist and pass validation. If no path is -configured, the pipeline uses the KIS account view and continues. It never -creates fake empty exposure files. - -## State and restart behavior - -The intent ledger is: - -```text -RUNTIME_ROOT/state/// -``` - -The strategy-position ledger persists across dates at: - -```text -RUNTIME_ROOT/state///positions.sqlite3 -``` - -The workflow descriptor is -`RUNTIME_ROOT/current/.json`, and date-specific artifacts are under -`RUNTIME_ROOT/workflows//`. - -The stages move through `PREPARED`, `PLANNED`, and `READY`. A same-date stage -may reuse its completed output. Entry writes an attempt marker before polling -and refuses a second entry for the same session. - -The runner still keeps a single-writer ledger, treats ambiguous paper -mutations as `UNKNOWN`, and reconciles nonterminal states. These rules protect -paper-mode compatibility; daily shadow mode itself sends zero mutations. - -## Status - -Each stage writes one readable JSON status containing its stage, status, -market, session date, reason, details, timestamp, `live_enabled`, and -`api_mutation_count`. Status files are operational records, not signatures; -they do not contain receipt hashes. - -When a generated systemd service has `QTA_NTFY_TOPIC_URL` in its protected -environment file, the outer systemd wrapper sends a best-effort completion or -block notification after this status is written. Notification configuration is -not part of the deterministic plan inputs, and delivery failure never retries -the stage or changes its status. diff --git a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/execution-contract.md b/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/execution-contract.md deleted file mode 100644 index edbd101..0000000 --- a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/execution-contract.md +++ /dev/null @@ -1,279 +0,0 @@ -# Open-one-hour execution contract - -## Version boundary - -Keep three independent contracts: - -```text -qta-1.0.0/2.0.0 completed T-1 adjusted daily data -> technical payload -qta-screen-1.0.0 legacy KR/US manifest -> deterministic candidates -qta-screen-1.1.0 selected-market manifest -> exchange-ranked candidates -open1h-exec-2.0.0 broker/account snapshots -> intents and order lifecycle -``` - -The entry window is `09:00–10:00 Asia/Seoul` for Korea and -`09:30–10:30 America/New_York` for the U.S. The planner validates those exact -local clocks with the IANA timezone database and rejects a supplied UTC offset -that disagrees with `ZoneInfo`, including an incorrect U.S. DST offset. It also -requires a frozen `qta-market-session/v1` JSON snapshot whose local file -SHA-256 and canonical `session_hash` match the policy. The snapshot binds the -scheduled open session, regular open/close, and previous session date. A -weekday check is only a second defense; holiday and early-close truth comes -from the frozen calendar snapshot. - -## Required frozen inputs - -- `qta-screen/v1` or `qta-screen/v2` artifact and canonical hash; -- broker, environment, nonsecret account alias, market, and trading date; -- broker account snapshot including settled cash, positions, open orders, and - snapshot time; -- cross-broker exposure snapshot, including fractional holdings even when - exact quantity is unavailable; -- numeric risk and execution policies; -- fee, tax, FX, tick, lot, fractional, and settlement semantics; -- hashed local market-calendar snapshot and entry-window timestamps; -- code, configuration, and universe hashes. - -A v2 screen requires `qta-exposure-snapshot/v2`. Every exposure position must -name one of `KOSPI`, `KOSDAQ`, `NYSE`, or `NASDAQ`; match positions by -`exchange:broker_symbol`. Never collapse a v2 position back to a market-only -key. Order intents use the manifest's verified `broker_symbol`, while -`canonical_symbol` and `data_symbol` remain provenance fields. - -For v2, broker symbols are qualified by the KIS stock-master snapshots used by -the universe builder. The planner therefore requires account `broker: "kis"`. -It rejects Toss planning even if a caller supplies a self-consistent screen -hash. Toss remains unavailable for v2 until a separate Toss-qualified -all-universe snapshot contract exists; a symbol lookup endpoint is not evidence -that every selected symbol was qualified for that broker. - -The v2 screen must match the complete `finalize_screen_v2` top-level contract: -source/schema/status/method and selector versions, analysis date, manifest and -selector hashes, counts and blocked fraction, normalized selector, all four -selected arrays, decisions, and screen hash. The planner validates the exact -field set, selector hash, counts, eligibility reasons, decision-to-selection -identity, instrument provenance, and canonical screen hash. It validates the -exact emitted READY or BLOCKED QTA schema, binds every READY payload's -`analysis_date` to the screen date, and recomputes the documented -total/medium/long/short/risk/ticker ranking before accepting exchange ranks or -selection order. It also checks the qta-1.0.0 score formula within the -two-decimal rounding envelope, score/opinion and score/setup relationships, -resolved-tick alignment, and the emitted 2R take-profit geometry. A smaller, -reranked, or semantically forged object cannot make itself acceptable merely -by hashing its own contents. - -For `qta-2.0.0`, the validator instead recomputes its first-hour total, -verifies the market-specific turnover floor, and requires the exact -research-only market-regime policy. QTA2 can run only in shadow mode. Before -stock quotes in each cycle, the runner reads KOSPI Composite, KOSDAQ -Composite, Nasdaq Composite, or the documented S&P 500 broad-market proxy for -NYSE candidates. It verifies the current-session timestamp, a maximum age of -90 seconds, and return from previous close of at least zero basis points. A -failed check skips new entries for that exchange and cycle. - -Each account position must include `market` and `symbol`, plus `exchange` for a -v2 screen. Each item in `open_orders` must represent a currently working order -and include `side, market, symbol`, plus `exchange` for v2. Existing positions -and working buy orders occupy exposure keys and concurrent-position slots -before any new intent is sized. - -For v2, `analysis_date` must equal the calendar artifact's -`previous_session_date`. For both legacy and v2 screens, account and -cross-broker exposure snapshots must be timezone-aware, share one exact frozen -`as_of` instant, fall on the entry session's local date, be no earlier than the -calendar `source_as_of`, and be no later than the window start. Their age at -the window start must not exceed the policy's `snapshot_max_age_seconds`; that -field is capped at 3,600 seconds. - -The exact market-session object contains: - -```text -schema, provider, source_id, source_as_of, source_path, source_sha256, -market, timezone, session_date, previous_session_date, scheduled_status, -regular_open, regular_close, session_hash -``` - -`source_path` must be absolute and contain a JSON object exactly matching the -same fields except `source_path, source_sha256, session_hash`. -`source_sha256` is recomputed from that file. `session_hash` is canonical -SHA-256 over the full market-session object excluding only `session_hash`. -The complete object and hash are copied into the plan context and revalidated -from disk by the session runner, not only by the planner. - -`scheduled_status: OPEN` means the calendar scheduled a regular session. It is -not evidence that the exchange is currently healthy. Emergency closures, -market-wide halts, broker incidents, and stale quotes remain runtime -`ENTRY_FREEZE` conditions. The current market-level contract treats KOSPI and -KOSDAQ as sharing the KR session, and NYSE and NASDAQ as sharing the U.S. -session. If venue calendars diverge, migrate to exchange-specific artifacts -instead of guessing. - -Lending availability is not settled cash. Keep it in a distinct field and -exclude it from quantity calculations. - -## Deterministic planning - -Calculate every quantity from the same pre-session snapshot: - -```text -effective_loss_per_share = - entry_limit - stop - + entry_limit * round_trip_cost_bps / 10000 - -quantity = floor_to_lot(min( - per_trade_risk_native / effective_loss_per_share, - max_symbol_notional_native / entry_limit, - settled_unborrowed_cash_native / worst_case_all_in_price -)) -``` - -Subtract existing and working exposure before creating an intent. If additions -to existing cross-broker exposure are disabled, reject the symbol even when it -is held at another broker or only as a fractional share. - -Do not reallocate unused cash after a skipped or rejected order. Keep ranking, -tie-breaking, cash reservation, and candidate count fixed for the session. - -Every plan embeds the complete normalized `screen, account, exposure, risk, -execution` objects under exact `frozen_inputs`. Before preview, polling, or -reconciliation, the runner validates the exact plan/context/window/policy and -intent schemas, verifies each deterministic intent ID and intent hash, then -runs the planner again from those frozen inputs and requires the rebuilt plan -to equal the supplied plan. Recomputing only `plan_hash`, or even making a -self-consistent quantity/cash/intent rehash, cannot change the result without -also changing the frozen inputs. - -These hashes prove deterministic internal consistency only. They are not -signatures or authorization gates. Keep account numbers and secrets out of -frozen inputs. Use the nonsecret account alias for the local state path; broker -routing values go directly from the credential loader to the adapter. - -## Trigger and polling - -Poll at absolute times `window_start + n * interval`. Skip late cycles rather -than replaying them. Run all slow work—universe construction, daily QTA, -ranking, sizing, account/exposure freezing, calendar validation, and request -serialization—before the opening bell. The hot path contains only a -deterministic rank-ordered quote, trigger evaluation, persisted transition, -and broker request. - -KIS KR and Toss need two REST calls per waiting intent: price and order book. -KIS-live US requires one same-source order-book call that contains last, bid, -ask, date, and time; its timestamp-less price fallback is forbidden. The -adapters cannot otherwise provide a safely joined bid/ask batch, so do not -claim batch latency. Before accepting a session, compute a broker- and -market-specific pacing budget from candidate count, quote-call count, and one -possible submit per intent in paper mode. Reserve one pacing interval for each -request, including response/persistence headroom: - -```text -KIS paper: ceil(intent_count * 3 * 1.05s) -KIS live KR shadow: ceil(intent_count * 2 * 0.12s) -KIS live US shadow: ceil(intent_count * 1 * 0.12s) -Toss shadow: ceil(intent_count * 2 * 0.10s) -``` - -QTA2 adds one KIS benchmark request per active exchange to the same cycle -budget. - -Reject a configured interval below that floor. This is only an API-pacing -feasibility bound, not proof of network latency. Production promotion also -requires observed shadow percentiles. - -Start the runner before the open. It acquires/refreshes the broker token -30 seconds before the scheduled open and blocks the session if authentication -fails or finishes late. Before warm-up, resolve every venue and serialize every -potential submit request; a missing venue or invalid request must fail before -the opening bell. Every cycle records scheduled/start/end times, lateness, -duration, quote/status/submit latency, skipped-cycle counts, and expected REST -request counts. Non-mutation quote telemetry is committed once per cycle; -`RESERVED`, `SUBMITTING`, acknowledgements, and other mutation states retain -individual `synchronous=FULL` commits. If a quote consumes the cycle deadline, -freeze new entries instead of evaluating later ranks from a delayed cycle. - -Do not start a submission inside its 10-second HTTP timeout plus broker pacing -margin before the window end. Pass the absolute window end into the broker -limiter, refuse a paced send that would cross it, and record both submit start -and acknowledgement time. If an already-sent request acknowledges after the -window, persist the acknowledgement, freeze entries, then immediately enter -the normal cancel/reconciliation phase. - -Require a fresh quote and a price-capped limit order: - -```text -last_price >= qta_entry -best_ask <= entry_limit -quote_age <= max_quote_age -spread_bps <= max_spread_bps -``` - -Freeze whether the trigger accepts a price already above entry at the first -poll (`AT_OR_ABOVE`) or requires a below-to-above transition -(`CROSS_FROM_BELOW`). Never infer this choice at runtime. -For `CROSS_FROM_BELOW`, update the prior-price baseline only after quote -status, instrument identity, positive bid/ask/last, receipt freshness, and both -source timestamps pass. A stale, missing-source, mismatched, or malformed -quote cannot seed a later crossing. - -If an adapter does not provide bid/ask, do not substitute a market order. -Either obtain the order book through the broker-specific endpoint or block the -candidate. - -At the window end, freeze new entries, cancel outstanding paper-mode entry -orders, and reconcile. Continue managing already filled paper positions. -Automatic market flattening is not implemented. - -The account/market strategy-position ledger is stored one directory above the -date-specific intent ledger. Authoritative cumulative entry fills create or -update a position with its original stop, target, tick, FX, and plan identity. -Before the next session, exact broker quantities are reconciled and valid -residuals roll to `CARRY_EXIT_ONLY`; they are excluded from new additions. -The bundled lifecycle module deterministically calculates stop, gap-stop, -target, close-liquidation, partial-fill, and daily-loss decisions, but the -current polling service does not yet send live exit mutations. Do not describe -that decision module as active automatic liquidation. - -## State machine - -```text -PLANNED -> WAIT_TRIGGER -> RESERVED -> SUBMITTING -SUBMITTING -> ACKNOWLEDGED | UNKNOWN -ACKNOWLEDGED -> PARTIALLY_FILLED | FILLED | CANCEL_PENDING | REJECTED -PARTIALLY_FILLED -> FILLED | CANCEL_PENDING -CANCEL_PENDING -> CANCELLED | FILLED | UNKNOWN -UNKNOWN -> RECONCILING -> ACKNOWLEDGED | PARTIALLY_FILLED | FILLED -UNKNOWN -> RECONCILING -> MANUAL_BLOCK -``` - -Persist the intent and exact request hash before `SUBMITTING`. Store broker -status values without discarding unknown values. - -One state directory may never silently bridge plans. Immediately after opening -the ledger, reject any nonterminal intent—or any unresolved `MANUAL_BLOCK`— -whose `plan_hash` or intent ID is outside the current plan. Also reject -`RESERVED`, `SUBMITTING`, `UNKNOWN`, `RECONCILING`, `CANCEL_PENDING`, and -`MANUAL_BLOCK` even when they belong to the same plan. Require broker evidence -and explicit clearance before warming authentication, reading quotes, or -sending a new mutation. - -When reconciliation observes `ACKNOWLEDGED` or `PARTIALLY_FILLED` after a -cancel request, keep the local state at `CANCEL_PENDING`, return `BLOCKED`, and -monitor again. A cancel acknowledgement is not terminal proof that the -original order can no longer fill. - -## Live trading - -Live trading is not supported. `kis-live/shadow` may read live account and -market data, but `live_enabled` remains false and the adapter sends no order, -correction, or cancellation mutation. Enabling live mutations requires a -separate implementation and a separate user request; it is not a hidden -promotion state in this skill. - -An offline request preview uses a placeholder account identifier. Its request -hash proves deterministic serialization for that placeholder only. A preview -lists potential requests for all planned intents and does not claim that an -intraday trigger was observed. - -Default kill behavior is `ENTRY_FREEZE`: stop mutations, cancel known open -entry orders when safe, preserve protective orders, and reconcile. Never infer -permission to liquidate existing user positions. diff --git a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/fixtures/aggressive-2m-risk-policy.json b/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/fixtures/aggressive-2m-risk-policy.json deleted file mode 100644 index 70acb21..0000000 --- a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/fixtures/aggressive-2m-risk-policy.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "allow_auto_fx": false, - "allow_borrowed_cash": false, - "allow_existing_additions": false, - "allow_margin": false, - "allow_short": false, - "base_currency": "KRW", - "cash_buffer_bps": "0", - "max_concurrent_positions": 10000, - "max_daily_loss_krw": "2000000", - "max_symbol_notional_krw": "2000000", - "per_trade_risk_krw": "2000000", - "round_trip_cost_bps": "30", - "schema": "qta-risk-policy/v1", - "whole_shares_only": true -} diff --git a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/fixtures/kr-market-session-2026-07-27.json b/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/fixtures/kr-market-session-2026-07-27.json deleted file mode 100644 index 1701817..0000000 --- a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/fixtures/kr-market-session-2026-07-27.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "schema": "qta-market-session/v1", - "provider": "DETERMINISTIC_TEST_FIXTURE", - "source_id": "fixture-kr-2026-07-27", - "source_as_of": "2026-07-27T08:00:00+09:00", - "market": "KR", - "timezone": "Asia/Seoul", - "session_date": "2026-07-27", - "previous_session_date": "2026-07-24", - "scheduled_status": "OPEN", - "regular_open": "2026-07-27T09:00:00+09:00", - "regular_close": "2026-07-27T15:30:00+09:00" -} diff --git a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/fixtures/us-market-session-2026-07-26-invalid.json b/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/fixtures/us-market-session-2026-07-26-invalid.json deleted file mode 100644 index 7f04e37..0000000 --- a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/fixtures/us-market-session-2026-07-26-invalid.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "schema": "qta-market-session/v1", - "provider": "DETERMINISTIC_TEST_FIXTURE", - "source_id": "fixture-us-invalid-sunday-2026-07-26", - "source_as_of": "2026-07-26T08:00:00-04:00", - "market": "US", - "timezone": "America/New_York", - "session_date": "2026-07-26", - "previous_session_date": "2026-07-24", - "scheduled_status": "OPEN", - "regular_open": "2026-07-26T09:30:00-04:00", - "regular_close": "2026-07-26T16:00:00-04:00" -} diff --git a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/fixtures/us-market-session-2026-07-27.json b/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/fixtures/us-market-session-2026-07-27.json deleted file mode 100644 index b1508cc..0000000 --- a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/fixtures/us-market-session-2026-07-27.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "schema": "qta-market-session/v1", - "provider": "DETERMINISTIC_TEST_FIXTURE", - "source_id": "fixture-us-2026-07-27", - "source_as_of": "2026-07-27T08:00:00-04:00", - "market": "US", - "timezone": "America/New_York", - "session_date": "2026-07-27", - "previous_session_date": "2026-07-24", - "scheduled_status": "OPEN", - "regular_open": "2026-07-27T09:30:00-04:00", - "regular_close": "2026-07-27T16:00:00-04:00" -} diff --git a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/fixtures/us-market-session-2026-12-07.json b/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/fixtures/us-market-session-2026-12-07.json deleted file mode 100644 index fa2381f..0000000 --- a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/fixtures/us-market-session-2026-12-07.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "schema": "qta-market-session/v1", - "provider": "DETERMINISTIC_TEST_FIXTURE", - "source_id": "fixture-us-2026-12-07", - "source_as_of": "2026-12-07T08:00:00-05:00", - "market": "US", - "timezone": "America/New_York", - "session_date": "2026-12-07", - "previous_session_date": "2026-12-04", - "scheduled_status": "OPEN", - "regular_open": "2026-12-07T09:30:00-05:00", - "regular_close": "2026-12-07T16:00:00-05:00" -} diff --git a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/policy-examples.md b/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/policy-examples.md deleted file mode 100644 index 0623d95..0000000 --- a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/policy-examples.md +++ /dev/null @@ -1,192 +0,0 @@ -# Frozen policy examples - -These values define a software canary, not an expected-return claim. Replace -dates and account-derived values only after a broker snapshot and calendar -preflight. - -## Risk policy - -```json -{ - "schema": "qta-risk-policy/v1", - "base_currency": "KRW", - "per_trade_risk_krw": "10000", - "max_symbol_notional_krw": "200000", - "max_concurrent_positions": 1, - "max_daily_loss_krw": "20000", - "round_trip_cost_bps": "30", - "cash_buffer_bps": "100", - "allow_existing_additions": false, - "allow_borrowed_cash": false, - "allow_margin": false, - "allow_short": false, - "allow_auto_fx": false, - "whole_shares_only": true -} -``` - -`borrowed_buying_power` may be present in an account snapshot for audit, but -this policy always excludes it. A `qta-screen/v2` plan requires -`"broker": "kis"` because its `broker_symbol` values are qualified against KIS -stock-master snapshots. Do not change this field to `toss`; build and validate -a separate Toss-qualified universe contract first. - -For the user's explicitly requested effectively uncapped 2,000,000 KRW shadow -allocation, use -`references/fixtures/aggressive-2m-risk-policy.json`. Its numeric fields are -set to the full operating capital and its concurrent-position cap is above any -feasible screen size, so they do not impose a smaller portfolio limit. -`round_trip_cost_bps: 30` remains part of the fill-cost calculation and -`cash_buffer_bps: 0` reserves no extra cash. This does not authorize borrowed -cash, margin, shorting, or live mutation. - -## Account snapshot - -```json -{ - "schema": "qta-account-snapshot/v2", - "broker": "kis", - "environment": "paper", - "account_alias": "kis-paper-kr", - "market": "KR", - "currency": "KRW", - "as_of": "2026-07-27T08:50:00+09:00", - "settled_cash": "200000", - "borrowed_buying_power": "5000000", - "fx_to_krw": "1", - "positions": [], - "open_orders": [] -} -``` - -Never put a real account number, account product code, Toss account sequence, -app key, or secret in this file. `account_alias` is a nonsecret local label; -the adapter reads the actual routing values from the credential environment. - -After planning, run and reconcile directly: - -```bash -python3 scripts/run_session.py run \ - --plan /secure/runtime/order-plan.json \ - --broker kis-paper \ - --mode paper \ - --state-dir /secure/runtime/state - -python3 scripts/reconcile.py \ - --plan /secure/runtime/order-plan.json \ - --broker kis-paper \ - --state-dir /secure/runtime/state -``` - -Both commands use the broker-routing environment. Store those values in an OS -secret store or process environment; do not write them into these JSON files. - -## Execution policy - -Korea: - -```json -{ - "schema": "qta-execution-policy/v1", - "market": "KR", - "timezone": "Asia/Seoul", - "entry_window_start": "2026-07-27T09:00:00+09:00", - "entry_window_end": "2026-07-27T10:00:00+09:00", - "market_session": { - "schema": "qta-market-session/v1", - "provider": "KRX", - "source_id": "replace-with-frozen-calendar-source-id", - "source_as_of": "2026-07-27T08:30:00+09:00", - "source_path": "/absolute/path/to/kr-market-session-2026-07-27.json", - "source_sha256": "replace-with-lowercase-sha256", - "market": "KR", - "timezone": "Asia/Seoul", - "session_date": "2026-07-27", - "previous_session_date": "2026-07-24", - "scheduled_status": "OPEN", - "regular_open": "2026-07-27T09:00:00+09:00", - "regular_close": "2026-07-27T15:30:00+09:00", - "session_hash": "replace-with-canonical-session-sha256" - }, - "snapshot_max_age_seconds": 1800, - "poll_interval_seconds": 4, - "quote_max_age_seconds": 5, - "max_spread_bps": "25", - "max_gap_bps": "20", - "trigger_mode": "AT_OR_ABOVE", - "order_ttl_seconds": 30, - "order_type": "LIMIT", - "time_in_force": "DAY", - "allow_partial_fill": true, - "cancel_remainder_at_window_end": true -} -``` - -The four-second example is the minimum admission value for one KIS-paper -intent: two quote calls plus one possible submit at the adapter's conservative -1.05-second pacing. Recompute it from the final number of planned intents; -two KIS-paper intents require at least seven seconds. KIS-live and Toss -shadow sessions have different pacing floors, but live mutation remains -disabled until the promotion gates pass. Start the runner at least 30 seconds -before the regular open so authentication is not on the first quote cycle. - -For the U.S., use `America/New_York` and `09:30–10:30` with the correct offset -for that session. Do not derive it by subtracting a fixed number of hours from -KST. The planner checks the supplied offset against `ZoneInfo`, so use `-04:00` -only when New York is actually on daylight time and `-05:00` when it is on -standard time. - -For a v2 run, stamp the account snapshot and -`qta-exposure-snapshot/v2` with the same timezone-aware `as_of` instant after -both reads are frozen. That instant must be on the local session date, at or -after `market_session.source_as_of`, no later than the entry-window start, and -no older than `snapshot_max_age_seconds`. The screen `analysis_date` must equal -the hashed calendar snapshot's `previous_session_date`. - -The file at `market_session.source_path` contains exactly the market-session -fields shown above except `source_path`, `source_sha256`, and `session_hash`. -Generate the two hashes with the execution helper after freezing that file; -the placeholder values above are intentionally not runnable. - -```bash -python3 scripts/freeze_market_session.py \ - --source /absolute/path/to/kr-market-session-2026-07-27.json \ - --output /absolute/path/to/kr-market-session-bound-2026-07-27.json -``` - -Copy the complete bound object into the execution policy without editing it. - -## Venue map - -Legacy `qta-screen/v1` plans require: - -```json -{ - "schema": "qta-venue-map/v1", - "venues": { - "KR:005930": "KRX", - "US:AAPL": "NASD" - } -} -``` - -An exchange-aware v2 plan carries the verified venue inside each hashed intent, -so `run_session.py` does not require `--venue-map`. If a v2 map is supplied for -an independent cross-check, prefer exchange-qualified keys: - -```json -{ - "schema": "qta-venue-map/v1", - "venues": { - "KOSPI:005930": "KRX", - "NASDAQ:AAPL": "NASD", - "NYSE:IBM": "NYSE" - } -} -``` - -A supplied map must match the hashed intent or the session blocks. KIS paper -uses KRX for Korea. KIS U.S. order venues are `NASD`, `NYSE`, and `AMEX`; quote -endpoints use different codes internally, which the adapter maps. Toss ignores -the venue map for legacy v1 previews, but v2 planning blocks Toss because the -universe has not been Toss-qualified. diff --git a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/systemd-contract.md b/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/systemd-contract.md deleted file mode 100644 index aa5336f..0000000 --- a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/references/systemd-contract.md +++ /dev/null @@ -1,105 +0,0 @@ -# Deterministic systemd supervision contract - -`systemd_units.py generate` creates user-level service and timer files without -changing the user manager. This is a safety property of the command, not a -reason for an agent to stop early. When the user explicitly requests actual -installation or automation, generation is an intermediate step: review and -verify the exact generated units, then install, enable, start, and inspect only -the requested scope. - -## Bundle - -The input is one exact `qta-systemd-bundle/v2` JSON object. It binds: - -- the exact Python executable and installed technical/trader skill roots; -- a user-owned, regular, non-symlink mode-0600 environment file; -- one absolute runtime directory containing all mutable jobs, plans, state, - venue maps, and status files; -- unique static EOD/account-snapshot/entry jobs or daily - prepare/snapshot/entry jobs for KR and US. - -Daily preparation starts at 01:00 in each market timezone on weekdays. That -provides eight hours before the Korean open and eight and a half hours before -the U.S. open for the measured multi-hour market refresh. An initial -multi-year bootstrap can take longer than the daily window and must be -completed before relying on the timer; subsequent runs resume and increment -the completed cache. Entry services start at 08:59 Asia/Seoul and 09:29 -America/New_York so the -runner can complete its 30-second token warm-up before the exchange open. -Account snapshot services run at 08:50 Asia/Seoul and 09:20 -America/New_York, use KIS live credentials in shadow mode, and must complete -before their market's entry service. -Entry timers are never persistent: a sleeping or offline host must not replay a -missed opening session. Account snapshot timers are also never persistent. -Daily preparation and legacy EOD timers are persistent; late daily preparation -still fails its pre-open deadline rather than replaying an entry. Entry and -snapshot timers are never persistent. - -For recurring automation, use `daily-prepare`, `daily-snapshot`, and -`daily-entry` jobs whose `input_path` is the stable -`qta-daily-shadow-config/v2` file described in -`daily-pipeline-contract.md`. The daily runner calculates the session date and -atomically publishes current descriptors. A timer bound to a date-stamped job -file remains session-specific and must not be reported as future-date -automation. Never reuse a prior session's plan, calendar, state directory, or -output path on a later date. - -The generator accepts only KIS paper/paper or KIS live/shadow entry pairs. -It refuses live mode. The generated service executes Python directly without a -shell, removes `PYTHONPATH` and `PYTHONHOME` before the wrapper starts and -again before the worker process, disables bytecode writes, applies systemd -filesystem/process hardening, and grants writes only to the bound runtime -directory. - -## Optional ntfy notification - -The mode-`0600` environment file may define `QTA_NTFY_TOPIC_URL` and, -for an authenticated topic, `QTA_NTFY_TOKEN`. The topic must be a plain -HTTP(S) topic URL without embedded credentials, a query, or a fragment. The -wrapper publishes one completion notification after each job and reports only -the channel result (`SENT`, `FAILED`, or `DISABLED`) to the journal. It never -prints the topic URL, token, or server response body. - -Notification failure is deliberately best-effort: it does not change the -worker's return code and cannot cause a successful shadow stage to be retried. -`BLOCKED` and `MANUAL_BLOCK` use urgent priority; entry completion uses high -priority. The notification is an operational side channel, not a broker API -mutation, and it does not change `live_enabled=false` or -`api_mutation_count=0`. - -## Single writer - -Every generated execution acquires a nonblocking, non-symlink regular-file -market lock below a non-symlink `runtime_directory/locks`. Preparation, -snapshot, and entry acquire only their selected market lock. Failure -to acquire a lock is `BLOCKED`, not a retry or a second writer. The wrapper -sets every opened lock file to mode `0600`, including a regular lock file that -predates the current bundle, and verifies the resulting mode before acquiring -it. - -Generate into a new or empty output directory so stale service/timer files -cannot survive from an older bundle. The generated `systemd-bundle.json` is a -normalized, hashed, exact `qta-systemd-bundle/v2` object that the `execute` -subcommand must be able to read without projection or manual editing. -Derived `schedule` and `persistent` values belong only in generated timer -files; they are not serialized as job input fields. - -Unit files and their SHA-256 values are recorded in -`generation-status.json` with: - -```text -activation_performed=false -live_enabled=false -api_mutation_count=0 -``` - -Run `systemd-analyze --user verify` against the generated files on the target -Linux host before copying them to `~/.config/systemd/user/`. Resolve the -intended filenames explicitly; do not install a wildcard that could include a -stale or unreviewed unit. After an explicitly requested activation, run -`systemctl --user daemon-reload`, enable/start only the approved timers or -services, and verify `is-enabled`, `is-active`, `list-timers`, and the service -journal. Entry or snapshot activation must not occur until the daily config, -credentials, runtime ownership, and installed skill roots are valid. The daily -producer itself creates the date-bound calendar, plan, and state path before -entry; a missing stage descriptor is `BLOCKED`. diff --git a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/account_snapshot.py b/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/account_snapshot.py deleted file mode 100644 index 4b96fb2..0000000 --- a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/account_snapshot.py +++ /dev/null @@ -1,1199 +0,0 @@ -#!/usr/bin/env python3 -"""Freeze KIS account state and merge verified cross-broker exposures.""" - -from __future__ import annotations - -import argparse -import json -import os -import tempfile -from datetime import datetime, timezone -from decimal import Decimal, InvalidOperation -from pathlib import Path -from typing import Any - -from broker_adapters import HttpResponse, KisBroker, QueueTransport, kis_result -from broker_credentials import ( - load_kis_credentials, - require_kis_runtime_credentials, -) -from execution_core import ( - BlockedError, - canonical_json, - normalized_account_snapshot, - normalized_exposure_snapshot, - parse_aware_datetime, - sha256_file, - sha256_json, -) - -JOB_SCHEMA = "qta-account-snapshot-job/v2" -COMPONENT_SCHEMA = "qta-exposure-component/v1" -STATUS_SCHEMA = "qta-account-snapshot-status/v2" -MANIFEST_SCHEMA = "qta-universe-manifest/v2" -ACCOUNT_SCHEMA = "qta-account-snapshot/v2" -EXPOSURE_SCHEMA = "qta-exposure-snapshot/v2" - -EXCHANGE_MARKET = { - "KOSPI": "KR", - "KOSDAQ": "KR", - "NYSE": "US", - "NASDAQ": "US", -} -US_EXCHANGE_CODES = { - "NAS": "NASDAQ", - "NASD": "NASDAQ", - "NASDAQ": "NASDAQ", - "NYS": "NYSE", - "NYSE": "NYSE", -} - - -def exact_fields(value: dict[str, Any], required: set[str], label: str) -> None: - missing = required - set(value) - extra = set(value) - required - if missing or extra: - raise BlockedError( - f"{label} fields mismatch: missing={sorted(missing)}, extra={sorted(extra)}" - ) - - -def decimal_value(value: Any, label: str) -> Decimal: - if isinstance(value, bool): - raise BlockedError(f"{label} must be a decimal") - try: - result = Decimal(str(value).strip()) - except (InvalidOperation, ValueError, AttributeError) as exc: - raise BlockedError(f"{label} must be a decimal") from exc - if not result.is_finite(): - raise BlockedError(f"{label} must be finite") - return result - - -def nonnegative_decimal(value: Any, label: str) -> Decimal: - result = decimal_value(value, label) - if result < 0: - raise BlockedError(f"{label} must be nonnegative") - return result - - -def positive_decimal(value: Any, label: str) -> Decimal: - result = decimal_value(value, label) - if result <= 0: - raise BlockedError(f"{label} must be positive") - return result - - -def read_json_object(path: Path, label: str) -> dict[str, Any]: - try: - value = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as exc: - raise BlockedError(f"{label} must be readable JSON: {path}") from exc - if not isinstance(value, dict): - raise BlockedError(f"{label} must be a JSON object") - return value - - -def atomic_write_json(path: Path, value: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) - temporary = Path(temporary_name) - try: - with os.fdopen(descriptor, "w", encoding="utf-8") as handle: - handle.write(json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2)) - handle.write("\n") - handle.flush() - os.fsync(handle.fileno()) - os.replace(temporary, path) - except BaseException: - temporary.unlink(missing_ok=True) - raise - - -def validate_absolute_path(value: Any, label: str) -> Path: - if not isinstance(value, str) or not value.strip(): - raise BlockedError(f"{label} must be a non-empty path") - path = Path(value) - if not path.is_absolute(): - raise BlockedError(f"{label} must be absolute") - return path - - -def load_job(path: Path) -> dict[str, Any]: - value = read_json_object(path, "account snapshot job") - required = { - "schema", - "broker", - "mode", - "market", - "account_alias", - "universe_manifest", - "fx_to_krw", - "manual_exposure_components", - "manual_component_max_age_seconds", - "output_account_path", - "output_exposure_path", - "output_status_path", - } - exact_fields(value, required, "account snapshot job") - if value["schema"] != JOB_SCHEMA: - raise BlockedError(f"unsupported account snapshot job schema: {value['schema']!r}") - if value["broker"] != "kis-live" or value["mode"] != "shadow": - raise BlockedError("account collector currently requires kis-live/shadow") - market = str(value["market"]).upper() - if market not in {"KR", "US"}: - raise BlockedError("job market must be KR or US") - alias = value["account_alias"] - if not isinstance(alias, str) or not alias.strip() or alias != alias.strip(): - raise BlockedError("account_alias must be a non-empty trimmed string") - manifest = validate_absolute_path(value["universe_manifest"], "universe_manifest") - raw_fx = value["fx_to_krw"] - if market == "US" and raw_fx == "KIS_PRESENT_BALANCE": - fx: Decimal | None = None - else: - fx = positive_decimal(raw_fx, "fx_to_krw") - if market == "KR" and fx != 1: - raise BlockedError("KR fx_to_krw must be 1") - components = value["manual_exposure_components"] - if not isinstance(components, list): - raise BlockedError("manual_exposure_components must be an array") - component_paths = [ - validate_absolute_path(item, f"manual_exposure_components[{index}]") - for index, item in enumerate(components) - ] - max_age = value["manual_component_max_age_seconds"] - if isinstance(max_age, bool) or not isinstance(max_age, int) or max_age <= 0: - raise BlockedError("manual_component_max_age_seconds must be a positive integer") - if max_age > 86400: - raise BlockedError("manual_component_max_age_seconds must not exceed 86400") - output_paths = { - key: validate_absolute_path(value[key], key) - for key in ( - "output_account_path", - "output_exposure_path", - "output_status_path", - ) - } - if len(set(output_paths.values())) != len(output_paths): - raise BlockedError("account, exposure, and status output paths must differ") - return { - **value, - "market": market, - "universe_manifest": manifest, - "fx_to_krw": fx, - "manual_exposure_components": component_paths, - "manual_component_max_age_seconds": max_age, - **output_paths, - } - - -def load_manifest_map(path: Path) -> tuple[dict[str, str], str]: - manifest = read_json_object(path, "universe manifest") - if manifest.get("schema") != MANIFEST_SCHEMA: - raise BlockedError(f"universe manifest must use {MANIFEST_SCHEMA}") - if manifest.get("build_status") != "READY": - raise BlockedError("universe manifest build_status must be READY") - supplied_hash = manifest.get("manifest_hash") - if ( - not isinstance(supplied_hash, str) - or len(supplied_hash) != 64 - or supplied_hash != sha256_json( - {key: item for key, item in manifest.items() if key != "manifest_hash"} - ) - ): - raise BlockedError("universe manifest hash is invalid") - instruments = manifest.get("instruments") - if not isinstance(instruments, list): - raise BlockedError("universe manifest instruments must be an array") - mapping: dict[str, str] = {} - for index, instrument in enumerate(instruments): - if not isinstance(instrument, dict): - raise BlockedError(f"manifest instruments[{index}] must be an object") - symbol = str(instrument.get("broker_symbol", "")).strip().upper() - exchange = str(instrument.get("exchange", "")).strip().upper() - market = str(instrument.get("market", "")).strip().upper() - if not symbol or exchange not in EXCHANGE_MARKET: - raise BlockedError(f"manifest instruments[{index}] has invalid identity") - if EXCHANGE_MARKET[exchange] != market: - raise BlockedError(f"manifest instruments[{index}] exchange/market mismatch") - key = f"{market}:{symbol}" - previous = mapping.setdefault(key, exchange) - if previous != exchange: - raise BlockedError(f"broker symbol maps to multiple exchanges: {key}") - return mapping, supplied_hash - - -def normalize_component( - value: dict[str, Any], - *, - frozen_at: datetime, - max_age_seconds: int, - path: Path, -) -> tuple[list[dict[str, Any]], dict[str, Any]]: - required = {"schema", "broker", "source_kind", "source_as_of", "positions"} - exact_fields(value, required, f"exposure component {path}") - if value["schema"] != COMPONENT_SCHEMA: - raise BlockedError(f"unsupported exposure component schema in {path}") - broker = value["broker"] - if broker not in {"toss", "nh"}: - raise BlockedError(f"manual exposure component broker must be toss or nh: {path}") - if value["source_kind"] not in {"broker-export", "verified-manual"}: - raise BlockedError(f"unsupported manual source_kind in {path}") - source_as_of = parse_aware_datetime(value["source_as_of"], f"{path} source_as_of") - age = (frozen_at.astimezone(timezone.utc) - source_as_of.astimezone(timezone.utc)).total_seconds() - if age < 0: - raise BlockedError(f"manual exposure component is future-dated: {path}") - if age > max_age_seconds: - raise BlockedError(f"manual exposure component is stale: {path}") - raw_positions = value["positions"] - if not isinstance(raw_positions, list): - raise BlockedError(f"manual exposure component positions must be an array: {path}") - positions: list[dict[str, Any]] = [] - for index, item in enumerate(raw_positions): - if not isinstance(item, dict): - raise BlockedError(f"{path} positions[{index}] must be an object") - exact_fields( - item, - {"market", "exchange", "symbol", "quantity", "market_value_krw"}, - f"{path} positions[{index}]", - ) - exchange = str(item["exchange"]).upper() - market = str(item["market"]).upper() - if exchange not in EXCHANGE_MARKET or EXCHANGE_MARKET[exchange] != market: - raise BlockedError(f"{path} positions[{index}] exchange/market mismatch") - symbol = str(item["symbol"]).strip().upper() - if not symbol: - raise BlockedError(f"{path} positions[{index}] symbol is empty") - quantity = item["quantity"] - if quantity is not None: - quantity = format( - nonnegative_decimal(quantity, f"{path} positions[{index}].quantity"), - "f", - ) - market_value = format( - nonnegative_decimal( - item["market_value_krw"], - f"{path} positions[{index}].market_value_krw", - ), - "f", - ) - positions.append( - { - "broker": broker, - "market": market, - "exchange": exchange, - "symbol": symbol, - "quantity": quantity, - "market_value_krw": market_value, - } - ) - return positions, { - "broker": broker, - "path": str(path), - "sha256": sha256_file(path), - "source_as_of": source_as_of.isoformat(), - "position_count": len(positions), - } - - -def response_list(body: dict[str, Any], field: str, label: str) -> list[dict[str, Any]]: - raw = body.get(field, []) - if raw in (None, ""): - return [] - if isinstance(raw, dict): - raw = [raw] - if not isinstance(raw, list) or any(not isinstance(item, dict) for item in raw): - raise BlockedError(f"KIS {label} {field} must be an object array") - return raw - - -def request_pages( - broker: KisBroker, - *, - path: str, - tr_id: str, - query: dict[str, str], - output_fields: tuple[str, ...], - context_suffix: str | None, - label: str, -) -> dict[str, list[dict[str, Any]]]: - collected = {field: [] for field in output_fields} - current_query = dict(query) - continuation = "" - for page in range(1, 51): - headers = broker.headers(tr_id) - if continuation: - headers["tr_cont"] = "N" - response = broker.transport.request( - "GET", - f"{broker.base_url}{path}", - headers=headers, - query=current_query, - timeout_seconds=10, - ) - body = kis_result(response, label) - for field in output_fields: - collected[field].extend(response_list(body, field, label)) - continuation = str(response.headers.get("tr_cont", "")).strip().upper() - if continuation not in {"M", "F"}: - return collected - if context_suffix is None: - raise BlockedError(f"KIS {label} requested unsupported continuation") - if context_suffix == "": - continue - fk = str(body.get(f"ctx_area_fk{context_suffix}", "")).strip() - nk = str(body.get(f"ctx_area_nk{context_suffix}", "")).strip() - if not fk or not nk: - raise BlockedError(f"KIS {label} continuation keys are missing") - current_query[f"CTX_AREA_FK{context_suffix}"] = fk - current_query[f"CTX_AREA_NK{context_suffix}"] = nk - raise BlockedError(f"KIS {label} exceeded 50 continuation pages") - - -def manifest_exchange( - mapping: dict[str, str], - market: str, - symbol: str, - *, - broker_exchange: Any = None, -) -> str: - normalized_symbol = symbol.strip().upper() - exchange = mapping.get(f"{market}:{normalized_symbol}") - if exchange is None and market == "US": - exchange = US_EXCHANGE_CODES.get(str(broker_exchange or "").strip().upper()) - if exchange is None: - raise BlockedError( - f"held or working symbol is absent from the selected-market map: " - f"{market}:{normalized_symbol}" - ) - return exchange - - -def side_from_kis(value: Any, label: str) -> str: - code = str(value or "").strip() - if code == "01": - return "SELL" - if code == "02": - return "BUY" - raise BlockedError(f"KIS {label} has unsupported side code: {code!r}") - - -def aggregate_positions(positions: list[dict[str, Any]]) -> list[dict[str, Any]]: - totals: dict[tuple[str, str, str], tuple[Decimal, Decimal]] = {} - for item in positions: - key = (item["market"], item["exchange"], item["symbol"]) - quantity, market_value = totals.get(key, (Decimal(0), Decimal(0))) - totals[key] = ( - quantity + nonnegative_decimal(item["quantity"], "position quantity"), - market_value - + nonnegative_decimal(item["market_value_krw"], "position market value"), - ) - return [ - { - "market": market, - "exchange": exchange, - "symbol": symbol, - "quantity": format(totals[(market, exchange, symbol)][0], "f"), - "market_value_krw": format(totals[(market, exchange, symbol)][1], "f"), - } - for market, exchange, symbol in sorted(totals) - ] - - -def minimum_nonnegative(rows: list[tuple[Any, str]]) -> Decimal: - return min(max(decimal_value(value, label), Decimal(0)) for value, label in rows) - - -def collect_kr( - broker: KisBroker, - mapping: dict[str, str], -) -> tuple[Decimal, Decimal, list[dict[str, Any]], list[dict[str, Any]]]: - balance = request_pages( - broker, - path="/uapi/domestic-stock/v1/trading/inquire-balance", - tr_id="TTTC8434R", - query={ - "CANO": broker.account_prefix, - "ACNT_PRDT_CD": broker.account_product, - "AFHR_FLPR_YN": "N", - "OFL_YN": "", - "INQR_DVSN": "02", - "UNPR_DVSN": "01", - "FUND_STTL_ICLD_YN": "N", - "FNCG_AMT_AUTO_RDPT_YN": "N", - "PRCS_DVSN": "00", - "CTX_AREA_FK100": "", - "CTX_AREA_NK100": "", - }, - output_fields=("output1", "output2"), - context_suffix="100", - label="KR balance", - ) - if not balance["output2"]: - raise BlockedError("KIS KR balance summary is missing") - summary = balance["output2"][-1] - settled_cash = minimum_nonnegative( - [ - (summary.get("dnca_tot_amt"), "KR dnca_tot_amt"), - (summary.get("nxdy_excc_amt"), "KR nxdy_excc_amt"), - (summary.get("prvs_rcdl_excc_amt"), "KR prvs_rcdl_excc_amt"), - ] - ) - borrowed = nonnegative_decimal(summary.get("tot_loan_amt", "0"), "KR tot_loan_amt") - positions: list[dict[str, Any]] = [] - for row in balance["output1"]: - quantity = nonnegative_decimal(row.get("hldg_qty"), "KR hldg_qty") - if quantity == 0: - continue - symbol = str(row.get("pdno", "")).strip().upper() - exchange = manifest_exchange(mapping, "KR", symbol) - positions.append( - { - "market": "KR", - "exchange": exchange, - "symbol": symbol, - "quantity": format(quantity, "f"), - "market_value_krw": format( - nonnegative_decimal(row.get("evlu_amt"), "KR evlu_amt"), "f" - ), - } - ) - orders = request_pages( - broker, - path="/uapi/domestic-stock/v1/trading/inquire-psbl-rvsecncl", - tr_id="TTTC0084R", - query={ - "CANO": broker.account_prefix, - "ACNT_PRDT_CD": broker.account_product, - "INQR_DVSN_1": "1", - "INQR_DVSN_2": "0", - "CTX_AREA_FK100": "", - "CTX_AREA_NK100": "", - }, - output_fields=("output",), - context_suffix="100", - label="KR open orders", - ) - open_orders: list[dict[str, Any]] = [] - for row in orders["output"]: - remaining = nonnegative_decimal(row.get("psbl_qty"), "KR psbl_qty") - if remaining == 0: - continue - symbol = str(row.get("pdno", "")).strip().upper() - open_orders.append( - { - "side": side_from_kis(row.get("sll_buy_dvsn_cd"), "KR open order"), - "market": "KR", - "exchange": manifest_exchange(mapping, "KR", symbol), - "symbol": symbol, - } - ) - return settled_cash, borrowed, positions, open_orders - - -def collect_us( - broker: KisBroker, - mapping: dict[str, str], - fx_to_krw: Decimal | None, -) -> tuple[ - Decimal, - Decimal, - list[dict[str, Any]], - list[dict[str, Any]], - Decimal | None, - str, -]: - present = request_pages( - broker, - path="/uapi/overseas-stock/v1/trading/inquire-present-balance", - tr_id="CTRP6504R", - query={ - "CANO": broker.account_prefix, - "ACNT_PRDT_CD": broker.account_product, - "WCRC_FRCR_DVSN_CD": "02", - "NATN_CD": "840", - "TR_MKET_CD": "00", - "INQR_DVSN_CD": "00", - }, - output_fields=("output1", "output2", "output3"), - context_suffix="", - label="US present balance", - ) - usd_cash_rows = [ - row - for row in present["output2"] - if str(row.get("crcy_cd", "")).strip().upper() == "USD" - ] - cash_snapshots: set[tuple[Decimal, Decimal, Decimal]] = set() - observed_cash_fx: set[Decimal] = set() - for index, cash in enumerate(usd_cash_rows): - cash_snapshots.add( - ( - nonnegative_decimal( - cash.get("frcr_dncl_amt_2"), - f"US output2[{index}].frcr_dncl_amt_2", - ), - nonnegative_decimal( - cash.get("frcr_drwg_psbl_amt_1"), - f"US output2[{index}].frcr_drwg_psbl_amt_1", - ), - nonnegative_decimal( - cash.get("nxdy_frcr_drwg_psbl_amt"), - f"US output2[{index}].nxdy_frcr_drwg_psbl_amt", - ), - ) - ) - if fx_to_krw is None: - observed_cash_fx.add( - positive_decimal( - cash.get("frst_bltn_exrt"), - f"US output2[{index}].frst_bltn_exrt", - ) - ) - if len(cash_snapshots) > 1: - raise BlockedError("KIS US present balance returned conflicting USD cash rows") - if len(observed_cash_fx) > 1: - raise BlockedError("KIS US present balance returned conflicting USD FX rates") - settled_cash = min(next(iter(cash_snapshots))) if cash_snapshots else Decimal(0) - - position_rows: list[tuple[dict[str, Any], Decimal]] = [] - observed_position_fx: set[Decimal] = set() - borrowed = Decimal(0) - for index, row in enumerate(present["output1"]): - quantity = nonnegative_decimal( - row.get("cblc_qty13"), f"US output1[{index}].cblc_qty13" - ) - borrowed += nonnegative_decimal( - row.get("loan_rmnd", "0"), f"US output1[{index}].loan_rmnd" - ) - if quantity == 0: - continue - position_rows.append((row, quantity)) - if fx_to_krw is None and not observed_cash_fx: - observed_position_fx.add( - positive_decimal( - row.get("bass_exrt"), - f"US output1[{index}].bass_exrt", - ) - ) - if len(observed_position_fx) > 1: - raise BlockedError( - "KIS US present balance returned conflicting position FX rates" - ) - - if fx_to_krw is not None: - effective_fx: Decimal | None = fx_to_krw - fx_source = "JOB_INPUT" - elif observed_cash_fx: - effective_fx = next(iter(observed_cash_fx)) - fx_source = "KIS_PRESENT_BALANCE:frst_bltn_exrt" - elif observed_position_fx: - effective_fx = next(iter(observed_position_fx)) - fx_source = "KIS_PRESENT_BALANCE:bass_exrt" - else: - effective_fx = None - fx_source = "NOT_APPLICABLE:NO_SETTLED_USD_OR_POSITION" - - positions: list[dict[str, Any]] = [] - for row, quantity in position_rows: - if effective_fx is None: - raise BlockedError("KIS US positions require a positive FX rate") - symbol = str(row.get("pdno", "")).strip().upper() - exchange = manifest_exchange( - mapping, - "US", - symbol, - broker_exchange=row.get("ovrs_excg_cd"), - ) - market_value_usd = nonnegative_decimal( - row.get("frcr_evlu_amt2"), "US frcr_evlu_amt2" - ) - positions.append( - { - "market": "US", - "exchange": exchange, - "symbol": symbol, - "quantity": format(quantity, "f"), - "market_value_krw": format(market_value_usd * effective_fx, "f"), - } - ) - orders = request_pages( - broker, - path="/uapi/overseas-stock/v1/trading/inquire-nccs", - tr_id="TTTS3018R", - query={ - "CANO": broker.account_prefix, - "ACNT_PRDT_CD": broker.account_product, - "OVRS_EXCG_CD": "NASD", - "SORT_SQN": "DS", - "CTX_AREA_FK200": "", - "CTX_AREA_NK200": "", - }, - output_fields=("output",), - context_suffix="200", - label="US open orders", - ) - open_orders: list[dict[str, Any]] = [] - for row in orders["output"]: - remaining = nonnegative_decimal(row.get("nccs_qty"), "US nccs_qty") - if remaining == 0: - continue - symbol = str(row.get("pdno", "")).strip().upper() - open_orders.append( - { - "side": side_from_kis(row.get("sll_buy_dvsn_cd"), "US open order"), - "market": "US", - "exchange": manifest_exchange( - mapping, - "US", - symbol, - broker_exchange=row.get("ovrs_excg_cd"), - ), - "symbol": symbol, - } - ) - return ( - settled_cash, - borrowed, - positions, - open_orders, - effective_fx, - fx_source, - ) - - -def collect( - job: dict[str, Any], - *, - transport: Any | None = None, - frozen_at: datetime | None = None, -) -> dict[str, Any]: - credential_status = load_kis_credentials("live") - require_kis_runtime_credentials(credential_status) - mapping, manifest_hash = load_manifest_map(job["universe_manifest"]) - broker = KisBroker( - app_key=os.environ["QTA_KIS_APP_KEY"], - app_secret=os.environ["QTA_KIS_APP_SECRET"], - account_prefix=os.environ["QTA_KIS_ACCOUNT_PREFIX"], - account_product=os.environ["QTA_KIS_ACCOUNT_PRODUCT"], - environment="live", - transport=transport, - ) - if job["market"] == "KR": - settled, borrowed, positions, open_orders = collect_kr(broker, mapping) - effective_fx = Decimal(1) - fx_source = "FIXED_KRW" - else: - ( - settled, - borrowed, - positions, - open_orders, - effective_fx, - fx_source, - ) = collect_us( - broker, - mapping, - job["fx_to_krw"], - ) - positions = aggregate_positions(positions) - freeze_time = frozen_at or datetime.now(timezone.utc) - if freeze_time.tzinfo is None: - raise BlockedError("frozen_at must include a timezone") - freeze_time = freeze_time.astimezone(timezone.utc) - as_of = freeze_time.isoformat() - account_positions = [ - { - "market": item["market"], - "exchange": item["exchange"], - "symbol": item["symbol"], - "quantity": item["quantity"], - "market_value_krw": item["market_value_krw"], - } - for item in positions - ] - account = normalized_account_snapshot( - { - "schema": ACCOUNT_SCHEMA, - "broker": "kis", - "environment": "shadow", - "account_alias": job["account_alias"], - "market": job["market"], - "currency": "KRW" if job["market"] == "KR" else "USD", - "as_of": as_of, - "settled_cash": format(settled, "f"), - "borrowed_buying_power": format(borrowed, "f"), - "fx_to_krw": ( - format(effective_fx, "f") if effective_fx is not None else None - ), - "positions": account_positions, - "open_orders": open_orders, - }, - screen_schema="qta-screen/v2", - ) - exposure_positions = [ - { - "broker": "kis", - "market": item["market"], - "exchange": item["exchange"], - "symbol": item["symbol"], - "quantity": item["quantity"], - "market_value_krw": item["market_value_krw"], - } - for item in positions - ] - component_receipts: list[dict[str, Any]] = [] - component_brokers: set[str] = set() - for path in job["manual_exposure_components"]: - component = read_json_object(path, "manual exposure component") - component_positions, component_receipt = normalize_component( - component, - frozen_at=freeze_time, - max_age_seconds=job["manual_component_max_age_seconds"], - path=path, - ) - component_broker = component_receipt["broker"] - if component_broker in component_brokers: - raise BlockedError( - f"manual exposure broker appears more than once: {component_broker}" - ) - component_brokers.add(component_broker) - exposure_positions.extend(component_positions) - component_receipts.append(component_receipt) - exposure_keys: set[tuple[str, str, str]] = set() - for item in exposure_positions: - key = (item["broker"], item["exchange"], item["symbol"]) - if key in exposure_keys: - raise BlockedError( - "duplicate cross-broker exposure position: " + ":".join(key) - ) - exposure_keys.add(key) - exposure = normalized_exposure_snapshot( - { - "schema": EXPOSURE_SCHEMA, - "as_of": as_of, - "positions": exposure_positions, - } - ) - atomic_write_json(job["output_account_path"], account) - atomic_write_json(job["output_exposure_path"], exposure) - receipt = { - "schema": STATUS_SCHEMA, - "status": "READY", - "broker": "kis", - "mode": "shadow", - "market": job["market"], - "as_of": as_of, - "universe_manifest_hash": manifest_hash, - "account_path": str(job["output_account_path"]), - "account_sha256": sha256_file(job["output_account_path"]), - "exposure_path": str(job["output_exposure_path"]), - "exposure_sha256": sha256_file(job["output_exposure_path"]), - "manual_components": component_receipts, - "fx_to_krw": account["fx_to_krw"], - "fx_source": fx_source, - "settled_cash": account["settled_cash"], - "borrowed_buying_power_excluded": True, - "position_count": len(account["positions"]), - "open_order_count": len(account["open_orders"]), - "cross_broker_position_count": len(exposure["positions"]), - "api_mutation_count": 0, - } - atomic_write_json(job["output_status_path"], receipt) - return receipt - - -def manual_template(broker: str, source_as_of: str) -> dict[str, Any]: - if broker not in {"toss", "nh"}: - raise BlockedError("manual template broker must be toss or nh") - parsed = parse_aware_datetime(source_as_of, "source_as_of") - return { - "schema": COMPONENT_SCHEMA, - "broker": broker, - "source_kind": "verified-manual", - "source_as_of": parsed.isoformat(), - "positions": [], - } - - -def self_test() -> dict[str, Any]: - with tempfile.TemporaryDirectory() as directory_name: - directory = Path(directory_name) - now = datetime(2026, 7, 27, 0, 50, tzinfo=timezone.utc) - manifest_without_hash = { - "schema": MANIFEST_SCHEMA, - "build_status": "READY", - "instruments": [ - {"market": "KR", "exchange": "KOSPI", "broker_symbol": "005930"}, - {"market": "KR", "exchange": "KOSPI", "broker_symbol": "000660"}, - {"market": "US", "exchange": "NASDAQ", "broker_symbol": "AAPL"}, - {"market": "US", "exchange": "NYSE", "broker_symbol": "IBM"}, - ], - } - manifest = { - **manifest_without_hash, - "manifest_hash": sha256_json(manifest_without_hash), - } - manifest_path = directory / "manifest.json" - atomic_write_json(manifest_path, manifest) - manual_path = directory / "toss.json" - atomic_write_json( - manual_path, - { - "schema": COMPONENT_SCHEMA, - "broker": "toss", - "source_kind": "verified-manual", - "source_as_of": "2026-07-27T09:45:00+09:00", - "positions": [ - { - "market": "US", - "exchange": "NASDAQ", - "symbol": "AAPL", - "quantity": "0.5", - "market_value_krw": "100000", - } - ], - }, - ) - job_raw = { - "schema": JOB_SCHEMA, - "broker": "kis-live", - "mode": "shadow", - "market": "KR", - "account_alias": "kis-live-kr", - "universe_manifest": str(manifest_path), - "fx_to_krw": "1", - "manual_exposure_components": [str(manual_path)], - "manual_component_max_age_seconds": 3600, - "output_account_path": str(directory / "account.json"), - "output_exposure_path": str(directory / "exposure.json"), - "output_status_path": str(directory / "status.json"), - } - job_path = directory / "job.json" - atomic_write_json(job_path, job_raw) - job = load_job(job_path) - responses = [ - HttpResponse( - 200, - {}, - { - "access_token": "fixture-token", - "token_type": "Bearer", - "expires_in": 86400, - }, - ), - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "output1": [ - { - "pdno": "005930", - "hldg_qty": "2", - "evlu_amt": "150000", - } - ], - "output2": [ - { - "dnca_tot_amt": "2500000", - "nxdy_excc_amt": "2100000", - "prvs_rcdl_excc_amt": "2000000", - "tot_loan_amt": "5000000", - } - ], - }, - ), - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "output": [ - { - "pdno": "000660", - "psbl_qty": "1", - "sll_buy_dvsn_cd": "02", - } - ], - }, - ), - ] - previous = { - name: os.environ.get(name) - for name in ( - "QTA_KIS_APP_KEY", - "QTA_KIS_APP_SECRET", - "QTA_KIS_ACCOUNT_PREFIX", - "QTA_KIS_ACCOUNT_PRODUCT", - ) - } - os.environ.update( - { - "QTA_KIS_APP_KEY": "test-key", - "QTA_KIS_APP_SECRET": "test-secret", - "QTA_KIS_ACCOUNT_PREFIX": "12345678", - "QTA_KIS_ACCOUNT_PRODUCT": "01", - } - ) - try: - receipt = collect( - job, - transport=QueueTransport(responses), - frozen_at=now, - ) - finally: - for name, value in previous.items(): - if value is None: - os.environ.pop(name, None) - else: - os.environ[name] = value - account = read_json_object(job["output_account_path"], "test account") - exposure = read_json_object(job["output_exposure_path"], "test exposure") - assert receipt["status"] == "READY" - assert receipt["api_mutation_count"] == 0 - assert account["settled_cash"] == "2000000" - assert account["borrowed_buying_power"] == "5000000" - assert account["positions"][0]["exchange"] == "KOSPI" - assert account["open_orders"][0]["side"] == "BUY" - assert account["as_of"] == exposure["as_of"] - assert len(exposure["positions"]) == 2 - - us_job_raw = { - **job_raw, - "market": "US", - "account_alias": "kis-live-us", - "fx_to_krw": "KIS_PRESENT_BALANCE", - "output_account_path": str(directory / "account-us.json"), - "output_exposure_path": str(directory / "exposure-us.json"), - "output_status_path": str(directory / "status-us.json"), - } - us_job_path = directory / "job-us.json" - atomic_write_json(us_job_path, us_job_raw) - us_responses = [ - HttpResponse( - 200, - {}, - { - "access_token": "fixture-token", - "token_type": "Bearer", - "expires_in": 86400, - }, - ), - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "output1": [ - { - "pdno": "AAPL", - "ovrs_excg_cd": "NASD", - "cblc_qty13": "1.25", - "frcr_evlu_amt2": "250", - "loan_rmnd": "0", - } - ], - "output2": [ - { - "crcy_cd": "USD", - "frcr_dncl_amt_2": "1500", - "frcr_drwg_psbl_amt_1": "1200", - "nxdy_frcr_drwg_psbl_amt": "1100", - "frst_bltn_exrt": "1400", - } - ], - "output3": [{}], - }, - ), - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "output": [ - { - "pdno": "IBM", - "ovrs_excg_cd": "NYSE", - "nccs_qty": "2", - "sll_buy_dvsn_cd": "01", - } - ], - }, - ), - ] - previous_us = { - name: os.environ.get(name) - for name in ( - "QTA_KIS_APP_KEY", - "QTA_KIS_APP_SECRET", - "QTA_KIS_ACCOUNT_PREFIX", - "QTA_KIS_ACCOUNT_PRODUCT", - ) - } - os.environ.update( - { - "QTA_KIS_APP_KEY": "test-key", - "QTA_KIS_APP_SECRET": "test-secret", - "QTA_KIS_ACCOUNT_PREFIX": "12345678", - "QTA_KIS_ACCOUNT_PRODUCT": "01", - } - ) - try: - us_receipt = collect( - load_job(us_job_path), - transport=QueueTransport(us_responses), - frozen_at=now, - ) - finally: - for name, value in previous_us.items(): - if value is None: - os.environ.pop(name, None) - else: - os.environ[name] = value - us_account = read_json_object(Path(us_job_raw["output_account_path"]), "US account") - assert us_receipt["status"] == "READY" - assert us_account["settled_cash"] == "1100" - assert us_account["positions"][0]["market_value_krw"] == "350000" - assert us_account["fx_to_krw"] == "1400" - assert us_receipt["fx_source"] == "KIS_PRESENT_BALANCE:frst_bltn_exrt" - assert us_account["open_orders"][0]["exchange"] == "NYSE" - assert us_account["open_orders"][0]["side"] == "SELL" - - empty_us_job_raw = { - **us_job_raw, - "manual_exposure_components": [], - "output_account_path": str(directory / "account-us-empty.json"), - "output_exposure_path": str(directory / "exposure-us-empty.json"), - "output_status_path": str(directory / "status-us-empty.json"), - } - empty_us_job_path = directory / "job-us-empty.json" - atomic_write_json(empty_us_job_path, empty_us_job_raw) - empty_us_responses = [ - HttpResponse( - 200, - {}, - { - "access_token": "fixture-token", - "token_type": "Bearer", - "expires_in": 86400, - }, - ), - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "output1": [], - "output2": [], - "output3": [{"tot_frcr_cblc_smtl": "0"}], - }, - ), - HttpResponse(200, {}, {"rt_cd": "0", "output": []}), - ] - previous_empty_us = { - name: os.environ.get(name) - for name in ( - "QTA_KIS_APP_KEY", - "QTA_KIS_APP_SECRET", - "QTA_KIS_ACCOUNT_PREFIX", - "QTA_KIS_ACCOUNT_PRODUCT", - ) - } - os.environ.update( - { - "QTA_KIS_APP_KEY": "test-key", - "QTA_KIS_APP_SECRET": "test-secret", - "QTA_KIS_ACCOUNT_PREFIX": "12345678", - "QTA_KIS_ACCOUNT_PRODUCT": "01", - } - ) - try: - empty_us_receipt = collect( - load_job(empty_us_job_path), - transport=QueueTransport(empty_us_responses), - frozen_at=now, - ) - finally: - for name, value in previous_empty_us.items(): - if value is None: - os.environ.pop(name, None) - else: - os.environ[name] = value - empty_us_account = read_json_object( - Path(empty_us_job_raw["output_account_path"]), - "empty US account", - ) - assert empty_us_receipt["status"] == "READY" - assert empty_us_receipt["fx_source"] == ( - "NOT_APPLICABLE:NO_SETTLED_USD_OR_POSITION" - ) - assert empty_us_account["settled_cash"] == "0" - assert empty_us_account["fx_to_krw"] is None - assert empty_us_account["positions"] == [] - assert empty_us_account["open_orders"] == [] - - stale = read_json_object(manual_path, "manual") - stale["source_as_of"] = "2026-07-25T09:00:00+09:00" - try: - normalize_component( - stale, - frozen_at=now, - max_age_seconds=3600, - path=manual_path, - ) - except BlockedError: - pass - else: - raise AssertionError("stale manual exposure component was accepted") - return { - "schema": "qta-account-snapshot-self-test/v1", - "status": "PASS", - "self_test": "PASS", - "api_mutation_count": 0, - } - - -def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description=__doc__) - subparsers = parser.add_subparsers(dest="command", required=True) - collect_parser = subparsers.add_parser("collect") - collect_parser.add_argument("--job", type=Path, required=True) - template_parser = subparsers.add_parser("manual-template") - template_parser.add_argument("--broker", choices=("toss", "nh"), required=True) - template_parser.add_argument("--source-as-of", required=True) - template_parser.add_argument("--output", type=Path, required=True) - subparsers.add_parser("self-test") - return parser - - -def main() -> int: - args = build_parser().parse_args() - try: - if args.command == "collect": - output = collect(load_job(args.job)) - elif args.command == "manual-template": - if not args.output.is_absolute(): - raise BlockedError("--output must be absolute") - output = manual_template(args.broker, args.source_as_of) - atomic_write_json(args.output, output) - else: - output = self_test() - except BlockedError as exc: - print( - canonical_json( - { - "schema": "qta-account-snapshot-error/v1", - "status": "BLOCKED", - "reason": str(exc), - "api_mutation_count": 0, - } - ) - ) - return 2 - print(canonical_json(output)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/broker_adapters.py b/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/broker_adapters.py deleted file mode 100755 index 8062df2..0000000 --- a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/broker_adapters.py +++ /dev/null @@ -1,2379 +0,0 @@ -#!/usr/bin/env python3 -"""Toss and KIS REST adapters with explicit request contracts.""" - -from __future__ import annotations - -import argparse -import json -import sys -import time -from collections.abc import Callable -from dataclasses import dataclass, replace -from datetime import date, datetime, timedelta, timezone -from decimal import Decimal, ROUND_HALF_UP -from typing import Any, ClassVar -from urllib.error import HTTPError, URLError -from urllib.parse import urlencode -from urllib.request import Request, urlopen -from zoneinfo import ZoneInfo - -from execution_core import ( - BlockedError, - canonical_json, - decimal_value, - normalize_symbol, - redact, - sha256_json, -) - - -class TransportFailure(RuntimeError): - """Network failure with no authoritative broker response.""" - - -class AmbiguousMutationError(TransportFailure): - """Mutation may have reached the broker; caller must reconcile.""" - - -class AuthoritativeMutationRejection(BlockedError): - """Broker explicitly rejected a mutation before returning an order ID.""" - - -@dataclass(frozen=True) -class HttpResponse: - status: int - headers: dict[str, str] - body: Any - request_started_at: str | None = None - received_at: str | None = None - - -class UrlLibTransport: - def request( - self, - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - query: dict[str, Any] | None = None, - json_body: dict[str, Any] | None = None, - form_body: dict[str, Any] | None = None, - timeout_seconds: float = 10.0, - mutation: bool = False, - ) -> HttpResponse: - if query: - url = f"{url}?{urlencode(query, doseq=True)}" - request_headers = dict(headers or {}) - data: bytes | None = None - if json_body is not None and form_body is not None: - raise ValueError("json_body and form_body are mutually exclusive") - if json_body is not None: - data = canonical_json(json_body).encode("utf-8") - request_headers.setdefault("Content-Type", "application/json") - elif form_body is not None: - data = urlencode(form_body).encode("utf-8") - request_headers.setdefault( - "Content-Type", "application/x-www-form-urlencoded" - ) - request = Request(url, data=data, headers=request_headers, method=method) - request_started_at = datetime.now(timezone.utc).isoformat() - try: - with urlopen(request, timeout=timeout_seconds) as response: - raw = response.read() - body = json.loads(raw) if raw else None - return HttpResponse( - status=response.status, - headers={ - key.lower(): value for key, value in response.headers.items() - }, - body=body, - request_started_at=request_started_at, - received_at=datetime.now(timezone.utc).isoformat(), - ) - except HTTPError as exc: - raw = exc.read() - try: - body = json.loads(raw) if raw else None - except json.JSONDecodeError: - body = {"raw": raw.decode("utf-8", errors="replace")} - return HttpResponse( - status=exc.code, - headers={key.lower(): value for key, value in exc.headers.items()}, - body=body, - request_started_at=request_started_at, - received_at=datetime.now(timezone.utc).isoformat(), - ) - except (URLError, TimeoutError) as exc: - if mutation: - raise AmbiguousMutationError(str(exc)) from exc - raise TransportFailure(str(exc)) from exc - - -class QueueTransport: - """Deterministic fixture transport used only by self-tests.""" - - def __init__(self, responses: list[HttpResponse | BaseException]): - self.responses = list(responses) - self.requests: list[dict[str, Any]] = [] - - def request(self, method: str, url: str, **kwargs: Any) -> HttpResponse: - self.requests.append({"method": method, "url": url, **kwargs}) - if not self.responses: - raise AssertionError("fixture transport exhausted") - request_started_at = datetime.now(timezone.utc).isoformat() - response = self.responses.pop(0) - if isinstance(response, BaseException): - raise response - return replace( - response, - request_started_at=response.request_started_at or request_started_at, - received_at=response.received_at or datetime.now(timezone.utc).isoformat(), - ) - - -class PacedTransport: - """Serialize requests with a deterministic minimum interval.""" - - def __init__(self, delegate: Any, minimum_interval_seconds: float): - self.delegate = delegate - self.minimum_interval_seconds = minimum_interval_seconds - self.last_started: float | None = None - - def request(self, method: str, url: str, **kwargs: Any) -> HttpResponse: - deadline_at = kwargs.pop("deadline_at", None) - deadline = normalize_deadline(deadline_at) - now = time.monotonic() - remaining = 0.0 - if self.last_started is not None: - remaining = self.minimum_interval_seconds - (now - self.last_started) - if deadline is not None: - projected_start = datetime.now(timezone.utc) + timedelta( - seconds=max(remaining, 0.0) - ) - if projected_start >= deadline: - raise BlockedError( - "mutation pacing would cross the absolute submit deadline" - ) - if remaining > 0: - time.sleep(remaining) - if deadline is not None and datetime.now(timezone.utc) >= deadline: - raise BlockedError("absolute submit deadline elapsed before HTTP send") - self.last_started = time.monotonic() - request_started_at = datetime.now(timezone.utc).isoformat() - response = self.delegate.request(method, url, **kwargs) - return replace( - response, - request_started_at=response.request_started_at or request_started_at, - received_at=response.received_at or datetime.now(timezone.utc).isoformat(), - ) - - -class MinimumIntervalRateLimiter: - """Conservative single-process pacing for one broker rate-limit group.""" - - def __init__( - self, - requests_per_second: float, - *, - monotonic: Callable[[], float] = time.monotonic, - sleeper: Callable[[float], None] = time.sleep, - ): - if isinstance(requests_per_second, bool) or requests_per_second <= 0: - raise ValueError("requests_per_second must be positive") - self.minimum_interval_seconds = 1.0 / float(requests_per_second) - self.monotonic = monotonic - self.sleeper = sleeper - self.last_started: float | None = None - - def acquire(self, *, deadline_at: datetime | None = None) -> None: - deadline = normalize_deadline(deadline_at) - now = self.monotonic() - remaining = 0.0 - if self.last_started is not None: - remaining = self.minimum_interval_seconds - (now - self.last_started) - if deadline is not None: - projected_start = datetime.now(timezone.utc) + timedelta( - seconds=max(remaining, 0.0) - ) - if projected_start >= deadline: - raise BlockedError( - "mutation pacing would cross the absolute submit deadline" - ) - if remaining > 0: - self.sleeper(remaining) - if deadline is not None and datetime.now(timezone.utc) >= deadline: - raise BlockedError("absolute submit deadline elapsed before HTTP send") - self.last_started = self.monotonic() - - def observe_headers(self, headers: dict[str, str]) -> None: - normalized = {str(key).lower(): value for key, value in headers.items()} - raw_limit = normalized.get("x-ratelimit-limit") - if raw_limit is None: - return - try: - observed_limit = float(raw_limit) - except (TypeError, ValueError): - return - if observed_limit > 0: - self.minimum_interval_seconds = max( - self.minimum_interval_seconds, - 1.0 / observed_limit, - ) - - -def require_mapping(value: Any, label: str) -> dict[str, Any]: - if not isinstance(value, dict): - raise BlockedError(f"{label} must be an object") - return value - - -def normalize_deadline(value: datetime | None) -> datetime | None: - if value is None: - return None - if not isinstance(value, datetime) or value.tzinfo is None: - raise BlockedError("absolute submit deadline must be timezone-aware") - return value.astimezone(timezone.utc) - - -def toss_result(response: HttpResponse, operation: str) -> Any: - body = require_mapping(response.body, f"Toss {operation} response") - if response.status < 200 or response.status >= 300: - error = body.get("error") - code = error.get("code") if isinstance(error, dict) else "unknown" - raise BlockedError(f"Toss {operation} failed: HTTP {response.status} {code}") - if "result" not in body: - raise BlockedError(f"Toss {operation} response missing result") - return body["result"] - - -def toss_mutation_result( - response: HttpResponse, - operation: str, -) -> dict[str, Any]: - """Return an acknowledged Toss mutation or classify it fail-closed.""" - if response.status >= 500: - raise AmbiguousMutationError( - f"Toss {operation} may have been accepted: HTTP {response.status}" - ) - if not isinstance(response.body, dict): - raise AmbiguousMutationError( - f"Toss {operation} returned a non-object mutation response" - ) - body = response.body - if response.status < 200 or response.status >= 300: - error = body.get("error") - code = error.get("code") if isinstance(error, dict) else None - if 400 <= response.status < 500 and str(code or "").strip(): - raise AuthoritativeMutationRejection( - f"Toss {operation} rejected: HTTP {response.status} {code}" - ) - raise AmbiguousMutationError( - f"Toss {operation} returned no authoritative rejection: " - f"HTTP {response.status}" - ) - result = body.get("result") - if not isinstance(result, dict): - raise AmbiguousMutationError( - f"Toss {operation} success response missing result object" - ) - return result - - -def kis_result(response: HttpResponse, operation: str) -> dict[str, Any]: - body = require_mapping(response.body, f"KIS {operation} response") - if response.status != 200: - raise BlockedError(f"KIS {operation} failed: HTTP {response.status}") - if str(body.get("rt_cd")) != "0": - raise BlockedError( - f"KIS {operation} failed: {body.get('msg_cd', 'unknown')} " - f"{body.get('msg1', '')}".strip() - ) - return body - - -def kis_mutation_result( - response: HttpResponse, - operation: str, -) -> dict[str, Any]: - """Return an acknowledged KIS mutation or classify it fail-closed.""" - if response.status >= 500: - raise AmbiguousMutationError( - f"KIS {operation} may have been accepted: HTTP {response.status}" - ) - if not isinstance(response.body, dict): - raise AmbiguousMutationError( - f"KIS {operation} returned a non-object mutation response" - ) - body = response.body - if response.status != 200: - code = body.get("error_code", body.get("msg_cd")) - if 400 <= response.status < 500 and str(code or "").strip(): - raise AuthoritativeMutationRejection( - f"KIS {operation} rejected: HTTP {response.status} {code}" - ) - raise AmbiguousMutationError( - f"KIS {operation} returned no authoritative rejection: " - f"HTTP {response.status}" - ) - result_code = str(body.get("rt_cd", "")).strip() - if result_code != "0": - message_code = str(body.get("msg_cd", "")).strip() - if result_code and message_code: - raise AuthoritativeMutationRejection( - f"KIS {operation} rejected: {message_code} " - f"{body.get('msg1', '')}".strip() - ) - raise AmbiguousMutationError( - f"KIS {operation} response missing an authoritative result code" - ) - output = body.get("output") - if not isinstance(output, dict): - raise AmbiguousMutationError( - f"KIS {operation} success response missing output object" - ) - return output - - -def mutation_order_id(value: Any, label: str) -> str: - """Require the acknowledgement identifier needed for reconciliation.""" - if isinstance(value, bool): - raise AmbiguousMutationError(f"{label} is missing") - order_id = str(value or "").strip() - if not order_id: - raise AmbiguousMutationError(f"{label} is missing") - return order_id - - -def decimal_string(value: Any, field: str) -> str: - return format(decimal_value(value, field), "f") - - -def parse_expiration_timestamp(value: Any, field: str) -> datetime: - if isinstance(value, datetime): - parsed = value - elif isinstance(value, str): - try: - parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) - except ValueError as exc: - raise BlockedError(f"{field} must be an ISO-8601 timestamp") from exc - else: - raise BlockedError(f"{field} must be an ISO-8601 timestamp") - if parsed.tzinfo is None: - raise BlockedError(f"{field} must include a timezone") - return parsed.astimezone(timezone.utc) - - -def positive_level_prices(value: Any, label: str) -> list[Any]: - if not isinstance(value, list) or not value: - raise BlockedError(f"{label} must be a non-empty array") - prices = [] - for index, level in enumerate(value): - if not isinstance(level, dict) or "price" not in level: - raise BlockedError(f"{label}[{index}] must contain price") - price = decimal_value(level["price"], f"{label}[{index}].price") - if price > 0: - prices.append(price) - if not prices: - raise BlockedError(f"{label} has no positive price level") - return prices - - -def kis_market_timestamp( - *, - date_value: Any, - time_value: Any, - timezone_name: str, -) -> str | None: - """Parse a KIS market-local YYYYMMDD/HHMMSS timestamp, or fail closed.""" - compact_time = str(time_value or "").strip() - if len(compact_time) != 6 or not compact_time.isdigit(): - return None - compact_date = str(date_value or "").strip() - zone = ZoneInfo(timezone_name) - if len(compact_date) != 8 or not compact_date.isdigit(): - return None - try: - parsed = datetime.strptime(compact_date + compact_time, "%Y%m%d%H%M%S").replace( - tzinfo=zone - ) - except ValueError: - return None - return parsed.isoformat() - - -class TossBroker: - LIVE_BASE_URL = "https://openapi.tossinvest.com" - CREATE_ORDER_PATH = "/api/v1/orders" - MARKET_DATA_TPS = 10 - ORDER_TPS = 3 - ORDER_HISTORY_TPS = 5 - - def __init__( - self, - *, - client_id: str, - client_secret: str, - account_seq: int, - transport: Any | None = None, - access_token: str | None = None, - access_token_expires_at: datetime | str | None = None, - market_data_limiter: MinimumIntervalRateLimiter | None = None, - order_limiter: MinimumIntervalRateLimiter | None = None, - order_history_limiter: MinimumIntervalRateLimiter | None = None, - ): - if not client_id or not client_secret: - raise BlockedError("Toss client_id and client_secret are required") - if isinstance(account_seq, bool) or not isinstance(account_seq, int): - raise BlockedError("Toss account_seq must be an integer") - self.client_id = client_id - self.client_secret = client_secret - self.account_seq = account_seq - self.transport = transport or UrlLibTransport() - if access_token is not None and ( - not isinstance(access_token, str) or not access_token - ): - raise BlockedError("Toss access_token must be a non-empty string") - if (access_token is None) != (access_token_expires_at is None): - raise BlockedError( - "Toss external access token and expiration timestamp " - "must be provided together" - ) - self.access_token = access_token - self.token_expires_at = ( - None - if access_token_expires_at is None - else parse_expiration_timestamp( - access_token_expires_at, - "Toss access token expiration", - ) - ) - self.market_data_limiter = market_data_limiter or MinimumIntervalRateLimiter( - self.MARKET_DATA_TPS - ) - self.order_limiter = order_limiter or MinimumIntervalRateLimiter(self.ORDER_TPS) - self.order_history_limiter = ( - order_history_limiter or MinimumIntervalRateLimiter(self.ORDER_HISTORY_TPS) - ) - - @staticmethod - def capabilities() -> dict[str, Any]: - return { - "broker": "toss", - "supported_markets": ["KR", "US"], - "paper": False, - "quote_batch_max": 200, - "batch_bid_ask": False, - "order_types": ["LIMIT", "MARKET"], - "initial_contract_order_types": ["LIMIT"], - "client_order_id": True, - "client_order_id_ttl_seconds": 600, - "query_by_client_order_id": False, - "modify_returns_new_id": True, - "cancel_returns_new_id": True, - "individual_fills": False, - "conditional_types": ["SINGLE", "OCO", "OTO"], - "client_rate_limits_tps": { - "MARKET_DATA": TossBroker.MARKET_DATA_TPS, - "ORDER": TossBroker.ORDER_TPS, - "ORDER_HISTORY": TossBroker.ORDER_HISTORY_TPS, - }, - "order_rate_limit_policy": "peak_safe", - } - - def token(self) -> str: - if ( - self.access_token - and self.token_expires_at is not None - and datetime.now(timezone.utc) + timedelta(seconds=60) - < self.token_expires_at - ): - return self.access_token - response = self.transport.request( - "POST", - f"{self.LIVE_BASE_URL}/oauth2/token", - form_body={ - "grant_type": "client_credentials", - "client_id": self.client_id, - "client_secret": self.client_secret, - }, - timeout_seconds=10, - mutation=False, - ) - body = require_mapping(response.body, "Toss token response") - if response.status != 200 or "access_token" not in body: - raise BlockedError( - f"Toss token request failed: HTTP {response.status} " - f"{body.get('error', 'unknown')}" - ) - expires_in = int(body.get("expires_in", 0)) - if expires_in <= 0: - raise BlockedError("Toss token response has invalid expires_in") - self.access_token = str(body["access_token"]) - self.token_expires_at = datetime.now(timezone.utc) + timedelta( - seconds=expires_in - ) - return self.access_token - - def headers(self, *, account: bool) -> dict[str, str]: - headers = {"Authorization": f"Bearer {self.token()}"} - if account: - headers["X-Tossinvest-Account"] = str(self.account_seq) - return headers - - def limited_request( - self, - limiter: MinimumIntervalRateLimiter, - method: str, - url: str, - *, - deadline_at: datetime | None = None, - **kwargs: Any, - ) -> HttpResponse: - limiter.acquire(deadline_at=deadline_at) - request_started_at = datetime.now(timezone.utc).isoformat() - response = self.transport.request(method, url, **kwargs) - limiter.observe_headers(response.headers) - return replace( - response, - request_started_at=response.request_started_at or request_started_at, - received_at=response.received_at or datetime.now(timezone.utc).isoformat(), - ) - - def mutation_request_hash( - self, - method: str, - path: str, - body: dict[str, Any], - ) -> str: - return sha256_json( - { - "method": method.upper(), - "path": path, - "account_seq": str(self.account_seq), - "body": body, - } - ) - - @staticmethod - def build_order_body(intent: dict[str, Any]) -> dict[str, Any]: - if intent.get("side") not in {"BUY", "SELL"}: - raise BlockedError("Toss intent side must be BUY or SELL") - if intent.get("order_type") != "LIMIT": - raise BlockedError("initial Toss contract requires LIMIT") - quantity = decimal_value(intent.get("quantity"), "quantity") - if quantity <= 0 or quantity != quantity.to_integral_value(): - raise BlockedError("initial Toss contract requires positive whole quantity") - client_order_id = str(intent.get("client_order_id", "")) - if ( - not client_order_id - or len(client_order_id) > 36 - or any( - character - not in "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" - for character in client_order_id - ) - ): - raise BlockedError("invalid Toss client_order_id") - return { - "clientOrderId": client_order_id, - "symbol": normalize_symbol(str(intent["market"]), str(intent["symbol"])), - "side": intent["side"], - "orderType": "LIMIT", - "timeInForce": "DAY", - "quantity": format(quantity, "f"), - "price": decimal_string(intent["limit_price"], "limit_price"), - "confirmHighValueOrder": False, - } - - @staticmethod - def build_oco_body(intent: dict[str, Any], expire_date: str) -> dict[str, Any]: - quantity = decimal_value(intent["quantity"], "quantity") - if quantity != quantity.to_integral_value(): - raise BlockedError("Toss OCO quantity must be a whole number") - take_profit = decimal_string(intent["take_profit_price"], "take_profit_price") - stop = decimal_string(intent["stop_price"], "stop_price") - return { - "symbol": normalize_symbol(str(intent["market"]), str(intent["symbol"])), - "type": "OCO", - "quantity": format(quantity, "f"), - "orderType": "LIMIT", - "clientOrderId": f"oco-{str(intent['intent_id'])[:28]}", - "expireDate": expire_date, - "first": { - "orderSide": "SELL", - "triggerPrice": take_profit, - "orderPrice": take_profit, - }, - "second": { - "orderSide": "SELL", - "triggerPrice": stop, - "orderPrice": stop, - }, - "confirmHighValueOrder": False, - } - - def preview_submit(self, intent: dict[str, Any]) -> dict[str, Any]: - body = self.build_order_body(intent) - request_hash = self.mutation_request_hash( - "POST", - self.CREATE_ORDER_PATH, - body, - ) - return { - "method": "POST", - "path": self.CREATE_ORDER_PATH, - "headers": { - "Authorization": "[REDACTED]", - "X-Tossinvest-Account": "[REDACTED]", - }, - "body": body, - "request_hash": request_hash, - "timeout_policy": "retry same clientOrderId and byte-equivalent body only within 600 seconds", - } - - def submit( - self, - intent: dict[str, Any], - *, - deadline_at: datetime | None = None, - ) -> dict[str, Any]: - body = self.build_order_body(intent) - request_hash = self.mutation_request_hash( - "POST", - self.CREATE_ORDER_PATH, - body, - ) - response = self.limited_request( - self.order_limiter, - "POST", - f"{self.LIVE_BASE_URL}{self.CREATE_ORDER_PATH}", - headers=self.headers(account=True), - json_body=body, - timeout_seconds=10, - mutation=True, - deadline_at=deadline_at, - ) - result = toss_mutation_result(response, "submit") - order_id = mutation_order_id( - result.get("orderId"), - "Toss submit result orderId", - ) - return { - "broker_order_id": order_id, - "client_order_id": result.get("clientOrderId"), - "request_hash": request_hash, - "submit_started_at": response.request_started_at, - "ack_received_at": response.received_at, - "raw": redact(result), - } - - def quote(self, market: str, symbol: str) -> dict[str, Any]: - normalized = normalize_symbol(market, symbol) - headers = self.headers(account=False) - price_response = self.limited_request( - self.market_data_limiter, - "GET", - f"{self.LIVE_BASE_URL}/api/v1/prices", - headers=headers, - query={"symbols": normalized}, - timeout_seconds=5, - ) - prices = toss_result(price_response, "price") - if not isinstance(prices, list): - raise BlockedError("Toss price result must be an array") - matching = [ - item - for item in prices - if isinstance(item, dict) and item.get("symbol") == normalized - ] - if len(matching) != 1: - raise BlockedError(f"Toss price missing or duplicated: {normalized}") - orderbook_response = self.limited_request( - self.market_data_limiter, - "GET", - f"{self.LIVE_BASE_URL}/api/v1/orderbook", - headers=headers, - query={"symbol": normalized}, - timeout_seconds=5, - ) - orderbook = require_mapping( - toss_result(orderbook_response, "orderbook"), "Toss orderbook result" - ) - asks = positive_level_prices(orderbook.get("asks"), "Toss asks") - bids = positive_level_prices(orderbook.get("bids"), "Toss bids") - return { - "market": market, - "symbol": normalized, - "currency": matching[0].get("currency"), - "last_price": decimal_string(matching[0].get("lastPrice"), "lastPrice"), - "best_ask": format(min(asks), "f"), - "best_bid": format(max(bids), "f"), - "trade_timestamp": matching[0].get("timestamp"), - "book_timestamp": orderbook.get("timestamp"), - "received_at": datetime.now(timezone.utc).isoformat(), - "raw_status": "OK", - } - - def get_order(self, broker_order_id: str) -> dict[str, Any]: - response = self.limited_request( - self.order_history_limiter, - "GET", - f"{self.LIVE_BASE_URL}/api/v1/orders/{broker_order_id}", - headers=self.headers(account=True), - timeout_seconds=10, - ) - return require_mapping(toss_result(response, "get order"), "Toss order result") - - def cancel(self, broker_order_id: str) -> dict[str, Any]: - path = f"/api/v1/orders/{broker_order_id}/cancel" - body: dict[str, Any] = {} - request_hash = self.mutation_request_hash("POST", path, body) - response = self.limited_request( - self.order_limiter, - "POST", - f"{self.LIVE_BASE_URL}{path}", - headers=self.headers(account=True), - json_body=body, - timeout_seconds=10, - mutation=True, - ) - result = toss_mutation_result(response, "cancel") - operation_order_id = mutation_order_id( - result.get("orderId"), - "Toss cancel result operation orderId", - ) - return { - "original_order_id": broker_order_id, - "operation_order_id": operation_order_id, - "request_hash": request_hash, - "raw": redact(result), - } - - -class KisBroker: - BASE_URLS: ClassVar[dict[str, str]] = { - "live": "https://openapi.koreainvestment.com:9443", - "paper": "https://openapivts.koreainvestment.com:29443", - } - KR_QUOTE_VENUE: ClassVar[dict[str, str]] = { - "KRX": "J", - "NXT": "NX", - "SOR": "UN", - } - US_ORDER_VENUES: ClassVar[set[str]] = {"NASD", "NYSE", "AMEX"} - US_QUOTE_VENUE: ClassVar[dict[str, str]] = { - "NASD": "NAS", - "NYSE": "NYS", - "AMEX": "AMS", - } - REGIME_BENCHMARKS: ClassVar[dict[str, dict[str, str]]] = { - "KOSPI": { - "market": "KR", - "benchmark_id": "KOSPI_COMPOSITE", - "provider_symbol": "0001", - "regime_proxy_id": "KOSPI_COMPOSITE", - "timezone": "Asia/Seoul", - }, - "KOSDAQ": { - "market": "KR", - "benchmark_id": "KOSDAQ_COMPOSITE", - "provider_symbol": "1001", - "regime_proxy_id": "KOSDAQ_COMPOSITE", - "timezone": "Asia/Seoul", - }, - "NYSE": { - "market": "US", - "benchmark_id": "NYSE_COMPOSITE", - "provider_symbol": "SPX", - "regime_proxy_id": "S&P_500", - "timezone": "America/New_York", - }, - "NASDAQ": { - "market": "US", - "benchmark_id": "NASDAQ_COMPOSITE", - "provider_symbol": "COMP", - "regime_proxy_id": "NASDAQ_COMPOSITE", - "timezone": "America/New_York", - }, - } - - def __init__( - self, - *, - app_key: str, - app_secret: str, - account_prefix: str, - account_product: str, - environment: str, - transport: Any | None = None, - access_token: str | None = None, - ): - if environment not in self.BASE_URLS: - raise BlockedError("KIS environment must be live or paper") - if not app_key or not app_secret: - raise BlockedError("KIS app_key and app_secret are required") - if len(account_prefix) != 8 or not account_prefix.isdigit(): - raise BlockedError("KIS account_prefix must be 8 digits") - if len(account_product) != 2 or not account_product.isdigit(): - raise BlockedError("KIS account_product must be 2 digits") - self.app_key = app_key - self.app_secret = app_secret - self.account_prefix = account_prefix - self.account_product = account_product - self.environment = environment - raw_transport = transport or UrlLibTransport() - interval = ( - 0.0 if transport is not None else (1.05 if environment == "paper" else 0.12) - ) - self.transport = PacedTransport(raw_transport, interval) - self.access_token = access_token - - @staticmethod - def capabilities(environment: str) -> dict[str, Any]: - if environment not in {"live", "paper"}: - raise BlockedError("KIS environment must be live or paper") - return { - "broker": "kis", - "environment": environment, - "supported_markets": ["KR", "US"], - "paper": True, - "polling_order_book_markets": ( - ["KR"] if environment == "paper" else ["KR", "US"] - ), - "us_paper_polling_session": False, - "quote_batch_max": 1, - "batch_bid_ask": False, - "order_types": ["LIMIT"], - "client_order_id": False, - "query_by_client_order_id": False, - "modify_returns_new_id": False, - "cancel_returns_new_id": False, - "individual_fills": False, - "server_oco": False, - "whole_shares_only": True, - "minimum_request_interval_ms": 1000 if environment == "paper" else 100, - } - - @property - def base_url(self) -> str: - return self.BASE_URLS[self.environment] - - def token(self) -> str: - if self.access_token: - return self.access_token - response = self.transport.request( - "POST", - f"{self.base_url}/oauth2/tokenP", - json_body={ - "grant_type": "client_credentials", - "appkey": self.app_key, - "appsecret": self.app_secret, - }, - timeout_seconds=10, - ) - body = require_mapping(response.body, "KIS token response") - if response.status != 200 or not body.get("access_token"): - raise BlockedError( - f"KIS token request failed: HTTP {response.status} " - f"{body.get('error_code', body.get('msg_cd', 'unknown'))}" - ) - self.access_token = str(body["access_token"]) - return self.access_token - - def headers(self, tr_id: str) -> dict[str, str]: - return { - "authorization": f"Bearer {self.token()}", - "appkey": self.app_key, - "appsecret": self.app_secret, - "tr_id": tr_id, - "custtype": "P", - "Content-Type": "application/json", - } - - def order_tr_id(self, market: str, side: str) -> str: - market = market.upper() - side = side.upper() - if market == "KR": - live = "TTTC0012U" if side == "BUY" else "TTTC0011U" - return live if self.environment == "live" else f"V{live[1:]}" - if market == "US": - if side == "BUY": - return "TTTT1002U" if self.environment == "live" else "VTTT1002U" - if side == "SELL": - return "TTTT1006U" if self.environment == "live" else "VTTT1001U" - raise BlockedError(f"unsupported KIS order market/side: {market}/{side}") - - def kr_quote_venue(self, venue: str) -> str: - venue = venue.upper() - if venue not in self.KR_QUOTE_VENUE: - raise BlockedError("KR venue must be KRX, NXT, or SOR") - if self.environment == "paper" and venue != "KRX": - raise BlockedError("KIS paper KR quotes require KRX") - return self.KR_QUOTE_VENUE[venue] - - def build_order_request( - self, intent: dict[str, Any], *, venue: str - ) -> dict[str, Any]: - market = str(intent["market"]).upper() - side = str(intent["side"]).upper() - if side not in {"BUY", "SELL"}: - raise BlockedError("KIS side must be BUY or SELL") - if intent.get("order_type") != "LIMIT": - raise BlockedError("initial KIS contract requires LIMIT") - quantity = decimal_value(intent["quantity"], "quantity") - if quantity <= 0 or quantity != quantity.to_integral_value(): - raise BlockedError("KIS stock quantity must be a positive whole number") - symbol = normalize_symbol(market, str(intent["symbol"])) - price = decimal_string(intent["limit_price"], "limit_price") - tr_id = self.order_tr_id(market, side) - if market == "KR": - if venue not in {"KRX", "NXT", "SOR"}: - raise BlockedError("KR venue must be KRX, NXT, or SOR") - if self.environment == "paper" and venue != "KRX": - raise BlockedError("KIS paper KR orders require KRX") - body = { - "CANO": self.account_prefix, - "ACNT_PRDT_CD": self.account_product, - "PDNO": symbol, - "ORD_DVSN": "00", - "ORD_QTY": format(quantity, "f"), - "ORD_UNPR": price, - "EXCG_ID_DVSN_CD": venue, - "SLL_TYPE": "01" if side == "SELL" else "", - "CNDT_PRIC": "", - } - path = "/uapi/domestic-stock/v1/trading/order-cash" - else: - if venue not in self.US_ORDER_VENUES: - raise BlockedError("US venue must be NASD, NYSE, or AMEX") - body = { - "CANO": self.account_prefix, - "ACNT_PRDT_CD": self.account_product, - "OVRS_EXCG_CD": venue, - "PDNO": symbol, - "ORD_QTY": format(quantity, "f"), - "OVRS_ORD_UNPR": price, - "CTAC_TLNO": "", - "MGCO_APTM_ODNO": "", - "SLL_TYPE": "00" if side == "SELL" else "", - "ORD_SVR_DVSN_CD": "0", - "ORD_DVSN": "00", - } - path = "/uapi/overseas-stock/v1/trading/order" - return { - "method": "POST", - "path": path, - "tr_id": tr_id, - "body": body, - "request_hash": sha256_json({"tr_id": tr_id, "body": body}), - } - - def preview_submit(self, intent: dict[str, Any], *, venue: str) -> dict[str, Any]: - request = self.build_order_request(intent, venue=venue) - return { - **request, - "headers": { - "authorization": "[REDACTED]", - "appkey": "[REDACTED]", - "appsecret": "[REDACTED]", - "tr_id": request["tr_id"], - "custtype": "P", - }, - "body": redact(request["body"]), - "timeout_policy": "never resubmit automatically; reconcile orders, fills, and balances", - } - - def submit( - self, - intent: dict[str, Any], - *, - venue: str, - deadline_at: datetime | None = None, - ) -> dict[str, Any]: - request = self.build_order_request(intent, venue=venue) - response = self.transport.request( - "POST", - f"{self.base_url}{request['path']}", - headers=self.headers(request["tr_id"]), - json_body=request["body"], - timeout_seconds=10, - mutation=True, - deadline_at=deadline_at, - ) - output = kis_mutation_result(response, "submit") - order_id = mutation_order_id( - output.get("ODNO"), - "KIS submit output ODNO", - ) - return { - "broker_order_id": order_id, - "request_hash": request["request_hash"], - "broker_time": output.get("ORD_TMD"), - "organization": output.get("KRX_FWDG_ORD_ORGNO"), - "submit_started_at": response.request_started_at, - "ack_received_at": response.received_at, - "raw": redact(output), - } - - def quote(self, market: str, symbol: str, *, venue: str) -> dict[str, Any]: - market = market.upper() - symbol = normalize_symbol(market, symbol) - if market == "KR": - quote_venue = self.kr_quote_venue(venue) - tr_id = "FHKST01010100" - price_response = self.transport.request( - "GET", - f"{self.base_url}/uapi/domestic-stock/v1/quotations/inquire-price", - headers=self.headers(tr_id), - query={ - "FID_COND_MRKT_DIV_CODE": quote_venue, - "FID_INPUT_ISCD": symbol, - }, - timeout_seconds=5, - ) - price_body = kis_result(price_response, "KR price") - output = require_mapping(price_body.get("output"), "KIS KR price output") - book_tr = "FHKST01010200" - book_response = self.transport.request( - "GET", - f"{self.base_url}/uapi/domestic-stock/v1/quotations/inquire-asking-price-exp-ccn", - headers=self.headers(book_tr), - query={ - "FID_COND_MRKT_DIV_CODE": quote_venue, - "FID_INPUT_ISCD": symbol, - }, - timeout_seconds=5, - ) - book_body = kis_result(book_response, "KR orderbook") - book = require_mapping(book_body.get("output1"), "KIS KR orderbook output") - received_at = datetime.now(timezone.utc) - price_date = str(output.get("stck_bsop_date") or "").strip() - book_date = str(book.get("stck_bsop_date") or "").strip() - if any( - len(value) != 8 or not value.isdigit() - for value in (price_date, book_date) - ): - raise BlockedError( - "KIS KR price and orderbook responses each require a valid " - "broker-supplied stck_bsop_date" - ) - if price_date != book_date: - raise BlockedError( - "KIS KR price and orderbook stck_bsop_date values must match" - ) - trade_timestamp = kis_market_timestamp( - date_value=price_date, - time_value=output.get("stck_cntg_hour"), - timezone_name="Asia/Seoul", - ) - book_timestamp = kis_market_timestamp( - date_value=book_date, - time_value=book.get("aspr_acpt_hour"), - timezone_name="Asia/Seoul", - ) - if trade_timestamp is None or book_timestamp is None: - raise BlockedError( - "KIS KR quote response requires independent broker-supplied " - "stck_cntg_hour and aspr_acpt_hour timestamps" - ) - last = output.get("stck_prpr") - if last is None: - raise BlockedError("KIS KR price response missing last price") - return { - "market": "KR", - "symbol": symbol, - "currency": "KRW", - "last_price": decimal_string(last, "stck_prpr"), - "best_ask": decimal_string(book.get("askp1"), "askp1"), - "best_bid": decimal_string(book.get("bidp1"), "bidp1"), - "trade_timestamp": trade_timestamp, - "book_timestamp": book_timestamp, - "received_at": received_at.isoformat(), - "source_timestamp_raw": { - "trade": { - "stck_bsop_date": price_date, - "stck_cntg_hour": output.get("stck_cntg_hour"), - }, - "book": { - "stck_bsop_date": book_date, - "aspr_acpt_hour": book.get("aspr_acpt_hour"), - }, - }, - "raw_status": "OK", - } - if self.environment == "paper": - raise BlockedError( - "KIS U.S. paper order book HHDFS76200100 is unsupported; " - "use production shadow quotes and separate paper serialization tests" - ) - if venue not in self.US_ORDER_VENUES: - raise BlockedError("US venue must be NASD, NYSE, or AMEX") - quote_venue = self.US_QUOTE_VENUE[venue] - book_tr = "HHDFS76200100" - book_response = self.transport.request( - "GET", - f"{self.base_url}/uapi/overseas-price/v1/quotations/inquire-asking-price", - headers=self.headers(book_tr), - query={"AUTH": "", "EXCD": quote_venue, "SYMB": symbol}, - timeout_seconds=5, - ) - book_body = kis_result(book_response, "US orderbook") - book = require_mapping(book_body.get("output1"), "KIS US orderbook output") - last = book.get("last") - ask = book.get("pask1") or book.get("pask") - bid = book.get("pbid1") or book.get("pbid") - if last is None or ask is None or bid is None: - raise BlockedError( - "KIS US orderbook response missing same-source last/bid/ask" - ) - received_at = datetime.now(timezone.utc) - source_timestamp = kis_market_timestamp( - date_value=book.get("dymd"), - time_value=book.get("dhms"), - timezone_name="America/New_York", - ) - if source_timestamp is None: - raise BlockedError( - "KIS US orderbook response requires broker-supplied dymd/dhms" - ) - return { - "market": "US", - "symbol": symbol, - "currency": "USD", - "last_price": decimal_string(last, "last"), - "best_ask": decimal_string(ask, "best ask"), - "best_bid": decimal_string(bid, "best bid"), - "trade_timestamp": source_timestamp, - "book_timestamp": source_timestamp, - "received_at": received_at.isoformat(), - "source_timestamp_raw": { - "dymd": book.get("dymd"), - "dhms": book.get("dhms"), - }, - "raw_status": "OK", - } - - def benchmark_quote( - self, - *, - exchange: str, - session_date: str, - ) -> dict[str, Any]: - """Return one same-session, read-only benchmark regime observation.""" - exchange = str(exchange).upper() - contract = self.REGIME_BENCHMARKS.get(exchange) - if contract is None: - raise BlockedError(f"unsupported benchmark exchange: {exchange}") - try: - parsed_session_date = date.fromisoformat(session_date) - except (TypeError, ValueError) as exc: - raise BlockedError("benchmark session_date must be YYYY-MM-DD") from exc - if parsed_session_date.isoformat() != session_date: - raise BlockedError("benchmark session_date must be YYYY-MM-DD") - compact_date = session_date.replace("-", "") - received_at = datetime.now(timezone.utc) - - if contract["market"] == "KR": - tr_id = "FHPUP02110100" - response = self.transport.request( - "GET", - ( - f"{self.base_url}/uapi/domestic-stock/v1/quotations/" - "inquire-index-tickprice" - ), - headers=self.headers(tr_id), - query={ - "FID_INPUT_ISCD": contract["provider_symbol"], - "FID_COND_MRKT_DIV_CODE": "U", - }, - timeout_seconds=5, - ) - body = kis_result(response, f"{exchange} benchmark") - rows = body.get("output") - if not isinstance(rows, list) or not rows: - raise BlockedError( - f"KIS {exchange} benchmark output must be a non-empty array" - ) - valid_rows = [ - row - for row in rows - if isinstance(row, dict) - and kis_market_timestamp( - date_value=compact_date, - time_value=row.get("stck_cntg_hour"), - timezone_name=contract["timezone"], - ) - is not None - ] - if not valid_rows: - raise BlockedError( - f"KIS {exchange} benchmark has no broker timestamp" - ) - latest = max( - valid_rows, - key=lambda row: str(row["stck_cntg_hour"]), - ) - source_timestamp = kis_market_timestamp( - date_value=compact_date, - time_value=latest["stck_cntg_hour"], - timezone_name=contract["timezone"], - ) - current = decimal_value( - latest.get("bstp_nmix_prpr"), - f"{exchange} benchmark current", - ) - difference = abs( - decimal_value( - latest.get("bstp_nmix_prdy_vrss"), - f"{exchange} benchmark previous difference", - ) - ) - sign = str(latest.get("prdy_vrss_sign") or "").strip() - if sign in {"1", "2"}: - signed_difference = difference - elif sign == "3": - signed_difference = Decimal(0) - elif sign in {"4", "5"}: - signed_difference = -difference - else: - raise BlockedError( - f"KIS {exchange} benchmark has an unknown previous-close sign" - ) - previous_close = current - signed_difference - if current <= 0 or previous_close <= 0: - raise BlockedError( - f"KIS {exchange} benchmark prices must be positive" - ) - else: - if self.environment == "paper": - raise BlockedError( - "KIS U.S. paper benchmark feed is unsupported; use " - "kis-live/shadow" - ) - tr_id = "FHKST03030200" - response = self.transport.request( - "GET", - ( - f"{self.base_url}/uapi/overseas-price/v1/quotations/" - "inquire-time-indexchartprice" - ), - headers=self.headers(tr_id), - query={ - "FID_COND_MRKT_DIV_CODE": "N", - "FID_INPUT_ISCD": contract["provider_symbol"], - "FID_HOUR_CLS_CODE": "0", - "FID_PW_DATA_INCU_YN": "N", - }, - timeout_seconds=5, - ) - body = kis_result(response, f"{exchange} benchmark") - summary = require_mapping( - body.get("output1"), - f"KIS {exchange} benchmark output1", - ) - rows = body.get("output2") - if not isinstance(rows, list) or not rows: - raise BlockedError( - f"KIS {exchange} benchmark output2 must be a non-empty array" - ) - valid_rows = [ - row - for row in rows - if isinstance(row, dict) - and str(row.get("stck_bsop_date") or "").strip() == compact_date - and kis_market_timestamp( - date_value=row.get("stck_bsop_date"), - time_value=row.get("stck_cntg_hour"), - timezone_name=contract["timezone"], - ) - is not None - ] - if not valid_rows: - raise BlockedError( - f"KIS {exchange} benchmark has no bar for {session_date}" - ) - latest = max( - valid_rows, - key=lambda row: str(row["stck_cntg_hour"]), - ) - source_timestamp = kis_market_timestamp( - date_value=latest["stck_bsop_date"], - time_value=latest["stck_cntg_hour"], - timezone_name=contract["timezone"], - ) - current = decimal_value( - latest.get("optn_prpr"), - f"{exchange} benchmark current", - ) - previous_close = decimal_value( - summary.get("ovrs_nmix_prdy_clpr"), - f"{exchange} benchmark previous close", - ) - if current <= 0 or previous_close <= 0: - raise BlockedError( - f"KIS {exchange} benchmark prices must be positive" - ) - - if source_timestamp is None: - raise BlockedError( - f"KIS {exchange} benchmark source timestamp is invalid" - ) - change_bps = ( - (current / previous_close - Decimal(1)) * Decimal(10000) - ).quantize(Decimal("0.01"), rounding=ROUND_HALF_UP) - return { - "schema": "qta-market-regime-quote/v1", - "broker": "kis", - "exchange": exchange, - "benchmark_id": contract["benchmark_id"], - "regime_proxy_id": contract["regime_proxy_id"], - "provider_symbol": contract["provider_symbol"], - "current": format(current, "f"), - "previous_close": format(previous_close, "f"), - "change_bps": format(change_bps, "f"), - "source_timestamp": source_timestamp, - "received_at": received_at.isoformat(), - "raw_status": "OK", - } - - def build_cancel_request( - self, - *, - market: str, - broker_order_id: str, - symbol: str, - quantity: str, - price: str, - venue: str, - organization: str = "", - ) -> dict[str, Any]: - market = market.upper() - quantity = decimal_string(quantity, "cancel quantity") - price = decimal_string(price, "cancel price") - if market == "KR": - if venue not in {"KRX", "NXT", "SOR"}: - raise BlockedError("KR venue must be KRX, NXT, or SOR") - tr_id = "TTTC0013U" if self.environment == "live" else "VTTC0013U" - path = "/uapi/domestic-stock/v1/trading/order-rvsecncl" - body = { - "CANO": self.account_prefix, - "ACNT_PRDT_CD": self.account_product, - "KRX_FWDG_ORD_ORGNO": organization, - "ORGN_ODNO": broker_order_id, - "ORD_DVSN": "00", - "RVSE_CNCL_DVSN_CD": "02", - "ORD_QTY": quantity, - "ORD_UNPR": price, - "QTY_ALL_ORD_YN": "Y", - "EXCG_ID_DVSN_CD": venue, - } - elif market == "US": - if venue not in self.US_ORDER_VENUES: - raise BlockedError("US venue must be NASD, NYSE, or AMEX") - tr_id = "TTTT1004U" if self.environment == "live" else "VTTT1004U" - path = "/uapi/overseas-stock/v1/trading/order-rvsecncl" - body = { - "CANO": self.account_prefix, - "ACNT_PRDT_CD": self.account_product, - "OVRS_EXCG_CD": venue, - "PDNO": normalize_symbol("US", symbol), - "ORGN_ODNO": broker_order_id, - "RVSE_CNCL_DVSN_CD": "02", - "ORD_QTY": quantity, - "OVRS_ORD_UNPR": "0", - "MGCO_APTM_ODNO": "", - "ORD_SVR_DVSN_CD": "0", - } - else: - raise BlockedError(f"unsupported market: {market}") - return { - "method": "POST", - "path": path, - "tr_id": tr_id, - "body": body, - "request_hash": sha256_json({"tr_id": tr_id, "body": body}), - } - - def cancel(self, **kwargs: Any) -> dict[str, Any]: - request = self.build_cancel_request(**kwargs) - response = self.transport.request( - "POST", - f"{self.base_url}{request['path']}", - headers=self.headers(request["tr_id"]), - json_body=request["body"], - timeout_seconds=10, - mutation=True, - ) - output = kis_mutation_result(response, "cancel") - operation_order_id = mutation_order_id( - output.get("ODNO"), - "KIS cancel output ODNO", - ) - return { - "original_order_id": kwargs["broker_order_id"], - "operation_order_id": operation_order_id, - "request_hash": request["request_hash"], - "raw": redact(output), - } - - @staticmethod - def _case_value(record: dict[str, Any], *names: str) -> Any: - lowered = {str(key).lower(): value for key, value in record.items()} - for name in names: - if name.lower() in lowered: - return lowered[name.lower()] - return None - - @staticmethod - def _same_order_id(left: Any, right: Any) -> bool: - a = str(left or "").strip() - b = str(right or "").strip() - if not a or not b: - return False - return a == b or a.lstrip("0") == b.lstrip("0") - - def order_history( - self, - *, - market: str, - trading_date: str, - symbol: str, - venue: str, - ) -> list[dict[str, Any]]: - market = market.upper() - compact_date = trading_date.replace("-", "") - if len(compact_date) != 8 or not compact_date.isdigit(): - raise BlockedError("trading_date must be YYYY-MM-DD") - if market == "KR": - tr_id = "TTTC0081R" if self.environment == "live" else "VTTC0081R" - response = self.transport.request( - "GET", - f"{self.base_url}/uapi/domestic-stock/v1/trading/inquire-daily-ccld", - headers=self.headers(tr_id), - query={ - "CANO": self.account_prefix, - "ACNT_PRDT_CD": self.account_product, - "INQR_STRT_DT": compact_date, - "INQR_END_DT": compact_date, - "SLL_BUY_DVSN_CD": "00", - "PDNO": normalize_symbol("KR", symbol), - "CCLD_DVSN": "00", - "INQR_DVSN": "00", - "INQR_DVSN_3": "00", - "ORD_GNO_BRNO": "", - "ODNO": "", - "INQR_DVSN_1": "", - "CTX_AREA_FK100": "", - "CTX_AREA_NK100": "", - "EXCG_ID_DVSN_CD": venue, - }, - timeout_seconds=10, - ) - body = kis_result(response, "KR order history") - records = body.get("output1", []) - elif market == "US": - tr_id = "TTTS3035R" if self.environment == "live" else "VTTS3035R" - response = self.transport.request( - "GET", - f"{self.base_url}/uapi/overseas-stock/v1/trading/inquire-ccnl", - headers=self.headers(tr_id), - query={ - "CANO": self.account_prefix, - "ACNT_PRDT_CD": self.account_product, - "PDNO": "" if self.environment == "paper" else "%", - "ORD_STRT_DT": compact_date, - "ORD_END_DT": compact_date, - "SLL_BUY_DVSN": "00", - "CCLD_NCCS_DVSN": "00", - "OVRS_EXCG_CD": "" if self.environment == "paper" else "NASD", - "SORT_SQN": "DS", - "ORD_DT": "", - "ORD_GNO_BRNO": "", - "ODNO": "", - "CTX_AREA_NK200": "", - "CTX_AREA_FK200": "", - }, - timeout_seconds=10, - ) - body = kis_result(response, "US order history") - records = body.get("output", []) - else: - raise BlockedError(f"unsupported KIS history market: {market}") - if isinstance(records, dict): - records = [records] - if not isinstance(records, list) or any( - not isinstance(item, dict) for item in records - ): - raise BlockedError("KIS order history output must be an object array") - return records - - def get_order( - self, - *, - market: str, - trading_date: str, - symbol: str, - venue: str, - broker_order_id: str, - ) -> dict[str, Any]: - records = self.order_history( - market=market, - trading_date=trading_date, - symbol=symbol, - venue=venue, - ) - matches = [ - record - for record in records - if self._same_order_id( - self._case_value(record, "ODNO", "odno"), broker_order_id - ) - ] - if len(matches) != 1: - raise BlockedError( - f"KIS order {broker_order_id} match count is {len(matches)}" - ) - record = matches[0] - ordered = self._case_value( - record, "ORD_QTY", "ord_qty", "FT_ORD_QTY", "ft_ord_qty" - ) - filled = self._case_value( - record, - "TOT_CCLD_QTY", - "tot_ccld_qty", - "FT_CCLD_QTY", - "ft_ccld_qty", - ) - average_fill_price = self._case_value( - record, - "AVG_PRVS", - "avg_prvs", - "FT_CCLD_UNPR3", - "ft_ccld_unpr3", - ) - remaining = self._case_value( - record, "RMN_QTY", "rmn_qty", "NCCS_QTY", "nccs_qty" - ) - ordered_decimal = ( - decimal_value(ordered, "ordered quantity") - if ordered not in (None, "") - else None - ) - filled_decimal = ( - decimal_value(filled, "filled quantity") - if filled not in (None, "") - else None - ) - remaining_decimal = ( - decimal_value(remaining, "remaining quantity") - if remaining not in (None, "") - else None - ) - average_fill_decimal = ( - decimal_value(average_fill_price, "average fill price") - if average_fill_price not in (None, "") - else None - ) - if average_fill_decimal is not None and average_fill_decimal <= 0: - if filled_decimal is not None and filled_decimal > 0: - raise BlockedError( - "KIS filled order has a nonpositive average fill price" - ) - average_fill_decimal = None - cancel_value = str( - self._case_value(record, "CNCL_YN", "cncl_yn", "CANCEL_YN") or "" - ).upper() - if ( - ordered_decimal is not None - and filled_decimal is not None - and ordered_decimal > 0 - and filled_decimal >= ordered_decimal - ): - normalized_status = "FILLED" - elif ( - filled_decimal is not None - and filled_decimal > 0 - and (remaining_decimal is None or remaining_decimal > 0) - ): - normalized_status = "PARTIALLY_FILLED" - elif cancel_value == "Y": - normalized_status = "CANCELLED" - elif remaining_decimal is not None and remaining_decimal > 0: - normalized_status = "ACKNOWLEDGED" - else: - normalized_status = "UNKNOWN" - return { - "broker_order_id": broker_order_id, - "normalized_status": normalized_status, - "ordered_quantity": None - if ordered_decimal is None - else format(ordered_decimal, "f"), - "filled_quantity": None - if filled_decimal is None - else format(filled_decimal, "f"), - "average_fill_price": None - if average_fill_decimal is None - else format(average_fill_decimal, "f"), - "remaining_quantity": None - if remaining_decimal is None - else format(remaining_decimal, "f"), - "raw": record, - } - - -def self_test() -> None: - intent = { - "intent_id": "1" * 32, - "client_order_id": "qta-" + "1" * 28, - "market": "US", - "symbol": "AAPL", - "side": "BUY", - "order_type": "LIMIT", - "time_in_force": "DAY", - "quantity": "1", - "limit_price": "200.00", - "stop_price": "180.00", - "take_profit_price": "240.00", - } - now = datetime.now(timezone.utc) - fake_clock = [0.0] - - def fake_monotonic() -> float: - return fake_clock[0] - - def fake_sleep(seconds: float) -> None: - fake_clock[0] += seconds - - market_data_limiter = MinimumIntervalRateLimiter( - 10, - monotonic=fake_monotonic, - sleeper=fake_sleep, - ) - toss_transport = QueueTransport( - [ - HttpResponse( - 200, - {}, - { - "result": { - "orderId": "toss-order-1", - "clientOrderId": intent["client_order_id"], - } - }, - ), - HttpResponse( - 200, - {"X-RateLimit-Limit": "10"}, - { - "result": [ - { - "symbol": "AAPL", - "timestamp": now.isoformat(), - "lastPrice": "200.50", - "currency": "USD", - } - ] - }, - ), - HttpResponse( - 200, - {"x-ratelimit-limit": "5"}, - { - "result": { - "timestamp": now.isoformat(), - "asks": [ - {"price": "202.00"}, - {"price": "0"}, - {"price": "201.00"}, - ], - "bids": [ - {"price": "199.00"}, - {"price": "0"}, - {"price": "200.00"}, - ], - } - }, - ), - ] - ) - toss = TossBroker( - client_id="test-client", - client_secret="test-secret", - account_seq=1, - transport=toss_transport, - access_token="test-token", - access_token_expires_at=now + timedelta(minutes=5), - market_data_limiter=market_data_limiter, - ) - preview = toss.preview_submit(intent) - assert preview["body"]["quantity"] == "1" - assert preview["body"]["price"] == "200.00" - assert preview["request_hash"] == toss.mutation_request_hash( - "POST", - "/api/v1/orders", - preview["body"], - ) - account_two = TossBroker( - client_id="test-client", - client_secret="test-secret", - account_seq=2, - transport=QueueTransport([]), - ) - assert account_two.preview_submit(intent)["request_hash"] != preview["request_hash"] - assert ( - toss.mutation_request_hash("PUT", "/api/v1/orders", preview["body"]) - != preview["request_hash"] - ) - assert ( - toss.mutation_request_hash("POST", "/api/v1/orders/other", preview["body"]) - != preview["request_hash"] - ) - changed_body = {**preview["body"], "price": "201.00"} - assert ( - toss.mutation_request_hash("POST", "/api/v1/orders", changed_body) - != preview["request_hash"] - ) - ack = toss.submit(intent) - assert ack["broker_order_id"] == "toss-order-1" - assert ack["request_hash"] == preview["request_hash"] - assert ack["submit_started_at"] - assert ack["ack_received_at"] - quote = toss.quote("US", "AAPL") - assert quote["best_ask"] == "201.00" - assert quote["best_bid"] == "200.00" - assert fake_clock[0] == 0.1 - assert market_data_limiter.minimum_interval_seconds == 0.2 - assert toss.capabilities()["client_order_id_ttl_seconds"] == 600 - assert toss.capabilities()["client_rate_limits_tps"]["ORDER"] == 3 - - try: - TossBroker( - client_id="test-client", - client_secret="test-secret", - account_seq=1, - access_token="test-token", - ) - except BlockedError: - pass - else: - raise AssertionError("Toss external token without expiration must be blocked") - - def toss_with_response(response: HttpResponse) -> TossBroker: - return TossBroker( - client_id="test-client", - client_secret="test-secret", - account_seq=1, - transport=QueueTransport([response]), - access_token="test-token", - access_token_expires_at=now + timedelta(minutes=5), - ) - - for ambiguous_response in ( - HttpResponse(503, {}, None), - HttpResponse(200, {}, {"result": {}}), - ): - try: - toss_with_response(ambiguous_response).submit(intent) - except AmbiguousMutationError: - pass - else: - raise AssertionError("Toss 5xx/missing-order-id mutation must be ambiguous") - try: - toss_with_response( - HttpResponse( - 400, - {}, - {"error": {"code": "INVALID_ORDER", "message": "rejected"}}, - ) - ).submit(intent) - except AuthoritativeMutationRejection: - pass - else: - raise AssertionError("explicit Toss rejection must be authoritative") - try: - toss_with_response(HttpResponse(200, {}, {"result": {}})).cancel( - "existing-order" - ) - except AmbiguousMutationError: - pass - else: - raise AssertionError("Toss cancel missing operation ID must be ambiguous") - malformed_toss_quote = TossBroker( - client_id="test-client", - client_secret="test-secret", - account_seq=1, - transport=QueueTransport( - [ - HttpResponse( - 200, - {}, - { - "result": [ - { - "symbol": "AAPL", - "timestamp": now.isoformat(), - "currency": "USD", - } - ] - }, - ), - HttpResponse( - 200, - {}, - { - "result": { - "timestamp": now.isoformat(), - "asks": [{"price": "201"}], - "bids": [{"price": "200"}], - } - }, - ), - ] - ), - access_token="test-token", - access_token_expires_at=now + timedelta(minutes=5), - ) - try: - malformed_toss_quote.quote("US", "AAPL") - except BlockedError as exc: - assert "lastPrice must be" in str(exc) - else: - raise AssertionError("missing Toss lastPrice must fail as BlockedError") - - kis_transport = QueueTransport( - [ - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "msg_cd": "0", - "msg1": "OK", - "output": {"ODNO": "12345", "ORD_TMD": "093001"}, - }, - ) - ] - ) - kis = KisBroker( - app_key="test-key", - app_secret="test-secret", - account_prefix="12345678", - account_product="01", - environment="paper", - transport=kis_transport, - access_token="test-token", - ) - kis_preview = kis.preview_submit(intent, venue="NASD") - assert kis_preview["tr_id"] == "VTTT1002U" - assert kis_preview["body"]["CANO"] == "[REDACTED]" - assert kis_preview["body"]["ACNT_PRDT_CD"] == "[REDACTED]" - kis_ack = kis.submit(intent, venue="NASD") - assert kis_ack["broker_order_id"] == "12345" - assert kis_ack["submit_started_at"] - assert kis_ack["ack_received_at"] - - def kis_with_response(response: HttpResponse) -> KisBroker: - return KisBroker( - app_key="test-key", - app_secret="test-secret", - account_prefix="12345678", - account_product="01", - environment="paper", - transport=QueueTransport([response]), - access_token="test-token", - ) - - for ambiguous_response in ( - HttpResponse(503, {}, {"rt_cd": "1", "msg_cd": "SERVER_ERROR"}), - HttpResponse( - 200, - {}, - {"rt_cd": "0", "msg_cd": "0", "msg1": "OK", "output": {}}, - ), - ): - try: - kis_with_response(ambiguous_response).submit(intent, venue="NASD") - except AmbiguousMutationError: - pass - else: - raise AssertionError("KIS 5xx/missing-order-id mutation must be ambiguous") - try: - kis_with_response( - HttpResponse( - 200, - {}, - { - "rt_cd": "1", - "msg_cd": "ORDER_REJECTED", - "msg1": "rejected", - }, - ) - ).submit(intent, venue="NASD") - except AuthoritativeMutationRejection: - pass - else: - raise AssertionError("explicit KIS rejection must be authoritative") - try: - kis_with_response( - HttpResponse( - 200, - {}, - {"rt_cd": "0", "msg_cd": "0", "msg1": "OK", "output": {}}, - ) - ).cancel( - market="US", - broker_order_id="12345", - symbol="AAPL", - quantity="1", - price="200", - venue="NASD", - ) - except AmbiguousMutationError: - pass - else: - raise AssertionError("KIS cancel missing operation ID must be ambiguous") - - deadline_delegate = QueueTransport( - [ - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "msg_cd": "0", - "msg1": "OK", - "output": {"ODNO": "late"}, - }, - ) - ] - ) - paced = PacedTransport(deadline_delegate, minimum_interval_seconds=10) - paced.last_started = time.monotonic() - try: - paced.request( - "POST", - "https://example.invalid/order", - deadline_at=datetime.now(timezone.utc) + timedelta(seconds=1), - ) - except BlockedError as exc: - assert "deadline" in str(exc) - else: - raise AssertionError("pacing past the submit deadline must be blocked") - assert deadline_delegate.requests == [] - - sell_intent = {**intent, "side": "SELL"} - assert kis.build_order_request(sell_intent, venue="NASD")["tr_id"] == "VTTT1001U" - kr_intent = { - **intent, - "market": "KR", - "symbol": "5930", - "limit_price": "70000", - } - kr_request = kis.build_order_request(kr_intent, venue="KRX") - assert kr_request["tr_id"] == "VTTC0012U" - assert kr_request["body"]["PDNO"] == "005930" - assert kis.kr_quote_venue("KRX") == "J" - try: - kis.kr_quote_venue("NXT") - except BlockedError: - pass - else: - raise AssertionError("KIS paper NXT quote must be blocked") - - live_kis = KisBroker( - app_key="test-key", - app_secret="test-secret", - account_prefix="12345678", - account_product="01", - environment="live", - transport=QueueTransport([]), - access_token="test-token", - ) - assert live_kis.kr_quote_venue("KRX") == "J" - assert live_kis.kr_quote_venue("NXT") == "NX" - assert live_kis.kr_quote_venue("SOR") == "UN" - - quote_transport = QueueTransport( - [ - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "msg_cd": "0", - "msg1": "OK", - "output": { - "stck_prpr": "70000", - "stck_bsop_date": "20260727", - "stck_cntg_hour": "093000", - }, - }, - ), - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "msg_cd": "0", - "msg1": "OK", - "output1": { - "askp1": "70100", - "bidp1": "69900", - "stck_bsop_date": "20260727", - "aspr_acpt_hour": "093001", - }, - "output2": {"stck_prpr": "70000"}, - }, - ), - ] - ) - quote_kis = KisBroker( - app_key="test-key", - app_secret="test-secret", - account_prefix="12345678", - account_product="01", - environment="live", - transport=quote_transport, - access_token="test-token", - ) - kr_quote = quote_kis.quote("KR", "005930", venue="NXT") - assert kr_quote["trade_timestamp"].endswith("T09:30:00+09:00") - assert kr_quote["book_timestamp"].endswith("T09:30:01+09:00") - assert kr_quote["trade_timestamp"] != kr_quote["book_timestamp"] - assert all( - request["query"]["FID_COND_MRKT_DIV_CODE"] == "NX" - for request in quote_transport.requests - ) - - malformed_kr_quote = KisBroker( - app_key="test-key", - app_secret="test-secret", - account_prefix="12345678", - account_product="01", - environment="live", - transport=QueueTransport( - [ - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "msg_cd": "0", - "msg1": "OK", - "output": { - "stck_prpr": "70000", - "stck_bsop_date": "20260727", - "stck_cntg_hour": "093000", - }, - }, - ), - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "msg_cd": "0", - "msg1": "OK", - "output1": { - "bidp1": "69900", - "stck_bsop_date": "20260727", - "aspr_acpt_hour": "093001", - }, - }, - ), - ] - ), - access_token="test-token", - ) - try: - malformed_kr_quote.quote("KR", "005930", venue="KRX") - except BlockedError as exc: - assert "askp1 must be" in str(exc) - else: - raise AssertionError("missing KIS askp1 must fail as BlockedError") - - undated_quote_kis = KisBroker( - app_key="test-key", - app_secret="test-secret", - account_prefix="12345678", - account_product="01", - environment="live", - transport=QueueTransport( - [ - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "msg_cd": "0", - "msg1": "OK", - "output": { - "stck_prpr": "70000", - "stck_cntg_hour": "093000", - }, - }, - ), - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "msg_cd": "0", - "msg1": "OK", - "output1": { - "askp1": "70100", - "bidp1": "69900", - "stck_bsop_date": "20260727", - "aspr_acpt_hour": "093001", - }, - "output2": {"stck_prpr": "70000"}, - }, - ), - ] - ), - access_token="test-token", - ) - try: - undated_quote_kis.quote("KR", "005930", venue="KRX") - except BlockedError as exc: - assert "each require a valid" in str(exc) - else: - raise AssertionError("KIS KR quote must not synthesize a trading date") - - single_timestamp_kis = KisBroker( - app_key="test-key", - app_secret="test-secret", - account_prefix="12345678", - account_product="01", - environment="live", - transport=QueueTransport( - [ - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "msg_cd": "0", - "msg1": "OK", - "output": { - "stck_prpr": "70000", - "stck_bsop_date": "20260727", - }, - }, - ), - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "msg_cd": "0", - "msg1": "OK", - "output1": { - "askp1": "70100", - "bidp1": "69900", - "stck_bsop_date": "20260727", - "aspr_acpt_hour": "093001", - }, - "output2": {"stck_prpr": "70000"}, - }, - ), - ] - ), - access_token="test-token", - ) - try: - single_timestamp_kis.quote("KR", "005930", venue="KRX") - except BlockedError as exc: - assert "independent broker-supplied" in str(exc) - else: - raise AssertionError("KIS KR price and book timestamps must be independent") - - us_cancel = kis.build_cancel_request( - market="US", - broker_order_id="12345", - symbol="AAPL", - quantity="1", - price="200.00", - venue="NASD", - ) - assert us_cancel["tr_id"] == "VTTT1004U" - assert us_cancel["body"]["PDNO"] == "AAPL" - assert us_cancel["body"]["OVRS_ORD_UNPR"] == "0" - assert us_cancel["body"]["ORD_SVR_DVSN_CD"] == "0" - - assert ( - kis_market_timestamp( - date_value="20260727", - time_value="093001", - timezone_name="America/New_York", - ) - == "2026-07-27T09:30:01-04:00" - ) - assert ( - kis_market_timestamp( - date_value="", - time_value="093001", - timezone_name="America/New_York", - ) - is None - ) - - for venue, expected_exchange_code, symbol in ( - ("NASD", "NAS", "AAPL"), - ("NYSE", "NYS", "IBM"), - ): - us_quote_transport = QueueTransport( - [ - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "msg_cd": "0", - "msg1": "OK", - "output1": { - "last": "200.25", - "pask1": "200.30", - "pbid1": "200.20", - "dymd": "20260727", - "dhms": "093001", - }, - }, - ) - ] - ) - us_quote_kis = KisBroker( - app_key="test-key", - app_secret="test-secret", - account_prefix="12345678", - account_product="01", - environment="live", - transport=us_quote_transport, - access_token="test-token", - ) - us_quote = us_quote_kis.quote("US", symbol, venue=venue) - assert us_quote["last_price"] == "200.25" - assert us_quote["trade_timestamp"] == us_quote["book_timestamp"] - assert us_quote["trade_timestamp"].endswith("T09:30:01-04:00") - assert len(us_quote_transport.requests) == 1 - assert us_quote_transport.requests[0]["query"]["EXCD"] == expected_exchange_code - assert us_quote_transport.requests[0]["url"].endswith( - "/uapi/overseas-price/v1/quotations/inquire-asking-price" - ) - - no_last_transport = QueueTransport( - [ - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "msg_cd": "0", - "msg1": "OK", - "output1": { - "pask1": "200.30", - "pbid1": "200.20", - "dymd": "20260727", - "dhms": "093001", - }, - }, - ), - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "msg_cd": "0", - "msg1": "OK", - "output": {"last": "199.00"}, - }, - ), - ] - ) - no_last_kis = KisBroker( - app_key="test-key", - app_secret="test-secret", - account_prefix="12345678", - account_product="01", - environment="live", - transport=no_last_transport, - access_token="test-token", - ) - try: - no_last_kis.quote("US", "AAPL", venue="NASD") - except BlockedError as exc: - assert "same-source last/bid/ask" in str(exc) - else: - raise AssertionError("KIS US quote must not borrow last from another endpoint") - assert len(no_last_transport.requests) == 1 - assert len(no_last_transport.responses) == 1 - - try: - kis.quote("US", "AAPL", venue="NASD") - except BlockedError as exc: - assert "paper order book" in str(exc) - else: - raise AssertionError("KIS U.S. paper quote must be blocked") - - ambiguous = QueueTransport([AmbiguousMutationError("accepted then timeout")]) - ambiguous_broker = KisBroker( - app_key="test-key", - app_secret="test-secret", - account_prefix="12345678", - account_product="01", - environment="paper", - transport=ambiguous, - access_token="test-token", - ) - try: - ambiguous_broker.submit(intent, venue="NASD") - except AmbiguousMutationError: - pass - else: - raise AssertionError("ambiguous KIS mutation did not raise") - - print( - canonical_json( - { - "self_test": "PASS", - "toss_request_hash": preview["request_hash"], - "kis_request_hash": kis_preview["request_hash"], - } - ) - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--self-test", action="store_true") - parser.add_argument("--capabilities", choices=("toss", "kis-paper", "kis-live")) - return parser.parse_args() - - -def main() -> int: - args = parse_args() - if args.self_test: - self_test() - return 0 - if args.capabilities == "toss": - print(canonical_json(TossBroker.capabilities())) - return 0 - if args.capabilities == "kis-paper": - print(canonical_json(KisBroker.capabilities("paper"))) - return 0 - if args.capabilities == "kis-live": - print(canonical_json(KisBroker.capabilities("live"))) - return 0 - print( - canonical_json( - { - "status": "BLOCKED", - "reason": "use --self-test or --capabilities", - } - ) - ) - return 2 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/broker_credentials.py b/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/broker_credentials.py deleted file mode 100644 index 4a7eef0..0000000 --- a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/broker_credentials.py +++ /dev/null @@ -1,469 +0,0 @@ -#!/usr/bin/env python3 -"""Load and configure KIS credentials without exposing their values.""" - -from __future__ import annotations - -import argparse -import getpass -import json -import os -import re -import stat -import sys -import tempfile -from datetime import datetime, timezone -from pathlib import Path -from typing import Any -from unittest.mock import patch - -from broker_adapters import KisBroker, TransportFailure -from execution_core import BlockedError, canonical_json - -STATUS_SCHEMA = "qta-broker-credential-status/v1" -DEFAULT_SECRETS_PATH = Path("~/.config/mrcha-skills/secrets.env").expanduser() -SECRETS_PATH_ENV = "QTA_SECRETS_FILE" -FIELD_SUFFIXES = { - "app_key": "APP_KEY", - "app_secret": "APP_SECRET", - "account_prefix": "ACCOUNT_PREFIX", - "account_product": "ACCOUNT_PRODUCT", -} -GENERIC_NAMES = { - field: f"QTA_KIS_{suffix}" for field, suffix in FIELD_SUFFIXES.items() -} -DOTENV_KEY = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") -_INJECTED_ENVIRONMENT: dict[str, str] = {} - - -def credential_scope(environment: str) -> str: - normalized = environment.strip().lower() - if normalized == "paper": - return "PAPER" - if normalized in {"shadow", "live"}: - return "LIVE" - raise BlockedError("KIS credential environment must be paper, shadow, or live") - - -def resolve_secrets_path(path: str | Path | None = None) -> Path: - candidate = path if path is not None else os.environ.get(SECRETS_PATH_ENV) - return Path(candidate).expanduser() if candidate else DEFAULT_SECRETS_PATH - - -def inspect_secrets_file(path: Path) -> dict[str, Any]: - try: - metadata = path.lstat() - except FileNotFoundError: - return { - "path": str(path), - "exists": False, - "secure": None, - "mode": None, - } - if stat.S_ISLNK(metadata.st_mode): - raise BlockedError(f"credential file must not be a symlink: {path}") - if not stat.S_ISREG(metadata.st_mode): - raise BlockedError(f"credential path must be a regular file: {path}") - if metadata.st_uid != os.getuid(): - raise BlockedError(f"credential file must be owned by the current user: {path}") - mode = stat.S_IMODE(metadata.st_mode) - if mode != 0o600: - raise BlockedError( - f"credential file permissions must be 0600, found {mode:04o}: {path}" - ) - return { - "path": str(path), - "exists": True, - "secure": True, - "mode": "0600", - } - - -def parse_dotenv_value(raw: str, *, line_number: int) -> str: - value = raw.strip() - if not value: - return "" - if value.startswith('"'): - try: - parsed = json.loads(value) - except json.JSONDecodeError as exc: - raise BlockedError( - f"invalid double-quoted dotenv value on line {line_number}" - ) from exc - if not isinstance(parsed, str): - raise BlockedError(f"dotenv value on line {line_number} must be a string") - return parsed - if value.startswith("'"): - if len(value) < 2 or not value.endswith("'"): - raise BlockedError( - f"invalid single-quoted dotenv value on line {line_number}" - ) - return value[1:-1].replace("\\'", "'").replace("\\\\", "\\") - return value - - -def parse_dotenv(path: Path) -> dict[str, str]: - inspection = inspect_secrets_file(path) - if not inspection["exists"]: - return {} - values: dict[str, str] = {} - for line_number, raw_line in enumerate( - path.read_text(encoding="utf-8").splitlines(), start=1 - ): - stripped = raw_line.strip() - if not stripped or stripped.startswith("#"): - continue - if stripped.startswith("export "): - stripped = stripped[7:].lstrip() - if "=" not in stripped: - raise BlockedError(f"invalid dotenv assignment on line {line_number}") - key, raw_value = stripped.split("=", 1) - key = key.strip() - if not DOTENV_KEY.fullmatch(key): - raise BlockedError(f"invalid dotenv key on line {line_number}") - if key in values: - raise BlockedError(f"duplicate dotenv key on line {line_number}: {key}") - values[key] = parse_dotenv_value(raw_value, line_number=line_number) - return values - - -def scoped_name(scope: str, field: str) -> str: - return f"QTA_KIS_{scope}_{FIELD_SUFFIXES[field]}" - - -def choose_value( - *, - generic_name: str, - scoped_variable: str, - dotenv_values: dict[str, str], -) -> tuple[str | None, str | None]: - candidates = ( - (os.environ.get(generic_name), f"environment:{generic_name}"), - (os.environ.get(scoped_variable), f"environment:{scoped_variable}"), - (dotenv_values.get(scoped_variable), f"dotenv:{scoped_variable}"), - (dotenv_values.get(generic_name), f"dotenv:{generic_name}"), - ) - for value, source in candidates: - if value: - return value, source - return None, None - - -def clear_injected_environment() -> None: - for name, injected_value in tuple(_INJECTED_ENVIRONMENT.items()): - if os.environ.get(name) == injected_value: - os.environ.pop(name, None) - _INJECTED_ENVIRONMENT.pop(name, None) - - -def load_kis_credentials( - environment: str, - *, - secrets_path: str | Path | None = None, -) -> dict[str, Any]: - clear_injected_environment() - scope = credential_scope(environment) - path = resolve_secrets_path(secrets_path) - inspection = inspect_secrets_file(path) - dotenv_values = parse_dotenv(path) if inspection["exists"] else {} - sources: dict[str, str] = {} - present: list[str] = [] - missing: list[str] = [] - - for field, generic_name in GENERIC_NAMES.items(): - value, source = choose_value( - generic_name=generic_name, - scoped_variable=scoped_name(scope, field), - dotenv_values=dotenv_values, - ) - if value is None or source is None: - missing.append(generic_name) - continue - os.environ[generic_name] = value - if source != f"environment:{generic_name}": - _INJECTED_ENVIRONMENT[generic_name] = value - present.append(generic_name) - sources[generic_name] = source - - return { - "schema": STATUS_SCHEMA, - "broker": "kis", - "credential_scope": scope.lower(), - "status": "READY" if not missing else "BLOCKED", - "secrets_file": inspection, - "present": sorted(present), - "missing": sorted(missing), - "sources": dict(sorted(sources.items())), - } - - -def require_kis_runtime_credentials(report: dict[str, Any]) -> None: - if report["missing"]: - raise BlockedError( - "missing KIS runtime credentials: " + ", ".join(report["missing"]) - ) - - -def dotenv_quote(value: str) -> str: - return json.dumps(value, ensure_ascii=False) - - -def assignment_key(raw_line: str) -> str | None: - stripped = raw_line.strip() - if not stripped or stripped.startswith("#"): - return None - if stripped.startswith("export "): - stripped = stripped[7:].lstrip() - if "=" not in stripped: - return None - key = stripped.split("=", 1)[0].strip() - return key if DOTENV_KEY.fullmatch(key) else None - - -def write_dotenv_updates(path: Path, updates: dict[str, str]) -> None: - inspection = inspect_secrets_file(path) - existing_lines = ( - path.read_text(encoding="utf-8").splitlines() if inspection["exists"] else [] - ) - output_lines: list[str] = [] - replaced: set[str] = set() - for raw_line in existing_lines: - key = assignment_key(raw_line) - if key not in updates: - output_lines.append(raw_line) - continue - if key not in replaced: - output_lines.append(f"{key}={dotenv_quote(updates[key])}") - replaced.add(key) - if output_lines and output_lines[-1] != "": - output_lines.append("") - for key in updates: - if key not in replaced: - output_lines.append(f"{key}={dotenv_quote(updates[key])}") - - parent_existed = path.parent.exists() - path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) - if not parent_existed: - path.parent.chmod(0o700) - file_descriptor, temporary_name = tempfile.mkstemp( - prefix=".secrets-", dir=path.parent - ) - temporary_path = Path(temporary_name) - try: - os.fchmod(file_descriptor, 0o600) - with os.fdopen(file_descriptor, "w", encoding="utf-8") as handle: - handle.write("\n".join(output_lines) + "\n") - handle.flush() - os.fsync(handle.fileno()) - os.replace(temporary_path, path) - path.chmod(0o600) - except BaseException: - temporary_path.unlink(missing_ok=True) - raise - - -def prompt_secret(label: str, *, existing: str | None = None) -> str: - suffix = " (Enter keeps the stored value)" if existing else "" - value = getpass.getpass(f"{label}{suffix}: ") - if value: - return value - if existing: - return existing - raise BlockedError(f"{label} is required") - - -def configure_kis(environment: str, path: Path) -> dict[str, Any]: - scope = credential_scope(environment) - inspection = inspect_secrets_file(path) - existing = parse_dotenv(path) if inspection["exists"] else {} - updates: dict[str, str] = {} - labels = { - "app_key": "KIS app key", - "app_secret": "KIS app secret", - "account_prefix": "KIS account number (first 8 digits)", - } - for field in ("app_key", "app_secret", "account_prefix"): - name = scoped_name(scope, field) - updates[name] = prompt_secret(labels[field], existing=existing.get(name)) - - product_name = scoped_name(scope, "account_product") - stored_product = existing.get(product_name) - product = getpass.getpass( - "KIS account product (2 digits" - + (", Enter keeps the stored value" if stored_product else ", Enter uses 01") - + "): " - ) - updates[product_name] = product or stored_product or "01" - if len(updates[scoped_name(scope, "account_prefix")]) != 8 or not updates[ - scoped_name(scope, "account_prefix") - ].isdigit(): - raise BlockedError("KIS account number prefix must be 8 digits") - if len(updates[product_name]) != 2 or not updates[product_name].isdigit(): - raise BlockedError("KIS account product must be 2 digits") - write_dotenv_updates(path, updates) - report = load_kis_credentials(environment, secrets_path=path) - require_kis_runtime_credentials(report) - return { - **report, - "configured": True, - "values_printed": False, - } - - -def auth_check(environment: str, path: Path) -> dict[str, Any]: - report = load_kis_credentials(environment, secrets_path=path) - require_kis_runtime_credentials(report) - broker = KisBroker( - app_key=os.environ["QTA_KIS_APP_KEY"], - app_secret=os.environ["QTA_KIS_APP_SECRET"], - account_prefix=os.environ["QTA_KIS_ACCOUNT_PREFIX"], - account_product=os.environ["QTA_KIS_ACCOUNT_PRODUCT"], - environment="paper" if credential_scope(environment) == "PAPER" else "live", - access_token=None, - ) - token = broker.token() - if not token: - raise BlockedError("KIS authentication returned an empty access token") - return { - "schema": STATUS_SCHEMA, - "broker": "kis", - "credential_scope": report["credential_scope"], - "status": "AUTHENTICATED", - "checked_at": datetime.now(timezone.utc).isoformat(), - "token_received": True, - "token_persisted": False, - "secrets_file": report["secrets_file"], - "sources": report["sources"], - } - - -def self_test() -> None: - with tempfile.TemporaryDirectory(prefix="qta-credentials-") as directory: - path = Path(directory) / "secrets.env" - path.write_text( - "\n".join( - ( - 'UNRELATED_VALUE="preserved"', - 'QTA_KIS_PAPER_APP_KEY="fixture-app"', - 'QTA_KIS_PAPER_APP_SECRET="fixture-secret"', - 'QTA_KIS_PAPER_ACCOUNT_PREFIX="00000000"', - 'QTA_KIS_PAPER_ACCOUNT_PRODUCT="01"', - 'QTA_KIS_LIVE_APP_KEY="fixture-live-app"', - 'QTA_KIS_LIVE_APP_SECRET="fixture-live-secret"', - 'QTA_KIS_LIVE_ACCOUNT_PREFIX="11111111"', - 'QTA_KIS_LIVE_ACCOUNT_PRODUCT="01"', - "", - ) - ), - encoding="utf-8", - ) - path.chmod(0o600) - with patch.dict(os.environ, {}, clear=True): - report = load_kis_credentials("paper", secrets_path=path) - assert report["status"] == "READY" - assert os.environ["QTA_KIS_APP_KEY"] == "fixture-app" - rendered = canonical_json(report) - assert "fixture-app" not in rendered - assert "fixture-secret" not in rendered - live_report = load_kis_credentials("live", secrets_path=path) - assert live_report["status"] == "READY" - assert os.environ["QTA_KIS_APP_KEY"] == "fixture-live-app" - assert os.environ["QTA_KIS_ACCOUNT_PREFIX"] == "11111111" - with patch.dict( - os.environ, - {"QTA_KIS_APP_KEY": "environment-app"}, - clear=True, - ): - report = load_kis_credentials("paper", secrets_path=path) - assert report["sources"]["QTA_KIS_APP_KEY"].startswith("environment:") - assert os.environ["QTA_KIS_APP_KEY"] == "environment-app" - write_dotenv_updates( - path, - {"QTA_KIS_PAPER_ACCOUNT_PRODUCT": "02"}, - ) - assert parse_dotenv(path)["UNRELATED_VALUE"] == "preserved" - assert parse_dotenv(path)["QTA_KIS_PAPER_ACCOUNT_PRODUCT"] == "02" - path.chmod(0o644) - try: - load_kis_credentials("paper", secrets_path=path) - except BlockedError as exc: - assert "0600" in str(exc) - else: - raise AssertionError("insecure credential file permissions must be blocked") - - configured_path = Path(directory) / "configured.env" - with ( - patch.dict(os.environ, {}, clear=True), - patch.object( - getpass, - "getpass", - side_effect=("new-app", "new-secret", "22222222", "01"), - ), - ): - configured = configure_kis("paper", configured_path) - assert configured["status"] == "READY" - assert configured_path.stat().st_mode & 0o777 == 0o600 - configured_rendered = canonical_json(configured) - assert "new-app" not in configured_rendered - assert "new-secret" not in configured_rendered - with patch.object(KisBroker, "token", return_value="sensitive-token"): - authenticated = auth_check("paper", configured_path) - authenticated_rendered = canonical_json(authenticated) - assert authenticated["status"] == "AUTHENTICATED" - assert "sensitive-token" not in authenticated_rendered - print(canonical_json({"self_test": "PASS", "secrets_exposed": False})) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - subparsers = parser.add_subparsers(dest="command") - for command in ("status", "configure", "auth-check"): - subparser = subparsers.add_parser(command) - subparser.add_argument( - "--environment", - required=True, - choices=("paper", "shadow", "live"), - ) - subparser.add_argument("--secrets-file") - subparsers.add_parser("self-test") - return parser.parse_args() - - -def main() -> int: - args = parse_args() - if args.command == "self-test": - self_test() - return 0 - if args.command not in {"status", "configure", "auth-check"}: - print( - canonical_json( - { - "schema": STATUS_SCHEMA, - "status": "BLOCKED", - "reason": "choose status, configure, auth-check, or self-test", - } - ) - ) - return 2 - path = resolve_secrets_path(args.secrets_file) - try: - if args.command == "configure": - output = configure_kis(args.environment, path) - elif args.command == "auth-check": - output = auth_check(args.environment, path) - else: - output = load_kis_credentials(args.environment, secrets_path=path) - except (BlockedError, OSError, TransportFailure, ValueError) as exc: - output = { - "schema": STATUS_SCHEMA, - "broker": "kis", - "status": "BLOCKED", - "reason": str(exc), - } - print(canonical_json(output)) - return 2 - print(canonical_json(output)) - return 0 if output["status"] in {"READY", "AUTHENTICATED"} else 2 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/daily_pipeline.py b/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/daily_pipeline.py deleted file mode 100644 index 3794470..0000000 --- a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/daily_pipeline.py +++ /dev/null @@ -1,1565 +0,0 @@ -#!/usr/bin/env python3 -"""Run the deterministic daily pre-open, snapshot, and shadow-entry pipeline.""" - -from __future__ import annotations - -import argparse -import json -import os -import shutil -import sqlite3 -import stat -import subprocess -import sys -import tempfile -import time -from datetime import date, datetime, timedelta, timezone -from decimal import Decimal -from pathlib import Path -from typing import Any -from zoneinfo import ZoneInfo - -import market_calendar -from execution_core import ( - ENTRY_WINDOWS, - TERMINAL_STATES, - BlockedError, - normalized_execution_policy, - normalized_risk_policy, - sha256_file, -) - -CONFIG_SCHEMA = "qta-daily-shadow-config/v2" -STATUS_SCHEMA = "qta-daily-shadow-status/v2" -DESCRIPTOR_SCHEMA = "qta-daily-shadow-descriptor/v2" -REQUIRED_CONFIG_FIELDS = { - "schema", - "runtime_root", - "broker", - "mode", - "history_start_date", - "minimum_sessions", - "request_interval_ms", - "catalog_coverage_contract", - "selector", - "risk_policy_path", - "execution_policy", - "account_aliases", - "prepare_complete_seconds_before_open", -} -OPTIONAL_CONFIG_FIELDS = { - "exposure_component_paths", - "exposure_component_max_age_seconds", - "qta_method_version", -} -EXECUTION_DEFAULT_FIELDS = { - "snapshot_max_age_seconds", - "poll_interval_seconds", - "quote_max_age_seconds", - "max_spread_bps", - "max_gap_bps", - "trigger_mode", - "order_ttl_seconds", - "order_type", - "time_in_force", - "allow_partial_fill", - "cancel_remainder_at_window_end", -} -TERMINAL_OR_MANUAL = TERMINAL_STATES -RECONCILE_ATTEMPTS = 20 -RECONCILE_INTERVAL_SECONDS = 15 - - -class PipelineBlockedError(BlockedError): - """Raised when a daily gate cannot be completed without guessing.""" - - -def atomic_write(path: Path, payload: bytes, mode: int = 0o600) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - descriptor, temporary_name = tempfile.mkstemp( - prefix=f".{path.name}.", suffix=".tmp", dir=str(path.parent) - ) - temporary = Path(temporary_name) - try: - with os.fdopen(descriptor, "wb") as handle: - handle.write(payload) - handle.flush() - os.fsync(handle.fileno()) - os.replace(temporary, path) - os.chmod(path, mode) - finally: - if temporary.exists(): - temporary.unlink() - - -def atomic_write_json(path: Path, value: Any) -> None: - atomic_write( - path, - (json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode( - "utf-8" - ), - ) - - -def load_json(path: Path, label: str) -> dict[str, Any]: - try: - value = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as exc: - raise PipelineBlockedError(f"{label} must be readable JSON: {path}") from exc - if not isinstance(value, dict): - raise PipelineBlockedError(f"{label} must be one JSON object") - return value - - -def exact_fields(value: dict[str, Any], expected: set[str], label: str) -> None: - if set(value) != expected: - raise PipelineBlockedError( - f"{label} fields mismatch; missing={sorted(expected - set(value))}, " - f"extra={sorted(set(value) - expected)}" - ) - - -def absolute_path(value: Any, label: str) -> Path: - path = Path(str(value)).expanduser() - if not path.is_absolute(): - raise PipelineBlockedError(f"{label} must be absolute") - if any(character in str(path) for character in ("\n", "\r", "\0")): - raise PipelineBlockedError(f"{label} contains an invalid character") - return path - - -def regular_nonsymlink(path: Path, label: str) -> None: - try: - metadata = path.lstat() - except FileNotFoundError as exc: - raise PipelineBlockedError(f"{label} does not exist: {path}") from exc - if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode): - raise PipelineBlockedError(f"{label} must be a regular non-symlink file") - - -def validate_sha256(value: Any, label: str) -> str: - text = str(value) - if len(text) != 64 or any(character not in "0123456789abcdef" for character in text): - raise PipelineBlockedError(f"{label} must be a lowercase SHA-256") - return text - - -def parse_iso_date(value: Any, label: str) -> date: - if not isinstance(value, str): - raise PipelineBlockedError(f"{label} must be YYYY-MM-DD") - try: - parsed = date.fromisoformat(value) - except ValueError as exc: - raise PipelineBlockedError(f"{label} must be YYYY-MM-DD") from exc - if parsed.isoformat() != value: - raise PipelineBlockedError(f"{label} must be YYYY-MM-DD") - return parsed - - -def normalize_config(raw: Any, path: Path) -> dict[str, Any]: - if not isinstance(raw, dict): - raise PipelineBlockedError("daily config must be one JSON object") - missing = REQUIRED_CONFIG_FIELDS - set(raw) - extra = set(raw) - REQUIRED_CONFIG_FIELDS - OPTIONAL_CONFIG_FIELDS - if missing or extra: - raise PipelineBlockedError( - "daily config fields mismatch; " - f"missing={sorted(missing)}, extra={sorted(extra)}" - ) - if raw["schema"] != CONFIG_SCHEMA: - raise PipelineBlockedError(f"daily config.schema must be {CONFIG_SCHEMA}") - runtime = absolute_path(raw["runtime_root"], "runtime_root") - if runtime.exists() and (runtime.is_symlink() or not runtime.is_dir()): - raise PipelineBlockedError("runtime_root must be a non-symlink directory") - if raw["broker"] != "kis-live" or raw["mode"] != "shadow": - raise PipelineBlockedError("daily pipeline requires kis-live/shadow") - history_start = parse_iso_date(raw["history_start_date"], "history_start_date") - minimum_sessions = raw["minimum_sessions"] - if ( - isinstance(minimum_sessions, bool) - or not isinstance(minimum_sessions, int) - or minimum_sessions < 756 - ): - raise PipelineBlockedError("minimum_sessions must be an integer >= 756") - interval = raw["request_interval_ms"] - if isinstance(interval, bool) or not isinstance(interval, int) or interval < 100: - raise PipelineBlockedError("request_interval_ms must be an integer >= 100") - coverage = raw["catalog_coverage_contract"] - if not isinstance(coverage, dict): - raise PipelineBlockedError("catalog_coverage_contract must be an object") - selector = raw["selector"] - if not isinstance(selector, dict) or selector.get("selector_version") != ( - "qta-screen-1.1.0" - ): - raise PipelineBlockedError("selector must use qta-screen-1.1.0") - risk_path = absolute_path(raw["risk_policy_path"], "risk_policy_path") - regular_nonsymlink(risk_path, "risk_policy_path") - risk = normalized_risk_policy(load_json(risk_path, "risk policy")) - for field in ( - "allow_existing_additions", - "allow_borrowed_cash", - "allow_margin", - "allow_short", - "allow_auto_fx", - ): - if risk[field] is not False: - raise PipelineBlockedError(f"risk policy {field} must be false") - if risk["whole_shares_only"] is not True: - raise PipelineBlockedError("risk policy whole_shares_only must be true") - execution = raw["execution_policy"] - if not isinstance(execution, dict): - raise PipelineBlockedError("execution_policy must be an object") - exact_fields(execution, EXECUTION_DEFAULT_FIELDS, "execution_policy") - qta_method_version = raw.get("qta_method_version", "qta-1.0.0") - if qta_method_version not in {"qta-1.0.0", "qta-2.0.0"}: - raise PipelineBlockedError( - "qta_method_version must be qta-1.0.0 or qta-2.0.0" - ) - component_paths = raw.get("exposure_component_paths", []) - if not isinstance(component_paths, list): - raise PipelineBlockedError("exposure_component_paths must be an array") - normalized_component_paths: list[str] = [] - for index, item in enumerate(component_paths): - if not isinstance(item, str) or not item: - raise PipelineBlockedError( - f"exposure_component_paths[{index}] must be a string" - ) - text = item - substituted = text.replace("{session_date}", "2000-01-01") - absolute_path( - substituted, - f"exposure_component_paths[{index}]", - ) - if text.count("{session_date}") > 1 or "{" in text.replace( - "{session_date}", "" - ): - raise PipelineBlockedError( - f"exposure_component_paths[{index}] has an unknown template" - ) - normalized_component_paths.append(text) - max_age = raw.get("exposure_component_max_age_seconds", 86400) - if ( - isinstance(max_age, bool) - or not isinstance(max_age, int) - or max_age <= 0 - or max_age > 86400 - ): - raise PipelineBlockedError( - "exposure_component_max_age_seconds must be 1..86400" - ) - aliases = raw["account_aliases"] - if not isinstance(aliases, dict) or set(aliases) != {"KR", "US"}: - raise PipelineBlockedError("account_aliases must contain exactly KR and US") - if any( - not isinstance(value, str) - or not value.strip() - or value != value.strip() - or any(character not in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_" for character in value) - for value in aliases.values() - ): - raise PipelineBlockedError( - "account aliases must use only letters, digits, hyphen, and underscore" - ) - deadline = raw["prepare_complete_seconds_before_open"] - if ( - isinstance(deadline, bool) - or not isinstance(deadline, int) - or deadline < 600 - or deadline > 7200 - ): - raise PipelineBlockedError( - "prepare_complete_seconds_before_open must be 600..7200" - ) - return { - **raw, - "runtime_root": runtime.resolve(strict=False), - "history_start_date": history_start.isoformat(), - "risk_policy_path": risk_path.resolve(), - "risk_policy": risk, - "exposure_component_paths": normalized_component_paths, - "exposure_component_max_age_seconds": max_age, - "qta_method_version": qta_method_version, - "config_path": path.resolve(), - } - - -def load_config(path: Path) -> dict[str, Any]: - regular_nonsymlink(path, "daily config") - return normalize_config(load_json(path, "daily config"), path) - - -def subprocess_environment() -> dict[str, str]: - environment = dict(os.environ) - environment["PYTHONDONTWRITEBYTECODE"] = "1" - environment.pop("PYTHONPATH", None) - environment.pop("PYTHONHOME", None) - return environment - - -def run_command(command: list[str], label: str) -> None: - result = subprocess.run( - command, - check=False, - capture_output=True, - text=True, - env=subprocess_environment(), - ) - if result.returncode != 0: - reason = result.stderr.strip() or result.stdout.strip() or ( - f"exit status {result.returncode}" - ) - if len(reason) > 2000: - reason = reason[-2000:] - raise PipelineBlockedError(f"{label} failed: {reason}") - - -def python_command(script: Path, *arguments: str) -> list[str]: - regular_nonsymlink(script, "pipeline script") - return [sys.executable, "-B", "-s", str(script), *arguments] - - -def workflow_directory(config: dict[str, Any], market: str, session: str) -> Path: - return config["runtime_root"] / "workflows" / market.lower() / session - - -def current_descriptor_path(config: dict[str, Any], market: str) -> Path: - return config["runtime_root"] / "current" / f"{market.lower()}.json" - - -def stage_status( - *, - stage: str, - status: str, - market: str, - session_date: str, - reason: str = "", - details: dict[str, Any] | None = None, -) -> dict[str, Any]: - return { - "schema": STATUS_SCHEMA, - "stage": stage, - "status": status, - "market": market, - "session_date": session_date, - "reason": reason, - "details": details or {}, - "recorded_at": datetime.now(timezone.utc).isoformat(), - "live_enabled": False, - "api_mutation_count": 0, - } - - -def current_market_date(market: str) -> str: - return datetime.now( - ZoneInfo(ENTRY_WINDOWS[market]["timezone"]) - ).date().isoformat() - - -def read_source_snapshot(path: Path, as_of: str, label: str) -> dict[str, Any]: - receipt = load_json(path, label) - if receipt.get("as_of") != as_of: - raise PipelineBlockedError(f"{label} as_of mismatch") - if not isinstance(receipt.get("official_sources"), list) or not isinstance( - receipt.get("broker_sources"), list - ): - raise PipelineBlockedError(f"{label} source arrays are missing") - return receipt - - -def ensure_source_snapshot( - *, - market: str, - as_of: str, - directory: Path, - technical_root: Path, -) -> dict[str, Any]: - receipt_path = directory / "source-snapshot.json" - if receipt_path.is_file() and not receipt_path.is_symlink(): - return read_source_snapshot(receipt_path, as_of, f"{market} source snapshot") - script = technical_root / "scripts" / ( - "fetch_kis_kr_eod.py" if market == "KR" else "fetch_kis_us_eod.py" - ) - run_command( - python_command( - script, - "snapshot-sources", - "--as-of", - as_of, - "--output-directory", - str(directory), - ), - f"{market} official source snapshot", - ) - return read_source_snapshot(receipt_path, as_of, f"{market} source snapshot") - - -def copy_cache_tree(source: Path, target: Path) -> None: - if not source.is_dir() or source.is_symlink() or target.exists(): - return - - def link_or_copy(from_path: str, to_path: str) -> str: - try: - os.link(from_path, to_path) - return to_path - except OSError: - return shutil.copy2(from_path, to_path) - - shutil.copytree(source, target, copy_function=link_or_copy) - - -def seed_eod_cache( - runtime_root: Path, - target_root: Path, - analysis_date: str, - component: str, -) -> None: - if target_root.exists() and any(target_root.iterdir()): - return - candidates: list[tuple[date, Path]] = [] - eod_root = runtime_root / "eod" - if not eod_root.is_dir(): - return - target_date = date.fromisoformat(analysis_date) - for catalog in eod_root.rglob("eod-catalog.csv"): - if catalog.parent.name != component: - continue - try: - relative_parts = catalog.relative_to(eod_root).parts - candidate_date = date.fromisoformat(relative_parts[0]) - except (ValueError, IndexError): - continue - candidate_root = catalog.parent - if candidate_date > target_date or candidate_root.resolve() == target_root.resolve(): - continue - receipt_path = candidate_root / "eod-bundle-receipt.json" - if ( - receipt_path.is_symlink() - or not receipt_path.is_file() - or catalog.is_symlink() - ): - continue - try: - receipt = json.loads(receipt_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - continue - if ( - not isinstance(receipt, dict) - or receipt.get("status") != "READY" - or receipt.get("analysis_date") != candidate_date.isoformat() - or receipt.get("api_mutation_count") != 0 - ): - continue - candidates.append((candidate_date, candidate_root)) - if not candidates: - return - source_root = max(candidates, key=lambda item: (item[0], str(item[1])))[1] - for name in ("stocks", "benchmarks"): - copy_cache_tree(source_root / name, target_root / name) - - -def ensure_eod( - *, - config: dict[str, Any], - technical_root: Path, - workflow: Path, - as_of: str, - analysis_date: str, - market: str, - sources: dict[str, Any], - cache_namespace: str, -) -> Path: - eod_base = config["runtime_root"] / "eod" / analysis_date / cache_namespace - component = market.lower() - output = eod_base / component - receipt_path = output / "eod-bundle-receipt.json" - jobs_directory = workflow / "jobs" - jobs_directory.mkdir(parents=True, exist_ok=True) - - if not receipt_path.is_file(): - seed_eod_cache(config["runtime_root"], output, analysis_date, component) - job = { - "schema": ( - "qta-kis-kr-eod-job/v1" - if market == "KR" - else "qta-kis-us-eod-job/v1" - ), - "as_of": as_of, - "analysis_date": analysis_date, - "environment": "live", - "output_directory": str(output.resolve()), - "history_start_date": config["history_start_date"], - "minimum_sessions": config["minimum_sessions"], - "request_interval_ms": config["request_interval_ms"], - "official_sources": sources["official_sources"], - "broker_sources": sources["broker_sources"], - "catalog_coverage_contract": config["catalog_coverage_contract"], - "base_eod_catalog": "", - } - job_path = jobs_directory / f"{component}-eod.json" - atomic_write_json(job_path, job) - script = technical_root / "scripts" / ( - "fetch_kis_kr_eod.py" - if market == "KR" - else "fetch_kis_us_eod.py" - ) - run_command( - python_command( - script, - "collect", - "--job", - str(job_path), - ), - f"KIS {market} adjusted EOD", - ) - receipt = load_json(receipt_path, f"{market} EOD receipt") - if ( - receipt.get("status") != "READY" - or receipt.get("analysis_date") != analysis_date - or receipt.get("api_mutation_count") != 0 - ): - raise PipelineBlockedError(f"{market} EOD receipt is not READY and read-only") - return output - - -def derive_universe_build_spec( - *, - config: dict[str, Any], - eod_output: Path, - workflow: Path, - market: str, -) -> Path: - if market not in {"KR", "US"}: - raise PipelineBlockedError("market must be KR or US") - receipt_path = eod_output / "eod-bundle-receipt.json" - regular_nonsymlink(receipt_path, "EOD receipt") - receipt = load_json(receipt_path, "EOD receipt") - binding = receipt.get("build_spec") - if not isinstance(binding, dict): - raise PipelineBlockedError("EOD receipt build_spec must be an object") - exact_fields(binding, {"path", "sha256"}, "EOD receipt build_spec") - source_path = absolute_path(binding["path"], "EOD build spec path") - expected_path = (eod_output / "universe-build-spec.json").resolve() - regular_nonsymlink(source_path, "EOD build spec") - if source_path.resolve() != expected_path: - raise PipelineBlockedError( - "EOD receipt build_spec path does not match its bundle" - ) - expected_sha256 = validate_sha256( - binding["sha256"], - "EOD receipt build_spec.sha256", - ) - if sha256_file(source_path) != expected_sha256: - raise PipelineBlockedError("EOD build spec hash mismatch") - derived = load_json(source_path, "EOD build spec") - expected_exchanges = ( - {"KOSPI", "KOSDAQ"} if market == "KR" else {"NYSE", "NASDAQ"} - ) - for field in ("official_sources", "broker_sources"): - sources = derived.get(field) - if not isinstance(sources, list) or any( - not isinstance(item, dict) or not isinstance(item.get("exchange"), str) - for item in sources - ): - raise PipelineBlockedError(f"EOD build spec {field} must be an array") - exchanges = {item["exchange"] for item in sources} - if len(sources) != len(exchanges) or exchanges != expected_exchanges: - raise PipelineBlockedError( - f"EOD build spec {field} must contain exactly " - f"{sorted(expected_exchanges)} for {market}" - ) - derived["catalog_coverage_contract"] = config["catalog_coverage_contract"] - output_path = workflow / "universe-build-spec.json" - atomic_write_json(output_path, derived) - return output_path - - -def prepare( - config: dict[str, Any], - market: str, - technical_root: Path, - trader_root: Path, -) -> dict[str, Any]: - session = current_market_date(market) - workflow = workflow_directory(config, market, session) - workflow.mkdir(parents=True, exist_ok=True) - current_path = current_descriptor_path(config, market) - if current_path.is_file() and not current_path.is_symlink(): - current = load_json(current_path, "current daily descriptor") - if ( - current.get("schema") == DESCRIPTOR_SCHEMA - and current.get("market") == market - and current.get("session_date") == session - ): - current_status = current.get("status") - if current_status in {"BLOCKED", "MANUAL_BLOCK"}: - raise PipelineBlockedError( - str( - current.get("blocked_reason") - or "current market session is terminally blocked" - ) - ) - if current_status in { - "MARKET_CLOSED", - "PREPARED", - "PLANNED", - "READY", - }: - return load_json( - workflow / "prepare-status.json", - "existing prepare status", - ) - calendar = market_calendar.snapshot( - market=market, - session_date=session, - output_directory=workflow / "calendar", - ) - if calendar["status"] == "MARKET_CLOSED": - receipt = stage_status( - stage="prepare", - status="MARKET_CLOSED", - market=market, - session_date=session, - details={"calendar_receipt": calendar}, - ) - descriptor = { - "schema": DESCRIPTOR_SCHEMA, - "status": "MARKET_CLOSED", - "market": market, - "session_date": session, - "workflow_directory": str(workflow.resolve()), - "prepare_status": str((workflow / "prepare-status.json").resolve()), - "live_enabled": False, - "api_mutation_count": 0, - } - atomic_write_json(workflow / "prepare-status.json", receipt) - atomic_write_json(current_path, descriptor) - return receipt - if calendar["status"] != "READY": - raise PipelineBlockedError("calendar snapshot is neither READY nor closed") - market_session_path = Path(calendar["market_session_path"]) - market_session = load_json(market_session_path, "market session") - regular_open = datetime.fromisoformat(market_session["regular_open"]) - deadline = regular_open - timedelta( - seconds=config["prepare_complete_seconds_before_open"] - ) - if datetime.now(timezone.utc) >= deadline.astimezone(timezone.utc): - raise PipelineBlockedError( - "prepare started too late to complete before the account snapshot gate" - ) - analysis_date = market_session["previous_session_date"] - if parse_iso_date(config["history_start_date"], "history_start_date") >= ( - parse_iso_date(analysis_date, "analysis_date") - ): - raise PipelineBlockedError("history_start_date must precede analysis_date") - source_root = config["runtime_root"] / "sources" / session - sources = ensure_source_snapshot( - market=market, - as_of=session, - directory=source_root / market.lower(), - technical_root=technical_root, - ) - eod_output = ensure_eod( - config=config, - technical_root=technical_root, - workflow=workflow, - as_of=session, - analysis_date=analysis_date, - market=market, - sources=sources, - cache_namespace="official-market-v1", - ) - if datetime.now(timezone.utc) >= deadline.astimezone(timezone.utc): - raise PipelineBlockedError( - "EOD preparation finished after the account snapshot deadline" - ) - build_spec = derive_universe_build_spec( - config=config, - eod_output=eod_output, - workflow=workflow, - market=market, - ) - manifest_path = workflow / "universe-manifest.json" - run_command( - python_command( - technical_root / "scripts" / "build_universe_manifest.py", - "--build-spec", - str(build_spec), - "--output", - str(manifest_path), - ), - f"{market} universe manifest", - ) - manifest = load_json(manifest_path, "universe manifest") - if manifest.get("build_status") != "READY": - raise PipelineBlockedError("universe manifest build_status is not READY") - selector_path = workflow / "selector.json" - atomic_write_json(selector_path, config["selector"]) - screen_path = workflow / "screen.json" - run_command( - python_command( - technical_root / "scripts" / "screen_universe.py", - "--manifest", - str(manifest_path), - "--selector", - str(selector_path), - "--output", - str(screen_path), - "--method-version", - config["qta_method_version"], - ), - f"{market} screen", - ) - screen = load_json(screen_path, "screen") - if screen.get("screen_status") != "READY": - raise PipelineBlockedError("qta-screen/v2 is not READY") - receipt = stage_status( - stage="prepare", - status="READY", - market=market, - session_date=session, - details={ - "analysis_date": analysis_date, - "manifest_hash": manifest["manifest_hash"], - "screen_hash": screen["screen_hash"], - "calendar_session_hash": market_session["session_hash"], - }, - ) - descriptor = { - "schema": DESCRIPTOR_SCHEMA, - "status": "PREPARED", - "market": market, - "session_date": session, - "analysis_date": analysis_date, - "workflow_directory": str(workflow.resolve()), - "market_session_path": str(market_session_path.resolve()), - "manifest_path": str(manifest_path.resolve()), - "screen_path": str(screen_path.resolve()), - "manifest_hash": manifest["manifest_hash"], - "screen_hash": screen["screen_hash"], - "live_enabled": False, - "api_mutation_count": 0, - } - atomic_write_json(workflow / "prepare-status.json", receipt) - atomic_write_json(current_path, descriptor) - return receipt - - -def resolve_exposure_components( - config: dict[str, Any], session_date: str -) -> list[Path]: - paths: list[Path] = [] - for value in config["exposure_component_paths"]: - path = Path(value.replace("{session_date}", session_date)) - regular_nonsymlink(path, "configured exposure component") - paths.append(path.resolve()) - brokers: set[str] = set() - for path in paths: - component = load_json(path, "configured exposure component") - broker = component.get("broker") - if broker not in {"toss", "nh"}: - raise PipelineBlockedError( - f"configured exposure component broker is invalid: {path}" - ) - if broker in brokers: - raise PipelineBlockedError( - f"configured exposure broker appears more than once: {broker}" - ) - brokers.add(broker) - return paths - - -def read_ledger_states(ledger_path: Path) -> list[dict[str, str]]: - if not ledger_path.is_file() or ledger_path.is_symlink(): - return [] - uri = f"file:{ledger_path.as_posix()}?mode=ro" - try: - connection = sqlite3.connect(uri, uri=True) - rows = connection.execute( - "SELECT intent_id, plan_hash, state FROM intents ORDER BY intent_id" - ).fetchall() - except sqlite3.Error as exc: - raise PipelineBlockedError(f"cannot inspect prior ledger: {ledger_path}") from exc - finally: - if "connection" in locals(): - connection.close() - return [ - {"intent_id": str(row[0]), "plan_hash": str(row[1]), "state": str(row[2])} - for row in rows - ] - - -def state_market_directory( - runtime_root: Path, account_alias: str, market: str -) -> Path: - account_root = runtime_root / "state" / account_alias - if account_root.exists() and ( - account_root.is_symlink() or not account_root.is_dir() - ): - raise PipelineBlockedError( - "account state root must be a non-symlink directory" - ) - market_root = account_root / market.lower() - if market_root.exists() and ( - market_root.is_symlink() or not market_root.is_dir() - ): - raise PipelineBlockedError( - "market state root must be a non-symlink directory" - ) - return market_root - - -def unresolved_state_directories( - market_root: Path, - *, - exclude_date: str | None = None, -) -> list[tuple[Path, list[dict[str, str]]]]: - if not market_root.exists(): - return [] - if market_root.is_symlink() or not market_root.is_dir(): - raise PipelineBlockedError("market state root must be a non-symlink directory") - output: list[tuple[Path, list[dict[str, str]]]] = [] - for directory in sorted(market_root.iterdir(), key=lambda item: item.name): - if ( - not directory.is_dir() - or directory.is_symlink() - or (exclude_date is not None and directory.name == exclude_date) - ): - continue - states = read_ledger_states(directory / "ledger.sqlite3") - unresolved = [ - item - for item in states - if item["state"] not in TERMINAL_OR_MANUAL - or item["state"] == "MANUAL_BLOCK" - ] - if unresolved: - output.append((directory, unresolved)) - return output - - -def prior_descriptor_for_state( - config: dict[str, Any], market: str, state_directory: Path -) -> dict[str, Any]: - session = state_directory.name - descriptor_path = ( - config["runtime_root"] - / "workflows" - / market.lower() - / session - / "descriptor.json" - ) - return load_json(descriptor_path, "prior workflow descriptor") - - -def reconcile_one( - *, - trader_root: Path, - descriptor: dict[str, Any], - output_path: Path, -) -> None: - command = python_command( - trader_root / "scripts" / "reconcile.py", - "--plan", - descriptor["plan_path"], - "--broker", - "kis-live", - "--state-dir", - descriptor["state_directory"], - "--output", - str(output_path), - ) - result = subprocess.run( - command, - check=False, - capture_output=True, - text=True, - env=subprocess_environment(), - ) - if result.returncode not in {0, 2}: - raise PipelineBlockedError( - "reconciliation process failed outside its receipt contract" - ) - - -def mark_manual_block(state_directory: Path, reason: str) -> None: - ledger_path = state_directory / "ledger.sqlite3" - states = read_ledger_states(ledger_path) - if not states: - return - from execution_core import Ledger - - ledger = Ledger(ledger_path) - try: - for item in states: - state = ledger.get(item["intent_id"]).state - if state in TERMINAL_OR_MANUAL: - continue - if state in {"ACKNOWLEDGED", "PARTIALLY_FILLED", "CANCEL_PENDING"}: - ledger.transition(item["intent_id"], "UNKNOWN", {"reason": reason}) - state = "UNKNOWN" - if state == "UNKNOWN": - ledger.transition(item["intent_id"], "RECONCILING", {}) - state = "RECONCILING" - if state in {"PLANNED", "WAIT_TRIGGER", "RESERVED", "RECONCILING"}: - ledger.transition(item["intent_id"], "MANUAL_BLOCK", {"reason": reason}) - elif state == "SUBMITTING": - ledger.transition(item["intent_id"], "UNKNOWN", {"reason": reason}) - ledger.transition(item["intent_id"], "RECONCILING", {}) - ledger.transition(item["intent_id"], "MANUAL_BLOCK", {"reason": reason}) - finally: - ledger.close() - - -def reconcile_prior_states( - *, - config: dict[str, Any], - trader_root: Path, - market: str, - account_alias: str, - current_session: str, - receipt_directory: Path, -) -> None: - market_root = state_market_directory( - config["runtime_root"], account_alias, market - ) - unresolved = unresolved_state_directories( - market_root, exclude_date=current_session - ) - for state_directory, states in unresolved: - if any(item["state"] == "MANUAL_BLOCK" for item in states): - raise PipelineBlockedError( - f"prior MANUAL_BLOCK requires explicit clearance: {state_directory}" - ) - descriptor = prior_descriptor_for_state(config, market, state_directory) - for attempt in range(1, RECONCILE_ATTEMPTS + 1): - reconcile_one( - trader_root=trader_root, - descriptor=descriptor, - output_path=( - receipt_directory - / f"prior-{state_directory.name}-reconcile-{attempt:02d}.json" - ), - ) - observed = read_ledger_states(state_directory / "ledger.sqlite3") - if any(item["state"] == "MANUAL_BLOCK" for item in observed): - raise PipelineBlockedError( - f"prior reconciliation produced MANUAL_BLOCK: {state_directory}" - ) - remaining = [ - item for item in observed if item["state"] not in TERMINAL_OR_MANUAL - ] - if not remaining: - break - if attempt < RECONCILE_ATTEMPTS: - time.sleep(RECONCILE_INTERVAL_SECONDS) - else: - mark_manual_block( - state_directory, - "read-only reconciliation exhausted 20 attempts", - ) - raise PipelineBlockedError( - f"prior state became MANUAL_BLOCK after reconciliation: " - f"{state_directory}" - ) - - -def snapshot_and_plan( - config: dict[str, Any], - market: str, - technical_root: Path, - trader_root: Path, -) -> dict[str, Any]: - session = current_market_date(market) - descriptor_path = current_descriptor_path(config, market) - descriptor = load_json(descriptor_path, "current daily descriptor") - if descriptor.get("session_date") != session or descriptor.get("market") != market: - raise PipelineBlockedError("current descriptor is not for this market session") - if descriptor.get("status") in {"BLOCKED", "MANUAL_BLOCK"}: - raise PipelineBlockedError( - str( - descriptor.get("blocked_reason") - or "current market session is terminally blocked" - ) - ) - if descriptor.get("status") == "MARKET_CLOSED": - receipt = stage_status( - stage="snapshot", - status="MARKET_CLOSED", - market=market, - session_date=session, - ) - atomic_write_json( - Path(descriptor["workflow_directory"]) / "snapshot-status.json", - receipt, - ) - return receipt - if descriptor.get("status") == "PLANNED": - return load_json( - Path(descriptor["workflow_directory"]) / "snapshot-status.json", - "existing snapshot status", - ) - if descriptor.get("status") != "PREPARED": - raise PipelineBlockedError("daily prepare stage is not PREPARED") - workflow = Path(descriptor["workflow_directory"]) - market_session = load_json( - Path(descriptor["market_session_path"]), "market session" - ) - regular_open = datetime.fromisoformat(market_session["regular_open"]) - now = datetime.now(timezone.utc) - age_until_open = (regular_open.astimezone(timezone.utc) - now).total_seconds() - snapshot_max_age = config["execution_policy"]["snapshot_max_age_seconds"] - if age_until_open <= 0: - raise PipelineBlockedError("account snapshot started after regular open") - if age_until_open > snapshot_max_age: - raise PipelineBlockedError( - "account snapshot started too early for snapshot_max_age_seconds" - ) - components = resolve_exposure_components(config, session) - account_directory = workflow / "account" - account_directory.mkdir(parents=True, exist_ok=True) - account_alias = config["account_aliases"][market] - reconcile_prior_states( - config=config, - trader_root=trader_root, - market=market, - account_alias=account_alias, - current_session=session, - receipt_directory=workflow / "reconciliation", - ) - state_market_root = state_market_directory( - config["runtime_root"], account_alias, market - ) - state_directory = state_market_root / session - account_path = account_directory / "account.json" - exposure_path = account_directory / "exposure.json" - account_status_path = account_directory / "status.json" - account_job = { - "schema": "qta-account-snapshot-job/v2", - "broker": "kis-live", - "mode": "shadow", - "market": market, - "account_alias": account_alias, - "universe_manifest": descriptor["manifest_path"], - "fx_to_krw": "1" if market == "KR" else "KIS_PRESENT_BALANCE", - "manual_exposure_components": [str(path) for path in components], - "manual_component_max_age_seconds": config[ - "exposure_component_max_age_seconds" - ], - "output_account_path": str(account_path.resolve()), - "output_exposure_path": str(exposure_path.resolve()), - "output_status_path": str(account_status_path.resolve()), - } - account_job_path = workflow / "jobs" / "account-snapshot.json" - atomic_write_json(account_job_path, account_job) - run_command( - python_command( - trader_root / "scripts" / "account_snapshot.py", - "collect", - "--job", - str(account_job_path), - ), - "KIS and cross-broker account snapshot", - ) - account_status = load_json(account_status_path, "account snapshot status") - if ( - account_status.get("status") != "READY" - or account_status.get("api_mutation_count") != 0 - ): - raise PipelineBlockedError("account snapshot status is not READY/read-only") - execution = { - "schema": "qta-execution-policy/v1", - "market": market, - "timezone": market_session["timezone"], - "entry_window_start": market_session["regular_open"], - "entry_window_end": ( - datetime.fromisoformat(market_session["regular_open"]) - + timedelta(hours=1) - ).isoformat(), - "market_session": market_session, - **config["execution_policy"], - } - execution = normalized_execution_policy(execution) - execution_path = workflow / "execution-policy.json" - atomic_write_json(execution_path, execution) - plan_path = workflow / "order-plan.json" - run_command( - python_command( - trader_root / "scripts" / "plan_orders.py", - "--screen", - descriptor["screen_path"], - "--account", - str(account_path), - "--exposure", - str(exposure_path), - "--risk", - str(config["risk_policy_path"]), - "--execution", - str(execution_path), - "--output", - str(plan_path), - ), - "deterministic order plan", - ) - plan = load_json(plan_path, "order plan") - if plan.get("plan_status") not in {"READY", "NO_ORDERS"}: - raise PipelineBlockedError("order plan is not READY or NO_ORDERS") - preview_path = workflow / "submit-preview.json" - run_command( - python_command( - trader_root / "scripts" / "run_session.py", - "preview", - "--plan", - str(plan_path), - "--broker", - "kis-live", - "--output", - str(preview_path), - ), - "venue and submit serialization preflight", - ) - receipt = stage_status( - stage="snapshot", - status="READY", - market=market, - session_date=session, - details={ - "plan_hash": plan["plan_hash"], - "settled_cash": account_status["settled_cash"], - "borrowed_buying_power_excluded": True, - "fx_to_krw": account_status["fx_to_krw"], - "additional_exposure_components": len(components), - "state_directory": str(state_directory.resolve()), - }, - ) - full_descriptor = { - **descriptor, - "status": "PLANNED", - "account_alias": account_alias, - "account_path": str(account_path.resolve()), - "exposure_path": str(exposure_path.resolve()), - "account_status_path": str(account_status_path.resolve()), - "execution_policy_path": str(execution_path.resolve()), - "plan_path": str(plan_path.resolve()), - "plan_hash": plan["plan_hash"], - "state_directory": str(state_directory.resolve()), - "session_status_path": str((workflow / "session-status.json").resolve()), - "reconciliation_status_path": str( - (workflow / "reconciliation-status.json").resolve() - ), - "live_enabled": False, - "api_mutation_count": 0, - } - atomic_write_json(workflow / "snapshot-status.json", receipt) - atomic_write_json(workflow / "descriptor.json", full_descriptor) - atomic_write_json(descriptor_path, full_descriptor) - return receipt - - -def reconcile_current( - *, - descriptor: dict[str, Any], - trader_root: Path, - output_path: Path, -) -> dict[str, Any]: - state_directory = Path(descriptor["state_directory"]) - for attempt in range(1, RECONCILE_ATTEMPTS + 1): - attempt_path = output_path.with_name( - f"{output_path.stem}-attempt-{attempt:02d}.json" - ) - reconcile_one( - trader_root=trader_root, - descriptor=descriptor, - output_path=attempt_path, - ) - states = read_ledger_states(state_directory / "ledger.sqlite3") - remaining = [ - item for item in states if item["state"] not in TERMINAL_OR_MANUAL - ] - if not remaining: - final = load_json(attempt_path, "reconciliation receipt") - atomic_write_json(output_path, final) - return final - if attempt < RECONCILE_ATTEMPTS: - time.sleep(RECONCILE_INTERVAL_SECONDS) - mark_manual_block( - state_directory, - "read-only reconciliation exhausted 20 attempts", - ) - states = read_ledger_states(state_directory / "ledger.sqlite3") - manual = stage_status( - stage="reconcile", - status="MANUAL_BLOCK", - market=descriptor["market"], - session_date=descriptor["session_date"], - reason="UNKNOWN or CANCEL_PENDING remained after 20 read-only attempts", - details={"intents": states, "emergency_alert": True}, - ) - atomic_write_json(output_path, manual) - return manual - - -def build_final_report( - *, - config: dict[str, Any], - descriptor: dict[str, Any], - session_receipt: dict[str, Any], - reconciliation: dict[str, Any], -) -> dict[str, Any]: - plan = load_json(Path(descriptor["plan_path"]), "order plan") - screen = load_json(Path(descriptor["screen_path"]), "screen") - account_status = load_json( - Path(descriptor["account_status_path"]), "account status" - ) - states = read_ledger_states( - Path(descriptor["state_directory"]) / "ledger.sqlite3" - ) - selected = sum( - len(screen["selected"][exchange]) - for exchange in ( - ("KOSPI", "KOSDAQ") if descriptor["market"] == "KR" else ("NYSE", "NASDAQ") - ) - ) - state_counts = { - state: sum(1 for item in states if item["state"] == state) - for state in sorted({item["state"] for item in states}) - } - metrics = session_receipt.get("polling_metrics", {}) - first_unmet = "" - final_status = "READY" - if state_counts.get("MANUAL_BLOCK", 0): - final_status = "MANUAL_BLOCK" - first_unmet = reconciliation.get( - "reason", "one or more intents require manual clearance" - ) - elif session_receipt.get("status") == "BLOCKED": - final_status = "BLOCKED" - first_unmet = session_receipt.get("reason", "session blocked") - elif reconciliation.get("status") not in {"READY", None}: - final_status = "BLOCKED" - first_unmet = "reconciliation did not reach READY" - submitted = int(metrics.get("submits_started", 0)) - if submitted != 0 or session_receipt.get("mutation_sent") is True: - raise PipelineBlockedError("shadow session reported an API mutation") - return { - "schema": "qta-daily-shadow-final-report/v1", - "status": final_status, - "session_date": descriptor["session_date"], - "market": descriptor["market"], - "broker": "kis-live", - "mode": "shadow", - "manifest_hash": descriptor["manifest_hash"], - "screen_hash": descriptor["screen_hash"], - "plan_hash": descriptor["plan_hash"], - "selected_count": selected, - "planned_count": len(plan.get("intents", [])), - "submitted_count": 0, - "filled_count": state_counts.get("FILLED", 0), - "cancelled_count": state_counts.get("CANCELLED", 0), - "blocked_count": state_counts.get("MANUAL_BLOCK", 0), - "cycles": int(metrics.get("cycles_completed", 0)), - "skipped_cycles": int(metrics.get("cycles_skipped", 0)), - "max_schedule_latency_ms": int( - metrics.get("max_schedule_lateness_ms", 0) - ), - "max_quote_latency_ms": int(metrics.get("max_quote_latency_ms", 0)), - "max_submit_latency_ms": int(metrics.get("max_submit_latency_ms", 0)), - "settled_cash_used": format( - Decimal(plan["settled_cash_start"]) - - Decimal(plan["settled_cash_unreserved"]), - "f", - ), - "settled_cash_available": account_status["settled_cash"], - "borrowed_cash_excluded": load_json( - Path(descriptor["account_path"]), "account" - )["borrowed_buying_power"], - "intent_final_states": states, - "first_unmet_gate": first_unmet, - "api_mutation_count": 0, - "live_enabled": False, - } - - -def entry( - config: dict[str, Any], - market: str, - technical_root: Path, - trader_root: Path, - max_cycles: int | None, -) -> dict[str, Any]: - session = current_market_date(market) - descriptor_path = current_descriptor_path(config, market) - descriptor = load_json(descriptor_path, "current daily descriptor") - if descriptor.get("session_date") != session or descriptor.get("market") != market: - raise PipelineBlockedError("current descriptor is not for this market session") - if descriptor.get("status") in {"BLOCKED", "MANUAL_BLOCK"}: - raise PipelineBlockedError( - str( - descriptor.get("blocked_reason") - or "current market session is terminally blocked" - ) - ) - if descriptor.get("status") == "MARKET_CLOSED": - receipt = stage_status( - stage="entry", - status="MARKET_CLOSED", - market=market, - session_date=session, - ) - atomic_write_json( - Path(descriptor["workflow_directory"]) / "entry-status.json", - receipt, - ) - return receipt - if descriptor.get("status") != "PLANNED": - raise PipelineBlockedError("daily snapshot and plan stage is not ready") - account_alias = config["account_aliases"][market] - if descriptor.get("account_alias") != account_alias: - raise PipelineBlockedError("current plan account alias does not match config") - expected_state = ( - state_market_directory(config["runtime_root"], account_alias, market) - / session - ).resolve() - if Path(descriptor["state_directory"]).resolve() != expected_state: - raise PipelineBlockedError( - "state directory does not follow the account/market/session rule" - ) - ledger_path = expected_state / "ledger.sqlite3" - current_states = read_ledger_states(ledger_path) - if ledger_path.exists(): - if any(item["state"] not in TERMINAL_OR_MANUAL for item in current_states): - reconciliation = reconcile_current( - descriptor=descriptor, - trader_root=trader_root, - output_path=Path(descriptor["reconciliation_status_path"]), - ) - if reconciliation.get("status") != "READY": - raise PipelineBlockedError( - "existing current-session ledger did not reconcile to terminal" - ) - raise PipelineBlockedError( - "current session was already started; automatic re-entry is forbidden" - ) - attempt_path = Path(descriptor["workflow_directory"]) / "entry-attempt.json" - if attempt_path.exists(): - raise PipelineBlockedError( - "current session already has an entry-attempt marker" - ) - atomic_write_json( - attempt_path, - { - "schema": "qta-shadow-entry-attempt/v1", - "market": market, - "session_date": session, - "plan_hash": descriptor["plan_hash"], - "started_at": datetime.now(timezone.utc).isoformat(), - "live_enabled": False, - "api_mutation_count": 0, - }, - ) - command = python_command( - trader_root / "scripts" / "run_session.py", - "run", - "--plan", - descriptor["plan_path"], - "--broker", - "kis-live", - "--mode", - "shadow", - "--state-dir", - descriptor["state_directory"], - "--output", - descriptor["session_status_path"], - ) - if max_cycles is not None: - if max_cycles <= 0: - raise PipelineBlockedError("max_cycles must be positive when supplied") - command.extend(["--max-cycles", str(max_cycles)]) - result = subprocess.run( - command, - check=False, - capture_output=True, - text=True, - env=subprocess_environment(), - ) - session_receipt = load_json( - Path(descriptor["session_status_path"]), "session status" - ) - reconciliation = reconcile_current( - descriptor=descriptor, - trader_root=trader_root, - output_path=Path(descriptor["reconciliation_status_path"]), - ) - report = build_final_report( - config=config, - descriptor=descriptor, - session_receipt=session_receipt, - reconciliation=reconciliation, - ) - workflow = Path(descriptor["workflow_directory"]) - atomic_write_json(workflow / "final-report.json", report) - atomic_write_json( - descriptor_path, - { - **descriptor, - "status": report["status"], - "final_report_path": str((workflow / "final-report.json").resolve()), - }, - ) - if result.returncode != 0 or report["status"] not in {"READY"}: - raise PipelineBlockedError( - report["first_unmet_gate"] or "shadow session did not finish READY" - ) - return report - - -def self_test() -> None: - with tempfile.TemporaryDirectory(prefix="qta-daily-pipeline-") as temporary: - root = Path(temporary) - config = { - "runtime_root": root, - "exposure_component_paths": [], - } - if resolve_exposure_components(config, "2026-07-28") != []: - raise AssertionError("additional exposure components must be optional") - receipt = stage_status( - stage="self-test", - status="PASS", - market="US", - session_date="2026-07-28", - ) - if receipt["api_mutation_count"] != 0 or receipt["live_enabled"]: - raise AssertionError("shadow safety status failed") - if "receipt_hash" in receipt: - raise AssertionError("daily status must not require a receipt signature") - print( - json.dumps( - { - "self_test": "PASS", - "config_schema": CONFIG_SCHEMA, - "external_exposure_required": False, - "arm_required": False, - "provenance_required": False, - "live_enabled": False, - "api_mutation_count": 0, - }, - sort_keys=True, - ) - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - subparsers = parser.add_subparsers(dest="command", required=True) - for name in ("prepare", "snapshot", "entry"): - stage = subparsers.add_parser(name) - stage.add_argument("--config", required=True) - stage.add_argument("--market", choices=("KR", "US"), required=True) - stage.add_argument("--technical-skill-root", required=True) - stage.add_argument("--trader-skill-root", required=True) - if name == "entry": - stage.add_argument("--max-cycles", type=int) - subparsers.add_parser("self-test") - return parser.parse_args() - - -def main() -> int: - args = parse_args() - if args.command == "self-test": - self_test() - return 0 - config_path = absolute_path(args.config, "config") - technical_root = absolute_path( - args.technical_skill_root, "technical_skill_root" - ) - trader_root = absolute_path(args.trader_skill_root, "trader_skill_root") - market = args.market - session = current_market_date(market) - try: - config = load_config(config_path) - config["runtime_root"].mkdir(parents=True, exist_ok=True, mode=0o700) - if args.command == "prepare": - output = prepare(config, market, technical_root, trader_root) - elif args.command == "snapshot": - output = snapshot_and_plan( - config, market, technical_root, trader_root - ) - else: - output = entry( - config, - market, - technical_root, - trader_root, - args.max_cycles, - ) - print(json.dumps(output, ensure_ascii=False, sort_keys=True)) - return 0 - except ( - PipelineBlockedError, - market_calendar.CalendarBlockedError, - OSError, - ValueError, - json.JSONDecodeError, - ) as exc: - receipt = stage_status( - stage=args.command, - status="BLOCKED", - market=market, - session_date=session, - reason=str(exc), - ) - try: - workflow = workflow_directory( - {"runtime_root": absolute_path( - load_json(config_path, "daily config")["runtime_root"], - "runtime_root", - )}, - market, - session, - ) - blocked_path = workflow / f"{args.command}-blocked.json" - stage_status_path = workflow / f"{args.command}-status.json" - atomic_write_json(blocked_path, receipt) - atomic_write_json(stage_status_path, receipt) - descriptor_path = current_descriptor_path( - {"runtime_root": workflow.parents[2]}, - market, - ) - existing: dict[str, Any] = {} - if descriptor_path.is_file() and not descriptor_path.is_symlink(): - candidate = load_json(descriptor_path, "current daily descriptor") - if ( - candidate.get("market") == market - and candidate.get("session_date") == session - ): - existing = candidate - final_status = ( - "MANUAL_BLOCK" - if existing.get("status") == "MANUAL_BLOCK" - else "BLOCKED" - ) - blocked_descriptor = { - **existing, - "schema": DESCRIPTOR_SCHEMA, - "status": final_status, - "market": market, - "session_date": session, - "workflow_directory": str(workflow.resolve()), - "blocked_stage": args.command, - "blocked_reason": str(exc), - "blocked_status_path": str(blocked_path.resolve()), - "live_enabled": False, - "api_mutation_count": 0, - } - atomic_write_json( - workflow / "descriptor.json", - blocked_descriptor, - ) - atomic_write_json(descriptor_path, blocked_descriptor) - except (OSError, ValueError, KeyError): - pass - print(json.dumps(receipt, ensure_ascii=False, sort_keys=True), file=sys.stderr) - return 2 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/execution_core.py b/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/execution_core.py deleted file mode 100755 index 2bd45fd..0000000 --- a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/execution_core.py +++ /dev/null @@ -1,2838 +0,0 @@ -#!/usr/bin/env python3 -"""Deterministic planning primitives and durable order-intent ledger.""" - -from __future__ import annotations - -import argparse -import fcntl -import hashlib -import json -import sqlite3 -import sys -import tempfile -from collections.abc import Iterator -from contextlib import contextmanager -from dataclasses import dataclass -from datetime import date, datetime, timedelta, timezone -from decimal import ROUND_DOWN, Decimal, InvalidOperation -from pathlib import Path -from typing import Any -from zoneinfo import ZoneInfo - -EXECUTION_VERSION = "open1h-exec-2.0.0" -PLAN_SCHEMA = "qta-order-plan/v2" -RISK_SCHEMA = "qta-risk-policy/v1" -EXECUTION_POLICY_SCHEMA = "qta-execution-policy/v1" -MARKET_SESSION_SCHEMA = "qta-market-session/v1" -ACCOUNT_SCHEMA = "qta-account-snapshot/v2" -EXPOSURE_SCHEMA = "qta-exposure-snapshot/v1" -EXPOSURE_SCHEMA_V2 = "qta-exposure-snapshot/v2" -SCREEN_SCHEMA = "qta-screen/v1" -SCREEN_SCHEMA_V2 = "qta-screen/v2" -MARKET_CURRENCY = {"KR": "KRW", "US": "USD"} -V2_SELECTOR_VERSION = "qta-screen-1.1.0" -QTA_METHOD_V1 = "qta-1.0.0" -QTA_METHOD_V2 = "qta-2.0.0" -SUPPORTED_QTA_METHODS = {QTA_METHOD_V1, QTA_METHOD_V2} -V2_EXCHANGES_BY_MARKET = { - "KR": ("KOSPI", "KOSDAQ"), - "US": ("NYSE", "NASDAQ"), -} -V2_EXCHANGE_CONTRACTS = { - "KOSPI": { - "market": "KR", - "benchmark_id": "KOSPI_COMPOSITE", - "currency": "KRW", - "venue": "KRX", - }, - "KOSDAQ": { - "market": "KR", - "benchmark_id": "KOSDAQ_COMPOSITE", - "currency": "KRW", - "venue": "KRX", - }, - "NYSE": { - "market": "US", - "benchmark_id": "NYSE_COMPOSITE", - "currency": "USD", - "venue": "NYSE", - }, - "NASDAQ": { - "market": "US", - "benchmark_id": "NASDAQ_COMPOSITE", - "currency": "USD", - "venue": "NASD", - }, -} -V2_INSTRUMENT_FIELDS = { - "market", - "exchange", - "canonical_symbol", - "data_symbol", - "broker_symbol", - "instrument_type", - "benchmark_id", - "currency", - "venue", - "ticker_csv", - "benchmark_csv", - "tick_contract", - "source_name", - "ticker_csv_sha256", - "benchmark_csv_sha256", - "broker_tradability_verified", - "official_source_id", - "broker_source_id", -} -V2_TICK_CONTRACT_FIELDS = { - "schema", - "kind", - "rule_id", - "effective_date", - "reference_price", - "resolved_tick_size", -} -V2_INSTRUMENT_TYPES = {"COMMON", "ADR", "REIT"} -V2_SCREEN_FIELDS = { - "source_skill", - "schema", - "screen_status", - "method_version", - "selector_version", - "analysis_date", - "manifest_hash", - "selector_hash", - "blocked_count", - "blocked_fraction", - "instrument_count", - "selector", - "selected", - "decisions", - "screen_hash", -} -V2_SELECTOR_FIELDS = { - "selector_version", - "min_total_score", - "eligible_setup_statuses", - "top_k_by_exchange", - "min_selected_by_exchange", - "max_blocked_fraction", -} -V2_DECISION_FIELDS = { - "market", - "exchange", - "canonical_symbol", - "instrument", - "eligible", - "reasons", - "exchange_rank", - "selected", - "qta", -} -QTA_READY_FIELDS = { - "source_skill", - "result_schema", - "calculation_status", - "setup_status", - "method_version", - "analysis_date", - "market", - "ticker", - "source_name", - "shared_sessions", - "score_basis", - "short", - "medium", - "long", - "risk", - "entry_price", - "stop_price", - "take_profit_price", - "total_score", - "reference_observations", - "assumptions", -} -QTA2_READY_FIELDS = QTA_READY_FIELDS | { - "validation_status", - "liquidity", - "market_regime", -} -QTA2_LIQUIDITY_FIELDS = { - "median_20_session_turnover", - "minimum_turnover", - "currency", - "status", -} -QTA2_MARKET_REGIME_FIELDS = { - "metric", - "minimum_change_bps", - "max_age_seconds", - "status", -} -QTA_BLOCKED_FIELDS = { - "source_skill", - "result_schema", - "calculation_status", - "reason", - "method_version", - "market", - "ticker", -} -QTA_HORIZON_FIELDS = {"opinion", "score"} -QTA_RISK_FIELDS = {"score", "counterpoint"} -QTA_REFERENCE_FIELDS = { - "return_5", - "return_20", - "close_over_sma_20", - "sma_5_over_sma_20", - "signed_log_volume_ratio_20", - "rsi_14", - "return_63", - "close_over_sma_50", - "sma_20_over_sma_50", - "relative_return_63", - "adx_direction_14", - "return_126", - "return_252", - "close_over_sma_200", - "sma_50_over_sma_200", - "relative_return_252", - "ATR14/close", - "20d annualized volatility", - "252d maximum drawdown", - "60d gap p95", -} -QTA_SCORE_BASIS = "ticker-relative historical percentile; not probability of profit" -QTA2_SCORE_BASIS = ( - "ticker-relative historical percentiles aligned to first-hour " - "continuation; not probability of profit" -) -QTA_ASSUMPTIONS = [ - "input rows are finalized completed daily sessions", - "prices are corporate-action adjusted and aligned to analysis-date raw price scale", - "fees, tax, FX, slippage, position size, and execution are excluded", -] -QTA2_ASSUMPTIONS = [ - *QTA_ASSUMPTIONS, - "QTA 2.0 is research-only until multi-session walk-forward validation", - "same-session benchmark regime admission is required downstream", -] -QTA2_LIQUIDITY_FLOOR = { - "KR": Decimal("1000000000"), - "US": Decimal("1000000"), -} -QTA2_REGIME_METRIC = "same_session_previous_close_return_bps" -QTA2_REGIME_MINIMUM_CHANGE_BPS = Decimal("0") -QTA2_REGIME_MAX_AGE_SECONDS = 90 -QTA_SETUP_STATUSES = {"READY", "CONDITIONAL"} -QTA_OPINIONS = {"강한 긍정", "긍정", "중립", "부정", "강한 부정"} -QTA_MIN_SHARED_SESSIONS = 756 -QTA_MIN_REFERENCE_OBSERVATIONS = 252 -QTA_MAX_REFERENCE_OBSERVATIONS = 756 -ENTRY_WINDOWS = { - "KR": { - "timezone": "Asia/Seoul", - "start": (9, 0), - "end": (10, 0), - }, - "US": { - "timezone": "America/New_York", - "start": (9, 30), - "end": (10, 30), - }, -} -MARKET_SESSION_SOURCE_FIELDS = { - "schema", - "provider", - "source_id", - "source_as_of", - "market", - "timezone", - "session_date", - "previous_session_date", - "scheduled_status", - "regular_open", - "regular_close", -} -MARKET_SESSION_FIELDS = MARKET_SESSION_SOURCE_FIELDS | { - "source_path", - "source_sha256", - "session_hash", -} -MAX_SNAPSHOT_AGE_SECONDS = 3600 - -TERMINAL_STATES = { - "FILLED", - "CANCELLED", - "REJECTED", - "MANUAL_BLOCK", -} -TRANSITIONS = { - "PLANNED": {"WAIT_TRIGGER", "MANUAL_BLOCK"}, - "WAIT_TRIGGER": {"RESERVED", "CANCELLED", "MANUAL_BLOCK"}, - "RESERVED": {"SUBMITTING", "CANCELLED", "MANUAL_BLOCK"}, - "SUBMITTING": {"ACKNOWLEDGED", "UNKNOWN", "REJECTED"}, - "ACKNOWLEDGED": { - "PARTIALLY_FILLED", - "FILLED", - "CANCEL_PENDING", - "CANCELLED", - "REJECTED", - "UNKNOWN", - }, - "PARTIALLY_FILLED": { - "PARTIALLY_FILLED", - "FILLED", - "CANCEL_PENDING", - "CANCELLED", - "UNKNOWN", - }, - "CANCEL_PENDING": { - "CANCELLED", - "FILLED", - "UNKNOWN", - }, - "UNKNOWN": {"RECONCILING", "MANUAL_BLOCK"}, - "RECONCILING": { - "ACKNOWLEDGED", - "PARTIALLY_FILLED", - "FILLED", - "CANCELLED", - "REJECTED", - "MANUAL_BLOCK", - }, -} - - -class BlockedError(ValueError): - """Raised when an execution input is incomplete or unsafe.""" - - -def canonical_json(value: Any) -> str: - return json.dumps( - value, - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - allow_nan=False, - ) - - -def sha256_json(value: Any) -> str: - return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest() - - -def sha256_file(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def decimal_value(value: Any, field: str) -> Decimal: - if isinstance(value, bool): - raise BlockedError(f"{field} must be a decimal string or number") - try: - result = Decimal(str(value)) - except (InvalidOperation, ValueError) as exc: - raise BlockedError(f"{field} must be a decimal string or number") from exc - if not result.is_finite(): - raise BlockedError(f"{field} must be finite") - return result - - -def nonnegative_decimal(value: Any, field: str) -> Decimal: - result = decimal_value(value, field) - if result < 0: - raise BlockedError(f"{field} must be >= 0") - return result - - -def positive_decimal(value: Any, field: str) -> Decimal: - result = decimal_value(value, field) - if result <= 0: - raise BlockedError(f"{field} must be > 0") - return result - - -def exact_fields(value: dict[str, Any], required: set[str], label: str) -> None: - if set(value) != required: - missing = sorted(required - set(value)) - extra = sorted(set(value) - required) - raise BlockedError(f"{label} fields mismatch; missing={missing}, extra={extra}") - - -def parse_aware_datetime(value: Any, field: str) -> datetime: - try: - parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) - except ValueError as exc: - raise BlockedError(f"{field} must be an ISO-8601 datetime") from exc - if parsed.tzinfo is None or parsed.utcoffset() is None: - raise BlockedError(f"{field} must include a UTC offset") - return parsed - - -def parse_iso_date(value: Any, field: str) -> date: - if not isinstance(value, str): - raise BlockedError(f"{field} must be YYYY-MM-DD") - try: - parsed = date.fromisoformat(value) - except ValueError as exc: - raise BlockedError(f"{field} must be YYYY-MM-DD") from exc - if parsed.isoformat() != value: - raise BlockedError(f"{field} must be YYYY-MM-DD") - return parsed - - -def load_json_object(path: str | Path) -> dict[str, Any]: - source = Path(path) - value = json.loads(source.read_text(encoding="utf-8")) - if not isinstance(value, dict): - raise BlockedError(f"{source} must contain a JSON object") - return value - - -def emit_json(value: dict[str, Any], output: str | None = None) -> None: - rendered = json.dumps( - value, ensure_ascii=False, sort_keys=True, indent=2, allow_nan=False - ) - if output: - Path(output).write_text(rendered + "\n", encoding="utf-8") - print(rendered) - - -def normalize_symbol(market: str, symbol: str) -> str: - normalized_market = market.upper() - normalized = symbol.strip().upper() - if normalized_market == "KR": - if normalized.isdigit() and len(normalized) <= 6: - return normalized.zfill(6) - if len(normalized) == 6 and all( - character in "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" - for character in normalized - ): - return normalized - raise BlockedError(f"invalid KR symbol: {symbol!r}") - if normalized_market == "US": - if not normalized or any( - character not in "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.-" - for character in normalized - ): - raise BlockedError(f"invalid US symbol: {symbol!r}") - return normalized - raise BlockedError(f"unsupported market: {market!r}") - - -def normalized_risk_policy(value: dict[str, Any]) -> dict[str, Any]: - required = { - "schema", - "base_currency", - "per_trade_risk_krw", - "max_symbol_notional_krw", - "max_concurrent_positions", - "max_daily_loss_krw", - "round_trip_cost_bps", - "cash_buffer_bps", - "allow_existing_additions", - "allow_borrowed_cash", - "allow_margin", - "allow_short", - "allow_auto_fx", - "whole_shares_only", - } - exact_fields(value, required, "risk policy") - if value["schema"] != RISK_SCHEMA: - raise BlockedError(f"unsupported risk policy schema: {value['schema']!r}") - if value["base_currency"] != "KRW": - raise BlockedError("base_currency must be KRW") - for field in ( - "allow_existing_additions", - "allow_borrowed_cash", - "allow_margin", - "allow_short", - "allow_auto_fx", - "whole_shares_only", - ): - if not isinstance(value[field], bool): - raise BlockedError(f"{field} must be boolean") - if value["allow_borrowed_cash"]: - raise BlockedError("borrowed cash is unsupported by this execution contract") - if value["allow_margin"] or value["allow_short"] or value["allow_auto_fx"]: - raise BlockedError("margin, shorting, and automatic FX must remain disabled") - if not value["whole_shares_only"]: - raise BlockedError("initial execution contract requires whole shares") - count = value["max_concurrent_positions"] - if isinstance(count, bool) or not isinstance(count, int) or count <= 0: - raise BlockedError("max_concurrent_positions must be a positive integer") - for field in ( - "per_trade_risk_krw", - "max_symbol_notional_krw", - "max_daily_loss_krw", - ): - positive_decimal(value[field], field) - for field in ("round_trip_cost_bps", "cash_buffer_bps"): - amount = nonnegative_decimal(value[field], field) - if amount > Decimal(10000): - raise BlockedError(f"{field} must be <= 10000") - return { - **value, - "per_trade_risk_krw": format( - positive_decimal(value["per_trade_risk_krw"], "per_trade_risk_krw"), - "f", - ), - "max_symbol_notional_krw": format( - positive_decimal( - value["max_symbol_notional_krw"], "max_symbol_notional_krw" - ), - "f", - ), - "max_daily_loss_krw": format( - positive_decimal(value["max_daily_loss_krw"], "max_daily_loss_krw"), - "f", - ), - "round_trip_cost_bps": format( - nonnegative_decimal(value["round_trip_cost_bps"], "round_trip_cost_bps"), - "f", - ), - "cash_buffer_bps": format( - nonnegative_decimal(value["cash_buffer_bps"], "cash_buffer_bps"), - "f", - ), - } - - -def canonical_market_session_hash(value: dict[str, Any]) -> str: - """Hash a market-session artifact without its self-referential hash.""" - try: - return sha256_json( - {key: item for key, item in value.items() if key != "session_hash"} - ) - except (TypeError, ValueError) as exc: - raise BlockedError("market_session must contain canonical JSON values") from exc - - -def market_session_from_source(path: str | Path) -> dict[str, Any]: - source_path = Path(path).resolve() - try: - source_payload = load_json_object(source_path) - source_sha256 = sha256_file(source_path) - except (OSError, json.JSONDecodeError) as exc: - raise BlockedError( - "market-session source must be a readable JSON snapshot" - ) from exc - exact_fields( - source_payload, - MARKET_SESSION_SOURCE_FIELDS, - "market-session source snapshot", - ) - result = { - **source_payload, - "source_path": str(source_path), - "source_sha256": source_sha256, - } - result["session_hash"] = canonical_market_session_hash(result) - return result - - -def normalized_market_session( - value: Any, - *, - expected_market: str, - expected_timezone: str, -) -> dict[str, Any]: - if not isinstance(value, dict): - raise BlockedError("market_session must be an object") - exact_fields(value, MARKET_SESSION_FIELDS, "market_session") - if value["schema"] != MARKET_SESSION_SCHEMA: - raise BlockedError(f"market_session.schema must be {MARKET_SESSION_SCHEMA}") - provider = validate_nonempty_string(value["provider"], "market_session.provider") - source_id = validate_nonempty_string(value["source_id"], "market_session.source_id") - if value["market"] != expected_market: - raise BlockedError(f"market_session.market must be {expected_market}") - if value["timezone"] != expected_timezone: - raise BlockedError(f"market_session.timezone must be {expected_timezone}") - if value["scheduled_status"] != "OPEN": - raise BlockedError("market_session.scheduled_status must be OPEN") - - session_date = parse_iso_date(value["session_date"], "market_session.session_date") - previous_session_date = parse_iso_date( - value["previous_session_date"], - "market_session.previous_session_date", - ) - if session_date.weekday() >= 5: - raise BlockedError("market_session.session_date must be a weekday") - if previous_session_date >= session_date: - raise BlockedError( - "market_session.previous_session_date must precede session_date" - ) - - zone = ZoneInfo(expected_timezone) - regular_open = parse_aware_datetime( - value["regular_open"], "market_session.regular_open" - ) - regular_close = parse_aware_datetime( - value["regular_close"], "market_session.regular_close" - ) - source_as_of = parse_aware_datetime( - value["source_as_of"], "market_session.source_as_of" - ) - for supplied, localized, field in ( - (regular_open, regular_open.astimezone(zone), "regular_open"), - (regular_close, regular_close.astimezone(zone), "regular_close"), - ): - if ( - supplied.replace(tzinfo=None) != localized.replace(tzinfo=None) - or supplied.utcoffset() != localized.utcoffset() - ): - raise BlockedError( - "market_session." - f"{field} local clock and UTC offset must match {expected_timezone}" - ) - if supplied.isoformat() != value[field]: - raise BlockedError( - f"market_session.{field} must be a canonical ISO-8601 datetime" - ) - if localized.date() != session_date: - raise BlockedError(f"market_session.{field} must fall on session_date") - if regular_close <= regular_open: - raise BlockedError("market_session.regular_close must be after regular_open") - if source_as_of.isoformat() != value["source_as_of"]: - raise BlockedError( - "market_session.source_as_of must be a canonical ISO-8601 datetime" - ) - if source_as_of > regular_open: - raise BlockedError("market_session.source_as_of must not be after regular_open") - - source_path_text = validate_nonempty_string( - value["source_path"], "market_session.source_path" - ) - source_path = Path(source_path_text) - if not source_path.is_absolute(): - raise BlockedError("market_session.source_path must be absolute") - validate_sha256(value["source_sha256"], "market_session.source_sha256") - try: - actual_source_sha256 = sha256_file(source_path) - source_payload = load_json_object(source_path) - except (OSError, json.JSONDecodeError) as exc: - raise BlockedError( - "market_session.source_path must be a readable JSON snapshot" - ) from exc - if actual_source_sha256 != value["source_sha256"]: - raise BlockedError("market_session.source_sha256 does not match source_path") - exact_fields( - source_payload, - MARKET_SESSION_SOURCE_FIELDS, - "market_session source snapshot", - ) - expected_source_payload = {key: value[key] for key in MARKET_SESSION_SOURCE_FIELDS} - if source_payload != expected_source_payload: - raise BlockedError( - "market_session fields do not match the hashed source snapshot" - ) - - validate_sha256(value["session_hash"], "market_session.session_hash") - if value["session_hash"] != canonical_market_session_hash(value): - raise BlockedError( - "market_session.session_hash does not match canonical contents" - ) - return { - **value, - "provider": provider, - "source_id": source_id, - "source_as_of": source_as_of.isoformat(), - "regular_open": regular_open.isoformat(), - "regular_close": regular_close.isoformat(), - } - - -def normalized_execution_policy(value: dict[str, Any]) -> dict[str, Any]: - required = { - "schema", - "market", - "timezone", - "entry_window_start", - "entry_window_end", - "market_session", - "snapshot_max_age_seconds", - "poll_interval_seconds", - "quote_max_age_seconds", - "max_spread_bps", - "max_gap_bps", - "trigger_mode", - "order_ttl_seconds", - "order_type", - "time_in_force", - "allow_partial_fill", - "cancel_remainder_at_window_end", - } - exact_fields(value, required, "execution policy") - if value["schema"] != EXECUTION_POLICY_SCHEMA: - raise BlockedError(f"unsupported execution policy schema: {value['schema']!r}") - market = str(value["market"]).upper() - if market not in MARKET_CURRENCY: - raise BlockedError("market must be KR or US") - window_contract = ENTRY_WINDOWS[market] - expected_timezone = window_contract["timezone"] - if value["timezone"] != expected_timezone: - raise BlockedError(f"{market} timezone must be {expected_timezone}") - start = parse_aware_datetime(value["entry_window_start"], "entry_window_start") - end = parse_aware_datetime(value["entry_window_end"], "entry_window_end") - if start >= end: - raise BlockedError("entry window start must be before end") - zone = ZoneInfo(expected_timezone) - start_local = start.astimezone(zone) - end_local = end.astimezone(zone) - for supplied, localized, field in ( - (start, start_local, "entry_window_start"), - (end, end_local, "entry_window_end"), - ): - if ( - supplied.replace(tzinfo=None) != localized.replace(tzinfo=None) - or supplied.utcoffset() != localized.utcoffset() - ): - raise BlockedError( - f"{field} local clock and UTC offset must match {expected_timezone}" - ) - expected_start = window_contract["start"] - expected_end = window_contract["end"] - if ( - (start_local.hour, start_local.minute) != expected_start - or start_local.second != 0 - or start_local.microsecond != 0 - or (end_local.hour, end_local.minute) != expected_end - or end_local.second != 0 - or end_local.microsecond != 0 - or start_local.date() != end_local.date() - ): - raise BlockedError( - f"{market} entry window must be " - f"{expected_start[0]:02d}:{expected_start[1]:02d}-" - f"{expected_end[0]:02d}:{expected_end[1]:02d} {expected_timezone}" - ) - if (end - start).total_seconds() != 3600: - raise BlockedError("entry window must be exactly one hour") - market_session = normalized_market_session( - value["market_session"], - expected_market=market, - expected_timezone=expected_timezone, - ) - regular_open = parse_aware_datetime( - market_session["regular_open"], "market_session.regular_open" - ) - regular_close = parse_aware_datetime( - market_session["regular_close"], "market_session.regular_close" - ) - if start != regular_open: - raise BlockedError("entry_window_start must equal market_session.regular_open") - if end != regular_open + timedelta(hours=1): - raise BlockedError( - "entry_window_end must equal one hour after market_session.regular_open" - ) - if end > regular_close: - raise BlockedError( - "entry_window_end must not exceed market_session.regular_close" - ) - for field in ( - "snapshot_max_age_seconds", - "poll_interval_seconds", - "quote_max_age_seconds", - "order_ttl_seconds", - ): - amount = value[field] - if isinstance(amount, bool) or not isinstance(amount, int) or amount <= 0: - raise BlockedError(f"{field} must be a positive integer") - if value["snapshot_max_age_seconds"] > MAX_SNAPSHOT_AGE_SECONDS: - raise BlockedError( - f"snapshot_max_age_seconds must be <= {MAX_SNAPSHOT_AGE_SECONDS}" - ) - if value["order_type"] != "LIMIT" or value["time_in_force"] != "DAY": - raise BlockedError("initial execution contract requires LIMIT + DAY") - if value["trigger_mode"] not in {"AT_OR_ABOVE", "CROSS_FROM_BELOW"}: - raise BlockedError("trigger_mode must be AT_OR_ABOVE or CROSS_FROM_BELOW") - for field in ("allow_partial_fill", "cancel_remainder_at_window_end"): - if not isinstance(value[field], bool): - raise BlockedError(f"{field} must be boolean") - for field in ("max_spread_bps", "max_gap_bps"): - amount = nonnegative_decimal(value[field], field) - if amount > Decimal(10000): - raise BlockedError(f"{field} must be <= 10000") - return { - **value, - "market": market, - "entry_window_start": start_local.isoformat(), - "entry_window_end": end_local.isoformat(), - "market_session": market_session, - "max_spread_bps": format( - nonnegative_decimal(value["max_spread_bps"], "max_spread_bps"), "f" - ), - "max_gap_bps": format( - nonnegative_decimal(value["max_gap_bps"], "max_gap_bps"), "f" - ), - } - - -def normalized_account_snapshot( - value: dict[str, Any], - *, - screen_schema: str = SCREEN_SCHEMA, -) -> dict[str, Any]: - required = { - "schema", - "broker", - "environment", - "account_alias", - "market", - "currency", - "as_of", - "settled_cash", - "borrowed_buying_power", - "fx_to_krw", - "positions", - "open_orders", - } - exact_fields(value, required, "account snapshot") - if value["schema"] != ACCOUNT_SCHEMA: - raise BlockedError(f"unsupported account snapshot schema: {value['schema']!r}") - market = str(value["market"]).upper() - if market not in MARKET_CURRENCY: - raise BlockedError("account market must be KR or US") - if value["currency"] != MARKET_CURRENCY[market]: - raise BlockedError( - f"{market} account currency must be {MARKET_CURRENCY[market]}" - ) - if value["environment"] not in {"paper", "shadow", "live"}: - raise BlockedError("account environment must be paper, shadow, or live") - broker = validate_nonempty_string(value["broker"], "broker") - account_alias = validate_nonempty_string(value["account_alias"], "account_alias") - as_of = parse_aware_datetime(value["as_of"], "account as_of") - settled = nonnegative_decimal(value["settled_cash"], "settled_cash") - borrowed = nonnegative_decimal( - value["borrowed_buying_power"], "borrowed_buying_power" - ) - if not isinstance(value["positions"], list) or not isinstance( - value["open_orders"], list - ): - raise BlockedError("positions and open_orders must be arrays") - raw_fx = value["fx_to_krw"] - if raw_fx is None: - if market != "US" or settled != 0 or value["positions"]: - raise BlockedError( - "fx_to_krw may be null only for an empty US account with " - "zero settled cash" - ) - fx: Decimal | None = None - else: - fx = positive_decimal(raw_fx, "fx_to_krw") - if screen_schema not in {SCREEN_SCHEMA, SCREEN_SCHEMA_V2}: - raise BlockedError(f"unsupported screen schema: {screen_schema!r}") - - positions: list[dict[str, Any]] = [] - for index, position in enumerate(value["positions"]): - label = f"account positions[{index}]" - if not isinstance(position, dict): - raise BlockedError(f"{label} must be an object") - if "market" not in position or "symbol" not in position: - raise BlockedError(f"{label} requires market and symbol") - position_market = str(position["market"]).upper() - if position_market != market: - raise BlockedError(f"{label}.market must match account market {market}") - symbol = normalize_symbol(position_market, str(position["symbol"])) - normalized_position = { - **position, - "market": position_market, - "symbol": symbol, - } - if screen_schema == SCREEN_SCHEMA_V2: - if "exchange" not in position: - raise BlockedError(f"{label} requires exchange for qta-screen/v2") - exchange = str(position["exchange"]).upper() - contract = V2_EXCHANGE_CONTRACTS.get(exchange) - if contract is None or contract["market"] != position_market: - raise BlockedError(f"{label}.exchange must match its market") - normalized_position["exchange"] = exchange - positions.append(normalized_position) - - open_orders: list[dict[str, Any]] = [] - for index, open_order in enumerate(value["open_orders"]): - label = f"account open_orders[{index}]" - if not isinstance(open_order, dict): - raise BlockedError(f"{label} must be an object") - required_order_fields = {"side", "market", "symbol"} - missing_order_fields = required_order_fields - set(open_order) - if missing_order_fields: - raise BlockedError( - f"{label} missing fields needed for exposure key: " - f"{sorted(missing_order_fields)}" - ) - side = str(open_order["side"]).upper() - if side not in {"BUY", "SELL"}: - raise BlockedError(f"{label}.side must be BUY or SELL") - order_market = str(open_order["market"]).upper() - if order_market != market: - raise BlockedError(f"{label}.market must match account market {market}") - symbol = normalize_symbol(order_market, str(open_order["symbol"])) - normalized_open_order = { - **open_order, - "side": side, - "market": order_market, - "symbol": symbol, - } - if screen_schema == SCREEN_SCHEMA_V2: - if "exchange" not in open_order: - raise BlockedError(f"{label} requires exchange for qta-screen/v2") - exchange = str(open_order["exchange"]).upper() - contract = V2_EXCHANGE_CONTRACTS.get(exchange) - if contract is None or contract["market"] != order_market: - raise BlockedError(f"{label}.exchange must match its market") - normalized_open_order["exchange"] = exchange - open_orders.append(normalized_open_order) - - positions.sort( - key=lambda item: ( - item["market"], - item.get("exchange", ""), - item["symbol"], - ) - ) - open_orders.sort( - key=lambda item: ( - item["market"], - item.get("exchange", ""), - item["symbol"], - item["side"], - ) - ) - return { - **value, - "broker": broker, - "account_alias": account_alias, - "market": market, - "as_of": as_of.isoformat(), - "settled_cash": format(settled, "f"), - "borrowed_buying_power": format(borrowed, "f"), - "fx_to_krw": format(fx, "f") if fx is not None else None, - "positions": positions, - "open_orders": open_orders, - } - - -def normalized_exposure_snapshot(value: dict[str, Any]) -> dict[str, Any]: - required = {"schema", "as_of", "positions"} - exact_fields(value, required, "exposure snapshot") - schema = value["schema"] - if schema not in {EXPOSURE_SCHEMA, EXPOSURE_SCHEMA_V2}: - raise BlockedError(f"unsupported exposure snapshot schema: {value['schema']!r}") - as_of = parse_aware_datetime(value["as_of"], "exposure as_of") - if not isinstance(value["positions"], list): - raise BlockedError("exposure positions must be an array") - positions: list[dict[str, Any]] = [] - required_position = { - "broker", - "market", - "symbol", - "quantity", - "market_value_krw", - } - if schema == EXPOSURE_SCHEMA_V2: - required_position = required_position | {"exchange"} - for index, position in enumerate(value["positions"]): - if not isinstance(position, dict): - raise BlockedError(f"positions[{index}] must be an object") - exact_fields(position, required_position, f"positions[{index}]") - market = str(position["market"]).upper() - symbol = normalize_symbol(market, str(position["symbol"])) - normalized_position = {**position, "market": market, "symbol": symbol} - if schema == EXPOSURE_SCHEMA_V2: - exchange = str(position["exchange"]).upper() - contract = V2_EXCHANGE_CONTRACTS.get(exchange) - if contract is None: - raise BlockedError( - f"positions[{index}].exchange is unsupported: {exchange!r}" - ) - if contract["market"] != market: - raise BlockedError( - f"positions[{index}] exchange and market are inconsistent" - ) - normalized_position["exchange"] = exchange - if position["quantity"] is not None: - nonnegative_decimal(position["quantity"], f"positions[{index}].quantity") - nonnegative_decimal( - position["market_value_krw"], - f"positions[{index}].market_value_krw", - ) - positions.append(normalized_position) - positions.sort( - key=lambda item: ( - item["market"], - item.get("exchange", ""), - item["symbol"], - item["broker"], - ) - ) - return {**value, "as_of": as_of.isoformat(), "positions": positions} - - -def validate_screen(screen: dict[str, Any]) -> None: - schema = screen.get("schema") - if schema not in {SCREEN_SCHEMA, SCREEN_SCHEMA_V2}: - raise BlockedError( - f"screen schema must be {SCREEN_SCHEMA} or {SCREEN_SCHEMA_V2}" - ) - if schema == SCREEN_SCHEMA_V2: - validate_v2_screen(screen) - return - if screen.get("screen_status") != "READY": - raise BlockedError("screen_status must be READY") - if screen.get("method_version") != "qta-1.0.0": - raise BlockedError("method_version must be qta-1.0.0") - if screen.get("selector_version") != "qta-screen-1.0.0": - raise BlockedError("selector_version must be qta-screen-1.0.0") - if not isinstance(screen.get("selected"), dict): - raise BlockedError("screen.selected must be an object") - if "screen_hash" not in screen: - raise BlockedError("screen_hash is required") - supplied_hash = screen["screen_hash"] - validate_sha256(supplied_hash, "screen_hash") - expected_hash = canonical_screen_hash(screen) - if supplied_hash != expected_hash: - raise BlockedError("screen_hash does not match canonical screen payload") - - -def validate_sha256(value: Any, field: str) -> None: - if ( - not isinstance(value, str) - or len(value) != 64 - or any(character not in "0123456789abcdef" for character in value) - ): - raise BlockedError(f"{field} must be a lowercase SHA-256 hex digest") - - -def canonical_screen_hash(screen: dict[str, Any]) -> str: - """Hash a screen's canonical content without its self-referential hash.""" - try: - return sha256_json( - {key: item for key, item in screen.items() if key != "screen_hash"} - ) - except (TypeError, ValueError) as exc: - raise BlockedError("screen must contain canonical JSON values") from exc - - -def validate_nonempty_string(value: Any, field: str) -> str: - if not isinstance(value, str) or not value or value != value.strip(): - raise BlockedError(f"{field} must be a non-empty trimmed string") - return value - - -def validate_positive_decimal_string(value: Any, field: str) -> Decimal: - if not isinstance(value, str) or value != value.strip(): - raise BlockedError(f"{field} must be a positive decimal string") - return positive_decimal(value, field) - - -def validate_decimal_string( - value: Any, - field: str, - *, - minimum: Decimal, - maximum: Decimal, -) -> Decimal: - if not isinstance(value, str) or value != value.strip(): - raise BlockedError(f"{field} must be a decimal string") - result = decimal_value(value, field) - if result < minimum or result > maximum: - raise BlockedError(f"{field} must be between {minimum} and {maximum}") - return result - - -def validate_nonnegative_integer(value: Any, field: str) -> int: - if isinstance(value, bool) or not isinstance(value, int) or value < 0: - raise BlockedError(f"{field} must be an integer >= 0") - return value - - -def validate_v2_selector(selector: Any) -> dict[str, Any]: - if not isinstance(selector, dict): - raise BlockedError("screen.selector must be an object") - exact_fields(selector, V2_SELECTOR_FIELDS, "screen.selector") - if selector["selector_version"] != V2_SELECTOR_VERSION: - raise BlockedError( - f"screen.selector.selector_version must be {V2_SELECTOR_VERSION}" - ) - validate_decimal_string( - selector["min_total_score"], - "screen.selector.min_total_score", - minimum=Decimal(0), - maximum=Decimal(100), - ) - validate_decimal_string( - selector["max_blocked_fraction"], - "screen.selector.max_blocked_fraction", - minimum=Decimal(0), - maximum=Decimal(1), - ) - statuses = selector["eligible_setup_statuses"] - if ( - not isinstance(statuses, list) - or not statuses - or any( - not isinstance(status, str) or not status or status != status.strip() - for status in statuses - ) - ): - raise BlockedError( - "screen.selector.eligible_setup_statuses must be a non-empty " - "trimmed string array" - ) - if statuses != sorted(set(statuses)): - raise BlockedError( - "screen.selector.eligible_setup_statuses must be sorted and unique" - ) - if not set(statuses).issubset(QTA_SETUP_STATUSES): - raise BlockedError( - "screen.selector.eligible_setup_statuses must be a non-empty " - "subset of READY and CONDITIONAL" - ) - exchanges = set(V2_EXCHANGE_CONTRACTS) - for field in ("top_k_by_exchange", "min_selected_by_exchange"): - mapping = selector[field] - if not isinstance(mapping, dict) or set(mapping) != exchanges: - raise BlockedError( - f"screen.selector.{field} must contain exactly " - + ", ".join(sorted(exchanges)) - ) - for exchange in sorted(exchanges): - top_k = validate_nonnegative_integer( - selector["top_k_by_exchange"][exchange], - f"screen.selector.top_k_by_exchange.{exchange}", - ) - minimum_selected = validate_nonnegative_integer( - selector["min_selected_by_exchange"][exchange], - f"screen.selector.min_selected_by_exchange.{exchange}", - ) - if minimum_selected > top_k: - raise BlockedError( - "screen.selector.min_selected_by_exchange." - f"{exchange} must be <= top_k_by_exchange.{exchange}" - ) - return selector - - -def validate_json_number( - value: Any, - field: str, - *, - minimum: Decimal, - maximum: Decimal | None = None, -) -> Decimal: - if isinstance(value, bool) or not isinstance(value, (int, float)): - raise BlockedError(f"{field} must be a JSON number") - parsed = decimal_value(value, field) - if parsed < minimum or (maximum is not None and parsed > maximum): - upper = "" if maximum is None else f" and <= {maximum}" - raise BlockedError(f"{field} must be >= {minimum}{upper}") - return parsed - - -def qta_allowed_opinions(score: Decimal) -> set[str]: - boundary_labels = { - Decimal(20): {"강한 부정", "부정"}, - Decimal(40): {"부정", "중립"}, - Decimal(60): {"중립", "긍정"}, - Decimal(80): {"긍정", "강한 긍정"}, - } - if score in boundary_labels: - return boundary_labels[score] - if score >= Decimal(80): - return {"강한 긍정"} - if score >= Decimal(60): - return {"긍정"} - if score >= Decimal(40): - return {"중립"} - if score >= Decimal(20): - return {"부정"} - return {"강한 부정"} - - -def validate_v2_qta_identity( - qta: dict[str, Any], - instrument: dict[str, Any], - label: str, -) -> None: - if ( - qta.get("source_skill") != "quant-stock-technical" - or qta.get("result_schema") != "quant-stock-technical/v1" - or qta.get("method_version") not in SUPPORTED_QTA_METHODS - ): - raise BlockedError(f"{label} provenance is invalid") - if qta.get("market") != instrument["market"]: - raise BlockedError(f"{label}.market must equal instrument.market") - if qta.get("ticker") != instrument["canonical_symbol"]: - raise BlockedError(f"{label}.ticker must equal instrument.canonical_symbol") - - -def validate_v2_ready_qta( - qta: dict[str, Any], - instrument: dict[str, Any], - label: str, - *, - screen_analysis_date: date, -) -> None: - method_version = qta.get("method_version") - exact_fields( - qta, - QTA2_READY_FIELDS - if method_version == QTA_METHOD_V2 - else QTA_READY_FIELDS, - label, - ) - validate_v2_qta_identity(qta, instrument, label) - if qta["calculation_status"] != "READY": - raise BlockedError(f"{label}.calculation_status must be READY") - qta_analysis_date = parse_iso_date(qta["analysis_date"], f"{label}.analysis_date") - if qta_analysis_date != screen_analysis_date: - raise BlockedError(f"{label}.analysis_date must equal screen.analysis_date") - source_name = validate_nonempty_string(qta["source_name"], f"{label}.source_name") - if source_name != instrument["source_name"]: - raise BlockedError(f"{label}.source_name must equal instrument.source_name") - shared_sessions = qta["shared_sessions"] - if ( - isinstance(shared_sessions, bool) - or not isinstance(shared_sessions, int) - or shared_sessions < QTA_MIN_SHARED_SESSIONS - ): - raise BlockedError( - f"{label}.shared_sessions must be an integer >= {QTA_MIN_SHARED_SESSIONS}" - ) - expected_score_basis = ( - QTA2_SCORE_BASIS - if method_version == QTA_METHOD_V2 - else QTA_SCORE_BASIS - ) - if qta["score_basis"] != expected_score_basis: - raise BlockedError(f"{label}.score_basis is invalid") - - horizon_scores: dict[str, Decimal] = {} - for horizon in ("short", "medium", "long"): - payload = qta[horizon] - horizon_label = f"{label}.{horizon}" - if not isinstance(payload, dict): - raise BlockedError(f"{horizon_label} must be an object") - exact_fields(payload, QTA_HORIZON_FIELDS, horizon_label) - horizon_score = validate_json_number( - payload["score"], - f"{horizon_label}.score", - minimum=Decimal(0), - maximum=Decimal(100), - ) - horizon_scores[horizon] = horizon_score - if payload["opinion"] not in qta_allowed_opinions(horizon_score): - raise BlockedError(f"{horizon_label}.opinion does not match its score") - - risk = qta["risk"] - if not isinstance(risk, dict): - raise BlockedError(f"{label}.risk must be an object") - exact_fields(risk, QTA_RISK_FIELDS, f"{label}.risk") - risk_score = validate_json_number( - risk["score"], - f"{label}.risk.score", - minimum=Decimal(0), - maximum=Decimal(100), - ) - validate_nonempty_string(risk["counterpoint"], f"{label}.risk.counterpoint") - - entry = validate_json_number( - qta["entry_price"], - f"{label}.entry_price", - minimum=Decimal(0), - ) - stop = validate_json_number( - qta["stop_price"], - f"{label}.stop_price", - minimum=Decimal(0), - ) - take_profit = validate_json_number( - qta["take_profit_price"], - f"{label}.take_profit_price", - minimum=Decimal(0), - ) - if stop <= 0 or not stop < entry < take_profit: - raise BlockedError( - f"{label} prices must satisfy stop_price < entry_price < take_profit_price" - ) - tick = validate_positive_decimal_string( - instrument["tick_contract"]["resolved_tick_size"], - f"{label}.instrument_tick_size", - ) - for price, field in ( - (entry, "entry_price"), - (stop, "stop_price"), - (take_profit, "take_profit_price"), - ): - if price % tick != 0: - raise BlockedError(f"{label}.{field} must align to the resolved tick") - expected_take_profit = entry + Decimal(2) * (entry - stop) - if take_profit != expected_take_profit: - raise BlockedError( - f"{label}.take_profit_price must equal the " - f"{method_version} 2R target" - ) - total_score = validate_json_number( - qta["total_score"], - f"{label}.total_score", - minimum=Decimal(0), - maximum=Decimal(100), - ) - if method_version == QTA_METHOD_V2: - recomputed_total = ( - Decimal("0.50") * horizon_scores["short"] - + Decimal("0.30") * horizon_scores["medium"] - + Decimal("0.20") * (Decimal(100) - risk_score) - ) - else: - recomputed_total = ( - Decimal("0.25") * horizon_scores["short"] - + Decimal("0.35") * horizon_scores["medium"] - + Decimal("0.40") * horizon_scores["long"] - - Decimal("0.20") * max(risk_score - Decimal(50), Decimal(0)) - ) - recomputed_total = min(Decimal(100), max(Decimal(0), recomputed_total)) - if abs(total_score - recomputed_total) > Decimal("0.02"): - raise BlockedError( - f"{label}.total_score does not match the {method_version} score formula" - ) - liquidity_ready = True - if method_version == QTA_METHOD_V2: - if qta["validation_status"] != "RESEARCH_ONLY": - raise BlockedError( - f"{label}.validation_status must be RESEARCH_ONLY" - ) - liquidity = qta["liquidity"] - if not isinstance(liquidity, dict): - raise BlockedError(f"{label}.liquidity must be an object") - exact_fields( - liquidity, - QTA2_LIQUIDITY_FIELDS, - f"{label}.liquidity", - ) - median_turnover = validate_json_number( - liquidity["median_20_session_turnover"], - f"{label}.liquidity.median_20_session_turnover", - minimum=Decimal(0), - ) - minimum_turnover = validate_json_number( - liquidity["minimum_turnover"], - f"{label}.liquidity.minimum_turnover", - minimum=Decimal(0), - ) - expected_currency = MARKET_CURRENCY[instrument["market"]] - if liquidity["currency"] != expected_currency: - raise BlockedError( - f"{label}.liquidity.currency must be {expected_currency}" - ) - expected_floor = QTA2_LIQUIDITY_FLOOR[instrument["market"]] - if minimum_turnover != expected_floor: - raise BlockedError( - f"{label}.liquidity.minimum_turnover does not match QTA 2.0" - ) - liquidity_ready = median_turnover >= minimum_turnover - expected_liquidity_status = "READY" if liquidity_ready else "BLOCKED" - if liquidity["status"] != expected_liquidity_status: - raise BlockedError( - f"{label}.liquidity.status does not match turnover" - ) - market_regime = qta["market_regime"] - if not isinstance(market_regime, dict): - raise BlockedError(f"{label}.market_regime must be an object") - exact_fields( - market_regime, - QTA2_MARKET_REGIME_FIELDS, - f"{label}.market_regime", - ) - if market_regime["metric"] != QTA2_REGIME_METRIC: - raise BlockedError(f"{label}.market_regime.metric is invalid") - minimum_change = validate_json_number( - market_regime["minimum_change_bps"], - f"{label}.market_regime.minimum_change_bps", - minimum=Decimal("-10000"), - maximum=Decimal("10000"), - ) - if minimum_change != QTA2_REGIME_MINIMUM_CHANGE_BPS: - raise BlockedError( - f"{label}.market_regime.minimum_change_bps does not match " - "QTA 2.0" - ) - if ( - isinstance(market_regime["max_age_seconds"], bool) - or market_regime["max_age_seconds"] - != QTA2_REGIME_MAX_AGE_SECONDS - ): - raise BlockedError( - f"{label}.market_regime.max_age_seconds does not match QTA 2.0" - ) - if market_regime["status"] != "REQUIRED": - raise BlockedError( - f"{label}.market_regime.status must be REQUIRED" - ) - allowed_setup_statuses = ( - {"READY", "CONDITIONAL"} - if total_score == Decimal(60) - else {"READY"} - if total_score > Decimal(60) - else {"CONDITIONAL"} - ) - if not liquidity_ready: - allowed_setup_statuses = {"CONDITIONAL"} - if qta["setup_status"] not in allowed_setup_statuses: - raise BlockedError(f"{label}.setup_status does not match total_score") - - references = qta["reference_observations"] - if not isinstance(references, dict): - raise BlockedError(f"{label}.reference_observations must be an object") - exact_fields( - references, - QTA_REFERENCE_FIELDS, - f"{label}.reference_observations", - ) - maximum_reference_count = min( - QTA_MAX_REFERENCE_OBSERVATIONS, - shared_sessions - 1, - ) - for feature in sorted(QTA_REFERENCE_FIELDS): - count = references[feature] - if ( - isinstance(count, bool) - or not isinstance(count, int) - or count < QTA_MIN_REFERENCE_OBSERVATIONS - or count > maximum_reference_count - ): - raise BlockedError( - f"{label}.reference_observations.{feature} must be an integer " - f"between {QTA_MIN_REFERENCE_OBSERVATIONS} and " - f"{maximum_reference_count}" - ) - expected_assumptions = ( - QTA2_ASSUMPTIONS - if method_version == QTA_METHOD_V2 - else QTA_ASSUMPTIONS - ) - if qta["assumptions"] != expected_assumptions: - raise BlockedError( - f"{label}.assumptions do not match {method_version}" - ) - - -def validate_v2_blocked_qta( - qta: dict[str, Any], - instrument: dict[str, Any], - label: str, -) -> None: - exact_fields(qta, QTA_BLOCKED_FIELDS, label) - validate_v2_qta_identity(qta, instrument, label) - if qta["calculation_status"] != "BLOCKED": - raise BlockedError(f"{label}.calculation_status must be BLOCKED") - validate_nonempty_string(qta["reason"], f"{label}.reason") - - -def validate_v2_qta( - qta: dict[str, Any], - instrument: dict[str, Any], - label: str, - *, - screen_analysis_date: date, - require_ready: bool, -) -> None: - calculation_status = qta.get("calculation_status") - if calculation_status == "READY": - validate_v2_ready_qta( - qta, - instrument, - label, - screen_analysis_date=screen_analysis_date, - ) - return - if calculation_status == "BLOCKED" and not require_ready: - validate_v2_blocked_qta(qta, instrument, label) - return - expected = "READY" if require_ready else "READY or BLOCKED" - raise BlockedError(f"{label}.calculation_status must be {expected}") - - -def qta_ranking_key(qta: dict[str, Any]) -> tuple[Any, ...]: - if qta.get("method_version") == QTA_METHOD_V2: - return ( - -decimal_value(qta["total_score"], "qta.total_score"), - -decimal_value(qta["short"]["score"], "qta.short.score"), - -decimal_value(qta["medium"]["score"], "qta.medium.score"), - decimal_value(qta["risk"]["score"], "qta.risk.score"), - -decimal_value(qta["long"]["score"], "qta.long.score"), - str(qta["market"]), - str(qta["ticker"]), - ) - return ( - -decimal_value(qta["total_score"], "qta.total_score"), - -decimal_value(qta["medium"]["score"], "qta.medium.score"), - -decimal_value(qta["long"]["score"], "qta.long.score"), - -decimal_value(qta["short"]["score"], "qta.short.score"), - decimal_value(qta["risk"]["score"], "qta.risk.score"), - str(qta["market"]), - str(qta["ticker"]), - ) - - -def validate_v2_instrument( - instrument: dict[str, Any], - exchange: str, - label: str, - *, - analysis_date: date | None = None, -) -> dict[str, Any]: - exact_fields(instrument, V2_INSTRUMENT_FIELDS, label) - contract = V2_EXCHANGE_CONTRACTS[exchange] - for field in ("market", "benchmark_id", "currency", "venue"): - if instrument[field] != contract[field]: - raise BlockedError( - f"{label}.{field} must be {contract[field]} for {exchange}" - ) - if instrument["exchange"] != exchange: - raise BlockedError(f"{label}.exchange must be {exchange}") - if instrument["instrument_type"] not in V2_INSTRUMENT_TYPES: - raise BlockedError(f"{label}.instrument_type must be COMMON, ADR, or REIT") - market = contract["market"] - canonical_symbol = normalize_symbol(market, str(instrument["canonical_symbol"])) - broker_symbol = normalize_symbol(market, str(instrument["broker_symbol"])) - if canonical_symbol != instrument["canonical_symbol"]: - raise BlockedError(f"{label}.canonical_symbol must be normalized") - if broker_symbol != instrument["broker_symbol"]: - raise BlockedError(f"{label}.broker_symbol must be normalized") - validate_nonempty_string(instrument["data_symbol"], f"{label}.data_symbol") - for field in ( - "source_name", - "official_source_id", - "broker_source_id", - ): - validate_nonempty_string(instrument[field], f"{label}.{field}") - for field in ("ticker_csv", "benchmark_csv"): - path = validate_nonempty_string(instrument[field], f"{label}.{field}") - if not Path(path).is_absolute(): - raise BlockedError(f"{label}.{field} must be an absolute path") - if not isinstance(instrument["broker_tradability_verified"], bool): - raise BlockedError(f"{label}.broker_tradability_verified must be boolean") - if not instrument["broker_tradability_verified"]: - raise BlockedError(f"{label} is not broker-tradability verified") - validate_sha256(instrument["ticker_csv_sha256"], f"{label}.ticker_csv_sha256") - validate_sha256( - instrument["benchmark_csv_sha256"], - f"{label}.benchmark_csv_sha256", - ) - tick_contract = instrument["tick_contract"] - if not isinstance(tick_contract, dict): - raise BlockedError(f"{label}.tick_contract must be an object") - exact_fields(tick_contract, V2_TICK_CONTRACT_FIELDS, f"{label}.tick_contract") - if tick_contract["schema"] != "qta-tick-contract/v1": - raise BlockedError(f"{label}.tick_contract.schema must be qta-tick-contract/v1") - if tick_contract["kind"] != "RESOLVED_PRICE_LADDER": - raise BlockedError(f"{label}.tick_contract.kind must be RESOLVED_PRICE_LADDER") - validate_nonempty_string( - tick_contract["rule_id"], - f"{label}.tick_contract.rule_id", - ) - effective_date = validate_nonempty_string( - tick_contract["effective_date"], - f"{label}.tick_contract.effective_date", - ) - try: - parsed_effective_date = date.fromisoformat(effective_date) - except ValueError as exc: - raise BlockedError( - f"{label}.tick_contract.effective_date must be YYYY-MM-DD" - ) from exc - if parsed_effective_date.isoformat() != effective_date: - raise BlockedError(f"{label}.tick_contract.effective_date must be YYYY-MM-DD") - if analysis_date is not None and parsed_effective_date > analysis_date: - raise BlockedError( - f"{label}.tick_contract.effective_date must not be after analysis_date" - ) - validate_positive_decimal_string( - tick_contract["reference_price"], - f"{label}.tick_contract.reference_price", - ) - validate_positive_decimal_string( - tick_contract["resolved_tick_size"], - f"{label}.tick_contract.resolved_tick_size", - ) - return instrument - - -def validate_v2_selected( - selected: dict[str, Any], - *, - analysis_date: date, - method_version: str, -) -> dict[tuple[str, str], dict[str, Any]]: - expected_exchanges = set(V2_EXCHANGE_CONTRACTS) - if set(selected) != expected_exchanges: - raise BlockedError( - "qta-screen/v2 selected must contain exactly " - + ", ".join(sorted(expected_exchanges)) - ) - seen: dict[tuple[str, str], dict[str, Any]] = {} - for exchange in sorted(expected_exchanges): - candidates = selected[exchange] - if not isinstance(candidates, list): - raise BlockedError(f"screen.selected.{exchange} must be an array") - ranks: list[int] = [] - for index, selected_item in enumerate(candidates): - label = f"screen.selected.{exchange}[{index}]" - if not isinstance(selected_item, dict): - raise BlockedError(f"{label} must be an object") - exact_fields( - selected_item, - {"exchange_rank", "instrument", "qta"}, - label, - ) - exchange_rank = selected_item["exchange_rank"] - if ( - isinstance(exchange_rank, bool) - or not isinstance(exchange_rank, int) - or exchange_rank <= 0 - ): - raise BlockedError(f"{label}.exchange_rank must be a positive integer") - ranks.append(exchange_rank) - instrument = selected_item["instrument"] - qta = selected_item["qta"] - if not isinstance(instrument, dict) or not isinstance(qta, dict): - raise BlockedError(f"{label}.instrument and qta must be objects") - validate_v2_instrument( - instrument, - exchange, - f"{label}.instrument", - analysis_date=analysis_date, - ) - expected_market = V2_EXCHANGE_CONTRACTS[exchange]["market"] - if instrument["market"] != expected_market: - raise BlockedError( - f"{label}.instrument.market must be {expected_market}" - ) - validate_v2_qta( - qta, - instrument, - f"{label}.qta", - screen_analysis_date=analysis_date, - require_ready=True, - ) - if qta["method_version"] != method_version: - raise BlockedError( - f"{label}.qta method differs from screen.method_version" - ) - key = (exchange, instrument["broker_symbol"]) - if key in seen: - raise BlockedError( - f"duplicate selected broker symbol: {exchange}:{key[1]}" - ) - seen[key] = selected_item - if ranks != list(range(1, len(ranks) + 1)): - raise BlockedError( - f"screen.selected.{exchange} exchange_rank values must be " - "ordered and contiguous" - ) - return seen - - -def validate_v2_screen(screen: dict[str, Any]) -> None: - exact_fields(screen, V2_SCREEN_FIELDS, "qta-screen/v2") - if screen["source_skill"] != "quant-stock-technical": - raise BlockedError("qta-screen/v2 source_skill must be quant-stock-technical") - if screen["schema"] != SCREEN_SCHEMA_V2: - raise BlockedError(f"qta-screen/v2 schema must be {SCREEN_SCHEMA_V2}") - if screen["screen_status"] != "READY": - raise BlockedError("screen_status must be READY") - if screen["method_version"] not in SUPPORTED_QTA_METHODS: - raise BlockedError( - "method_version must be qta-1.0.0 or qta-2.0.0" - ) - if screen["selector_version"] != V2_SELECTOR_VERSION: - raise BlockedError(f"selector_version must be {V2_SELECTOR_VERSION}") - validate_sha256(screen["screen_hash"], "screen_hash") - if screen["screen_hash"] != canonical_screen_hash(screen): - raise BlockedError("screen_hash does not match canonical screen payload") - - analysis_date = parse_iso_date(screen["analysis_date"], "screen.analysis_date") - validate_sha256(screen["manifest_hash"], "screen.manifest_hash") - validate_sha256(screen["selector_hash"], "screen.selector_hash") - selector = validate_v2_selector(screen["selector"]) - if selector["selector_version"] != screen["selector_version"]: - raise BlockedError("screen selector versions are inconsistent") - if screen["selector_hash"] != sha256_json(selector): - raise BlockedError("selector_hash does not match screen.selector") - - blocked_count = validate_nonnegative_integer( - screen["blocked_count"], "screen.blocked_count" - ) - instrument_count = validate_nonnegative_integer( - screen["instrument_count"], "screen.instrument_count" - ) - if instrument_count <= 0: - raise BlockedError("screen.instrument_count must be positive") - if blocked_count > instrument_count: - raise BlockedError("screen.blocked_count must not exceed instrument_count") - blocked_fraction = validate_decimal_string( - screen["blocked_fraction"], - "screen.blocked_fraction", - minimum=Decimal(0), - maximum=Decimal(1), - ) - expected_blocked_fraction = Decimal(blocked_count) / Decimal(instrument_count) - if screen["blocked_fraction"] != format(expected_blocked_fraction, "f"): - raise BlockedError( - "screen.blocked_fraction does not match blocked_count/instrument_count" - ) - if blocked_fraction > Decimal(selector["max_blocked_fraction"]): - raise BlockedError("READY screen exceeds selector.max_blocked_fraction") - - selected = screen["selected"] - if not isinstance(selected, dict): - raise BlockedError("screen.selected must be an object") - selected_by_key = validate_v2_selected( - selected, - analysis_date=analysis_date, - method_version=screen["method_version"], - ) - decisions = screen["decisions"] - if not isinstance(decisions, list): - raise BlockedError("screen.decisions must be an array") - if len(decisions) != instrument_count: - raise BlockedError("screen.decisions length must equal instrument_count") - - statuses = set(selector["eligible_setup_statuses"]) - minimum_score = Decimal(selector["min_total_score"]) - decision_by_key: dict[tuple[str, str], dict[str, Any]] = {} - eligible_decisions: dict[str, list[dict[str, Any]]] = { - exchange: [] for exchange in V2_EXCHANGE_CONTRACTS - } - computed_blocked_count = 0 - for index, decision in enumerate(decisions): - label = f"screen.decisions[{index}]" - if not isinstance(decision, dict): - raise BlockedError(f"{label} must be an object") - exact_fields(decision, V2_DECISION_FIELDS, label) - exchange = decision["exchange"] - if exchange not in V2_EXCHANGE_CONTRACTS: - raise BlockedError(f"{label}.exchange is unsupported") - instrument = decision["instrument"] - qta = decision["qta"] - if not isinstance(instrument, dict) or not isinstance(qta, dict): - raise BlockedError(f"{label}.instrument and qta must be objects") - validate_v2_instrument( - instrument, - exchange, - f"{label}.instrument", - analysis_date=analysis_date, - ) - if decision["market"] != instrument["market"]: - raise BlockedError(f"{label}.market must equal instrument.market") - if decision["canonical_symbol"] != instrument["canonical_symbol"]: - raise BlockedError( - f"{label}.canonical_symbol must equal instrument.canonical_symbol" - ) - validate_v2_qta( - qta, - instrument, - f"{label}.qta", - screen_analysis_date=analysis_date, - require_ready=False, - ) - if qta["method_version"] != screen["method_version"]: - raise BlockedError( - f"{label}.qta method differs from screen.method_version" - ) - calculation_ready = qta["calculation_status"] == "READY" - if not calculation_ready: - computed_blocked_count += 1 - expected_reasons: list[str] = [] - if not calculation_ready: - expected_reasons.append("calculation_not_ready") - else: - if qta.get("setup_status") not in statuses: - expected_reasons.append("setup_status_ineligible") - if ( - decimal_value(qta.get("total_score"), f"{label}.qta.total_score") - < minimum_score - ): - expected_reasons.append("score_below_minimum") - reasons = decision["reasons"] - if ( - not isinstance(reasons, list) - or any(not isinstance(reason, str) for reason in reasons) - or reasons != expected_reasons - ): - raise BlockedError(f"{label}.reasons do not match the selector decision") - eligible = decision["eligible"] - selected_flag = decision["selected"] - if not isinstance(eligible, bool) or not isinstance(selected_flag, bool): - raise BlockedError(f"{label}.eligible and selected must be boolean") - if eligible != (not expected_reasons): - raise BlockedError(f"{label}.eligible does not match reasons") - exchange_rank = decision["exchange_rank"] - if eligible: - if ( - isinstance(exchange_rank, bool) - or not isinstance(exchange_rank, int) - or exchange_rank <= 0 - ): - raise BlockedError( - f"{label}.exchange_rank must be positive for an eligible decision" - ) - eligible_decisions[exchange].append(decision) - elif exchange_rank is not None: - raise BlockedError( - f"{label}.exchange_rank must be null for an ineligible decision" - ) - - key = (exchange, instrument["broker_symbol"]) - if key in decision_by_key: - raise BlockedError(f"duplicate decision broker symbol: {exchange}:{key[1]}") - decision_by_key[key] = decision - selected_item = selected_by_key.get(key) - if selected_flag != (selected_item is not None): - raise BlockedError(f"{label}.selected disagrees with screen.selected") - if selected_item is not None and ( - selected_item["exchange_rank"] != exchange_rank - or selected_item["instrument"] != instrument - or selected_item["qta"] != qta - ): - raise BlockedError( - f"{label} does not exactly match its screen.selected item" - ) - - if computed_blocked_count != blocked_count: - raise BlockedError("screen.blocked_count does not match calculation decisions") - if set(selected_by_key) - set(decision_by_key): - raise BlockedError("screen.selected contains an item absent from decisions") - for exchange in sorted(V2_EXCHANGE_CONTRACTS): - ranked = sorted( - eligible_decisions[exchange], - key=lambda decision: qta_ranking_key(decision["qta"]), - ) - for expected_rank, decision in enumerate(ranked, start=1): - if decision["exchange_rank"] != expected_rank: - raise BlockedError( - f"screen.decisions {exchange} exchange_rank does not match " - "qta ranking_key" - ) - minimum_selected = selector["min_selected_by_exchange"][exchange] - if len(ranked) < minimum_selected: - raise BlockedError( - f"READY screen does not meet minimum selection for {exchange}" - ) - expected_selected_count = min( - selector["top_k_by_exchange"][exchange], - len(ranked), - ) - if len(selected[exchange]) != expected_selected_count: - raise BlockedError( - f"screen.selected.{exchange} count does not match selector" - ) - expected_selected = ranked[:expected_selected_count] - for selected_item, expected_decision in zip( - selected[exchange], - expected_selected, - ): - if ( - selected_item["exchange_rank"] != expected_decision["exchange_rank"] - or selected_item["instrument"] != expected_decision["instrument"] - or selected_item["qta"] != expected_decision["qta"] - ): - raise BlockedError( - f"screen.selected.{exchange} order does not match qta ranking_key" - ) - - -def selected_for_execution( - screen: dict[str, Any], - market: str, -) -> list[dict[str, Any]]: - if screen["schema"] == SCREEN_SCHEMA: - selected = screen["selected"].get(market) - if not isinstance(selected, list): - raise BlockedError(f"screen.selected.{market} must be an array") - return sorted(selected, key=lambda item: int(item["rank"])) - - candidates: list[dict[str, Any]] = [] - for exchange_order, exchange in enumerate(V2_EXCHANGES_BY_MARKET[market], start=1): - for selected_item in screen["selected"][exchange]: - candidates.append( - { - **selected_item, - "_exchange": exchange, - "_exchange_order": exchange_order, - } - ) - candidates.sort( - key=lambda item: ( - item["_exchange_order"], - int(item["exchange_rank"]), - item["instrument"]["broker_symbol"], - ) - ) - return [ - {**item, "_execution_rank": index} - for index, item in enumerate(candidates, start=1) - ] - - -def round_down_to_tick(value: Decimal, tick: Decimal) -> Decimal: - if tick <= 0: - raise BlockedError("tick size must be positive") - return (value / tick).to_integral_value(rounding=ROUND_DOWN) * tick - - -def whole_quantity(value: Decimal) -> int: - return int(value.to_integral_value(rounding=ROUND_DOWN)) - - -def existing_exposure_keys( - exposure: dict[str, Any], -) -> set[tuple[str, str]]: - if exposure["schema"] == EXPOSURE_SCHEMA: - return { - (position["market"], position["symbol"]) - for position in exposure["positions"] - if decimal_value(position["market_value_krw"], "market_value_krw") > 0 - or ( - position["quantity"] is not None - and decimal_value(position["quantity"], "quantity") > 0 - ) - } - return { - (position["exchange"], position["symbol"]) - for position in exposure["positions"] - if decimal_value(position["market_value_krw"], "market_value_krw") > 0 - or ( - position["quantity"] is not None - and decimal_value(position["quantity"], "quantity") > 0 - ) - } - - -def account_position_keys( - account: dict[str, Any], - screen_schema: str, -) -> set[tuple[str, str]]: - if screen_schema == SCREEN_SCHEMA: - return { - (position["market"], position["symbol"]) - for position in account["positions"] - } - return { - (position["exchange"], position["symbol"]) for position in account["positions"] - } - - -def account_open_buy_keys( - account: dict[str, Any], - screen_schema: str, -) -> set[tuple[str, str]]: - open_buys = [ - open_order - for open_order in account["open_orders"] - if open_order["side"] == "BUY" - ] - if screen_schema == SCREEN_SCHEMA: - return { - (open_order["market"], open_order["symbol"]) for open_order in open_buys - } - return {(open_order["exchange"], open_order["symbol"]) for open_order in open_buys} - - -def validate_temporal_snapshots( - account: dict[str, Any], - exposure: dict[str, Any], - execution: dict[str, Any], -) -> None: - zone = ZoneInfo(execution["timezone"]) - window_start = parse_aware_datetime( - execution["entry_window_start"], "entry_window_start" - ) - market_session = execution["market_session"] - session_date = parse_iso_date( - market_session["session_date"], - "market_session.session_date", - ) - if window_start.astimezone(zone).date() != session_date: - raise BlockedError( - "entry_window_start must fall on market_session.session_date" - ) - account_as_of = parse_aware_datetime(account["as_of"], "account as_of") - exposure_as_of = parse_aware_datetime(exposure["as_of"], "exposure as_of") - if account_as_of != exposure_as_of: - raise BlockedError( - "account and exposure snapshots must share one frozen as_of instant" - ) - snapshot_local = account_as_of.astimezone(zone) - if snapshot_local.date() != session_date: - raise BlockedError( - "account and exposure snapshots must be from the entry session local date" - ) - if account_as_of > window_start: - raise BlockedError( - "account and exposure snapshots must be frozen no later than " - "entry_window_start" - ) - source_as_of = parse_aware_datetime( - market_session["source_as_of"], - "market_session.source_as_of", - ) - if source_as_of > account_as_of: - raise BlockedError( - "market_session.source_as_of must not be after the frozen snapshots" - ) - snapshot_age_seconds = (window_start - account_as_of).total_seconds() - if snapshot_age_seconds > execution["snapshot_max_age_seconds"]: - raise BlockedError("account snapshots exceed snapshot_max_age_seconds") - - -def validate_v2_analysis_date( - screen: dict[str, Any], - execution: dict[str, Any], -) -> None: - market_session = execution["market_session"] - analysis_date = parse_iso_date(screen["analysis_date"], "screen.analysis_date") - previous_session_date = parse_iso_date( - market_session["previous_session_date"], - "market_session.previous_session_date", - ) - if analysis_date != previous_session_date: - raise BlockedError( - "qta-screen/v2 analysis_date must equal " - "market_session.previous_session_date" - ) - - -def plan_orders( - screen_raw: dict[str, Any], - account_raw: dict[str, Any], - exposure_raw: dict[str, Any], - risk_raw: dict[str, Any], - execution_raw: dict[str, Any], -) -> dict[str, Any]: - validate_screen(screen_raw) - screen_schema = screen_raw["schema"] - account = normalized_account_snapshot( - account_raw, - screen_schema=screen_schema, - ) - exposure = normalized_exposure_snapshot(exposure_raw) - risk = normalized_risk_policy(risk_raw) - execution = normalized_execution_policy(execution_raw) - market = execution["market"] - if account["market"] != market: - raise BlockedError("account and execution markets differ") - expected_exposure_schema = ( - EXPOSURE_SCHEMA if screen_schema == SCREEN_SCHEMA else EXPOSURE_SCHEMA_V2 - ) - if exposure["schema"] != expected_exposure_schema: - raise BlockedError( - f"{screen_schema} requires exposure schema {expected_exposure_schema}" - ) - validate_temporal_snapshots(account, exposure, execution) - if screen_schema == SCREEN_SCHEMA_V2: - if account["broker"] != "kis": - raise BlockedError( - "qta-screen/v2 broker_symbol is KIS-qualified and requires " - "account broker kis; Toss requires a separate Toss-qualified " - "universe snapshot contract" - ) - validate_v2_analysis_date(screen_raw, execution) - selected = selected_for_execution(screen_raw, market) - - settled_cash = Decimal(account["settled_cash"]) - raw_fx = account["fx_to_krw"] - if raw_fx is None: - if settled_cash != 0: - raise BlockedError("a positive settled cash balance requires fx_to_krw") - per_trade_risk_native = Decimal(0) - max_notional_native = Decimal(0) - else: - fx = Decimal(raw_fx) - per_trade_risk_native = Decimal(risk["per_trade_risk_krw"]) / fx - max_notional_native = Decimal(risk["max_symbol_notional_krw"]) / fx - cost_bps = Decimal(risk["round_trip_cost_bps"]) - cash_buffer_bps = Decimal(risk["cash_buffer_bps"]) - gap_bps = Decimal(execution["max_gap_bps"]) - position_keys = existing_exposure_keys(exposure) - position_keys.update(account_position_keys(account, screen_schema)) - open_buy_keys = account_open_buy_keys(account, screen_schema) - occupied_keys = position_keys | open_buy_keys - - context = { - "execution_version": EXECUTION_VERSION, - "screen_hash": canonical_screen_hash(screen_raw), - "account_hash": sha256_json(account), - "exposure_hash": sha256_json(exposure), - "risk_hash": sha256_json(risk), - "execution_policy_hash": sha256_json(execution), - "execution_policy": execution, - "market_session": execution["market_session"], - "market_session_hash": execution["market_session"]["session_hash"], - "snapshot_max_age_seconds": execution["snapshot_max_age_seconds"], - "broker": account["broker"], - "environment": account["environment"], - "account_alias": account["account_alias"], - "market": market, - } - if screen_schema == SCREEN_SCHEMA_V2: - context.update( - { - "screen_schema": SCREEN_SCHEMA_V2, - "candidate_order_contract": ( - "exchange_contract_order_then_exchange_rank_then_broker_symbol" - ), - "exchange_order": list(V2_EXCHANGES_BY_MARKET[market]), - "broker_symbol_qualification": "KIS", - "analysis_date": screen_raw["analysis_date"], - "snapshot_as_of": account["as_of"], - } - ) - plan_seed = sha256_json(context) - intents: list[dict[str, Any]] = [] - skipped: list[dict[str, Any]] = [] - planned_keys: set[tuple[str, str]] = set() - cash_remaining = settled_cash - - for selected_item in selected: - is_v2 = screen_schema == SCREEN_SCHEMA_V2 - qta = selected_item.get("qta") - instrument = selected_item.get("instrument") - if not isinstance(qta, dict) or not isinstance(instrument, dict): - raise BlockedError("selected item must contain qta and instrument objects") - exchange = selected_item["_exchange"] if is_v2 else None - rank = ( - int(selected_item["_execution_rank"]) - if is_v2 - else int(selected_item["rank"]) - ) - symbol = normalize_symbol( - market, - str(instrument["broker_symbol"] if is_v2 else qta["ticker"]), - ) - if ( - qta.get("source_skill") != "quant-stock-technical" - or qta.get("result_schema") != "quant-stock-technical/v1" - or qta.get("calculation_status") != "READY" - or qta.get("method_version") != screen_raw["method_version"] - ): - raise BlockedError("selected QTA payload contract is invalid") - if is_v2: - if qta.get("setup_status") not in set( - screen_raw["selector"]["eligible_setup_statuses"] - ): - raise BlockedError( - "selected QTA setup_status is not allowed by the v2 selector" - ) - if qta.get("market") != market: - raise BlockedError("selected QTA market differs from execution market") - if qta.get("ticker") != instrument["canonical_symbol"]: - raise BlockedError("selected QTA ticker differs from canonical_symbol") - key = (str(exchange), symbol) - else: - if qta.get("setup_status") != "READY": - raise BlockedError("legacy selected QTA setup_status must be READY") - symbol = normalize_symbol(market, str(qta["ticker"])) - key = (market, symbol) - if key in planned_keys: - skipped_item = { - "market": market, - "symbol": symbol, - "reason": "duplicate_planned_exposure", - } - if is_v2: - skipped_item.update( - { - "exchange": exchange, - "exchange_rank": int(selected_item["exchange_rank"]), - "canonical_symbol": instrument["canonical_symbol"], - } - ) - skipped.append(skipped_item) - continue - if key in open_buy_keys: - skipped_item = { - "market": market, - "symbol": symbol, - "reason": "existing_open_buy_order", - } - if is_v2: - skipped_item.update( - { - "exchange": exchange, - "exchange_rank": int(selected_item["exchange_rank"]), - "canonical_symbol": instrument["canonical_symbol"], - } - ) - skipped.append(skipped_item) - continue - if key in position_keys and not risk["allow_existing_additions"]: - skipped_item = { - "market": market, - "symbol": symbol, - "reason": "existing_exposure", - } - if is_v2: - skipped_item.update( - { - "exchange": exchange, - "exchange_rank": int(selected_item["exchange_rank"]), - "canonical_symbol": instrument["canonical_symbol"], - } - ) - skipped.append(skipped_item) - continue - if ( - key not in occupied_keys - and len(occupied_keys | planned_keys) >= risk["max_concurrent_positions"] - ): - skipped_item = { - "market": market, - "symbol": symbol, - "reason": "max_concurrent_positions", - } - if is_v2: - skipped_item.update( - { - "exchange": exchange, - "exchange_rank": int(selected_item["exchange_rank"]), - "canonical_symbol": instrument["canonical_symbol"], - } - ) - skipped.append(skipped_item) - continue - entry = positive_decimal(qta["entry_price"], f"{symbol}.entry_price") - stop = positive_decimal(qta["stop_price"], f"{symbol}.stop_price") - if stop >= entry: - raise BlockedError(f"{symbol} stop must be below entry") - tick_value = ( - instrument["tick_contract"]["resolved_tick_size"] - if is_v2 - else instrument["tick_size"] - ) - tick = positive_decimal(tick_value, f"{symbol}.tick_size") - limit_price = round_down_to_tick( - entry * (Decimal(1) + gap_bps / Decimal(10000)), tick - ) - limit_price = max(limit_price, entry) - effective_loss = limit_price - stop + limit_price * cost_bps / Decimal(10000) - if effective_loss <= 0: - raise BlockedError(f"{symbol} effective loss per share is non-positive") - risk_quantity = whole_quantity(per_trade_risk_native / effective_loss) - notional_quantity = whole_quantity(max_notional_native / limit_price) - worst_case_unit = limit_price * (Decimal(1) + cash_buffer_bps / Decimal(10000)) - cash_quantity = whole_quantity(cash_remaining / worst_case_unit) - quantity = min(risk_quantity, notional_quantity, cash_quantity) - if quantity <= 0: - skipped.append( - {"market": market, "symbol": symbol, "reason": "quantity_below_one"} - ) - continue - reserved_cash = worst_case_unit * Decimal(quantity) - intent_seed = { - "plan_seed": plan_seed, - "rank": rank, - "market": market, - "symbol": symbol, - "side": "BUY", - "order_type": "LIMIT", - "time_in_force": "DAY", - "quantity": str(quantity), - "limit_price": format(limit_price, "f"), - } - if is_v2: - intent_seed.update( - { - "exchange": exchange, - "exchange_rank": int(selected_item["exchange_rank"]), - "canonical_symbol": instrument["canonical_symbol"], - "data_symbol": instrument["data_symbol"], - "broker_symbol": instrument["broker_symbol"], - "venue": instrument["venue"], - } - ) - intent_id = hashlib.sha256( - canonical_json(intent_seed).encode("utf-8") - ).hexdigest()[:32] - client_order_id = f"qta-{intent_id[:28]}" - intent = { - "intent_id": intent_id, - "client_order_id": client_order_id, - "rank": rank, - "market": market, - "symbol": symbol, - "currency": account["currency"], - "side": "BUY", - "order_type": "LIMIT", - "time_in_force": "DAY", - "quantity": str(quantity), - "entry_trigger": format(entry, "f"), - "limit_price": format(limit_price, "f"), - "stop_price": format(stop, "f"), - "take_profit_price": format( - positive_decimal( - qta["take_profit_price"], f"{symbol}.take_profit_price" - ), - "f", - ), - "effective_loss_per_share": format(effective_loss, "f"), - "reserved_cash": format(reserved_cash, "f"), - "qta_payload_hash": sha256_json(qta), - "initial_state": "PLANNED", - } - if is_v2: - intent.update( - { - "exchange": exchange, - "exchange_rank": int(selected_item["exchange_rank"]), - "canonical_symbol": instrument["canonical_symbol"], - "data_symbol": instrument["data_symbol"], - "broker_symbol": instrument["broker_symbol"], - "instrument_type": instrument["instrument_type"], - "benchmark_id": instrument["benchmark_id"], - "venue": instrument["venue"], - "resolved_tick_size": format(tick, "f"), - "tick_contract_hash": sha256_json(instrument["tick_contract"]), - } - ) - intent["intent_hash"] = sha256_json(intent) - intents.append(intent) - planned_keys.add(key) - cash_remaining -= reserved_cash - - output = { - "schema": PLAN_SCHEMA, - "plan_status": "READY" if intents else "NO_ORDERS", - "execution_version": EXECUTION_VERSION, - "context": context, - "entry_window": { - "timezone": execution["timezone"], - "start": execution["entry_window_start"], - "end": execution["entry_window_end"], - "poll_interval_seconds": execution["poll_interval_seconds"], - }, - "quote_policy": { - "max_age_seconds": execution["quote_max_age_seconds"], - "max_spread_bps": execution["max_spread_bps"], - "max_gap_bps": execution["max_gap_bps"], - "trigger_mode": execution["trigger_mode"], - }, - "order_policy": { - "ttl_seconds": execution["order_ttl_seconds"], - "allow_partial_fill": execution["allow_partial_fill"], - "cancel_remainder_at_window_end": execution[ - "cancel_remainder_at_window_end" - ], - }, - "settled_cash_start": format(settled_cash, "f"), - "borrowed_buying_power_excluded": account["borrowed_buying_power"], - "settled_cash_unreserved": format(cash_remaining, "f"), - "intents": intents, - "skipped": skipped, - "frozen_inputs": { - "screen": screen_raw, - "account": account, - "exposure": exposure, - "risk": risk, - "execution": execution, - }, - } - output["plan_hash"] = sha256_json(output) - return output - - -@dataclass(frozen=True) -class IntentRecord: - intent_id: str - state: str - broker_order_id: str | None - request_hash: str | None - payload: dict[str, Any] - - -class Ledger: - """SQLite ledger that persists intent state before broker mutation.""" - - def __init__(self, path: str | Path): - self.path = Path(path) - self.path.parent.mkdir(parents=True, exist_ok=True) - self.connection = sqlite3.connect(str(self.path)) - self.connection.execute("PRAGMA journal_mode=WAL") - self.connection.execute("PRAGMA synchronous=FULL") - self.connection.executescript( - """ - CREATE TABLE IF NOT EXISTS intents ( - intent_id TEXT PRIMARY KEY, - plan_hash TEXT NOT NULL, - state TEXT NOT NULL, - broker_order_id TEXT, - request_hash TEXT, - payload_json TEXT NOT NULL, - updated_at TEXT NOT NULL - ); - CREATE TABLE IF NOT EXISTS events ( - sequence INTEGER PRIMARY KEY AUTOINCREMENT, - intent_id TEXT NOT NULL, - event_type TEXT NOT NULL, - payload_json TEXT NOT NULL, - created_at TEXT NOT NULL - ); - """ - ) - self.connection.commit() - - def close(self) -> None: - self.connection.close() - - @contextmanager - def transaction(self) -> Iterator[sqlite3.Connection]: - self.connection.execute("BEGIN IMMEDIATE") - try: - yield self.connection - except BaseException: - self.connection.rollback() - raise - else: - self.connection.commit() - - def create_intent(self, plan_hash: str, intent: dict[str, Any]) -> None: - intent_id = str(intent["intent_id"]) - now = datetime.now(timezone.utc).isoformat() - payload_json = canonical_json(intent) - with self.transaction() as connection: - existing = connection.execute( - "SELECT plan_hash, payload_json FROM intents WHERE intent_id = ?", - (intent_id,), - ).fetchone() - if existing: - if existing != (plan_hash, payload_json): - raise BlockedError(f"intent collision: {intent_id}") - return - connection.execute( - """ - INSERT INTO intents - (intent_id, plan_hash, state, payload_json, updated_at) - VALUES (?, ?, 'PLANNED', ?, ?) - """, - (intent_id, plan_hash, payload_json, now), - ) - connection.execute( - """ - INSERT INTO events (intent_id, event_type, payload_json, created_at) - VALUES (?, 'INTENT_CREATED', ?, ?) - """, - (intent_id, canonical_json({"plan_hash": plan_hash}), now), - ) - - def get(self, intent_id: str) -> IntentRecord: - row = self.connection.execute( - """ - SELECT intent_id, state, broker_order_id, request_hash, payload_json - FROM intents WHERE intent_id = ? - """, - (intent_id,), - ).fetchone() - if row is None: - raise BlockedError(f"unknown intent: {intent_id}") - return IntentRecord( - intent_id=row[0], - state=row[1], - broker_order_id=row[2], - request_hash=row[3], - payload=json.loads(row[4]), - ) - - def transition( - self, - intent_id: str, - new_state: str, - event_payload: dict[str, Any], - *, - broker_order_id: str | None = None, - request_hash: str | None = None, - ) -> None: - current = self.get(intent_id) - allowed = TRANSITIONS.get(current.state, set()) - if new_state not in allowed: - raise BlockedError( - f"illegal intent transition: {current.state} -> {new_state}" - ) - now = datetime.now(timezone.utc).isoformat() - next_broker_id = broker_order_id or current.broker_order_id - next_request_hash = request_hash or current.request_hash - with self.transaction() as connection: - changed = connection.execute( - """ - UPDATE intents - SET state = ?, broker_order_id = ?, request_hash = ?, updated_at = ? - WHERE intent_id = ? AND state = ? - """, - ( - new_state, - next_broker_id, - next_request_hash, - now, - intent_id, - current.state, - ), - ).rowcount - if changed != 1: - raise BlockedError(f"concurrent intent mutation: {intent_id}") - connection.execute( - """ - INSERT INTO events (intent_id, event_type, payload_json, created_at) - VALUES (?, ?, ?, ?) - """, - ( - intent_id, - f"STATE_{new_state}", - canonical_json(event_payload), - now, - ), - ) - - def append_event( - self, intent_id: str, event_type: str, event_payload: dict[str, Any] - ) -> None: - self.append_events([(intent_id, event_type, event_payload)]) - - def append_events( - self, - events: list[tuple[str, str, dict[str, Any]]], - ) -> None: - """Persist non-mutation telemetry in one FULL-synchronous transaction.""" - if not events: - return - for intent_id, event_type, event_payload in events: - self.get(intent_id) - if not isinstance(event_type, str) or not event_type: - raise BlockedError("event_type must be a non-empty string") - if not isinstance(event_payload, dict): - raise BlockedError("event payload must be an object") - now = datetime.now(timezone.utc).isoformat() - with self.transaction() as connection: - connection.executemany( - """ - INSERT INTO events (intent_id, event_type, payload_json, created_at) - VALUES (?, ?, ?, ?) - """, - [ - ( - intent_id, - event_type, - canonical_json(event_payload), - now, - ) - for intent_id, event_type, event_payload in events - ], - ) - - def nonterminal(self) -> list[IntentRecord]: - rows = self.connection.execute( - """ - SELECT intent_id, state, broker_order_id, request_hash, payload_json - FROM intents ORDER BY intent_id - """ - ).fetchall() - return [ - IntentRecord(row[0], row[1], row[2], row[3], json.loads(row[4])) - for row in rows - if row[1] not in TERMINAL_STATES - ] - - def nonterminal_bindings(self) -> list[dict[str, str]]: - rows = self.connection.execute( - """ - SELECT intent_id, plan_hash, state - FROM intents ORDER BY intent_id - """ - ).fetchall() - return [ - { - "intent_id": str(row[0]), - "plan_hash": str(row[1]), - "state": str(row[2]), - } - for row in rows - if row[2] not in TERMINAL_STATES or row[2] == "MANUAL_BLOCK" - ] - - def events(self, intent_id: str) -> list[dict[str, Any]]: - rows = self.connection.execute( - """ - SELECT sequence, event_type, payload_json, created_at - FROM events WHERE intent_id = ? ORDER BY sequence - """, - (intent_id,), - ).fetchall() - return [ - { - "sequence": row[0], - "event_type": row[1], - "payload": json.loads(row[2]), - "created_at": row[3], - } - for row in rows - ] - - -class SingleWriterLock: - """Non-blocking process lock for one account/session state directory.""" - - def __init__(self, path: str | Path): - self.path = Path(path) - self.handle: Any = None - - def __enter__(self): - self.path.parent.mkdir(parents=True, exist_ok=True) - self.handle = self.path.open("a+", encoding="utf-8") - try: - fcntl.flock(self.handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) - except BlockingIOError as exc: - self.handle.close() - self.handle = None - raise BlockedError(f"another writer holds {self.path}") from exc - return self - - def __exit__( - self, - exc_type: object, - exc: object, - traceback: object, - ) -> None: - if self.handle is not None: - fcntl.flock(self.handle.fileno(), fcntl.LOCK_UN) - self.handle.close() - self.handle = None - - -def redact(value: Any) -> Any: - secret_fragments = ( - "secret", - "token", - "authorization", - "appkey", - "app_key", - "password", - "account_number", - "accountnumber", - "accountno", - "account_prefix", - "accountprefix", - "account_product", - "accountproduct", - "account_seq", - "accountseq", - "cano", - "acnt_no", - "acnt_prdt_cd", - "x-tossinvest-account", - ) - if isinstance(value, dict): - result: dict[str, Any] = {} - for key, item in value.items(): - lowered = key.lower() - result[key] = ( - "[REDACTED]" - if any(fragment in lowered for fragment in secret_fragments) - else redact(item) - ) - return result - if isinstance(value, list): - return [redact(item) for item in value] - return value - - -def self_test() -> None: - qta_payload = { - "source_skill": "quant-stock-technical", - "result_schema": "quant-stock-technical/v1", - "calculation_status": "READY", - "setup_status": "READY", - "method_version": "qta-1.0.0", - "ticker": "NEW", - "entry_price": 100, - "stop_price": 90, - "take_profit_price": 120, - } - screen = { - "schema": SCREEN_SCHEMA, - "screen_status": "READY", - "method_version": "qta-1.0.0", - "selector_version": "qta-screen-1.0.0", - "selected": { - "KR": [], - "US": [ - { - "rank": 1, - "instrument": { - "market": "US", - "ticker": "NEW", - "tick_size": "0.01", - }, - "qta": qta_payload, - } - ], - }, - } - screen["screen_hash"] = canonical_screen_hash(screen) - account = { - "schema": ACCOUNT_SCHEMA, - "broker": "kis", - "environment": "paper", - "account_alias": "paper-us", - "market": "US", - "currency": "USD", - "as_of": "2026-07-27T09:00:00-04:00", - "settled_cash": "200", - "borrowed_buying_power": "5000", - "fx_to_krw": "1400", - "positions": [], - "open_orders": [], - } - exposure = { - "schema": EXPOSURE_SCHEMA, - "as_of": "2026-07-27T09:00:00-04:00", - "positions": [], - } - risk = { - "schema": RISK_SCHEMA, - "base_currency": "KRW", - "per_trade_risk_krw": "15000", - "max_symbol_notional_krw": "280000", - "max_concurrent_positions": 1, - "max_daily_loss_krw": "28000", - "round_trip_cost_bps": "10", - "cash_buffer_bps": "100", - "allow_existing_additions": False, - "allow_borrowed_cash": False, - "allow_margin": False, - "allow_short": False, - "allow_auto_fx": False, - "whole_shares_only": True, - } - execution = { - "schema": EXECUTION_POLICY_SCHEMA, - "market": "US", - "timezone": "America/New_York", - "entry_window_start": "2026-07-27T09:30:00-04:00", - "entry_window_end": "2026-07-27T10:30:00-04:00", - "market_session": market_session_from_source( - Path(__file__).resolve().parents[1] - / "references" - / "fixtures" - / "us-market-session-2026-07-27.json" - ), - "snapshot_max_age_seconds": 1800, - "poll_interval_seconds": 2, - "quote_max_age_seconds": 5, - "max_spread_bps": "25", - "max_gap_bps": "20", - "trigger_mode": "AT_OR_ABOVE", - "order_ttl_seconds": 30, - "order_type": "LIMIT", - "time_in_force": "DAY", - "allow_partial_fill": True, - "cancel_remainder_at_window_end": True, - } - first = plan_orders(screen, account, exposure, risk, execution) - second = plan_orders(screen, account, exposure, risk, execution) - assert first == second - assert first["plan_status"] == "READY" - assert first["borrowed_buying_power_excluded"] == "5000" - assert len(first["intents"]) == 1 - assert first["intents"][0]["quantity"] == "1" - - empty_account = { - **account, - "settled_cash": "0", - "fx_to_krw": None, - } - empty_plan = plan_orders(screen, empty_account, exposure, risk, execution) - assert empty_plan["plan_status"] == "NO_ORDERS" - assert empty_plan["frozen_inputs"]["account"]["fx_to_krw"] is None - assert empty_plan["skipped"][0]["reason"] == "quantity_below_one" - - held = json.loads(canonical_json(exposure)) - held["positions"] = [ - { - "broker": "toss", - "market": "US", - "symbol": "NEW", - "quantity": "0.5", - "market_value_krw": "70000", - } - ] - no_orders = plan_orders(screen, account, held, risk, execution) - assert no_orders["plan_status"] == "NO_ORDERS" - assert no_orders["skipped"][0]["reason"] == "existing_exposure" - - with tempfile.TemporaryDirectory(prefix="qta-ledger-") as directory: - ledger = Ledger(Path(directory) / "ledger.sqlite3") - intent = first["intents"][0] - ledger.create_intent(first["plan_hash"], intent) - ledger.create_intent(first["plan_hash"], intent) - ledger.transition(intent["intent_id"], "WAIT_TRIGGER", {}) - ledger.transition(intent["intent_id"], "RESERVED", {}) - ledger.transition( - intent["intent_id"], - "SUBMITTING", - {"request_hash": "b" * 64}, - request_hash="b" * 64, - ) - ledger.transition(intent["intent_id"], "UNKNOWN", {"reason": "timeout"}) - ledger.transition(intent["intent_id"], "RECONCILING", {}) - ledger.transition(intent["intent_id"], "MANUAL_BLOCK", {"reason": "ambiguous"}) - assert ledger.get(intent["intent_id"]).state == "MANUAL_BLOCK" - assert len(ledger.events(intent["intent_id"])) == 7 - ledger.close() - - redacted = redact( - { - "authorization": "Bearer secret", - "nested": {"appsecret": "secret", "safe": "ok"}, - } - ) - assert redacted["authorization"] == "[REDACTED]" - assert redacted["nested"]["appsecret"] == "[REDACTED]" - assert redacted["nested"]["safe"] == "ok" - print( - canonical_json( - { - "self_test": "PASS", - "execution_version": EXECUTION_VERSION, - "plan_hash": first["plan_hash"], - } - ) - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--self-test", action="store_true") - return parser.parse_args() - - -def main() -> int: - args = parse_args() - if args.self_test: - self_test() - return 0 - emit_json( - { - "schema": PLAN_SCHEMA, - "plan_status": "BLOCKED", - "reason": "use plan_orders.py or pass --self-test", - } - ) - return 2 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/freeze_market_session.py b/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/freeze_market_session.py deleted file mode 100644 index b3116e7..0000000 --- a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/freeze_market_session.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python3 -"""Hash and validate one frozen market-session source JSON file.""" - -from __future__ import annotations - -import argparse -import sys - -from execution_core import ( - ENTRY_WINDOWS, - BlockedError, - emit_json, - market_session_from_source, - normalized_market_session, -) - - -def build(source: str) -> dict[str, object]: - session = market_session_from_source(source) - market = session.get("market") - if market not in ENTRY_WINDOWS: - raise BlockedError("market-session source market must be KR or US") - return normalized_market_session( - session, - expected_market=market, - expected_timezone=ENTRY_WINDOWS[market]["timezone"], - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--source") - parser.add_argument("--output") - parser.add_argument("--self-test", action="store_true") - return parser.parse_args() - - -def main() -> int: - args = parse_args() - source = args.source - if args.self_test: - from pathlib import Path - - source = str( - Path(__file__).resolve().parents[1] - / "references" - / "fixtures" - / "kr-market-session-2026-07-27.json" - ) - if source is None: - raise BlockedError("--source is required") - result = build(source) - if args.self_test: - result = { - "self_test": "PASS", - "schema": result["schema"], - "session_hash": result["session_hash"], - } - emit_json(result, args.output) - return 0 - - -if __name__ == "__main__": - try: - sys.exit(main()) - except (BlockedError, OSError, ValueError) as exc: - emit_json({"status": "BLOCKED", "reason": str(exc)}) - sys.exit(2) diff --git a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/market_calendar.py b/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/market_calendar.py deleted file mode 100644 index 0b25f50..0000000 --- a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/market_calendar.py +++ /dev/null @@ -1,670 +0,0 @@ -#!/usr/bin/env python3 -"""Freeze an official KR or US equity-market calendar snapshot.""" - -from __future__ import annotations - -import argparse -import hashlib -import html -import json -import os -import re -import sys -import tempfile -from collections.abc import Callable -from datetime import date, datetime, time, timedelta -from html.parser import HTMLParser -from pathlib import Path -from typing import Any -from urllib.parse import urlencode -from urllib.request import Request, urlopen -from zoneinfo import ZoneInfo - -from execution_core import ( - ENTRY_WINDOWS, - BlockedError, - market_session_from_source, - normalized_market_session, -) - -RECEIPT_SCHEMA = "qta-market-calendar-receipt/v1" -KRX_HOLIDAY_PAGE = ( - "https://global.krx.co.kr/contents/GLB/05/0501/0501110000/" - "GLB0501110000.jsp" -) -KRX_HOURS_PAGE = ( - "https://global.krx.co.kr/contents/GLB/06/0602/0602020204/" - "GLB0602020204T1.jsp" -) -KRX_OTP_URL = "https://global.krx.co.kr/contents/COM/GenerateOTP.jspx" -KRX_DATA_URL = "https://global.krx.co.kr/contents/GLB/99/GLB99000001.jspx" -KRX_BLD = "GLB/05/0501/0501110000/glb0501110000_01" -NYSE_CALENDAR_URL = "https://www.nyse.com/markets/hours-calendars" -NASDAQ_CALENDAR_URL = "https://www.nasdaqtrader.com/Trader.aspx?id=calendar" -USER_AGENT = "qta-market-calendar/1.0 (+official read-only snapshot)" -MONTH_PATTERN = ( - r"January|February|March|April|May|June|July|August|" - r"September|October|November|December" -) -FULL_DATE = re.compile( - rf"\b({MONTH_PATTERN})\s+(\d{{1,2}}),\s+(\d{{4}})\b", - re.IGNORECASE, -) -MONTH_DAY = re.compile( - rf"\b({MONTH_PATTERN})\s+(\d{{1,2}})\b", - re.IGNORECASE, -) - - -class CalendarBlockedError(BlockedError): - """Raised when official calendar evidence cannot support one session.""" - - -class CalendarHtmlParser(HTMLParser): - """Collect text rows and block text without executing page scripts.""" - - def __init__(self) -> None: - super().__init__(convert_charrefs=True) - self.rows: list[list[str]] = [] - self.blocks: list[str] = [] - self._row: list[str] | None = None - self._cell: list[str] | None = None - self._block: list[str] | None = None - self._block_depth = 0 - - def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: - del attrs - tag = tag.lower() - if tag == "tr": - self._row = [] - elif tag in {"th", "td"} and self._row is not None: - self._cell = [] - if tag in {"p", "li"}: - if self._block is None: - self._block = [] - self._block_depth = 1 - else: - self._block_depth += 1 - - def handle_endtag(self, tag: str) -> None: - tag = tag.lower() - if tag in {"th", "td"} and self._row is not None and self._cell is not None: - self._row.append(normalize_text(" ".join(self._cell))) - self._cell = None - elif tag == "tr" and self._row is not None: - if any(self._row): - self.rows.append(self._row) - self._row = None - self._cell = None - if tag in {"p", "li"} and self._block is not None: - self._block_depth -= 1 - if self._block_depth == 0: - text = normalize_text(" ".join(self._block)) - if text: - self.blocks.append(text) - self._block = None - - def handle_data(self, data: str) -> None: - if self._cell is not None: - self._cell.append(data) - if self._block is not None: - self._block.append(data) - - -def normalize_text(value: str) -> str: - return " ".join(html.unescape(value).replace("\xa0", " ").split()) - - -def canonical_json(value: Any) -> str: - return json.dumps( - value, - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - allow_nan=False, - ) - - -def sha256_bytes(payload: bytes) -> str: - return hashlib.sha256(payload).hexdigest() - - -def sha256_file(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def atomic_write(path: Path, payload: bytes) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - descriptor, temporary = tempfile.mkstemp( - prefix=f".{path.name}.", suffix=".tmp", dir=str(path.parent) - ) - temporary_path = Path(temporary) - try: - with os.fdopen(descriptor, "wb") as handle: - handle.write(payload) - handle.flush() - os.fsync(handle.fileno()) - os.replace(temporary_path, path) - os.chmod(path, 0o600) - finally: - if temporary_path.exists(): - temporary_path.unlink() - - -def atomic_write_json(path: Path, value: Any) -> None: - atomic_write( - path, - (json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode( - "utf-8" - ), - ) - - -def fetch_bytes(url: str, data: bytes | None = None) -> bytes: - headers = { - "User-Agent": USER_AGENT, - "Accept": "application/json,text/html,application/xhtml+xml", - } - if data is not None: - headers["Content-Type"] = "application/x-www-form-urlencoded" - request = Request(url, data=data, headers=headers, method="POST" if data else "GET") - with urlopen(request, timeout=30) as response: - status = getattr(response, "status", 200) - if status != 200: - raise CalendarBlockedError(f"official calendar HTTP status {status}: {url}") - payload = response.read() - if not payload: - raise CalendarBlockedError(f"official calendar response is empty: {url}") - return payload - - -def parse_iso_date(value: str, label: str) -> date: - try: - return date.fromisoformat(value) - except (TypeError, ValueError) as exc: - raise CalendarBlockedError(f"{label} must be YYYY-MM-DD") from exc - - -def parsed_html(payload: bytes, label: str) -> CalendarHtmlParser: - try: - text = payload.decode("utf-8") - except UnicodeDecodeError as exc: - raise CalendarBlockedError(f"{label} is not UTF-8") from exc - parser = CalendarHtmlParser() - parser.feed(text) - parser.close() - if not parser.rows: - raise CalendarBlockedError(f"{label} contains no calendar table rows") - return parser - - -def full_date_from_match(match: re.Match[str]) -> date: - return datetime.strptime( - f"{match.group(1)} {match.group(2)}, {match.group(3)}", - "%B %d, %Y", - ).date() - - -def parse_nasdaq_calendar(payload: bytes, year: int) -> tuple[set[date], set[date]]: - parser = parsed_html(payload, "Nasdaq calendar") - closed: set[date] = set() - early: set[date] = set() - for row in parser.rows: - if len(row) < 3: - continue - match = FULL_DATE.search(row[0]) - if match is None: - continue - session = full_date_from_match(match) - if session.year != year: - continue - status = row[-1].lower() - if status == "closed": - closed.add(session) - elif re.fullmatch(r"1:00\s*p\.?m\.?", status): - early.add(session) - else: - raise CalendarBlockedError( - f"Nasdaq calendar has unknown status for {session}: {row[-1]}" - ) - if not closed: - raise CalendarBlockedError(f"Nasdaq calendar has no closed dates for {year}") - return closed, early - - -def parse_nyse_calendar(payload: bytes, year: int) -> tuple[set[date], set[date]]: - parser = parsed_html(payload, "NYSE calendar") - header: list[str] | None = None - closed: set[date] = set() - for row in parser.rows: - if "Holiday" in row and str(year) in row: - header = row - continue - if header is None or len(row) < len(header): - continue - year_index = header.index(str(year)) - value = row[year_index] - match = MONTH_DAY.search(value) - if match is None: - continue - closed.add( - datetime.strptime( - f"{match.group(1)} {match.group(2)}, {year}", - "%B %d, %Y", - ).date() - ) - early: set[date] = set() - for block in parser.blocks: - lowered = block.lower() - if "close early at 1:00 p.m." not in lowered: - continue - for match in FULL_DATE.finditer(block): - parsed = full_date_from_match(match) - if parsed.year == year: - early.add(parsed) - if not closed: - raise CalendarBlockedError(f"NYSE calendar has no closed dates for {year}") - if "Core Trading Session: 9:30 a.m. to 4:00 p.m. ET" not in normalize_text( - payload.decode("utf-8") - ): - raise CalendarBlockedError("NYSE core trading-hour contract is missing") - return closed, early - - -def parse_krx_holidays(payload: bytes, year: int) -> set[date]: - try: - decoded = json.loads(payload) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise CalendarBlockedError("KRX holiday payload is not valid JSON") from exc - if not isinstance(decoded, dict) or set(decoded) != {"block1"}: - raise CalendarBlockedError("KRX holiday payload fields changed") - rows = decoded["block1"] - if not isinstance(rows, list) or not rows: - raise CalendarBlockedError(f"KRX holiday payload is empty for {year}") - holidays: set[date] = set() - for index, row in enumerate(rows): - if not isinstance(row, dict) or "calnd_dd" not in row: - raise CalendarBlockedError(f"KRX holiday row {index} is malformed") - parsed = parse_iso_date(row["calnd_dd"], f"KRX holiday row {index}.calnd_dd") - if parsed.year != year: - raise CalendarBlockedError("KRX holiday payload contains another year") - holidays.add(parsed) - return holidays - - -def validate_krx_hours(payload: bytes) -> None: - text = normalize_text(payload.decode("utf-8")) - if "Regular market session" not in text or "09:00 - 15:30" not in text: - raise CalendarBlockedError("KRX regular trading-hour contract is missing") - - -def validate_krx_holiday_page(payload: bytes) -> None: - try: - text = payload.decode("utf-8") - except UnicodeDecodeError as exc: - raise CalendarBlockedError("KRX holiday page is not UTF-8") from exc - if KRX_BLD.lower() not in text.lower() or "search_bas_yy" not in text: - raise CalendarBlockedError("KRX holiday-page request contract is missing") - - -def previous_open_session(session: date, closed: set[date]) -> date: - candidate = session - timedelta(days=1) - for _ in range(14): - if candidate.weekday() < 5 and candidate not in closed: - return candidate - candidate -= timedelta(days=1) - raise CalendarBlockedError("official calendar cannot resolve previous session") - - -def source_record(path: Path, source_id: str) -> dict[str, Any]: - return { - "source_id": source_id, - "path": str(path.resolve()), - "sha256": sha256_file(path), - "bytes": path.stat().st_size, - } - - -def krx_year_payload( - year: int, - fetcher: Callable[[str, bytes | None], bytes], -) -> bytes: - otp = fetcher(KRX_OTP_URL, urlencode({"bld": KRX_BLD}).encode("ascii")) - try: - otp_text = otp.decode("ascii").strip() - except UnicodeDecodeError as exc: - raise CalendarBlockedError("KRX OTP is not ASCII") from exc - if not otp_text or not re.fullmatch(r"[A-Za-z0-9+/=_-]+", otp_text): - raise CalendarBlockedError("KRX OTP format changed") - return fetcher( - KRX_DATA_URL, - urlencode( - {"search_bas_yy": str(year), "gridTp": "KRX", "code": otp_text} - ).encode("ascii"), - ) - - -def build_kr_snapshot( - *, - session: date, - now: datetime, - output_directory: Path, - fetcher: Callable[[str, bytes | None], bytes], -) -> tuple[dict[str, Any] | None, dict[str, Any]]: - holiday_page = fetcher(KRX_HOLIDAY_PAGE, None) - hours_page = fetcher(KRX_HOURS_PAGE, None) - validate_krx_holiday_page(holiday_page) - validate_krx_hours(hours_page) - years = {session.year, (session - timedelta(days=14)).year} - holiday_payloads = {year: krx_year_payload(year, fetcher) for year in years} - closed = set().union( - *(parse_krx_holidays(payload, year) for year, payload in holiday_payloads.items()) - ) - - raw_directory = output_directory / "official" - records: list[dict[str, Any]] = [] - for path, payload, source_id in ( - (raw_directory / "krx-holiday-page.html", holiday_page, KRX_HOLIDAY_PAGE), - (raw_directory / "krx-trading-hours.html", hours_page, KRX_HOURS_PAGE), - ): - atomic_write(path, payload) - records.append(source_record(path, source_id)) - for year, payload in sorted(holiday_payloads.items()): - path = raw_directory / f"krx-holidays-{year}.json" - atomic_write(path, payload) - records.append(source_record(path, f"{KRX_DATA_URL}#KRX-{year}")) - - is_open = session.weekday() < 5 and session not in closed - previous = previous_open_session(session, closed) - if not is_open: - return None, { - "market": "KR", - "session_date": session.isoformat(), - "previous_session_date": previous.isoformat(), - "scheduled_status": "MARKET_CLOSED", - "official_sources": sorted(records, key=lambda item: item["source_id"]), - } - zone = ZoneInfo("Asia/Seoul") - regular_open = datetime.combine(session, time(9, 0), zone) - regular_close = datetime.combine(session, time(15, 30), zone) - if now > regular_open: - raise CalendarBlockedError("KR calendar snapshot completed after regular open") - source = { - "schema": "qta-market-session/v1", - "provider": "KRX", - "source_id": KRX_HOLIDAY_PAGE, - "source_as_of": now.isoformat(), - "market": "KR", - "timezone": "Asia/Seoul", - "session_date": session.isoformat(), - "previous_session_date": previous.isoformat(), - "scheduled_status": "OPEN", - "regular_open": regular_open.isoformat(), - "regular_close": regular_close.isoformat(), - } - return source, { - "market": "KR", - "session_date": session.isoformat(), - "previous_session_date": previous.isoformat(), - "scheduled_status": "OPEN", - "official_sources": sorted(records, key=lambda item: item["source_id"]), - } - - -def build_us_snapshot( - *, - session: date, - now: datetime, - output_directory: Path, - fetcher: Callable[[str, bytes | None], bytes], -) -> tuple[dict[str, Any] | None, dict[str, Any]]: - nyse_payload = fetcher(NYSE_CALENDAR_URL, None) - nasdaq_payload = fetcher(NASDAQ_CALENDAR_URL, None) - years = {session.year, (session - timedelta(days=14)).year} - nyse_closed: set[date] = set() - nyse_early: set[date] = set() - for year in sorted(years): - year_nyse_closed, year_nyse_early = parse_nyse_calendar( - nyse_payload, year - ) - year_nasdaq_closed, year_nasdaq_early = parse_nasdaq_calendar( - nasdaq_payload, year - ) - if year_nyse_closed != year_nasdaq_closed: - raise CalendarBlockedError( - f"NYSE and Nasdaq official closed-date sets do not match for {year}" - ) - if year_nyse_early != year_nasdaq_early: - raise CalendarBlockedError( - f"NYSE and Nasdaq official early-close date sets do not match for {year}" - ) - nyse_closed.update(year_nyse_closed) - nyse_early.update(year_nyse_early) - raw_directory = output_directory / "official" - records: list[dict[str, Any]] = [] - for path, payload, source_id in ( - (raw_directory / "nyse-hours-calendar.html", nyse_payload, NYSE_CALENDAR_URL), - ( - raw_directory / "nasdaq-trader-calendar.html", - nasdaq_payload, - NASDAQ_CALENDAR_URL, - ), - ): - atomic_write(path, payload) - records.append(source_record(path, source_id)) - - is_open = session.weekday() < 5 and session not in nyse_closed - previous = previous_open_session(session, nyse_closed) - if not is_open: - return None, { - "market": "US", - "session_date": session.isoformat(), - "previous_session_date": previous.isoformat(), - "scheduled_status": "MARKET_CLOSED", - "official_sources": sorted(records, key=lambda item: item["source_id"]), - } - zone = ZoneInfo("America/New_York") - regular_open = datetime.combine(session, time(9, 30), zone) - close_clock = time(13, 0) if session in nyse_early else time(16, 0) - regular_close = datetime.combine(session, close_clock, zone) - if now > regular_open: - raise CalendarBlockedError("US calendar snapshot completed after regular open") - source = { - "schema": "qta-market-session/v1", - "provider": "NYSE+NASDAQ", - "source_id": f"{NYSE_CALENDAR_URL}|{NASDAQ_CALENDAR_URL}", - "source_as_of": now.isoformat(), - "market": "US", - "timezone": "America/New_York", - "session_date": session.isoformat(), - "previous_session_date": previous.isoformat(), - "scheduled_status": "OPEN", - "regular_open": regular_open.isoformat(), - "regular_close": regular_close.isoformat(), - } - return source, { - "market": "US", - "session_date": session.isoformat(), - "previous_session_date": previous.isoformat(), - "scheduled_status": "OPEN", - "official_sources": sorted(records, key=lambda item: item["source_id"]), - } - - -def snapshot( - *, - market: str, - session_date: str, - output_directory: Path, - now: datetime | None = None, - fetcher: Callable[[str, bytes | None], bytes] = fetch_bytes, -) -> dict[str, Any]: - normalized_market = market.upper() - if normalized_market not in ENTRY_WINDOWS: - raise CalendarBlockedError("market must be KR or US") - session = parse_iso_date(session_date, "session_date") - zone = ZoneInfo(ENTRY_WINDOWS[normalized_market]["timezone"]) - observed_now = (now or datetime.now(zone)).astimezone(zone).replace(microsecond=0) - if observed_now.date() != session: - raise CalendarBlockedError( - "session_date must equal the current date in the market timezone" - ) - if output_directory.exists() and ( - output_directory.is_symlink() or not output_directory.is_dir() - ): - raise CalendarBlockedError("output_directory must be a non-symlink directory") - output_directory.mkdir(parents=True, exist_ok=True) - - builder = build_kr_snapshot if normalized_market == "KR" else build_us_snapshot - source, evidence = builder( - session=session, - now=observed_now, - output_directory=output_directory, - fetcher=fetcher, - ) - base_receipt = { - "schema": RECEIPT_SCHEMA, - **evidence, - "source_as_of": observed_now.isoformat(), - "live_enabled": False, - "api_mutation_count": 0, - } - if source is None: - receipt_without_hash = { - **base_receipt, - "status": "MARKET_CLOSED", - "market_session_source_path": "", - "market_session_source_sha256": "", - "market_session_path": "", - "session_hash": "", - } - else: - source_path = output_directory / "market-session-source.json" - atomic_write_json(source_path, source) - bound = market_session_from_source(source_path) - bound = normalized_market_session( - bound, - expected_market=normalized_market, - expected_timezone=ENTRY_WINDOWS[normalized_market]["timezone"], - ) - bound_path = output_directory / "market-session.json" - atomic_write_json(bound_path, bound) - receipt_without_hash = { - **base_receipt, - "status": "READY", - "market_session_source_path": str(source_path.resolve()), - "market_session_source_sha256": sha256_file(source_path), - "market_session_path": str(bound_path.resolve()), - "session_hash": bound["session_hash"], - } - receipt = { - **receipt_without_hash, - "receipt_hash": sha256_bytes( - canonical_json(receipt_without_hash).encode("utf-8") - ), - } - atomic_write_json(output_directory / "calendar-receipt.json", receipt) - return receipt - - -def self_test() -> None: - nasdaq = b""" - - - -
2026HolidayStatus
January 1, 2026New YearClosed
November 27, 2026Early Close1:00 p.m.
- """ - nyse = b""" -

Core Trading Session: 9:30 a.m. to 4:00 p.m. ET

- -
Holiday2026
New YearThursday, January 1
-

Each market will close early at 1:00 p.m. on Friday, - November 27, 2026.

- """ - if parse_nasdaq_calendar(nasdaq, 2026) != parse_nyse_calendar(nyse, 2026): - raise AssertionError("US calendar parser parity failed") - krx = json.dumps( - { - "block1": [ - { - "calnd_dd": "2026-01-01", - "dy_tp_cd": "THU", - "calnd_dd_dy": "2026-01-01", - "kr_dy_tp": "Thursday", - "holdy_eng_nm": "New Year's Day", - } - ] - } - ).encode() - if parse_krx_holidays(krx, 2026) != {date(2026, 1, 1)}: - raise AssertionError("KRX calendar parser failed") - print( - json.dumps( - { - "self_test": "PASS", - "live_enabled": False, - "api_mutation_count": 0, - }, - sort_keys=True, - ) - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - subparsers = parser.add_subparsers(dest="command", required=True) - snapshot_parser = subparsers.add_parser("snapshot") - snapshot_parser.add_argument("--market", choices=("KR", "US"), required=True) - snapshot_parser.add_argument("--session-date", required=True) - snapshot_parser.add_argument("--output-directory", required=True) - subparsers.add_parser("self-test") - return parser.parse_args() - - -def main() -> int: - args = parse_args() - if args.command == "self-test": - self_test() - return 0 - output_directory = Path(args.output_directory).expanduser() - if not output_directory.is_absolute(): - print( - json.dumps( - {"status": "BLOCKED", "reason": "output_directory must be absolute"}, - sort_keys=True, - ), - file=sys.stderr, - ) - return 2 - try: - receipt = snapshot( - market=args.market, - session_date=args.session_date, - output_directory=output_directory, - ) - print(json.dumps(receipt, sort_keys=True)) - return 0 - except (CalendarBlockedError, OSError, ValueError) as exc: - blocked = { - "schema": RECEIPT_SCHEMA, - "status": "BLOCKED", - "market": args.market, - "session_date": args.session_date, - "reason": str(exc), - "live_enabled": False, - "api_mutation_count": 0, - } - try: - atomic_write_json(output_directory / "calendar-receipt.json", blocked) - except OSError: - pass - print(json.dumps(blocked, sort_keys=True), file=sys.stderr) - return 2 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/plan_orders.py b/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/plan_orders.py deleted file mode 100755 index 9ad8e56..0000000 --- a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/plan_orders.py +++ /dev/null @@ -1,217 +0,0 @@ -#!/usr/bin/env python3 -"""Create a side-effect-free deterministic order plan from frozen JSON inputs.""" - -from __future__ import annotations - -import argparse -import json -import sys -import tempfile -from pathlib import Path -from typing import Any - -from execution_core import ( - ACCOUNT_SCHEMA, - EXECUTION_POLICY_SCHEMA, - EXPOSURE_SCHEMA, - PLAN_SCHEMA, - RISK_SCHEMA, - SCREEN_SCHEMA, - BlockedError, - canonical_json, - canonical_screen_hash, - emit_json, - load_json_object, - market_session_from_source, - plan_orders, -) - - -def build_from_paths(args: argparse.Namespace) -> dict[str, Any]: - return plan_orders( - load_json_object(args.screen), - load_json_object(args.account), - load_json_object(args.exposure), - load_json_object(args.risk), - load_json_object(args.execution), - ) - - -def fixture_inputs() -> tuple[dict[str, Any], ...]: - qta = { - "source_skill": "quant-stock-technical", - "result_schema": "quant-stock-technical/v1", - "calculation_status": "READY", - "setup_status": "READY", - "method_version": "qta-1.0.0", - "ticker": "005930", - "entry_price": 70000, - "stop_price": 65000, - "take_profit_price": 80000, - } - screen = { - "schema": SCREEN_SCHEMA, - "screen_status": "READY", - "method_version": "qta-1.0.0", - "selector_version": "qta-screen-1.0.0", - "selected": { - "KR": [ - { - "rank": 1, - "instrument": { - "market": "KR", - "ticker": "005930", - "tick_size": "100", - }, - "qta": qta, - } - ], - "US": [], - }, - } - screen["screen_hash"] = canonical_screen_hash(screen) - account = { - "schema": ACCOUNT_SCHEMA, - "broker": "kis", - "environment": "paper", - "account_alias": "paper-kr", - "market": "KR", - "currency": "KRW", - "as_of": "2026-07-27T08:50:00+09:00", - "settled_cash": "200000", - "borrowed_buying_power": "5000000", - "fx_to_krw": "1", - "positions": [], - "open_orders": [], - } - exposure = { - "schema": EXPOSURE_SCHEMA, - "as_of": "2026-07-27T08:50:00+09:00", - "positions": [], - } - risk = { - "schema": RISK_SCHEMA, - "base_currency": "KRW", - "per_trade_risk_krw": "10000", - "max_symbol_notional_krw": "200000", - "max_concurrent_positions": 1, - "max_daily_loss_krw": "20000", - "round_trip_cost_bps": "20", - "cash_buffer_bps": "100", - "allow_existing_additions": False, - "allow_borrowed_cash": False, - "allow_margin": False, - "allow_short": False, - "allow_auto_fx": False, - "whole_shares_only": True, - } - execution = { - "schema": EXECUTION_POLICY_SCHEMA, - "market": "KR", - "timezone": "Asia/Seoul", - "entry_window_start": "2026-07-27T09:00:00+09:00", - "entry_window_end": "2026-07-27T10:00:00+09:00", - "market_session": market_session_from_source( - Path(__file__).resolve().parents[1] - / "references" - / "fixtures" - / "kr-market-session-2026-07-27.json" - ), - "snapshot_max_age_seconds": 1800, - "poll_interval_seconds": 2, - "quote_max_age_seconds": 5, - "max_spread_bps": "25", - "max_gap_bps": "20", - "trigger_mode": "AT_OR_ABOVE", - "order_ttl_seconds": 30, - "order_type": "LIMIT", - "time_in_force": "DAY", - "allow_partial_fill": True, - "cancel_remainder_at_window_end": True, - } - return screen, account, exposure, risk, execution - - -def self_test() -> None: - values = fixture_inputs() - direct = plan_orders(*values) - with tempfile.TemporaryDirectory(prefix="qta-plan-cli-") as directory: - names = ("screen", "account", "exposure", "risk", "execution") - paths: list[str] = [] - for name, value in zip(names, values): - path = Path(directory) / f"{name}.json" - path.write_text(canonical_json(value), encoding="utf-8") - paths.append(str(path)) - args = argparse.Namespace( - screen=paths[0], - account=paths[1], - exposure=paths[2], - risk=paths[3], - execution=paths[4], - ) - loaded = build_from_paths(args) - assert direct == loaded - assert direct["plan_status"] == "READY" - assert direct["borrowed_buying_power_excluded"] == "5000000" - assert direct["intents"][0]["symbol"] == "005930" - print( - canonical_json( - { - "self_test": "PASS", - "schema": PLAN_SCHEMA, - "plan_hash": direct["plan_hash"], - } - ) - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--screen") - parser.add_argument("--account") - parser.add_argument("--exposure") - parser.add_argument("--risk") - parser.add_argument("--execution") - parser.add_argument("--output") - parser.add_argument("--self-test", action="store_true") - return parser.parse_args() - - -def main() -> int: - args = parse_args() - if args.self_test: - self_test() - return 0 - missing = [ - name - for name in ("screen", "account", "exposure", "risk", "execution") - if not getattr(args, name) - ] - if missing: - emit_json( - { - "schema": PLAN_SCHEMA, - "plan_status": "BLOCKED", - "reason": "missing arguments: " + ", ".join(missing), - }, - args.output, - ) - return 2 - try: - plan = build_from_paths(args) - except (BlockedError, OSError, ValueError, json.JSONDecodeError) as exc: - emit_json( - { - "schema": PLAN_SCHEMA, - "plan_status": "BLOCKED", - "reason": str(exc), - }, - args.output, - ) - return 2 - emit_json(plan, args.output) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/position_lifecycle.py b/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/position_lifecycle.py deleted file mode 100644 index e60c987..0000000 --- a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/position_lifecycle.py +++ /dev/null @@ -1,1236 +0,0 @@ -#!/usr/bin/env python3 -"""Deterministic strategy-position lifecycle and exit decisions.""" - -from __future__ import annotations - -import argparse -import json -import os -import sqlite3 -import sys -import tempfile -from dataclasses import dataclass -from datetime import date, datetime, timedelta, timezone -from decimal import Decimal -from pathlib import Path -from typing import Any - -from execution_core import ( - SCREEN_SCHEMA_V2, - BlockedError, - canonical_json, - decimal_value, - exact_fields, - normalize_symbol, - normalized_account_snapshot, - parse_aware_datetime, - positive_decimal, - round_down_to_tick, - sha256_json, -) - -POSITION_POLICY_SCHEMA = "qta-position-policy/v1" -STRATEGY_POSITION_SCHEMA = "qta-strategy-position/v1" -EXIT_DECISION_SCHEMA = "qta-exit-decision/v1" -POSITION_RECONCILIATION_SCHEMA = "qta-position-reconciliation/v1" -POSITION_POLICY_FIELDS = { - "schema", - "close_liquidation_seconds_before_close", - "exit_order_ttl_seconds", - "max_exit_replacements", - "quote_max_age_seconds", - "partial_entry_action", - "gap_down_action", - "daily_loss_action", - "overnight_residual_action", -} -STRATEGY_POSITION_FIELDS = { - "schema", - "position_id", - "plan_hash", - "entry_intent_id", - "market", - "exchange", - "symbol", - "venue", - "currency", - "entry_session_date", - "last_managed_session_date", - "planned_quantity", - "acquired_quantity", - "exited_quantity", - "average_entry_price", - "average_exit_price", - "stop_price", - "take_profit_price", - "tick_size", - "fx_to_krw", - "daily_baseline_session_date", - "daily_baseline_price", - "realized_pnl_native", - "daily_realized_pnl_native", - "status", -} -POSITION_STATUSES = { - "PENDING_ENTRY", - "OPEN", - "EXIT_PENDING", - "EXIT_PARTIAL", - "CARRY_EXIT_ONLY", - "CLOSED", - "MANUAL_BLOCK", -} -MARKET_EXCHANGES = { - "KR": {"KOSPI", "KOSDAQ"}, - "US": {"NYSE", "NASDAQ"}, -} -MARKET_CURRENCY = {"KR": "KRW", "US": "USD"} - - -def require_iso_date(value: Any, field: str) -> str: - if not isinstance(value, str): - raise BlockedError(f"{field} must be YYYY-MM-DD") - try: - parsed = date.fromisoformat(value) - except ValueError as exc: - raise BlockedError(f"{field} must be YYYY-MM-DD") from exc - if parsed.isoformat() != value: - raise BlockedError(f"{field} must be YYYY-MM-DD") - return value - - -def require_nonnegative_integer(value: Any, field: str) -> int: - if isinstance(value, bool) or not isinstance(value, int) or value < 0: - raise BlockedError(f"{field} must be a nonnegative integer") - return value - - -def require_positive_integer(value: Any, field: str) -> int: - amount = require_nonnegative_integer(value, field) - if amount == 0: - raise BlockedError(f"{field} must be a positive integer") - return amount - - -def normalized_position_policy(value: dict[str, Any]) -> dict[str, Any]: - exact_fields(value, POSITION_POLICY_FIELDS, "position policy") - if value["schema"] != POSITION_POLICY_SCHEMA: - raise BlockedError( - f"position policy schema must be {POSITION_POLICY_SCHEMA}" - ) - close_seconds = require_positive_integer( - value["close_liquidation_seconds_before_close"], - "close_liquidation_seconds_before_close", - ) - if close_seconds > 3600: - raise BlockedError( - "close_liquidation_seconds_before_close must be <= 3600" - ) - ttl = require_positive_integer( - value["exit_order_ttl_seconds"], "exit_order_ttl_seconds" - ) - if ttl > 300: - raise BlockedError("exit_order_ttl_seconds must be <= 300") - replacements = require_nonnegative_integer( - value["max_exit_replacements"], "max_exit_replacements" - ) - if replacements > 20: - raise BlockedError("max_exit_replacements must be <= 20") - quote_age = require_positive_integer( - value["quote_max_age_seconds"], "quote_max_age_seconds" - ) - if quote_age > 60: - raise BlockedError("quote_max_age_seconds must be <= 60") - expected_enums = { - "partial_entry_action": "CANCEL_REMAINDER_MANAGE_FILLED", - "gap_down_action": "MARKETABLE_LIMIT", - "daily_loss_action": "ENTRY_FREEZE_LIQUIDATE", - "overnight_residual_action": "EXIT_ONLY_NEXT_SESSION", - } - for field, expected in expected_enums.items(): - if value[field] != expected: - raise BlockedError(f"{field} must be {expected}") - return { - **value, - "close_liquidation_seconds_before_close": close_seconds, - "exit_order_ttl_seconds": ttl, - "max_exit_replacements": replacements, - "quote_max_age_seconds": quote_age, - } - - -def strategy_position_id(plan_hash: str, intent: dict[str, Any]) -> str: - if len(plan_hash) != 64 or any( - character not in "0123456789abcdef" for character in plan_hash - ): - raise BlockedError("plan_hash must be lowercase SHA-256") - seed = { - "plan_hash": plan_hash, - "entry_intent_id": intent.get("intent_id"), - "market": intent.get("market"), - "exchange": intent.get("exchange"), - "symbol": intent.get("symbol"), - "venue": intent.get("venue"), - } - return sha256_json(seed)[:32] - - -def position_from_intent( - *, - plan_hash: str, - intent: dict[str, Any], - session_date: str, - fx_to_krw: Any, -) -> dict[str, Any]: - session = require_iso_date(session_date, "session_date") - market = str(intent.get("market", "")).upper() - if market not in MARKET_EXCHANGES: - raise BlockedError("intent market must be KR or US") - exchange = str(intent.get("exchange", "")).upper() - if exchange not in MARKET_EXCHANGES[market]: - raise BlockedError("intent exchange does not match market") - symbol = normalize_symbol(market, str(intent.get("symbol", ""))) - venue = str(intent.get("venue", "")).upper() - if not venue: - raise BlockedError("intent venue is required") - quantity = positive_decimal(intent.get("quantity"), "intent quantity") - if quantity != quantity.to_integral_value(): - raise BlockedError("strategy positions require whole-share quantity") - currency = str(intent.get("currency", "")).upper() - if currency != MARKET_CURRENCY[market]: - raise BlockedError("intent currency does not match market") - fx = positive_decimal(fx_to_krw, "fx_to_krw") - stop = positive_decimal(intent.get("stop_price"), "stop_price") - take_profit = positive_decimal( - intent.get("take_profit_price"), "take_profit_price" - ) - tick_source = intent.get("resolved_tick_size") - if tick_source is None: - tick_source = intent.get("tick_size") - if tick_source is None: - raise BlockedError("intent resolved tick size is required") - tick = positive_decimal(tick_source, "tick_size") - entry_intent_id = str(intent.get("intent_id", "")) - if len(entry_intent_id) != 32: - raise BlockedError("entry intent ID must contain 32 characters") - output = { - "schema": STRATEGY_POSITION_SCHEMA, - "position_id": strategy_position_id(plan_hash, intent), - "plan_hash": plan_hash, - "entry_intent_id": entry_intent_id, - "market": market, - "exchange": exchange, - "symbol": symbol, - "venue": venue, - "currency": currency, - "entry_session_date": session, - "last_managed_session_date": session, - "planned_quantity": format(quantity, "f"), - "acquired_quantity": "0", - "exited_quantity": "0", - "average_entry_price": None, - "average_exit_price": None, - "stop_price": format(stop, "f"), - "take_profit_price": format(take_profit, "f"), - "tick_size": format(tick, "f"), - "fx_to_krw": format(fx, "f"), - "daily_baseline_session_date": None, - "daily_baseline_price": None, - "realized_pnl_native": "0", - "daily_realized_pnl_native": "0", - "status": "PENDING_ENTRY", - } - return normalize_strategy_position(output) - - -def normalize_strategy_position(value: dict[str, Any]) -> dict[str, Any]: - exact_fields(value, STRATEGY_POSITION_FIELDS, "strategy position") - if value["schema"] != STRATEGY_POSITION_SCHEMA: - raise BlockedError( - f"strategy position schema must be {STRATEGY_POSITION_SCHEMA}" - ) - position_id = str(value["position_id"]) - if len(position_id) != 32 or any( - character not in "0123456789abcdef" for character in position_id - ): - raise BlockedError("position_id must be 32 lowercase hex characters") - plan_hash = str(value["plan_hash"]) - if len(plan_hash) != 64 or any( - character not in "0123456789abcdef" for character in plan_hash - ): - raise BlockedError("plan_hash must be lowercase SHA-256") - market = str(value["market"]).upper() - exchange = str(value["exchange"]).upper() - if market not in MARKET_EXCHANGES or exchange not in MARKET_EXCHANGES[market]: - raise BlockedError("strategy position exchange does not match market") - symbol = normalize_symbol(market, str(value["symbol"])) - currency = str(value["currency"]).upper() - if currency != MARKET_CURRENCY[market]: - raise BlockedError("strategy position currency does not match market") - entry_date = require_iso_date( - value["entry_session_date"], "entry_session_date" - ) - managed_date = require_iso_date( - value["last_managed_session_date"], "last_managed_session_date" - ) - if managed_date < entry_date: - raise BlockedError("last managed session cannot precede entry session") - planned = positive_decimal(value["planned_quantity"], "planned_quantity") - acquired = decimal_value(value["acquired_quantity"], "acquired_quantity") - exited = decimal_value(value["exited_quantity"], "exited_quantity") - for amount, field in ( - (planned, "planned_quantity"), - (acquired, "acquired_quantity"), - (exited, "exited_quantity"), - ): - if amount < 0 or amount != amount.to_integral_value(): - raise BlockedError(f"{field} must be a nonnegative whole number") - if acquired > planned: - raise BlockedError("acquired quantity exceeds planned quantity") - if exited > acquired: - raise BlockedError("exited quantity exceeds acquired quantity") - average_entry = value["average_entry_price"] - if acquired > 0: - average_entry = format( - positive_decimal(average_entry, "average_entry_price"), "f" - ) - elif average_entry is not None: - raise BlockedError("zero acquired quantity cannot have an entry price") - average_exit = value["average_exit_price"] - if exited > 0: - average_exit = format( - positive_decimal(average_exit, "average_exit_price"), "f" - ) - elif average_exit is not None: - raise BlockedError("zero exited quantity cannot have an exit price") - baseline_date = value["daily_baseline_session_date"] - baseline_price = value["daily_baseline_price"] - if baseline_date is None: - if baseline_price is not None: - raise BlockedError("daily baseline date and price must both be null") - else: - baseline_date = require_iso_date( - baseline_date, "daily_baseline_session_date" - ) - if baseline_date != managed_date: - raise BlockedError("daily baseline must match last managed session") - baseline_price = format( - positive_decimal(baseline_price, "daily_baseline_price"), "f" - ) - stop = positive_decimal(value["stop_price"], "stop_price") - take_profit = positive_decimal(value["take_profit_price"], "take_profit_price") - if stop >= take_profit: - raise BlockedError("stop price must be below take-profit price") - tick = positive_decimal(value["tick_size"], "tick_size") - fx = positive_decimal(value["fx_to_krw"], "fx_to_krw") - realized = decimal_value(value["realized_pnl_native"], "realized_pnl_native") - daily_realized = decimal_value( - value["daily_realized_pnl_native"], "daily_realized_pnl_native" - ) - status = str(value["status"]) - if status not in POSITION_STATUSES: - raise BlockedError(f"unsupported strategy position status: {status}") - if acquired == 0 and status != "PENDING_ENTRY": - raise BlockedError("zero acquired quantity must be PENDING_ENTRY") - if acquired > 0 and exited == acquired and status != "CLOSED": - raise BlockedError("fully exited position must be CLOSED") - if acquired > exited and status in {"PENDING_ENTRY", "CLOSED"}: - raise BlockedError("open quantity requires an open position status") - return { - **value, - "market": market, - "exchange": exchange, - "symbol": symbol, - "currency": currency, - "entry_session_date": entry_date, - "last_managed_session_date": managed_date, - "planned_quantity": format(planned, "f"), - "acquired_quantity": format(acquired, "f"), - "exited_quantity": format(exited, "f"), - "average_entry_price": average_entry, - "average_exit_price": average_exit, - "stop_price": format(stop, "f"), - "take_profit_price": format(take_profit, "f"), - "tick_size": format(tick, "f"), - "fx_to_krw": format(fx, "f"), - "daily_baseline_session_date": baseline_date, - "daily_baseline_price": baseline_price, - "realized_pnl_native": format(realized, "f"), - "daily_realized_pnl_native": format(daily_realized, "f"), - "status": status, - } - - -def open_quantity(position: dict[str, Any]) -> Decimal: - normalized = normalize_strategy_position(position) - return Decimal(normalized["acquired_quantity"]) - Decimal( - normalized["exited_quantity"] - ) - - -def record_entry_fill( - position: dict[str, Any], - *, - cumulative_filled_quantity: Any, - average_fill_price: Any, - session_date: str, -) -> dict[str, Any]: - normalized = normalize_strategy_position(position) - session = require_iso_date(session_date, "session_date") - cumulative = decimal_value( - cumulative_filled_quantity, "cumulative_filled_quantity" - ) - prior = Decimal(normalized["acquired_quantity"]) - planned = Decimal(normalized["planned_quantity"]) - if ( - cumulative < prior - or cumulative > planned - or cumulative != cumulative.to_integral_value() - ): - raise BlockedError("entry cumulative fill is non-monotonic or out of bounds") - if cumulative == 0: - return normalized - average = positive_decimal(average_fill_price, "average_fill_price") - if session < normalized["entry_session_date"]: - raise BlockedError("entry fill session precedes the planned entry session") - exited = Decimal(normalized["exited_quantity"]) - next_status = "CLOSED" if cumulative == exited else "OPEN" - output = { - **normalized, - "last_managed_session_date": session, - "acquired_quantity": format(cumulative, "f"), - "average_entry_price": format(average, "f"), - "daily_baseline_session_date": session, - "daily_baseline_price": format(average, "f"), - "status": next_status, - } - return normalize_strategy_position(output) - - -def roll_position_session( - position: dict[str, Any], - *, - session_date: str, -) -> dict[str, Any]: - normalized = normalize_strategy_position(position) - session = require_iso_date(session_date, "session_date") - if session < normalized["last_managed_session_date"]: - raise BlockedError("cannot roll a position backward") - if session == normalized["last_managed_session_date"]: - return normalized - status = ( - "CLOSED" - if open_quantity(normalized) == 0 - else "CARRY_EXIT_ONLY" - ) - return normalize_strategy_position( - { - **normalized, - "last_managed_session_date": session, - "daily_baseline_session_date": None, - "daily_baseline_price": None, - "daily_realized_pnl_native": "0", - "status": status, - } - ) - - -def set_daily_baseline( - position: dict[str, Any], - *, - session_date: str, - price: Any, -) -> dict[str, Any]: - normalized = normalize_strategy_position(position) - session = require_iso_date(session_date, "session_date") - if session != normalized["last_managed_session_date"]: - raise BlockedError("daily baseline session must be the managed session") - baseline = positive_decimal(price, "daily baseline price") - existing_date = normalized["daily_baseline_session_date"] - if existing_date is not None: - if ( - existing_date != session - or Decimal(str(normalized["daily_baseline_price"])) != baseline - ): - raise BlockedError("daily baseline is immutable within one session") - return normalized - return normalize_strategy_position( - { - **normalized, - "daily_baseline_session_date": session, - "daily_baseline_price": format(baseline, "f"), - } - ) - - -def record_exit_fill( - position: dict[str, Any], - *, - cumulative_exited_quantity: Any, - average_exit_price: Any, -) -> dict[str, Any]: - normalized = normalize_strategy_position(position) - cumulative = decimal_value( - cumulative_exited_quantity, "cumulative_exited_quantity" - ) - prior = Decimal(normalized["exited_quantity"]) - acquired = Decimal(normalized["acquired_quantity"]) - if ( - cumulative < prior - or cumulative > acquired - or cumulative != cumulative.to_integral_value() - ): - raise BlockedError("exit cumulative fill is non-monotonic or out of bounds") - if cumulative == 0: - return normalized - average_exit = positive_decimal(average_exit_price, "average_exit_price") - entry = positive_decimal( - normalized["average_entry_price"], "average_entry_price" - ) - prior_average_exit = ( - Decimal(str(normalized["average_exit_price"])) - if prior > 0 - else Decimal(0) - ) - prior_exit_proceeds = prior_average_exit * prior - cumulative_exit_proceeds = average_exit * cumulative - incremental_quantity = cumulative - prior - incremental_exit_proceeds = cumulative_exit_proceeds - prior_exit_proceeds - realized = ( - Decimal(normalized["realized_pnl_native"]) - + incremental_exit_proceeds - - entry * incremental_quantity - ) - baseline_value = normalized["daily_baseline_price"] - if baseline_value is None: - raise BlockedError("daily baseline is required before recording an exit") - baseline = positive_decimal(baseline_value, "daily_baseline_price") - daily_realized = ( - Decimal(normalized["daily_realized_pnl_native"]) - + incremental_exit_proceeds - - baseline * incremental_quantity - ) - status = "CLOSED" if cumulative == acquired else "EXIT_PARTIAL" - return normalize_strategy_position( - { - **normalized, - "exited_quantity": format(cumulative, "f"), - "average_exit_price": format(average_exit, "f"), - "realized_pnl_native": format(realized, "f"), - "daily_realized_pnl_native": format(daily_realized, "f"), - "status": status, - } - ) - - -def partial_entry_requires_cancel( - position: dict[str, Any], - *, - broker_remaining_quantity: Any, -) -> bool: - normalized = normalize_strategy_position(position) - remaining = decimal_value( - broker_remaining_quantity, "broker_remaining_quantity" - ) - return Decimal(normalized["acquired_quantity"]) > 0 and remaining > 0 - - -def normalize_exit_quote( - quote: dict[str, Any], - *, - position: dict[str, Any], - observed_at: datetime, - max_age_seconds: int, -) -> dict[str, Any]: - normalized = normalize_strategy_position(position) - if str(quote.get("market", "")).upper() != normalized["market"]: - raise BlockedError("exit quote market does not match position") - if normalize_symbol( - normalized["market"], str(quote.get("symbol", "")) - ) != normalized["symbol"]: - raise BlockedError("exit quote symbol does not match position") - last = positive_decimal(quote.get("last_price"), "exit quote last_price") - bid = positive_decimal(quote.get("best_bid"), "exit quote best_bid") - ask = positive_decimal(quote.get("best_ask"), "exit quote best_ask") - if bid > ask: - raise BlockedError("exit quote best bid exceeds best ask") - trade_time = parse_aware_datetime( - quote.get("trade_timestamp"), "exit quote trade_timestamp" - ) - book_time = parse_aware_datetime( - quote.get("book_timestamp"), "exit quote book_timestamp" - ) - received = parse_aware_datetime( - quote.get("received_at"), "exit quote received_at" - ) - now = observed_at.astimezone(timezone.utc) - for supplied, field in ( - (trade_time, "trade_timestamp"), - (book_time, "book_timestamp"), - (received, "received_at"), - ): - age = (now - supplied.astimezone(timezone.utc)).total_seconds() - if age < -1: - raise BlockedError(f"exit quote {field} is future-dated") - if age > max_age_seconds: - raise BlockedError(f"exit quote {field} is stale") - tick = Decimal(normalized["tick_size"]) - return { - "last_price": format(last, "f"), - "best_bid": format(bid, "f"), - "best_ask": format(ask, "f"), - "limit_price": format(round_down_to_tick(bid, tick), "f"), - "trade_timestamp": trade_time.isoformat(), - "book_timestamp": book_time.isoformat(), - "received_at": received.isoformat(), - } - - -def exit_decision( - position: dict[str, Any], - quote: dict[str, Any], - policy: dict[str, Any], - *, - observed_at: datetime, - regular_close: datetime, - first_valid_quote_of_session: bool, - daily_loss_breached: bool, -) -> dict[str, Any]: - normalized = normalize_strategy_position(position) - normalized_policy = normalized_position_policy(policy) - quantity = open_quantity(normalized) - if quantity <= 0: - action = "CLOSED" - reason = "NO_OPEN_QUANTITY" - normalized_quote = None - limit_price = None - else: - if observed_at.tzinfo is None or observed_at.utcoffset() is None: - raise BlockedError("observed_at must be timezone-aware") - if regular_close.tzinfo is None or regular_close.utcoffset() is None: - raise BlockedError("regular_close must be timezone-aware") - if observed_at >= regular_close: - action = "MANUAL_BLOCK" - reason = "MARKET_CLOSED_WITH_RESIDUAL" - normalized_quote = None - limit_price = None - else: - normalized_quote = normalize_exit_quote( - quote, - position=normalized, - observed_at=observed_at, - max_age_seconds=normalized_policy["quote_max_age_seconds"], - ) - last = Decimal(normalized_quote["last_price"]) - stop = Decimal(normalized["stop_price"]) - target = Decimal(normalized["take_profit_price"]) - close_trigger = regular_close - timedelta( - seconds=normalized_policy[ - "close_liquidation_seconds_before_close" - ] - ) - action = "HOLD" - reason = "NO_EXIT_TRIGGER" - if daily_loss_breached: - action = "SUBMIT_EXIT" - reason = "DAILY_LOSS_LIMIT" - elif last <= stop: - action = "SUBMIT_EXIT" - reason = ( - "STOP_GAP" - if first_valid_quote_of_session and last < stop - else "STOP" - ) - elif last >= target: - action = "SUBMIT_EXIT" - reason = "TAKE_PROFIT" - elif observed_at >= close_trigger: - action = "SUBMIT_EXIT" - reason = "MARKET_CLOSE" - limit_price = ( - normalized_quote["limit_price"] - if action == "SUBMIT_EXIT" - else None - ) - without_hash = { - "schema": EXIT_DECISION_SCHEMA, - "position_id": normalized["position_id"], - "session_date": normalized["last_managed_session_date"], - "observed_at": observed_at.isoformat(), - "action": action, - "reason": reason, - "quantity": format(max(quantity, Decimal(0)), "f"), - "order_type": "LIMIT" if action == "SUBMIT_EXIT" else None, - "limit_price": limit_price, - "quote": normalized_quote, - "daily_loss_breached": bool(daily_loss_breached), - } - return {**without_hash, "decision_hash": sha256_json(without_hash)} - - -def marked_daily_pnl_krw( - position: dict[str, Any], - *, - best_bid: Any, - estimated_exit_cost_bps: Any, -) -> Decimal: - normalized = normalize_strategy_position(position) - if normalized["daily_baseline_price"] is None: - raise BlockedError("daily baseline is required for daily P&L") - bid = positive_decimal(best_bid, "best_bid") - cost_bps = decimal_value(estimated_exit_cost_bps, "estimated_exit_cost_bps") - if cost_bps < 0 or cost_bps > 10000: - raise BlockedError("estimated_exit_cost_bps must be between 0 and 10000") - quantity = open_quantity(normalized) - baseline = Decimal(str(normalized["daily_baseline_price"])) - daily_realized = Decimal(normalized["daily_realized_pnl_native"]) - exit_cost = bid * quantity * cost_bps / Decimal(10000) - pnl_native = daily_realized + (bid - baseline) * quantity - exit_cost - return pnl_native * Decimal(normalized["fx_to_krw"]) - - -def portfolio_daily_loss_krw( - marked_pnl_by_position: dict[str, Decimal], -) -> Decimal: - pnl = sum(marked_pnl_by_position.values(), Decimal(0)) - return max(-pnl, Decimal(0)) - - -def daily_loss_limit_breached(loss_krw: Any, maximum_loss_krw: Any) -> bool: - loss = decimal_value(loss_krw, "loss_krw") - maximum = positive_decimal(maximum_loss_krw, "maximum_loss_krw") - if loss < 0: - raise BlockedError("loss_krw must be nonnegative") - return loss >= maximum - - -def exit_order_id( - decision: dict[str, Any], - *, - attempt: int, -) -> str: - attempt_number = require_nonnegative_integer(attempt, "exit attempt") - if decision.get("schema") != EXIT_DECISION_SCHEMA: - raise BlockedError("exit decision schema is invalid") - if decision.get("action") != "SUBMIT_EXIT": - raise BlockedError("exit order requires a SUBMIT_EXIT decision") - seed = { - "position_id": decision["position_id"], - "session_date": decision["session_date"], - "reason": decision["reason"], - "quantity": decision["quantity"], - "limit_price": decision["limit_price"], - "attempt": attempt_number, - } - return sha256_json(seed)[:32] - - -@dataclass(frozen=True) -class StoredPosition: - position_id: str - status: str - payload: dict[str, Any] - - -class PositionLedger: - """Account/market-level position registry that survives session dates.""" - - def __init__(self, path: str | Path): - self.path = Path(path) - self.path.parent.mkdir(parents=True, exist_ok=True) - self.connection = sqlite3.connect(str(self.path)) - self.connection.execute("PRAGMA journal_mode=WAL") - self.connection.execute("PRAGMA synchronous=FULL") - self.connection.executescript( - """ - CREATE TABLE IF NOT EXISTS strategy_positions ( - position_id TEXT PRIMARY KEY, - status TEXT NOT NULL, - payload_json TEXT NOT NULL, - updated_at TEXT NOT NULL - ); - CREATE TABLE IF NOT EXISTS position_events ( - sequence INTEGER PRIMARY KEY AUTOINCREMENT, - position_id TEXT NOT NULL, - event_type TEXT NOT NULL, - payload_json TEXT NOT NULL, - created_at TEXT NOT NULL - ); - """ - ) - self.connection.commit() - - def close(self) -> None: - self.connection.close() - - def put( - self, - position: dict[str, Any], - *, - event_type: str, - event_payload: dict[str, Any], - ) -> None: - normalized = normalize_strategy_position(position) - if not event_type: - raise BlockedError("position event type is required") - if not isinstance(event_payload, dict): - raise BlockedError("position event payload must be an object") - now = datetime.now(timezone.utc).isoformat() - rendered = canonical_json(normalized) - self.connection.execute("BEGIN IMMEDIATE") - try: - existing = self.connection.execute( - """ - SELECT payload_json FROM strategy_positions - WHERE position_id = ? - """, - (normalized["position_id"],), - ).fetchone() - if existing is None: - self.connection.execute( - """ - INSERT INTO strategy_positions - (position_id, status, payload_json, updated_at) - VALUES (?, ?, ?, ?) - """, - ( - normalized["position_id"], - normalized["status"], - rendered, - now, - ), - ) - else: - self.connection.execute( - """ - UPDATE strategy_positions - SET status = ?, payload_json = ?, updated_at = ? - WHERE position_id = ? - """, - ( - normalized["status"], - rendered, - now, - normalized["position_id"], - ), - ) - self.connection.execute( - """ - INSERT INTO position_events - (position_id, event_type, payload_json, created_at) - VALUES (?, ?, ?, ?) - """, - ( - normalized["position_id"], - event_type, - canonical_json(event_payload), - now, - ), - ) - except BaseException: - self.connection.rollback() - raise - else: - self.connection.commit() - - def get(self, position_id: str) -> StoredPosition: - row = self.connection.execute( - """ - SELECT position_id, status, payload_json - FROM strategy_positions WHERE position_id = ? - """, - (position_id,), - ).fetchone() - if row is None: - raise BlockedError(f"unknown strategy position: {position_id}") - return StoredPosition( - position_id=str(row[0]), - status=str(row[1]), - payload=normalize_strategy_position(json.loads(row[2])), - ) - - def get_optional(self, position_id: str) -> StoredPosition | None: - row = self.connection.execute( - """ - SELECT position_id, status, payload_json - FROM strategy_positions WHERE position_id = ? - """, - (position_id,), - ).fetchone() - if row is None: - return None - return StoredPosition( - position_id=str(row[0]), - status=str(row[1]), - payload=normalize_strategy_position(json.loads(row[2])), - ) - - def open_positions(self) -> list[StoredPosition]: - rows = self.connection.execute( - """ - SELECT position_id, status, payload_json - FROM strategy_positions - WHERE status != 'CLOSED' - ORDER BY position_id - """ - ).fetchall() - return [ - StoredPosition( - position_id=str(row[0]), - status=str(row[1]), - payload=normalize_strategy_position(json.loads(row[2])), - ) - for row in rows - ] - - def events(self, position_id: str) -> list[dict[str, Any]]: - rows = self.connection.execute( - """ - SELECT sequence, event_type, payload_json, created_at - FROM position_events - WHERE position_id = ? - ORDER BY sequence - """, - (position_id,), - ).fetchall() - return [ - { - "sequence": int(row[0]), - "event_type": str(row[1]), - "payload": json.loads(row[2]), - "created_at": str(row[3]), - } - for row in rows - ] - - -def reconcile_account_positions( - ledger: PositionLedger, - *, - account_snapshot: dict[str, Any], - session_date: str, -) -> dict[str, Any]: - """Match bot-owned positions to the broker snapshot and roll valid carries.""" - session = require_iso_date(session_date, "session_date") - account = normalized_account_snapshot( - account_snapshot, - screen_schema=SCREEN_SCHEMA_V2, - ) - market = account["market"] - account_quantities: dict[tuple[str, str], Decimal] = {} - for item in account["positions"]: - key = (str(item["exchange"]), str(item["symbol"])) - quantity = decimal_value(item.get("quantity"), "account position quantity") - if quantity < 0: - raise BlockedError("account position quantity must be nonnegative") - account_quantities[key] = account_quantities.get(key, Decimal(0)) + quantity - open_order_keys = { - (str(item["exchange"]), str(item["symbol"])) - for item in account["open_orders"] - } - - stored_positions = [ - item for item in ledger.open_positions() if item.payload["market"] == market - ] - by_key: dict[tuple[str, str], list[StoredPosition]] = {} - for stored in stored_positions: - key = (stored.payload["exchange"], stored.payload["symbol"]) - by_key.setdefault(key, []).append(stored) - - results: list[dict[str, Any]] = [] - managed_keys: set[tuple[str, str]] = set() - for key in sorted(by_key): - positions = by_key[key] - managed_keys.add(key) - expected = sum( - (open_quantity(item.payload) for item in positions), Decimal(0) - ) - observed = account_quantities.get(key, Decimal(0)) - reasons: list[str] = [] - if len(positions) != 1: - reasons.append("duplicate_strategy_positions_for_symbol") - if observed != expected: - reasons.append("broker_quantity_differs_from_strategy_ledger") - if key in open_order_keys: - reasons.append("broker_has_open_order_for_strategy_symbol") - if any(item.status == "MANUAL_BLOCK" for item in positions): - reasons.append("prior_manual_block_requires_explicit_repair") - - if reasons: - for stored in positions: - if stored.status != "MANUAL_BLOCK": - blocked_position = normalize_strategy_position( - {**stored.payload, "status": "MANUAL_BLOCK"} - ) - ledger.put( - blocked_position, - event_type="ACCOUNT_RECONCILIATION_BLOCKED", - event_payload={ - "session_date": session, - "expected_quantity": format(expected, "f"), - "observed_quantity": format(observed, "f"), - "reasons": reasons, - }, - ) - state = "MANUAL_BLOCK" - else: - stored = positions[0] - rolled = roll_position_session( - stored.payload, - session_date=session, - ) - if rolled != stored.payload: - ledger.put( - rolled, - event_type="POSITION_CARRIED", - event_payload={ - "from_session_date": stored.payload[ - "last_managed_session_date" - ], - "to_session_date": session, - "quantity": format(expected, "f"), - }, - ) - state = rolled["status"] - results.append( - { - "exchange": key[0], - "symbol": key[1], - "strategy_position_ids": [ - item.position_id for item in positions - ], - "expected_quantity": format(expected, "f"), - "observed_quantity": format(observed, "f"), - "state": state, - "reasons": reasons, - } - ) - - unmanaged = [ - { - "exchange": key[0], - "symbol": key[1], - "quantity": format(quantity, "f"), - } - for key, quantity in sorted(account_quantities.items()) - if key not in managed_keys and quantity > 0 - ] - without_hash = { - "schema": POSITION_RECONCILIATION_SCHEMA, - "account_as_of": account["as_of"], - "account_alias": account["account_alias"], - "market": market, - "session_date": session, - "status": ( - "MANUAL_BLOCK" - if any(item["state"] == "MANUAL_BLOCK" for item in results) - else "READY" - ), - "managed_positions": results, - "unmanaged_broker_positions": unmanaged, - "api_mutation_count": 0, - "live_enabled": False, - } - return { - **without_hash, - "reconciliation_hash": sha256_json(without_hash), - } - - -def atomic_write_json(path: Path, value: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - descriptor, temporary_name = tempfile.mkstemp( - prefix=f".{path.name}.", dir=path.parent - ) - temporary = Path(temporary_name) - try: - with os.fdopen(descriptor, "w", encoding="utf-8") as handle: - handle.write(json.dumps(value, sort_keys=True, separators=(",", ":"))) - handle.write("\n") - handle.flush() - os.fsync(handle.fileno()) - os.replace(temporary, path) - except BaseException: - temporary.unlink(missing_ok=True) - raise - - -def self_test() -> None: - plan_hash = "a" * 64 - intent = { - "intent_id": "b" * 32, - "market": "KR", - "exchange": "KOSPI", - "symbol": "005930", - "venue": "KRX", - "currency": "KRW", - "quantity": "10", - "stop_price": "65000", - "take_profit_price": "80000", - "resolved_tick_size": "100", - } - policy = normalized_position_policy( - { - "schema": POSITION_POLICY_SCHEMA, - "close_liquidation_seconds_before_close": 600, - "exit_order_ttl_seconds": 15, - "max_exit_replacements": 3, - "quote_max_age_seconds": 5, - "partial_entry_action": "CANCEL_REMAINDER_MANAGE_FILLED", - "gap_down_action": "MARKETABLE_LIMIT", - "daily_loss_action": "ENTRY_FREEZE_LIQUIDATE", - "overnight_residual_action": "EXIT_ONLY_NEXT_SESSION", - } - ) - position = position_from_intent( - plan_hash=plan_hash, - intent=intent, - session_date="2026-07-29", - fx_to_krw="1", - ) - position = record_entry_fill( - position, - cumulative_filled_quantity="4", - average_fill_price="70000", - session_date="2026-07-29", - ) - assert partial_entry_requires_cancel( - position, broker_remaining_quantity="6" - ) - now = datetime(2026, 7, 29, 9, 1, tzinfo=timezone(timedelta(hours=9))) - quote = { - "market": "KR", - "symbol": "005930", - "last_price": "64000", - "best_bid": "63900", - "best_ask": "64000", - "trade_timestamp": now.isoformat(), - "book_timestamp": now.isoformat(), - "received_at": now.isoformat(), - } - decision = exit_decision( - position, - quote, - policy, - observed_at=now, - regular_close=datetime( - 2026, 7, 29, 15, 30, tzinfo=timezone(timedelta(hours=9)) - ), - first_valid_quote_of_session=True, - daily_loss_breached=False, - ) - assert decision["reason"] == "STOP_GAP" - assert decision["quantity"] == "4" - assert decision["limit_price"] == "63900" - assert len(exit_order_id(decision, attempt=0)) == 32 - assert marked_daily_pnl_krw( - position, - best_bid="64000", - estimated_exit_cost_bps="30", - ) < 0 - position = record_exit_fill( - position, - cumulative_exited_quantity="2", - average_exit_price="64000", - ) - assert position["status"] == "EXIT_PARTIAL" - assert open_quantity(position) == Decimal(2) - carried = roll_position_session(position, session_date="2026-07-30") - assert carried["status"] == "CARRY_EXIT_ONLY" - with tempfile.TemporaryDirectory(prefix="qta-position-ledger-") as temporary: - ledger = PositionLedger(Path(temporary) / "positions.sqlite3") - try: - ledger.put( - carried, - event_type="POSITION_CARRIED", - event_payload={"session_date": "2026-07-30"}, - ) - assert ledger.open_positions()[0].status == "CARRY_EXIT_ONLY" - assert ledger.events(carried["position_id"])[0]["event_type"] == ( - "POSITION_CARRIED" - ) - finally: - ledger.close() - print( - json.dumps( - { - "schema": STRATEGY_POSITION_SCHEMA, - "self_test": "PASS", - "decision_hash": decision["decision_hash"], - "live_enabled": False, - "api_mutation_count": 0, - }, - sort_keys=True, - ) - ) - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--self-test", action="store_true") - subparsers = parser.add_subparsers(dest="command") - reconcile_parser = subparsers.add_parser("reconcile-account") - reconcile_parser.add_argument("--ledger", required=True) - reconcile_parser.add_argument("--account-snapshot", required=True) - reconcile_parser.add_argument("--session-date", required=True) - reconcile_parser.add_argument("--output", required=True) - args = parser.parse_args(argv) - try: - if args.self_test: - self_test() - return 0 - if args.command != "reconcile-account": - parser.error("use --self-test or reconcile-account") - paths = { - name: Path(getattr(args, name)) - for name in ("ledger", "account_snapshot", "output") - } - if any(not path.is_absolute() for path in paths.values()): - raise BlockedError("ledger, account snapshot, and output paths must be absolute") - if ( - not paths["account_snapshot"].is_file() - or paths["account_snapshot"].is_symlink() - ): - raise BlockedError( - "account snapshot must be a regular non-symlink file" - ) - account_value = json.loads( - paths["account_snapshot"].read_text(encoding="utf-8") - ) - if not isinstance(account_value, dict): - raise BlockedError("account snapshot must contain one JSON object") - ledger = PositionLedger(paths["ledger"]) - try: - receipt = reconcile_account_positions( - ledger, - account_snapshot=account_value, - session_date=args.session_date, - ) - finally: - ledger.close() - atomic_write_json(paths["output"], receipt) - print(json.dumps(receipt, sort_keys=True)) - return 0 - except ( - AssertionError, - BlockedError, - ValueError, - OSError, - json.JSONDecodeError, - ) as exc: - print( - json.dumps( - { - "status": "BLOCKED", - "reason": str(exc), - "api_mutation_count": 0, - "live_enabled": False, - }, - sort_keys=True, - ) - ) - return 2 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/reconcile.py b/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/reconcile.py deleted file mode 100755 index 7251d7d..0000000 --- a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/reconcile.py +++ /dev/null @@ -1,390 +0,0 @@ -#!/usr/bin/env python3 -"""Reconcile local nonterminal intents against broker order records.""" - -from __future__ import annotations - -import argparse -import json -import sys -import tempfile -from datetime import datetime, timezone -from pathlib import Path -from typing import Any -from unittest.mock import patch - -from broker_adapters import TransportFailure -from execution_core import ( - BlockedError, - Ledger, - SingleWriterLock, - canonical_json, - emit_json, - load_json_object, - plan_orders as build_order_plan, -) -from broker_credentials import load_kis_credentials, require_kis_runtime_credentials -from plan_orders import fixture_inputs -from run_session import ( - create_broker, - current_order, - normalize_venue_map, - parse_timestamp, - validate_broker_binding, - validate_plan, - venue_for, -) - -RECONCILIATION_SCHEMA = "qta-reconciliation-status/v2" - - -def latest_ack(ledger: Ledger, intent_id: str) -> dict[str, Any] | None: - for event in reversed(ledger.events(intent_id)): - if event["event_type"] == "STATE_ACKNOWLEDGED": - ack = event["payload"].get("ack") - return ack if isinstance(ack, dict) else None - return None - - -def reconcile( - plan: dict[str, Any], - broker_name: str, - venues: dict[str, str], - state_directory: Path, - *, - broker: Any | None = None, -) -> dict[str, Any]: - validate_plan(plan) - validate_broker_binding(plan, broker_name) - for intent in plan["intents"]: - venue_for(venues, intent) - state_directory.mkdir(parents=True, exist_ok=True) - trading_date = ( - parse_timestamp(plan["entry_window"]["start"], "entry window start") - .date() - .isoformat() - ) - broker = broker or create_broker(broker_name) - changes: list[dict[str, Any]] = [] - blocked: list[dict[str, Any]] = [] - - with SingleWriterLock(state_directory / "writer.lock"): - ledger = Ledger(state_directory / "ledger.sqlite3") - try: - intent_by_id = {intent["intent_id"]: intent for intent in plan["intents"]} - for record in ledger.nonterminal(): - intent = intent_by_id.get(record.intent_id) - if intent is None: - raise BlockedError( - f"ledger intent is absent from plan: {record.intent_id}" - ) - if record.state in { - "PLANNED", - "WAIT_TRIGGER", - "RESERVED", - "SUBMITTING", - }: - if record.state == "SUBMITTING": - ledger.transition( - record.intent_id, - "UNKNOWN", - {"reason": "restart_during_submitting"}, - ) - ledger.transition(record.intent_id, "RECONCILING", {}) - ledger.transition( - record.intent_id, - "MANUAL_BLOCK", - {"reason": "restart_during_submitting"}, - ) - else: - ledger.transition( - record.intent_id, - "MANUAL_BLOCK", - {"reason": f"restart_from_{record.state.lower()}"}, - ) - blocked.append( - { - "intent_id": record.intent_id, - "reason": f"unsafe restart state {record.state}", - } - ) - continue - if record.state == "UNKNOWN" and not record.broker_order_id: - ledger.transition(record.intent_id, "RECONCILING", {}) - ledger.transition( - record.intent_id, - "MANUAL_BLOCK", - {"reason": "unknown mutation without broker order id"}, - ) - blocked.append( - { - "intent_id": record.intent_id, - "reason": "unknown mutation without broker order id", - } - ) - continue - ack = latest_ack(ledger, record.intent_id) - if ack is None and record.broker_order_id: - ack = {"broker_order_id": record.broker_order_id} - if not isinstance(ack, dict): - if record.state == "UNKNOWN": - ledger.transition(record.intent_id, "RECONCILING", {}) - ledger.transition( - record.intent_id, - "MANUAL_BLOCK", - {"reason": "broker acknowledgement unavailable"}, - ) - blocked.append( - { - "intent_id": record.intent_id, - "reason": "broker acknowledgement unavailable", - } - ) - continue - if record.state == "UNKNOWN": - ledger.transition(record.intent_id, "RECONCILING", {}) - record = ledger.get(record.intent_id) - try: - snapshot = current_order( - broker, - broker_name, - intent=intent, - ack=ack, - venue=venue_for(venues, intent), - trading_date=trading_date, - ) - except (BlockedError, TransportFailure) as exc: - ledger.append_event( - record.intent_id, - "RECONCILE_BLOCKED", - {"reason": str(exc)}, - ) - blocked.append({"intent_id": record.intent_id, "reason": str(exc)}) - continue - next_state = snapshot["normalized_status"] - if next_state == "UNKNOWN": - if record.state != "RECONCILING": - ledger.transition(record.intent_id, "UNKNOWN", snapshot) - ledger.transition(record.intent_id, "RECONCILING", {}) - ledger.transition( - record.intent_id, - "MANUAL_BLOCK", - {"snapshot": snapshot}, - ) - blocked.append( - { - "intent_id": record.intent_id, - "reason": "broker status is unknown", - } - ) - continue - if record.state == "CANCEL_PENDING" and next_state in { - "ACKNOWLEDGED", - "CANCEL_PENDING", - "PARTIALLY_FILLED", - }: - ledger.append_event( - record.intent_id, "CANCEL_STILL_PENDING", snapshot - ) - blocked.append( - { - "intent_id": record.intent_id, - "reason": ( - "cancel remains pending while broker order is " - f"{next_state.lower()}" - ), - } - ) - continue - if next_state == record.state: - ledger.append_event( - record.intent_id, "RECONCILE_NO_CHANGE", snapshot - ) - continue - ledger.transition(record.intent_id, next_state, {"snapshot": snapshot}) - changes.append( - { - "intent_id": record.intent_id, - "from": record.state, - "to": next_state, - } - ) - final = [ - { - "intent_id": intent["intent_id"], - "state": ledger.get(intent["intent_id"]).state, - "broker_order_id": ledger.get(intent["intent_id"]).broker_order_id, - } - for intent in plan["intents"] - ] - finally: - ledger.close() - output = { - "schema": RECONCILIATION_SCHEMA, - "plan_hash": plan["plan_hash"], - "broker": broker_name, - "reconciled_at": datetime.now(timezone.utc).isoformat(), - "status": "BLOCKED" if blocked else "READY", - "changes": changes, - "blocked": blocked, - "intents": final, - } - return output - - -def self_test() -> None: - plan = build_order_plan(*fixture_inputs()) - intent = plan["intents"][0] - venues = {"KR:005930": "KRX"} - try: - reconcile( - plan, - "toss", - venues, - Path("/unused/reconcile-state"), - broker=object(), - ) - except BlockedError as exc: - assert "plan broker does not match selected adapter" in str(exc) - else: - raise AssertionError("reconcile must bind the adapter to the plan broker") - with patch.dict("os.environ", {}, clear=False): - with tempfile.TemporaryDirectory(prefix="qta-reconcile-") as directory: - ledger = Ledger(Path(directory) / "ledger.sqlite3") - ledger.create_intent(plan["plan_hash"], intent) - ledger.transition(intent["intent_id"], "WAIT_TRIGGER", {}) - ledger.transition(intent["intent_id"], "RESERVED", {}) - ledger.transition( - intent["intent_id"], - "SUBMITTING", - {"request_hash": "a" * 64}, - request_hash="a" * 64, - ) - ledger.transition( - intent["intent_id"], "UNKNOWN", {"reason": "accepted then timeout"} - ) - ledger.close() - - class NeverCalledBroker: - pass - - receipt = reconcile( - plan, - "kis-paper", - venues, - Path(directory), - broker=NeverCalledBroker(), - ) - assert receipt["status"] == "BLOCKED" - assert receipt["intents"][0]["state"] == "MANUAL_BLOCK" - - for pending_status in ( - "ACKNOWLEDGED", - "CANCEL_PENDING", - "PARTIALLY_FILLED", - ): - with tempfile.TemporaryDirectory(prefix="qta-cancel-pending-") as directory: - ledger = Ledger(Path(directory) / "ledger.sqlite3") - ledger.create_intent(plan["plan_hash"], intent) - ledger.transition(intent["intent_id"], "WAIT_TRIGGER", {}) - ledger.transition(intent["intent_id"], "RESERVED", {}) - ledger.transition( - intent["intent_id"], - "SUBMITTING", - {"request_hash": "b" * 64}, - request_hash="b" * 64, - ) - ledger.transition( - intent["intent_id"], - "ACKNOWLEDGED", - {"ack": {"broker_order_id": "fixture-order"}}, - broker_order_id="fixture-order", - ) - ledger.transition(intent["intent_id"], "CANCEL_PENDING", {}) - ledger.close() - - class PendingCancelBroker: - def get_order(self, **_: Any) -> dict[str, Any]: - return {"normalized_status": pending_status} - - receipt = reconcile( - plan, - "kis-paper", - venues, - Path(directory), - broker=PendingCancelBroker(), - ) - assert receipt["status"] == "BLOCKED" - assert receipt["intents"][0]["state"] == "CANCEL_PENDING" - print( - canonical_json( - { - "self_test": "PASS", - "schema": RECONCILIATION_SCHEMA, - "unknown_without_order_id": "MANUAL_BLOCK", - } - ) - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--plan") - parser.add_argument("--broker", choices=("toss", "kis-paper", "kis-live")) - parser.add_argument("--venue-map") - parser.add_argument("--state-dir") - parser.add_argument("--output") - parser.add_argument("--self-test", action="store_true") - return parser.parse_args() - - -def main() -> int: - args = parse_args() - if args.self_test: - self_test() - return 0 - missing = [ - name - for name in ("plan", "broker", "state_dir") - if not getattr(args, name) - ] - if missing: - emit_json( - { - "schema": RECONCILIATION_SCHEMA, - "status": "BLOCKED", - "reason": "missing arguments: " + ", ".join(missing), - }, - args.output, - ) - return 2 - try: - if args.broker in {"kis-paper", "kis-live"}: - require_kis_runtime_credentials( - load_kis_credentials(args.broker.split("-", 1)[1]) - ) - receipt = reconcile( - load_json_object(args.plan), - args.broker, - ( - normalize_venue_map(load_json_object(args.venue_map)) - if args.venue_map - else {} - ), - Path(args.state_dir).resolve(), - ) - except (BlockedError, OSError, ValueError, json.JSONDecodeError) as exc: - emit_json( - { - "schema": RECONCILIATION_SCHEMA, - "status": "BLOCKED", - "reason": str(exc), - }, - args.output, - ) - return 2 - emit_json(receipt, args.output) - return 0 if receipt["status"] == "READY" else 2 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/run_session.py b/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/run_session.py deleted file mode 100755 index 1a500b3..0000000 --- a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/run_session.py +++ /dev/null @@ -1,2655 +0,0 @@ -#!/usr/bin/env python3 -"""Preview or run a deterministic first-hour shadow/paper polling session.""" - -from __future__ import annotations - -import argparse -import json -import os -import sys -import time -from datetime import datetime, timedelta, timezone -from decimal import ROUND_CEILING, Decimal -from pathlib import Path -from typing import Any -from zoneinfo import ZoneInfo - -from broker_adapters import ( - AmbiguousMutationError, - AuthoritativeMutationRejection, - KisBroker, - TossBroker, - TransportFailure, -) -from broker_credentials import load_kis_credentials, require_kis_runtime_credentials -from execution_core import ( - ENTRY_WINDOWS, - EXECUTION_VERSION, - MARKET_CURRENCY, - MAX_SNAPSHOT_AGE_SECONDS, - PLAN_SCHEMA, - QTA2_REGIME_MAX_AGE_SECONDS, - QTA2_REGIME_MINIMUM_CHANGE_BPS, - QTA2_REGIME_METRIC, - QTA_METHOD_V2, - SCREEN_SCHEMA_V2, - V2_EXCHANGE_CONTRACTS, - V2_EXCHANGES_BY_MARKET, - BlockedError, - Ledger, - SingleWriterLock, - canonical_json, - decimal_value, - emit_json, - exact_fields, - load_json_object, - nonnegative_decimal, - normalize_symbol, - normalized_execution_policy, - normalized_market_session, - parse_aware_datetime, - parse_iso_date, - plan_orders as rebuild_order_plan, - positive_decimal, - redact, - sha256_json, - validate_nonempty_string, - validate_sha256, -) -from position_lifecycle import ( - PositionLedger, - position_from_intent, - reconcile_account_positions, - record_entry_fill, - strategy_position_id, -) - -VENUE_SCHEMA = "qta-venue-map/v1" -SESSION_STATUS_SCHEMA = "qta-session-status/v2" -PREOPEN_WARMUP_SECONDS = 30 -SUBMIT_HTTP_TIMEOUT_SECONDS = 10 -DEFAULT_QUOTE_HTTP_REQUESTS_PER_INTENT = 2 -BROKER_PACING_SECONDS = { - "toss": Decimal("0.1"), - "kis-paper": Decimal("1.05"), - "kis-live": Decimal("0.12"), -} -PLAN_FIELDS = { - "schema", - "plan_status", - "execution_version", - "context", - "entry_window", - "quote_policy", - "order_policy", - "settled_cash_start", - "borrowed_buying_power_excluded", - "settled_cash_unreserved", - "intents", - "skipped", - "frozen_inputs", - "plan_hash", -} -FROZEN_INPUT_FIELDS = { - "screen", - "account", - "exposure", - "risk", - "execution", -} -BASE_CONTEXT_FIELDS = { - "execution_version", - "screen_hash", - "account_hash", - "exposure_hash", - "risk_hash", - "execution_policy_hash", - "execution_policy", - "market_session", - "market_session_hash", - "snapshot_max_age_seconds", - "broker", - "environment", - "account_alias", - "market", -} -V2_CONTEXT_FIELDS = BASE_CONTEXT_FIELDS | { - "screen_schema", - "candidate_order_contract", - "exchange_order", - "broker_symbol_qualification", - "analysis_date", - "snapshot_as_of", -} -ENTRY_WINDOW_FIELDS = { - "timezone", - "start", - "end", - "poll_interval_seconds", -} -QUOTE_POLICY_FIELDS = { - "max_age_seconds", - "max_spread_bps", - "max_gap_bps", - "trigger_mode", -} -ORDER_POLICY_FIELDS = { - "ttl_seconds", - "allow_partial_fill", - "cancel_remainder_at_window_end", -} -BASE_INTENT_FIELDS = { - "intent_id", - "client_order_id", - "rank", - "market", - "symbol", - "currency", - "side", - "order_type", - "time_in_force", - "quantity", - "entry_trigger", - "limit_price", - "stop_price", - "take_profit_price", - "effective_loss_per_share", - "reserved_cash", - "qta_payload_hash", - "initial_state", - "intent_hash", -} -V2_INTENT_FIELDS = BASE_INTENT_FIELDS | { - "exchange", - "exchange_rank", - "canonical_symbol", - "data_symbol", - "broker_symbol", - "instrument_type", - "benchmark_id", - "venue", - "resolved_tick_size", - "tick_contract_hash", -} -SKIP_REASONS = { - "duplicate_planned_exposure", - "existing_open_buy_order", - "existing_exposure", - "max_concurrent_positions", - "quantity_below_one", -} -MARKET_REGIME_QUOTE_FIELDS = { - "schema", - "broker", - "exchange", - "benchmark_id", - "regime_proxy_id", - "provider_symbol", - "current", - "previous_close", - "change_bps", - "source_timestamp", - "received_at", - "raw_status", -} -REGIME_PROXY_BY_EXCHANGE = { - "KOSPI": "KOSPI_COMPOSITE", - "KOSDAQ": "KOSDAQ_COMPOSITE", - "NYSE": "S&P_500", - "NASDAQ": "NASDAQ_COMPOSITE", -} - - -def validate_positive_integer(value: Any, field: str) -> int: - if isinstance(value, bool) or not isinstance(value, int) or value <= 0: - raise BlockedError(f"{field} must be a positive integer") - return value - - -def validate_decimal_text( - value: Any, - field: str, - *, - positive: bool, -) -> Decimal: - if not isinstance(value, str) or not value or value != value.strip(): - raise BlockedError(f"{field} must be a decimal string") - result = ( - positive_decimal(value, field) - if positive - else nonnegative_decimal(value, field) - ) - if format(result, "f") != value: - raise BlockedError(f"{field} must be a canonical decimal string") - return result - - -def validate_intent( - intent: Any, - *, - index: int, - context: dict[str, Any], - is_v2: bool, -) -> tuple[int, Decimal, tuple[str, str]]: - label = f"plan.intents[{index}]" - if not isinstance(intent, dict): - raise BlockedError(f"{label} must be an object") - exact_fields( - intent, - V2_INTENT_FIELDS if is_v2 else BASE_INTENT_FIELDS, - label, - ) - validate_sha256(intent["intent_hash"], f"{label}.intent_hash") - expected_intent_hash = sha256_json( - {key: value for key, value in intent.items() if key != "intent_hash"} - ) - if intent["intent_hash"] != expected_intent_hash: - raise BlockedError(f"{label}.intent_hash does not match intent contents") - validate_sha256(intent["qta_payload_hash"], f"{label}.qta_payload_hash") - - market = context["market"] - if intent["market"] != market: - raise BlockedError(f"{label}.market must equal plan.context.market") - symbol = normalize_symbol(market, str(intent["symbol"])) - if symbol != intent["symbol"]: - raise BlockedError(f"{label}.symbol must be normalized") - if intent["currency"] != MARKET_CURRENCY[market]: - raise BlockedError(f"{label}.currency does not match market") - if ( - intent["side"] != "BUY" - or intent["order_type"] != "LIMIT" - or intent["time_in_force"] != "DAY" - or intent["initial_state"] != "PLANNED" - ): - raise BlockedError(f"{label} must be BUY, LIMIT, DAY, and initially PLANNED") - rank = validate_positive_integer(intent["rank"], f"{label}.rank") - quantity_text = intent["quantity"] - if ( - not isinstance(quantity_text, str) - or not quantity_text.isdigit() - or str(int(quantity_text)) != quantity_text - or int(quantity_text) <= 0 - ): - raise BlockedError(f"{label}.quantity must be a canonical positive integer") - quantity = Decimal(quantity_text) - entry = validate_decimal_text( - intent["entry_trigger"], - f"{label}.entry_trigger", - positive=True, - ) - limit_price = validate_decimal_text( - intent["limit_price"], - f"{label}.limit_price", - positive=True, - ) - stop = validate_decimal_text( - intent["stop_price"], - f"{label}.stop_price", - positive=True, - ) - take_profit = validate_decimal_text( - intent["take_profit_price"], - f"{label}.take_profit_price", - positive=True, - ) - effective_loss = validate_decimal_text( - intent["effective_loss_per_share"], - f"{label}.effective_loss_per_share", - positive=True, - ) - reserved_cash = validate_decimal_text( - intent["reserved_cash"], - f"{label}.reserved_cash", - positive=True, - ) - if not stop < entry <= limit_price < take_profit: - raise BlockedError( - f"{label} prices must satisfy stop < entry <= limit < take_profit" - ) - if effective_loss < limit_price - stop: - raise BlockedError(f"{label}.effective_loss_per_share is below price loss") - if reserved_cash < limit_price * quantity: - raise BlockedError(f"{label}.reserved_cash is below limit notional") - - intent_seed = { - "plan_seed": sha256_json(context), - "rank": rank, - "market": market, - "symbol": symbol, - "side": "BUY", - "order_type": "LIMIT", - "time_in_force": "DAY", - "quantity": quantity_text, - "limit_price": intent["limit_price"], - } - exposure_namespace = market - if is_v2: - exchange = intent["exchange"] - contract = V2_EXCHANGE_CONTRACTS.get(exchange) - if contract is None or contract["market"] != market: - raise BlockedError(f"{label}.exchange does not match market") - exchange_rank = validate_positive_integer( - intent["exchange_rank"], - f"{label}.exchange_rank", - ) - canonical_symbol = normalize_symbol( - market, - validate_nonempty_string( - intent["canonical_symbol"], - f"{label}.canonical_symbol", - ), - ) - broker_symbol = normalize_symbol( - market, - validate_nonempty_string( - intent["broker_symbol"], - f"{label}.broker_symbol", - ), - ) - if canonical_symbol != intent["canonical_symbol"]: - raise BlockedError(f"{label}.canonical_symbol must be normalized") - if broker_symbol != intent["broker_symbol"] or symbol != broker_symbol: - raise BlockedError( - f"{label}.symbol and broker_symbol must match and be normalized" - ) - validate_nonempty_string(intent["data_symbol"], f"{label}.data_symbol") - if intent["instrument_type"] not in {"COMMON", "ADR", "REIT"}: - raise BlockedError(f"{label}.instrument_type is unsupported") - if ( - intent["benchmark_id"] != contract["benchmark_id"] - or intent["venue"] != contract["venue"] - ): - raise BlockedError(f"{label}.benchmark_id and venue must match exchange") - validate_sha256( - intent["tick_contract_hash"], - f"{label}.tick_contract_hash", - ) - tick_size = validate_decimal_text( - intent["resolved_tick_size"], - f"{label}.resolved_tick_size", - positive=True, - ) - for price, field in ( - (entry, "entry_trigger"), - (limit_price, "limit_price"), - (stop, "stop_price"), - (take_profit, "take_profit_price"), - ): - if price % tick_size != 0: - raise BlockedError( - f"{label}.{field} must align to resolved_tick_size" - ) - intent_seed.update( - { - "exchange": exchange, - "exchange_rank": exchange_rank, - "canonical_symbol": canonical_symbol, - "data_symbol": intent["data_symbol"], - "broker_symbol": broker_symbol, - "venue": intent["venue"], - } - ) - exposure_namespace = exchange - expected_intent_id = sha256_json(intent_seed)[:32] - if intent["intent_id"] != expected_intent_id: - raise BlockedError(f"{label}.intent_id does not match deterministic seed") - if intent["client_order_id"] != f"qta-{expected_intent_id[:28]}": - raise BlockedError( - f"{label}.client_order_id does not match deterministic intent_id" - ) - return rank, reserved_cash, (exposure_namespace, symbol) - - -def validate_skipped( - skipped: Any, - *, - market: str, -) -> None: - if not isinstance(skipped, list): - raise BlockedError("plan.skipped must be an array") - for index, item in enumerate(skipped): - label = f"plan.skipped[{index}]" - if not isinstance(item, dict): - raise BlockedError(f"{label} must be an object") - base_fields = {"market", "symbol", "reason"} - extended_fields = base_fields | { - "exchange", - "exchange_rank", - "canonical_symbol", - } - item_fields = frozenset(item) - if item_fields not in { - frozenset(base_fields), - frozenset(extended_fields), - }: - raise BlockedError(f"{label} fields mismatch") - if item["market"] != market: - raise BlockedError(f"{label}.market must equal plan market") - if normalize_symbol(market, str(item["symbol"])) != item["symbol"]: - raise BlockedError(f"{label}.symbol must be normalized") - if item["reason"] not in SKIP_REASONS: - raise BlockedError(f"{label}.reason is unsupported") - if item_fields == frozenset(extended_fields): - exchange = item["exchange"] - contract = V2_EXCHANGE_CONTRACTS.get(exchange) - if contract is None or contract["market"] != market: - raise BlockedError(f"{label}.exchange does not match market") - validate_positive_integer( - item["exchange_rank"], - f"{label}.exchange_rank", - ) - canonical_symbol = normalize_symbol( - market, - str(item["canonical_symbol"]), - ) - if canonical_symbol != item["canonical_symbol"]: - raise BlockedError(f"{label}.canonical_symbol must be normalized") - - -def validate_plan(plan: dict[str, Any]) -> None: - if not isinstance(plan, dict): - raise BlockedError("plan must be an object") - exact_fields(plan, PLAN_FIELDS, "plan") - if plan.get("schema") != PLAN_SCHEMA: - raise BlockedError(f"plan schema must be {PLAN_SCHEMA}") - if plan.get("plan_status") not in {"READY", "NO_ORDERS"}: - raise BlockedError("plan_status must be READY or NO_ORDERS") - if plan["execution_version"] != EXECUTION_VERSION: - raise BlockedError(f"execution_version must be {EXECUTION_VERSION}") - claimed = plan.get("plan_hash") - validate_sha256(claimed, "plan_hash") - unhashed = dict(plan) - unhashed.pop("plan_hash") - if sha256_json(unhashed) != claimed: - raise BlockedError("plan_hash does not match plan contents") - - context = plan.get("context") - if not isinstance(context, dict): - raise BlockedError("plan.context must be an object") - is_v2 = context.get("screen_schema") == SCREEN_SCHEMA_V2 - exact_fields( - context, - V2_CONTEXT_FIELDS if is_v2 else BASE_CONTEXT_FIELDS, - "plan.context", - ) - if context["execution_version"] != plan["execution_version"]: - raise BlockedError( - "plan.context.execution_version must equal plan.execution_version" - ) - for field in ( - "screen_hash", - "account_hash", - "exposure_hash", - "risk_hash", - "execution_policy_hash", - ): - validate_sha256(context[field], f"plan.context.{field}") - validate_nonempty_string(context["account_alias"], "plan.context.account_alias") - if context["broker"] not in {"kis", "toss"}: - raise BlockedError("plan.context.broker must be kis or toss") - if context["environment"] not in {"paper", "shadow", "live"}: - raise BlockedError("plan.context.environment must be paper, shadow, or live") - market = context.get("market") - if market not in ENTRY_WINDOWS: - raise BlockedError("plan.context.market is unsupported") - snapshot_max_age = validate_positive_integer( - context["snapshot_max_age_seconds"], - "plan.context.snapshot_max_age_seconds", - ) - if snapshot_max_age > MAX_SNAPSHOT_AGE_SECONDS: - raise BlockedError( - "plan.context.snapshot_max_age_seconds exceeds the hard maximum" - ) - market_session = context.get("market_session") - normalized_session = normalized_market_session( - market_session, - expected_market=market, - expected_timezone=ENTRY_WINDOWS[market]["timezone"], - ) - if normalized_session != market_session: - raise BlockedError("plan.context.market_session is not normalized") - if context.get("market_session_hash") != normalized_session["session_hash"]: - raise BlockedError( - "plan.context.market_session_hash does not match market_session" - ) - execution_policy = context["execution_policy"] - if not isinstance(execution_policy, dict): - raise BlockedError("plan.context.execution_policy must be an object") - normalized_policy = normalized_execution_policy(execution_policy) - if normalized_policy != execution_policy: - raise BlockedError("plan.context.execution_policy is not normalized") - if sha256_json(normalized_policy) != context["execution_policy_hash"]: - raise BlockedError( - "plan.context.execution_policy_hash does not match execution_policy" - ) - if ( - normalized_policy["market"] != market - or normalized_policy["market_session"] != normalized_session - or normalized_policy["snapshot_max_age_seconds"] != snapshot_max_age - ): - raise BlockedError("plan.context execution policy duplicates are inconsistent") - if is_v2: - if ( - context["candidate_order_contract"] - != "exchange_contract_order_then_exchange_rank_then_broker_symbol" - or context["exchange_order"] != list(V2_EXCHANGES_BY_MARKET[market]) - or context["broker_symbol_qualification"] != "KIS" - or context["broker"] != "kis" - ): - raise BlockedError("qta-screen/v2 plan context contract is invalid") - analysis_date = parse_iso_date( - context["analysis_date"], - "plan.context.analysis_date", - ) - if analysis_date.isoformat() != normalized_session["previous_session_date"]: - raise BlockedError( - "plan.context.analysis_date must equal previous_session_date" - ) - snapshot_as_of = parse_aware_datetime( - context["snapshot_as_of"], - "plan.context.snapshot_as_of", - ) - session_zone = ZoneInfo(normalized_session["timezone"]) - if ( - snapshot_as_of.astimezone(session_zone).date().isoformat() - != normalized_session["session_date"] - ): - raise BlockedError("plan.context.snapshot_as_of must fall on session_date") - source_as_of = parse_aware_datetime( - normalized_session["source_as_of"], - "market_session.source_as_of", - ) - regular_open = parse_aware_datetime( - normalized_session["regular_open"], - "market_session.regular_open", - ) - if not source_as_of <= snapshot_as_of <= regular_open: - raise BlockedError( - "plan.context.snapshot_as_of must be between source and open" - ) - if (regular_open - snapshot_as_of).total_seconds() > snapshot_max_age: - raise BlockedError("plan.context.snapshot_as_of exceeds snapshot freshness") - - entry_window = plan["entry_window"] - if not isinstance(entry_window, dict): - raise BlockedError("plan.entry_window must be an object") - exact_fields(entry_window, ENTRY_WINDOW_FIELDS, "plan.entry_window") - if entry_window["timezone"] != ENTRY_WINDOWS[market]["timezone"]: - raise BlockedError("plan.entry_window.timezone does not match market") - window_start = parse_aware_datetime( - entry_window["start"], "plan.entry_window.start" - ) - window_end = parse_aware_datetime(entry_window["end"], "plan.entry_window.end") - regular_open = parse_aware_datetime( - normalized_session["regular_open"], - "market_session.regular_open", - ) - regular_close = parse_aware_datetime( - normalized_session["regular_close"], - "market_session.regular_close", - ) - if ( - window_start != regular_open - or window_end != regular_open + timedelta(hours=1) - or window_end > regular_close - ): - raise BlockedError( - "plan.entry_window must be the first hour of the regular session" - ) - validate_positive_integer( - entry_window["poll_interval_seconds"], - "plan.entry_window.poll_interval_seconds", - ) - if entry_window != { - "timezone": normalized_policy["timezone"], - "start": normalized_policy["entry_window_start"], - "end": normalized_policy["entry_window_end"], - "poll_interval_seconds": normalized_policy["poll_interval_seconds"], - }: - raise BlockedError( - "plan.entry_window does not match the embedded execution policy" - ) - - quote_policy = plan["quote_policy"] - if not isinstance(quote_policy, dict): - raise BlockedError("plan.quote_policy must be an object") - exact_fields(quote_policy, QUOTE_POLICY_FIELDS, "plan.quote_policy") - validate_positive_integer( - quote_policy["max_age_seconds"], - "plan.quote_policy.max_age_seconds", - ) - for field in ("max_spread_bps", "max_gap_bps"): - value = validate_decimal_text( - quote_policy[field], - f"plan.quote_policy.{field}", - positive=False, - ) - if value > Decimal(10000): - raise BlockedError(f"plan.quote_policy.{field} must be <= 10000") - if quote_policy["trigger_mode"] not in {"AT_OR_ABOVE", "CROSS_FROM_BELOW"}: - raise BlockedError("plan.quote_policy.trigger_mode is unsupported") - if quote_policy != { - "max_age_seconds": normalized_policy["quote_max_age_seconds"], - "max_spread_bps": normalized_policy["max_spread_bps"], - "max_gap_bps": normalized_policy["max_gap_bps"], - "trigger_mode": normalized_policy["trigger_mode"], - }: - raise BlockedError( - "plan.quote_policy does not match the embedded execution policy" - ) - - order_policy = plan["order_policy"] - if not isinstance(order_policy, dict): - raise BlockedError("plan.order_policy must be an object") - exact_fields(order_policy, ORDER_POLICY_FIELDS, "plan.order_policy") - validate_positive_integer( - order_policy["ttl_seconds"], - "plan.order_policy.ttl_seconds", - ) - for field in ("allow_partial_fill", "cancel_remainder_at_window_end"): - if not isinstance(order_policy[field], bool): - raise BlockedError(f"plan.order_policy.{field} must be boolean") - if order_policy != { - "ttl_seconds": normalized_policy["order_ttl_seconds"], - "allow_partial_fill": normalized_policy["allow_partial_fill"], - "cancel_remainder_at_window_end": normalized_policy[ - "cancel_remainder_at_window_end" - ], - }: - raise BlockedError( - "plan.order_policy does not match the embedded execution policy" - ) - - settled_cash_start = validate_decimal_text( - plan["settled_cash_start"], - "plan.settled_cash_start", - positive=False, - ) - validate_decimal_text( - plan["borrowed_buying_power_excluded"], - "plan.borrowed_buying_power_excluded", - positive=False, - ) - settled_cash_unreserved = validate_decimal_text( - plan["settled_cash_unreserved"], - "plan.settled_cash_unreserved", - positive=False, - ) - if settled_cash_unreserved > settled_cash_start: - raise BlockedError( - "plan.settled_cash_unreserved must not exceed settled_cash_start" - ) - intents = plan["intents"] - if not isinstance(intents, list): - raise BlockedError("plan.intents must be an array") - if plan["plan_status"] != ("READY" if intents else "NO_ORDERS"): - raise BlockedError("plan_status does not match intents") - intent_ranks: list[int] = [] - intent_keys: set[tuple[str, str]] = set() - reserved_total = Decimal(0) - for index, intent in enumerate(intents): - rank, reserved_cash, exposure_key = validate_intent( - intent, - index=index, - context=context, - is_v2=is_v2, - ) - if exposure_key in intent_keys: - raise BlockedError("plan.intents contain duplicate exposure keys") - intent_keys.add(exposure_key) - intent_ranks.append(rank) - reserved_total += reserved_cash - if intent_ranks != sorted(set(intent_ranks)): - raise BlockedError("plan.intent ranks must be strictly increasing and unique") - if settled_cash_start - reserved_total != settled_cash_unreserved: - raise BlockedError("plan cash totals do not match reserved_cash across intents") - validate_skipped(plan["skipped"], market=market) - frozen_inputs = plan["frozen_inputs"] - if not isinstance(frozen_inputs, dict): - raise BlockedError("plan.frozen_inputs must be an object") - exact_fields(frozen_inputs, FROZEN_INPUT_FIELDS, "plan.frozen_inputs") - if any(not isinstance(frozen_inputs[field], dict) for field in FROZEN_INPUT_FIELDS): - raise BlockedError("every plan.frozen_inputs value must be an object") - rebuilt_plan = rebuild_order_plan( - frozen_inputs["screen"], - frozen_inputs["account"], - frozen_inputs["exposure"], - frozen_inputs["risk"], - frozen_inputs["execution"], - ) - if rebuilt_plan != plan: - raise BlockedError( - "plan does not match deterministic rebuild from frozen_inputs" - ) - - -def validate_broker_binding(plan: dict[str, Any], broker_name: str) -> None: - context = plan["context"] - expected_broker = "toss" if broker_name == "toss" else "kis" - if context["broker"] != expected_broker: - raise BlockedError("plan broker does not match selected adapter") - environment = context["environment"] - if broker_name == "kis-paper": - if environment != "paper": - raise BlockedError("KIS paper adapter requires a paper plan") - elif broker_name == "kis-live": - if environment not in {"shadow", "live"}: - raise BlockedError("KIS live adapter requires a shadow/live plan") - elif broker_name == "toss": - if environment not in {"shadow", "live"}: - raise BlockedError("Toss adapter requires a shadow/live plan") - else: - raise BlockedError(f"unsupported broker: {broker_name}") - - -def normalize_venue_map(value: dict[str, Any]) -> dict[str, str]: - if set(value) != {"schema", "venues"} or value.get("schema") != VENUE_SCHEMA: - raise BlockedError(f"venue map schema must be {VENUE_SCHEMA}") - venues = value["venues"] - if not isinstance(venues, dict): - raise BlockedError("venue map venues must be an object") - normalized: dict[str, str] = {} - for key, venue in venues.items(): - if not isinstance(key, str) or ":" not in key or not isinstance(venue, str): - raise BlockedError( - "venue keys must be MARKET:SYMBOL or EXCHANGE:SYMBOL strings" - ) - namespace, symbol = key.split(":", 1) - normalized_key = f"{namespace.strip().upper()}:{symbol.strip().upper()}" - normalized_venue = venue.strip().upper() - if not namespace.strip() or not symbol.strip() or not normalized_venue: - raise BlockedError("venue keys and values must be non-empty") - if ( - normalized_key in normalized - and normalized[normalized_key] != normalized_venue - ): - raise BlockedError(f"conflicting venue map entry for {normalized_key}") - normalized[normalized_key] = normalized_venue - return dict(sorted(normalized.items())) - - -def venue_for(venues: dict[str, str], intent: dict[str, Any]) -> str: - market = str(intent["market"]).upper() - symbol = str(intent["symbol"]).upper() - market_key = f"{market}:{symbol}" - intent_venue = intent.get("venue") - exchange = intent.get("exchange") - if intent_venue is None: - if market_key not in venues: - raise BlockedError(f"venue map missing {market_key}") - return venues[market_key] - - if not isinstance(intent_venue, str) or not intent_venue.strip(): - raise BlockedError("intent venue must be a non-empty string") - venue = intent_venue.strip().upper() - exchange_contracts = { - "KOSPI": ("KR", "KRX"), - "KOSDAQ": ("KR", "KRX"), - "NYSE": ("US", "NYSE"), - "NASDAQ": ("US", "NASD"), - } - if not isinstance(exchange, str) or exchange.upper() not in exchange_contracts: - raise BlockedError("intent venue requires a supported exchange") - normalized_exchange = exchange.upper() - expected_market, expected_venue = exchange_contracts[normalized_exchange] - if market != expected_market or venue != expected_venue: - raise BlockedError( - f"{normalized_exchange} intent requires market {expected_market} " - f"and venue {expected_venue}" - ) - - exchange_key = f"{normalized_exchange}:{symbol}" - mapped = venues.get(exchange_key, venues.get(market_key)) - if mapped is not None and mapped != venue: - raise BlockedError( - f"venue map {mapped} conflicts with hashed intent venue {venue} " - f"for {exchange_key}" - ) - return venue - - -def parse_timestamp(value: Any, field: str) -> datetime: - if not isinstance(value, str): - raise BlockedError(f"{field} must be an ISO-8601 string") - try: - parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) - except ValueError as exc: - raise BlockedError(f"{field} must be an ISO-8601 string") from exc - if parsed.tzinfo is None: - raise BlockedError(f"{field} must include a timezone") - return parsed - - -def evaluate_quote( - intent: dict[str, Any], - quote: dict[str, Any], - quote_policy: dict[str, Any], - now: datetime, - previous_last: Decimal | None, - *, - require_source_timestamp: bool, -) -> dict[str, Any]: - quality_reasons: list[str] = [] - if quote.get("raw_status") != "OK": - quality_reasons.append("quote_status_not_ok") - if quote.get("market") != intent.get("market") or quote.get("symbol") != intent.get( - "symbol" - ): - quality_reasons.append("quote_instrument_mismatch") - last = decimal_value(quote.get("last_price"), "last_price") - ask = decimal_value(quote.get("best_ask"), "best_ask") - bid = decimal_value(quote.get("best_bid"), "best_bid") - if bid <= 0 or ask <= 0 or last <= 0 or ask < bid: - quality_reasons.append("malformed_bid_ask") - received = parse_timestamp(quote.get("received_at"), "received_at") - age = ( - now.astimezone(timezone.utc) - received.astimezone(timezone.utc) - ).total_seconds() - if age < -1 or age > int(quote_policy["max_age_seconds"]): - quality_reasons.append("stale_quote") - source_timestamp_present = all( - bool(quote.get(field)) for field in ("trade_timestamp", "book_timestamp") - ) - if require_source_timestamp and not source_timestamp_present: - quality_reasons.append("source_timestamp_missing") - source_ages: list[float] = [] - for field in ("trade_timestamp", "book_timestamp"): - value = quote.get(field) - if not value: - continue - try: - source_time = parse_timestamp(value, field) - except BlockedError: - quality_reasons.append(f"{field}_invalid") - continue - source_ages.append( - ( - now.astimezone(timezone.utc) - source_time.astimezone(timezone.utc) - ).total_seconds() - ) - source_age = max(source_ages) if source_ages else None - if any( - item < -1 or item > int(quote_policy["max_age_seconds"]) for item in source_ages - ): - quality_reasons.append("stale_source_quote") - reasons = list(quality_reasons) - midpoint = (ask + bid) / Decimal(2) if ask >= bid > 0 else Decimal(0) - spread_bps = ( - Decimal(10000) * (ask - bid) / midpoint if midpoint > 0 else Decimal("Infinity") - ) - if spread_bps > decimal_value(quote_policy["max_spread_bps"], "max_spread_bps"): - reasons.append("spread_too_wide") - entry = decimal_value(intent["entry_trigger"], "entry_trigger") - limit_price = decimal_value(intent["limit_price"], "limit_price") - if ask > limit_price: - reasons.append("ask_above_limit") - trigger_mode = quote_policy["trigger_mode"] - if trigger_mode == "AT_OR_ABOVE": - crossed = last >= entry - elif trigger_mode == "CROSS_FROM_BELOW": - crossed = previous_last is not None and previous_last < entry <= last - else: - raise BlockedError(f"unsupported trigger_mode: {trigger_mode}") - if not crossed: - reasons.append("entry_not_triggered") - return { - "triggered": not reasons, - "quality_valid": not quality_reasons, - "reasons": reasons, - "last_price": format(last, "f"), - "best_ask": format(ask, "f"), - "best_bid": format(bid, "f"), - "spread_bps": format(spread_bps, "f"), - "quote_age_seconds": f"{age:.6f}", - "source_timestamp_present": source_timestamp_present, - "source_age_seconds": None if source_age is None else f"{source_age:.6f}", - } - - -def normalize_toss_order(raw: dict[str, Any]) -> dict[str, Any]: - raw_status = str(raw.get("status", "UNKNOWN")) - status_map = { - "PENDING": "ACKNOWLEDGED", - "PENDING_CANCEL": "CANCEL_PENDING", - "PENDING_REPLACE": "UNKNOWN", - "PARTIAL_FILLED": "PARTIALLY_FILLED", - "FILLED": "FILLED", - "CANCELED": "CANCELLED", - "REJECTED": "REJECTED", - "CANCEL_REJECTED": "UNKNOWN", - "REPLACE_REJECTED": "UNKNOWN", - "REPLACED": "UNKNOWN", - } - execution = raw.get("execution") - if not isinstance(execution, dict): - execution = {} - return { - "normalized_status": status_map.get(raw_status, "UNKNOWN"), - "raw_status": raw_status, - "filled_quantity": execution.get("filledQuantity"), - "average_fill_price": execution.get("averageFilledPrice"), - "raw": redact(raw), - } - - -def preview_requests( - plan: dict[str, Any], broker_name: str, venues: dict[str, str] -) -> dict[str, Any]: - validate_plan(plan) - previews: list[dict[str, Any]] = [] - if broker_name == "toss": - broker: Any = TossBroker( - client_id="preview", - client_secret="preview", - account_seq=1, - ) - for intent in plan["intents"]: - previews.append( - { - "intent_id": intent["intent_id"], - "request": broker.preview_submit(intent), - } - ) - elif broker_name in {"kis-paper", "kis-live"}: - environment = broker_name.split("-", 1)[1] - broker = KisBroker( - app_key="preview", - app_secret="preview", - account_prefix="00000000", - account_product="01", - environment=environment, - access_token="preview", - ) - for intent in plan["intents"]: - previews.append( - { - "intent_id": intent["intent_id"], - "request": broker.preview_submit( - intent, venue=venue_for(venues, intent) - ), - } - ) - else: - raise BlockedError(f"unsupported broker: {broker_name}") - output = { - "schema": "qta-request-preview/v1", - "status": "READY", - "plan_hash": plan["plan_hash"], - "broker": broker_name, - "account_bound": False, - "request_hash_scope": "placeholder_account", - "trigger_evaluated": False, - "mutation_sent": False, - "previews": previews, - } - output["preview_hash"] = sha256_json(output) - return output - - -def require_environment(name: str) -> str: - value = os.environ.get(name) - if not value: - raise BlockedError(f"required environment variable is absent: {name}") - return value - - -def runtime_toss_account_seq() -> int: - raw = require_environment("QTA_TOSS_ACCOUNT_SEQ") - try: - account_seq = int(raw) - except ValueError as exc: - raise BlockedError( - "QTA_TOSS_ACCOUNT_SEQ must be a canonical positive integer" - ) from exc - if account_seq <= 0 or str(account_seq) != raw: - raise BlockedError("QTA_TOSS_ACCOUNT_SEQ must be a canonical positive integer") - return account_seq - - -def create_broker(broker_name: str) -> Any: - if broker_name == "toss": - return TossBroker( - client_id=require_environment("QTA_TOSS_CLIENT_ID"), - client_secret=require_environment("QTA_TOSS_CLIENT_SECRET"), - account_seq=runtime_toss_account_seq(), - access_token=os.environ.get("QTA_TOSS_ACCESS_TOKEN"), - access_token_expires_at=os.environ.get("QTA_TOSS_ACCESS_TOKEN_EXPIRES_AT"), - ) - if broker_name in {"kis-paper", "kis-live"}: - environment = broker_name.split("-", 1)[1] - return KisBroker( - app_key=require_environment("QTA_KIS_APP_KEY"), - app_secret=require_environment("QTA_KIS_APP_SECRET"), - account_prefix=require_environment("QTA_KIS_ACCOUNT_PREFIX"), - account_product=require_environment("QTA_KIS_ACCOUNT_PRODUCT"), - environment=environment, - access_token=os.environ.get("QTA_KIS_ACCESS_TOKEN"), - ) - raise BlockedError(f"unsupported broker: {broker_name}") - - -def validate_mode(plan: dict[str, Any], broker_name: str, mode: str) -> None: - validate_broker_binding(plan, broker_name) - context = plan["context"] - screen_method = plan["frozen_inputs"]["screen"].get("method_version") - if screen_method == "qta-2.0.0" and mode != "shadow": - raise BlockedError( - "qta-2.0.0 is RESEARCH_ONLY and may run only in shadow mode" - ) - if mode == "live": - raise BlockedError( - "live promotion is intentionally disabled until accepted-timeout " - "recovery, terminal cancel verification, and OCO protection are complete" - ) - if mode == "paper": - if broker_name != "kis-paper" or context.get("environment") != "paper": - raise BlockedError("paper mode requires a KIS paper plan and adapter") - elif mode == "shadow": - if context.get("environment") not in {"shadow", "live"}: - raise BlockedError("shadow mode requires a shadow/live account snapshot") - else: - raise BlockedError("mode must be paper, shadow, or live") - - -def validate_runtime_capabilities(plan: dict[str, Any], broker_name: str) -> None: - if broker_name == "kis-paper" and any( - str(intent.get("market", "")).upper() == "US" - for intent in plan.get("intents", []) - if isinstance(intent, dict) - ): - raise BlockedError( - "KIS U.S. paper sessions are unsupported because " - "HHDFS76200100 order-book quotes have no paper TR; " - "use KIS live production shadow quotes and test paper order " - "serialization separately" - ) - - -def preflight_venues( - plan: dict[str, Any], - venues: dict[str, str], -) -> dict[str, str]: - return { - str(intent["intent_id"]): venue_for(venues, intent) - for intent in plan["intents"] - } - - -def preflight_submit_requests( - broker: Any, - broker_name: str, - plan: dict[str, Any], - venue_by_intent: dict[str, str], -) -> dict[str, dict[str, Any]]: - return { - str(intent["intent_id"]): ( - broker.preview_submit(intent) - if broker_name == "toss" - else broker.preview_submit( - intent, - venue=venue_by_intent[str(intent["intent_id"])], - ) - ) - for intent in plan["intents"] - } - - -def quote_http_request_count( - broker_name: str, - intent: dict[str, Any], -) -> int: - if broker_name == "kis-live" and str(intent.get("market", "")).upper() == "US": - return 1 - return DEFAULT_QUOTE_HTTP_REQUESTS_PER_INTENT - - -def plan_uses_qta2(plan: dict[str, Any]) -> bool: - frozen_inputs = plan.get("frozen_inputs") - if not isinstance(frozen_inputs, dict): - return False - screen = frozen_inputs.get("screen") - return ( - isinstance(screen, dict) - and screen.get("method_version") == QTA_METHOD_V2 - ) - - -def active_qta2_exchanges(plan: dict[str, Any]) -> list[str]: - if not plan_uses_qta2(plan): - return [] - present = { - str(intent.get("exchange") or "").upper() - for intent in plan.get("intents", []) - if isinstance(intent, dict) - } - market = str(plan.get("context", {}).get("market") or "").upper() - return [ - exchange - for exchange in V2_EXCHANGES_BY_MARKET.get(market, ()) - if exchange in present - ] - - -def polling_admission( - plan: dict[str, Any], - broker_name: str, - mode: str, -) -> dict[str, Any]: - """Fail closed when a full deterministic cycle cannot fit broker pacing.""" - if broker_name not in BROKER_PACING_SECONDS: - raise BlockedError(f"unsupported broker: {broker_name}") - intent_count = len(plan["intents"]) - quote_requests = sum( - quote_http_request_count(broker_name, intent) for intent in plan["intents"] - ) - regime_requests = len(active_qta2_exchanges(plan)) - mutation_requests = intent_count if mode == "paper" else 0 - request_budget = quote_requests + regime_requests + mutation_requests - pacing_seconds = BROKER_PACING_SECONDS[broker_name] * request_budget - minimum_interval = max( - 1, - int(pacing_seconds.to_integral_value(rounding=ROUND_CEILING)), - ) - configured_interval = int(plan["entry_window"]["poll_interval_seconds"]) - if configured_interval < minimum_interval: - raise BlockedError( - f"{broker_name} {mode} cycle with {intent_count} intents requires " - f"poll_interval_seconds >= {minimum_interval} for " - f"{request_budget} paced requests; got {configured_interval}" - ) - return { - "intent_count": intent_count, - "quote_request_budget": quote_requests, - "quote_request_contract": ( - "KIS-live US=1; KIS KR and Toss=2 per waiting intent" - ), - "market_regime_request_budget": regime_requests, - "worst_case_mutation_requests": mutation_requests, - "worst_case_request_budget": request_budget, - "minimum_request_interval_ms": int(BROKER_PACING_SECONDS[broker_name] * 1000), - "minimum_poll_interval_seconds": minimum_interval, - "configured_poll_interval_seconds": configured_interval, - "network_and_persistence_headroom_seconds": format( - Decimal(configured_interval) - pacing_seconds, - "f", - ), - } - - -def evaluate_market_regime( - *, - exchange: str, - quote: dict[str, Any], - observed_at: datetime, - session_date: str, -) -> dict[str, Any]: - """Validate and deterministically admit one QTA2 exchange regime quote.""" - exchange = str(exchange).upper() - if exchange not in V2_EXCHANGE_CONTRACTS: - raise BlockedError(f"unsupported market regime exchange: {exchange}") - if observed_at.tzinfo is None or observed_at.utcoffset() is None: - raise BlockedError("market regime observed_at must be timezone-aware") - exact_fields(quote, MARKET_REGIME_QUOTE_FIELDS, "market regime quote") - reasons: list[str] = [] - expected_benchmark = V2_EXCHANGE_CONTRACTS[exchange]["benchmark_id"] - if quote["schema"] != "qta-market-regime-quote/v1": - reasons.append("invalid_schema") - if quote["broker"] != "kis": - reasons.append("invalid_broker") - if quote["exchange"] != exchange: - reasons.append("exchange_mismatch") - if quote["benchmark_id"] != expected_benchmark: - reasons.append("benchmark_mismatch") - if quote["regime_proxy_id"] != REGIME_PROXY_BY_EXCHANGE[exchange]: - reasons.append("proxy_mismatch") - if quote["raw_status"] != "OK": - reasons.append("raw_status_not_ok") - - current = positive_decimal(quote["current"], "market regime current") - previous_close = positive_decimal( - quote["previous_close"], - "market regime previous_close", - ) - change_bps = decimal_value( - quote["change_bps"], - "market regime change_bps", - ) - recomputed_change = ( - (current / previous_close - Decimal(1)) * Decimal(10000) - ) - if abs(change_bps - recomputed_change) > Decimal("0.02"): - reasons.append("change_bps_mismatch") - - zone_name = ( - "Asia/Seoul" - if exchange in {"KOSPI", "KOSDAQ"} - else "America/New_York" - ) - for timestamp_field in ("source_timestamp", "received_at"): - try: - timestamp = parse_aware_datetime( - quote[timestamp_field], - f"market regime {timestamp_field}", - ) - except BlockedError: - reasons.append(f"{timestamp_field}_invalid") - continue - age_seconds = ( - observed_at.astimezone(timezone.utc) - - timestamp.astimezone(timezone.utc) - ).total_seconds() - if age_seconds < -1 or age_seconds > QTA2_REGIME_MAX_AGE_SECONDS: - reasons.append(f"{timestamp_field}_stale") - if ( - timestamp_field == "source_timestamp" - and timestamp.astimezone(ZoneInfo(zone_name)).date().isoformat() - != session_date - ): - reasons.append("source_session_date_mismatch") - - quality_valid = not reasons - admitted = ( - quality_valid - and change_bps >= QTA2_REGIME_MINIMUM_CHANGE_BPS - ) - if quality_valid and not admitted: - reasons.append("benchmark_below_minimum") - without_hash = { - "metric": QTA2_REGIME_METRIC, - "exchange": exchange, - "benchmark_id": expected_benchmark, - "regime_proxy_id": quote["regime_proxy_id"], - "minimum_change_bps": format( - QTA2_REGIME_MINIMUM_CHANGE_BPS, - "f", - ), - "observed_change_bps": format(change_bps, "f"), - "max_age_seconds": QTA2_REGIME_MAX_AGE_SECONDS, - "quality_valid": quality_valid, - "admitted": admitted, - "reasons": reasons, - "quote": quote, - "observed_at": observed_at.isoformat(), - } - return {**without_hash, "decision_hash": sha256_json(without_hash)} - - -def wait_until(target: datetime) -> None: - while True: - remaining = target.timestamp() - datetime.now(timezone.utc).timestamp() - if remaining <= 0: - return - time.sleep(min(remaining, 60)) - - -def warm_broker_before_open(broker: Any, start: datetime) -> dict[str, Any]: - warmup_at = start - timedelta(seconds=PREOPEN_WARMUP_SECONDS) - now = datetime.now(timezone.utc) - if now >= start: - raise BlockedError( - "session runner must start before the regular open for pre-open " - "authentication warm-up" - ) - wait_until(warmup_at) - started_at = datetime.now(timezone.utc) - started_monotonic = time.monotonic() - try: - broker.token() - except (BlockedError, TransportFailure) as exc: - raise BlockedError(f"pre-open broker authentication failed: {exc}") from exc - completed_at = datetime.now(timezone.utc) - if completed_at >= start: - raise BlockedError( - "broker authentication did not complete before the regular open" - ) - return { - "scheduled_at": warmup_at.isoformat(), - "started_at": started_at.isoformat(), - "completed_at": completed_at.isoformat(), - "latency_ms": int((time.monotonic() - started_monotonic) * 1000), - "token_material_logged": False, - } - - -def entry_window_open(end: datetime, now: datetime | None = None) -> bool: - observed = now or datetime.now(timezone.utc) - return observed.timestamp() < end.timestamp() - - -def require_remaining_quantity(snapshot: dict[str, Any]) -> str: - remaining = decimal_value(snapshot.get("remaining_quantity"), "remaining_quantity") - if remaining <= 0 or remaining != remaining.to_integral_value(): - raise BlockedError( - "KIS cancel requires a positive whole current remaining quantity" - ) - return format(remaining, "f") - - -def current_order( - broker: Any, - broker_name: str, - *, - intent: dict[str, Any], - ack: dict[str, Any], - venue: str, - trading_date: str, -) -> dict[str, Any]: - if broker_name == "toss": - return normalize_toss_order(broker.get_order(ack["broker_order_id"])) - return broker.get_order( - market=intent["market"], - trading_date=trading_date, - symbol=intent["symbol"], - venue=venue, - broker_order_id=ack["broker_order_id"], - ) - - -def cancel_order( - broker: Any, - broker_name: str, - *, - intent: dict[str, Any], - ack: dict[str, Any], - venue: str, - quantity: str | None = None, -) -> dict[str, Any]: - if broker_name == "toss": - return broker.cancel(ack["broker_order_id"]) - if quantity is None: - raise BlockedError("KIS cancel requires current remaining quantity") - return broker.cancel( - market=intent["market"], - broker_order_id=ack["broker_order_id"], - symbol=intent["symbol"], - quantity=quantity, - price=intent["limit_price"], - venue=venue, - organization=str(ack.get("organization") or ""), - ) - - -def session_status( - plan: dict[str, Any], - broker_name: str, - mode: str, - ledger: Ledger, - started_at: datetime, - stopped_at: datetime, - freeze_reason: str | None, - warmup: dict[str, Any], - polling_metrics: dict[str, Any], -) -> dict[str, Any]: - intents = [] - for intent in plan["intents"]: - record = ledger.get(intent["intent_id"]) - intents.append( - { - "intent_id": record.intent_id, - "state": record.state, - "broker_order_id": record.broker_order_id, - "request_hash": record.request_hash, - "event_count": len(ledger.events(record.intent_id)), - } - ) - output = { - "schema": SESSION_STATUS_SCHEMA, - "plan_hash": plan["plan_hash"], - "broker": broker_name, - "mode": mode, - "started_at": started_at.isoformat(), - "stopped_at": stopped_at.isoformat(), - "entry_freeze": freeze_reason is not None, - "freeze_reason": freeze_reason, - "preopen_warmup": warmup, - "polling_metrics": polling_metrics, - "intents": intents, - } - return output - - -def validate_ledger_scope(ledger: Ledger, plan: dict[str, Any]) -> None: - current_intent_ids = { - str(intent["intent_id"]) - for intent in plan["intents"] - if isinstance(intent, dict) and "intent_id" in intent - } - foreign = [ - binding - for binding in ledger.nonterminal_bindings() - if binding["plan_hash"] != plan["plan_hash"] - or binding["intent_id"] not in current_intent_ids - ] - if foreign: - states = sorted({binding["state"] for binding in foreign}) - raise BlockedError( - "state directory contains nonterminal intents outside the current " - f"plan (states={states}); reconcile the prior plan before running " - "a new one" - ) - unresolved_states = { - "RESERVED", - "SUBMITTING", - "UNKNOWN", - "RECONCILING", - "CANCEL_PENDING", - "MANUAL_BLOCK", - } - unresolved = [ - binding - for binding in ledger.nonterminal_bindings() - if binding["state"] in unresolved_states - ] - if unresolved: - states = sorted({binding["state"] for binding in unresolved}) - raise BlockedError( - "state directory contains unresolved mutation states " - f"(states={states}); broker evidence and explicit clearance are " - "required before new entries" - ) - - -def position_ledger_path(state_directory: Path) -> Path: - """Return the account/market ledger shared by all session-date directories.""" - if not state_directory.is_absolute(): - raise BlockedError("state_directory must be absolute") - return state_directory.parent / "positions.sqlite3" - - -def persist_entry_fill( - position_ledger: PositionLedger, - *, - plan: dict[str, Any], - intent: dict[str, Any], - snapshot: dict[str, Any], - session_date: str, -) -> dict[str, Any] | None: - """Persist an authoritative cumulative entry fill exactly once per change.""" - raw_filled = snapshot.get("filled_quantity") - if raw_filled in (None, ""): - return None - cumulative = decimal_value(raw_filled, "broker filled_quantity") - if cumulative < 0 or cumulative != cumulative.to_integral_value(): - raise BlockedError( - "broker filled_quantity must be a nonnegative whole number" - ) - if cumulative == 0: - return None - raw_average = snapshot.get("average_fill_price") - if raw_average in (None, ""): - raise BlockedError( - "broker reported an entry fill without average_fill_price" - ) - average = positive_decimal(raw_average, "broker average_fill_price") - account = plan["frozen_inputs"]["account"] - fx_to_krw = account.get("fx_to_krw") - if intent["market"] == "KR": - fx_to_krw = "1" - elif fx_to_krw in (None, ""): - raise BlockedError( - "a U.S. strategy position requires frozen fx_to_krw" - ) - - position_id = strategy_position_id(plan["plan_hash"], intent) - stored = position_ledger.get_optional(position_id) - if stored is None: - position = position_from_intent( - plan_hash=plan["plan_hash"], - intent=intent, - session_date=session_date, - fx_to_krw=fx_to_krw, - ) - event_type = "ENTRY_FILL_CREATED" - else: - position = stored.payload - if ( - position["plan_hash"] != plan["plan_hash"] - or position["entry_intent_id"] != intent["intent_id"] - ): - raise BlockedError( - "strategy position identity differs from plan entry intent" - ) - prior_quantity = Decimal(position["acquired_quantity"]) - prior_average = ( - None - if position["average_entry_price"] is None - else Decimal(position["average_entry_price"]) - ) - if cumulative == prior_quantity and average == prior_average: - return position - if Decimal(position["exited_quantity"]) > 0: - raise BlockedError( - "entry fill changed after exit fills were recorded" - ) - event_type = "ENTRY_FILL_UPDATED" - - updated = record_entry_fill( - position, - cumulative_filled_quantity=format(cumulative, "f"), - average_fill_price=format(average, "f"), - session_date=session_date, - ) - position_ledger.put( - updated, - event_type=event_type, - event_payload={ - "broker_order_id": snapshot.get("broker_order_id"), - "normalized_status": snapshot.get("normalized_status"), - "ordered_quantity": snapshot.get("ordered_quantity"), - "filled_quantity": format(cumulative, "f"), - "average_fill_price": format(average, "f"), - "remaining_quantity": snapshot.get("remaining_quantity"), - "session_date": session_date, - }, - ) - return updated - - -def run_session( - plan: dict[str, Any], - broker_name: str, - mode: str, - venues: dict[str, str], - state_directory: Path, - *, - max_cycles: int | None = None, -) -> dict[str, Any]: - validate_plan(plan) - validate_mode(plan, broker_name, mode) - if plan["plan_status"] == "NO_ORDERS": - state_directory.mkdir(parents=True, exist_ok=True) - with SingleWriterLock(state_directory / "writer.lock"): - ledger = Ledger(state_directory / "ledger.sqlite3") - try: - validate_ledger_scope(ledger, plan) - finally: - ledger.close() - return { - "schema": SESSION_STATUS_SCHEMA, - "plan_hash": plan["plan_hash"], - "broker": broker_name, - "mode": mode, - "status": "NO_ORDERS", - "mutation_sent": False, - } - validate_runtime_capabilities(plan, broker_name) - venue_by_intent = preflight_venues(plan, venues) - start = parse_timestamp(plan["entry_window"]["start"], "entry window start") - end = parse_timestamp(plan["entry_window"]["end"], "entry window end") - interval = int(plan["entry_window"]["poll_interval_seconds"]) - admission = polling_admission(plan, broker_name, mode) - broker = create_broker(broker_name) - preview_by_intent = preflight_submit_requests( - broker, - broker_name, - plan, - venue_by_intent, - ) - trading_date = start.date().isoformat() - state_directory.mkdir(parents=True, exist_ok=True) - started_at = datetime.now(timezone.utc) - freeze_reason: str | None = None - previous_last: dict[str, Decimal] = {} - acknowledgements: dict[str, dict[str, Any]] = {} - warmup: dict[str, Any] = {} - polling_metrics: dict[str, Any] = { - "admission": admission, - "telemetry_anchor_intent_id": plan["intents"][0]["intent_id"], - "cycles_started": 0, - "cycles_completed": 0, - "cycles_skipped": 0, - "quotes_evaluated": 0, - "market_regime_queries": 0, - "market_regime_passes": 0, - "market_regime_blocks": 0, - "order_status_queries": 0, - "submits_started": 0, - "expected_http_requests_started": 0, - "max_schedule_lateness_ms": 0, - "max_cycle_duration_ms": 0, - "max_quote_latency_ms": 0, - "max_market_regime_latency_ms": 0, - "max_order_status_latency_ms": 0, - "max_submit_latency_ms": 0, - "position_reconciliation": None, - } - - with SingleWriterLock(state_directory / "writer.lock"): - ledger = Ledger(state_directory / "ledger.sqlite3") - position_ledger = PositionLedger(position_ledger_path(state_directory)) - try: - validate_ledger_scope(ledger, plan) - position_reconciliation = reconcile_account_positions( - position_ledger, - account_snapshot=plan["frozen_inputs"]["account"], - session_date=trading_date, - ) - polling_metrics["position_reconciliation"] = { - "status": position_reconciliation["status"], - "reconciliation_hash": position_reconciliation[ - "reconciliation_hash" - ], - "managed_position_count": len( - position_reconciliation["managed_positions"] - ), - "unmanaged_broker_position_count": len( - position_reconciliation["unmanaged_broker_positions"] - ), - } - if position_reconciliation["status"] != "READY": - raise BlockedError( - "strategy position ledger differs from the frozen broker " - "account snapshot; reconcile MANUAL_BLOCK before new entries" - ) - warmup = warm_broker_before_open(broker, start) - for intent in plan["intents"]: - ledger.create_intent(plan["plan_hash"], intent) - if ledger.get(intent["intent_id"]).state == "PLANNED": - ledger.transition(intent["intent_id"], "WAIT_TRIGGER", {}) - - cycle = 0 - while True: - target = start.timestamp() + cycle * interval - now = datetime.now(timezone.utc) - if now.timestamp() >= end.timestamp(): - freeze_reason = "entry_window_closed" - break - if max_cycles is not None and cycle >= max_cycles: - freeze_reason = "max_cycles_reached" - break - if now.timestamp() < target: - time.sleep(min(target - now.timestamp(), 60)) - continue - if now.timestamp() >= target + interval: - skipped_count = max( - 1, - int((now.timestamp() - target) // interval), - ) - if max_cycles is not None: - skipped_count = min( - skipped_count, - max_cycles - cycle, - ) - lateness_ms = int((now.timestamp() - target) * 1000) - polling_metrics["cycles_skipped"] += skipped_count - polling_metrics["max_schedule_lateness_ms"] = max( - polling_metrics["max_schedule_lateness_ms"], - lateness_ms, - ) - ledger.append_event( - plan["intents"][0]["intent_id"], - "CYCLE_SKIPPED", - { - "first_cycle": cycle, - "last_cycle": cycle + skipped_count - 1, - "skipped_count": skipped_count, - "scheduled_at": datetime.fromtimestamp( - target, timezone.utc - ).isoformat(), - "observed_at": now.isoformat(), - "lateness_ms": lateness_ms, - "reason": "scheduled_cycle_expired", - }, - ) - cycle += skipped_count - continue - - cycle_started_at = now - cycle_started_monotonic = time.monotonic() - cycle_deadline = min(target + interval, end.timestamp()) - lateness_ms = max(0, int((now.timestamp() - target) * 1000)) - polling_metrics["cycles_started"] += 1 - polling_metrics["max_schedule_lateness_ms"] = max( - polling_metrics["max_schedule_lateness_ms"], - lateness_ms, - ) - cycle_events: list[tuple[str, str, dict[str, Any]]] = [ - ( - plan["intents"][0]["intent_id"], - "CYCLE_START", - { - "cycle": cycle, - "scheduled_at": datetime.fromtimestamp( - target, timezone.utc - ).isoformat(), - "started_at": cycle_started_at.isoformat(), - "deadline_at": datetime.fromtimestamp( - cycle_deadline, timezone.utc - ).isoformat(), - "lateness_ms": lateness_ms, - }, - ) - ] - regime_by_exchange: dict[str, dict[str, Any]] = {} - if plan_uses_qta2(plan): - waiting_by_exchange: dict[str, list[dict[str, Any]]] = {} - for candidate in plan["intents"]: - if ledger.get(candidate["intent_id"]).state == "WAIT_TRIGGER": - waiting_by_exchange.setdefault( - candidate["exchange"], - [], - ).append(candidate) - for exchange in active_qta2_exchanges(plan): - waiting = waiting_by_exchange.get(exchange, []) - if not waiting: - continue - if datetime.now(timezone.utc).timestamp() >= cycle_deadline: - freeze_reason = "cycle_latency_budget_exceeded" - break - regime_started_at = datetime.now(timezone.utc) - regime_started_monotonic = time.monotonic() - polling_metrics["market_regime_queries"] += 1 - polling_metrics["expected_http_requests_started"] += 1 - try: - regime_quote = broker.benchmark_quote( - exchange=exchange, - session_date=trading_date, - ) - regime_observed_at = datetime.now(timezone.utc) - regime_decision = evaluate_market_regime( - exchange=exchange, - quote=regime_quote, - observed_at=regime_observed_at, - session_date=trading_date, - ) - except (BlockedError, TransportFailure) as exc: - regime_decision = { - "metric": QTA2_REGIME_METRIC, - "exchange": exchange, - "benchmark_id": V2_EXCHANGE_CONTRACTS[exchange][ - "benchmark_id" - ], - "minimum_change_bps": format( - QTA2_REGIME_MINIMUM_CHANGE_BPS, - "f", - ), - "max_age_seconds": QTA2_REGIME_MAX_AGE_SECONDS, - "quality_valid": False, - "admitted": False, - "reasons": ["benchmark_quote_unavailable"], - "error": str(exc), - "observed_at": datetime.now( - timezone.utc - ).isoformat(), - } - regime_latency_ms = int( - (time.monotonic() - regime_started_monotonic) * 1000 - ) - regime_decision["latency_ms"] = regime_latency_ms - regime_decision["started_at"] = ( - regime_started_at.isoformat() - ) - regime_by_exchange[exchange] = regime_decision - polling_metrics["max_market_regime_latency_ms"] = max( - polling_metrics["max_market_regime_latency_ms"], - regime_latency_ms, - ) - metric = ( - "market_regime_passes" - if regime_decision["admitted"] - else "market_regime_blocks" - ) - polling_metrics[metric] += 1 - cycle_events.append( - ( - waiting[0]["intent_id"], - "MARKET_REGIME_EVALUATED", - regime_decision, - ) - ) - if freeze_reason == "cycle_latency_budget_exceeded": - ledger.append_events(cycle_events) - break - for intent in plan["intents"]: - if datetime.now(timezone.utc).timestamp() >= cycle_deadline: - freeze_reason = "cycle_latency_budget_exceeded" - break - record = ledger.get(intent["intent_id"]) - venue = venue_by_intent[intent["intent_id"]] - if record.state == "WAIT_TRIGGER": - if plan_uses_qta2(plan): - regime_decision = regime_by_exchange.get( - intent["exchange"] - ) - if ( - regime_decision is None - or not regime_decision["admitted"] - ): - cycle_events.append( - ( - intent["intent_id"], - "ENTRY_BLOCKED_BY_MARKET_REGIME", - { - "exchange": intent["exchange"], - "decision": regime_decision, - }, - ) - ) - continue - if not entry_window_open(end): - freeze_reason = "entry_window_closed" - break - quote_started_at = datetime.now(timezone.utc) - quote_started_monotonic = time.monotonic() - polling_metrics["expected_http_requests_started"] += ( - quote_http_request_count(broker_name, intent) - ) - try: - quote = ( - broker.quote( - intent["market"], intent["symbol"], venue=venue - ) - if broker_name != "toss" - else broker.quote(intent["market"], intent["symbol"]) - ) - except (BlockedError, TransportFailure) as exc: - quote_latency_ms = int( - (time.monotonic() - quote_started_monotonic) * 1000 - ) - polling_metrics["max_quote_latency_ms"] = max( - polling_metrics["max_quote_latency_ms"], - quote_latency_ms, - ) - cycle_events.append( - ( - intent["intent_id"], - "QUOTE_BLOCKED", - { - "reason": str(exc), - "started_at": quote_started_at.isoformat(), - "completed_at": datetime.now( - timezone.utc - ).isoformat(), - "latency_ms": quote_latency_ms, - }, - ) - ) - freeze_reason = "quote_failure" - break - quote_completed_at = datetime.now(timezone.utc) - quote_latency_ms = int( - (time.monotonic() - quote_started_monotonic) * 1000 - ) - polling_metrics["quotes_evaluated"] += 1 - polling_metrics["max_quote_latency_ms"] = max( - polling_metrics["max_quote_latency_ms"], - quote_latency_ms, - ) - decision = evaluate_quote( - intent, - quote, - plan["quote_policy"], - quote_completed_at, - previous_last.get(intent["intent_id"]), - require_source_timestamp=True, - ) - decision.update( - { - "quote_started_at": quote_started_at.isoformat(), - "quote_completed_at": quote_completed_at.isoformat(), - "quote_latency_ms": quote_latency_ms, - } - ) - if quote_completed_at.timestamp() >= cycle_deadline: - decision["triggered"] = False - decision["reasons"].append("cycle_latency_budget_exceeded") - freeze_reason = "cycle_latency_budget_exceeded" - if decision["quality_valid"]: - previous_last[intent["intent_id"]] = decimal_value( - quote["last_price"], "last_price" - ) - cycle_events.append( - ( - intent["intent_id"], - "QUOTE_EVALUATED", - decision, - ) - ) - if freeze_reason == "cycle_latency_budget_exceeded": - break - if not decision["triggered"]: - continue - if not entry_window_open(end): - freeze_reason = "entry_window_closed" - break - if mode == "shadow": - shadow_request = preview_by_intent[intent["intent_id"]] - cycle_events.append( - ( - intent["intent_id"], - "SHADOW_WOULD_SUBMIT", - {"request": shadow_request}, - ) - ) - ledger.append_events(cycle_events) - cycle_events = [] - ledger.transition( - intent["intent_id"], - "CANCELLED", - {"reason": "shadow_only"}, - ) - continue - submit_safety_seconds = SUBMIT_HTTP_TIMEOUT_SECONDS + float( - BROKER_PACING_SECONDS[broker_name] - ) - submit_cutoff = end - timedelta(seconds=submit_safety_seconds) - if not entry_window_open(submit_cutoff): - ledger.append_events(cycle_events) - cycle_events = [] - ledger.transition( - intent["intent_id"], - "CANCELLED", - {"reason": "submit_safety_cutoff_reached"}, - ) - freeze_reason = "submit_safety_cutoff_reached" - break - ledger.append_events(cycle_events) - cycle_events = [] - ledger.transition(intent["intent_id"], "RESERVED", decision) - preview = preview_by_intent[intent["intent_id"]] - if not entry_window_open(end): - ledger.transition( - intent["intent_id"], - "CANCELLED", - {"reason": "entry_window_closed_before_submit"}, - ) - freeze_reason = "entry_window_closed" - break - if datetime.now(timezone.utc).timestamp() >= cycle_deadline: - ledger.transition( - intent["intent_id"], - "CANCELLED", - {"reason": "cycle_latency_budget_exceeded"}, - ) - freeze_reason = "cycle_latency_budget_exceeded" - break - ledger.transition( - intent["intent_id"], - "SUBMITTING", - {"request_hash": preview["request_hash"]}, - request_hash=preview["request_hash"], - ) - submit_started_at = datetime.now(timezone.utc) - submit_started_monotonic = time.monotonic() - polling_metrics["submits_started"] += 1 - polling_metrics["expected_http_requests_started"] += 1 - try: - ack = ( - broker.submit(intent, deadline_at=end) - if broker_name == "toss" - else broker.submit( - intent, - venue=venue, - deadline_at=end, - ) - ) - except AmbiguousMutationError as exc: - polling_metrics["max_submit_latency_ms"] = max( - polling_metrics["max_submit_latency_ms"], - int( - (time.monotonic() - submit_started_monotonic) * 1000 - ), - ) - ledger.transition( - intent["intent_id"], - "UNKNOWN", - { - "reason": str(exc), - "submit_started_at": ( - submit_started_at.isoformat() - ), - "completed_at": datetime.now( - timezone.utc - ).isoformat(), - }, - ) - freeze_reason = "ambiguous_mutation" - break - except AuthoritativeMutationRejection as exc: - ledger.transition( - intent["intent_id"], - "REJECTED", - { - "reason": str(exc), - "submit_started_at": ( - submit_started_at.isoformat() - ), - "completed_at": datetime.now( - timezone.utc - ).isoformat(), - }, - ) - freeze_reason = "submit_rejected" - break - except (BlockedError, TransportFailure) as exc: - ledger.transition( - intent["intent_id"], - "UNKNOWN", - { - "reason": str(exc), - "submit_started_at": ( - submit_started_at.isoformat() - ), - "completed_at": datetime.now( - timezone.utc - ).isoformat(), - }, - ) - freeze_reason = "ambiguous_mutation" - break - submit_latency_ms = int( - (time.monotonic() - submit_started_monotonic) * 1000 - ) - polling_metrics["max_submit_latency_ms"] = max( - polling_metrics["max_submit_latency_ms"], - submit_latency_ms, - ) - acknowledgements[intent["intent_id"]] = ack - ledger.transition( - intent["intent_id"], - "ACKNOWLEDGED", - { - "ack": ack, - "runner_submit_started_at": ( - submit_started_at.isoformat() - ), - "runner_ack_received_at": datetime.now( - timezone.utc - ).isoformat(), - "latency_ms": submit_latency_ms, - }, - broker_order_id=ack["broker_order_id"], - ) - if not entry_window_open(end): - freeze_reason = "ack_after_entry_window" - break - elif record.state in {"ACKNOWLEDGED", "PARTIALLY_FILLED"}: - ack = acknowledgements.get(intent["intent_id"]) - if ack is None: - for event in reversed(ledger.events(intent["intent_id"])): - payload = event["payload"] - if event["event_type"] == "STATE_ACKNOWLEDGED": - ack = payload.get("ack") - break - if not isinstance(ack, dict): - ledger.transition( - intent["intent_id"], - "UNKNOWN", - {"reason": "ack_missing_after_restart"}, - ) - freeze_reason = "ack_missing" - break - status_started_at = datetime.now(timezone.utc) - status_started_monotonic = time.monotonic() - polling_metrics["order_status_queries"] += 1 - polling_metrics["expected_http_requests_started"] += 1 - try: - snapshot = current_order( - broker, - broker_name, - intent=intent, - ack=ack, - venue=venue, - trading_date=trading_date, - ) - except (BlockedError, TransportFailure) as exc: - status_latency_ms = int( - (time.monotonic() - status_started_monotonic) * 1000 - ) - polling_metrics["max_order_status_latency_ms"] = max( - polling_metrics["max_order_status_latency_ms"], - status_latency_ms, - ) - cycle_events.append( - ( - intent["intent_id"], - "RECONCILE_RETRY", - { - "reason": str(exc), - "started_at": status_started_at.isoformat(), - "completed_at": datetime.now( - timezone.utc - ).isoformat(), - "latency_ms": status_latency_ms, - }, - ) - ) - continue - status_latency_ms = int( - (time.monotonic() - status_started_monotonic) * 1000 - ) - polling_metrics["max_order_status_latency_ms"] = max( - polling_metrics["max_order_status_latency_ms"], - status_latency_ms, - ) - next_state = snapshot["normalized_status"] - try: - persist_entry_fill( - position_ledger, - plan=plan, - intent=intent, - snapshot=snapshot, - session_date=trading_date, - ) - except BlockedError as exc: - ledger.transition( - intent["intent_id"], - "UNKNOWN", - { - "reason": ( - "strategy position ledger sync failed: " - f"{exc}" - ), - "snapshot": snapshot, - }, - ) - freeze_reason = "position_ledger_sync_failed" - break - if next_state != record.state: - if next_state not in { - "ACKNOWLEDGED", - "PARTIALLY_FILLED", - "FILLED", - "CANCELLED", - "REJECTED", - }: - ledger.transition( - intent["intent_id"], - "UNKNOWN", - {"snapshot": snapshot}, - ) - freeze_reason = "unknown_broker_status" - break - ledger.transition( - intent["intent_id"], next_state, {"snapshot": snapshot} - ) - cycle_duration_ms = int( - (time.monotonic() - cycle_started_monotonic) * 1000 - ) - polling_metrics["cycles_completed"] += 1 - polling_metrics["max_cycle_duration_ms"] = max( - polling_metrics["max_cycle_duration_ms"], - cycle_duration_ms, - ) - if ( - not freeze_reason - and datetime.now(timezone.utc).timestamp() >= cycle_deadline - ): - freeze_reason = "cycle_latency_budget_exceeded" - cycle_events.append( - ( - plan["intents"][0]["intent_id"], - "CYCLE_END", - { - "cycle": cycle, - "completed_at": datetime.now(timezone.utc).isoformat(), - "duration_ms": cycle_duration_ms, - "freeze_reason": freeze_reason, - }, - ) - ) - ledger.append_events(cycle_events) - if freeze_reason and freeze_reason != "entry_window_closed": - break - cycle += 1 - - for intent in plan["intents"]: - record = ledger.get(intent["intent_id"]) - if record.state == "WAIT_TRIGGER": - ledger.transition( - intent["intent_id"], - "CANCELLED", - { - "reason": ( - "entry_window_closed" - if freeze_reason == "entry_window_closed" - else str(freeze_reason or "entry_window_ended") - ) - }, - ) - - if plan["order_policy"]["cancel_remainder_at_window_end"]: - for intent in plan["intents"]: - record = ledger.get(intent["intent_id"]) - if record.state not in {"ACKNOWLEDGED", "PARTIALLY_FILLED"}: - continue - ack = acknowledgements.get(intent["intent_id"]) - if ack is None: - for event in reversed(ledger.events(intent["intent_id"])): - if event["event_type"] == "STATE_ACKNOWLEDGED": - ack = event["payload"].get("ack") - break - if not isinstance(ack, dict): - ledger.transition( - intent["intent_id"], - "UNKNOWN", - {"reason": "cannot_cancel_without_ack"}, - ) - continue - cancel_quantity: str | None = None - if broker_name in {"kis-paper", "kis-live"}: - try: - cancel_snapshot = current_order( - broker, - broker_name, - intent=intent, - ack=ack, - venue=venue_by_intent[intent["intent_id"]], - trading_date=trading_date, - ) - persist_entry_fill( - position_ledger, - plan=plan, - intent=intent, - snapshot=cancel_snapshot, - session_date=trading_date, - ) - next_state = cancel_snapshot["normalized_status"] - if next_state in {"FILLED", "CANCELLED"} or ( - next_state == "REJECTED" - and record.state == "ACKNOWLEDGED" - ): - if next_state != record.state: - ledger.transition( - intent["intent_id"], - next_state, - {"snapshot": cancel_snapshot}, - ) - continue - if next_state == "REJECTED": - raise BlockedError( - "KIS partially filled order cannot regress to rejected" - ) - if next_state not in { - "ACKNOWLEDGED", - "PARTIALLY_FILLED", - }: - raise BlockedError( - "KIS cancel preflight returned an unknown state" - ) - if ( - next_state == "PARTIALLY_FILLED" - and record.state == "ACKNOWLEDGED" - ): - ledger.transition( - intent["intent_id"], - "PARTIALLY_FILLED", - {"snapshot": cancel_snapshot}, - ) - cancel_quantity = require_remaining_quantity( - cancel_snapshot - ) - ledger.append_event( - intent["intent_id"], - "CANCEL_PREFLIGHT", - {"snapshot": cancel_snapshot}, - ) - except (BlockedError, TransportFailure) as exc: - ledger.transition( - intent["intent_id"], - "UNKNOWN", - {"reason": f"cancel preflight failed: {exc}"}, - ) - continue - ledger.transition(intent["intent_id"], "CANCEL_PENDING", {}) - try: - result = cancel_order( - broker, - broker_name, - intent=intent, - ack=ack, - venue=venue_by_intent[intent["intent_id"]], - quantity=cancel_quantity, - ) - except (AmbiguousMutationError, TransportFailure) as exc: - ledger.transition( - intent["intent_id"], - "UNKNOWN", - {"reason": str(exc)}, - ) - except BlockedError as exc: - ledger.transition( - intent["intent_id"], - "UNKNOWN", - {"reason": str(exc)}, - ) - else: - ledger.append_event( - intent["intent_id"], "CANCEL_ACCEPTED", result - ) - return session_status( - plan, - broker_name, - mode, - ledger, - started_at, - datetime.now(timezone.utc), - freeze_reason, - warmup, - polling_metrics, - ) - finally: - position_ledger.close() - ledger.close() - - -def self_test() -> None: - now = datetime.now(timezone.utc) - intent = { - "market": "US", - "symbol": "AAPL", - "entry_trigger": "100", - "limit_price": "101", - } - policy = { - "max_age_seconds": 5, - "max_spread_bps": "25", - "max_gap_bps": "100", - "trigger_mode": "AT_OR_ABOVE", - } - quote = { - "market": "US", - "symbol": "AAPL", - "last_price": "100.5", - "best_ask": "100.6", - "best_bid": "100.4", - "received_at": now.isoformat(), - "trade_timestamp": now.isoformat(), - "book_timestamp": now.isoformat(), - "raw_status": "OK", - } - decision = evaluate_quote( - intent, quote, policy, now, None, require_source_timestamp=True - ) - assert decision["triggered"] - crossed_policy = {**policy, "trigger_mode": "CROSS_FROM_BELOW"} - assert not evaluate_quote( - intent, quote, crossed_policy, now, None, require_source_timestamp=True - )["triggered"] - assert evaluate_quote( - intent, - quote, - crossed_policy, - now, - Decimal(99), - require_source_timestamp=True, - )["triggered"] - stale_source = { - **quote, - "trade_timestamp": ( - now.replace(microsecond=0) - timedelta(seconds=10) - ).isoformat(), - "book_timestamp": ( - now.replace(microsecond=0) - timedelta(seconds=10) - ).isoformat(), - } - stale_decision = evaluate_quote( - intent, - stale_source, - policy, - now, - None, - require_source_timestamp=True, - ) - assert "stale_source_quote" in stale_decision["reasons"] - assert not stale_decision["quality_valid"] - assert not evaluate_quote( - intent, - quote, - crossed_policy, - now, - ( - Decimal(stale_decision["last_price"]) - if stale_decision["quality_valid"] - else None - ), - require_source_timestamp=True, - )["triggered"] - mixed_future_source = { - **quote, - "trade_timestamp": (now + timedelta(seconds=10)).isoformat(), - } - future_decision = evaluate_quote( - intent, - mixed_future_source, - policy, - now, - None, - require_source_timestamp=True, - ) - assert "stale_source_quote" in future_decision["reasons"] - missing_source = { - **quote, - "trade_timestamp": None, - "book_timestamp": None, - } - missing_decision = evaluate_quote( - intent, - missing_source, - policy, - now, - None, - require_source_timestamp=True, - ) - assert "source_timestamp_missing" in missing_decision["reasons"] - one_missing_source = { - **quote, - "book_timestamp": None, - } - one_missing_decision = evaluate_quote( - intent, - one_missing_source, - policy, - now, - None, - require_source_timestamp=True, - ) - assert "source_timestamp_missing" in one_missing_decision["reasons"] - invalid_source = { - **quote, - "trade_timestamp": "not-a-timestamp", - } - invalid_decision = evaluate_quote( - intent, - invalid_source, - policy, - now, - None, - require_source_timestamp=True, - ) - assert "trade_timestamp_invalid" in invalid_decision["reasons"] - for raw_status in ( - "PENDING_REPLACE", - "REPLACED", - "CANCEL_REJECTED", - "REPLACE_REJECTED", - ): - assert ( - normalize_toss_order({"status": raw_status})["normalized_status"] - == "UNKNOWN" - ) - try: - validate_runtime_capabilities({"intents": [intent]}, "kis-paper") - except BlockedError as exc: - assert "U.S. paper sessions" in str(exc) - else: - raise AssertionError("KIS U.S. paper session must be blocked") - one_intent_plan = { - "intents": [intent], - "entry_window": {"poll_interval_seconds": 4}, - } - assert ( - polling_admission(one_intent_plan, "kis-paper", "paper")[ - "minimum_poll_interval_seconds" - ] - == 4 - ) - too_fast_plan = { - "intents": [intent], - "entry_window": {"poll_interval_seconds": 3}, - } - try: - polling_admission(too_fast_plan, "kis-paper", "paper") - except BlockedError as exc: - assert "poll_interval_seconds >= 4" in str(exc) - else: - raise AssertionError("KIS paper pacing must reject an infeasible cycle") - four_intent_shadow_plan = { - "intents": [intent, intent, intent, intent], - "entry_window": {"poll_interval_seconds": 1}, - } - assert quote_http_request_count("kis-live", intent) == 1 - assert quote_http_request_count("toss", intent) == 2 - assert ( - polling_admission(four_intent_shadow_plan, "kis-live", "shadow")[ - "minimum_poll_interval_seconds" - ] - == 1 - ) - - class FailingWarmupBroker: - def token(self) -> str: - raise TransportFailure("fixture authentication outage") - - try: - warm_broker_before_open( - FailingWarmupBroker(), - datetime.now(timezone.utc) + timedelta(seconds=1), - ) - except BlockedError as exc: - assert "pre-open broker authentication failed" in str(exc) - else: - raise AssertionError("pre-open transport failure must fail closed") - assert entry_window_open(now + timedelta(seconds=1), now) - assert not entry_window_open(now, now) - assert require_remaining_quantity({"remaining_quantity": "2"}) == "2" - v2_intent = { - **intent, - "exchange": "NASDAQ", - "venue": "NASD", - } - assert venue_for({}, v2_intent) == "NASD" - assert venue_for({"NASDAQ:AAPL": "NASD"}, v2_intent) == "NASD" - try: - venue_for({"US:AAPL": "NYSE"}, v2_intent) - except BlockedError as exc: - assert "conflicts with hashed intent venue" in str(exc) - else: - raise AssertionError("a venue map must not override the hashed v2 venue") - try: - venue_for({}, intent) - except BlockedError as exc: - assert "venue map missing US:AAPL" in str(exc) - else: - raise AssertionError("legacy intents still require an external venue map") - - class CaptureCancelBroker: - def __init__(self) -> None: - self.kwargs: dict[str, Any] | None = None - - def cancel(self, **kwargs: Any) -> dict[str, Any]: - self.kwargs = kwargs - return {"accepted": True} - - capture = CaptureCancelBroker() - cancel_order( - capture, - "kis-paper", - intent=intent, - ack={"broker_order_id": "12345", "organization": "06000"}, - venue="NASD", - quantity="2", - ) - assert capture.kwargs is not None - assert capture.kwargs["symbol"] == "AAPL" - assert capture.kwargs["quantity"] == "2" - print( - canonical_json( - { - "self_test": "PASS", - "spread_bps": decision["spread_bps"], - "live_gate": "BLOCKED", - } - ) - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - subparsers = parser.add_subparsers(dest="command") - - preview = subparsers.add_parser("preview") - preview.add_argument("--plan", required=True) - preview.add_argument( - "--broker", required=True, choices=("toss", "kis-paper", "kis-live") - ) - preview.add_argument("--venue-map") - preview.add_argument("--output") - - run = subparsers.add_parser("run") - run.add_argument("--plan", required=True) - run.add_argument( - "--broker", required=True, choices=("toss", "kis-paper", "kis-live") - ) - run.add_argument("--mode", required=True, choices=("paper", "shadow", "live")) - run.add_argument("--venue-map") - run.add_argument("--state-dir", required=True) - run.add_argument("--output") - run.add_argument("--max-cycles", type=int) - - subparsers.add_parser("self-test") - return parser.parse_args() - - -def main() -> int: - args = parse_args() - if args.command == "self-test": - self_test() - return 0 - if args.command not in {"preview", "run"}: - emit_json( - { - "schema": SESSION_STATUS_SCHEMA, - "status": "BLOCKED", - "reason": "choose preview, run, or self-test", - } - ) - return 2 - try: - if args.command == "run" and args.broker in {"kis-paper", "kis-live"}: - require_kis_runtime_credentials( - load_kis_credentials(args.broker.split("-", 1)[1]) - ) - plan = load_json_object(args.plan) - venues = ( - normalize_venue_map(load_json_object(args.venue_map)) - if args.venue_map - else {} - ) - if args.command == "preview": - output = preview_requests(plan, args.broker, venues) - else: - if args.max_cycles is not None and args.max_cycles <= 0: - raise BlockedError("--max-cycles must be positive") - output = run_session( - plan, - args.broker, - args.mode, - venues, - Path(args.state_dir).resolve(), - max_cycles=args.max_cycles, - ) - except (BlockedError, OSError, ValueError, json.JSONDecodeError) as exc: - output = { - "schema": SESSION_STATUS_SCHEMA, - "status": "BLOCKED", - "reason": str(exc), - } - emit_json(output, args.output) - return 2 - emit_json(output, args.output) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/systemd_units.py b/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/systemd_units.py deleted file mode 100644 index 3b16cfb..0000000 --- a/plugins/quant-stock-polling-trader/skills/quant-stock-polling-trader/scripts/systemd_units.py +++ /dev/null @@ -1,1026 +0,0 @@ -#!/usr/bin/env python3 -"""Generate and execute deterministic user-level systemd jobs for QTA.""" - -from __future__ import annotations - -import argparse -import fcntl -import hashlib -import json -import os -import re -import stat -import subprocess -import sys -import time -import urllib.error -import urllib.parse -import urllib.request -from pathlib import Path -from typing import Any, Mapping - - -BUNDLE_SCHEMA = "qta-systemd-bundle/v2" -STATUS_SCHEMA = "qta-systemd-generation-status/v2" -TOP_FIELDS = { - "schema", - "unit_prefix", - "python_executable", - "technical_skill_root", - "trader_skill_root", - "environment_file", - "runtime_directory", - "jobs", -} -JOB_FIELDS = { - "name", - "kind", - "market", - "input_path", - "plan_path", - "state_directory", - "output_path", - "broker", - "mode", - "venue_map", - "max_cycles", - "timeout_start_seconds", -} -SCHEDULES = { - ("eod", "KR"): "Mon..Fri *-*-* 07:00:00 Asia/Seoul", - ("eod", "US"): "Mon..Fri *-*-* 18:00:00 America/New_York", - ("snapshot", "KR"): "Mon..Fri *-*-* 08:50:00 Asia/Seoul", - ("snapshot", "US"): "Mon..Fri *-*-* 09:20:00 America/New_York", - ("entry", "KR"): "Mon..Fri *-*-* 08:59:00 Asia/Seoul", - ("entry", "US"): "Mon..Fri *-*-* 09:29:00 America/New_York", - ("daily-prepare", "KR"): "Mon..Fri *-*-* 01:00:00 Asia/Seoul", - ("daily-prepare", "US"): "Mon..Fri *-*-* 01:00:00 America/New_York", - ("daily-snapshot", "KR"): "Mon..Fri *-*-* 08:50:00 Asia/Seoul", - ("daily-snapshot", "US"): "Mon..Fri *-*-* 09:20:00 America/New_York", - ("daily-entry", "KR"): "Mon..Fri *-*-* 08:59:00 Asia/Seoul", - ("daily-entry", "US"): "Mon..Fri *-*-* 09:29:00 America/New_York", -} -UNIT_NAME = re.compile(r"^[a-z0-9][a-z0-9-]{0,47}$") -NTFY_TOPIC_URL_ENV = "QTA_NTFY_TOPIC_URL" -NTFY_TOKEN_ENV = "QTA_NTFY_TOKEN" -NTFY_TIMEOUT_SECONDS = 10 - - -class UnitBlockedError(ValueError): - """Raised when a unit bundle would rely on an unsafe or ambiguous value.""" - - -def sha256_file(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def atomic_write(path: Path, payload: bytes, mode: int = 0o600) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") - descriptor = os.open( - temporary, - os.O_WRONLY | os.O_CREAT | os.O_EXCL, - mode, - ) - try: - with os.fdopen(descriptor, "wb") as handle: - handle.write(payload) - handle.flush() - os.fsync(handle.fileno()) - os.replace(temporary, path) - os.chmod(path, mode) - finally: - if temporary.exists(): - temporary.unlink() - - -def exact_fields(value: Mapping[str, Any], expected: set[str], label: str) -> None: - if set(value) != expected: - raise UnitBlockedError( - f"{label} fields mismatch; missing={sorted(expected - set(value))}, " - f"extra={sorted(set(value) - expected)}" - ) - - -def absolute_path(value: Any, label: str) -> Path: - path = Path(str(value)).expanduser() - if not path.is_absolute(): - raise UnitBlockedError(f"{label} must be absolute") - if any(character in str(path) for character in ("\n", "\r", "\0")): - raise UnitBlockedError(f"{label} contains an invalid character") - return path - - -def regular_nonsymlink(path: Path, label: str, mode: int | None = None) -> None: - try: - metadata = path.lstat() - except FileNotFoundError as exc: - raise UnitBlockedError(f"{label} does not exist: {path}") from exc - if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode): - raise UnitBlockedError(f"{label} must be a regular non-symlink file") - if mode is not None and stat.S_IMODE(metadata.st_mode) != mode: - raise UnitBlockedError(f"{label} must have mode {mode:04o}") - - -def within(path: Path, root: Path, label: str) -> Path: - resolved = path.resolve(strict=False) - try: - resolved.relative_to(root.resolve(strict=False)) - except ValueError as exc: - raise UnitBlockedError(f"{label} must be inside runtime_directory") from exc - return resolved - - -def normalize_bundle(raw: Any) -> dict[str, Any]: - if not isinstance(raw, dict): - raise UnitBlockedError("bundle must be one JSON object") - exact_fields(raw, TOP_FIELDS, "bundle") - if raw["schema"] != BUNDLE_SCHEMA: - raise UnitBlockedError(f"bundle.schema must be {BUNDLE_SCHEMA}") - prefix = str(raw["unit_prefix"]) - if not UNIT_NAME.fullmatch(prefix): - raise UnitBlockedError("unit_prefix must be lowercase letters, digits, hyphens") - python_path = absolute_path(raw["python_executable"], "python_executable") - try: - python = python_path.resolve(strict=True) - except FileNotFoundError as exc: - raise UnitBlockedError("python_executable does not exist") from exc - regular_nonsymlink(python, "resolved python_executable") - technical_root = absolute_path(raw["technical_skill_root"], "technical_skill_root") - trader_root = absolute_path(raw["trader_skill_root"], "trader_skill_root") - for root, label in ( - (technical_root, "technical_skill_root"), - (trader_root, "trader_skill_root"), - ): - if not root.is_dir() or root.is_symlink(): - raise UnitBlockedError(f"{label} must be a non-symlink directory") - required_scripts = ( - technical_root / "scripts" / "fetch_kis_kr_eod.py", - technical_root / "scripts" / "fetch_kis_us_eod.py", - trader_root / "scripts" / "account_snapshot.py", - trader_root / "scripts" / "daily_pipeline.py", - trader_root / "scripts" / "market_calendar.py", - trader_root / "scripts" / "run_session.py", - trader_root / "scripts" / "systemd_units.py", - ) - for script in required_scripts: - regular_nonsymlink(script, "required skill script") - environment = absolute_path(raw["environment_file"], "environment_file") - regular_nonsymlink(environment, "environment_file", 0o600) - runtime = absolute_path(raw["runtime_directory"], "runtime_directory") - if runtime.exists() and (not runtime.is_dir() or runtime.is_symlink()): - raise UnitBlockedError("runtime_directory must be a non-symlink directory") - jobs = raw["jobs"] - if not isinstance(jobs, list) or not jobs: - raise UnitBlockedError("jobs must be a non-empty array") - normalized_jobs: list[dict[str, Any]] = [] - names: set[str] = set() - pairs: set[tuple[str, str]] = set() - for index, job in enumerate(jobs): - label = f"jobs[{index}]" - if not isinstance(job, dict): - raise UnitBlockedError(f"{label} must be an object") - exact_fields(job, JOB_FIELDS, label) - name = str(job["name"]) - if not UNIT_NAME.fullmatch(name) or name in names: - raise UnitBlockedError(f"{label}.name is invalid or duplicate") - names.add(name) - kind = str(job["kind"]).lower() - market = str(job["market"]).upper() - if (kind, market) not in SCHEDULES or (kind, market) in pairs: - raise UnitBlockedError(f"{label} has an unsupported or duplicate job scope") - pairs.add((kind, market)) - timeout = job["timeout_start_seconds"] - if ( - isinstance(timeout, bool) - or not isinstance(timeout, int) - or timeout < 60 - or timeout > 86400 - ): - raise UnitBlockedError(f"{label}.timeout_start_seconds must be 60..86400") - max_cycles = job["max_cycles"] - if ( - isinstance(max_cycles, bool) - or not isinstance(max_cycles, int) - or max_cycles < 0 - ): - raise UnitBlockedError(f"{label}.max_cycles must be an integer >= 0") - path_values = { - field: ( - within(absolute_path(job[field], f"{label}.{field}"), runtime, label) - if str(job[field]) - else None - ) - for field in ( - "input_path", - "plan_path", - "state_directory", - "output_path", - "venue_map", - ) - } - broker = str(job["broker"]) - mode = str(job["mode"]) - if kind.startswith("daily-"): - if path_values["input_path"] is None: - raise UnitBlockedError( - f"{label}.input_path is required for a daily pipeline job" - ) - if any( - path_values[field] is not None - for field in ( - "plan_path", - "state_directory", - "output_path", - "venue_map", - ) - ): - raise UnitBlockedError( - f"{label} has static paths on a daily pipeline job" - ) - if broker != "kis-live" or mode != "shadow": - raise UnitBlockedError( - f"{label} daily pipeline requires kis-live/shadow" - ) - if kind != "daily-entry" and max_cycles: - raise UnitBlockedError( - f"{label}.max_cycles is allowed only for daily-entry" - ) - elif kind == "eod": - if path_values["input_path"] is None: - raise UnitBlockedError(f"{label}.input_path is required for EOD") - if any( - path_values[field] is not None - for field in ( - "plan_path", - "state_directory", - "output_path", - "venue_map", - ) - ): - raise UnitBlockedError(f"{label} has entry-only paths on an EOD job") - if broker or mode or max_cycles: - raise UnitBlockedError(f"{label} has entry-only values on an EOD job") - elif kind == "snapshot": - if path_values["input_path"] is None: - raise UnitBlockedError( - f"{label}.input_path is required for an account snapshot" - ) - if any( - path_values[field] is not None - for field in ( - "plan_path", - "state_directory", - "output_path", - "venue_map", - ) - ): - raise UnitBlockedError( - f"{label} has entry-only paths on an account snapshot job" - ) - if broker != "kis-live" or mode != "shadow": - raise UnitBlockedError( - f"{label} account snapshot requires kis-live/shadow" - ) - if max_cycles: - raise UnitBlockedError( - f"{label}.max_cycles must be zero for an account snapshot" - ) - else: - required = ("plan_path", "state_directory", "output_path") - if any(path_values[field] is None for field in required): - raise UnitBlockedError(f"{label} is missing an entry path") - if path_values["input_path"] is not None: - raise UnitBlockedError(f"{label}.input_path must be empty for entry") - if broker not in {"kis-paper", "kis-live"}: - raise UnitBlockedError(f"{label}.broker must be kis-paper or kis-live") - if mode not in {"paper", "shadow"}: - raise UnitBlockedError( - f"{label}.mode must be paper or shadow; live is not generated" - ) - if broker == "kis-paper" and mode != "paper": - raise UnitBlockedError(f"{label} kis-paper requires paper mode") - if broker == "kis-live" and mode != "shadow": - raise UnitBlockedError(f"{label} kis-live requires shadow mode") - normalized_jobs.append( - { - "name": name, - "kind": kind, - "market": market, - "schedule": SCHEDULES[(kind, market)], - "persistent": kind in {"eod", "daily-prepare"}, - "input_path": str(path_values["input_path"] or ""), - "plan_path": str(path_values["plan_path"] or ""), - "state_directory": str(path_values["state_directory"] or ""), - "output_path": str(path_values["output_path"] or ""), - "broker": broker, - "mode": mode, - "venue_map": str(path_values["venue_map"] or ""), - "max_cycles": max_cycles, - "timeout_start_seconds": timeout, - } - ) - normalized_jobs.sort(key=lambda item: item["name"]) - return { - "schema": BUNDLE_SCHEMA, - "unit_prefix": prefix, - "python_executable": str(python.resolve()), - "technical_skill_root": str(technical_root.resolve()), - "trader_skill_root": str(trader_root.resolve()), - "environment_file": str(environment.resolve()), - "runtime_directory": str(runtime.resolve(strict=False)), - "jobs": normalized_jobs, - } - - -def execution_bundle(bundle: dict[str, Any]) -> dict[str, Any]: - """Project a normalized bundle back to the exact executable v1 schema.""" - return { - key: ( - [ - {field: job[field] for field in JOB_FIELDS} - for job in bundle["jobs"] - ] - if key == "jobs" - else bundle[key] - ) - for key in TOP_FIELDS - } - - -def systemd_quote(value: str) -> str: - escaped = ( - value.replace("\\", "\\\\") - .replace('"', '\\"') - .replace("$", "\\$") - .replace("%", "%%") - ) - return f'"{escaped}"' - - -def systemd_path(value: str) -> str: - """Escape one absolute path for directives that do not accept quoting.""" - escaped: list[str] = [] - for octet in value.encode("utf-8"): - character = chr(octet) - if character == "%": - escaped.append("%%") - elif 0x21 <= octet <= 0x7E and character not in {'"', "'", "\\"}: - escaped.append(character) - else: - escaped.append(f"\\x{octet:02x}") - return "".join(escaped) - - -def service_text(bundle: dict[str, Any], job: dict[str, Any], bundle_path: Path) -> str: - root = Path(bundle["runtime_directory"]) - conditions = [] - for field in ("input_path", "plan_path", "venue_map"): - if job[field]: - conditions.append(f"ConditionPathExists={systemd_path(job[field])}") - command = " ".join( - systemd_quote(item) - for item in ( - bundle["python_executable"], - "-B", - "-s", - str(Path(bundle["trader_skill_root"]) / "scripts" / "systemd_units.py"), - "execute", - "--bundle", - str(bundle_path), - "--name", - job["name"], - ) - ) - dependencies: list[str] = ["network-online.target"] - predecessor_kind = { - "daily-snapshot": "daily-prepare", - "daily-entry": "daily-snapshot", - }.get(job["kind"]) - if predecessor_kind is not None: - predecessor = next( - ( - item - for item in bundle["jobs"] - if item["kind"] == predecessor_kind - and item["market"] == job["market"] - ), - None, - ) - if predecessor is not None: - dependencies.append( - f"{bundle['unit_prefix']}-{predecessor['name']}.service" - ) - return ( - "[Unit]\n" - f"Description=QTA {job['kind']} {job['market']} ({job['name']})\n" - f"After={' '.join(dependencies)}\n" - "Wants=network-online.target\n" - + ("\n".join(conditions) + "\n" if conditions else "") - + "\n[Service]\n" - "Type=oneshot\n" - f"WorkingDirectory={systemd_path(str(root))}\n" - f"EnvironmentFile={systemd_path(bundle['environment_file'])}\n" - 'Environment="PYTHONDONTWRITEBYTECODE=1"\n' - "UnsetEnvironment=PYTHONPATH PYTHONHOME\n" - f"ExecStart={command}\n" - f"TimeoutStartSec={job['timeout_start_seconds']}s\n" - "UMask=0077\n" - "NoNewPrivileges=yes\n" - "PrivateTmp=yes\n" - "PrivateDevices=yes\n" - "ProtectSystem=strict\n" - "ProtectHome=read-only\n" - "ProtectKernelTunables=yes\n" - "ProtectKernelModules=yes\n" - "ProtectControlGroups=yes\n" - "RestrictSUIDSGID=yes\n" - "LockPersonality=yes\n" - "MemoryDenyWriteExecute=yes\n" - "RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\n" - f"ReadWritePaths={systemd_path(str(root))}\n" - f"ReadOnlyPaths={systemd_path(bundle['technical_skill_root'])} " - f"{systemd_path(bundle['trader_skill_root'])} " - f"{systemd_path(bundle['environment_file'])}\n" - ) - - -def timer_text(prefix: str, job: dict[str, Any]) -> str: - return ( - "[Unit]\n" - f"Description=Schedule QTA {job['kind']} {job['market']} ({job['name']})\n" - "\n[Timer]\n" - f"OnCalendar={job['schedule']}\n" - "AccuracySec=1s\n" - "RandomizedDelaySec=0\n" - f"Persistent={'yes' if job['persistent'] else 'no'}\n" - f"Unit={prefix}-{job['name']}.service\n" - "\n[Install]\n" - "WantedBy=timers.target\n" - ) - - -def generate(bundle: dict[str, Any], output_directory: Path) -> dict[str, Any]: - if output_directory.exists(): - if output_directory.is_symlink() or not output_directory.is_dir(): - raise UnitBlockedError( - "output_directory must be a non-symlink directory" - ) - if any(output_directory.iterdir()): - raise UnitBlockedError( - "output_directory must be empty to prevent stale unit files" - ) - else: - output_directory.mkdir(parents=True) - bundle_path = output_directory / "systemd-bundle.json" - runnable_bundle = execution_bundle(bundle) - atomic_write( - bundle_path, - (json.dumps(runnable_bundle, indent=2, sort_keys=True) + "\n").encode( - "utf-8" - ), - ) - files: list[dict[str, Any]] = [] - prefix = bundle["unit_prefix"] - for job in bundle["jobs"]: - service_path = output_directory / f"{prefix}-{job['name']}.service" - timer_path = output_directory / f"{prefix}-{job['name']}.timer" - atomic_write(service_path, service_text(bundle, job, bundle_path).encode()) - atomic_write(timer_path, timer_text(prefix, job).encode()) - for path in (service_path, timer_path): - files.append( - { - "path": str(path.resolve()), - "sha256": sha256_file(path), - "bytes": path.stat().st_size, - } - ) - files.sort(key=lambda item: item["path"]) - status = { - "schema": STATUS_SCHEMA, - "status": "READY", - "bundle_path": str(bundle_path.resolve()), - "bundle_sha256": sha256_file(bundle_path), - "unit_files": files, - "activation_performed": False, - "live_enabled": False, - "api_mutation_count": 0, - } - atomic_write( - output_directory / "generation-status.json", - (json.dumps(status, indent=2, sort_keys=True) + "\n").encode(), - ) - return status - - -def command_for(bundle: dict[str, Any], job: dict[str, Any]) -> list[str]: - python = bundle["python_executable"] - if job["kind"].startswith("daily-"): - stage = job["kind"].removeprefix("daily-") - command = [ - python, - "-B", - "-s", - str( - Path(bundle["trader_skill_root"]) - / "scripts" - / "daily_pipeline.py" - ), - stage, - "--config", - job["input_path"], - "--market", - job["market"], - "--technical-skill-root", - bundle["technical_skill_root"], - "--trader-skill-root", - bundle["trader_skill_root"], - ] - if job["kind"] == "daily-entry" and job["max_cycles"]: - command.extend(["--max-cycles", str(job["max_cycles"])]) - return command - if job["kind"] == "eod": - script_name = ( - "fetch_kis_kr_eod.py" if job["market"] == "KR" else "fetch_kis_us_eod.py" - ) - return [ - python, - "-B", - "-s", - str(Path(bundle["technical_skill_root"]) / "scripts" / script_name), - "collect", - "--job", - job["input_path"], - ] - if job["kind"] == "snapshot": - return [ - python, - "-B", - "-s", - str( - Path(bundle["trader_skill_root"]) - / "scripts" - / "account_snapshot.py" - ), - "collect", - "--job", - job["input_path"], - ] - command = [ - python, - "-B", - "-s", - str(Path(bundle["trader_skill_root"]) / "scripts" / "run_session.py"), - "run", - "--plan", - job["plan_path"], - "--broker", - job["broker"], - "--mode", - job["mode"], - "--state-dir", - job["state_directory"], - "--output", - job["output_path"], - ] - if job["venue_map"]: - command.extend(["--venue-map", job["venue_map"]]) - if job["max_cycles"]: - command.extend(["--max-cycles", str(job["max_cycles"])]) - return command - - -def acquire_market_locks( - runtime: Path, markets: list[str] -) -> list[int]: - lock_directory = runtime / "locks" - try: - lock_directory.mkdir(parents=True, exist_ok=True, mode=0o700) - lock_metadata = lock_directory.lstat() - except OSError as exc: - raise UnitBlockedError("cannot create or inspect the lock directory") from exc - if stat.S_ISLNK(lock_metadata.st_mode) or not stat.S_ISDIR( - lock_metadata.st_mode - ): - raise UnitBlockedError("lock directory must be a non-symlink directory") - descriptors: list[int] = [] - try: - for market in sorted(set(markets)): - lock_path = lock_directory / f"{market.lower()}.lock" - lock_flags = os.O_RDWR | os.O_CREAT - if hasattr(os, "O_NOFOLLOW"): - lock_flags |= os.O_NOFOLLOW - try: - descriptor = os.open(lock_path, lock_flags, 0o600) - except OSError as exc: - raise UnitBlockedError( - f"cannot securely open the {market} market lock" - ) from exc - if not stat.S_ISREG(os.fstat(descriptor).st_mode): - os.close(descriptor) - raise UnitBlockedError("market lock must be a regular file") - try: - os.fchmod(descriptor, 0o600) - except OSError as exc: - os.close(descriptor) - raise UnitBlockedError( - f"cannot restrict the {market} market lock permissions" - ) from exc - if stat.S_IMODE(os.fstat(descriptor).st_mode) != 0o600: - os.close(descriptor) - raise UnitBlockedError( - f"{market} market lock permissions must be 0600" - ) - try: - fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) - except BlockingIOError as exc: - os.close(descriptor) - raise UnitBlockedError( - f"another {market} QTA process owns the single-writer lock" - ) from exc - os.set_inheritable(descriptor, True) - descriptors.append(descriptor) - except BaseException: - for descriptor in descriptors: - os.close(descriptor) - raise - return descriptors - - -def ntfy_topic_url(environment: Mapping[str, str]) -> str | None: - value = environment.get(NTFY_TOPIC_URL_ENV, "").strip() - if not value: - return None - try: - parsed = urllib.parse.urlsplit(value) - except ValueError as exc: - raise UnitBlockedError( - f"{NTFY_TOPIC_URL_ENV} must be a plain HTTP(S) topic URL" - ) from exc - if ( - parsed.scheme not in {"http", "https"} - or not parsed.netloc - or parsed.username is not None - or parsed.password is not None - or parsed.query - or parsed.fragment - or not parsed.path.strip("/") - or any(ord(character) < 0x20 for character in value) - ): - raise UnitBlockedError( - f"{NTFY_TOPIC_URL_ENV} must be a plain HTTP(S) topic URL" - ) - return value - - -def read_daily_outcome( - runtime: Path, - job: Mapping[str, Any], - returncode: int, -) -> tuple[str, list[str]]: - outcome = "PASS" if returncode == 0 else "BLOCKED" - details: list[str] = [] - if not str(job["kind"]).startswith("daily-"): - return outcome, details - descriptor_path = runtime / "current" / f"{str(job['market']).lower()}.json" - try: - descriptor = json.loads(descriptor_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - return outcome, details - if not isinstance(descriptor, dict) or descriptor.get("market") != job["market"]: - return outcome, details - status = descriptor.get("status") - if ( - isinstance(status, str) - and status - and ( - returncode == 0 - or status in {"BLOCKED", "MANUAL_BLOCK"} - ) - ): - outcome = status - session_date = descriptor.get("session_date") - if isinstance(session_date, str) and session_date: - details.append(f"session={session_date}") - if job["kind"] == "daily-snapshot": - plan_hash = descriptor.get("plan_hash") - if isinstance(plan_hash, str) and plan_hash: - details.append(f"plan={plan_hash[:12]}") - if job["kind"] == "daily-entry": - report_path = descriptor.get("final_report_path") - if isinstance(report_path, str) and report_path: - try: - report = json.loads(Path(report_path).read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - report = {} - if isinstance(report, dict): - counts = report.get("counts") - if isinstance(counts, dict): - for field in ( - "planned", - "submitted", - "filled", - "cancelled", - "blocked", - ): - value = counts.get(field) - if isinstance(value, int) and not isinstance(value, bool): - details.append(f"{field}={value}") - return outcome, details - - -def publish_ntfy( - *, - environment: Mapping[str, str], - job: Mapping[str, Any], - outcome: str, - duration_seconds: int, - details: list[str], -) -> dict[str, str]: - try: - topic_url = ntfy_topic_url(environment) - except UnitBlockedError: - return { - "channel": "ntfy", - "status": "FAILED", - "reason": "INVALID_TOPIC_URL", - } - if topic_url is None: - return {"channel": "ntfy", "status": "DISABLED"} - blocked = outcome in {"BLOCKED", "MANUAL_BLOCK"} - title = f"QTA {job['market']} {job['kind']} {outcome}" - message_parts = [ - f"market={job['market']}", - f"stage={job['kind']}", - f"status={outcome}", - f"duration={duration_seconds}s", - *details, - "live_enabled=false", - "broker_api_mutations=0", - ] - headers = { - "Content-Type": "text/plain; charset=utf-8", - "Title": title, - "Priority": "urgent" if blocked else ( - "high" if job["kind"] == "daily-entry" else "default" - ), - "Tags": "warning" if blocked else "white_check_mark", - } - token = environment.get(NTFY_TOKEN_ENV, "").strip() - if token: - if any(ord(character) < 0x21 or ord(character) > 0x7E for character in token): - return { - "channel": "ntfy", - "status": "FAILED", - "reason": "INVALID_TOKEN", - } - headers["Authorization"] = f"Bearer {token}" - try: - request = urllib.request.Request( - topic_url, - data=" ".join(message_parts).encode("utf-8"), - headers=headers, - method="POST", - ) - with urllib.request.urlopen( - request, - timeout=NTFY_TIMEOUT_SECONDS, - ) as response: - status = getattr(response, "status", 200) - if not 200 <= status < 300: - return { - "channel": "ntfy", - "status": "FAILED", - "reason": f"HTTP_{status}", - } - except urllib.error.HTTPError as exc: - return { - "channel": "ntfy", - "status": "FAILED", - "reason": f"HTTP_{exc.code}", - } - except (urllib.error.URLError, TimeoutError, OSError, ValueError): - return { - "channel": "ntfy", - "status": "FAILED", - "reason": "TRANSPORT_ERROR", - } - return {"channel": "ntfy", "status": "SENT"} - - -def execute(bundle: dict[str, Any], name: str) -> int: - job = next((item for item in bundle["jobs"] if item["name"] == name), None) - if job is None: - raise UnitBlockedError(f"unknown job: {name}") - runtime = Path(bundle["runtime_directory"]) - lock_markets = [job["market"]] - acquire_market_locks(runtime, lock_markets) - environment = dict(os.environ) - environment["PYTHONDONTWRITEBYTECODE"] = "1" - environment.pop("PYTHONPATH", None) - environment.pop("PYTHONHOME", None) - started = time.monotonic() - result = subprocess.run( - command_for(bundle, job), - check=False, - env=environment, - ) - duration = max(0, round(time.monotonic() - started)) - outcome, details = read_daily_outcome(runtime, job, result.returncode) - notification = publish_ntfy( - environment=environment, - job=job, - outcome=outcome, - duration_seconds=duration, - details=details, - ) - print(json.dumps(notification, sort_keys=True)) - return result.returncode - - -def self_test() -> None: - import tempfile - - with tempfile.TemporaryDirectory(prefix="qta-systemd-") as temporary: - root = Path(temporary) - technical = root / "technical" - trader = root / "trader" - runtime = root / "runtime" - output = root / "units" - for path in ( - technical / "scripts", - trader / "scripts", - runtime, - ): - path.mkdir(parents=True) - for path in ( - technical / "scripts" / "fetch_kis_kr_eod.py", - technical / "scripts" / "fetch_kis_us_eod.py", - trader / "scripts" / "run_session.py", - trader / "scripts" / "account_snapshot.py", - trader / "scripts" / "daily_pipeline.py", - trader / "scripts" / "market_calendar.py", - trader / "scripts" / "systemd_units.py", - runtime / "kr-eod.json", - runtime / "kr-account-snapshot.json", - runtime / "us-plan.json", - ): - path.write_text("{}\n", encoding="utf-8") - environment = root / "secrets.env" - environment.write_text("QTA_SELF_TEST=1\n", encoding="utf-8") - environment.chmod(0o600) - raw = { - "schema": BUNDLE_SCHEMA, - "unit_prefix": "qta", - "python_executable": str(Path(sys.executable).resolve()), - "technical_skill_root": str(technical), - "trader_skill_root": str(trader), - "environment_file": str(environment), - "runtime_directory": str(runtime), - "jobs": [ - { - "name": "eod-kr", - "kind": "eod", - "market": "KR", - "input_path": str(runtime / "kr-eod.json"), - "plan_path": "", - "state_directory": "", - "output_path": "", - "broker": "", - "mode": "", - "venue_map": "", - "max_cycles": 0, - "timeout_start_seconds": 7200, - }, - { - "name": "snapshot-kr", - "kind": "snapshot", - "market": "KR", - "input_path": str(runtime / "kr-account-snapshot.json"), - "plan_path": "", - "state_directory": "", - "output_path": "", - "broker": "kis-live", - "mode": "shadow", - "venue_map": "", - "max_cycles": 0, - "timeout_start_seconds": 300, - }, - { - "name": "entry-us", - "kind": "entry", - "market": "US", - "input_path": "", - "plan_path": str(runtime / "us-plan.json"), - "state_directory": str(runtime / "state-us"), - "output_path": str(runtime / "us-receipt.json"), - "broker": "kis-live", - "mode": "shadow", - "venue_map": "", - "max_cycles": 1, - "timeout_start_seconds": 3900, - }, - ], - } - bundle = normalize_bundle(raw) - receipt = generate(bundle, output) - if len(receipt["unit_files"]) != 6: - raise AssertionError("systemd self-test unit count mismatch") - service = (output / "qta-entry-us.service").read_text() - timer = (output / "qta-entry-us.timer").read_text() - if "ProtectSystem=strict" not in service or "live" in service: - raise AssertionError("systemd self-test hardening failed") - if ( - f"WorkingDirectory={systemd_path(bundle['runtime_directory'])}\n" - not in service - or f"EnvironmentFile={systemd_path(bundle['environment_file'])}\n" - not in service - or "UnsetEnvironment=PYTHONPATH PYTHONHOME\n" not in service - ): - raise AssertionError("systemd self-test path rendering failed") - if "09:29:00 America/New_York" not in timer or "Persistent=no" not in timer: - raise AssertionError("systemd self-test timer failed") - if read_bundle(output / "systemd-bundle.json") != bundle: - raise AssertionError("generated bundle is not executable by the reader") - permissive_lock = runtime / "locks" / "us.lock" - permissive_lock.parent.mkdir(mode=0o700) - permissive_lock.write_bytes(b"") - permissive_lock.chmod(0o644) - descriptors = acquire_market_locks(runtime, ["US"]) - try: - if stat.S_IMODE(permissive_lock.stat().st_mode) != 0o600: - raise AssertionError( - "existing market lock permissions were not restricted" - ) - finally: - for descriptor in descriptors: - os.close(descriptor) - print( - json.dumps( - { - "self_test": "PASS", - "unit_files": 6, - "activation_performed": False, - "live_enabled": False, - }, - sort_keys=True, - ) - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - subparsers = parser.add_subparsers(dest="command", required=True) - generate_parser = subparsers.add_parser("generate") - generate_parser.add_argument("--bundle", required=True) - generate_parser.add_argument("--output-directory", required=True) - execute_parser = subparsers.add_parser("execute") - execute_parser.add_argument("--bundle", required=True) - execute_parser.add_argument("--name", required=True) - subparsers.add_parser("self-test") - return parser.parse_args() - - -def read_bundle(path: Path) -> dict[str, Any]: - regular_nonsymlink(path, "bundle") - try: - raw = json.loads(path.read_text(encoding="utf-8")) - except json.JSONDecodeError as exc: - raise UnitBlockedError("bundle is not valid JSON") from exc - return normalize_bundle(raw) - - -def main() -> int: - args = parse_args() - try: - if args.command == "self-test": - self_test() - return 0 - bundle_path = absolute_path(args.bundle, "bundle") - bundle = read_bundle(bundle_path) - if args.command == "generate": - receipt = generate( - bundle, absolute_path(args.output_directory, "output_directory") - ) - print(json.dumps(receipt, sort_keys=True)) - return 0 - return execute(bundle, args.name) - except (UnitBlockedError, OSError) as exc: - print( - json.dumps( - {"status": "BLOCKED", "reason": str(exc)}, - ensure_ascii=False, - sort_keys=True, - ), - file=sys.stderr, - ) - return 2 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/plugins/quant-stock-technical/.claude-plugin/plugin.json b/plugins/quant-stock-technical/.claude-plugin/plugin.json deleted file mode 100644 index 7287cc0..0000000 --- a/plugins/quant-stock-technical/.claude-plugin/plugin.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", - "name": "quant-stock-technical", - "displayName": "Quant Stock Technical", - "version": "0.4.6", - "description": "Acquire audited KIS EOD data, screen four exchanges, and evaluate selected opening-hour outcomes.", - "author": { - "name": "mrcha033", - "url": "https://github.com/mrcha033" - }, - "homepage": "https://github.com/mrcha033/skills", - "repository": "https://github.com/mrcha033/skills", - "keywords": [ - "stocks", - "technical-analysis", - "quantitative" - ], - "skills": "./skills/" -} diff --git a/plugins/quant-stock-technical/.codex-plugin/plugin.json b/plugins/quant-stock-technical/.codex-plugin/plugin.json deleted file mode 100644 index 4328d9e..0000000 --- a/plugins/quant-stock-technical/.codex-plugin/plugin.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "name": "quant-stock-technical", - "version": "0.4.6", - "description": "Acquire audited KIS EOD data, screen four exchanges, and evaluate selected opening-hour outcomes.", - "author": { - "name": "mrcha033", - "url": "https://github.com/mrcha033" - }, - "homepage": "https://github.com/mrcha033/skills", - "repository": "https://github.com/mrcha033/skills", - "keywords": [ - "stocks", - "technical-analysis", - "quantitative" - ], - "skills": "./skills/", - "interface": { - "displayName": "Quant Stock Technical", - "shortDescription": "Four-exchange screening and opening-hour research", - "longDescription": "Acquire resumable read-only KIS Korean and U.S. adjusted EOD data, build a source-hashed four-exchange universe, and evaluate frozen selected candidates against opening-hour outcomes.", - "developerName": "mrcha033", - "category": "Finance", - "capabilities": [ - "Interactive", - "Deterministic analysis" - ], - "websiteURL": "https://github.com/mrcha033/skills", - "defaultPrompt": [ - "Acquire resumable read-only KIS Korean and U.S. adjusted-EOD bundles.", - "Calculate a technical report from these completed OHLCV files.", - "Evaluate frozen selected candidates against opening-hour outcomes." - ] - } -} diff --git a/plugins/quant-stock-technical/skills/quant-stock-technical/SKILL.md b/plugins/quant-stock-technical/skills/quant-stock-technical/SKILL.md deleted file mode 100644 index 609588b..0000000 --- a/plugins/quant-stock-technical/skills/quant-stock-technical/SKILL.md +++ /dev/null @@ -1,131 +0,0 @@ ---- -name: quant-stock-technical -description: Acquire resumable read-only KIS Korean and U.S. adjusted-EOD bundles, calculate reproducible numbers-only stock technical analysis, screen deterministic KOSPI, KOSDAQ, NYSE, and NASDAQ target universes, and evaluate frozen selected candidates against opening-hour outcomes. Use when the user needs completed historical OHLCV and benchmarks, an as-of-date market/ticker report, short-, medium-, and long-horizon opinions, a quantitative risk counterpoint, entry/stop/take-profit levels, a 0-100 technical-strength score, a frozen exchange-aware screen, or a read-only point-in-time or explicitly survivor-biased walk-forward diagnostic. Do not use news, fundamentals, analyst views, narrative judgment, intraday execution, or live-order claims. ---- - -# Quant Stock Technical - -Produce technical-analysis outputs only through the bundled deterministic calculator. Do not estimate indicators, prices, labels, or scores in prose. - -## Required inputs - -Obtain two finalized daily CSV files, or build the Korean files with the -read-only KIS EOD workflow below: - -- ticker OHLCV with `date,open,high,low,close,adjusted_close,volume` -- benchmark OHLCV with the same columns - -Also obtain `market`, `ticker`, and the market's valid `tick_size`. Treat the last shared CSV date as the default analysis date. Use `--analysis-date` for historical cutoffs. Require at least 756 shared completed sessions; recommend 1,260. - -Never treat an unfinished daily candle as completed. Never silently substitute unadjusted data, another ticker, another benchmark, a guessed tick size, or synthetic observations. - -## Korean EOD acquisition - -When completed Korean history is missing, read -`references/eod-provider-contract.md`. Do not stop merely because the caller -did not prebuild thousands of CSVs. Before the entry session: - -1. Run `scripts/fetch_kis_kr_eod.py snapshot-sources` to freeze KRX KIND and - KIS KOSPI/KOSDAQ master files. -2. Bind `analysis_date` to the frozen calendar's previous completed session. -3. Run `scripts/fetch_kis_kr_eod.py collect --job ...` with at least 756 - sessions and the existing secure KIS credential environment. -4. Preserve `eod-bundle-receipt.json`, `eod-catalog.csv`, benchmarks, stock - CSVs, and `universe-build-spec.json` together. -5. Continue to universe build and screening only when the required market - scope is complete. Never lower coverage or fabricate inactive exchanges. - -The collector is resumable: the first run backfills in bounded pages and later -runs update only missing completed sessions. Run it outside the first-hour -polling path. It performs authentication and market-data reads only; -`api_mutation_count` must remain zero and no account identifier is required. -Do not issue a separate token-only auth probe immediately before starting the -collector; KIS limits token issuance to once per minute. Treat the collector's -first successful read as authentication evidence or inject one -supervisor-held token through the environment. - -## U.S. EOD acquisition - -When NYSE/NASDAQ history is missing, read -`references/us-eod-provider-contract.md`. Freeze Nasdaq Trader official symbol -directories and KIS overseas masters with -`scripts/fetch_kis_us_eod.py snapshot-sources`, then run `collect --job ...` -outside the U.S. entry window. KIS stock history must use adjusted `MODP=1`; -preserve exchange-specific membership, broker symbols, benchmarks, and tick -mappings. A U.S.-scoped build uses NYSE/NASDAQ only; combine a completed -Korean catalog only when the caller explicitly requests an all-four-exchange -research universe. If KIS supplies an interior -adjusted row with invalid OHLC geometry, preserve its hash and values in the -invalid-row audit and exclude only that real row without interpolation. An -invalid cutoff row remains blocking. - -## Workflow - -1. Read `references/methodology.md` when explaining, reviewing, or changing the model. -2. Run `scripts/analyze_stock.py` for the stable QTA1 baseline. Use - `scripts/analyze_stock_v2.py` only for first-hour research/shadow work. -3. Return the calculator output without changing values or labels. -4. State the data source and cutoff. Keep `score_basis` visible whenever a user could mistake the score for a probability. -5. If the calculator returns `BLOCKED`, report the missing or invalid input. Do not manufacture a partial recommendation. - -Example: - -```bash -python3 scripts/analyze_stock.py \ - --ticker-csv arm.csv \ - --benchmark-csv soxx.csv \ - --market NASDAQ \ - --ticker ARM \ - --tick-size 0.01 \ - --source-name "licensed-eod-export" \ - --format markdown -``` - -Use `--format json` for machine-readable output. Run `python3 scripts/analyze_stock.py --self-test` after modifying the calculator. - -For a Korean, U.S., or all-four-exchange target universe, first read -`references/universe-contract.md`, build `qta-universe-manifest/v2` with -`scripts/build_universe_manifest.py`, then read `references/screen-contract.md` -and run `scripts/screen_universe.py`. Use `qta-universe-manifest/v1` only for -legacy, explicitly curated KR/US inputs. The selector is a separate -cross-sectional strategy assumption; it does not turn either score into a -probability. `screen_universe.py --method-version qta-2.0.0` selects the -research-only first-hour model; its output additionally requires a fresh -same-session benchmark-regime admission in the execution runner. Pass the -complete frozen screen artifact to -`$quant-stock-polling-trader`; never submit orders from this skill. - -After changing universe or screening code, run: - -```bash -python3 scripts/fetch_kis_kr_eod.py --self-test -python3 scripts/fetch_kis_us_eod.py --self-test -python3 scripts/build_universe_manifest.py --self-test -python3 scripts/analyze_stock_v2.py --self-test -python3 scripts/collect_open1h_research.py --self-test -python3 scripts/evaluate_open1h_research.py --self-test -python3 scripts/selected_open1h_outcomes.py --self-test -python3 scripts/walk_forward_open1h.py --self-test -python3 scripts/screen_universe.py --self-test -``` - -For a longer selected-candidate diagnostic, use -`scripts/walk_forward_open1h.py`. It holds one source manifest membership -static across prior sessions and therefore labels the result -`STATIC_SOURCE_MANIFEST_SURVIVOR_BIASED`; it is research evidence, not a -point-in-time universe backtest or promotion evidence. - -When `$stock-scenario-story` will run next, save the unmodified JSON output to a file. That downstream skill requires `source_skill`, `result_schema`, `method_version`, and the complete calculator payload; do not summarize or reconstruct the handoff manually. - -## Output boundaries - -- Interpret the 0-100 score as historical technical strength under the emitted - method version, not as probability of profit. `qta-2.0.0` remains - `RESEARCH_ONLY` until multi-session walk-forward evidence exists. -- Treat entry, stop, and take-profit as calculated price levels, not execution or suitability advice. -- Keep brokerage fees, tax, FX, slippage, position sizing, account constraints, and order semantics outside this technical score unless a later version explicitly models them. -- Do not claim real-time monitoring. This version consumes completed end-of-day data. -- Keep account identifiers, intraday quotes, and broker mutations outside this - skill. KIS app credentials may be resolved only by the read-only EOD - collector through the existing secure environment or mode-0600 fallback; - never persist or print them. diff --git a/plugins/quant-stock-technical/skills/quant-stock-technical/agents/openai.yaml b/plugins/quant-stock-technical/skills/quant-stock-technical/agents/openai.yaml deleted file mode 100644 index 5602b74..0000000 --- a/plugins/quant-stock-technical/skills/quant-stock-technical/agents/openai.yaml +++ /dev/null @@ -1,4 +0,0 @@ -interface: - display_name: "Quant Stock Technical" - short_description: "Four-exchange screening and opening-hour research" - default_prompt: "Use $quant-stock-technical to acquire read-only KIS EOD data, calculate reproducible technical scores, screen a frozen four-exchange universe, or evaluate selected opening-hour outcomes." diff --git a/plugins/quant-stock-technical/skills/quant-stock-technical/references/eod-provider-contract.md b/plugins/quant-stock-technical/skills/quant-stock-technical/references/eod-provider-contract.md deleted file mode 100644 index 3193569..0000000 --- a/plugins/quant-stock-technical/skills/quant-stock-technical/references/eod-provider-contract.md +++ /dev/null @@ -1,139 +0,0 @@ -# KIS Korean adjusted-EOD provider contract - -`fetch_kis_kr_eod.py` fills the operational input gap between a frozen Korean -market session and the offline universe builder. It is a read-only, pre-open -data stage. It never belongs in the first-hour polling loop and never calls a -broker order, account, balance, fill, correction, or cancellation endpoint. - -## Source contract - -Freeze dated source masters first: - -```bash -python3 scripts/fetch_kis_kr_eod.py snapshot-sources \ - --as-of 2026-07-27 \ - --output-directory /secure/qta/sources/2026-07-27/kr -``` - -The command downloads the KRX KIND corporate-list HTML export and the official -KIS KOSPI/KOSDAQ master ZIPs, verifies their expected format, extracts only the -named `.mst` members, hashes every file, and emits `source-snapshot.json`. -Review the provider URLs when they change; never silently replace a failed -official snapshot with a scraped or synthetic list. - -## EOD job - -Create one exact `qta-kis-kr-eod-job/v1` JSON object: - -```json -{ - "schema": "qta-kis-kr-eod-job/v1", - "as_of": "2026-07-27", - "analysis_date": "2026-07-24", - "environment": "live", - "output_directory": "/secure/qta/eod/kr", - "history_start_date": "2021-01-01", - "minimum_sessions": 756, - "request_interval_ms": 120, - "official_sources": [], - "broker_sources": [], - "catalog_coverage_contract": { - "schema": "qta-catalog-coverage-contract/v1", - "minimum_ratio_by_exchange": { - "KOSPI": "1", - "KOSDAQ": "1", - "NYSE": "1", - "NASDAQ": "1" - }, - "minimum_screenable_ratio_by_exchange": { - "KOSPI": "0.5", - "KOSDAQ": "0.5", - "NYSE": "0.5", - "NASDAQ": "0.5" - } - }, - "base_eod_catalog": "" -} -``` - -Copy the exact KOSPI/KOSDAQ descriptors from `source-snapshot.json` into the -two source arrays. This produces a complete Korean-scoped v2 build spec. -Other exchange descriptors and a previously frozen cross-market catalog may -be included only for an explicit all-four-exchange research build. Relative -paths resolve from the job file. - -Run: - -```bash -python3 scripts/fetch_kis_kr_eod.py collect --job kr-eod-job.json -``` - -The KIS domestic daily endpoint is queried with -`FID_ORG_ADJ_PRC=0`, which the KIS interface defines as adjusted price. Its -adjusted OHLC is written to the calculator schema with `adjusted_close` equal -to the returned adjusted `close`. The same cutoff is enforced for KOSPI index -code `0001` and KOSDAQ index code `1001`. - -## Cache and completeness - -- The first run walks backward using the broker's bounded page size. The stock - endpoint currently returns up to 100 rows and the index endpoint may return - 50; completion is determined by the date cursor, never by assuming a short - page is final. -- Each successful symbol CSV is written atomically and survives an interrupted - bootstrap. -- Later runs reuse complete history and request only a missing cutoff. -- The exact `analysis_date` must exist in every admitted stock and benchmark. -- At least 756 sessions are required; fewer rows are a per-symbol failure, not - permission to synthesize observations. -- A conflicting duplicate date, invalid OHLC geometry, malformed response, - persistent rate limit, or missing completed candle is `BLOCKED`. -- The output catalog still records every official Korean member. Unsupported - instruments and missing KIS matches remain explicit exclusions. - -An individual new listing with fewer than 756 sessions does not make the -collector itself fail. Its catalog row remains present but has no admissible -EOD path or tick contract. The downstream hashed screenable-coverage policy, -not the collector, decides whether the complete market scope is `READY`. - -The output directory contains: - -```text -benchmarks/ -stocks/KOSPI/ -stocks/KOSDAQ/ -eod-catalog.csv -universe-build-spec.json -eod-bundle-receipt.json -``` - -If only Korean source descriptors are supplied, the emitted build spec is a -complete KOSPI/KOSDAQ input. The v2 builder emits zero counts for inactive -NYSE/NASDAQ keys and evaluates coverage only for the active Korean pair. Do -not add U.S. inputs to a Korean scheduled task or lower coverage thresholds. - -## Credentials and mutation boundary - -The collector resolves only KIS app key and secret from existing -`QTA_KIS_*` environment variables or the user-owned mode-0600 fallback already -used by the polling skill. An optional access token may be injected through -`QTA_KIS_ACCESS_TOKEN` or its scoped form; tokens are never written to disk or -receipts. When no token is injected, one token is requested for the collector -process. KIS limits token issuance to once per minute. Do not run a separate -token-only `auth-check` immediately before a collector process. Prefer using -the collector's first successful market-data response as the authentication -gate, or inject the same in-memory supervisor token through the process -environment. If KIS returns `EGW00133`, the collector waits 61 seconds and -retries token issuance exactly once. - -Every receipt fixes `api_mutation_count` to zero. Authentication and quotation -reads do not arm live trading. - -## Tick contract - -The collector resolves the current KRX equity quotation unit from the final -completed adjusted close and records an effective, source-observed rule ID, -reference price, and tick size for every admitted symbol. KOSPI and KOSDAQ -remain distinct ladders. ETF, ETN, ELW, preferred, SPAC, right, warrant, -abnormal, halted, liquidation, and administrative issues are not promoted into -the common-stock screen. diff --git a/plugins/quant-stock-technical/skills/quant-stock-technical/references/methodology.md b/plugins/quant-stock-technical/skills/quant-stock-technical/references/methodology.md deleted file mode 100644 index edebb15..0000000 --- a/plugins/quant-stock-technical/skills/quant-stock-technical/references/methodology.md +++ /dev/null @@ -1,162 +0,0 @@ -# QTA 1.0.0 methodology - -## Scope - -`qta-1.0.0` is a long-only, daily, trend-following technical-strength model. It uses adjusted OHLCV and a same-market benchmark. It contains no news, fundamentals, sentiment, analyst estimates, sector narrative, or discretionary overrides. - -## Causal data handling - -- Use only sessions on or before the analysis date. -- Intersect ticker and benchmark dates. -- Align adjusted historical prices to the ticker's raw close scale on the analysis date. -- Exclude the current feature observation from its historical reference distribution. -- Require at least 756 shared sessions and at least 252 valid prior observations for every scored feature. - -## Percentile transform - -For each feature, compare the current value with up to 756 preceding valid values: - -```text -percentile = 100 * (count(prior < current) + 0.5 * count(prior = current)) / count(prior) -``` - -This is a ticker-relative historical percentile. It is not a forecast probability and is not a cross-sectional rank against other securities. - -## Horizon scores - -Short score: - -```text -25% return_5 -20% return_20 -20% close_over_sma_20 -15% sma_5_over_sma_20 -15% signed_log_volume_ratio_20 - 5% rsi_14 -``` - -Medium score: - -```text -25% return_63 -20% close_over_sma_50 -20% sma_20_over_sma_50 -20% relative_return_63 -15% adx_direction_14 -``` - -Long score: - -```text -25% return_126 -25% return_252 -20% close_over_sma_200 -15% sma_50_over_sma_200 -15% relative_return_252 -``` - -`signed_log_volume_ratio_20` is the sign of the one-day return multiplied by `ln(volume / median_volume_20)`. `adx_direction_14` is `(+DI - -DI) * min(ADX / 25, 1)`. - -## Risk score and total score - -Risk score: - -```text -30% percentile(ATR_14 / close) -25% percentile(20-session annualized log-return volatility) -25% percentile(252-session maximum drawdown magnitude) -20% percentile(60-session 95th-percentile absolute overnight gap) -``` - -Total score: - -```text -clip( - 0.25 * short - + 0.35 * medium - + 0.40 * long - - 0.20 * max(risk - 50, 0), - 0, - 100 -) -``` - -Opinion labels are mechanical: - -| Score | Label | -|---:|---| -| 80-100 | 강한 긍정 | -| 60-79.999 | 긍정 | -| 40-59.999 | 중립 | -| 20-39.999 | 부정 | -| 0-19.999 | 강한 부정 | - -## Price levels - -Use the market tick size supplied by the caller: - -```text -entry = round_up_to_tick(max(current_close, prior_20_session_high + tick)) -stop = round_down_to_tick(entry - 2 * ATR_14) -take_profit = round_down_to_tick(entry + 2 * (entry - stop)) -``` - -The setup is `READY` when total score is at least 60 and otherwise `CONDITIONAL`. Data or calculation failures return `BLOCKED`. - -## Quantitative counterpoint - -Select the risk component with the highest historical percentile. Render only its metric value, historical percentile, and entry-to-stop percentage using a fixed template. Do not add narrative judgment. - -## Downstream handoff contract - -Machine-readable results include: - -```text -source_skill = quant-stock-technical -result_schema = quant-stock-technical/v1 -method_version = qta-1.0.0 -calculation_status = READY -``` - -`stock-scenario-story` accepts only the complete unmodified JSON result. A Markdown rendering, hand-entered score summary, incomplete payload, or failed calculation is not a valid handoff. - -## QTA 2.0.0 first-hour research model - -`qta-2.0.0` reuses the same causal daily features, cutoff rules, price levels, -and per-ticker historical percentile transform. It changes only the horizon -aggregation for the first regular-session hour: - -```text -clip( - 0.50 * short - + 0.30 * medium - + 0.20 * (100 - risk), - 0, - 100 -) -``` - -Long-horizon strength remains visible in the payload but has zero weight in -the first-hour total. A setup also requires median 20-session turnover of at -least KRW 1,000,000,000 for Korean instruments or USD 1,000,000 for U.S. -instruments. - -The frozen payload requires a downstream same-session market-regime check: - -```text -metric = same_session_previous_close_return_bps -minimum_change_bps = 0 -max_age_seconds = 90 -``` - -The runner admits a new entry only when the exchange benchmark or documented -broad-market proxy is fresh, belongs to the current session, and is not below -the previous close. It never applies this check to exit management. - -The model was created after a full-universe diagnostic for the 2026-07-28 -opening hour. On that single session relative to QTA1, Spearman rank -correlation increased on KOSDAQ, KOSPI, NASDAQ, and NYSE. This is diagnostic -evidence, not sufficient promotion evidence. The payload therefore emits -`validation_status=RESEARCH_ONLY`, and the execution skill accepts it only in -shadow mode until multiple later sessions are evaluated without retuning the -formula. diff --git a/plugins/quant-stock-technical/skills/quant-stock-technical/references/screen-contract.md b/plugins/quant-stock-technical/skills/quant-stock-technical/references/screen-contract.md deleted file mode 100644 index bf70deb..0000000 --- a/plugins/quant-stock-technical/skills/quant-stock-technical/references/screen-contract.md +++ /dev/null @@ -1,163 +0,0 @@ -# QTA screen contracts - -`qta-screen-1.0.0` applies a frozen cross-sectional selector to unmodified -`quant-stock-technical/v1` results. It does not modify `qta-1.0.0`. - -## Legacy v1 input - -Pass a universe manifest: - -```json -{ - "schema": "qta-universe-manifest/v1", - "analysis_date": "2026-07-25", - "instruments": [ - { - "market": "KR", - "ticker": "005930", - "ticker_csv": "data/005930.csv", - "benchmark_csv": "data/kospi.csv", - "tick_size": "100", - "source_name": "licensed-adjusted-eod" - } - ] -} -``` - -Relative CSV paths resolve from the manifest directory. Require unique -`(market, ticker)` pairs and `market` equal to `KR` or `US`. - -Pass an explicit selector: - -```json -{ - "selector_version": "qta-screen-1.0.0", - "min_total_score": "60", - "eligible_setup_statuses": ["READY"], - "top_k_by_market": {"KR": 5, "US": 5}, - "max_blocked_fraction": "0" -} -``` - -Do not hide strategy choices in defaults. Every field above is required. -`eligible_setup_statuses` must be a non-empty subset of the exact uppercase -values `READY` and `CONDITIONAL`; no other calculator or caller-defined status -is eligible. - -## Calculation and ordering - -Run the original calculator once per instrument with the manifest analysis -date. Preserve each complete calculator payload. - -Filter by calculation status, setup status, and minimum score. Sort eligible -results with this total ordering: - -```text -total_score descending -medium.score descending -long.score descending -short.score descending -risk.score ascending -market ascending -ticker ascending -``` - -The score is ticker-relative rather than a cross-sectional rank. Treat this -ordering as a new strategy assumption that needs its own historical, -walk-forward, and opening-hour execution validation. - -If the fraction of `BLOCKED` instruments exceeds `max_blocked_fraction`, return -the whole screen as `BLOCKED`. Never silently drop enough invalid instruments -to change the effective universe beyond that frozen tolerance. - -## Output - -Emit `qta-screen/v1` with: - -- selector and method versions; -- canonical hashes of the manifest and selector; -- the complete unmodified QTA payload for every instrument; -- deterministic eligibility decisions and ranks; -- selected candidates per market; -- explicit blocked reasons. - -Use the output only as an input to a separate execution skill. It is not an -order, suitability decision, or profit forecast. - -## Exchange-scoped v2 input - -Read `universe-contract.md` first. Pass the complete, unmodified -`qta-universe-manifest/v2` emitted by `build_universe_manifest.py`; the screen -recomputes its canonical hash and verifies source, CSV, tick, count, and broker -tradability metadata. The manifest may cover the Korean pair, the U.S. pair, -or all four exchanges. It rejects a `BLOCKED` manifest. - -Pass this exact selector: - -```json -{ - "selector_version": "qta-screen-1.1.0", - "min_total_score": "60", - "eligible_setup_statuses": ["READY"], - "top_k_by_exchange": { - "KOSPI": 2, - "KOSDAQ": 2, - "NYSE": 2, - "NASDAQ": 2 - }, - "min_selected_by_exchange": { - "KOSPI": 0, - "KOSDAQ": 0, - "NYSE": 0, - "NASDAQ": 0 - }, - "max_blocked_fraction": "0" -} -``` - -Every field is required. `top_k_by_exchange` must contain all four keys, -including any exchange intentionally assigned zero slots. -`min_selected_by_exchange` must also contain all four keys, and every value -must be between zero and that exchange's top K. Use zero to allow a valid -no-trade result when no setup passes; use one or more only when underfilling -that exchange should block the whole screen. The same exact -`eligible_setup_statuses` restriction from v1 applies. - -By default, run the stable `qta-1.0.0` calculator once per included -instrument. Pass `--method-version qta-2.0.0` only for the research-only -first-hour model described in `methodology.md`. Use -`canonical_symbol` as the QTA ticker, the explicit exchange benchmark CSV, and -`tick_contract.resolved_tick_size`; never use `data_symbol` or `broker_symbol` -interchangeably. - -Apply the method-specific score ordering independently inside each exchange. -QTA1 orders medium, long, short, then risk after total score. QTA2 orders -short, medium, risk, then long after total score. The final -tie-break is canonical symbol ascending. Set `exchange_rank` before taking the -configured top K. A score remains ticker-relative rather than a -cross-sectional percentile or probability. - -If the fraction of calculation-blocked included instruments exceeds -`max_blocked_fraction`, the entire v2 screen is `BLOCKED` and every exchange's -selected list is empty. The same happens when an exchange has fewer eligible -names than its explicit `min_selected_by_exchange`. Listings excluded by the -builder remain visible in the manifest and are not counted as attempted -calculations. - -Emit `qta-screen/v2` with: - -- the exact manifest hash, selector hash, and final screen hash; -- the complete unmodified QTA payload for each attempted instrument; -- explicit eligibility decisions and exchange ranks; -- exact selected keys `KOSPI`, `KOSDAQ`, `NYSE`, and `NASDAQ`; -- the complete v2 instrument metadata alongside every selected payload. - -The polling trader combines KOSPI then KOSDAQ for a Korean session, and NYSE -then NASDAQ for a U.S. session. It orders by the frozen exchange order, -`exchange_rank`, then `broker_symbol`; it does not rescore candidates. - -`screen_universe.py` uses up to eight worker processes by default. `--workers -1` provides the serial reference path. Both paths must emit byte-identical -artifacts. When `--output` is supplied, the complete screen is written only to -that file and stdout contains a compact hash/status summary rather than -thousands of instrument payloads. diff --git a/plugins/quant-stock-technical/skills/quant-stock-technical/references/universe-contract.md b/plugins/quant-stock-technical/skills/quant-stock-technical/references/universe-contract.md deleted file mode 100644 index 5339ebe..0000000 --- a/plugins/quant-stock-technical/skills/quant-stock-technical/references/universe-contract.md +++ /dev/null @@ -1,252 +0,0 @@ -# Exchange-scoped universe contract - -`qta-universe-manifest/v2` freezes a reproducible, screen-ready intersection -for exactly one supported scope: KOSPI/KOSDAQ, NYSE/NASDAQ, or all four -exchanges. It is not a live quote feed and it does not make every listed -security eligible. - -## Source boundary - -Build each exchange from dated local snapshots: - -- an official listing or symbol-directory snapshot; -- a broker symbol-master snapshot; -- explicit symbol-to-adjusted-EOD-CSV mappings; -- an explicit same-exchange benchmark CSV mapping; -- explicit, effective-dated tick contracts. - -Hash every input file before parsing it. Keep the raw files immutable with the -result. Never query thousands of symbols one by one during the entry window. -Refresh and screen from completed T-1 data before the session, then pass only -the selected candidates to the polling trader. - -Run the offline builder with: - -```bash -python3 scripts/build_universe_manifest.py \ - --build-spec universe-build-spec.json \ - --output universe-manifest.json -``` - -The build spec has exactly: - -```text -schema, as_of, analysis_date, official_sources, broker_sources, eod_catalog, -catalog_coverage_contract -``` - -`schema` is `qta-universe-build-spec/v1`. Supply exactly one official source -and one KIS broker source for each exchange in the selected supported scope. -Do not mix a single exchange from either market. Every source descriptor has -exactly: - -```text -source_id, provider, exchange, as_of, path, format, encoding, -delimiter, skip_rows, columns, normal_status_values -``` - -All inputs are local snapshot files. The built-in direct formats are: - -| role | exchange | provider | format | -| --- | --- | --- | --- | -| official | KOSPI/KOSDAQ | `KRX` | `KRX_KIND_HTML` | -| official | NASDAQ | `NASDAQ_TRADER` | `NASDAQ_LISTED` | -| official | NYSE | `NASDAQ_TRADER` | `NASDAQ_OTHER` | -| broker | KOSPI/KOSDAQ | `KIS` | `KIS_KRX_MASTER` | -| broker | NASDAQ/NYSE | `KIS` | `KIS_OVERSEAS_MASTER` | - -`KRX_KIND_HTML` consumes the CP949 KIND corporate-list HTML-table export. -`KIS_KRX_MASTER` consumes the raw domestic `.mst` layout, including its -exchange-specific trailing flag block. `KIS_OVERSEAS_MASTER` consumes the raw, -headerless 24-column tab-separated `.COD` file. Set `columns` to `{}` for -these built-ins. Generic normalized inputs may instead use `KRX_CSV`, -`KIS_CSV`, or `KIS_FIXED_WIDTH` with explicit logical-column mappings. Never -apply decoded-character fixed offsets copied from one raw KIS market to -another. - -Acquire and timestamp snapshots outside the builder. Current official source -locations are: - -```text -KRX KIND: -https://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13 - -Nasdaq Trader: -https://www.nasdaqtrader.com/dynamic/SymDir/nasdaqlisted.txt -https://www.nasdaqtrader.com/dynamic/SymDir/otherlisted.txt - -KIS: -https://new.real.download.dws.co.kr/common/master/kospi_code.mst.zip -https://new.real.download.dws.co.kr/common/master/kosdaq_code.mst.zip -https://new.real.download.dws.co.kr/common/master/nasmst.cod.zip -https://new.real.download.dws.co.kr/common/master/nysmst.cod.zip -``` - -Check the provider's current documentation before automating downloads. The -builder deliberately performs no network access. - -`eod_catalog` has exactly `source_id, provider, as_of, path, encoding`. Its CSV -header is exactly, in this order: - -```text -exchange,canonical_symbol,data_symbol,broker_symbol,instrument_type, -benchmark_id,ticker_csv,benchmark_csv,tick_rule_id,tick_effective_date, -tick_reference_price,resolved_tick_size,source_name -``` - -The catalog is an explicit join contract, not a security master. A listed -symbol absent from it is excluded with `missing_eod_mapping`. Blank, invalid, -or nonexistent data and tick fields become per-symbol exclusion reasons rather -than guessed values. - -`catalog_coverage_contract` is a mandatory readiness gate with exactly: - -```json -{ - "schema": "qta-catalog-coverage-contract/v1", - "minimum_ratio_by_exchange": { - "KOSPI": "1", - "KOSDAQ": "1", - "NYSE": "1", - "NASDAQ": "1" - }, - "minimum_screenable_ratio_by_exchange": { - "KOSPI": "0.5", - "KOSDAQ": "0.5", - "NYSE": "0.5", - "NASDAQ": "0.5" - } -} -``` - -Each minimum is a decimal greater than zero and at most one. The builder -canonicalizes it to a decimal string. Both ratios use the frozen official-row -count as their denominator and are evaluated independently for every exchange: - -- `minimum_ratio_by_exchange` gates `catalog_mapped / official`. A value of - `1` requires an EOD-catalog row for every official member. -- `minimum_screenable_ratio_by_exchange` gates `included / official`, after - instrument classification, broker tradability, completed adjusted EOD, - benchmark, file-hash, and tick-contract checks. - -Both gates must pass. A catalog row alone never makes a symbol screenable, and -a high screenable ratio never excuses missing catalog mappings. -`screen_universe.py` independently recomputes both ratios from the hashed -manifest counts and rejects a forged `READY` manifest that misses either -minimum. - -The normalized contract is copied into the manifest before `manifest_hash` is -computed. This makes lowering or changing a threshold visible in the manifest -hash. For a full-universe build, use `1` for all four catalog-mapping -minimums. Choose screenable minimums from an explicit coverage policy that -accounts for intentionally excluded security classes, and do not lower either -set of values merely to turn a sparse proof-of-concept catalog `READY`. - -The canonical exchange mapping is: - -| exchange | market | venue | currency | benchmark_id | -| --- | --- | --- | --- | --- | -| `KOSPI` | `KR` | `KRX` | `KRW` | `KOSPI_COMPOSITE` | -| `KOSDAQ` | `KR` | `KRX` | `KRW` | `KOSDAQ_COMPOSITE` | -| `NYSE` | `US` | `NYSE` | `USD` | `NYSE_COMPOSITE` | -| `NASDAQ` | `US` | `NASD` | `USD` | `NASDAQ_COMPOSITE` | - -## Eligible instruments - -Include only `COMMON`, `ADR`, and `REIT`; `ADR` is U.S.-only. Exclude ETFs, -ETNs, preferred shares, SPACs, units, rights, warrants, test issues, abnormal -listing-status issues, and any row that cannot be classified deterministically. - -An otherwise eligible listing is still excluded when it lacks: - -- an unambiguous broker-master match; -- confirmed broker tradability; -- a ticker CSV and benchmark CSV whose last shared completed date reaches the - requested analysis date; -- file hashes and valid adjusted-OHLCV columns; -- an effective tick contract. - -Every exclusion must carry sorted, explicit reason codes. Never silently shrink -the universe. - -## Exact manifest - -The top-level object has exactly: - -```text -schema, build_status, as_of, analysis_date, source_hashes, -catalog_coverage_contract, instruments, exclusions, counts, manifest_hash -``` - -`schema` is `qta-universe-manifest/v2`. `build_status` is `READY` only when -every active exchange has at least one included instrument and meets both -hashed catalog-mapping and screenable-coverage minimums; otherwise it is -`BLOCKED`. Inactive exchanges must have zero counts. - -Each instrument has exactly: - -```text -market, exchange, canonical_symbol, data_symbol, broker_symbol, -instrument_type, benchmark_id, currency, venue, ticker_csv, benchmark_csv, -tick_contract, source_name, ticker_csv_sha256, benchmark_csv_sha256, -broker_tradability_verified, official_source_id, broker_source_id -``` - -All paths are absolute in the emitted manifest. Symbols have separate -canonical, data-provider, and broker representations; never assume they are -identical. - -The required tick object has exactly: - -```json -{ - "schema": "qta-tick-contract/v1", - "kind": "RESOLVED_PRICE_LADDER", - "rule_id": "provider-rule-identifier", - "effective_date": "2026-07-25", - "reference_price": "123.45", - "resolved_tick_size": "0.005" -} -``` - -The two numeric values are positive canonical decimal strings. Do not hard-code -one U.S. tick or one Korean tick for an entire exchange. - -`source_hashes` is sorted by `source_id`. Each entry has exactly -`source_id, role, provider, exchange, as_of, path, sha256`; `exchange` may be -`GLOBAL` for a cross-exchange mapping. Hashes are lowercase SHA-256 values. - -Each exclusion has exactly: - -```text -exchange, canonical_symbol, broker_symbol, name, reasons, -official_source_id, broker_source_id -``` - -`broker_symbol` and `broker_source_id` may be null. `reasons` is a non-empty, -sorted, unique list. - -`counts` has exact totals `official_rows, broker_rows, catalog_mapped_rows, -included, excluded` and a `by_exchange` object with all four exchanges. Each -exchange has exact `official, broker, catalog_mapped, included, excluded` -counts; totals must equal their sums. Keys remain present with all-zero values -for inactive exchanges so the v2 shape stays stable. `catalog_mapped` counts official symbols -with a catalog row, regardless of whether that row later passes EOD, tick, or -broker validation. `excluded` means exclusion-record count. It can include an -EOD-catalog row that is no longer an official member, so it is not defined as -`official - included`. - -`manifest_hash` is SHA-256 over UTF-8 canonical JSON for the entire object -excluding `manifest_hash`, using sorted keys, compact separators, and -non-ASCII characters unescaped. - -## Reproducibility and history - -Treat `as_of` as the membership/master snapshot date and `analysis_date` as the -completed EOD cutoff. Keep both dates and all hashes. Re-running from identical -snapshots must produce byte-identical JSON. - -A current listing snapshot cannot reconstruct a historical universe. For -historical or walk-forward testing, use the membership and broker-master -snapshots that were actually frozen for each historical date; otherwise label -the result as survivorship-biased and do not use it as promotion evidence. diff --git a/plugins/quant-stock-technical/skills/quant-stock-technical/references/us-eod-provider-contract.md b/plugins/quant-stock-technical/skills/quant-stock-technical/references/us-eod-provider-contract.md deleted file mode 100644 index 58dc9d1..0000000 --- a/plugins/quant-stock-technical/skills/quant-stock-technical/references/us-eod-provider-contract.md +++ /dev/null @@ -1,60 +0,0 @@ -# KIS U.S. adjusted-EOD provider contract - -`fetch_kis_us_eod.py` produces a frozen NYSE/NASDAQ catalog before the U.S. -session. It performs authentication and quotation -reads only and fixes `api_mutation_count` to zero. - -## Frozen membership and tradability - -Run `snapshot-sources` with the source date. The command freezes and hashes: - -- Nasdaq Trader `nasdaqlisted.txt` for official NASDAQ membership; -- Nasdaq Trader `otherlisted.txt`, filtered to exchange `N`, for official NYSE - membership; -- KIS `nasmst.cod` and `nysmst.cod` for broker symbol qualification. - -The admitted universe is the deterministic intersection after the existing -common-stock, ADR, REIT, ETF, test-issue, abnormal-status, and broker -tradability rules. Never substitute AMEX, ARCA, OTC, a search result, or an -unqualified symbol. - -## Adjusted history - -KIS overseas daily history uses -`/uapi/overseas-price/v1/quotations/dailyprice`, TR `HHDFS76240000`, daily -`GUBN=0`, and adjusted-price `MODP=1`. NYSE uses `EXCD=NYS`; NASDAQ uses -`EXCD=NAS`. The cursor walks backward from the completed cutoff and writes -atomic, resumable CSV files. Every admitted ticker and benchmark requires the -exact cutoff and at least 756 sessions. - -KIS can return an interior adjusted row whose reported open, high, low, and -close do not form valid OHLC geometry. Exclude that exact provider row without -interpolation, record its exchange, symbol, date, normalized values, size, and -SHA-256 in `invalid-eod-rows.json`, and continue only if the remaining real -rows still meet the complete-session contract. An invalid row on the requested -cutoff remains `BLOCKED`; never invent the latest completed candle. The -receipt binds the audit file and reports policy -`EXCLUDED_WITHOUT_INTERPOLATION`. - -The existing v2 universe contract names `NYSE_COMPOSITE` and -`NASDAQ_COMPOSITE`. Their public benchmark symbols are `^NYA` and `^IXIC`. -Benchmark downloads are source-labeled and hashed; a malformed, incomplete, or -rate-limited response blocks the benchmark rather than silently substituting -S&P 500, an ETF, or another index. - -U.S. order-price resolution uses a conservative valid ladder: `0.0001` below -USD 1 and `0.01` at or above USD 1. A later version may consume an -effective-dated Regulation NMS variable tick table for finer price increments, -but it must not infer one from price alone. - -## Scope - -With only U.S. descriptors and an empty `base_eod_catalog`, the emitted -`universe-build-spec.json` is a complete NYSE/NASDAQ v2 input. The job may -also include all four descriptors and a completed Korean base catalog for an -explicit all-four-exchange research build. Daily U.S. execution uses the -U.S.-only form and does not refresh Korean EOD. - -The initial bootstrap belongs outside both first-hour entry windows. Completed -symbol files survive interruption; later runs fetch only the missing completed -cutoff. diff --git a/plugins/quant-stock-technical/skills/quant-stock-technical/scripts/analyze_stock.py b/plugins/quant-stock-technical/skills/quant-stock-technical/scripts/analyze_stock.py deleted file mode 100644 index 4a62bc1..0000000 --- a/plugins/quant-stock-technical/skills/quant-stock-technical/scripts/analyze_stock.py +++ /dev/null @@ -1,693 +0,0 @@ -#!/usr/bin/env python3 -"""Deterministic daily stock technical analysis for quant-stock-technical.""" - -from __future__ import annotations - -import argparse -import csv -import json -import math -import statistics -import sys -import tempfile -from dataclasses import dataclass -from datetime import date, timedelta -from decimal import Decimal -from pathlib import Path -from typing import Any, Iterable, Optional - - -METHOD_VERSION = "qta-1.0.0" -MIN_SHARED_SESSIONS = 756 -PERCENTILE_LOOKBACK = 756 -MIN_REFERENCE_OBSERVATIONS = 252 - - -class BlockedError(ValueError): - pass - - -@dataclass(frozen=True) -class RawBar: - day: date - open: float - high: float - low: float - close: float - adjusted_close: float - volume: float - - -@dataclass(frozen=True) -class Bar: - day: date - open: float - high: float - low: float - close: float - volume: float - - -def _number(row: dict[str, str], field: str, row_number: int) -> float: - try: - value = float(row[field]) - except (KeyError, TypeError, ValueError) as exc: - raise BlockedError(f"row {row_number}: invalid or missing {field}") from exc - if not math.isfinite(value): - raise BlockedError(f"row {row_number}: non-finite {field}") - return value - - -def read_csv(path: str) -> list[RawBar]: - required = {"date", "open", "high", "low", "close", "adjusted_close", "volume"} - rows: list[RawBar] = [] - with Path(path).open("r", encoding="utf-8-sig", newline="") as handle: - reader = csv.DictReader(handle) - fields = {str(x).strip().lower() for x in (reader.fieldnames or [])} - if not required.issubset(fields): - missing = ", ".join(sorted(required - fields)) - raise BlockedError(f"CSV missing columns: {missing}") - for row_number, source_row in enumerate(reader, start=2): - row = {str(k).strip().lower(): v for k, v in source_row.items()} - try: - day = date.fromisoformat(str(row["date"]).strip()) - except (KeyError, ValueError) as exc: - raise BlockedError(f"row {row_number}: date must be YYYY-MM-DD") from exc - values = {name: _number(row, name, row_number) for name in required - {"date"}} - if min(values["open"], values["high"], values["low"], values["close"], values["adjusted_close"]) <= 0: - raise BlockedError(f"row {row_number}: prices must be positive") - if values["volume"] < 0: - raise BlockedError(f"row {row_number}: volume must be non-negative") - if values["high"] < max(values["open"], values["low"], values["close"]): - raise BlockedError(f"row {row_number}: high is inconsistent") - if values["low"] > min(values["open"], values["high"], values["close"]): - raise BlockedError(f"row {row_number}: low is inconsistent") - rows.append(RawBar(day=day, **values)) - rows.sort(key=lambda item: item.day) - if len({item.day for item in rows}) != len(rows): - raise BlockedError("CSV contains duplicate dates") - if not rows: - raise BlockedError("CSV contains no observations") - return rows - - -def normalize_to_latest_scale(rows: list[RawBar]) -> list[Bar]: - latest_factor = rows[-1].adjusted_close / rows[-1].close - if latest_factor <= 0: - raise BlockedError("invalid latest adjustment factor") - normalized: list[Bar] = [] - for row in rows: - factor = (row.adjusted_close / row.close) / latest_factor - normalized.append( - Bar( - day=row.day, - open=row.open * factor, - high=row.high * factor, - low=row.low * factor, - close=row.close * factor, - volume=row.volume, - ) - ) - return normalized - - -def cutoff(rows: list[RawBar], analysis_date: Optional[date]) -> list[RawBar]: - selected = [row for row in rows if analysis_date is None or row.day <= analysis_date] - if not selected: - raise BlockedError("no observations on or before analysis date") - if analysis_date is not None and selected[-1].day != analysis_date: - raise BlockedError("analysis date is not present in both completed-session files") - return selected - - -def align(ticker_rows: list[RawBar], benchmark_rows: list[RawBar], analysis_date: Optional[date]) -> tuple[list[Bar], list[Bar]]: - ticker_cut = cutoff(ticker_rows, analysis_date) - benchmark_cut = cutoff(benchmark_rows, analysis_date) - ticker_map = {row.day: row for row in ticker_cut} - benchmark_map = {row.day: row for row in benchmark_cut} - days = sorted(set(ticker_map) & set(benchmark_map)) - if not days: - raise BlockedError("ticker and benchmark have no shared dates") - target = analysis_date or min(ticker_cut[-1].day, benchmark_cut[-1].day) - days = [day for day in days if day <= target] - if not days or days[-1] != target: - raise BlockedError("latest requested date is not shared by ticker and benchmark") - ticker_aligned = normalize_to_latest_scale([ticker_map[day] for day in days]) - benchmark_aligned = normalize_to_latest_scale([benchmark_map[day] for day in days]) - if len(days) < MIN_SHARED_SESSIONS: - raise BlockedError(f"need at least {MIN_SHARED_SESSIONS} shared sessions; found {len(days)}") - return ticker_aligned, benchmark_aligned - - -def sma(values: list[float], period: int) -> list[Optional[float]]: - output: list[Optional[float]] = [None] * len(values) - running = 0.0 - for index, value in enumerate(values): - running += value - if index >= period: - running -= values[index - period] - if index >= period - 1: - output[index] = running / period - return output - - -def returns(values: list[float], period: int) -> list[Optional[float]]: - output: list[Optional[float]] = [None] * len(values) - for index in range(period, len(values)): - output[index] = values[index] / values[index - period] - 1.0 - return output - - -def rsi(values: list[float], period: int = 14) -> list[Optional[float]]: - output: list[Optional[float]] = [None] * len(values) - if len(values) <= period: - return output - gains = [0.0] * len(values) - losses = [0.0] * len(values) - for index in range(1, len(values)): - delta = values[index] - values[index - 1] - gains[index] = max(delta, 0.0) - losses[index] = max(-delta, 0.0) - avg_gain = sum(gains[1 : period + 1]) / period - avg_loss = sum(losses[1 : period + 1]) / period - for index in range(period, len(values)): - if index > period: - avg_gain = ((period - 1) * avg_gain + gains[index]) / period - avg_loss = ((period - 1) * avg_loss + losses[index]) / period - output[index] = 50.0 if avg_gain == avg_loss == 0 else (100.0 if avg_loss == 0 else 100.0 - 100.0 / (1.0 + avg_gain / avg_loss)) - return output - - -def atr_adx(bars: list[Bar], period: int = 14) -> tuple[list[Optional[float]], list[Optional[float]], list[Optional[float]], list[Optional[float]]]: - length = len(bars) - tr = [0.0] * length - plus_dm = [0.0] * length - minus_dm = [0.0] * length - for index, bar in enumerate(bars): - if index == 0: - tr[index] = bar.high - bar.low - continue - tr[index] = max(bar.high - bar.low, abs(bar.high - bars[index - 1].close), abs(bar.low - bars[index - 1].close)) - up = bar.high - bars[index - 1].high - down = bars[index - 1].low - bar.low - plus_dm[index] = up if up > down and up > 0 else 0.0 - minus_dm[index] = down if down > up and down > 0 else 0.0 - - atr_values: list[Optional[float]] = [None] * length - plus_di: list[Optional[float]] = [None] * length - minus_di: list[Optional[float]] = [None] * length - adx: list[Optional[float]] = [None] * length - if length <= period: - return atr_values, adx, plus_di, minus_di - - atr_value = sum(tr[1 : period + 1]) / period - plus_value = sum(plus_dm[1 : period + 1]) / period - minus_value = sum(minus_dm[1 : period + 1]) / period - dx: list[Optional[float]] = [None] * length - for index in range(period, length): - if index > period: - atr_value = ((period - 1) * atr_value + tr[index]) / period - plus_value = ((period - 1) * plus_value + plus_dm[index]) / period - minus_value = ((period - 1) * minus_value + minus_dm[index]) / period - atr_values[index] = atr_value - if atr_value > 0: - plus_di[index] = 100.0 * plus_value / atr_value - minus_di[index] = 100.0 * minus_value / atr_value - denominator = plus_di[index] + minus_di[index] - dx[index] = 0.0 if denominator == 0 else 100.0 * abs(plus_di[index] - minus_di[index]) / denominator - else: - # A newly listed or previously suspended instrument can begin with - # a flat true-range seed. Wilder's directional movement is zero - # in that case; leaving DX unavailable would prevent ADX from ever - # initializing, even after the instrument starts trading. - plus_di[index] = 0.0 - minus_di[index] = 0.0 - dx[index] = 0.0 - - first_adx = 2 * period - 1 - initial_dx = [value for value in dx[period : first_adx + 1] if value is not None] - if len(initial_dx) == period: - adx_value = sum(initial_dx) / period - adx[first_adx] = adx_value - for index in range(first_adx + 1, length): - if dx[index] is None: - continue - adx_value = ((period - 1) * adx_value + dx[index]) / period - adx[index] = adx_value - return atr_values, adx, plus_di, minus_di - - -def rolling_volatility(values: list[float], period: int = 20) -> list[Optional[float]]: - log_returns: list[Optional[float]] = [None] + [math.log(values[i] / values[i - 1]) for i in range(1, len(values))] - output: list[Optional[float]] = [None] * len(values) - for index in range(period, len(values)): - window = [value for value in log_returns[index - period + 1 : index + 1] if value is not None] - if len(window) == period: - output[index] = statistics.stdev(window) * math.sqrt(252.0) - return output - - -def rolling_max_drawdown(values: list[float], period: int = 252) -> list[Optional[float]]: - """Calculate exact sliding-window drawdown in amortized linear time.""" - if period <= 0: - raise BlockedError("maximum drawdown period must be positive") - output: list[Optional[float]] = [None] * len(values) - - # A segment stores (maximum price, minimum price, largest earlier/later - # peak-to-trough ratio). The operation is associative, so a two-stack - # aggregate queue supports an exact ordered sliding-window query. - Segment = tuple[float, float, float] - - def singleton(value: float) -> Segment: - return (value, value, 1.0) - - def combine(left: Segment, right: Segment) -> Segment: - return ( - max(left[0], right[0]), - min(left[1], right[1]), - max(left[2], right[2], left[0] / right[1]), - ) - - incoming: list[tuple[Segment, Segment]] = [] - outgoing: list[tuple[Segment, Segment]] = [] - - def push(value: float) -> None: - item = singleton(value) - aggregate = item if not incoming else combine(incoming[-1][1], item) - incoming.append((item, aggregate)) - - def pop_oldest() -> None: - if not outgoing: - while incoming: - item, _ = incoming.pop() - aggregate = ( - item - if not outgoing - else combine(item, outgoing[-1][1]) - ) - outgoing.append((item, aggregate)) - outgoing.pop() - - def aggregate() -> Segment: - if outgoing and incoming: - return combine(outgoing[-1][1], incoming[-1][1]) - if outgoing: - return outgoing[-1][1] - if incoming: - return incoming[-1][1] - raise BlockedError("cannot aggregate an empty drawdown window") - - window_size = 0 - for index, value in enumerate(values): - push(value) - window_size += 1 - if window_size > period: - pop_oldest() - window_size -= 1 - if window_size == period: - maximum_ratio = aggregate()[2] - output[index] = 1.0 - 1.0 / maximum_ratio - return output - - -def quantile(values: Iterable[float], probability: float) -> float: - ordered = sorted(values) - if not ordered: - raise BlockedError("cannot calculate quantile from empty data") - location = (len(ordered) - 1) * probability - lower = math.floor(location) - upper = math.ceil(location) - if lower == upper: - return ordered[lower] - return ordered[lower] * (upper - location) + ordered[upper] * (location - lower) - - -def rolling_gap95(bars: list[Bar], period: int = 60) -> list[Optional[float]]: - gaps: list[Optional[float]] = [None] + [abs(bars[i].open / bars[i - 1].close - 1.0) for i in range(1, len(bars))] - output: list[Optional[float]] = [None] * len(bars) - for index in range(period, len(bars)): - window = [value for value in gaps[index - period + 1 : index + 1] if value is not None] - if len(window) == period: - output[index] = quantile(window, 0.95) - return output - - -def rolling_signed_volume(bars: list[Bar], period: int = 20) -> list[Optional[float]]: - output: list[Optional[float]] = [None] * len(bars) - for index in range(period - 1, len(bars)): - median_volume = statistics.median(bar.volume for bar in bars[index - period + 1 : index + 1]) - if index == 0 or median_volume <= 0 or bars[index].volume <= 0: - continue - direction = 1.0 if bars[index].close > bars[index - 1].close else (-1.0 if bars[index].close < bars[index - 1].close else 0.0) - output[index] = direction * math.log(bars[index].volume / median_volume) - return output - - -def ratio_series(left: list[Optional[float]], right: list[Optional[float]]) -> list[Optional[float]]: - output: list[Optional[float]] = [None] * len(left) - for index, (a, b) in enumerate(zip(left, right)): - if a is not None and b not in (None, 0): - output[index] = a / b - 1.0 - return output - - -def close_over(values: list[float], average: list[Optional[float]]) -> list[Optional[float]]: - return [None if avg in (None, 0) else value / avg - 1.0 for value, avg in zip(values, average)] - - -def relative_returns(ticker: list[float], benchmark: list[float], period: int) -> list[Optional[float]]: - output: list[Optional[float]] = [None] * len(ticker) - for index in range(period, len(ticker)): - output[index] = (ticker[index] / ticker[index - period] - 1.0) - (benchmark[index] / benchmark[index - period] - 1.0) - return output - - -def percentile_rank(series: list[Optional[float]], index: int) -> tuple[float, int]: - current = series[index] - if current is None or not math.isfinite(current): - raise BlockedError("current feature value is unavailable") - start = max(0, index - PERCENTILE_LOOKBACK) - reference = [value for value in series[start:index] if value is not None and math.isfinite(value)] - if len(reference) < MIN_REFERENCE_OBSERVATIONS: - raise BlockedError(f"feature has only {len(reference)} prior observations; need {MIN_REFERENCE_OBSERVATIONS}") - less = sum(value < current for value in reference) - equal = sum(math.isclose(value, current, rel_tol=1e-12, abs_tol=1e-15) for value in reference) - return 100.0 * (less + 0.5 * equal) / len(reference), len(reference) - - -def weighted_score(percentiles: dict[str, float], weights: dict[str, float]) -> float: - return sum(percentiles[name] * weight for name, weight in weights.items()) - - -def opinion(score: float) -> str: - if score >= 80: - return "강한 긍정" - if score >= 60: - return "긍정" - if score >= 40: - return "중립" - if score >= 20: - return "부정" - return "강한 부정" - - -def decimals_for_tick(tick: float) -> int: - return max(0, -Decimal(str(tick)).normalize().as_tuple().exponent) - - -def round_up_tick(value: float, tick: float) -> float: - return math.ceil((value - 1e-12) / tick) * tick - - -def round_down_tick(value: float, tick: float) -> float: - return math.floor((value + 1e-12) / tick) * tick - - -def calculate(ticker_bars: list[Bar], benchmark_bars: list[Bar], market: str, ticker: str, tick_size: float, source_name: str) -> dict[str, Any]: - if tick_size <= 0 or not math.isfinite(tick_size): - raise BlockedError("tick size must be a positive finite number") - closes = [bar.close for bar in ticker_bars] - benchmark_closes = [bar.close for bar in benchmark_bars] - index = len(ticker_bars) - 1 - - sma5 = sma(closes, 5) - sma20 = sma(closes, 20) - sma50 = sma(closes, 50) - sma200 = sma(closes, 200) - ret5 = returns(closes, 5) - ret20 = returns(closes, 20) - ret63 = returns(closes, 63) - ret126 = returns(closes, 126) - ret252 = returns(closes, 252) - rsi14 = rsi(closes, 14) - atr14, adx14, plus_di14, minus_di14 = atr_adx(ticker_bars, 14) - signed_volume20 = rolling_signed_volume(ticker_bars, 20) - vol20 = rolling_volatility(closes, 20) - max_drawdown252 = rolling_max_drawdown(closes, 252) - gap95_60 = rolling_gap95(ticker_bars, 60) - - close_sma20 = close_over(closes, sma20) - close_sma50 = close_over(closes, sma50) - close_sma200 = close_over(closes, sma200) - sma5_sma20 = ratio_series(sma5, sma20) - sma20_sma50 = ratio_series(sma20, sma50) - sma50_sma200 = ratio_series(sma50, sma200) - relative63 = relative_returns(closes, benchmark_closes, 63) - relative252 = relative_returns(closes, benchmark_closes, 252) - atr_pct = [None if atr is None else atr / closes[i] for i, atr in enumerate(atr14)] - adx_direction: list[Optional[float]] = [None] * len(closes) - for i in range(len(closes)): - if adx14[i] is not None and plus_di14[i] is not None and minus_di14[i] is not None: - adx_direction[i] = (plus_di14[i] - minus_di14[i]) * min(adx14[i] / 25.0, 1.0) - - features = { - "return_5": ret5, - "return_20": ret20, - "close_over_sma_20": close_sma20, - "sma_5_over_sma_20": sma5_sma20, - "signed_log_volume_ratio_20": signed_volume20, - "rsi_14": rsi14, - "return_63": ret63, - "close_over_sma_50": close_sma50, - "sma_20_over_sma_50": sma20_sma50, - "relative_return_63": relative63, - "adx_direction_14": adx_direction, - "return_126": ret126, - "return_252": ret252, - "close_over_sma_200": close_sma200, - "sma_50_over_sma_200": sma50_sma200, - "relative_return_252": relative252, - } - risk_features = { - "ATR14/close": atr_pct, - "20d annualized volatility": vol20, - "252d maximum drawdown": max_drawdown252, - "60d gap p95": gap95_60, - } - percentiles: dict[str, float] = {} - reference_counts: dict[str, int] = {} - for name, series in {**features, **risk_features}.items(): - try: - percentiles[name], reference_counts[name] = percentile_rank(series, index) - except BlockedError as exc: - raise BlockedError(f"{name}: {exc}") from exc - - short_weights = { - "return_5": 0.25, - "return_20": 0.20, - "close_over_sma_20": 0.20, - "sma_5_over_sma_20": 0.15, - "signed_log_volume_ratio_20": 0.15, - "rsi_14": 0.05, - } - medium_weights = { - "return_63": 0.25, - "close_over_sma_50": 0.20, - "sma_20_over_sma_50": 0.20, - "relative_return_63": 0.20, - "adx_direction_14": 0.15, - } - long_weights = { - "return_126": 0.25, - "return_252": 0.25, - "close_over_sma_200": 0.20, - "sma_50_over_sma_200": 0.15, - "relative_return_252": 0.15, - } - risk_weights = { - "ATR14/close": 0.30, - "20d annualized volatility": 0.25, - "252d maximum drawdown": 0.25, - "60d gap p95": 0.20, - } - short_score = weighted_score(percentiles, short_weights) - medium_score = weighted_score(percentiles, medium_weights) - long_score = weighted_score(percentiles, long_weights) - risk_score = weighted_score(percentiles, risk_weights) - total_score = min(100.0, max(0.0, 0.25 * short_score + 0.35 * medium_score + 0.40 * long_score - 0.20 * max(risk_score - 50.0, 0.0))) - - prior_high = max(bar.high for bar in ticker_bars[-21:-1]) - current_close = closes[-1] - current_atr = atr14[-1] - if current_atr is None: - raise BlockedError("ATR14 is unavailable") - entry = round_up_tick(max(current_close, prior_high + tick_size), tick_size) - stop = round_down_tick(entry - 2.0 * current_atr, tick_size) - if stop <= 0: - raise BlockedError("calculated stop is non-positive") - take_profit = round_down_tick(entry + 2.0 * (entry - stop), tick_size) - decimals = decimals_for_tick(tick_size) - - dominant_risk = max(risk_features, key=lambda name: percentiles[name]) - raw_risk = risk_features[dominant_risk][-1] - if raw_risk is None: - raise BlockedError("dominant risk value is unavailable") - risk_value = f"{100.0 * raw_risk:.2f}%" - stop_distance = 100.0 * (entry - stop) / entry - risk_line = f"{dominant_risk} {risk_value} (과거 백분위 {percentiles[dominant_risk]:.1f}), 진입-손절 거리 {stop_distance:.2f}%" - - result: dict[str, Any] = { - "source_skill": "quant-stock-technical", - "result_schema": "quant-stock-technical/v1", - "calculation_status": "READY", - "setup_status": "READY" if total_score >= 60.0 else "CONDITIONAL", - "method_version": METHOD_VERSION, - "analysis_date": ticker_bars[-1].day.isoformat(), - "market": market, - "ticker": ticker, - "source_name": source_name, - "shared_sessions": len(ticker_bars), - "score_basis": "ticker-relative historical percentile; not probability of profit", - "short": {"opinion": opinion(short_score), "score": round(short_score, 2)}, - "medium": {"opinion": opinion(medium_score), "score": round(medium_score, 2)}, - "long": {"opinion": opinion(long_score), "score": round(long_score, 2)}, - "risk": {"score": round(risk_score, 2), "counterpoint": risk_line}, - "entry_price": round(entry, decimals), - "stop_price": round(stop, decimals), - "take_profit_price": round(take_profit, decimals), - "total_score": round(total_score, 2), - "reference_observations": {name: reference_counts[name] for name in sorted(reference_counts)}, - "assumptions": [ - "input rows are finalized completed daily sessions", - "prices are corporate-action adjusted and aligned to analysis-date raw price scale", - "fees, tax, FX, slippage, position size, and execution are excluded", - ], - } - return result - - -def render_markdown(result: dict[str, Any]) -> str: - if result.get("calculation_status") != "READY": - return f"**계산 상태:** BLOCKED\n\n**사유:** {result.get('reason', 'unknown')}" - return "\n".join( - [ - f"## {result['ticker']} quantitative technical analysis", - "", - f"- 분석 기준일: {result['analysis_date']}", - f"- 시장: {result['market']}", - f"- 티커: {result['ticker']}", - f"- 계산 상태: {result['calculation_status']}", - f"- 진입 상태: {result['setup_status']}", - f"- 방법론: {result['method_version']}", - "", - "| 항목 | 의견 | 점수 |", - "|---|---|---:|", - f"| 단기 | {result['short']['opinion']} | {result['short']['score']:.2f} |", - f"| 중기 | {result['medium']['opinion']} | {result['medium']['score']:.2f} |", - f"| 장기 | {result['long']['opinion']} | {result['long']['score']:.2f} |", - f"| 종합 | {opinion(result['total_score'])} | {result['total_score']:.2f} |", - "", - f"- 리스크/반박: {result['risk']['counterpoint']}", - f"- 진입가: {result['entry_price']}", - f"- 손절가: {result['stop_price']}", - f"- 익절가: {result['take_profit_price']}", - f"- 점수 기준: {result['score_basis']}", - f"- 데이터: {result['source_name']} ({result['shared_sessions']} shared sessions)", - ] - ) - - -def synthetic_bars(length: int, drift: float, phase: float) -> list[Bar]: - output: list[Bar] = [] - day = date(2021, 1, 1) - price = 100.0 - index = 0 - while len(output) < length: - if day.weekday() < 5: - cycle = 0.004 * math.sin(index / 17.0 + phase) - change = drift + cycle - previous = price - price = max(1.0, price * (1.0 + change)) - opening = previous * (1.0 + 0.001 * math.sin(index / 9.0 + phase)) - high = max(opening, price) * 1.01 - low = min(opening, price) * 0.99 - volume = 1_000_000.0 * (1.0 + 0.2 * math.sin(index / 13.0 + phase)) - output.append(Bar(day, opening, high, low, price, volume)) - index += 1 - day += timedelta(days=1) - return output - - -def self_test() -> None: - ticker_bars = synthetic_bars(1100, 0.0006, 0.0) - benchmark_bars = synthetic_bars(1100, 0.0003, 1.0) - flat_then_active = [ - Bar( - day=date(2020, 1, 1) + timedelta(days=index), - open=100.0 if index < 20 else 100.0 + index - 20, - high=100.0 if index < 20 else 101.0 + index - 20, - low=100.0 if index < 20 else 99.0 + index - 20, - close=100.0 if index < 20 else 100.5 + index - 20, - volume=0.0 if index < 20 else 1_000.0, - ) - for index in range(80) - ] - _, restarted_adx, _, _ = atr_adx(flat_then_active, 14) - assert restarted_adx[-1] is not None - assert math.isfinite(restarted_adx[-1]) - with tempfile.TemporaryDirectory(prefix="qta-self-test-") as directory: - paths = [Path(directory) / "ticker.csv", Path(directory) / "benchmark.csv"] - for path, bars in zip(paths, (ticker_bars, benchmark_bars)): - with path.open("w", encoding="utf-8", newline="") as handle: - writer = csv.DictWriter(handle, fieldnames=["date", "open", "high", "low", "close", "adjusted_close", "volume"]) - writer.writeheader() - for bar in bars: - writer.writerow( - { - "date": bar.day.isoformat(), - "open": bar.open, - "high": bar.high, - "low": bar.low, - "close": bar.close, - "adjusted_close": bar.close, - "volume": bar.volume, - } - ) - parsed_ticker, parsed_benchmark = align(read_csv(str(paths[0])), read_csv(str(paths[1])), ticker_bars[-1].day) - result = calculate(parsed_ticker, parsed_benchmark, "TEST", "SYNTH", 0.01, "deterministic-self-test") - assert result["calculation_status"] == "READY" - assert result["source_skill"] == "quant-stock-technical" - assert result["result_schema"] == "quant-stock-technical/v1" - assert 0 <= result["total_score"] <= 100 - assert result["stop_price"] < result["entry_price"] < result["take_profit_price"] - print(json.dumps({"self_test": "PASS", "method_version": METHOD_VERSION, "total_score": result["total_score"]}, ensure_ascii=False)) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--ticker-csv") - parser.add_argument("--benchmark-csv") - parser.add_argument("--market") - parser.add_argument("--ticker") - parser.add_argument("--tick-size", type=float) - parser.add_argument("--source-name", default="user-supplied-eod-csv") - parser.add_argument("--analysis-date") - parser.add_argument("--format", choices=("json", "markdown"), default="json") - parser.add_argument("--self-test", action="store_true") - return parser.parse_args() - - -def main() -> int: - args = parse_args() - if args.self_test: - self_test() - return 0 - missing = [name for name in ("ticker_csv", "benchmark_csv", "market", "ticker", "tick_size") if getattr(args, name) is None] - if missing: - result = {"source_skill": "quant-stock-technical", "result_schema": "quant-stock-technical/v1", "calculation_status": "BLOCKED", "reason": "missing arguments: " + ", ".join(missing), "method_version": METHOD_VERSION} - print(json.dumps(result, ensure_ascii=False, indent=2)) - return 2 - try: - requested_date = date.fromisoformat(args.analysis_date) if args.analysis_date else None - ticker_rows, benchmark_rows = align(read_csv(args.ticker_csv), read_csv(args.benchmark_csv), requested_date) - result = calculate(ticker_rows, benchmark_rows, args.market, args.ticker, args.tick_size, args.source_name) - except (BlockedError, OSError, ValueError) as exc: - result = {"source_skill": "quant-stock-technical", "result_schema": "quant-stock-technical/v1", "calculation_status": "BLOCKED", "reason": str(exc), "method_version": METHOD_VERSION} - print(render_markdown(result) if args.format == "markdown" else json.dumps(result, ensure_ascii=False, indent=2)) - return 2 - print(render_markdown(result) if args.format == "markdown" else json.dumps(result, ensure_ascii=False, indent=2)) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/plugins/quant-stock-technical/skills/quant-stock-technical/scripts/analyze_stock_v2.py b/plugins/quant-stock-technical/skills/quant-stock-technical/scripts/analyze_stock_v2.py deleted file mode 100644 index 11c92fb..0000000 --- a/plugins/quant-stock-technical/skills/quant-stock-technical/scripts/analyze_stock_v2.py +++ /dev/null @@ -1,221 +0,0 @@ -#!/usr/bin/env python3 -"""First-hour-aligned QTA 2.0 daily setup score.""" - -from __future__ import annotations - -import argparse -import json -import statistics -import sys -from datetime import date -from pathlib import Path -from typing import Any - -import analyze_stock as qta1 - - -METHOD_VERSION = "qta-2.0.0" -VALIDATION_STATUS = "RESEARCH_ONLY" -LIQUIDITY_FLOOR = { - "KR": 1_000_000_000.0, - "US": 1_000_000.0, -} -MARKET_REGIME_POLICY = { - "metric": "same_session_previous_close_return_bps", - "minimum_change_bps": 0.0, - "max_age_seconds": 90, - "status": "REQUIRED", -} - - -def calculate( - ticker_bars: list[qta1.Bar], - benchmark_bars: list[qta1.Bar], - market: str, - ticker: str, - tick_size: float, - source_name: str, -) -> dict[str, Any]: - """Reuse the causal QTA1 features but score the first-hour horizon.""" - market = market.upper() - if market not in LIQUIDITY_FLOOR: - raise qta1.BlockedError("QTA 2.0 market must be KR or US") - result = qta1.calculate( - ticker_bars, - benchmark_bars, - market, - ticker, - tick_size, - source_name, - ) - short = float(result["short"]["score"]) - medium = float(result["medium"]["score"]) - risk = float(result["risk"]["score"]) - total = min( - 100.0, - max( - 0.0, - 0.50 * short - + 0.30 * medium - + 0.20 * (100.0 - risk), - ), - ) - median_turnover = statistics.median( - bar.close * bar.volume for bar in ticker_bars[-20:] - ) - minimum_turnover = LIQUIDITY_FLOOR[market] - liquidity_ready = median_turnover >= minimum_turnover - result.update( - { - "method_version": METHOD_VERSION, - "validation_status": VALIDATION_STATUS, - "score_basis": ( - "ticker-relative historical percentiles aligned to first-hour " - "continuation; not probability of profit" - ), - "liquidity": { - "median_20_session_turnover": round(median_turnover, 2), - "minimum_turnover": minimum_turnover, - "currency": "KRW" if market == "KR" else "USD", - "status": "READY" if liquidity_ready else "BLOCKED", - }, - "market_regime": dict(MARKET_REGIME_POLICY), - "total_score": round(total, 2), - "setup_status": ( - "READY" - if total >= 60.0 and liquidity_ready - else "CONDITIONAL" - ), - "assumptions": [ - *result["assumptions"], - "QTA 2.0 is research-only until multi-session walk-forward validation", - "same-session benchmark regime admission is required downstream", - ], - } - ) - return result - - -def self_test() -> None: - ticker = qta1.synthetic_bars(1100, 0.0006, 0.0) - benchmark = qta1.synthetic_bars(1100, 0.0003, 1.0) - result = calculate( - ticker, - benchmark, - "US", - "SYNTH", - 0.01, - "deterministic-self-test", - ) - expected = min( - 100.0, - max( - 0.0, - 0.50 * result["short"]["score"] - + 0.30 * result["medium"]["score"] - + 0.20 * (100.0 - result["risk"]["score"]), - ), - ) - assert abs(result["total_score"] - round(expected, 2)) <= 0.01 - assert result["method_version"] == METHOD_VERSION - assert result["validation_status"] == VALIDATION_STATUS - print( - json.dumps( - { - "self_test": "PASS", - "method_version": METHOD_VERSION, - "validation_status": VALIDATION_STATUS, - "total_score": result["total_score"], - }, - sort_keys=True, - ) - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--ticker-csv") - parser.add_argument("--benchmark-csv") - parser.add_argument("--market") - parser.add_argument("--ticker") - parser.add_argument("--tick-size", type=float) - parser.add_argument("--source-name", default="user-supplied-eod-csv") - parser.add_argument("--analysis-date") - parser.add_argument("--self-test", action="store_true") - return parser.parse_args() - - -def blocked(reason: str, market: Any, ticker: Any) -> dict[str, Any]: - return { - "source_skill": "quant-stock-technical", - "result_schema": "quant-stock-technical/v1", - "calculation_status": "BLOCKED", - "reason": reason, - "method_version": METHOD_VERSION, - "market": str(market or "").upper(), - "ticker": str(ticker or "").strip().upper(), - } - - -def main() -> int: - args = parse_args() - if args.self_test: - self_test() - return 0 - missing = [ - name - for name in ( - "ticker_csv", - "benchmark_csv", - "market", - "ticker", - "tick_size", - ) - if getattr(args, name) is None - ] - if missing: - print( - json.dumps( - blocked( - "missing arguments: " + ", ".join(missing), - args.market, - args.ticker, - ), - indent=2, - ) - ) - return 2 - try: - requested = ( - date.fromisoformat(args.analysis_date) - if args.analysis_date - else None - ) - ticker_rows, benchmark_rows = qta1.align( - qta1.read_csv(str(Path(args.ticker_csv))), - qta1.read_csv(str(Path(args.benchmark_csv))), - requested, - ) - result = calculate( - ticker_rows, - benchmark_rows, - args.market, - args.ticker, - args.tick_size, - args.source_name, - ) - except (qta1.BlockedError, OSError, ValueError) as exc: - print( - json.dumps( - blocked(str(exc), args.market, args.ticker), - ensure_ascii=False, - indent=2, - ) - ) - return 2 - print(json.dumps(result, ensure_ascii=False, indent=2)) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/plugins/quant-stock-technical/skills/quant-stock-technical/scripts/build_universe_manifest.py b/plugins/quant-stock-technical/skills/quant-stock-technical/scripts/build_universe_manifest.py deleted file mode 100755 index 7cd7ce7..0000000 --- a/plugins/quant-stock-technical/skills/quant-stock-technical/scripts/build_universe_manifest.py +++ /dev/null @@ -1,2194 +0,0 @@ -#!/usr/bin/env python3 -"""Build a deterministic market-scoped qta-universe-manifest/v2 offline.""" - -# Python 3.9 compatibility intentionally uses Optional instead of PEP 604 unions. -# ruff: noqa: UP045 - -from __future__ import annotations - -import argparse -import csv -import hashlib -import json -import re -import sys -import tempfile -from collections.abc import Iterable, Mapping, Sequence -from dataclasses import dataclass -from datetime import date -from decimal import Decimal, InvalidOperation -from html.parser import HTMLParser -from pathlib import Path -from typing import Any, Optional - -MANIFEST_SCHEMA = "qta-universe-manifest/v2" -BUILD_SPEC_SCHEMA = "qta-universe-build-spec/v1" -TICK_CONTRACT_SCHEMA = "qta-tick-contract/v1" -TICK_CONTRACT_KIND = "RESOLVED_PRICE_LADDER" -CATALOG_COVERAGE_SCHEMA = "qta-catalog-coverage-contract/v1" - -EXCHANGES = ("KOSPI", "KOSDAQ", "NYSE", "NASDAQ") -SUPPORTED_EXCHANGE_SCOPES = { - frozenset(("KOSPI", "KOSDAQ")), - frozenset(("NYSE", "NASDAQ")), - frozenset(EXCHANGES), -} -EXCHANGE_CONTRACTS: dict[str, dict[str, str]] = { - "KOSPI": { - "market": "KR", - "benchmark_id": "KOSPI_COMPOSITE", - "currency": "KRW", - "venue": "KRX", - }, - "KOSDAQ": { - "market": "KR", - "benchmark_id": "KOSDAQ_COMPOSITE", - "currency": "KRW", - "venue": "KRX", - }, - "NYSE": { - "market": "US", - "benchmark_id": "NYSE_COMPOSITE", - "currency": "USD", - "venue": "NYSE", - }, - "NASDAQ": { - "market": "US", - "benchmark_id": "NASDAQ_COMPOSITE", - "currency": "USD", - "venue": "NASD", - }, -} - -INCLUDED_INSTRUMENT_TYPES = {"COMMON", "ADR", "REIT"} -EXCLUDED_CATALOG_TYPES = { - "ETF": "instrument_etf", - "ETN": "instrument_etn", - "PREFERRED": "instrument_preferred", - "SPAC": "instrument_spac", - "UNIT": "instrument_unit", - "RIGHT": "instrument_right", - "WARRANT": "instrument_warrant", - "TEST": "test_issue", - "ABNORMAL": "abnormal_status", -} -CATALOG_TYPES = INCLUDED_INSTRUMENT_TYPES | set(EXCLUDED_CATALOG_TYPES) - -BUILD_SPEC_FIELDS = { - "schema", - "as_of", - "analysis_date", - "official_sources", - "broker_sources", - "eod_catalog", - "catalog_coverage_contract", -} -CATALOG_COVERAGE_FIELDS = { - "schema", - "minimum_ratio_by_exchange", - "minimum_screenable_ratio_by_exchange", -} -SOURCE_FIELDS = { - "source_id", - "provider", - "exchange", - "as_of", - "path", - "format", - "encoding", - "delimiter", - "skip_rows", - "columns", - "normal_status_values", -} -EOD_CATALOG_SPEC_FIELDS = { - "source_id", - "provider", - "as_of", - "path", - "encoding", -} -EOD_CATALOG_FIELDS = ( - "exchange", - "canonical_symbol", - "data_symbol", - "broker_symbol", - "instrument_type", - "benchmark_id", - "ticker_csv", - "benchmark_csv", - "tick_rule_id", - "tick_effective_date", - "tick_reference_price", - "resolved_tick_size", - "source_name", -) -LOGICAL_COLUMNS = { - "symbol", - "name", - "instrument_type", - "status", - "etf", - "etn", - "preferred", - "spac", - "unit", - "right", - "warrant", - "test_issue", - "abnormal", - "etp", - "trading_halt", - "liquidation", - "administrative", -} -FORMATS = { - "KRX_CSV", - "KRX_KIND_HTML", - "KIS_CSV", - "KIS_FIXED_WIDTH", - "KIS_KRX_MASTER", - "KIS_OVERSEAS_MASTER", - "NASDAQ_LISTED", - "NASDAQ_OTHER", -} -TRUE_VALUES = {"1", "T", "TRUE", "Y", "YES"} -FALSE_VALUES = {"", "0", "F", "FALSE", "N", "NO"} -HEX64 = re.compile(r"^[0-9a-f]{64}$") - -# KIS official stocks_info master layouts. The downloaded domestic rows contain -# a variable-width Korean name prefix followed by these fixed ASCII tails. -KIS_KOSPI_WIDTHS = ( - 2, - 1, - 4, - 4, - 4, - *([1] * 26), - 9, - 5, - 5, - 1, - 1, - 1, - 2, - 1, - 1, - 1, - 2, - 2, - 2, - 3, - 1, - 3, - 12, - 12, - 8, - 15, - 21, - 2, - 7, - 1, - 1, - 1, - 1, - 1, - 9, - 9, - 9, - 5, - 9, - 8, - 9, - 3, - 1, - 1, - 1, -) -KIS_KOSDAQ_WIDTHS = ( - 2, - 1, - 4, - 4, - 4, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 9, - 5, - 5, - 1, - 1, - 1, - 2, - 1, - 1, - 1, - 2, - 2, - 2, - 3, - 1, - 3, - 12, - 12, - 8, - 15, - 21, - 2, - 7, - 1, - 1, - 1, - 1, - 9, - 9, - 9, - 5, - 9, - 8, - 9, - 3, - 1, - 1, - 1, -) -KIS_KRX_LAYOUTS = { - "KOSPI": { - "tail_length": 227, - "widths": KIS_KOSPI_WIDTHS, - "etp_index": 12, - "spac_index": 19, - "halt_index": 34, - "liquidation_index": 35, - "admin_index": 36, - "preferred_index": 54, - }, - "KOSDAQ": { - "tail_length": 221, - "widths": KIS_KOSDAQ_WIDTHS, - "etp_index": 8, - "spac_index": 14, - "halt_index": 29, - "liquidation_index": 30, - "admin_index": 31, - "preferred_index": 49, - }, -} - - -class UniverseBlockedError(ValueError): - """Raised when the frozen universe build contract is incomplete or invalid.""" - - -@dataclass(frozen=True) -class MasterRow: - exchange: str - symbol: str - name: str - instrument_type: str - status: str - flags: Mapping[str, str] - source_id: str - - -@dataclass(frozen=True) -class CatalogRow: - exchange: str - canonical_symbol: str - data_symbol: str - broker_symbol: str - instrument_type: str - benchmark_id: str - ticker_csv: str - benchmark_csv: str - tick_rule_id: str - tick_effective_date: str - tick_reference_price: str - resolved_tick_size: str - source_name: str - - -def canonical_json(value: Any) -> str: - return json.dumps( - value, - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - allow_nan=False, - ) - - -def sha256_json(value: Any) -> str: - return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest() - - -def sha256_file(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - while True: - chunk = handle.read(1024 * 1024) - if not chunk: - break - digest.update(chunk) - return digest.hexdigest() - - -def load_json_object(path: Path) -> dict[str, Any]: - value = json.loads(path.read_text(encoding="utf-8")) - if not isinstance(value, dict): - raise UniverseBlockedError(f"{path} must contain one JSON object") - return value - - -def require_exact_fields( - value: Mapping[str, Any], expected: Iterable[str], label: str -) -> None: - expected_set = set(expected) - actual = set(value) - if actual != expected_set: - missing = sorted(expected_set - actual) - extra = sorted(actual - expected_set) - raise UniverseBlockedError( - f"{label} fields mismatch; missing={missing}, extra={extra}" - ) - - -def require_iso_date(value: Any, label: str) -> str: - if not isinstance(value, str): - raise UniverseBlockedError(f"{label} must be a YYYY-MM-DD string") - rendered = value - try: - date.fromisoformat(rendered) - except ValueError as exc: - raise UniverseBlockedError(f"{label} must be YYYY-MM-DD") from exc - return rendered - - -def require_nonempty(value: Any, label: str) -> str: - if not isinstance(value, str): - raise UniverseBlockedError(f"{label} must be a non-empty string") - rendered = value.strip() - if not rendered: - raise UniverseBlockedError(f"{label} must be a non-empty string") - return rendered - - -def normalized_symbol(value: Any, label: str) -> str: - return require_nonempty(value, label).upper() - - -def decimal_string(value: Any, label: str, positive: bool = False) -> str: - if isinstance(value, bool): - raise UniverseBlockedError(f"{label} must be a decimal string or number") - try: - parsed = Decimal(str(value)) - except (InvalidOperation, ValueError) as exc: - raise UniverseBlockedError( - f"{label} must be a decimal string or number" - ) from exc - if not parsed.is_finite(): - raise UniverseBlockedError(f"{label} must be finite") - if positive and parsed <= 0: - raise UniverseBlockedError(f"{label} must be positive") - return format(parsed, "f") - - -def canonical_ratio_string(value: Any, label: str) -> str: - if isinstance(value, bool): - raise UniverseBlockedError(f"{label} must be a decimal string or number") - try: - parsed = Decimal(str(value)) - except (InvalidOperation, ValueError) as exc: - raise UniverseBlockedError( - f"{label} must be a decimal string or number" - ) from exc - if not parsed.is_finite() or parsed <= 0 or parsed > 1: - raise UniverseBlockedError(f"{label} must be greater than 0 and at most 1") - rendered = format(parsed.normalize(), "f") - return rendered.rstrip("0").rstrip(".") if "." in rendered else rendered - - -def meets_minimum_coverage_ratio( - numerator: int, - official_rows: int, - minimum: str, -) -> bool: - if official_rows <= 0: - return False - return Decimal(numerator) / Decimal(official_rows) >= Decimal(minimum) - - -def normalize_catalog_coverage_contract(raw: Any) -> dict[str, Any]: - label = "catalog_coverage_contract" - if not isinstance(raw, dict): - raise UniverseBlockedError(f"{label} must be an object") - require_exact_fields(raw, CATALOG_COVERAGE_FIELDS, label) - if raw["schema"] != CATALOG_COVERAGE_SCHEMA: - raise UniverseBlockedError(f"{label}.schema must be {CATALOG_COVERAGE_SCHEMA}") - normalized_ratios: dict[str, dict[str, str]] = {} - for field in ( - "minimum_ratio_by_exchange", - "minimum_screenable_ratio_by_exchange", - ): - ratios = raw[field] - if not isinstance(ratios, dict) or set(ratios) != set(EXCHANGES): - raise UniverseBlockedError( - f"{label}.{field} must contain exactly {list(EXCHANGES)}" - ) - normalized_ratios[field] = { - exchange: canonical_ratio_string( - ratios[exchange], - f"{label}.{field}.{exchange}", - ) - for exchange in EXCHANGES - } - return { - "schema": CATALOG_COVERAGE_SCHEMA, - **normalized_ratios, - } - - -def resolve_input_path(raw: Any, base_directory: Path, label: str) -> Path: - rendered = require_nonempty(raw, label) - path = Path(rendered) - if not path.is_absolute(): - path = base_directory / path - path = path.resolve() - if not path.is_file(): - raise UniverseBlockedError(f"{label} does not exist or is not a file: {path}") - return path - - -def normalize_source_descriptor( - raw: Any, - role: str, - build_as_of: str, - spec_directory: Path, - index: int, -) -> dict[str, Any]: - label = f"{role.lower()}_sources[{index}]" - if not isinstance(raw, dict): - raise UniverseBlockedError(f"{label} must be an object") - require_exact_fields(raw, SOURCE_FIELDS, label) - - source_id = require_nonempty(raw["source_id"], f"{label}.source_id") - provider = require_nonempty(raw["provider"], f"{label}.provider").upper() - exchange = str(raw["exchange"]).upper() - if exchange not in EXCHANGES: - raise UniverseBlockedError(f"{label}.exchange must be one of {EXCHANGES}") - source_as_of = require_iso_date(raw["as_of"], f"{label}.as_of") - if source_as_of != build_as_of: - raise UniverseBlockedError( - f"{label}.as_of {source_as_of} does not match build as_of {build_as_of}" - ) - source_format = str(raw["format"]).upper() - if source_format not in FORMATS: - raise UniverseBlockedError(f"{label}.format must be one of {sorted(FORMATS)}") - encoding = require_nonempty(raw["encoding"], f"{label}.encoding") - delimiter = str(raw["delimiter"]) - if len(delimiter) > 1: - raise UniverseBlockedError(f"{label}.delimiter must have at most one character") - skip_rows = raw["skip_rows"] - if isinstance(skip_rows, bool) or not isinstance(skip_rows, int) or skip_rows < 0: - raise UniverseBlockedError(f"{label}.skip_rows must be an integer >= 0") - columns = raw["columns"] - if not isinstance(columns, dict): - raise UniverseBlockedError(f"{label}.columns must be an object") - unknown_columns = set(columns) - LOGICAL_COLUMNS - if unknown_columns: - raise UniverseBlockedError( - f"{label}.columns has unknown logical fields: {sorted(unknown_columns)}" - ) - statuses = raw["normal_status_values"] - if not isinstance(statuses, list) or any( - not isinstance(item, str) for item in statuses - ): - raise UniverseBlockedError( - f"{label}.normal_status_values must be a string array" - ) - normal_status_values = sorted({item.strip().upper() for item in statuses}) - - if role == "OFFICIAL_MASTER": - expected_provider = ( - "KRX" if exchange in {"KOSPI", "KOSDAQ"} else "NASDAQ_TRADER" - ) - if provider != expected_provider: - raise UniverseBlockedError( - f"{label}.provider must be {expected_provider} for {exchange}" - ) - expected_formats = ( - {"KRX_CSV", "KRX_KIND_HTML"} - if exchange in {"KOSPI", "KOSDAQ"} - else {"NASDAQ_LISTED" if exchange == "NASDAQ" else "NASDAQ_OTHER"} - ) - if source_format not in expected_formats: - raise UniverseBlockedError( - f"{label}.format {source_format} is invalid for {exchange}" - ) - elif role == "BROKER_MASTER": - if provider != "KIS": - raise UniverseBlockedError(f"{label}.provider must be KIS") - if source_format not in { - "KIS_CSV", - "KIS_FIXED_WIDTH", - "KIS_KRX_MASTER", - "KIS_OVERSEAS_MASTER", - }: - raise UniverseBlockedError( - f"{label}.format is not a supported KIS master format" - ) - if source_format == "KIS_KRX_MASTER" and exchange not in {"KOSPI", "KOSDAQ"}: - raise UniverseBlockedError( - f"{label}.format KIS_KRX_MASTER is domestic only" - ) - if source_format == "KIS_OVERSEAS_MASTER" and exchange not in { - "NYSE", - "NASDAQ", - }: - raise UniverseBlockedError( - f"{label}.format KIS_OVERSEAS_MASTER is U.S. only" - ) - else: - raise UniverseBlockedError(f"unsupported source role: {role}") - - if source_format in { - "NASDAQ_LISTED", - "NASDAQ_OTHER", - "KRX_KIND_HTML", - "KIS_KRX_MASTER", - "KIS_OVERSEAS_MASTER", - }: - if columns: - raise UniverseBlockedError( - f"{label}.columns must be empty for built-in master formats" - ) - else: - if "symbol" not in columns: - raise UniverseBlockedError(f"{label}.columns.symbol is required") - if source_format == "KIS_FIXED_WIDTH": - for logical, bounds in columns.items(): - if ( - not isinstance(bounds, list) - or len(bounds) != 2 - or any( - isinstance(item, bool) or not isinstance(item, int) - for item in bounds - ) - or bounds[0] < 0 - or bounds[1] <= bounds[0] - ): - raise UniverseBlockedError( - f"{label}.columns.{logical} must be [start,end] " - "zero-based half-open integers" - ) - else: - for logical, header in columns.items(): - require_nonempty(header, f"{label}.columns.{logical}") - - status_is_available = source_format == "NASDAQ_LISTED" or "status" in columns - if status_is_available and not normal_status_values: - raise UniverseBlockedError( - f"{label}.normal_status_values cannot be empty when status is mapped" - ) - if not status_is_available and normal_status_values: - raise UniverseBlockedError( - f"{label}.normal_status_values must be empty without a status field" - ) - - path = resolve_input_path(raw["path"], spec_directory, f"{label}.path") - return { - "source_id": source_id, - "provider": provider, - "exchange": exchange, - "as_of": source_as_of, - "path": str(path), - "format": source_format, - "encoding": encoding, - "delimiter": delimiter, - "skip_rows": skip_rows, - "columns": columns, - "normal_status_values": normal_status_values, - "role": role, - } - - -def normalize_build_spec(raw: dict[str, Any], spec_directory: Path) -> dict[str, Any]: - require_exact_fields(raw, BUILD_SPEC_FIELDS, "build spec") - if raw["schema"] != BUILD_SPEC_SCHEMA: - raise UniverseBlockedError(f"unsupported build spec schema: {raw['schema']!r}") - as_of = require_iso_date(raw["as_of"], "build spec.as_of") - analysis_date = require_iso_date(raw["analysis_date"], "build spec.analysis_date") - if date.fromisoformat(analysis_date) > date.fromisoformat(as_of): - raise UniverseBlockedError("analysis_date cannot be later than as_of") - - official_raw = raw["official_sources"] - broker_raw = raw["broker_sources"] - if not isinstance(official_raw, list) or not isinstance(broker_raw, list): - raise UniverseBlockedError("official_sources and broker_sources must be arrays") - official = [ - normalize_source_descriptor( - item, "OFFICIAL_MASTER", as_of, spec_directory, index - ) - for index, item in enumerate(official_raw) - ] - broker = [ - normalize_source_descriptor(item, "BROKER_MASTER", as_of, spec_directory, index) - for index, item in enumerate(broker_raw) - ] - - official_by_exchange = {item["exchange"]: item for item in official} - if len(official_by_exchange) != len(official): - raise UniverseBlockedError( - "official_sources may contain only one source per exchange" - ) - exchange_scope = frozenset(official_by_exchange) - if exchange_scope not in SUPPORTED_EXCHANGE_SCOPES: - raise UniverseBlockedError( - "official_sources must contain exactly KOSPI/KOSDAQ, " - "NYSE/NASDAQ, or all four exchanges" - ) - broker_by_exchange = {item["exchange"]: item for item in broker} - if len(broker_by_exchange) != len(broker): - raise UniverseBlockedError( - "broker_sources may contain only one source per exchange" - ) - if frozenset(broker_by_exchange) != exchange_scope: - raise UniverseBlockedError( - "broker_sources must cover the same exchanges as official_sources" - ) - exchanges = tuple(exchange for exchange in EXCHANGES if exchange in exchange_scope) - - source_ids = [item["source_id"] for item in official + broker] - if len(source_ids) != len(set(source_ids)): - raise UniverseBlockedError("master source_id values must be unique") - - catalog_raw = raw["eod_catalog"] - if not isinstance(catalog_raw, dict): - raise UniverseBlockedError("eod_catalog must be an object") - require_exact_fields(catalog_raw, EOD_CATALOG_SPEC_FIELDS, "eod_catalog") - catalog_as_of = require_iso_date(catalog_raw["as_of"], "eod_catalog.as_of") - if catalog_as_of != as_of: - raise UniverseBlockedError( - f"eod_catalog.as_of {catalog_as_of} does not match build as_of {as_of}" - ) - catalog = { - "source_id": require_nonempty( - catalog_raw["source_id"], "eod_catalog.source_id" - ), - "provider": require_nonempty(catalog_raw["provider"], "eod_catalog.provider"), - "as_of": catalog_as_of, - "path": str( - resolve_input_path(catalog_raw["path"], spec_directory, "eod_catalog.path") - ), - "encoding": require_nonempty(catalog_raw["encoding"], "eod_catalog.encoding"), - } - if catalog["source_id"] in source_ids: - raise UniverseBlockedError("eod_catalog.source_id must be unique") - coverage_contract = normalize_catalog_coverage_contract( - raw["catalog_coverage_contract"] - ) - - return { - "schema": BUILD_SPEC_SCHEMA, - "as_of": as_of, - "analysis_date": analysis_date, - "official_sources": sorted( - official, key=lambda item: EXCHANGES.index(item["exchange"]) - ), - "broker_sources": sorted( - broker, key=lambda item: EXCHANGES.index(item["exchange"]) - ), - "eod_catalog": catalog, - "catalog_coverage_contract": coverage_contract, - "exchanges": exchanges, - } - - -def normalize_delimited_row( - raw: Mapping[str, Any], columns: Mapping[str, Any] -) -> dict[str, str]: - output: dict[str, str] = {} - for logical, header in columns.items(): - value = raw.get(str(header)) - output[logical] = "" if value is None else str(value).strip() - return output - - -def normalize_fixed_width_row( - line: bytes, columns: Mapping[str, Any], encoding: str -) -> dict[str, str]: - return { - logical: line[bounds[0] : bounds[1]].decode(encoding).strip() - for logical, bounds in columns.items() - } - - -def row_from_fields( - fields: Mapping[str, str], descriptor: Mapping[str, Any], row_number: int -) -> MasterRow: - label = f"{descriptor['source_id']} row {row_number}" - symbol = normalized_symbol(fields.get("symbol", ""), f"{label}.symbol") - return MasterRow( - exchange=str(descriptor["exchange"]), - symbol=symbol, - name=str(fields.get("name", "")).strip(), - instrument_type=str(fields.get("instrument_type", "")).strip(), - status=str(fields.get("status", "")).strip(), - flags={ - logical: str(fields.get(logical, "")).strip() - for logical in ( - "etf", - "etn", - "preferred", - "spac", - "unit", - "right", - "warrant", - "test_issue", - "abnormal", - "etp", - "trading_halt", - "liquidation", - "administrative", - ) - if logical in fields - }, - source_id=str(descriptor["source_id"]), - ) - - -def parse_csv_source(descriptor: Mapping[str, Any]) -> list[MasterRow]: - path = Path(str(descriptor["path"])) - delimiter = str(descriptor["delimiter"]) - if len(delimiter) != 1: - raise UniverseBlockedError( - f"{descriptor['source_id']}.delimiter must be exactly one character" - ) - with path.open("r", encoding=str(descriptor["encoding"]), newline="") as handle: - for _ in range(int(descriptor["skip_rows"])): - next(handle, None) - reader = csv.DictReader(handle, delimiter=delimiter) - if reader.fieldnames is None: - raise UniverseBlockedError(f"{descriptor['source_id']} has no header row") - required_headers = {str(header) for header in descriptor["columns"].values()} - missing_headers = required_headers - set(reader.fieldnames) - if missing_headers: - raise UniverseBlockedError( - f"{descriptor['source_id']} missing headers: {sorted(missing_headers)}" - ) - rows: list[MasterRow] = [] - for row_number, raw in enumerate( - reader, start=2 + int(descriptor["skip_rows"]) - ): - fields = normalize_delimited_row(raw, descriptor["columns"]) - if not any(fields.values()): - continue - rows.append(row_from_fields(fields, descriptor, row_number)) - return rows - - -def parse_fixed_width_source(descriptor: Mapping[str, Any]) -> list[MasterRow]: - path = Path(str(descriptor["path"])) - lines = path.read_bytes().splitlines() - rows: list[MasterRow] = [] - skip_rows = int(descriptor["skip_rows"]) - for row_number, line in enumerate(lines[skip_rows:], start=skip_rows + 1): - if not line.strip(): - continue - fields = normalize_fixed_width_row( - line, - descriptor["columns"], - str(descriptor["encoding"]), - ) - rows.append(row_from_fields(fields, descriptor, row_number)) - return rows - - -def split_fixed_fields(value: str, widths: Sequence[int]) -> list[str]: - output: list[str] = [] - offset = 0 - for width in widths: - output.append(value[offset : offset + width].strip()) - offset += width - if len(value) != offset: - raise UniverseBlockedError( - f"fixed-width tail has {len(value)} characters; expected exactly {offset}" - ) - return output - - -def parse_kis_krx_master(descriptor: Mapping[str, Any]) -> list[MasterRow]: - exchange = str(descriptor["exchange"]) - layout = KIS_KRX_LAYOUTS[exchange] - path = Path(str(descriptor["path"])) - lines = path.read_text(encoding=str(descriptor["encoding"])).splitlines() - rows: list[MasterRow] = [] - skip_rows = int(descriptor["skip_rows"]) - for row_number, line in enumerate(lines[skip_rows:], start=skip_rows + 1): - if not line.strip(): - continue - tail_length = int(layout["tail_length"]) - if len(line) <= tail_length + 21: - raise UniverseBlockedError( - f"{descriptor['source_id']} row {row_number} is shorter than the " - "KIS domestic master layout" - ) - prefix = line[:-tail_length] - tail = line[-tail_length:] - symbol = prefix[0:9].rstrip() - standard_symbol = prefix[9:21].rstrip() - name = prefix[21:].strip() - if not symbol or not standard_symbol or not name: - raise UniverseBlockedError( - f"{descriptor['source_id']} row {row_number} has an incomplete " - "KIS domestic prefix" - ) - parts = split_fixed_fields(tail, layout["widths"]) - group_code = parts[0].upper() - etp_code = parts[int(layout["etp_index"])].upper() - flags = { - "etf": "Y" if etp_code in {"1", "2", "5"} else "N", - "etn": "Y" if etp_code in {"3", "4"} else "N", - "etp": ("Y" if etp_code not in {"", "0", "1", "2", "3", "4", "5"} else "N"), - "spac": parts[int(layout["spac_index"])], - "preferred": ( - "Y" if parts[int(layout["preferred_index"])] not in {"", "0"} else "N" - ), - "trading_halt": parts[int(layout["halt_index"])], - "liquidation": parts[int(layout["liquidation_index"])], - "administrative": parts[int(layout["admin_index"])], - "warrant": "Y" if group_code in {"EW", "SW"} else "N", - "right": "Y" if group_code == "SR" else "N", - } - rows.append( - row_from_fields( - { - "symbol": symbol, - "name": name, - "instrument_type": group_code, - **flags, - }, - descriptor, - row_number, - ) - ) - return rows - - -def parse_kis_overseas_master( - descriptor: Mapping[str, Any], -) -> list[MasterRow]: - path = Path(str(descriptor["path"])) - exchange = str(descriptor["exchange"]) - expected_exchange_code = "NAS" if exchange == "NASDAQ" else "NYS" - lines = path.read_bytes().splitlines() - rows: list[MasterRow] = [] - skip_rows = int(descriptor["skip_rows"]) - for row_number, raw_line in enumerate(lines[skip_rows:], start=skip_rows + 1): - if not raw_line.strip(): - continue - fields = raw_line.decode(str(descriptor["encoding"])).split("\t") - if len(fields) != 24: - raise UniverseBlockedError( - f"{descriptor['source_id']} row {row_number} has {len(fields)} " - "columns; expected 24" - ) - if fields[2].strip().upper() != expected_exchange_code: - raise UniverseBlockedError( - f"{descriptor['source_id']} row {row_number} exchange code " - f"{fields[2]!r} does not match {expected_exchange_code}" - ) - symbol = fields[4].strip() - korean_name = fields[6].strip() - english_name = fields[7].strip() - security_type = fields[8].strip() - dr_flag = fields[17].strip().upper() - classification = fields[22].strip() - if dr_flag not in {"Y", "N"}: - raise UniverseBlockedError( - f"{descriptor['source_id']} row {row_number} has invalid DR flag " - f"{dr_flag!r}" - ) - is_etp = security_type == "3" - flags = { - "etf": ("Y" if is_etp and classification in {"001", "005"} else "N"), - "etn": ("Y" if is_etp and classification in {"002", "006"} else "N"), - "etp": ( - "Y" - if is_etp and classification not in {"001", "002", "005", "006"} - else "N" - ), - "warrant": "Y" if security_type == "4" else "N", - } - if security_type == "1": - normalized_type = "INDEX" - elif security_type == "2" and dr_flag == "Y": - normalized_type = "DR" - elif security_type == "2": - normalized_type = "STOCK" - elif security_type == "3": - normalized_type = "ETP" - elif security_type == "4": - normalized_type = "WARRANT" - else: - normalized_type = f"UNKNOWN_SECURITY_TYPE_{security_type or 'BLANK'}" - rows.append( - row_from_fields( - { - "symbol": symbol, - "name": english_name or korean_name, - "instrument_type": normalized_type, - **flags, - }, - descriptor, - row_number, - ) - ) - return rows - - -class KindTableParser(HTMLParser): - """Collect text cells from the offline KIND HTML-table Excel export.""" - - def __init__(self) -> None: - super().__init__(convert_charrefs=True) - self.rows: list[list[str]] = [] - self._row: Optional[list[str]] = None - self._cell_parts: Optional[list[str]] = None - - def handle_starttag(self, tag: str, attrs: list[tuple[str, Optional[str]]]) -> None: - del attrs - lowered = tag.lower() - if lowered == "tr": - self._row = [] - elif lowered in {"th", "td"} and self._row is not None: - self._cell_parts = [] - - def handle_data(self, data: str) -> None: - if self._cell_parts is not None: - self._cell_parts.append(data) - - def handle_endtag(self, tag: str) -> None: - lowered = tag.lower() - if lowered in {"th", "td"} and self._cell_parts is not None: - if self._row is not None: - self._row.append(" ".join("".join(self._cell_parts).split())) - self._cell_parts = None - elif lowered == "tr" and self._row is not None: - if self._row: - self.rows.append(self._row) - self._row = None - self._cell_parts = None - - -def parse_kind_html_source(descriptor: Mapping[str, Any]) -> list[MasterRow]: - path = Path(str(descriptor["path"])) - parser = KindTableParser() - parser.feed(path.read_text(encoding=str(descriptor["encoding"]))) - required_headers = {"회사명", "시장구분", "종목코드"} - header_index: Optional[int] = None - headers: list[str] = [] - for index, cells in enumerate(parser.rows): - if required_headers.issubset(set(cells)): - header_index = index - headers = cells - break - if header_index is None: - raise UniverseBlockedError( - f"{descriptor['source_id']} KIND table is missing headers " - f"{sorted(required_headers)}" - ) - if len(headers) != len(set(headers)): - raise UniverseBlockedError( - f"{descriptor['source_id']} KIND table has duplicate headers" - ) - market_values = { - "KOSPI": {"유가", "유가증권"}, - "KOSDAQ": {"코스닥"}, - } - exchange = str(descriptor["exchange"]) - expected_values = market_values[exchange] - rows: list[MasterRow] = [] - for row_number, cells in enumerate( - parser.rows[header_index + 1 :], start=header_index + 2 - ): - if len(cells) != len(headers): - raise UniverseBlockedError( - f"{descriptor['source_id']} KIND row {row_number} has " - f"{len(cells)} cells; expected {len(headers)}" - ) - raw = dict(zip(headers, cells)) - if raw["시장구분"].strip() not in expected_values: - continue - fields = { - "symbol": raw["종목코드"].strip(), - "name": raw["회사명"].strip(), - } - rows.append(row_from_fields(fields, descriptor, row_number)) - return rows - - -def parse_nasdaq_source(descriptor: Mapping[str, Any]) -> list[MasterRow]: - path = Path(str(descriptor["path"])) - source_format = str(descriptor["format"]) - with path.open("r", encoding=str(descriptor["encoding"]), newline="") as handle: - reader = csv.DictReader(handle, delimiter="|") - if reader.fieldnames is None: - raise UniverseBlockedError( - f"{descriptor['source_id']} has no Nasdaq header" - ) - if source_format == "NASDAQ_LISTED": - required = { - "Symbol", - "Security Name", - "ETF", - "Test Issue", - "Financial Status", - } - else: - required = { - "ACT Symbol", - "Security Name", - "Exchange", - "ETF", - "Test Issue", - } - missing = required - set(reader.fieldnames) - if missing: - raise UniverseBlockedError( - f"{descriptor['source_id']} missing Nasdaq headers: {sorted(missing)}" - ) - rows: list[MasterRow] = [] - for row_number, raw in enumerate(reader, start=2): - first_value = next(iter(raw.values()), "") - if str(first_value).startswith("File Creation Time:"): - continue - if ( - source_format == "NASDAQ_OTHER" - and str(raw.get("Exchange", "")).strip().upper() != "N" - ): - continue - symbol_header = ( - "Symbol" if source_format == "NASDAQ_LISTED" else "ACT Symbol" - ) - fields = { - "symbol": str(raw.get(symbol_header, "") or "").strip(), - "name": str(raw.get("Security Name", "") or "").strip(), - "etf": str(raw.get("ETF", "") or "").strip(), - "test_issue": str(raw.get("Test Issue", "") or "").strip(), - } - if source_format == "NASDAQ_LISTED": - fields["status"] = str(raw.get("Financial Status", "") or "").strip() - if not fields["symbol"]: - raise UniverseBlockedError( - f"{descriptor['source_id']} row {row_number} has no symbol" - ) - rows.append(row_from_fields(fields, descriptor, row_number)) - return rows - - -def parse_master_source(descriptor: Mapping[str, Any]) -> list[MasterRow]: - source_format = str(descriptor["format"]) - if source_format in {"KRX_CSV", "KIS_CSV"}: - rows = parse_csv_source(descriptor) - elif source_format == "KIS_FIXED_WIDTH": - rows = parse_fixed_width_source(descriptor) - elif source_format == "KIS_KRX_MASTER": - rows = parse_kis_krx_master(descriptor) - elif source_format == "KIS_OVERSEAS_MASTER": - rows = parse_kis_overseas_master(descriptor) - elif source_format == "KRX_KIND_HTML": - rows = parse_kind_html_source(descriptor) - else: - rows = parse_nasdaq_source(descriptor) - if not rows: - raise UniverseBlockedError(f"{descriptor['source_id']} contains no rows") - by_symbol: dict[str, MasterRow] = {} - for row in rows: - if row.symbol in by_symbol: - if by_symbol[row.symbol] == row: - continue - raise UniverseBlockedError( - f"{descriptor['source_id']} contains conflicting duplicate " - f"symbol {row.symbol}" - ) - by_symbol[row.symbol] = row - return [by_symbol[symbol] for symbol in sorted(by_symbol)] - - -def parse_bool_flag(value: str, label: str) -> bool: - normalized = value.strip().upper() - if normalized in TRUE_VALUES: - return True - if normalized in FALSE_VALUES: - return False - raise UniverseBlockedError( - f"{label} must be one of {sorted(TRUE_VALUES | FALSE_VALUES)}" - ) - - -def metadata_exclusion_reasons( - row: MasterRow, descriptor: Mapping[str, Any] -) -> list[str]: - reasons: list[str] = [] - flag_reasons = { - "etf": "instrument_etf", - "etn": "instrument_etn", - "preferred": "instrument_preferred", - "spac": "instrument_spac", - "unit": "instrument_unit", - "right": "instrument_right", - "warrant": "instrument_warrant", - "test_issue": "test_issue", - "abnormal": "abnormal_status", - "etp": "instrument_etp", - "trading_halt": "trading_halt", - "liquidation": "liquidation", - "administrative": "administrative_issue", - } - for flag, reason in flag_reasons.items(): - if flag in row.flags and parse_bool_flag( - row.flags[flag], f"{row.source_id}:{row.symbol}.{flag}" - ): - reasons.append(reason) - - normal_statuses = set(descriptor["normal_status_values"]) - if normal_statuses and row.status.strip().upper() not in normal_statuses: - reasons.append("abnormal_status") - - raw_type = row.instrument_type.strip().upper() - raw_type_reasons = { - "EF": "instrument_etf", - "FE": "instrument_etf", - "EN": "instrument_etn", - "PF": "instrument_preferred", - "EW": "instrument_warrant", - "SW": "instrument_warrant", - "SR": "instrument_right", - "ETF": "instrument_etf", - "ETN": "instrument_etn", - "WARRANT": "instrument_warrant", - "INDEX": "unsupported_instrument_type", - "MF": "unsupported_instrument_type", - "SC": "unsupported_instrument_type", - "IF": "unsupported_instrument_type", - "BC": "unsupported_instrument_type", - } - if raw_type in raw_type_reasons: - reasons.append(raw_type_reasons[raw_type]) - elif raw_type == "ETP" and not { - "instrument_etf", - "instrument_etn", - }.intersection(reasons): - reasons.append("instrument_etp") - elif raw_type.startswith("UNKNOWN_SECURITY_TYPE_"): - reasons.append("unsupported_instrument_type") - - searchable = f"{row.instrument_type} {row.name}".upper() - token_patterns = ( - (r"\bETF\b|상장지수펀드", "instrument_etf"), - (r"\bETN\b|상장지수증권", "instrument_etn"), - (r"\bPREFERRED\b|우선주", "instrument_preferred"), - (r"\bSPAC\b|\bACQUISITION CORP(?:ORATION)?\b|기업인수목적", "instrument_spac"), - (r"\bUNITS?\b", "instrument_unit"), - (r"\bRIGHTS?\b", "instrument_right"), - (r"\bWARRANTS?\b", "instrument_warrant"), - (r"\bTEST (?:ISSUE|STOCK)\b", "test_issue"), - ) - for pattern, reason in token_patterns: - if re.search(pattern, searchable): - reasons.append(reason) - return sorted(set(reasons)) - - -def parse_eod_catalog( - descriptor: Mapping[str, Any], -) -> tuple[dict[tuple[str, str], CatalogRow], Path]: - path = Path(str(descriptor["path"])) - with path.open("r", encoding=str(descriptor["encoding"]), newline="") as handle: - reader = csv.DictReader(handle) - if reader.fieldnames is None: - raise UniverseBlockedError("eod_catalog has no header") - if tuple(reader.fieldnames) != EOD_CATALOG_FIELDS: - raise UniverseBlockedError( - "eod_catalog headers must be exactly, in order: " - + ",".join(EOD_CATALOG_FIELDS) - ) - output: dict[tuple[str, str], CatalogRow] = {} - for row_number, raw in enumerate(reader, start=2): - if not any(str(value or "").strip() for value in raw.values()): - continue - if None in raw: - raise UniverseBlockedError( - f"eod_catalog row {row_number} has extra cells" - ) - cells = { - field: str(raw[field] or "").strip() for field in EOD_CATALOG_FIELDS - } - exchange = cells["exchange"].upper() - if exchange not in EXCHANGES: - raise UniverseBlockedError( - f"eod_catalog row {row_number}.exchange must be one of {EXCHANGES}" - ) - canonical_symbol = normalized_symbol( - cells["canonical_symbol"], - f"eod_catalog row {row_number}.canonical_symbol", - ) - data_symbol = cells["data_symbol"] - broker_symbol = cells["broker_symbol"].upper() - instrument_type = cells["instrument_type"].upper() - catalog_row = CatalogRow( - exchange=exchange, - canonical_symbol=canonical_symbol, - data_symbol=data_symbol, - broker_symbol=broker_symbol, - instrument_type=instrument_type, - benchmark_id=cells["benchmark_id"].upper(), - ticker_csv=cells["ticker_csv"], - benchmark_csv=cells["benchmark_csv"], - tick_rule_id=cells["tick_rule_id"], - tick_effective_date=cells["tick_effective_date"], - tick_reference_price=cells["tick_reference_price"], - resolved_tick_size=cells["resolved_tick_size"], - source_name=cells["source_name"], - ) - key = (exchange, canonical_symbol) - if key in output: - raise UniverseBlockedError( - f"duplicate eod_catalog row: {exchange}:{canonical_symbol}" - ) - output[key] = catalog_row - if not output: - raise UniverseBlockedError("eod_catalog contains no rows") - return output, path.parent - - -def optional_data_path(raw: str, catalog_directory: Path) -> Optional[Path]: - if not raw: - return None - path = Path(raw) - if not path.is_absolute(): - path = catalog_directory / path - path = path.resolve() - if not path.is_file(): - return None - return path - - -def normalize_tick_contract( - row: CatalogRow, analysis_date: str -) -> tuple[Optional[dict[str, str]], Optional[str]]: - fields = ( - row.tick_rule_id, - row.tick_effective_date, - row.tick_reference_price, - row.resolved_tick_size, - ) - if any(not value for value in fields): - return None, "missing_tick_contract" - try: - effective_date = require_iso_date( - row.tick_effective_date, "tick_effective_date" - ) - if date.fromisoformat(effective_date) > date.fromisoformat(analysis_date): - raise UniverseBlockedError( - "tick_effective_date cannot be later than analysis_date" - ) - reference_price = decimal_string( - row.tick_reference_price, "tick_reference_price", positive=True - ) - resolved_tick_size = decimal_string( - row.resolved_tick_size, "resolved_tick_size", positive=True - ) - except UniverseBlockedError: - return None, "invalid_tick_contract" - return ( - { - "schema": TICK_CONTRACT_SCHEMA, - "kind": TICK_CONTRACT_KIND, - "rule_id": row.tick_rule_id, - "effective_date": effective_date, - "reference_price": reference_price, - "resolved_tick_size": resolved_tick_size, - }, - None, - ) - - -def catalog_exclusion_reasons( - row: CatalogRow, - catalog_directory: Path, - analysis_date: str, -) -> tuple[ - list[str], - Optional[Path], - Optional[Path], - Optional[dict[str, str]], -]: - reasons: list[str] = [] - if row.instrument_type in EXCLUDED_CATALOG_TYPES: - reasons.append(EXCLUDED_CATALOG_TYPES[row.instrument_type]) - elif row.instrument_type not in INCLUDED_INSTRUMENT_TYPES: - reasons.append("unsupported_instrument_type") - if not row.data_symbol: - reasons.append("missing_data_symbol") - if not row.broker_symbol: - reasons.append("missing_broker_symbol") - if row.benchmark_id != EXCHANGE_CONTRACTS[row.exchange]["benchmark_id"]: - reasons.append("invalid_benchmark_id") - if not row.source_name: - reasons.append("missing_eod_source") - - ticker_path = optional_data_path(row.ticker_csv, catalog_directory) - if ticker_path is None: - reasons.append("missing_ticker_csv") - benchmark_path = optional_data_path(row.benchmark_csv, catalog_directory) - if benchmark_path is None: - reasons.append("missing_benchmark_csv") - tick_contract, tick_reason = normalize_tick_contract(row, analysis_date) - if tick_reason is not None: - reasons.append(tick_reason) - return ( - sorted(set(reasons)), - ticker_path, - benchmark_path, - tick_contract, - ) - - -def source_hash_entry( - descriptor: Mapping[str, Any], role: str, exchange: str -) -> dict[str, str]: - path = Path(str(descriptor["path"])) - entry = { - "source_id": str(descriptor["source_id"]), - "role": role, - "provider": str(descriptor["provider"]), - "exchange": exchange, - "as_of": str(descriptor["as_of"]), - "path": str(path.resolve()), - "sha256": sha256_file(path), - } - if not HEX64.fullmatch(entry["sha256"]): - raise AssertionError("sha256_file returned an invalid digest") - return entry - - -def exclusion_record( - exchange: str, - canonical_symbol: str, - broker_symbol: Optional[str], - name: str, - reasons: Sequence[str], - official_source_id: str, - broker_source_id: Optional[str], -) -> dict[str, Any]: - normalized_reasons = sorted(set(reasons)) - if not normalized_reasons: - raise AssertionError("an exclusion must have at least one reason") - return { - "exchange": exchange, - "canonical_symbol": canonical_symbol, - "broker_symbol": broker_symbol, - "name": name, - "reasons": normalized_reasons, - "official_source_id": official_source_id, - "broker_source_id": broker_source_id, - } - - -def build_manifest(raw_spec: dict[str, Any], spec_directory: Path) -> dict[str, Any]: - spec = normalize_build_spec(raw_spec, spec_directory) - exchanges = spec["exchanges"] - official_descriptors = {item["exchange"]: item for item in spec["official_sources"]} - broker_descriptors = {item["exchange"]: item for item in spec["broker_sources"]} - official_rows = { - exchange: { - row.symbol: row - for row in parse_master_source(official_descriptors[exchange]) - } - for exchange in exchanges - } - broker_rows = { - exchange: { - row.symbol: row for row in parse_master_source(broker_descriptors[exchange]) - } - for exchange in broker_descriptors - } - catalog, catalog_directory = parse_eod_catalog(spec["eod_catalog"]) - - source_hashes = [ - source_hash_entry(item, "OFFICIAL_MASTER", item["exchange"]) - for item in spec["official_sources"] - ] - source_hashes.extend( - source_hash_entry(item, "BROKER_MASTER", item["exchange"]) - for item in spec["broker_sources"] - ) - source_hashes.append( - source_hash_entry(spec["eod_catalog"], "EOD_CATALOG", "GLOBAL") - ) - source_hashes.sort(key=lambda item: item["source_id"]) - - instruments: list[dict[str, Any]] = [] - exclusions: list[dict[str, Any]] = [] - path_hash_cache: dict[Path, str] = {} - - for exchange in exchanges: - official_descriptor = official_descriptors[exchange] - broker_descriptor = broker_descriptors[exchange] - broker_by_symbol = broker_rows[exchange] - for canonical_symbol in sorted(official_rows[exchange]): - official_row = official_rows[exchange][canonical_symbol] - catalog_row = catalog.get((exchange, canonical_symbol)) - reasons = metadata_exclusion_reasons(official_row, official_descriptor) - broker_row: Optional[MasterRow] = None - broker_symbol: Optional[str] = None - broker_source_id: Optional[str] = str(broker_descriptor["source_id"]) - ticker_path: Optional[Path] = None - benchmark_path: Optional[Path] = None - tick_contract: Optional[dict[str, str]] = None - - if catalog_row is None: - reasons.append("missing_eod_mapping") - else: - broker_symbol = catalog_row.broker_symbol or None - ( - catalog_reasons, - ticker_path, - benchmark_path, - tick_contract, - ) = catalog_exclusion_reasons( - catalog_row, - catalog_directory, - spec["analysis_date"], - ) - reasons.extend(catalog_reasons) - if catalog_row.broker_symbol: - broker_row = broker_by_symbol.get(catalog_row.broker_symbol.upper()) - if broker_row is None: - reasons.append("not_kis_tradable") - else: - reasons.extend( - metadata_exclusion_reasons(broker_row, broker_descriptor) - ) - - reasons = sorted(set(reasons)) - if reasons: - exclusions.append( - exclusion_record( - exchange=exchange, - canonical_symbol=canonical_symbol, - broker_symbol=broker_symbol, - name=official_row.name, - reasons=reasons, - official_source_id=official_row.source_id, - broker_source_id=broker_source_id, - ) - ) - continue - - if catalog_row is None: - raise AssertionError("catalog row unexpectedly missing") - if catalog_row.instrument_type not in INCLUDED_INSTRUMENT_TYPES: - raise AssertionError("excluded catalog type reached inclusion path") - if ticker_path is None or benchmark_path is None or tick_contract is None: - raise AssertionError("validated EOD inputs unexpectedly missing") - for data_path in (ticker_path, benchmark_path): - if data_path not in path_hash_cache: - path_hash_cache[data_path] = sha256_file(data_path) - - contract = EXCHANGE_CONTRACTS[exchange] - instruments.append( - { - "market": contract["market"], - "exchange": exchange, - "canonical_symbol": canonical_symbol, - "data_symbol": catalog_row.data_symbol, - "broker_symbol": catalog_row.broker_symbol, - "instrument_type": catalog_row.instrument_type, - "benchmark_id": catalog_row.benchmark_id, - "currency": contract["currency"], - "venue": contract["venue"], - "ticker_csv": str(ticker_path), - "benchmark_csv": str(benchmark_path), - "tick_contract": tick_contract, - "source_name": catalog_row.source_name, - "ticker_csv_sha256": path_hash_cache[ticker_path], - "benchmark_csv_sha256": path_hash_cache[benchmark_path], - "broker_tradability_verified": True, - "official_source_id": official_row.source_id, - "broker_source_id": broker_source_id, - } - ) - - official_symbols = set(official_rows[exchange]) - for catalog_key, catalog_row in sorted(catalog.items()): - catalog_exchange, canonical_symbol = catalog_key - if catalog_exchange != exchange or canonical_symbol in official_symbols: - continue - exclusions.append( - exclusion_record( - exchange=exchange, - canonical_symbol=canonical_symbol, - broker_symbol=catalog_row.broker_symbol or None, - name="", - reasons=["not_official_exchange_member"], - official_source_id=str(official_descriptor["source_id"]), - broker_source_id=str(broker_descriptor["source_id"]), - ) - ) - - instruments.sort( - key=lambda item: ( - EXCHANGES.index(item["exchange"]), - item["canonical_symbol"], - item["broker_symbol"], - ) - ) - exclusions.sort( - key=lambda item: ( - EXCHANGES.index(item["exchange"]), - item["canonical_symbol"], - tuple(item["reasons"]), - ) - ) - - by_exchange: dict[str, dict[str, int]] = {} - for exchange in EXCHANGES: - exchange_official_rows = official_rows.get(exchange, {}) - catalog_mapped = sum( - (exchange, symbol) in catalog for symbol in exchange_official_rows - ) - by_exchange[exchange] = { - "official": len(exchange_official_rows), - "broker": len(broker_rows.get(exchange, {})), - "catalog_mapped": catalog_mapped, - "included": sum(item["exchange"] == exchange for item in instruments), - "excluded": sum(item["exchange"] == exchange for item in exclusions), - } - counts = { - "official_rows": sum(item["official"] for item in by_exchange.values()), - "broker_rows": sum(item["broker"] for item in by_exchange.values()), - "catalog_mapped_rows": sum( - item["catalog_mapped"] for item in by_exchange.values() - ), - "included": sum(item["included"] for item in by_exchange.values()), - "excluded": sum(item["excluded"] for item in by_exchange.values()), - "by_exchange": by_exchange, - } - coverage_minimums = spec["catalog_coverage_contract"]["minimum_ratio_by_exchange"] - coverage_ready = all( - meets_minimum_coverage_ratio( - by_exchange[exchange]["catalog_mapped"], - by_exchange[exchange]["official"], - coverage_minimums[exchange], - ) - for exchange in exchanges - ) - screenable_minimums = spec["catalog_coverage_contract"][ - "minimum_screenable_ratio_by_exchange" - ] - screenable_ready = all( - meets_minimum_coverage_ratio( - by_exchange[exchange]["included"], - by_exchange[exchange]["official"], - screenable_minimums[exchange], - ) - for exchange in exchanges - ) - build_status = ( - "READY" - if coverage_ready - and screenable_ready - and all(by_exchange[exchange]["included"] > 0 for exchange in exchanges) - else "BLOCKED" - ) - output: dict[str, Any] = { - "schema": MANIFEST_SCHEMA, - "build_status": build_status, - "as_of": spec["as_of"], - "analysis_date": spec["analysis_date"], - "catalog_coverage_contract": spec["catalog_coverage_contract"], - "source_hashes": source_hashes, - "instruments": instruments, - "exclusions": exclusions, - "counts": counts, - } - output["manifest_hash"] = sha256_json(output) - return output - - -def write_csv( - path: Path, fieldnames: Sequence[str], rows: Sequence[Mapping[str, Any]] -) -> None: - with path.open("w", encoding="utf-8", newline="") as handle: - writer = csv.DictWriter(handle, fieldnames=fieldnames) - writer.writeheader() - writer.writerows(rows) - - -def synthetic_kis_krx_line( - exchange: str, - symbol: str, - name: str, - *, - group_code: str = "ST", - etp_code: str = "0", - spac: str = "N", - preferred: str = "0", - trading_halt: str = "N", - liquidation: str = "N", - administrative: str = "N", -) -> str: - layout = KIS_KRX_LAYOUTS[exchange] - widths = layout["widths"] - values = [""] * len(widths) - values[0] = group_code - values[int(layout["etp_index"])] = etp_code - values[int(layout["spac_index"])] = spac - values[int(layout["halt_index"])] = trading_halt - values[int(layout["liquidation_index"])] = liquidation - values[int(layout["admin_index"])] = administrative - values[int(layout["preferred_index"])] = preferred - tail = "".join( - str(value).ljust(width)[:width] for value, width in zip(values, widths) - ) - tail = tail.ljust(int(layout["tail_length"])) - standard_symbol = f"KR7{symbol}".ljust(12)[:12] - return f"{symbol.ljust(9)}{standard_symbol}{name}{tail}" - - -def synthetic_kis_overseas_line( - exchange: str, - symbol: str, - security_type: str, - dr_flag: str, - classification: str = "", -) -> str: - fields = [""] * 24 - fields[0] = "US" - fields[1] = "22" if exchange == "NASDAQ" else "21" - fields[2] = "NAS" if exchange == "NASDAQ" else "NYS" - fields[3] = exchange - fields[4] = symbol - fields[5] = f"{fields[2]}{symbol}" - fields[6] = symbol - fields[7] = f"{symbol} SYNTHETIC" - fields[8] = security_type - fields[9] = "USD" - fields[17] = dr_flag - fields[22] = classification - return "\t".join(fields) - - -def self_test() -> None: - with tempfile.TemporaryDirectory(prefix="qta-universe-v2-") as directory: - root = Path(directory) - master_paths: dict[str, Path] = {} - broker_paths: dict[str, Path] = {} - kind_path = root / "kind-corpList.xls" - kind_path.write_text( - "" - "" - "" - "" - "" - "" - "" - "" - "
회사명시장구분종목코드
KOSPI Common유가000001
KOSPI ETF유가000002
KOSDAQ Common코스닥100001
KOSDAQ Missing코스닥100002
KOSDAQ Alpha코스닥0156T0
KONEX Ignored코넥스900001
", - encoding="cp949", - ) - for exchange in ("KOSPI", "KOSDAQ"): - master_paths[exchange] = kind_path - broker_path = root / f"{exchange.lower()}_code.mst" - broker_path.write_text( - synthetic_kis_krx_line( - exchange, - "000001" if exchange == "KOSPI" else "100001", - f"{exchange} Common", - ) - + "\n", - encoding="cp949", - ) - broker_paths[exchange] = broker_path - - nasdaq_path = root / "nasdaqlisted.txt" - nasdaq_path.write_text( - "Symbol|Security Name|Market Category|Test Issue|Financial Status|" - "Round Lot Size|ETF|NextShares\n" - "NDAQ1|Nasdaq Common|Q|N|N|100|N|N\n" - "NDAQW|Nasdaq Warrants|Q|N|N|100|N|N\n" - "File Creation Time: 0726202600|||||||\n", - encoding="utf-8", - ) - nyse_path = root / "otherlisted.txt" - nyse_path.write_text( - "ACT Symbol|Security Name|Exchange|CQS Symbol|ETF|Round Lot Size|" - "Test Issue|NASDAQ Symbol\n" - "NYSE1|NYSE ADR|N|NYSE1|N|100|N|NYSE1\n" - "NYSEX|Other Market|A|NYSEX|N|100|N|NYSEX\n" - "File Creation Time: 0726202600|||||||\n", - encoding="utf-8", - ) - master_paths["NASDAQ"] = nasdaq_path - master_paths["NYSE"] = nyse_path - - for exchange, symbol in (("NASDAQ", "NDAQ1"), ("NYSE", "NYSE1")): - broker_path = root / ( - "nasmst.cod" if exchange == "NASDAQ" else "nysmst.cod" - ) - broker_path.write_text( - synthetic_kis_overseas_line( - exchange, - symbol, - "2", - "Y" if exchange == "NYSE" else "N", - ) - + "\n", - encoding="cp949", - ) - broker_paths[exchange] = broker_path - - ticker_paths: dict[str, Path] = {} - benchmark_paths: dict[str, Path] = {} - included_symbols = { - "KOSPI": "000001", - "KOSDAQ": "100001", - "NYSE": "NYSE1", - "NASDAQ": "NDAQ1", - } - for exchange, symbol in included_symbols.items(): - ticker_path = root / f"{exchange.lower()}-{symbol}.csv" - ticker_path.write_text( - "date,adjusted_close\n2026-07-24,100\n", encoding="utf-8" - ) - ticker_paths[exchange] = ticker_path - benchmark_path = root / f"{exchange.lower()}-benchmark.csv" - benchmark_path.write_text( - "date,adjusted_close\n2026-07-24,100\n", encoding="utf-8" - ) - benchmark_paths[exchange] = benchmark_path - - catalog_path = root / "eod-catalog.csv" - catalog_rows: list[dict[str, str]] = [] - for exchange, symbol in included_symbols.items(): - catalog_rows.append( - { - "exchange": exchange, - "canonical_symbol": symbol, - "data_symbol": ( - f"{symbol}.KS" - if exchange == "KOSPI" - else f"{symbol}.KQ" - if exchange == "KOSDAQ" - else symbol - ), - "broker_symbol": symbol, - "instrument_type": "ADR" if exchange == "NYSE" else "COMMON", - "benchmark_id": EXCHANGE_CONTRACTS[exchange]["benchmark_id"], - "ticker_csv": str(ticker_paths[exchange]), - "benchmark_csv": str(benchmark_paths[exchange]), - "tick_rule_id": f"SELFTEST-{exchange}", - "tick_effective_date": "2026-01-01", - "tick_reference_price": "100", - "resolved_tick_size": ( - "1" if exchange in {"KOSPI", "KOSDAQ"} else "0.01" - ), - "source_name": "synthetic-adjusted-eod", - } - ) - catalog_rows.extend( - [ - { - "exchange": "KOSPI", - "canonical_symbol": "000002", - "data_symbol": "", - "broker_symbol": "", - "instrument_type": "ETF", - "benchmark_id": "KOSPI_COMPOSITE", - "ticker_csv": "", - "benchmark_csv": "", - "tick_rule_id": "SELFTEST-KOSPI", - "tick_effective_date": "2026-01-01", - "tick_reference_price": "100", - "resolved_tick_size": "1", - "source_name": "", - }, - { - "exchange": "NASDAQ", - "canonical_symbol": "NDAQW", - "data_symbol": "", - "broker_symbol": "", - "instrument_type": "WARRANT", - "benchmark_id": "NASDAQ_COMPOSITE", - "ticker_csv": "", - "benchmark_csv": "", - "tick_rule_id": "SELFTEST-NASDAQ", - "tick_effective_date": "2026-01-01", - "tick_reference_price": "1", - "resolved_tick_size": "0.01", - "source_name": "", - }, - { - "exchange": "KOSDAQ", - "canonical_symbol": "100002", - "data_symbol": "100002.KQ", - "broker_symbol": "", - "instrument_type": "COMMON", - "benchmark_id": "KOSDAQ_COMPOSITE", - "ticker_csv": "", - "benchmark_csv": "", - "tick_rule_id": "", - "tick_effective_date": "", - "tick_reference_price": "", - "resolved_tick_size": "", - "source_name": "synthetic-adjusted-eod", - }, - { - "exchange": "KOSDAQ", - "canonical_symbol": "0156T0", - "data_symbol": "0156T0.KQ", - "broker_symbol": "0156T0", - "instrument_type": "COMMON", - "benchmark_id": "KOSDAQ_COMPOSITE", - "ticker_csv": str(ticker_paths["KOSDAQ"]), - "benchmark_csv": str(benchmark_paths["KOSDAQ"]), - "tick_rule_id": "SELFTEST-KOSDAQ", - "tick_effective_date": "2026-01-01", - "tick_reference_price": "100", - "resolved_tick_size": "0", - "source_name": "synthetic-adjusted-eod", - }, - ] - ) - write_csv(catalog_path, EOD_CATALOG_FIELDS, catalog_rows) - - official_sources: list[dict[str, Any]] = [] - broker_sources: list[dict[str, Any]] = [] - for exchange in EXCHANGES: - if exchange in {"KOSPI", "KOSDAQ"}: - official_format = "KRX_KIND_HTML" - official_columns = {} - official_delimiter = "" - official_statuses = [] - elif exchange == "NASDAQ": - official_format = "NASDAQ_LISTED" - official_columns = {} - official_delimiter = "|" - official_statuses = ["N"] - else: - official_format = "NASDAQ_OTHER" - official_columns = {} - official_delimiter = "|" - official_statuses = [] - official_sources.append( - { - "source_id": f"official-{exchange.lower()}", - "provider": ( - "KRX" if exchange in {"KOSPI", "KOSDAQ"} else "NASDAQ_TRADER" - ), - "exchange": exchange, - "as_of": "2026-07-26", - "path": str(master_paths[exchange]), - "format": official_format, - "encoding": ( - "cp949" if official_format == "KRX_KIND_HTML" else "utf-8" - ), - "delimiter": official_delimiter, - "skip_rows": 0, - "columns": official_columns, - "normal_status_values": official_statuses, - } - ) - broker_sources.append( - { - "source_id": f"kis-{exchange.lower()}", - "provider": "KIS", - "exchange": exchange, - "as_of": "2026-07-26", - "path": str(broker_paths[exchange]), - "format": ( - "KIS_KRX_MASTER" - if exchange in {"KOSPI", "KOSDAQ"} - else "KIS_OVERSEAS_MASTER" - ), - "encoding": "cp949", - "delimiter": ("" if exchange in {"KOSPI", "KOSDAQ"} else "\t"), - "skip_rows": 0, - "columns": {}, - "normal_status_values": [], - } - ) - - spec = { - "schema": BUILD_SPEC_SCHEMA, - "as_of": "2026-07-26", - "analysis_date": "2026-07-24", - "catalog_coverage_contract": { - "schema": CATALOG_COVERAGE_SCHEMA, - "minimum_ratio_by_exchange": {exchange: "1" for exchange in EXCHANGES}, - "minimum_screenable_ratio_by_exchange": { - exchange: "0.3" for exchange in EXCHANGES - }, - }, - "official_sources": official_sources, - "broker_sources": broker_sources, - "eod_catalog": { - "source_id": "eod-catalog", - "provider": "synthetic-eod", - "as_of": "2026-07-26", - "path": str(catalog_path), - "encoding": "utf-8", - }, - } - first = build_manifest(spec, root) - second = build_manifest( - { - **spec, - "official_sources": list(reversed(official_sources)), - "broker_sources": list(reversed(broker_sources)), - }, - root, - ) - assert first == second - assert first["build_status"] == "READY" - assert first["counts"]["included"] == 4 - assert first["counts"]["by_exchange"]["KOSPI"]["excluded"] == 1 - assert first["counts"]["by_exchange"]["KOSDAQ"]["excluded"] == 2 - assert first["counts"]["by_exchange"]["NASDAQ"]["excluded"] == 1 - assert ( - first["counts"]["catalog_mapped_rows"] == first["counts"]["official_rows"] - ) - assert all( - first["counts"]["by_exchange"][exchange]["catalog_mapped"] - == first["counts"]["by_exchange"][exchange]["official"] - for exchange in EXCHANGES - ) - assert any( - item["canonical_symbol"] == "000002" and "instrument_etf" in item["reasons"] - for item in first["exclusions"] - ) - assert any( - item["canonical_symbol"] == "NDAQW" - and "instrument_warrant" in item["reasons"] - for item in first["exclusions"] - ) - assert any( - item["canonical_symbol"] == "0156T0" - and "invalid_tick_contract" in item["reasons"] - for item in first["exclusions"] - ) - assert any( - item["canonical_symbol"] == "100002" - and { - "missing_tick_contract", - "missing_ticker_csv", - "missing_benchmark_csv", - }.issubset(set(item["reasons"])) - for item in first["exclusions"] - ) - assert all(item["broker_tradability_verified"] for item in first["instruments"]) - assert all( - set(item["tick_contract"]) - == { - "schema", - "kind", - "rule_id", - "effective_date", - "reference_price", - "resolved_tick_size", - } - for item in first["instruments"] - ) - expected_hash = sha256_json( - {key: value for key, value in first.items() if key != "manifest_hash"} - ) - assert first["manifest_hash"] == expected_hash - kr_only = build_manifest( - { - **spec, - "official_sources": [ - item - for item in official_sources - if item["exchange"] in {"KOSPI", "KOSDAQ"} - ], - "broker_sources": [ - item - for item in broker_sources - if item["exchange"] in {"KOSPI", "KOSDAQ"} - ], - }, - root, - ) - assert kr_only["build_status"] == "READY" - assert {item["market"] for item in kr_only["instruments"]} == {"KR"} - assert all( - value == 0 - for exchange in ("NYSE", "NASDAQ") - for value in kr_only["counts"]["by_exchange"][exchange].values() - ) - relaxed_manifest = build_manifest( - { - **spec, - "catalog_coverage_contract": { - "schema": CATALOG_COVERAGE_SCHEMA, - "minimum_ratio_by_exchange": { - exchange: "0.5" for exchange in EXCHANGES - }, - "minimum_screenable_ratio_by_exchange": { - exchange: "0.3" for exchange in EXCHANGES - }, - }, - }, - root, - ) - assert relaxed_manifest["build_status"] == "READY" - assert relaxed_manifest["manifest_hash"] != first["manifest_hash"] - - strict_screenable_manifest = build_manifest( - { - **spec, - "catalog_coverage_contract": { - "schema": CATALOG_COVERAGE_SCHEMA, - "minimum_ratio_by_exchange": { - exchange: "1" for exchange in EXCHANGES - }, - "minimum_screenable_ratio_by_exchange": { - exchange: "0.6" for exchange in EXCHANGES - }, - }, - }, - root, - ) - assert ( - strict_screenable_manifest["counts"]["catalog_mapped_rows"] - == strict_screenable_manifest["counts"]["official_rows"] - ) - assert strict_screenable_manifest["build_status"] == "BLOCKED" - assert strict_screenable_manifest["manifest_hash"] != first["manifest_hash"] - - minimal_catalog_path = root / "minimal-eod-catalog.csv" - write_csv(minimal_catalog_path, EOD_CATALOG_FIELDS, catalog_rows[:4]) - minimal_manifest = build_manifest( - { - **spec, - "eod_catalog": { - **spec["eod_catalog"], - "path": str(minimal_catalog_path), - }, - }, - root, - ) - assert minimal_manifest["counts"]["included"] == 4 - assert minimal_manifest["counts"]["catalog_mapped_rows"] == 4 - assert minimal_manifest["build_status"] == "BLOCKED" - assert minimal_manifest["manifest_hash"] != first["manifest_hash"] - print( - canonical_json( - { - "self_test": "PASS", - "schema": MANIFEST_SCHEMA, - "manifest_hash": first["manifest_hash"], - "counts": first["counts"], - } - ) - ) - - -def emit(value: Mapping[str, Any], output: Optional[str]) -> None: - rendered = json.dumps( - value, - ensure_ascii=False, - sort_keys=True, - indent=2, - allow_nan=False, - ) - if output: - output_path = Path(output).resolve() - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(rendered + "\n", encoding="utf-8") - print( - canonical_json( - { - "schema": value.get("schema"), - "build_status": value.get("build_status"), - "output": str(output_path), - "manifest_hash": value.get("manifest_hash"), - "counts": value.get("counts"), - } - ) - ) - else: - print(rendered) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--build-spec") - parser.add_argument("--output") - parser.add_argument("--self-test", action="store_true") - return parser.parse_args() - - -def main() -> int: - args = parse_args() - if args.self_test: - self_test() - return 0 - if not args.build_spec: - print( - canonical_json( - { - "schema": "qta-universe-build-error/v1", - "build_status": "BLOCKED", - "reason": "--build-spec is required", - } - ) - ) - return 2 - spec_path = Path(args.build_spec).resolve() - try: - manifest = build_manifest( - load_json_object(spec_path), - spec_path.parent, - ) - except ( - UniverseBlockedError, - OSError, - UnicodeError, - json.JSONDecodeError, - ) as exc: - error = { - "schema": "qta-universe-build-error/v1", - "build_status": "BLOCKED", - "reason": str(exc), - } - emit(error, None) - return 2 - emit(manifest, args.output) - return 0 if manifest["build_status"] == "READY" else 2 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/plugins/quant-stock-technical/skills/quant-stock-technical/scripts/collect_open1h_research.py b/plugins/quant-stock-technical/skills/quant-stock-technical/scripts/collect_open1h_research.py deleted file mode 100644 index 88fc3c1..0000000 --- a/plugins/quant-stock-technical/skills/quant-stock-technical/scripts/collect_open1h_research.py +++ /dev/null @@ -1,920 +0,0 @@ -#!/usr/bin/env python3 -"""Collect deterministic first-60-minute KIS bars for QTA research.""" - -from __future__ import annotations - -import argparse -import json -import sys -import tempfile -from datetime import date -from decimal import Decimal, InvalidOperation, ROUND_HALF_EVEN -from pathlib import Path -from typing import Any, Mapping - -import fetch_kis_kr_eod as shared -import screen_universe as screen - - -JOB_SCHEMA = "qta-open1h-research-job/v1" -BARS_SCHEMA = "qta-open1h-bars/v1" -SNAPSHOT_SCHEMA = "qta-open1h-snapshot/v1" -JOB_FIELDS = { - "schema", - "manifest_path", - "market", - "session_date", - "output_directory", - "environment", - "request_interval_ms", -} -MARKET_CONTRACTS = { - "KR": { - "exchanges": ("KOSPI", "KOSDAQ"), - "interval_minutes": 1, - "window_start": "090000", - "window_end": "100000", - "expected_bars": 61, - "path": ( - "/uapi/domestic-stock/v1/quotations/" - "inquire-time-dailychartprice" - ), - "tr_id": "FHKST03010230", - }, - "US": { - "exchanges": ("NYSE", "NASDAQ"), - "interval_minutes": 5, - "window_start": "093000", - "window_end": "103000", - "expected_bars": 13, - "path": ( - "/uapi/overseas-price/v1/quotations/" - "inquire-time-itemchartprice" - ), - "tr_id": "HHDFS76950200", - }, -} -US_EXCHANGE_CODES = {"NYSE": "NYS", "NASDAQ": "NAS"} -BAR_FIELDS = { - "time", - "open", - "high", - "low", - "close", - "volume", - "turnover", -} -BARS_FIELDS = { - "schema", - "source", - "market", - "exchange", - "canonical_symbol", - "broker_symbol", - "session_date", - "interval_minutes", - "window_start", - "window_end", - "bars", - "bars_hash", - "summary", -} -SUMMARY_FIELDS = { - "bar_count", - "start_open", - "end_close", - "window_high", - "window_low", - "window_volume", - "window_turnover", - "return_60m_bps", - "maximum_excursion_bps", - "minimum_excursion_bps", -} - - -class ResearchBlockedError(ValueError): - """Raised when first-hour research data cannot be used without guessing.""" - - -def exact_fields(value: Mapping[str, Any], expected: set[str], label: str) -> None: - if set(value) != expected: - raise ResearchBlockedError( - f"{label} fields mismatch; missing={sorted(expected - set(value))}, " - f"extra={sorted(set(value) - expected)}" - ) - - -def require_date(value: Any, field: str) -> str: - if not isinstance(value, str): - raise ResearchBlockedError(f"{field} must be YYYY-MM-DD") - try: - parsed = date.fromisoformat(value) - except ValueError as exc: - raise ResearchBlockedError(f"{field} must be YYYY-MM-DD") from exc - if parsed.isoformat() != value: - raise ResearchBlockedError(f"{field} must be YYYY-MM-DD") - return value - - -def require_absolute_path(value: Any, field: str) -> Path: - path = Path(str(value)).expanduser() - if not path.is_absolute(): - raise ResearchBlockedError(f"{field} must be absolute") - return path - - -def decimal_value( - value: Any, - field: str, - *, - allow_zero: bool = False, -) -> Decimal: - if isinstance(value, bool): - raise ResearchBlockedError(f"{field} must be decimal") - try: - parsed = Decimal(str(value).replace(",", "").strip()) - except (InvalidOperation, ValueError) as exc: - raise ResearchBlockedError(f"{field} must be decimal") from exc - if not parsed.is_finite() or ( - parsed < 0 if allow_zero else parsed <= 0 - ): - qualifier = "nonnegative" if allow_zero else "positive" - raise ResearchBlockedError(f"{field} must be {qualifier}") - return parsed - - -def decimal_text(value: Decimal) -> str: - rendered = format(value, "f") - if "." in rendered: - rendered = rendered.rstrip("0").rstrip(".") - return rendered or "0" - - -def bps(numerator: Decimal, denominator: Decimal) -> str: - value = ( - (numerator / denominator - Decimal(1)) * Decimal(10000) - ).quantize(Decimal("0.01"), rounding=ROUND_HALF_EVEN) - return format(value, "f") - - -def load_job(path: Path) -> dict[str, Any]: - try: - raw = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as exc: - raise ResearchBlockedError(f"job must be readable JSON: {path}") from exc - if not isinstance(raw, dict): - raise ResearchBlockedError("job must contain one JSON object") - exact_fields(raw, JOB_FIELDS, "job") - if raw["schema"] != JOB_SCHEMA: - raise ResearchBlockedError(f"job.schema must be {JOB_SCHEMA}") - market = str(raw["market"]).upper() - if market not in MARKET_CONTRACTS: - raise ResearchBlockedError("job.market must be KR or US") - session_date = require_date(raw["session_date"], "job.session_date") - manifest_path = require_absolute_path(raw["manifest_path"], "job.manifest_path") - if not manifest_path.is_file() or manifest_path.is_symlink(): - raise ResearchBlockedError( - "job.manifest_path must be a regular non-symlink file" - ) - output_directory = require_absolute_path( - raw["output_directory"], "job.output_directory" - ) - if output_directory.exists() and ( - not output_directory.is_dir() or output_directory.is_symlink() - ): - raise ResearchBlockedError( - "job.output_directory must be a non-symlink directory" - ) - environment = str(raw["environment"]).lower() - if environment not in shared.BASE_URLS: - raise ResearchBlockedError("job.environment must be live or paper") - interval = raw["request_interval_ms"] - if ( - isinstance(interval, bool) - or not isinstance(interval, int) - or interval < (1000 if environment == "paper" else 100) - ): - floor = 1000 if environment == "paper" else 100 - raise ResearchBlockedError( - f"job.request_interval_ms must be an integer >= {floor}" - ) - try: - manifest_raw = json.loads(manifest_path.read_text(encoding="utf-8")) - manifest = screen.normalize_manifest_v2(manifest_raw) - except ( - OSError, - json.JSONDecodeError, - screen.ScreenBlockedError, - ) as exc: - raise ResearchBlockedError(f"manifest validation failed: {exc}") from exc - if manifest["as_of"] != session_date: - raise ResearchBlockedError( - "manifest.as_of must equal the research session_date" - ) - if manifest["analysis_date"] >= session_date: - raise ResearchBlockedError( - "manifest.analysis_date must precede the research session_date" - ) - instruments = [ - instrument - for instrument in manifest["instruments"] - if instrument["market"] == market - ] - if not instruments: - raise ResearchBlockedError("manifest has no instruments for job.market") - return { - "schema": JOB_SCHEMA, - "manifest_path": str(manifest_path), - "manifest": manifest, - "market": market, - "session_date": session_date, - "output_directory": output_directory, - "environment": environment, - "request_interval_ms": interval, - "instruments": sorted( - instruments, - key=lambda item: ( - MARKET_CONTRACTS[market]["exchanges"].index(item["exchange"]), - item["canonical_symbol"], - ), - ), - } - - -def normalize_bar( - raw: Mapping[str, Any], - *, - market: str, - session_date: str, -) -> dict[str, str] | None: - compact_date = session_date.replace("-", "") - if market == "KR": - raw_date = str(raw.get("stck_bsop_date", "")).strip() - raw_time = str(raw.get("stck_cntg_hour", "")).strip() - if raw_date != compact_date: - return None - fields = { - "open": raw.get("stck_oprc"), - "high": raw.get("stck_hgpr"), - "low": raw.get("stck_lwpr"), - "close": raw.get("stck_prpr"), - "volume": raw.get("cntg_vol", "0") or "0", - "turnover": raw.get("acml_tr_pbmn", "0") or "0", - } - else: - raw_date = str(raw.get("xymd", "")).strip() - raw_time = str(raw.get("xhms", "")).strip() - if raw_date != compact_date: - return None - fields = { - "open": raw.get("open"), - "high": raw.get("high"), - "low": raw.get("low"), - "close": raw.get("last"), - "volume": raw.get("evol", "0") or "0", - "turnover": raw.get("eamt", "0") or "0", - } - if len(raw_time) != 6 or not raw_time.isdigit(): - raise ResearchBlockedError("KIS minute row has invalid local time") - output = { - "time": raw_time, - "open": decimal_text(decimal_value(fields["open"], "minute open")), - "high": decimal_text(decimal_value(fields["high"], "minute high")), - "low": decimal_text(decimal_value(fields["low"], "minute low")), - "close": decimal_text(decimal_value(fields["close"], "minute close")), - "volume": decimal_text( - decimal_value(fields["volume"], "minute volume", allow_zero=True) - ), - "turnover": decimal_text( - decimal_value(fields["turnover"], "minute turnover", allow_zero=True) - ), - } - high = Decimal(output["high"]) - low = Decimal(output["low"]) - if high < max( - Decimal(output["open"]), - low, - Decimal(output["close"]), - ): - raise ResearchBlockedError("KIS minute row high is inconsistent") - if low > min( - Decimal(output["open"]), - high, - Decimal(output["close"]), - ): - raise ResearchBlockedError("KIS minute row low is inconsistent") - return output - - -def normalized_window_bars( - raw_rows: Any, - *, - market: str, - session_date: str, -) -> list[dict[str, str]]: - if not isinstance(raw_rows, list) or any( - not isinstance(item, dict) for item in raw_rows - ): - raise ResearchBlockedError("KIS minute output must be an object array") - contract = MARKET_CONTRACTS[market] - by_time: dict[str, dict[str, str]] = {} - for raw in raw_rows: - bar = normalize_bar( - raw, - market=market, - session_date=session_date, - ) - if bar is None: - continue - if not ( - contract["window_start"] - <= bar["time"] - <= contract["window_end"] - ): - continue - existing = by_time.get(bar["time"]) - if existing is not None and existing != bar: - raise ResearchBlockedError( - f"conflicting duplicate minute bar at {bar['time']}" - ) - by_time[bar["time"]] = bar - bars = [by_time[key] for key in sorted(by_time)] - if not bars: - raise ResearchBlockedError("KIS response has no first-hour bars") - return bars - - -def minute_of_day(value: str) -> int: - if len(value) != 6 or not value.isdigit(): - raise ResearchBlockedError("bar time must be HHMMSS") - hour = int(value[:2]) - minute = int(value[2:4]) - second = int(value[4:]) - if hour > 23 or minute > 59 or second != 0: - raise ResearchBlockedError("bar time must be a whole valid minute") - return hour * 60 + minute - - -def coverage_summary( - bars: list[dict[str, str]], - *, - market: str, -) -> dict[str, Any]: - contract = MARKET_CONTRACTS[market] - first_time = bars[0]["time"] - last_time = bars[-1]["time"] - first_minute = minute_of_day(first_time) - last_minute = minute_of_day(last_time) - start_minute = minute_of_day(contract["window_start"]) - end_minute = minute_of_day(contract["window_end"]) - expected = int(contract["expected_bars"]) - interval = int(contract["interval_minutes"]) - if ( - first_time == contract["window_start"] - and last_time == contract["window_end"] - and len(bars) == expected - ): - status = "COMPLETE_GRID" - elif ( - first_time == contract["window_start"] - and last_time == contract["window_end"] - ): - status = "EXACT_ENDPOINTS_SPARSE" - elif ( - first_minute <= start_minute + interval - and last_minute >= end_minute - interval - ): - status = "NEAR_ENDPOINTS" - else: - status = "PARTIAL_WINDOW" - return { - "coverage_status": status, - "first_bar_time": first_time, - "last_bar_time": last_time, - "expected_bar_count": expected, - "missing_bar_count": max(0, expected - len(bars)), - "coverage_fraction": format( - Decimal(len(bars)) / Decimal(expected), "f" - ), - "observed_span_minutes": last_minute - first_minute, - } - - -def summarize_bars( - bars: list[dict[str, str]], - *, - market: str, -) -> dict[str, Any]: - start_open = Decimal(bars[0]["open"]) - end_close = Decimal(bars[-1]["close"]) - window_high = max(Decimal(bar["high"]) for bar in bars) - window_low = min(Decimal(bar["low"]) for bar in bars) - window_volume = sum( - (Decimal(bar["volume"]) for bar in bars), Decimal(0) - ) - if market == "KR": - # The domestic endpoint exposes cumulative traded value on each bar. - window_turnover = Decimal(bars[-1]["turnover"]) - else: - window_turnover = sum( - (Decimal(bar["turnover"]) for bar in bars), Decimal(0) - ) - return { - "bar_count": len(bars), - "start_open": decimal_text(start_open), - "end_close": decimal_text(end_close), - "window_high": decimal_text(window_high), - "window_low": decimal_text(window_low), - "window_volume": decimal_text(window_volume), - "window_turnover": decimal_text(window_turnover), - "return_60m_bps": bps(end_close, start_open), - "maximum_excursion_bps": bps(window_high, start_open), - "minimum_excursion_bps": bps(window_low, start_open), - } - - -def fetch_raw_rows( - client: shared.KisReadClient, - *, - market: str, - exchange: str, - broker_symbol: str, - session_date: str, -) -> list[dict[str, Any]]: - contract = MARKET_CONTRACTS[market] - if market == "KR": - body = client.get( - contract["path"], - contract["tr_id"], - { - "FID_COND_MRKT_DIV_CODE": "J", - "FID_INPUT_ISCD": broker_symbol, - "FID_INPUT_HOUR_1": contract["window_end"], - "FID_INPUT_DATE_1": session_date.replace("-", ""), - "FID_PW_DATA_INCU_YN": "N", - "FID_FAKE_TICK_INCU_YN": "", - }, - ) - else: - body = client.get( - contract["path"], - contract["tr_id"], - { - "AUTH": "", - "EXCD": US_EXCHANGE_CODES[exchange], - "SYMB": broker_symbol, - "NMIN": str(contract["interval_minutes"]), - "PINC": "1", - "NEXT": "", - "NREC": "120", - "FILL": "", - "KEYB": "", - }, - ) - rows = body.get("output2") - if not isinstance(rows, list): - raise ResearchBlockedError("KIS minute response output2 must be an array") - return rows - - -def bars_artifact( - *, - instrument: dict[str, Any], - market: str, - session_date: str, - raw_rows: list[dict[str, Any]], -) -> dict[str, Any]: - contract = MARKET_CONTRACTS[market] - bars = normalized_window_bars( - raw_rows, - market=market, - session_date=session_date, - ) - summary = summarize_bars(bars, market=market) - output = { - "schema": BARS_SCHEMA, - "source": { - "provider": "KIS_OPEN_API", - "endpoint": contract["path"], - "tr_id": contract["tr_id"], - "read_only": True, - }, - "market": market, - "exchange": instrument["exchange"], - "canonical_symbol": instrument["canonical_symbol"], - "broker_symbol": instrument["broker_symbol"], - "session_date": session_date, - "interval_minutes": contract["interval_minutes"], - "window_start": contract["window_start"], - "window_end": contract["window_end"], - "bars": bars, - "bars_hash": shared.sha256_bytes( - shared.canonical_json(bars).encode("utf-8") - ), - "summary": summary, - } - return normalize_bars_artifact(output) - - -def normalize_bars_artifact(value: Any) -> dict[str, Any]: - if not isinstance(value, dict): - raise ResearchBlockedError("bars artifact must be an object") - exact_fields(value, BARS_FIELDS, "bars artifact") - if value["schema"] != BARS_SCHEMA: - raise ResearchBlockedError(f"bars artifact schema must be {BARS_SCHEMA}") - market = str(value["market"]).upper() - if market not in MARKET_CONTRACTS: - raise ResearchBlockedError("bars artifact market must be KR or US") - exchange = str(value["exchange"]).upper() - if exchange not in MARKET_CONTRACTS[market]["exchanges"]: - raise ResearchBlockedError("bars artifact exchange does not match market") - session_date = require_date(value["session_date"], "bars session_date") - source = value["source"] - if not isinstance(source, dict) or source != { - "provider": "KIS_OPEN_API", - "endpoint": MARKET_CONTRACTS[market]["path"], - "tr_id": MARKET_CONTRACTS[market]["tr_id"], - "read_only": True, - }: - raise ResearchBlockedError("bars source contract is invalid") - if value["interval_minutes"] != MARKET_CONTRACTS[market]["interval_minutes"]: - raise ResearchBlockedError("bars interval does not match market") - if value["window_start"] != MARKET_CONTRACTS[market]["window_start"]: - raise ResearchBlockedError("bars window_start does not match market") - if value["window_end"] != MARKET_CONTRACTS[market]["window_end"]: - raise ResearchBlockedError("bars window_end does not match market") - bars = value["bars"] - if not isinstance(bars, list): - raise ResearchBlockedError("bars must be an array") - for index, bar in enumerate(bars): - if not isinstance(bar, dict): - raise ResearchBlockedError(f"bars[{index}] must be an object") - exact_fields(bar, BAR_FIELDS, f"bars[{index}]") - normalized_bars = normalized_window_bars( - [ - ( - { - "stck_bsop_date": session_date.replace("-", ""), - "stck_cntg_hour": bar["time"], - "stck_oprc": bar["open"], - "stck_hgpr": bar["high"], - "stck_lwpr": bar["low"], - "stck_prpr": bar["close"], - "cntg_vol": bar["volume"], - "acml_tr_pbmn": bar["turnover"], - } - if market == "KR" - else { - "xymd": session_date.replace("-", ""), - "xhms": bar["time"], - "open": bar["open"], - "high": bar["high"], - "low": bar["low"], - "last": bar["close"], - "evol": bar["volume"], - "eamt": bar["turnover"], - } - ) - for bar in bars - ], - market=market, - session_date=session_date, - ) - expected_hash = shared.sha256_bytes( - shared.canonical_json(normalized_bars).encode("utf-8") - ) - if value["bars_hash"] != expected_hash: - raise ResearchBlockedError("bars_hash does not match bars") - summary = value["summary"] - if not isinstance(summary, dict): - raise ResearchBlockedError("bars summary must be an object") - exact_fields(summary, SUMMARY_FIELDS, "bars summary") - expected_summary = summarize_bars(normalized_bars, market=market) - if summary != expected_summary: - raise ResearchBlockedError("bars summary does not match bars") - canonical_symbol = str(value["canonical_symbol"]).strip() - broker_symbol = str(value["broker_symbol"]).strip() - if not canonical_symbol or not broker_symbol: - raise ResearchBlockedError("bars symbols must be non-empty") - return { - **value, - "market": market, - "exchange": exchange, - "canonical_symbol": canonical_symbol, - "broker_symbol": broker_symbol, - "session_date": session_date, - "bars": normalized_bars, - "bars_hash": expected_hash, - "summary": expected_summary, - } - - -def record_from_artifact(path: Path, artifact: dict[str, Any]) -> dict[str, Any]: - return { - "market": artifact["market"], - "exchange": artifact["exchange"], - "canonical_symbol": artifact["canonical_symbol"], - "broker_symbol": artifact["broker_symbol"], - "session_date": artifact["session_date"], - "interval_minutes": artifact["interval_minutes"], - "bars_path": str(path.resolve()), - "bars_file_sha256": shared.sha256_file(path), - "bars_hash": artifact["bars_hash"], - **coverage_summary(artifact["bars"], market=artifact["market"]), - **artifact["summary"], - } - - -def cache_path( - output_directory: Path, - instrument: dict[str, Any], -) -> Path: - return ( - output_directory - / "bars" - / instrument["exchange"] - / f"{instrument['canonical_symbol']}.json" - ) - - -def collect( - job: dict[str, Any], - *, - client: shared.KisReadClient | None = None, -) -> dict[str, Any]: - output_directory = job["output_directory"] - output_directory.mkdir(parents=True, exist_ok=True) - if client is None: - key, secret, token = shared.load_api_credentials(job["environment"]) - client = shared.KisReadClient( - environment=job["environment"], - app_key=key, - app_secret=secret, - access_token=token, - interval_ms=job["request_interval_ms"], - ) - records: list[dict[str, Any]] = [] - failures: list[dict[str, str]] = [] - cache_hits = 0 - instruments = job["instruments"] - for index, instrument in enumerate(instruments, start=1): - path = cache_path(output_directory, instrument) - try: - if path.exists(): - artifact = normalize_bars_artifact( - json.loads(path.read_text(encoding="utf-8")) - ) - if ( - artifact["market"] != job["market"] - or artifact["exchange"] != instrument["exchange"] - or artifact["canonical_symbol"] - != instrument["canonical_symbol"] - or artifact["broker_symbol"] != instrument["broker_symbol"] - or artifact["session_date"] != job["session_date"] - ): - raise ResearchBlockedError( - "cached bars identity does not match instrument" - ) - cache_hits += 1 - else: - raw_rows = fetch_raw_rows( - client, - market=job["market"], - exchange=instrument["exchange"], - broker_symbol=instrument["broker_symbol"], - session_date=job["session_date"], - ) - artifact = bars_artifact( - instrument=instrument, - market=job["market"], - session_date=job["session_date"], - raw_rows=raw_rows, - ) - shared.atomic_write_json(path, artifact) - records.append(record_from_artifact(path, artifact)) - except ( - OSError, - json.JSONDecodeError, - shared.EodBlockedError, - ResearchBlockedError, - ) as exc: - failures.append( - { - "market": job["market"], - "exchange": instrument["exchange"], - "canonical_symbol": instrument["canonical_symbol"], - "broker_symbol": instrument["broker_symbol"], - "reason": str(exc), - } - ) - if index % 100 == 0 or index == len(instruments): - print( - shared.canonical_json( - { - "progress": index, - "total": len(instruments), - "ready": len(records), - "blocked": len(failures), - "requests": client.request_count, - } - ), - file=sys.stderr, - flush=True, - ) - records.sort(key=lambda item: (item["exchange"], item["canonical_symbol"])) - failures.sort( - key=lambda item: (item["exchange"], item["canonical_symbol"]) - ) - coverage_counts = { - status: sum( - record["coverage_status"] == status for record in records - ) - for status in ( - "COMPLETE_GRID", - "EXACT_ENDPOINTS_SPARSE", - "NEAR_ENDPOINTS", - "PARTIAL_WINDOW", - ) - } - without_hash = { - "schema": SNAPSHOT_SCHEMA, - "source_skill": "quant-stock-technical", - "source_contract": { - "provider": "KIS_OPEN_API", - "read_only": True, - "market": job["market"], - "tr_id": MARKET_CONTRACTS[job["market"]]["tr_id"], - "endpoint": MARKET_CONTRACTS[job["market"]]["path"], - }, - "manifest_path": job["manifest_path"], - "manifest_hash": job["manifest"]["manifest_hash"], - "analysis_date": job["manifest"]["analysis_date"], - "session_date": job["session_date"], - "market": job["market"], - "window_start": MARKET_CONTRACTS[job["market"]]["window_start"], - "window_end": MARKET_CONTRACTS[job["market"]]["window_end"], - "interval_minutes": MARKET_CONTRACTS[job["market"]][ - "interval_minutes" - ], - "instrument_count": len(instruments), - "ready_count": len(records), - "blocked_count": len(failures), - "cache_hit_count": cache_hits, - "request_count": client.request_count, - "retry_count": client.retry_count, - "coverage_counts": coverage_counts, - "records": records, - "failures": failures, - "api_mutation_count": 0, - "live_enabled": False, - } - snapshot = { - **without_hash, - "snapshot_hash": shared.sha256_bytes( - shared.canonical_json(without_hash).encode("utf-8") - ), - } - shared.atomic_write_json(output_directory / "open1h-snapshot.json", snapshot) - return snapshot - - -def synthetic_rows( - *, - market: str, - session_date: str, -) -> list[dict[str, str]]: - contract = MARKET_CONTRACTS[market] - start_hour = 9 - start_minute = 0 if market == "KR" else 30 - rows: list[dict[str, str]] = [] - for index in range(contract["expected_bars"]): - total_minutes = start_hour * 60 + start_minute + ( - index * contract["interval_minutes"] - ) - stamp = f"{total_minutes // 60:02d}{total_minutes % 60:02d}00" - opening = Decimal("100") + Decimal(index) / Decimal("10") - row = { - "open": decimal_text(opening), - "high": decimal_text(opening + Decimal("1")), - "low": decimal_text(opening - Decimal("1")), - "close": decimal_text(opening + Decimal("0.5")), - "volume": "10", - "turnover": str(1000 + index), - } - if market == "KR": - rows.append( - { - "stck_bsop_date": session_date.replace("-", ""), - "stck_cntg_hour": stamp, - "stck_oprc": row["open"], - "stck_hgpr": row["high"], - "stck_lwpr": row["low"], - "stck_prpr": row["close"], - "cntg_vol": row["volume"], - "acml_tr_pbmn": row["turnover"], - } - ) - else: - rows.append( - { - "xymd": session_date.replace("-", ""), - "xhms": stamp, - "open": row["open"], - "high": row["high"], - "low": row["low"], - "last": row["close"], - "evol": row["volume"], - "eamt": row["turnover"], - } - ) - return list(reversed(rows)) - - -def self_test() -> None: - for market in ("KR", "US"): - exchange = MARKET_CONTRACTS[market]["exchanges"][0] - instrument = { - "exchange": exchange, - "canonical_symbol": "005930" if market == "KR" else "TEST", - "broker_symbol": "005930" if market == "KR" else "TEST", - } - artifact = bars_artifact( - instrument=instrument, - market=market, - session_date="2026-07-28", - raw_rows=synthetic_rows( - market=market, - session_date="2026-07-28", - ), - ) - assert artifact["summary"]["bar_count"] == ( - MARKET_CONTRACTS[market]["expected_bars"] - ) - assert Decimal(artifact["summary"]["return_60m_bps"]) > 0 - with tempfile.TemporaryDirectory( - prefix="qta-open1h-research-" - ) as temporary: - path = Path(temporary) / "bars.json" - shared.atomic_write_json(path, artifact) - loaded = normalize_bars_artifact( - json.loads(path.read_text(encoding="utf-8")) - ) - assert loaded == artifact - print( - shared.canonical_json( - { - "self_test": "PASS", - "schema": SNAPSHOT_SCHEMA, - "markets": ["KR", "US"], - "api_mutation_count": 0, - "live_enabled": False, - } - ) - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--self-test", action="store_true") - subparsers = parser.add_subparsers(dest="command") - collect_parser = subparsers.add_parser("collect") - collect_parser.add_argument("--job", required=True) - return parser.parse_args() - - -def main() -> int: - args = parse_args() - try: - if args.self_test: - self_test() - return 0 - if args.command != "collect": - raise ResearchBlockedError("use collect --job or --self-test") - job_path = require_absolute_path(args.job, "job path") - result = collect(load_job(job_path)) - except ( - ResearchBlockedError, - shared.EodBlockedError, - screen.ScreenBlockedError, - OSError, - json.JSONDecodeError, - ) as exc: - print( - shared.canonical_json( - { - "status": "BLOCKED", - "reason": str(exc), - "api_mutation_count": 0, - "live_enabled": False, - } - ), - file=sys.stderr, - ) - return 2 - print(shared.canonical_json(result)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/plugins/quant-stock-technical/skills/quant-stock-technical/scripts/evaluate_open1h_research.py b/plugins/quant-stock-technical/skills/quant-stock-technical/scripts/evaluate_open1h_research.py deleted file mode 100644 index 88bf812..0000000 --- a/plugins/quant-stock-technical/skills/quant-stock-technical/scripts/evaluate_open1h_research.py +++ /dev/null @@ -1,721 +0,0 @@ -#!/usr/bin/env python3 -"""Evaluate frozen QTA scores against completed first-60-minute outcomes.""" - -from __future__ import annotations - -import argparse -import csv -import json -import math -import statistics -import sys -import tempfile -from decimal import Decimal, InvalidOperation, ROUND_CEILING, ROUND_HALF_EVEN -from pathlib import Path -from typing import Any, Iterable - -import collect_open1h_research as intraday -import fetch_kis_kr_eod as shared -import screen_universe as screen - - -JOB_SCHEMA = "qta-open1h-evaluation-job/v1" -DATASET_SCHEMA = "qta-open1h-evaluation/v1" -JOB_FIELDS = { - "schema", - "screen_path", - "snapshot_path", - "output_path", - "top_fraction", -} -SNAPSHOT_FIELDS = { - "schema", - "source_skill", - "source_contract", - "manifest_path", - "manifest_hash", - "analysis_date", - "session_date", - "market", - "window_start", - "window_end", - "interval_minutes", - "instrument_count", - "ready_count", - "blocked_count", - "cache_hit_count", - "request_count", - "retry_count", - "coverage_counts", - "records", - "failures", - "api_mutation_count", - "live_enabled", - "snapshot_hash", -} - - -class EvaluationBlockedError(ValueError): - """Raised when a QTA/opening-hour comparison would be incomplete.""" - - -def exact_fields(value: dict[str, Any], expected: set[str], label: str) -> None: - if set(value) != expected: - raise EvaluationBlockedError( - f"{label} fields mismatch; missing={sorted(expected - set(value))}, " - f"extra={sorted(set(value) - expected)}" - ) - - -def decimal_value(value: Any, field: str) -> Decimal: - if isinstance(value, bool): - raise EvaluationBlockedError(f"{field} must be decimal") - try: - parsed = Decimal(str(value)) - except (InvalidOperation, ValueError) as exc: - raise EvaluationBlockedError(f"{field} must be decimal") from exc - if not parsed.is_finite(): - raise EvaluationBlockedError(f"{field} must be finite") - return parsed - - -def positive_fraction(value: Any, field: str) -> Decimal: - parsed = decimal_value(value, field) - if parsed <= 0 or parsed >= 1: - raise EvaluationBlockedError(f"{field} must be greater than 0 and below 1") - return parsed - - -def absolute_file(value: Any, field: str, *, must_exist: bool) -> Path: - path = Path(str(value)).expanduser() - if not path.is_absolute(): - raise EvaluationBlockedError(f"{field} must be absolute") - if must_exist and (not path.is_file() or path.is_symlink()): - raise EvaluationBlockedError( - f"{field} must be a regular non-symlink file" - ) - return path - - -def load_object(path: Path, label: str) -> dict[str, Any]: - try: - value = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as exc: - raise EvaluationBlockedError(f"{label} must be readable JSON") from exc - if not isinstance(value, dict): - raise EvaluationBlockedError(f"{label} must be one JSON object") - return value - - -def load_job(path: Path) -> dict[str, Any]: - raw = load_object(path, "evaluation job") - exact_fields(raw, JOB_FIELDS, "evaluation job") - if raw["schema"] != JOB_SCHEMA: - raise EvaluationBlockedError(f"job.schema must be {JOB_SCHEMA}") - screen_path = absolute_file(raw["screen_path"], "screen_path", must_exist=True) - snapshot_path = absolute_file( - raw["snapshot_path"], "snapshot_path", must_exist=True - ) - output_path = absolute_file(raw["output_path"], "output_path", must_exist=False) - return { - "schema": JOB_SCHEMA, - "screen_path": screen_path, - "snapshot_path": snapshot_path, - "output_path": output_path, - "top_fraction": positive_fraction(raw["top_fraction"], "top_fraction"), - } - - -def normalized_screen(path: Path) -> dict[str, Any]: - value = load_object(path, "screen") - supplied_hash = value.get("screen_hash") - if not isinstance(supplied_hash, str): - raise EvaluationBlockedError("screen_hash is required") - without_hash = {key: item for key, item in value.items() if key != "screen_hash"} - if screen.sha256_json(without_hash) != supplied_hash: - raise EvaluationBlockedError("screen_hash does not match screen") - if ( - value.get("schema") != screen.SCREEN_SCHEMA_V2 - or value.get("source_skill") != "quant-stock-technical" - or value.get("method_version") not in {"qta-1.0.0", "qta-2.0.0"} - ): - raise EvaluationBlockedError( - "evaluation requires a qta-screen/v2 QTA 1.0.0 or 2.0.0 artifact" - ) - decisions = value.get("decisions") - if not isinstance(decisions, list) or not decisions: - raise EvaluationBlockedError("screen decisions must be a non-empty array") - return value - - -def normalized_snapshot(path: Path) -> dict[str, Any]: - value = load_object(path, "opening-hour snapshot") - exact_fields(value, SNAPSHOT_FIELDS, "opening-hour snapshot") - if value["schema"] != intraday.SNAPSHOT_SCHEMA: - raise EvaluationBlockedError( - f"snapshot.schema must be {intraday.SNAPSHOT_SCHEMA}" - ) - if value["api_mutation_count"] != 0 or value["live_enabled"] is not False: - raise EvaluationBlockedError("research snapshot must be read-only") - supplied_hash = value["snapshot_hash"] - without_hash = { - key: item for key, item in value.items() if key != "snapshot_hash" - } - expected_hash = shared.sha256_bytes( - shared.canonical_json(without_hash).encode("utf-8") - ) - if supplied_hash != expected_hash: - raise EvaluationBlockedError("snapshot_hash does not match snapshot") - if ( - value["instrument_count"] - != value["ready_count"] + value["blocked_count"] - ): - raise EvaluationBlockedError("snapshot counts do not reconcile") - if value["ready_count"] != len(value["records"]): - raise EvaluationBlockedError("snapshot ready_count does not match records") - if value["blocked_count"] != len(value["failures"]): - raise EvaluationBlockedError("snapshot blocked_count does not match failures") - return value - - -def eod_features( - ticker_path: Path, - *, - analysis_date: str, -) -> dict[str, str]: - try: - with ticker_path.open("r", encoding="utf-8-sig", newline="") as handle: - rows = list(csv.DictReader(handle)) - except OSError as exc: - raise EvaluationBlockedError(f"cannot read ticker CSV: {ticker_path}") from exc - selected = [row for row in rows if str(row.get("date")) <= analysis_date] - if not selected or str(selected[-1].get("date")) != analysis_date: - raise EvaluationBlockedError( - f"ticker CSV has no completed {analysis_date} cutoff" - ) - if len(selected) < 253: - raise EvaluationBlockedError("ticker CSV needs at least 253 cutoff rows") - closes = [ - decimal_value(row.get("close"), "ticker close") for row in selected - ] - volumes = [ - decimal_value(row.get("volume"), "ticker volume") for row in selected - ] - dollar_turnover = [ - closes[index] * volumes[index] - for index in range(max(0, len(selected) - 20), len(selected)) - ] - return { - "previous_close": format(closes[-1], "f"), - "return_5_bps": intraday.bps(closes[-1], closes[-6]), - "return_20_bps": intraday.bps(closes[-1], closes[-21]), - "return_63_bps": intraday.bps(closes[-1], closes[-64]), - "median_20d_dollar_turnover": format( - statistics.median(dollar_turnover), "f" - ), - } - - -def first_bar_close_cross( - bars: list[dict[str, str]], - entry: Decimal, -) -> str | None: - previous: Decimal | None = None - for bar in bars: - current = Decimal(bar["close"]) - if previous is not None and previous < entry <= current: - return bar["time"] - previous = current - return None - - -def combine_rows( - *, - screen_value: dict[str, Any], - snapshot: dict[str, Any], -) -> tuple[list[dict[str, Any]], list[dict[str, str]]]: - if screen_value["manifest_hash"] != snapshot["manifest_hash"]: - raise EvaluationBlockedError("screen and snapshot manifest hashes differ") - if screen_value["analysis_date"] != snapshot["analysis_date"]: - raise EvaluationBlockedError("screen and snapshot analysis dates differ") - decision_by_key: dict[tuple[str, str], dict[str, Any]] = {} - for decision in screen_value["decisions"]: - if not isinstance(decision, dict): - raise EvaluationBlockedError("screen decision must be an object") - key = (str(decision.get("exchange")), str(decision.get("canonical_symbol"))) - if key in decision_by_key: - raise EvaluationBlockedError(f"duplicate screen decision: {key}") - decision_by_key[key] = decision - rows: list[dict[str, Any]] = [] - blocked: list[dict[str, str]] = [] - for record in snapshot["records"]: - if not isinstance(record, dict): - raise EvaluationBlockedError("snapshot record must be an object") - key = (str(record.get("exchange")), str(record.get("canonical_symbol"))) - decision = decision_by_key.get(key) - if decision is None: - raise EvaluationBlockedError(f"snapshot record has no screen decision: {key}") - qta = decision.get("qta") - if ( - not isinstance(qta, dict) - or qta.get("calculation_status") != "READY" - or qta.get("method_version") != screen_value["method_version"] - ): - blocked.append( - { - "exchange": key[0], - "canonical_symbol": key[1], - "reason": str( - qta.get("reason", "QTA calculation is not READY") - if isinstance(qta, dict) - else "QTA payload is missing" - ), - } - ) - continue - bars_path = absolute_file( - record.get("bars_path"), "record.bars_path", must_exist=True - ) - if shared.sha256_file(bars_path) != record.get("bars_file_sha256"): - raise EvaluationBlockedError(f"bars file hash mismatch: {bars_path}") - artifact = intraday.normalize_bars_artifact( - load_object(bars_path, "bars artifact") - ) - if artifact["bars_hash"] != record.get("bars_hash"): - raise EvaluationBlockedError(f"bars hash mismatch: {bars_path}") - instrument = decision.get("instrument") - if not isinstance(instrument, dict): - raise EvaluationBlockedError("screen decision instrument is missing") - ticker_path = absolute_file( - instrument.get("ticker_csv"), "ticker_csv", must_exist=True - ) - if shared.sha256_file(ticker_path) != instrument.get("ticker_csv_sha256"): - raise EvaluationBlockedError(f"ticker CSV hash mismatch: {ticker_path}") - eod = eod_features( - ticker_path, - analysis_date=screen_value["analysis_date"], - ) - entry = decimal_value(qta["entry_price"], "QTA entry price") - start_open = Decimal(record["start_open"]) - end_close = Decimal(record["end_close"]) - cross_time = first_bar_close_cross(artifact["bars"], entry) - rows.append( - { - "market": decision["market"], - "exchange": decision["exchange"], - "canonical_symbol": decision["canonical_symbol"], - "broker_symbol": instrument["broker_symbol"], - "instrument_type": instrument["instrument_type"], - "analysis_date": screen_value["analysis_date"], - "session_date": snapshot["session_date"], - "qta_method_version": qta["method_version"], - "qta_total_score": format( - decimal_value(qta["total_score"], "QTA total score"), "f" - ), - "qta_short_score": format( - decimal_value(qta["short"]["score"], "QTA short score"), "f" - ), - "qta_medium_score": format( - decimal_value(qta["medium"]["score"], "QTA medium score"), "f" - ), - "qta_long_score": format( - decimal_value(qta["long"]["score"], "QTA long score"), "f" - ), - "qta_risk_score": format( - decimal_value(qta["risk"]["score"], "QTA risk score"), "f" - ), - "qta_entry_price": format(entry, "f"), - "qta_stop_price": format( - decimal_value(qta["stop_price"], "QTA stop price"), "f" - ), - "qta_take_profit_price": format( - decimal_value( - qta["take_profit_price"], "QTA take-profit price" - ), - "f", - ), - **eod, - "opening_gap_bps": intraday.bps( - start_open, Decimal(eod["previous_close"]) - ), - "coverage_status": record["coverage_status"], - "first_bar_time": record["first_bar_time"], - "last_bar_time": record["last_bar_time"], - "bar_count": record["bar_count"], - "expected_bar_count": record["expected_bar_count"], - "missing_bar_count": record["missing_bar_count"], - "coverage_fraction": record["coverage_fraction"], - "observed_span_minutes": record["observed_span_minutes"], - "return_60m_bps": record["return_60m_bps"], - "maximum_excursion_bps": record["maximum_excursion_bps"], - "minimum_excursion_bps": record["minimum_excursion_bps"], - "window_volume": record["window_volume"], - "window_turnover": record["window_turnover"], - "entry_distance_from_previous_close_bps": intraday.bps( - entry, Decimal(eod["previous_close"]) - ), - "entry_touched_by_minute_high": ( - Decimal(record["window_high"]) >= entry - ), - "entry_crossed_by_bar_close": cross_time is not None, - "entry_cross_bar_time": cross_time, - "end_return_from_entry_bps": intraday.bps(end_close, entry), - } - ) - rows.sort(key=lambda item: (item["exchange"], item["canonical_symbol"])) - blocked.sort(key=lambda item: (item["exchange"], item["canonical_symbol"])) - return rows, blocked - - -def average_ranks(values: list[Decimal]) -> list[float]: - order = sorted(range(len(values)), key=lambda index: (values[index], index)) - ranks = [0.0] * len(values) - cursor = 0 - while cursor < len(order): - end = cursor + 1 - while end < len(order) and values[order[end]] == values[order[cursor]]: - end += 1 - average = (cursor + 1 + end) / 2.0 - for position in range(cursor, end): - ranks[order[position]] = average - cursor = end - return ranks - - -def pearson(left: Iterable[float], right: Iterable[float]) -> float | None: - x = list(left) - y = list(right) - if len(x) != len(y) or len(x) < 2: - return None - x_mean = sum(x) / len(x) - y_mean = sum(y) / len(y) - numerator = sum((a - x_mean) * (b - y_mean) for a, b in zip(x, y)) - x_scale = math.sqrt(sum((a - x_mean) ** 2 for a in x)) - y_scale = math.sqrt(sum((b - y_mean) ** 2 for b in y)) - if x_scale == 0 or y_scale == 0: - return None - return numerator / (x_scale * y_scale) - - -def rounded_metric(value: float | None) -> str | None: - if value is None: - return None - return format( - Decimal(str(value)).quantize( - Decimal("0.000001"), rounding=ROUND_HALF_EVEN - ), - "f", - ) - - -def group_summary(rows: list[dict[str, Any]]) -> dict[str, Any]: - returns = [Decimal(row["return_60m_bps"]) for row in rows] - touched = sum(bool(row["entry_touched_by_minute_high"]) for row in rows) - crossed = sum(bool(row["entry_crossed_by_bar_close"]) for row in rows) - positive = sum(value > 0 for value in returns) - return { - "count": len(rows), - "mean_return_60m_bps": ( - format(sum(returns, Decimal(0)) / len(returns), "f") - if returns - else None - ), - "median_return_60m_bps": ( - format(statistics.median(returns), "f") if returns else None - ), - "positive_fraction": ( - format(Decimal(positive) / len(rows), "f") if rows else None - ), - "entry_touch_fraction": ( - format(Decimal(touched) / len(rows), "f") if rows else None - ), - "bar_close_cross_fraction": ( - format(Decimal(crossed) / len(rows), "f") if rows else None - ), - } - - -def exchange_diagnostics( - rows: list[dict[str, Any]], - *, - top_fraction: Decimal, -) -> dict[str, Any]: - if not rows: - raise EvaluationBlockedError("exchange diagnostics require rows") - qta_scores = [Decimal(row["qta_total_score"]) for row in rows] - returns = [Decimal(row["return_60m_bps"]) for row in rows] - pearson_value = pearson( - [float(value) for value in qta_scores], - [float(value) for value in returns], - ) - spearman_value = pearson( - average_ranks(qta_scores), - average_ranks(returns), - ) - top_count = max( - 1, - int( - (Decimal(len(rows)) * top_fraction).to_integral_value( - rounding=ROUND_CEILING - ) - ), - ) - by_outcome = sorted( - rows, - key=lambda item: ( - -Decimal(item["return_60m_bps"]), - item["canonical_symbol"], - ), - ) - by_score = sorted( - rows, - key=lambda item: ( - -Decimal(item["qta_total_score"]), - item["canonical_symbol"], - ), - ) - winner_keys = { - row["canonical_symbol"] for row in by_outcome[:top_count] - } - score_keys = {row["canonical_symbol"] for row in by_score[:top_count]} - overlap = len(winner_keys & score_keys) - baseline = Decimal(top_count) / Decimal(len(rows)) - precision = Decimal(overlap) / Decimal(top_count) - lift = precision / baseline - deciles: list[dict[str, Any]] = [] - for index in range(10): - start = len(by_score) * index // 10 - end = len(by_score) * (index + 1) // 10 - bucket = by_score[start:end] - deciles.append( - { - "qta_score_decile": index + 1, - **group_summary(bucket), - } - ) - - def compact(row: dict[str, Any]) -> dict[str, Any]: - return { - key: row[key] - for key in ( - "canonical_symbol", - "qta_total_score", - "qta_short_score", - "qta_medium_score", - "qta_long_score", - "qta_risk_score", - "return_5_bps", - "return_20_bps", - "median_20d_dollar_turnover", - "opening_gap_bps", - "coverage_status", - "first_bar_time", - "last_bar_time", - "bar_count", - "return_60m_bps", - "maximum_excursion_bps", - "minimum_excursion_bps", - "entry_touched_by_minute_high", - "entry_crossed_by_bar_close", - "end_return_from_entry_bps", - ) - } - - return { - "count": len(rows), - "top_fraction": format(top_fraction, "f"), - "top_count": top_count, - "pearson_qta_total_vs_return_60m": rounded_metric(pearson_value), - "spearman_qta_total_vs_return_60m": rounded_metric(spearman_value), - "top_outcome_overlap_count": overlap, - "top_outcome_precision": format(precision, "f"), - "top_outcome_recall": format(precision, "f"), - "top_outcome_lift": format(lift, "f"), - "overall": group_summary(rows), - "qta_score_deciles": deciles, - "top_20_outcomes": [compact(row) for row in by_outcome[:20]], - "bottom_20_outcomes": [compact(row) for row in by_outcome[-20:]], - "top_20_qta_scores": [compact(row) for row in by_score[:20]], - } - - -def evaluate(job: dict[str, Any]) -> dict[str, Any]: - screen_value = normalized_screen(job["screen_path"]) - snapshot = normalized_snapshot(job["snapshot_path"]) - rows, qta_blocked = combine_rows( - screen_value=screen_value, - snapshot=snapshot, - ) - if not rows: - raise EvaluationBlockedError("no READY QTA/opening-hour joined rows") - analysis_rows = [ - row - for row in rows - if row["coverage_status"] - in { - "COMPLETE_GRID", - "EXACT_ENDPOINTS_SPARSE", - "NEAR_ENDPOINTS", - } - ] - if not analysis_rows: - raise EvaluationBlockedError( - "no joined rows have complete or near-complete opening-hour endpoints" - ) - diagnostics = { - exchange: exchange_diagnostics( - [row for row in analysis_rows if row["exchange"] == exchange], - top_fraction=job["top_fraction"], - ) - for exchange in intraday.MARKET_CONTRACTS[snapshot["market"]]["exchanges"] - if any(row["exchange"] == exchange for row in analysis_rows) - } - without_hash = { - "schema": DATASET_SCHEMA, - "source_skill": "quant-stock-technical", - "baseline_method_version": screen_value["method_version"], - "target_definition": ( - "last observed close divided by first observed open; no interpolation; " - "COMPLETE_GRID, EXACT_ENDPOINTS_SPARSE, and NEAR_ENDPOINTS only" - ), - "market": snapshot["market"], - "analysis_date": snapshot["analysis_date"], - "session_date": snapshot["session_date"], - "screen_path": str(job["screen_path"]), - "screen_hash": screen_value["screen_hash"], - "snapshot_path": str(job["snapshot_path"]), - "snapshot_hash": snapshot["snapshot_hash"], - "top_fraction": format(job["top_fraction"], "f"), - "joined_count": len(rows), - "analysis_eligible_count": len(analysis_rows), - "coverage_counts": snapshot["coverage_counts"], - "intraday_blocked_count": snapshot["blocked_count"], - "qta_blocked_after_join_count": len(qta_blocked), - "diagnostics_by_exchange": diagnostics, - "rows": rows, - "qta_blocked_after_join": qta_blocked, - "api_mutation_count": 0, - "live_enabled": False, - } - output = { - **without_hash, - "evaluation_hash": shared.sha256_bytes( - shared.canonical_json(without_hash).encode("utf-8") - ), - } - shared.atomic_write_json(job["output_path"], output) - return output - - -def self_test() -> None: - sample = [ - { - "canonical_symbol": f"T{index}", - "qta_total_score": str(index), - "qta_short_score": str(index), - "qta_medium_score": str(index), - "qta_long_score": str(index), - "qta_risk_score": str(100 - index), - "return_5_bps": str(index), - "return_20_bps": str(index), - "median_20d_dollar_turnover": str(index * 1000000), - "opening_gap_bps": "0", - "coverage_status": "COMPLETE_GRID", - "first_bar_time": "093000", - "last_bar_time": "103000", - "bar_count": 13, - "return_60m_bps": str(index * 2), - "maximum_excursion_bps": str(index * 3), - "minimum_excursion_bps": str(-index), - "entry_touched_by_minute_high": index % 2 == 0, - "entry_crossed_by_bar_close": index % 3 == 0, - "end_return_from_entry_bps": str(index), - } - for index in range(1, 21) - ] - diagnostics = exchange_diagnostics(sample, top_fraction=Decimal("0.1")) - assert diagnostics["spearman_qta_total_vs_return_60m"] == "1.000000" - assert diagnostics["top_outcome_overlap_count"] == 2 - bars = intraday.synthetic_rows(market="US", session_date="2026-07-28") - artifact = intraday.bars_artifact( - instrument={ - "exchange": "NASDAQ", - "canonical_symbol": "TEST", - "broker_symbol": "TEST", - }, - market="US", - session_date="2026-07-28", - raw_rows=bars, - ) - assert first_bar_close_cross(artifact["bars"], Decimal("101.2")) is not None - with tempfile.TemporaryDirectory(prefix="qta-open1h-evaluation-") as temporary: - output = Path(temporary) / "result.json" - shared.atomic_write_json(output, {"self_test": "PASS"}) - assert load_object(output, "self-test")["self_test"] == "PASS" - print( - shared.canonical_json( - { - "self_test": "PASS", - "schema": DATASET_SCHEMA, - "api_mutation_count": 0, - "live_enabled": False, - } - ) - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--self-test", action="store_true") - subparsers = parser.add_subparsers(dest="command") - evaluate_parser = subparsers.add_parser("evaluate") - evaluate_parser.add_argument("--job", required=True) - return parser.parse_args() - - -def main() -> int: - args = parse_args() - try: - if args.self_test: - self_test() - return 0 - if args.command != "evaluate": - raise EvaluationBlockedError("use evaluate --job or --self-test") - job_path = absolute_file(args.job, "job path", must_exist=True) - result = evaluate(load_job(job_path)) - except ( - EvaluationBlockedError, - intraday.ResearchBlockedError, - screen.ScreenBlockedError, - OSError, - json.JSONDecodeError, - ) as exc: - print( - shared.canonical_json( - { - "status": "BLOCKED", - "reason": str(exc), - "api_mutation_count": 0, - "live_enabled": False, - } - ), - file=sys.stderr, - ) - return 2 - print( - shared.canonical_json( - { - "status": "READY", - "market": result["market"], - "joined_count": result["joined_count"], - "evaluation_hash": result["evaluation_hash"], - "api_mutation_count": 0, - "live_enabled": False, - } - ) - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/plugins/quant-stock-technical/skills/quant-stock-technical/scripts/fetch_kis_kr_eod.py b/plugins/quant-stock-technical/skills/quant-stock-technical/scripts/fetch_kis_kr_eod.py deleted file mode 100644 index 12181da..0000000 --- a/plugins/quant-stock-technical/skills/quant-stock-technical/scripts/fetch_kis_kr_eod.py +++ /dev/null @@ -1,1253 +0,0 @@ -#!/usr/bin/env python3 -"""Build a resumable, source-hashed Korean adjusted-EOD bundle from KIS.""" - -from __future__ import annotations - -import argparse -import csv -import hashlib -import json -import os -import re -import stat -import sys -import tempfile -import time -import zipfile -from datetime import date, datetime, timedelta, timezone -from decimal import Decimal, InvalidOperation -from pathlib import Path -from typing import Any, Callable, Mapping, Optional -from urllib.error import HTTPError, URLError -from urllib.parse import urlencode -from urllib.request import Request, urlopen - -import build_universe_manifest as universe - - -JOB_SCHEMA = "qta-kis-kr-eod-job/v1" -RECEIPT_SCHEMA = "qta-kis-kr-eod-receipt/v1" -SNAPSHOT_SCHEMA = "qta-kr-source-snapshot/v1" -CATALOG_FIELDS = universe.EOD_CATALOG_FIELDS -KR_EXCHANGES = ("KOSPI", "KOSDAQ") -JOB_FIELDS = { - "schema", - "as_of", - "analysis_date", - "environment", - "output_directory", - "history_start_date", - "minimum_sessions", - "request_interval_ms", - "official_sources", - "broker_sources", - "catalog_coverage_contract", - "base_eod_catalog", -} -KIND_URL = ( - "https://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13" -) -KIS_MASTER_URLS = { - "KOSPI": ("https://new.real.download.dws.co.kr/common/master/kospi_code.mst.zip"), - "KOSDAQ": ("https://new.real.download.dws.co.kr/common/master/kosdaq_code.mst.zip"), -} -KIS_MASTER_NAMES = { - "KOSPI": "kospi_code.mst", - "KOSDAQ": "kosdaq_code.mst", -} -BASE_URLS = { - "live": "https://openapi.koreainvestment.com:9443", - "paper": "https://openapivts.koreainvestment.com:29443", -} -TOKEN_PATH = "/oauth2/tokenP" -STOCK_PATH = "/uapi/domestic-stock/v1/quotations/inquire-daily-itemchartprice" -INDEX_PATH = "/uapi/domestic-stock/v1/quotations/inquire-daily-indexchartprice" -STOCK_TR_ID = "FHKST03010100" -INDEX_TR_ID = "FHKUP03500100" -INDEX_CODES = {"KOSPI": "0001", "KOSDAQ": "1001"} -BENCHMARK_IDS = { - "KOSPI": "KOSPI_COMPOSITE", - "KOSDAQ": "KOSDAQ_COMPOSITE", -} -DEFAULT_SECRETS_PATH = Path("~/.config/mrcha-skills/secrets.env").expanduser() -DOTENV_KEY = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") - - -class EodBlockedError(ValueError): - """Raised when an EOD bundle cannot be completed without guessing.""" - - -def canonical_json(value: Any) -> str: - return json.dumps( - value, - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - allow_nan=False, - ) - - -def sha256_bytes(value: bytes) -> str: - return hashlib.sha256(value).hexdigest() - - -def sha256_file(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def atomic_write(path: Path, payload: bytes) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - descriptor, temporary = tempfile.mkstemp( - prefix=f".{path.name}.", suffix=".tmp", dir=str(path.parent) - ) - temporary_path = Path(temporary) - try: - with os.fdopen(descriptor, "wb") as handle: - handle.write(payload) - handle.flush() - os.fsync(handle.fileno()) - os.replace(temporary_path, path) - finally: - if temporary_path.exists(): - temporary_path.unlink() - - -def atomic_write_json(path: Path, value: Any) -> None: - atomic_write( - path, - (json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode( - "utf-8" - ), - ) - - -def require_exact_fields( - value: Mapping[str, Any], expected: set[str], label: str -) -> None: - if set(value) != expected: - raise EodBlockedError( - f"{label} fields mismatch; " - f"missing={sorted(expected - set(value))}, " - f"extra={sorted(set(value) - expected)}" - ) - - -def require_iso_date(value: Any, label: str) -> str: - if not isinstance(value, str): - raise EodBlockedError(f"{label} must be YYYY-MM-DD") - try: - date.fromisoformat(value) - except ValueError as exc: - raise EodBlockedError(f"{label} must be YYYY-MM-DD") from exc - return value - - -def require_positive_integer(value: Any, label: str) -> int: - if isinstance(value, bool) or not isinstance(value, int) or value <= 0: - raise EodBlockedError(f"{label} must be a positive integer") - return value - - -def normalize_job(raw: Any, job_directory: Path) -> dict[str, Any]: - if not isinstance(raw, dict): - raise EodBlockedError("job must be one JSON object") - require_exact_fields(raw, JOB_FIELDS, "job") - if raw["schema"] != JOB_SCHEMA: - raise EodBlockedError(f"job.schema must be {JOB_SCHEMA}") - as_of = require_iso_date(raw["as_of"], "job.as_of") - analysis_date = require_iso_date(raw["analysis_date"], "job.analysis_date") - history_start = require_iso_date( - raw["history_start_date"], "job.history_start_date" - ) - if history_start >= analysis_date: - raise EodBlockedError("history_start_date must be before analysis_date") - if as_of < analysis_date: - raise EodBlockedError("as_of cannot be before analysis_date") - environment = str(raw["environment"]).lower() - if environment not in BASE_URLS: - raise EodBlockedError("environment must be live or paper") - output_directory = Path(str(raw["output_directory"])).expanduser() - if not output_directory.is_absolute(): - output_directory = (job_directory / output_directory).resolve() - minimum_sessions = require_positive_integer( - raw["minimum_sessions"], "job.minimum_sessions" - ) - if minimum_sessions < 756: - raise EodBlockedError("minimum_sessions must be at least 756") - request_interval_ms = require_positive_integer( - raw["request_interval_ms"], "job.request_interval_ms" - ) - if environment == "paper" and request_interval_ms < 1000: - raise EodBlockedError("paper request_interval_ms must be at least 1000") - if environment == "live" and request_interval_ms < 100: - raise EodBlockedError("live request_interval_ms must be at least 100") - - def normalize_sources(role: str) -> list[dict[str, Any]]: - sources = raw[role] - if not isinstance(sources, list): - raise EodBlockedError(f"job.{role} must be an array") - normalized: list[dict[str, Any]] = [] - seen: set[str] = set() - for index, source in enumerate(sources): - if not isinstance(source, dict): - raise EodBlockedError(f"job.{role}[{index}] must be an object") - item = universe.normalize_source_descriptor( - source, - ("OFFICIAL_MASTER" if role == "official_sources" else "BROKER_MASTER"), - as_of, - job_directory, - index, - ) - if item["exchange"] in seen: - raise EodBlockedError(f"duplicate {role} exchange {item['exchange']}") - seen.add(item["exchange"]) - normalized.append(item) - missing = set(KR_EXCHANGES) - seen - if missing: - raise EodBlockedError(f"job.{role} missing KR exchanges {sorted(missing)}") - return sorted(normalized, key=lambda item: item["exchange"]) - - base_catalog = str(raw["base_eod_catalog"] or "").strip() - if base_catalog: - base_path = Path(base_catalog).expanduser() - if not base_path.is_absolute(): - base_path = (job_directory / base_path).resolve() - if not base_path.is_file(): - raise EodBlockedError(f"base_eod_catalog is not a file: {base_path}") - base_catalog = str(base_path) - - coverage = universe.normalize_catalog_coverage_contract( - raw["catalog_coverage_contract"] - ) - return { - "schema": JOB_SCHEMA, - "as_of": as_of, - "analysis_date": analysis_date, - "environment": environment, - "output_directory": str(output_directory), - "history_start_date": history_start, - "minimum_sessions": minimum_sessions, - "request_interval_ms": request_interval_ms, - "official_sources": normalize_sources("official_sources"), - "broker_sources": normalize_sources("broker_sources"), - "catalog_coverage_contract": coverage, - "base_eod_catalog": base_catalog, - } - - -def build_spec_sources( - sources: list[dict[str, Any]], -) -> list[dict[str, Any]]: - """Project normalized records back to the exact build-spec input schema.""" - return [ - { - field: source[field] - for field in sorted(universe.SOURCE_FIELDS) - } - for source in sources - ] - - -def inspect_secrets_file(path: Path) -> bool: - try: - metadata = path.lstat() - except FileNotFoundError: - return False - if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode): - raise EodBlockedError("KIS secrets path must be a regular non-symlink file") - if metadata.st_uid != os.getuid() or stat.S_IMODE(metadata.st_mode) != 0o600: - raise EodBlockedError("KIS secrets file must be user-owned mode 0600") - return True - - -def parse_dotenv(path: Path) -> dict[str, str]: - if not inspect_secrets_file(path): - return {} - output: dict[str, str] = {} - for line_number, raw_line in enumerate( - path.read_text(encoding="utf-8").splitlines(), start=1 - ): - stripped = raw_line.strip() - if not stripped or stripped.startswith("#"): - continue - if stripped.startswith("export "): - stripped = stripped[7:].lstrip() - if "=" not in stripped: - raise EodBlockedError(f"invalid dotenv line {line_number}") - key, value = stripped.split("=", 1) - key = key.strip() - if not DOTENV_KEY.fullmatch(key) or key in output: - raise EodBlockedError( - f"invalid or duplicate dotenv key on line {line_number}" - ) - value = value.strip() - if value.startswith('"'): - try: - decoded = json.loads(value) - except json.JSONDecodeError as exc: - raise EodBlockedError( - f"invalid quoted dotenv value on line {line_number}" - ) from exc - if not isinstance(decoded, str): - raise EodBlockedError(f"dotenv line {line_number} is not a string") - value = decoded - elif value.startswith("'"): - if len(value) < 2 or not value.endswith("'"): - raise EodBlockedError( - f"invalid quoted dotenv value on line {line_number}" - ) - value = value[1:-1] - output[key] = value - return output - - -def load_api_credentials(environment: str) -> tuple[str, str, Optional[str]]: - scope = "LIVE" if environment == "live" else "PAPER" - path = Path( - os.environ.get("QTA_SECRETS_FILE", str(DEFAULT_SECRETS_PATH)) - ).expanduser() - dotenv = parse_dotenv(path) - - def value(generic: str, scoped: str) -> Optional[str]: - return ( - os.environ.get(generic) - or os.environ.get(scoped) - or dotenv.get(scoped) - or dotenv.get(generic) - ) - - app_key = value("QTA_KIS_APP_KEY", f"QTA_KIS_{scope}_APP_KEY") - app_secret = value("QTA_KIS_APP_SECRET", f"QTA_KIS_{scope}_APP_SECRET") - access_token = value("QTA_KIS_ACCESS_TOKEN", f"QTA_KIS_{scope}_ACCESS_TOKEN") - missing = [ - name - for name, item in ( - ("QTA_KIS_APP_KEY", app_key), - ("QTA_KIS_APP_SECRET", app_secret), - ) - if not item - ] - if missing: - raise EodBlockedError("missing KIS credentials: " + ", ".join(missing)) - return str(app_key), str(app_secret), access_token - - -class KisReadClient: - """Read-only KIS HTTP client with deterministic pacing and bounded retries.""" - - def __init__( - self, - *, - environment: str, - app_key: str, - app_secret: str, - interval_ms: int, - access_token: Optional[str] = None, - transport: Optional[ - Callable[[str, str, Mapping[str, str], Optional[dict[str, Any]]], Any] - ] = None, - ) -> None: - self.base_url = BASE_URLS[environment] - self.app_key = app_key - self.app_secret = app_secret - self.interval_seconds = interval_ms / 1000.0 - self.access_token = access_token - self.last_request_at = 0.0 - self.request_count = 0 - self.retry_count = 0 - self.transport = transport or self._urllib_transport - - @staticmethod - def _urllib_transport( - method: str, - url: str, - headers: Mapping[str, str], - body: Optional[dict[str, Any]], - ) -> Any: - payload = None - if body is not None: - payload = canonical_json(body).encode("utf-8") - request = Request(url, data=payload, headers=dict(headers), method=method) - try: - with urlopen(request, timeout=15) as response: - raw = response.read() - status_code = int(response.status) - except HTTPError as exc: - raw = exc.read() - status_code = int(exc.code) - except (URLError, TimeoutError, OSError) as exc: - raise EodBlockedError(f"KIS network failure: {type(exc).__name__}") from exc - try: - decoded = json.loads(raw.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise EodBlockedError(f"KIS returned non-JSON HTTP {status_code}") from exc - return status_code, decoded - - def _pace(self) -> None: - now = time.monotonic() - remaining = self.interval_seconds - (now - self.last_request_at) - if remaining > 0: - time.sleep(remaining) - self.last_request_at = time.monotonic() - - def token(self) -> str: - if self.access_token: - return self.access_token - for attempt in range(2): - status, body = self.transport( - "POST", - f"{self.base_url}{TOKEN_PATH}", - {"Content-Type": "application/json"}, - { - "grant_type": "client_credentials", - "appkey": self.app_key, - "appsecret": self.app_secret, - }, - ) - self.request_count += 1 - if status == 200 and isinstance(body, dict) and body.get("access_token"): - self.access_token = str(body["access_token"]) - return self.access_token - code = ( - str(body.get("error_code", body.get("msg_cd", "unknown"))) - if isinstance(body, dict) - else "invalid_body" - ) - if code == "EGW00133" and attempt == 0: - self.retry_count += 1 - time.sleep(61.0) - continue - raise EodBlockedError(f"KIS token request failed: HTTP {status} {code}") - raise AssertionError("bounded KIS token loop fell through") - - def get(self, path: str, tr_id: str, params: Mapping[str, str]) -> dict[str, Any]: - query = urlencode(sorted(params.items())) - headers = { - "authorization": f"Bearer {self.token()}", - "appkey": self.app_key, - "appsecret": self.app_secret, - "tr_id": tr_id, - "custtype": "P", - "Content-Type": "application/json", - } - for attempt in range(4): - self._pace() - status, body = self.transport( - "GET", f"{self.base_url}{path}?{query}", headers, None - ) - self.request_count += 1 - code = ( - str(body.get("msg_cd", body.get("error_code", ""))) - if isinstance(body, dict) - else "" - ) - if status == 200 and isinstance(body, dict) and body.get("rt_cd") == "0": - return body - if status == 429 or code == "EGW00201": - if attempt == 3: - break - self.retry_count += 1 - time.sleep(float(2**attempt)) - continue - message = ( - str(body.get("msg1", body.get("error_description", code))) - if isinstance(body, dict) - else "invalid_body" - ) - raise EodBlockedError( - f"KIS read failed: HTTP {status} {code or 'unknown'} {message}" - ) - raise EodBlockedError("KIS read rate limit persisted after bounded retries") - - -def decimal_text(value: Any, label: str, *, allow_zero: bool = False) -> str: - try: - parsed = Decimal(str(value).replace(",", "").strip()) - except (InvalidOperation, ValueError) as exc: - raise EodBlockedError(f"{label} is not decimal") from exc - if not parsed.is_finite() or (parsed < 0 if allow_zero else parsed <= 0): - raise EodBlockedError( - f"{label} must be {'non-negative' if allow_zero else 'positive'}" - ) - rendered = format(parsed, "f") - return rendered.rstrip("0").rstrip(".") if "." in rendered else rendered - - -def normalize_stock_row(raw: Mapping[str, Any]) -> dict[str, str]: - try: - day = datetime.strptime(str(raw["stck_bsop_date"]), "%Y%m%d").date() - except (KeyError, ValueError) as exc: - raise EodBlockedError("KIS stock row has invalid stck_bsop_date") from exc - output = { - "date": day.isoformat(), - "open": decimal_text(raw.get("stck_oprc"), "stck_oprc"), - "high": decimal_text(raw.get("stck_hgpr"), "stck_hgpr"), - "low": decimal_text(raw.get("stck_lwpr"), "stck_lwpr"), - "close": decimal_text(raw.get("stck_clpr"), "stck_clpr"), - "volume": decimal_text(raw.get("acml_vol", "0"), "acml_vol", allow_zero=True), - } - output["adjusted_close"] = output["close"] - if Decimal(output["high"]) < max( - Decimal(output["open"]), Decimal(output["low"]), Decimal(output["close"]) - ): - raise EodBlockedError("KIS stock row high is inconsistent") - if Decimal(output["low"]) > min( - Decimal(output["open"]), Decimal(output["high"]), Decimal(output["close"]) - ): - raise EodBlockedError("KIS stock row low is inconsistent") - return output - - -def normalize_index_row(raw: Mapping[str, Any]) -> dict[str, str]: - day_value = raw.get("stck_bsop_date", raw.get("bstp_nmix_prdy_vrss_sign")) - try: - day = datetime.strptime(str(day_value), "%Y%m%d").date() - except ValueError as exc: - raise EodBlockedError("KIS index row has invalid stck_bsop_date") from exc - output = { - "date": day.isoformat(), - "open": decimal_text(raw.get("bstp_nmix_oprc"), "bstp_nmix_oprc"), - "high": decimal_text(raw.get("bstp_nmix_hgpr"), "bstp_nmix_hgpr"), - "low": decimal_text(raw.get("bstp_nmix_lwpr"), "bstp_nmix_lwpr"), - "close": decimal_text(raw.get("bstp_nmix_prpr"), "bstp_nmix_prpr"), - "volume": decimal_text( - raw.get("acml_vol", "0") or "0", "acml_vol", allow_zero=True - ), - } - output["adjusted_close"] = output["close"] - if Decimal(output["high"]) < max( - Decimal(output["open"]), Decimal(output["low"]), Decimal(output["close"]) - ): - raise EodBlockedError("KIS index row high is inconsistent") - if Decimal(output["low"]) > min( - Decimal(output["open"]), Decimal(output["high"]), Decimal(output["close"]) - ): - raise EodBlockedError("KIS index row low is inconsistent") - return output - - -def fetch_history( - client: KisReadClient, - *, - kind: str, - symbol: str, - start: date, - end: date, -) -> list[dict[str, str]]: - if kind == "stock": - path, tr_id, market_code = STOCK_PATH, STOCK_TR_ID, "J" - parser = normalize_stock_row - elif kind == "index": - path, tr_id, market_code = INDEX_PATH, INDEX_TR_ID, "U" - parser = normalize_index_row - else: - raise EodBlockedError("history kind must be stock or index") - cursor = end - output: dict[str, dict[str, str]] = {} - for _ in range(32): - params = { - "FID_COND_MRKT_DIV_CODE": market_code, - "FID_INPUT_ISCD": symbol, - "FID_INPUT_DATE_1": start.strftime("%Y%m%d"), - "FID_INPUT_DATE_2": cursor.strftime("%Y%m%d"), - "FID_PERIOD_DIV_CODE": "D", - } - if kind == "stock": - params["FID_ORG_ADJ_PRC"] = "0" - body = client.get(path, tr_id, params) - rows = body.get("output2") - if not isinstance(rows, list): - raise EodBlockedError("KIS history output2 must be an array") - if not rows: - break - normalized = [parser(row) for row in rows if isinstance(row, dict)] - if not normalized: - raise EodBlockedError("KIS history page has no valid rows") - for row in normalized: - day = date.fromisoformat(row["date"]) - if start <= day <= end: - existing = output.get(row["date"]) - if existing is not None and existing != row: - raise EodBlockedError( - f"KIS returned conflicting duplicate date {row['date']}" - ) - output[row["date"]] = row - oldest = min(date.fromisoformat(row["date"]) for row in normalized) - if oldest <= start: - break - next_cursor = oldest - timedelta(days=1) - if next_cursor >= cursor: - raise EodBlockedError("KIS history pagination did not move backward") - cursor = next_cursor - else: - raise EodBlockedError("KIS history exceeded 32 pages") - return [output[key] for key in sorted(output)] - - -def read_csv_rows(path: Path) -> list[dict[str, str]]: - if not path.is_file(): - return [] - with path.open("r", encoding="utf-8", newline="") as handle: - reader = csv.DictReader(handle) - if tuple(reader.fieldnames or ()) != ( - "date", - "open", - "high", - "low", - "close", - "adjusted_close", - "volume", - ): - raise EodBlockedError(f"cached EOD header mismatch: {path}") - rows = [dict(row) for row in reader] - normalized: list[dict[str, str]] = [] - for row in rows: - canonical = { - "date": require_iso_date(row["date"], f"{path}.date"), - "open": decimal_text(row["open"], f"{path}.open"), - "high": decimal_text(row["high"], f"{path}.high"), - "low": decimal_text(row["low"], f"{path}.low"), - "close": decimal_text(row["close"], f"{path}.close"), - "adjusted_close": decimal_text( - row["adjusted_close"], f"{path}.adjusted_close" - ), - "volume": decimal_text(row["volume"], f"{path}.volume", allow_zero=True), - } - normalized.append(canonical) - if len({row["date"] for row in normalized}) != len(normalized): - raise EodBlockedError(f"cached EOD contains duplicate dates: {path}") - return sorted(normalized, key=lambda row: row["date"]) - - -def write_csv_rows(path: Path, rows: list[dict[str, str]]) -> None: - fields = ( - "date", - "open", - "high", - "low", - "close", - "adjusted_close", - "volume", - ) - with tempfile.SpooledTemporaryFile( - mode="w+", encoding="utf-8", newline="" - ) as handle: - writer = csv.DictWriter(handle, fieldnames=fields, lineterminator="\n") - writer.writeheader() - writer.writerows(rows) - handle.seek(0) - atomic_write(path, handle.read().encode("utf-8")) - - -def update_history_file( - client: KisReadClient, - *, - kind: str, - symbol: str, - path: Path, - start: date, - end: date, - minimum_sessions: int, -) -> tuple[list[dict[str, str]], int]: - cached = read_csv_rows(path) - cached_map = { - row["date"]: row - for row in cached - if start <= date.fromisoformat(row["date"]) <= end - } - before_requests = client.request_count - if end.isoformat() not in cached_map or len(cached_map) < minimum_sessions: - fetched = fetch_history(client, kind=kind, symbol=symbol, start=start, end=end) - for row in fetched: - existing = cached_map.get(row["date"]) - if existing is not None and existing != row: - cached_map[row["date"]] = row - else: - cached_map[row["date"]] = row - rows = [cached_map[key] for key in sorted(cached_map)] - if end.isoformat() not in cached_map: - raise EodBlockedError(f"{symbol} has no completed row on {end.isoformat()}") - if len(rows) < minimum_sessions: - raise EodBlockedError( - f"{symbol} needs {minimum_sessions} sessions; found {len(rows)}" - ) - if rows != cached: - write_csv_rows(path, rows) - return rows, client.request_count - before_requests - - -def tick_size(exchange: str, price: Decimal) -> Decimal: - if exchange == "KOSPI": - bands = ( - (Decimal("1000"), Decimal("1")), - (Decimal("5000"), Decimal("5")), - (Decimal("10000"), Decimal("10")), - (Decimal("50000"), Decimal("50")), - (Decimal("100000"), Decimal("100")), - (Decimal("500000"), Decimal("500")), - ) - fallback = Decimal("1000") - elif exchange == "KOSDAQ": - bands = ( - (Decimal("1000"), Decimal("1")), - (Decimal("5000"), Decimal("5")), - (Decimal("10000"), Decimal("10")), - (Decimal("50000"), Decimal("50")), - ) - fallback = Decimal("100") - else: - raise EodBlockedError("tick exchange must be KOSPI or KOSDAQ") - for ceiling, tick in bands: - if price < ceiling: - return tick - return fallback - - -def catalog_type(reasons: list[str]) -> str: - mappings = ( - ("instrument_etf", "ETF"), - ("instrument_etn", "ETN"), - ("instrument_preferred", "PREFERRED"), - ("instrument_spac", "SPAC"), - ("instrument_unit", "UNIT"), - ("instrument_right", "RIGHT"), - ("instrument_warrant", "WARRANT"), - ("test_issue", "TEST"), - ("abnormal_status", "ABNORMAL"), - ) - for reason, rendered in mappings: - if reason in reasons: - return rendered - return "COMMON" - - -def load_base_catalog(path: str) -> dict[tuple[str, str], dict[str, str]]: - if not path: - return {} - output: dict[tuple[str, str], dict[str, str]] = {} - with Path(path).open("r", encoding="utf-8", newline="") as handle: - reader = csv.DictReader(handle) - if tuple(reader.fieldnames or ()) != CATALOG_FIELDS: - raise EodBlockedError("base_eod_catalog header mismatch") - for row in reader: - exchange = str(row["exchange"]).upper() - symbol = str(row["canonical_symbol"]).upper() - if exchange not in universe.EXCHANGES or not symbol: - raise EodBlockedError( - "base_eod_catalog has invalid exchange or canonical_symbol" - ) - key = (exchange, symbol) - if key in output: - raise EodBlockedError(f"duplicate base catalog row {key}") - output[key] = {field: str(row[field]) for field in CATALOG_FIELDS} - return output - - -def write_catalog(path: Path, rows: list[dict[str, str]]) -> None: - with tempfile.SpooledTemporaryFile( - mode="w+", encoding="utf-8", newline="" - ) as handle: - writer = csv.DictWriter(handle, fieldnames=CATALOG_FIELDS, lineterminator="\n") - writer.writeheader() - writer.writerows(rows) - handle.seek(0) - atomic_write(path, handle.read().encode("utf-8")) - - -def build_bundle( - job: dict[str, Any], - *, - client: Optional[KisReadClient] = None, -) -> dict[str, Any]: - output_root = Path(job["output_directory"]) - output_root.mkdir(parents=True, exist_ok=True) - if client is None: - app_key, app_secret, access_token = load_api_credentials(job["environment"]) - client = KisReadClient( - environment=job["environment"], - app_key=app_key, - app_secret=app_secret, - access_token=access_token, - interval_ms=job["request_interval_ms"], - ) - start = date.fromisoformat(job["history_start_date"]) - end = date.fromisoformat(job["analysis_date"]) - minimum_sessions = int(job["minimum_sessions"]) - official_descriptors = { - item["exchange"]: item - for item in job["official_sources"] - if item["exchange"] in KR_EXCHANGES - } - broker_descriptors = { - item["exchange"]: item - for item in job["broker_sources"] - if item["exchange"] in KR_EXCHANGES - } - official_rows = { - exchange: { - row.symbol: row - for row in universe.parse_master_source(official_descriptors[exchange]) - } - for exchange in KR_EXCHANGES - } - broker_rows = { - exchange: { - row.symbol: row - for row in universe.parse_master_source(broker_descriptors[exchange]) - } - for exchange in KR_EXCHANGES - } - - benchmark_paths: dict[str, Path] = {} - benchmark_results: dict[str, dict[str, Any]] = {} - for exchange in KR_EXCHANGES: - benchmark_path = output_root / "benchmarks" / f"{BENCHMARK_IDS[exchange]}.csv" - rows, requests = update_history_file( - client, - kind="index", - symbol=INDEX_CODES[exchange], - path=benchmark_path, - start=start, - end=end, - minimum_sessions=minimum_sessions, - ) - benchmark_paths[exchange] = benchmark_path.resolve() - benchmark_results[exchange] = { - "symbol": INDEX_CODES[exchange], - "path": str(benchmark_path.resolve()), - "sha256": sha256_file(benchmark_path), - "sessions": len(rows), - "requests": requests, - } - - catalog = load_base_catalog(job["base_eod_catalog"]) - failures: list[dict[str, str]] = [] - eligible_count = 0 - ready_count = 0 - cache_hits = 0 - coverage_by_exchange = { - exchange: { - "official": len(official_rows[exchange]), - "eligible": 0, - "ready": 0, - "failed": 0, - } - for exchange in KR_EXCHANGES - } - for exchange in KR_EXCHANGES: - official_descriptor = official_descriptors[exchange] - broker_descriptor = broker_descriptors[exchange] - for symbol in sorted(official_rows[exchange]): - official = official_rows[exchange][symbol] - broker = broker_rows[exchange].get(symbol) - reasons = universe.metadata_exclusion_reasons(official, official_descriptor) - if broker is None: - reasons.append("not_kis_tradable") - else: - reasons.extend( - universe.metadata_exclusion_reasons(broker, broker_descriptor) - ) - reasons = sorted(set(reasons)) - instrument_type = catalog_type(reasons) - ticker_path = output_root / "stocks" / exchange / f"{symbol}.csv" - latest_close = "" - resolved_tick = "" - source_name = "KIS_OPEN_API_ADJUSTED_DAILY:FHKST03010100" - data_path = "" - if not reasons: - eligible_count += 1 - coverage_by_exchange[exchange]["eligible"] += 1 - before = client.request_count - try: - rows, _ = update_history_file( - client, - kind="stock", - symbol=symbol, - path=ticker_path, - start=start, - end=end, - minimum_sessions=minimum_sessions, - ) - if client.request_count == before: - cache_hits += 1 - latest_close = rows[-1]["close"] - resolved_tick = format( - tick_size(exchange, Decimal(latest_close)), "f" - ) - data_path = str(ticker_path.resolve()) - ready_count += 1 - coverage_by_exchange[exchange]["ready"] += 1 - except EodBlockedError as exc: - coverage_by_exchange[exchange]["failed"] += 1 - failures.append( - { - "exchange": exchange, - "symbol": symbol, - "reason": str(exc), - } - ) - catalog[(exchange, symbol)] = { - "exchange": exchange, - "canonical_symbol": symbol, - "data_symbol": symbol, - "broker_symbol": symbol if broker is not None else "", - "instrument_type": instrument_type, - "benchmark_id": BENCHMARK_IDS[exchange], - "ticker_csv": data_path, - "benchmark_csv": str(benchmark_paths[exchange]), - "tick_rule_id": ( - f"KRX_GUIDE_OBSERVED_{job['as_of']}_{exchange}" - if resolved_tick - else "" - ), - "tick_effective_date": job["analysis_date"] if resolved_tick else "", - "tick_reference_price": latest_close, - "resolved_tick_size": resolved_tick, - "source_name": source_name, - } - - catalog_rows = [ - catalog[key] - for key in sorted( - catalog, - key=lambda item: ( - universe.EXCHANGES.index(item[0]), - item[1], - ), - ) - ] - catalog_path = output_root / "eod-catalog.csv" - write_catalog(catalog_path, catalog_rows) - - build_spec = { - "schema": universe.BUILD_SPEC_SCHEMA, - "as_of": job["as_of"], - "analysis_date": job["analysis_date"], - "official_sources": build_spec_sources(job["official_sources"]), - "broker_sources": build_spec_sources(job["broker_sources"]), - "eod_catalog": { - "source_id": f"kis-eod-catalog-{job['as_of']}", - "provider": "KIS", - "as_of": job["as_of"], - "path": str(catalog_path.resolve()), - "encoding": "utf-8", - }, - "catalog_coverage_contract": job["catalog_coverage_contract"], - } - build_spec_path = output_root / "universe-build-spec.json" - atomic_write_json(build_spec_path, build_spec) - - source_hashes = [ - { - "source_id": item["source_id"], - "exchange": item["exchange"], - "role": role, - "path": item["path"], - "sha256": sha256_file(Path(item["path"])), - } - for role, sources in ( - ("OFFICIAL_MASTER", job["official_sources"]), - ("BROKER_MASTER", job["broker_sources"]), - ) - for item in sources - ] - source_hashes.sort(key=lambda item: item["source_id"]) - receipt_without_hash = { - "schema": RECEIPT_SCHEMA, - "status": "READY", - "as_of": job["as_of"], - "analysis_date": job["analysis_date"], - "environment": job["environment"], - "api_mutation_count": 0, - "adjusted_price_parameter": "FID_ORG_ADJ_PRC=0", - "minimum_sessions": minimum_sessions, - "eligible_symbols": eligible_count, - "ready_symbols": ready_count, - "cache_hits": cache_hits, - "coverage_by_exchange": coverage_by_exchange, - "failed_symbols": failures, - "request_count": client.request_count, - "retry_count": client.retry_count, - "benchmarks": benchmark_results, - "source_hashes": source_hashes, - "catalog": { - "path": str(catalog_path.resolve()), - "sha256": sha256_file(catalog_path), - "rows": len(catalog_rows), - }, - "build_spec": { - "path": str(build_spec_path.resolve()), - "sha256": sha256_file(build_spec_path), - }, - "completed_at": datetime.now(timezone.utc).isoformat(), - } - receipt = { - **receipt_without_hash, - "receipt_hash": sha256_bytes( - canonical_json(receipt_without_hash).encode("utf-8") - ), - } - atomic_write_json(output_root / "eod-bundle-receipt.json", receipt) - return receipt - - -def download_bytes(url: str) -> bytes: - request = Request( - url, - headers={ - "User-Agent": "mrcha-skills/quant-stock-technical", - "Accept": "*/*", - }, - ) - try: - with urlopen(request, timeout=30) as response: - return response.read() - except (HTTPError, URLError, TimeoutError, OSError) as exc: - raise EodBlockedError( - f"source snapshot download failed: {url} ({type(exc).__name__})" - ) from exc - - -def snapshot_sources(as_of: str, output_directory: Path) -> dict[str, Any]: - require_iso_date(as_of, "as_of") - output_directory.mkdir(parents=True, exist_ok=True) - kind_path = output_directory / "krx-kind-corporate-list.xls" - atomic_write(kind_path, download_bytes(KIND_URL)) - try: - text = kind_path.read_text(encoding="cp949") - except UnicodeDecodeError as exc: - raise EodBlockedError("KRX KIND snapshot is not CP949 HTML") from exc - if "종목코드" not in text or "시장구분" not in text: - raise EodBlockedError("KRX KIND snapshot is missing expected headers") - - master_paths: dict[str, Path] = {} - for exchange in KR_EXCHANGES: - payload = download_bytes(KIS_MASTER_URLS[exchange]) - with tempfile.TemporaryDirectory() as temporary: - archive_path = Path(temporary) / "master.zip" - archive_path.write_bytes(payload) - try: - with zipfile.ZipFile(archive_path) as archive: - names = archive.namelist() - expected = KIS_MASTER_NAMES[exchange] - matches = [ - name - for name in names - if Path(name).name.lower() == expected.lower() - ] - if len(matches) != 1: - raise EodBlockedError( - f"{exchange} master archive must contain {expected}" - ) - master_payload = archive.read(matches[0]) - except zipfile.BadZipFile as exc: - raise EodBlockedError( - f"{exchange} master download is not a ZIP" - ) from exc - master_path = output_directory / KIS_MASTER_NAMES[exchange] - atomic_write(master_path, master_payload) - master_paths[exchange] = master_path - - official_sources = [ - { - "source_id": f"krx-kind-{exchange.lower()}-{as_of}", - "provider": "KRX", - "exchange": exchange, - "as_of": as_of, - "path": str(kind_path.resolve()), - "format": "KRX_KIND_HTML", - "encoding": "cp949", - "delimiter": "", - "skip_rows": 0, - "columns": {}, - "normal_status_values": [], - } - for exchange in KR_EXCHANGES - ] - broker_sources = [ - { - "source_id": f"kis-master-{exchange.lower()}-{as_of}", - "provider": "KIS", - "exchange": exchange, - "as_of": as_of, - "path": str(master_paths[exchange].resolve()), - "format": "KIS_KRX_MASTER", - "encoding": "cp949", - "delimiter": "", - "skip_rows": 0, - "columns": {}, - "normal_status_values": [], - } - for exchange in KR_EXCHANGES - ] - receipt_without_hash = { - "schema": SNAPSHOT_SCHEMA, - "as_of": as_of, - "official_sources": official_sources, - "broker_sources": broker_sources, - "files": [ - { - "path": str(path.resolve()), - "sha256": sha256_file(path), - } - for path in [kind_path, *master_paths.values()] - ], - "source_urls": [KIND_URL, *KIS_MASTER_URLS.values()], - "captured_at": datetime.now(timezone.utc).isoformat(), - } - receipt = { - **receipt_without_hash, - "receipt_hash": sha256_bytes( - canonical_json(receipt_without_hash).encode("utf-8") - ), - } - atomic_write_json(output_directory / "source-snapshot.json", receipt) - return receipt - - -def _synthetic_rows(end: date, count: int, *, index: bool) -> list[dict[str, str]]: - rows: list[dict[str, str]] = [] - cursor = end - for offset in range(count): - while cursor.weekday() >= 5: - cursor -= timedelta(days=1) - close = Decimal("1000") + Decimal(offset) - if index: - rows.append( - { - "stck_bsop_date": cursor.strftime("%Y%m%d"), - "bstp_nmix_oprc": str(close), - "bstp_nmix_hgpr": str(close + 10), - "bstp_nmix_lwpr": str(close - 10), - "bstp_nmix_prpr": str(close), - "acml_vol": "0", - } - ) - else: - rows.append( - { - "stck_bsop_date": cursor.strftime("%Y%m%d"), - "stck_oprc": str(close), - "stck_hgpr": str(close + 10), - "stck_lwpr": str(close - 10), - "stck_clpr": str(close), - "acml_vol": str(100000 + offset), - } - ) - cursor -= timedelta(days=1) - return rows - - -def self_test() -> None: - end = date(2026, 7, 24) - stock_rows = _synthetic_rows(end, 800, index=False) - index_rows = _synthetic_rows(end, 800, index=True) - - def transport( - method: str, - url: str, - headers: Mapping[str, str], - body: Optional[dict[str, Any]], - ) -> Any: - del headers, body - if method == "POST": - return 200, {"access_token": "fixture-token"} - query = dict(item.split("=", 1) for item in url.split("?", 1)[1].split("&")) - cursor = datetime.strptime(query["FID_INPUT_DATE_2"], "%Y%m%d").date() - source = index_rows if INDEX_PATH in url else stock_rows - page_size = 50 if index_rows is source else 100 - selected = [ - row - for row in source - if datetime.strptime(row["stck_bsop_date"], "%Y%m%d").date() <= cursor - ][:page_size] - return 200, {"rt_cd": "0", "output2": selected} - - client = KisReadClient( - environment="live", - app_key="fixture", - app_secret="fixture", - interval_ms=100, - access_token="fixture-token", - transport=transport, - ) - client.interval_seconds = 0 - start = date(2023, 1, 1) - stocks = fetch_history(client, kind="stock", symbol="005930", start=start, end=end) - indexes = fetch_history(client, kind="index", symbol="0001", start=start, end=end) - assert len(stocks) >= 756 - assert len(indexes) >= 756 - assert stocks[-1]["date"] == end.isoformat() - assert stocks[-1]["adjusted_close"] == stocks[-1]["close"] - assert tick_size("KOSPI", Decimal("999")) == Decimal("1") - assert tick_size("KOSPI", Decimal("100000")) == Decimal("500") - assert tick_size("KOSDAQ", Decimal("500000")) == Decimal("100") - with tempfile.TemporaryDirectory() as temporary: - path = Path(temporary) / "history.csv" - write_csv_rows(path, stocks) - assert read_csv_rows(path) == stocks - print( - json.dumps( - { - "self_test": "PASS", - "stock_sessions": len(stocks), - "index_sessions": len(indexes), - "requests": client.request_count, - }, - sort_keys=True, - ) - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - parser.add_argument("--self-test", action="store_true") - subparsers = parser.add_subparsers(dest="command") - collect = subparsers.add_parser("collect") - collect.add_argument("--job", required=True) - snapshot = subparsers.add_parser("snapshot-sources") - snapshot.add_argument("--as-of", required=True) - snapshot.add_argument("--output-directory", required=True) - return parser.parse_args() - - -def main() -> int: - args = parse_args() - try: - if args.self_test: - self_test() - return 0 - if args.command == "snapshot-sources": - receipt = snapshot_sources( - args.as_of, Path(args.output_directory).expanduser().resolve() - ) - elif args.command == "collect": - job_path = Path(args.job).expanduser().resolve() - raw = json.loads(job_path.read_text(encoding="utf-8")) - receipt = build_bundle(normalize_job(raw, job_path.parent)) - else: - raise EodBlockedError("use collect --job, snapshot-sources, or --self-test") - except ( - EodBlockedError, - universe.UniverseBlockedError, - json.JSONDecodeError, - ) as exc: - print( - json.dumps( - {"status": "BLOCKED", "reason": str(exc)}, - ensure_ascii=False, - sort_keys=True, - ), - file=sys.stderr, - ) - return 2 - print(json.dumps(receipt, ensure_ascii=False, sort_keys=True)) - return 0 if receipt.get("status", "READY") == "READY" else 2 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/plugins/quant-stock-technical/skills/quant-stock-technical/scripts/fetch_kis_us_eod.py b/plugins/quant-stock-technical/skills/quant-stock-technical/scripts/fetch_kis_us_eod.py deleted file mode 100644 index 2c1db51..0000000 --- a/plugins/quant-stock-technical/skills/quant-stock-technical/scripts/fetch_kis_us_eod.py +++ /dev/null @@ -1,997 +0,0 @@ -#!/usr/bin/env python3 -"""Build a resumable, source-hashed U.S. adjusted-EOD bundle from KIS.""" - -from __future__ import annotations - -import argparse -import json -import sys -import tempfile -import zipfile -from collections.abc import Callable, Mapping -from datetime import date, datetime, timedelta, timezone -from decimal import Decimal -from pathlib import Path -from typing import Any, Optional -from urllib.error import HTTPError, URLError -from urllib.parse import urlencode -from urllib.request import Request, urlopen - -import build_universe_manifest as universe -import fetch_kis_kr_eod as shared - -JOB_SCHEMA = "qta-kis-us-eod-job/v1" -RECEIPT_SCHEMA = "qta-kis-us-eod-receipt/v1" -SNAPSHOT_SCHEMA = "qta-us-source-snapshot/v1" -US_EXCHANGES = ("NYSE", "NASDAQ") -JOB_FIELDS = shared.JOB_FIELDS -STOCK_PATH = "/uapi/overseas-price/v1/quotations/dailyprice" -STOCK_TR_ID = "HHDFS76240000" -EXCHANGE_CODES = {"NYSE": "NYS", "NASDAQ": "NAS"} -BENCHMARK_IDS = {"NYSE": "NYSE_COMPOSITE", "NASDAQ": "NASDAQ_COMPOSITE"} -BENCHMARK_SYMBOLS = {"NYSE": "^NYA", "NASDAQ": "^IXIC"} -NASDAQ_URLS = { - "NASDAQ": "https://www.nasdaqtrader.com/dynamic/SymDir/nasdaqlisted.txt", - "NYSE": "https://www.nasdaqtrader.com/dynamic/SymDir/otherlisted.txt", -} -KIS_MASTER_URLS = { - "NASDAQ": "https://new.real.download.dws.co.kr/common/master/nasmst.cod.zip", - "NYSE": "https://new.real.download.dws.co.kr/common/master/nysmst.cod.zip", -} -KIS_MASTER_NAMES = {"NASDAQ": "nasmst.cod", "NYSE": "nysmst.cod"} -YAHOO_CHART_URL = "https://query1.finance.yahoo.com/v8/finance/chart" - - -class UsEodBlockedError(shared.EodBlockedError): - """Raised when a U.S. EOD bundle cannot be completed without guessing.""" - - -class InvalidOhlcGeometry(UsEodBlockedError): - """One parseable provider row has OHLC values that cannot be admitted.""" - - def __init__(self, reason: str, row: dict[str, str]): - super().__init__(reason) - self.reason = reason - self.row = row - - -def validate_ohlc_geometry(row: dict[str, str]) -> None: - if Decimal(row["high"]) < max( - Decimal(row["open"]), Decimal(row["low"]), Decimal(row["close"]) - ): - raise InvalidOhlcGeometry( - "KIS overseas row high is inconsistent", - row, - ) - if Decimal(row["low"]) > min( - Decimal(row["open"]), Decimal(row["high"]), Decimal(row["close"]) - ): - raise InvalidOhlcGeometry( - "KIS overseas row low is inconsistent", - row, - ) - - -def normalize_job(raw: Any, job_directory: Path) -> dict[str, Any]: - if not isinstance(raw, dict): - raise UsEodBlockedError("job must be one JSON object") - shared.require_exact_fields(raw, JOB_FIELDS, "job") - if raw["schema"] != JOB_SCHEMA: - raise UsEodBlockedError(f"job.schema must be {JOB_SCHEMA}") - as_of = shared.require_iso_date(raw["as_of"], "job.as_of") - analysis_date = shared.require_iso_date(raw["analysis_date"], "job.analysis_date") - history_start = shared.require_iso_date( - raw["history_start_date"], "job.history_start_date" - ) - if history_start >= analysis_date: - raise UsEodBlockedError("history_start_date must be before analysis_date") - if as_of < analysis_date: - raise UsEodBlockedError("as_of cannot be before analysis_date") - environment = str(raw["environment"]).lower() - if environment not in shared.BASE_URLS: - raise UsEodBlockedError("environment must be live or paper") - output_directory = Path(str(raw["output_directory"])).expanduser() - if not output_directory.is_absolute(): - output_directory = (job_directory / output_directory).resolve() - minimum_sessions = shared.require_positive_integer( - raw["minimum_sessions"], "job.minimum_sessions" - ) - if minimum_sessions < 756: - raise UsEodBlockedError("minimum_sessions must be at least 756") - interval = shared.require_positive_integer( - raw["request_interval_ms"], "job.request_interval_ms" - ) - if environment == "paper" and interval < 1000: - raise UsEodBlockedError("paper request_interval_ms must be at least 1000") - if environment == "live" and interval < 100: - raise UsEodBlockedError("live request_interval_ms must be at least 100") - - def sources_for(role: str) -> list[dict[str, Any]]: - raw_sources = raw[role] - if not isinstance(raw_sources, list): - raise UsEodBlockedError(f"job.{role} must be an array") - normalized: list[dict[str, Any]] = [] - seen: set[str] = set() - expected_role = ( - "OFFICIAL_MASTER" if role == "official_sources" else "BROKER_MASTER" - ) - for index, source in enumerate(raw_sources): - if not isinstance(source, dict): - raise UsEodBlockedError(f"job.{role}[{index}] must be an object") - item = universe.normalize_source_descriptor( - source, expected_role, as_of, job_directory, index - ) - if item["exchange"] in seen: - raise UsEodBlockedError(f"duplicate {role} exchange {item['exchange']}") - seen.add(item["exchange"]) - normalized.append(item) - missing = set(US_EXCHANGES) - seen - if missing: - raise UsEodBlockedError( - f"job.{role} missing U.S. exchanges {sorted(missing)}" - ) - return sorted( - normalized, key=lambda item: universe.EXCHANGES.index(item["exchange"]) - ) - - base_catalog = str(raw["base_eod_catalog"] or "").strip() - if base_catalog: - base_path = Path(base_catalog).expanduser() - if not base_path.is_absolute(): - base_path = (job_directory / base_path).resolve() - if not base_path.is_file(): - raise UsEodBlockedError(f"base_eod_catalog is not a file: {base_path}") - base_catalog = str(base_path) - coverage = universe.normalize_catalog_coverage_contract( - raw["catalog_coverage_contract"] - ) - return { - "schema": JOB_SCHEMA, - "as_of": as_of, - "analysis_date": analysis_date, - "environment": environment, - "output_directory": str(output_directory), - "history_start_date": history_start, - "minimum_sessions": minimum_sessions, - "request_interval_ms": interval, - "official_sources": sources_for("official_sources"), - "broker_sources": sources_for("broker_sources"), - "catalog_coverage_contract": coverage, - "base_eod_catalog": base_catalog, - } - - -def normalize_stock_row(raw: Mapping[str, Any]) -> dict[str, str]: - try: - day = datetime.strptime(str(raw["xymd"]), "%Y%m%d").date() - except (KeyError, ValueError) as exc: - raise UsEodBlockedError("KIS overseas row has invalid xymd") from exc - output = { - "date": day.isoformat(), - "open": shared.decimal_text(raw.get("open"), "open"), - "high": shared.decimal_text(raw.get("high"), "high"), - "low": shared.decimal_text(raw.get("low"), "low"), - "close": shared.decimal_text(raw.get("clos"), "clos"), - "volume": shared.decimal_text( - raw.get("tvol", "0") or "0", "tvol", allow_zero=True - ), - } - output["adjusted_close"] = output["close"] - validate_ohlc_geometry(output) - return output - - -def fetch_stock_history( - client: shared.KisReadClient, - *, - exchange: str, - symbol: str, - start: date, - end: date, - invalid_rows: Optional[list[dict[str, Any]]] = None, -) -> list[dict[str, str]]: - cursor = end - output: dict[str, dict[str, str]] = {} - for _ in range(32): - body = client.get( - STOCK_PATH, - STOCK_TR_ID, - { - "AUTH": "", - "EXCD": EXCHANGE_CODES[exchange], - "SYMB": symbol, - "GUBN": "0", - "BYMD": cursor.strftime("%Y%m%d"), - "MODP": "1", - }, - ) - rows = body.get("output2") - if not isinstance(rows, list): - raise UsEodBlockedError("KIS overseas output2 must be an array") - if not rows: - break - page_dates: list[date] = [] - normalized: list[dict[str, str]] = [] - for raw_row in rows: - if not isinstance(raw_row, dict): - continue - try: - page_dates.append( - datetime.strptime(str(raw_row["xymd"]), "%Y%m%d").date() - ) - except (KeyError, ValueError) as exc: - raise UsEodBlockedError( - "KIS overseas row has invalid xymd" - ) from exc - try: - normalized.append(normalize_stock_row(raw_row)) - except InvalidOhlcGeometry as exc: - invalid_day = date.fromisoformat(exc.row["date"]) - if invalid_day == end: - raise UsEodBlockedError( - f"{symbol} completed cutoff row has invalid OHLC geometry: " - f"{exc.reason}" - ) from exc - if invalid_rows is not None and start <= invalid_day <= end: - invalid_rows.append( - { - "date": exc.row["date"], - "reason": exc.reason, - "row": exc.row, - "row_sha256": shared.sha256_bytes( - shared.canonical_json(exc.row).encode("utf-8") - ), - } - ) - if not normalized: - if page_dates: - oldest = min(page_dates) - if oldest <= start: - break - next_cursor = oldest - timedelta(days=1) - if next_cursor >= cursor: - raise UsEodBlockedError( - "KIS overseas pagination did not move backward" - ) - cursor = next_cursor - continue - raise UsEodBlockedError("KIS overseas page has no valid rows") - for row in normalized: - day = date.fromisoformat(row["date"]) - if start <= day <= end: - existing = output.get(row["date"]) - if existing is not None and existing != row: - raise UsEodBlockedError( - f"KIS returned conflicting duplicate date {row['date']}" - ) - output[row["date"]] = row - oldest = min(page_dates) - if oldest <= start: - break - next_cursor = oldest - timedelta(days=1) - if next_cursor >= cursor: - raise UsEodBlockedError("KIS overseas pagination did not move backward") - cursor = next_cursor - else: - raise UsEodBlockedError("KIS overseas history exceeded 32 pages") - return [output[key] for key in sorted(output)] - - -def update_stock_file( - client: shared.KisReadClient, - *, - exchange: str, - symbol: str, - path: Path, - start: date, - end: date, - minimum_sessions: int, -) -> tuple[list[dict[str, str]], int, list[dict[str, Any]]]: - invalid_rows: list[dict[str, Any]] = [] - cached: list[dict[str, str]] = [] - cached_rows = shared.read_csv_rows(path) - for row in cached_rows: - try: - validate_ohlc_geometry(row) - except InvalidOhlcGeometry as exc: - invalid_day = date.fromisoformat(exc.row["date"]) - if invalid_day == end: - raise UsEodBlockedError( - f"{symbol} cached cutoff row has invalid OHLC geometry: " - f"{exc.reason}" - ) from exc - if start <= invalid_day <= end: - invalid_rows.append( - { - "date": exc.row["date"], - "reason": exc.reason, - "row": exc.row, - "row_sha256": shared.sha256_bytes( - shared.canonical_json(exc.row).encode("utf-8") - ), - } - ) - continue - cached.append(row) - merged = { - row["date"]: row - for row in cached - if start <= date.fromisoformat(row["date"]) <= end - } - before = client.request_count - if end.isoformat() not in merged or len(merged) < minimum_sessions: - for row in fetch_stock_history( - client, - exchange=exchange, - symbol=symbol, - start=start, - end=end, - invalid_rows=invalid_rows, - ): - merged[row["date"]] = row - rows = [merged[key] for key in sorted(merged)] - if end.isoformat() not in merged: - raise UsEodBlockedError(f"{symbol} has no completed row on {end.isoformat()}") - if len(rows) < minimum_sessions: - raise UsEodBlockedError( - f"{symbol} needs {minimum_sessions} sessions; found {len(rows)}" - ) - if rows != cached_rows: - shared.write_csv_rows(path, rows) - return rows, client.request_count - before, invalid_rows - - -def load_invalid_row_audit(path: Path) -> list[dict[str, Any]]: - if not path.exists(): - return [] - try: - value = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as exc: - raise UsEodBlockedError("invalid-row audit is not readable JSON") from exc - if ( - not isinstance(value, dict) - or value.get("schema") != "qta-kis-us-invalid-eod-rows/v1" - or not isinstance(value.get("records"), list) - ): - raise UsEodBlockedError("invalid-row audit has an unsupported schema") - return [item for item in value["records"] if isinstance(item, dict)] - - -def fetch_yahoo_benchmark(symbol: str, start: date, end: date) -> list[dict[str, str]]: - period1 = int( - datetime.combine(start, datetime.min.time(), timezone.utc).timestamp() - ) - period2 = int( - datetime.combine( - end + timedelta(days=1), datetime.min.time(), timezone.utc - ).timestamp() - ) - query = urlencode( - { - "period1": str(period1), - "period2": str(period2), - "interval": "1d", - "events": "div,splits", - } - ) - request = Request( - f"{YAHOO_CHART_URL}/{symbol}?{query}", - headers={"User-Agent": "mrcha-skills/quant-stock-technical"}, - ) - try: - with urlopen(request, timeout=30) as response: - payload = json.loads(response.read().decode("utf-8")) - except ( - HTTPError, - URLError, - TimeoutError, - OSError, - UnicodeDecodeError, - json.JSONDecodeError, - ) as exc: - raise UsEodBlockedError( - f"benchmark download failed for {symbol}: {type(exc).__name__}" - ) from exc - try: - result = payload["chart"]["result"][0] - timestamps = result["timestamp"] - quote = result["indicators"]["quote"][0] - adjusted = result["indicators"]["adjclose"][0]["adjclose"] - except (KeyError, IndexError, TypeError) as exc: - raise UsEodBlockedError(f"benchmark response malformed for {symbol}") from exc - rows: list[dict[str, str]] = [] - for index, stamp in enumerate(timestamps): - try: - day = datetime.fromtimestamp(int(stamp), timezone.utc).date() - close = Decimal(str(quote["close"][index])) - adjusted_close = Decimal(str(adjusted[index])) - ratio = adjusted_close / close - row = { - "date": day.isoformat(), - "open": shared.decimal_text( - Decimal(str(quote["open"][index])) * ratio, "benchmark.open" - ), - "high": shared.decimal_text( - Decimal(str(quote["high"][index])) * ratio, "benchmark.high" - ), - "low": shared.decimal_text( - Decimal(str(quote["low"][index])) * ratio, "benchmark.low" - ), - "close": shared.decimal_text(adjusted_close, "benchmark.close"), - "adjusted_close": shared.decimal_text( - adjusted_close, "benchmark.adjusted_close" - ), - "volume": shared.decimal_text( - quote["volume"][index] or "0", - "benchmark.volume", - allow_zero=True, - ), - } - except (IndexError, TypeError, ArithmeticError): - continue - if start <= day <= end: - rows.append(row) - if len({row["date"] for row in rows}) != len(rows): - raise UsEodBlockedError(f"benchmark contains duplicate dates for {symbol}") - return sorted(rows, key=lambda row: row["date"]) - - -def update_benchmark_file( - *, - symbol: str, - path: Path, - start: date, - end: date, - minimum_sessions: int, - loader: Callable[[str, date, date], list[dict[str, str]]], -) -> list[dict[str, str]]: - cached = shared.read_csv_rows(path) - merged = { - row["date"]: row - for row in cached - if start <= date.fromisoformat(row["date"]) <= end - } - if end.isoformat() not in merged or len(merged) < minimum_sessions: - for row in loader(symbol, start, end): - merged[row["date"]] = row - rows = [merged[key] for key in sorted(merged)] - if end.isoformat() not in merged: - raise UsEodBlockedError( - f"benchmark {symbol} has no completed row on {end.isoformat()}" - ) - if len(rows) < minimum_sessions: - raise UsEodBlockedError( - f"benchmark {symbol} needs {minimum_sessions} sessions; found {len(rows)}" - ) - if rows != cached: - shared.write_csv_rows(path, rows) - return rows - - -def tick_size(price: Decimal) -> Decimal: - if price <= 0: - raise UsEodBlockedError("reference price must be positive") - return Decimal("0.0001") if price < Decimal("1") else Decimal("0.01") - - -def build_bundle( - job: dict[str, Any], - *, - client: Optional[shared.KisReadClient] = None, - benchmark_loader: Callable[ - [str, date, date], list[dict[str, str]] - ] = fetch_yahoo_benchmark, -) -> dict[str, Any]: - output_root = Path(job["output_directory"]) - output_root.mkdir(parents=True, exist_ok=True) - if client is None: - key, secret, token = shared.load_api_credentials(job["environment"]) - client = shared.KisReadClient( - environment=job["environment"], - app_key=key, - app_secret=secret, - access_token=token, - interval_ms=job["request_interval_ms"], - ) - start = date.fromisoformat(job["history_start_date"]) - end = date.fromisoformat(job["analysis_date"]) - minimum = int(job["minimum_sessions"]) - official_descriptors = { - item["exchange"]: item - for item in job["official_sources"] - if item["exchange"] in US_EXCHANGES - } - broker_descriptors = { - item["exchange"]: item - for item in job["broker_sources"] - if item["exchange"] in US_EXCHANGES - } - official_rows = { - exchange: { - row.symbol: row - for row in universe.parse_master_source(official_descriptors[exchange]) - } - for exchange in US_EXCHANGES - } - broker_rows = { - exchange: { - row.symbol: row - for row in universe.parse_master_source(broker_descriptors[exchange]) - } - for exchange in US_EXCHANGES - } - - benchmark_paths: dict[str, Path] = {} - benchmark_results: dict[str, dict[str, Any]] = {} - for exchange in US_EXCHANGES: - path = output_root / "benchmarks" / f"{BENCHMARK_IDS[exchange]}.csv" - rows = update_benchmark_file( - symbol=BENCHMARK_SYMBOLS[exchange], - path=path, - start=start, - end=end, - minimum_sessions=minimum, - loader=benchmark_loader, - ) - benchmark_paths[exchange] = path.resolve() - benchmark_results[exchange] = { - "symbol": BENCHMARK_SYMBOLS[exchange], - "provider": "YAHOO_CHART", - "path": str(path.resolve()), - "sha256": shared.sha256_file(path), - "sessions": len(rows), - } - - catalog = shared.load_base_catalog(job["base_eod_catalog"]) - failures: list[dict[str, str]] = [] - invalid_audit_path = output_root / "invalid-eod-rows.json" - invalid_records = load_invalid_row_audit(invalid_audit_path) - ready_count = 0 - eligible_count = 0 - cache_hits = 0 - coverage = { - exchange: { - "official": len(official_rows[exchange]), - "eligible": 0, - "ready": 0, - "failed": 0, - } - for exchange in US_EXCHANGES - } - for exchange in US_EXCHANGES: - official_descriptor = official_descriptors[exchange] - broker_descriptor = broker_descriptors[exchange] - for symbol in sorted(official_rows[exchange]): - official = official_rows[exchange][symbol] - broker = broker_rows[exchange].get(symbol) - reasons = universe.metadata_exclusion_reasons(official, official_descriptor) - if broker is None: - reasons.append("not_kis_tradable") - else: - reasons.extend( - universe.metadata_exclusion_reasons(broker, broker_descriptor) - ) - reasons = sorted(set(reasons)) - instrument_type = shared.catalog_type(reasons) - ticker_path = output_root / "stocks" / exchange / f"{symbol}.csv" - data_path = "" - latest_close = "" - resolved_tick = "" - if not reasons: - eligible_count += 1 - coverage[exchange]["eligible"] += 1 - before = client.request_count - try: - rows, _, observed_invalid = update_stock_file( - client, - exchange=exchange, - symbol=symbol, - path=ticker_path, - start=start, - end=end, - minimum_sessions=minimum, - ) - if client.request_count != before or observed_invalid: - invalid_records = [ - item - for item in invalid_records - if not ( - item.get("exchange") == exchange - and item.get("symbol") == symbol - ) - ] - invalid_records.extend( - { - "exchange": exchange, - "symbol": symbol, - **item, - } - for item in observed_invalid - ) - if client.request_count == before: - cache_hits += 1 - latest_close = rows[-1]["close"] - resolved_tick = format(tick_size(Decimal(latest_close)), "f") - data_path = str(ticker_path.resolve()) - ready_count += 1 - coverage[exchange]["ready"] += 1 - except shared.EodBlockedError as exc: - coverage[exchange]["failed"] += 1 - failures.append( - { - "exchange": exchange, - "symbol": symbol, - "reason": str(exc), - } - ) - catalog[(exchange, symbol)] = { - "exchange": exchange, - "canonical_symbol": symbol, - "data_symbol": symbol, - "broker_symbol": symbol if broker is not None else "", - "instrument_type": instrument_type, - "benchmark_id": BENCHMARK_IDS[exchange], - "ticker_csv": data_path, - "benchmark_csv": str(benchmark_paths[exchange]), - "tick_rule_id": ( - f"US_NMS_CONSERVATIVE_{job['as_of']}" if resolved_tick else "" - ), - "tick_effective_date": job["analysis_date"] if resolved_tick else "", - "tick_reference_price": latest_close, - "resolved_tick_size": resolved_tick, - "source_name": "KIS_OPEN_API_ADJUSTED_DAILY:HHDFS76240000", - } - - rows = [ - catalog[key] - for key in sorted( - catalog, - key=lambda item: (universe.EXCHANGES.index(item[0]), item[1]), - ) - ] - catalog_path = output_root / "eod-catalog.csv" - shared.write_catalog(catalog_path, rows) - invalid_records.sort( - key=lambda item: ( - str(item.get("exchange", "")), - str(item.get("symbol", "")), - str(item.get("date", "")), - ) - ) - invalid_records = [ - item - for _, item in { - ( - str(item.get("exchange", "")), - str(item.get("symbol", "")), - str(item.get("date", "")), - ): item - for item in invalid_records - }.items() - ] - invalid_records.sort( - key=lambda item: ( - str(item.get("exchange", "")), - str(item.get("symbol", "")), - str(item.get("date", "")), - ) - ) - invalid_audit = { - "schema": "qta-kis-us-invalid-eod-rows/v1", - "analysis_date": job["analysis_date"], - "records": invalid_records, - } - shared.atomic_write_json(invalid_audit_path, invalid_audit) - build_spec = { - "schema": universe.BUILD_SPEC_SCHEMA, - "as_of": job["as_of"], - "analysis_date": job["analysis_date"], - "official_sources": shared.build_spec_sources(job["official_sources"]), - "broker_sources": shared.build_spec_sources(job["broker_sources"]), - "eod_catalog": { - "source_id": f"kis-cross-market-eod-catalog-{job['as_of']}", - "provider": "KIS", - "as_of": job["as_of"], - "path": str(catalog_path.resolve()), - "encoding": "utf-8", - }, - "catalog_coverage_contract": job["catalog_coverage_contract"], - } - build_spec_path = output_root / "universe-build-spec.json" - shared.atomic_write_json(build_spec_path, build_spec) - source_hashes = [ - { - "source_id": item["source_id"], - "exchange": item["exchange"], - "role": role, - "path": item["path"], - "sha256": shared.sha256_file(Path(item["path"])), - } - for role, sources in ( - ("OFFICIAL_MASTER", job["official_sources"]), - ("BROKER_MASTER", job["broker_sources"]), - ) - for item in sources - ] - source_hashes.sort(key=lambda item: item["source_id"]) - without_hash = { - "schema": RECEIPT_SCHEMA, - "status": "READY", - "as_of": job["as_of"], - "analysis_date": job["analysis_date"], - "environment": job["environment"], - "api_mutation_count": 0, - "adjusted_price_parameter": "MODP=1", - "minimum_sessions": minimum, - "eligible_symbols": eligible_count, - "ready_symbols": ready_count, - "cache_hits": cache_hits, - "coverage_by_exchange": coverage, - "failed_symbols": failures, - "invalid_eod_rows": { - "path": str(invalid_audit_path.resolve()), - "sha256": shared.sha256_file(invalid_audit_path), - "count": len(invalid_records), - "policy": "EXCLUDED_WITHOUT_INTERPOLATION", - }, - "request_count": client.request_count, - "retry_count": client.retry_count, - "benchmarks": benchmark_results, - "source_hashes": source_hashes, - "catalog": { - "path": str(catalog_path.resolve()), - "sha256": shared.sha256_file(catalog_path), - "rows": len(rows), - }, - "build_spec": { - "path": str(build_spec_path.resolve()), - "sha256": shared.sha256_file(build_spec_path), - }, - "completed_at": datetime.now(timezone.utc).isoformat(), - } - receipt = { - **without_hash, - "receipt_hash": shared.sha256_bytes( - shared.canonical_json(without_hash).encode("utf-8") - ), - } - shared.atomic_write_json(output_root / "eod-bundle-receipt.json", receipt) - return receipt - - -def download_bytes(url: str) -> bytes: - request = Request( - url, - headers={ - "User-Agent": "mrcha-skills/quant-stock-technical", - "Accept": "*/*", - }, - ) - try: - with urlopen(request, timeout=30) as response: - return response.read() - except (HTTPError, URLError, TimeoutError, OSError) as exc: - raise UsEodBlockedError( - f"source snapshot download failed: {url} ({type(exc).__name__})" - ) from exc - - -def snapshot_sources(as_of: str, output_directory: Path) -> dict[str, Any]: - shared.require_iso_date(as_of, "as_of") - output_directory.mkdir(parents=True, exist_ok=True) - official_paths: dict[str, Path] = {} - broker_paths: dict[str, Path] = {} - for exchange in US_EXCHANGES: - official_path = output_directory / ( - "otherlisted.txt" if exchange == "NYSE" else "nasdaqlisted.txt" - ) - shared.atomic_write(official_path, download_bytes(NASDAQ_URLS[exchange])) - official_paths[exchange] = official_path - payload = download_bytes(KIS_MASTER_URLS[exchange]) - with tempfile.TemporaryDirectory() as temporary: - archive_path = Path(temporary) / "master.zip" - archive_path.write_bytes(payload) - try: - with zipfile.ZipFile(archive_path) as archive: - expected = KIS_MASTER_NAMES[exchange] - matches = [ - name - for name in archive.namelist() - if Path(name).name.lower() == expected.lower() - ] - if len(matches) != 1: - raise UsEodBlockedError( - f"{exchange} master archive must contain {expected}" - ) - master_payload = archive.read(matches[0]) - except zipfile.BadZipFile as exc: - raise UsEodBlockedError( - f"{exchange} master download is not a ZIP" - ) from exc - broker_path = output_directory / KIS_MASTER_NAMES[exchange] - shared.atomic_write(broker_path, master_payload) - broker_paths[exchange] = broker_path - - official_sources = [ - { - "source_id": f"nasdaq-trader-{exchange.lower()}-{as_of}", - "provider": "NASDAQ_TRADER", - "exchange": exchange, - "as_of": as_of, - "path": str(official_paths[exchange].resolve()), - "format": ("NASDAQ_LISTED" if exchange == "NASDAQ" else "NASDAQ_OTHER"), - "encoding": "utf-8", - "delimiter": "|", - "skip_rows": 0, - "columns": {}, - "normal_status_values": ["N"] if exchange == "NASDAQ" else [], - } - for exchange in US_EXCHANGES - ] - broker_sources = [ - { - "source_id": f"kis-{exchange.lower()}-master-{as_of}", - "provider": "KIS", - "exchange": exchange, - "as_of": as_of, - "path": str(broker_paths[exchange].resolve()), - "format": "KIS_OVERSEAS_MASTER", - "encoding": "cp949", - "delimiter": "\t", - "skip_rows": 0, - "columns": {}, - "normal_status_values": [], - } - for exchange in US_EXCHANGES - ] - for index, descriptor in enumerate(official_sources): - normalized = universe.normalize_source_descriptor( - descriptor, "OFFICIAL_MASTER", as_of, output_directory, index - ) - universe.parse_master_source(normalized) - for index, descriptor in enumerate(broker_sources): - normalized = universe.normalize_source_descriptor( - descriptor, "BROKER_MASTER", as_of, output_directory, index - ) - universe.parse_master_source(normalized) - receipt = { - "schema": SNAPSHOT_SCHEMA, - "as_of": as_of, - "official_sources": official_sources, - "broker_sources": broker_sources, - "files": sorted( - [ - { - "path": str(path.resolve()), - "sha256": shared.sha256_file(path), - "bytes": path.stat().st_size, - } - for path in (*official_paths.values(), *broker_paths.values()) - ], - key=lambda item: item["path"], - ), - "api_mutation_count": 0, - "completed_at": datetime.now(timezone.utc).isoformat(), - } - shared.atomic_write_json(output_directory / "source-snapshot.json", receipt) - return receipt - - -def self_test() -> None: - end = date(2026, 7, 24) - raw = shared._synthetic_rows(end, 800, index=False) - overseas = [ - { - "xymd": item["stck_bsop_date"], - "open": item["stck_oprc"], - "high": item["stck_hgpr"], - "low": item["stck_lwpr"], - "clos": item["stck_clpr"], - "tvol": item["acml_vol"], - } - for item in raw - ] - - def transport( - method: str, - url: str, - headers: Mapping[str, str], - body: Optional[dict[str, Any]], - ) -> tuple[int, dict[str, Any]]: - del headers, body - if method != "GET": - raise AssertionError("self-test must be read-only") - query = dict(item.split("=", 1) for item in url.split("?", 1)[1].split("&")) - cursor = datetime.strptime(query["BYMD"], "%Y%m%d").date() - selected = [ - row - for row in overseas - if datetime.strptime(row["xymd"], "%Y%m%d").date() <= cursor - ][:100] - return 200, {"rt_cd": "0", "output2": selected} - - client = shared.KisReadClient( - environment="live", - app_key="fixture", - app_secret="fixture", - interval_ms=100, - access_token="fixture-token", - transport=transport, - ) - client.interval_seconds = 0 - rows = fetch_stock_history( - client, - exchange="NASDAQ", - symbol="AAPL", - start=end - timedelta(days=1200), - end=end, - ) - if len(rows) != 800 or rows[-1]["date"] != end.isoformat(): - raise AssertionError("U.S. pagination self-test failed") - if tick_size(Decimal("0.5")) != Decimal("0.0001"): - raise AssertionError("sub-dollar tick self-test failed") - if tick_size(Decimal("10")) != Decimal("0.01"): - raise AssertionError("dollar tick self-test failed") - print( - json.dumps( - { - "self_test": "PASS", - "stock_sessions": len(rows), - "requests": client.request_count, - "api_mutation_count": 0, - }, - sort_keys=True, - ) - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - parser.add_argument("--self-test", action="store_true") - subparsers = parser.add_subparsers(dest="command") - collect = subparsers.add_parser("collect") - collect.add_argument("--job", required=True) - snapshot = subparsers.add_parser("snapshot-sources") - snapshot.add_argument("--as-of", required=True) - snapshot.add_argument("--output-directory", required=True) - return parser.parse_args() - - -def main() -> int: - args = parse_args() - try: - if args.self_test: - self_test() - return 0 - if args.command == "snapshot-sources": - receipt = snapshot_sources( - args.as_of, Path(args.output_directory).expanduser().resolve() - ) - elif args.command == "collect": - job_path = Path(args.job).expanduser().resolve() - raw = json.loads(job_path.read_text(encoding="utf-8")) - receipt = build_bundle(normalize_job(raw, job_path.parent)) - else: - raise UsEodBlockedError( - "use collect --job, snapshot-sources, or --self-test" - ) - except ( - shared.EodBlockedError, - universe.UniverseBlockedError, - json.JSONDecodeError, - ) as exc: - print( - json.dumps( - {"status": "BLOCKED", "reason": str(exc)}, - ensure_ascii=False, - sort_keys=True, - ), - file=sys.stderr, - ) - return 2 - print(json.dumps(receipt, ensure_ascii=False, sort_keys=True)) - return 0 if receipt.get("status", "READY") == "READY" else 2 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/plugins/quant-stock-technical/skills/quant-stock-technical/scripts/screen_universe.py b/plugins/quant-stock-technical/skills/quant-stock-technical/scripts/screen_universe.py deleted file mode 100755 index c4f97f8..0000000 --- a/plugins/quant-stock-technical/skills/quant-stock-technical/scripts/screen_universe.py +++ /dev/null @@ -1,1951 +0,0 @@ -#!/usr/bin/env python3 -"""Run an approved QTA method across a frozen v1/v2 universe.""" - -from __future__ import annotations - -import argparse -import concurrent.futures -import hashlib -import json -import os -import re -import sys -from datetime import date -from decimal import Decimal, InvalidOperation -from pathlib import Path -from typing import Any - -import analyze_stock as qta -import analyze_stock_v2 as qta2 - -MANIFEST_SCHEMA = "qta-universe-manifest/v1" -SCREEN_SCHEMA = "qta-screen/v1" -SELECTOR_VERSION = "qta-screen-1.0.0" -SUPPORTED_MARKETS = {"KR", "US"} -MANIFEST_SCHEMA_V2 = "qta-universe-manifest/v2" -SCREEN_SCHEMA_V2 = "qta-screen/v2" -SELECTOR_VERSION_V2 = "qta-screen-1.1.0" -CATALOG_COVERAGE_SCHEMA = "qta-catalog-coverage-contract/v1" -EXCHANGE_ORDER = ("KOSPI", "KOSDAQ", "NYSE", "NASDAQ") -SUPPORTED_EXCHANGES = {"KOSPI", "KOSDAQ", "NYSE", "NASDAQ"} -SUPPORTED_EXCHANGE_SCOPES = { - frozenset(("KOSPI", "KOSDAQ")), - frozenset(("NYSE", "NASDAQ")), - frozenset(SUPPORTED_EXCHANGES), -} -SUPPORTED_SETUP_STATUSES = {"READY", "CONDITIONAL"} -EXCHANGE_CONTRACTS = { - "KOSPI": { - "market": "KR", - "currency": "KRW", - "venue": "KRX", - "benchmark_id": "KOSPI_COMPOSITE", - }, - "KOSDAQ": { - "market": "KR", - "currency": "KRW", - "venue": "KRX", - "benchmark_id": "KOSDAQ_COMPOSITE", - }, - "NYSE": { - "market": "US", - "currency": "USD", - "venue": "NYSE", - "benchmark_id": "NYSE_COMPOSITE", - }, - "NASDAQ": { - "market": "US", - "currency": "USD", - "venue": "NASD", - "benchmark_id": "NASDAQ_COMPOSITE", - }, -} -SUPPORTED_INSTRUMENT_TYPES = {"COMMON", "ADR", "REIT"} -SOURCE_HASH_FIELDS = { - "source_id", - "role", - "provider", - "exchange", - "as_of", - "path", - "sha256", -} -EXCLUSION_FIELDS = { - "exchange", - "canonical_symbol", - "broker_symbol", - "name", - "reasons", - "official_source_id", - "broker_source_id", -} -COUNT_FIELDS = { - "official_rows", - "broker_rows", - "catalog_mapped_rows", - "included", - "excluded", - "by_exchange", -} -EXCHANGE_COUNT_FIELDS = { - "official", - "broker", - "catalog_mapped", - "included", - "excluded", -} -CATALOG_COVERAGE_FIELDS = { - "schema", - "minimum_ratio_by_exchange", - "minimum_screenable_ratio_by_exchange", -} -MANIFEST_V2_FIELDS = { - "schema", - "build_status", - "as_of", - "analysis_date", - "catalog_coverage_contract", - "source_hashes", - "instruments", - "exclusions", - "counts", - "manifest_hash", -} -INSTRUMENT_V2_FIELDS = { - "market", - "exchange", - "canonical_symbol", - "data_symbol", - "broker_symbol", - "instrument_type", - "benchmark_id", - "currency", - "venue", - "ticker_csv", - "benchmark_csv", - "tick_contract", - "source_name", - "ticker_csv_sha256", - "benchmark_csv_sha256", - "broker_tradability_verified", - "official_source_id", - "broker_source_id", -} -TICK_CONTRACT_FIELDS = { - "schema", - "kind", - "rule_id", - "effective_date", - "reference_price", - "resolved_tick_size", -} -SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") - - -class ScreenBlockedError(ValueError): - """Raised when a frozen screen contract is invalid.""" - - -def canonical_json(value: Any) -> str: - return json.dumps( - value, - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - allow_nan=False, - ) - - -def sha256_json(value: Any) -> str: - return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest() - - -def sha256_file(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def load_object(path: Path) -> dict[str, Any]: - value = json.loads(path.read_text(encoding="utf-8")) - if not isinstance(value, dict): - raise ScreenBlockedError(f"{path} must contain a JSON object") - return value - - -def as_decimal(value: Any, field: str) -> Decimal: - if isinstance(value, bool): - raise ScreenBlockedError(f"{field} must be a decimal string or number") - try: - parsed = Decimal(str(value)) - except (InvalidOperation, ValueError) as exc: - raise ScreenBlockedError(f"{field} must be a decimal string or number") from exc - if not parsed.is_finite(): - raise ScreenBlockedError(f"{field} must be finite") - return parsed - - -def as_nonempty_string(value: Any, field: str) -> str: - if not isinstance(value, str) or not value.strip(): - raise ScreenBlockedError(f"{field} must be a non-empty string") - return value.strip() - - -def as_string(value: Any, field: str) -> str: - if not isinstance(value, str): - raise ScreenBlockedError(f"{field} must be a string") - return value - - -def as_iso_date(value: Any, field: str) -> str: - rendered = as_nonempty_string(value, field) - try: - date.fromisoformat(rendered) - except ValueError as exc: - raise ScreenBlockedError(f"{field} must be YYYY-MM-DD") from exc - return rendered - - -def positive_decimal_string(value: Any, field: str) -> str: - parsed = as_decimal(value, field) - if parsed <= 0: - raise ScreenBlockedError(f"{field} must be positive") - return format(parsed, "f") - - -def canonical_coverage_ratio(value: Any, field: str) -> str: - parsed = as_decimal(value, field) - if parsed <= 0 or parsed > 1: - raise ScreenBlockedError(f"{field} must be greater than 0 and at most 1") - rendered = format(parsed.normalize(), "f") - return rendered.rstrip("0").rstrip(".") if "." in rendered else rendered - - -def meets_minimum_coverage_ratio( - numerator: int, - official_rows: int, - minimum: str, -) -> bool: - if official_rows <= 0: - return False - return Decimal(numerator) / Decimal(official_rows) >= Decimal(minimum) - - -def normalize_catalog_coverage_contract(raw: Any) -> dict[str, Any]: - field = "catalog_coverage_contract" - if not isinstance(raw, dict) or set(raw) != CATALOG_COVERAGE_FIELDS: - raise ScreenBlockedError( - f"{field} fields must be exactly {sorted(CATALOG_COVERAGE_FIELDS)}" - ) - if raw["schema"] != CATALOG_COVERAGE_SCHEMA: - raise ScreenBlockedError(f"{field}.schema must be {CATALOG_COVERAGE_SCHEMA}") - normalized_ratios: dict[str, dict[str, str]] = {} - for ratio_field in ( - "minimum_ratio_by_exchange", - "minimum_screenable_ratio_by_exchange", - ): - ratios = raw[ratio_field] - if not isinstance(ratios, dict) or set(ratios) != SUPPORTED_EXCHANGES: - raise ScreenBlockedError( - f"{field}.{ratio_field} must contain exactly " - "KOSPI, KOSDAQ, NYSE, NASDAQ" - ) - normalized_ratios[ratio_field] = { - exchange: canonical_coverage_ratio( - ratios[exchange], - f"{field}.{ratio_field}.{exchange}", - ) - for exchange in EXCHANGE_ORDER - } - return { - "schema": CATALOG_COVERAGE_SCHEMA, - **normalized_ratios, - } - - -def normalize_manifest_v1(manifest: dict[str, Any]) -> dict[str, Any]: - if set(manifest) != {"schema", "analysis_date", "instruments"}: - raise ScreenBlockedError( - "manifest fields must be exactly schema, analysis_date, instruments" - ) - if manifest["schema"] != MANIFEST_SCHEMA: - raise ScreenBlockedError(f"unsupported manifest schema: {manifest['schema']!r}") - try: - date.fromisoformat(str(manifest["analysis_date"])) - except ValueError as exc: - raise ScreenBlockedError("analysis_date must be YYYY-MM-DD") from exc - instruments = manifest["instruments"] - if not isinstance(instruments, list) or not instruments: - raise ScreenBlockedError("instruments must be a non-empty array") - - required = { - "market", - "ticker", - "ticker_csv", - "benchmark_csv", - "tick_size", - "source_name", - } - normalized: list[dict[str, str]] = [] - seen: set[tuple[str, str]] = set() - for index, raw in enumerate(instruments): - if not isinstance(raw, dict) or set(raw) != required: - raise ScreenBlockedError( - f"instruments[{index}] fields must be exactly {sorted(required)}" - ) - market = str(raw["market"]).upper() - ticker = str(raw["ticker"]).upper() - if market not in SUPPORTED_MARKETS: - raise ScreenBlockedError(f"instruments[{index}].market must be KR or US") - if not ticker: - raise ScreenBlockedError(f"instruments[{index}].ticker is empty") - key = (market, ticker) - if key in seen: - raise ScreenBlockedError(f"duplicate instrument: {market}:{ticker}") - seen.add(key) - tick_size = as_decimal(raw["tick_size"], f"instruments[{index}].tick_size") - if tick_size <= 0: - raise ScreenBlockedError(f"instruments[{index}].tick_size must be positive") - normalized.append( - { - "market": market, - "ticker": ticker, - "ticker_csv": str(raw["ticker_csv"]), - "benchmark_csv": str(raw["benchmark_csv"]), - "tick_size": format(tick_size, "f"), - "source_name": str(raw["source_name"]), - } - ) - normalized.sort(key=lambda item: (item["market"], item["ticker"])) - return { - "schema": MANIFEST_SCHEMA, - "analysis_date": str(manifest["analysis_date"]), - "instruments": normalized, - } - - -def normalize_tick_contract( - raw: Any, - field: str, - analysis_date: str, -) -> dict[str, str]: - if not isinstance(raw, dict) or set(raw) != TICK_CONTRACT_FIELDS: - raise ScreenBlockedError( - f"{field} fields must be exactly {sorted(TICK_CONTRACT_FIELDS)}" - ) - if raw["schema"] != "qta-tick-contract/v1": - raise ScreenBlockedError(f"{field}.schema must be qta-tick-contract/v1") - if raw["kind"] != "RESOLVED_PRICE_LADDER": - raise ScreenBlockedError(f"{field}.kind must be RESOLVED_PRICE_LADDER") - effective_date = as_iso_date(raw["effective_date"], f"{field}.effective_date") - if effective_date > analysis_date: - raise ScreenBlockedError( - f"{field}.effective_date must not be after analysis_date" - ) - return { - "schema": "qta-tick-contract/v1", - "kind": "RESOLVED_PRICE_LADDER", - "rule_id": as_nonempty_string(raw["rule_id"], f"{field}.rule_id"), - "effective_date": effective_date, - "reference_price": positive_decimal_string( - raw["reference_price"], f"{field}.reference_price" - ), - "resolved_tick_size": positive_decimal_string( - raw["resolved_tick_size"], f"{field}.resolved_tick_size" - ), - } - - -def exact_sha256(value: Any, field: str) -> str: - rendered = as_nonempty_string(value, field) - if SHA256_PATTERN.fullmatch(rendered) is None: - raise ScreenBlockedError(f"{field} must be 64 lowercase hexadecimal characters") - return rendered - - -def exact_upper_string(value: Any, field: str) -> str: - rendered = as_nonempty_string(value, field) - if rendered != rendered.upper(): - raise ScreenBlockedError(f"{field} must be uppercase") - return rendered - - -def nonnegative_integer(value: Any, field: str) -> int: - if isinstance(value, bool) or not isinstance(value, int) or value < 0: - raise ScreenBlockedError(f"{field} must be an integer >= 0") - return value - - -def normalize_eligible_setup_statuses(raw: Any) -> list[str]: - if ( - not isinstance(raw, list) - or not raw - or any(not isinstance(item, str) for item in raw) - or not set(raw).issubset(SUPPORTED_SETUP_STATUSES) - ): - raise ScreenBlockedError( - "eligible_setup_statuses must be a non-empty subset of " - "READY and CONDITIONAL" - ) - return sorted(set(raw)) - - -def normalize_source_hashes(raw: Any, manifest_as_of: str) -> list[dict[str, str]]: - if not isinstance(raw, list) or not raw: - raise ScreenBlockedError("source_hashes must be a non-empty array") - normalized: list[dict[str, str]] = [] - seen: set[str] = set() - for index, source in enumerate(raw): - field = f"source_hashes[{index}]" - if not isinstance(source, dict) or set(source) != SOURCE_HASH_FIELDS: - raise ScreenBlockedError( - f"{field} fields must be exactly {sorted(SOURCE_HASH_FIELDS)}" - ) - source_id = as_nonempty_string(source["source_id"], f"{field}.source_id") - if source_id in seen: - raise ScreenBlockedError(f"duplicate source_id: {source_id}") - seen.add(source_id) - exchange = exact_upper_string(source["exchange"], f"{field}.exchange") - if exchange not in SUPPORTED_EXCHANGES | {"GLOBAL"}: - raise ScreenBlockedError( - f"{field}.exchange must be GLOBAL or one of " - f"{sorted(SUPPORTED_EXCHANGES)}" - ) - source_date = as_iso_date(source["as_of"], f"{field}.as_of") - if source_date != manifest_as_of: - raise ScreenBlockedError(f"{field}.as_of must equal manifest as_of") - source_path = Path(as_nonempty_string(source["path"], f"{field}.path")) - if not source_path.is_absolute(): - raise ScreenBlockedError(f"{field}.path must be absolute") - normalized.append( - { - "source_id": source_id, - "role": as_nonempty_string(source["role"], f"{field}.role"), - "provider": as_nonempty_string(source["provider"], f"{field}.provider"), - "exchange": exchange, - "as_of": source_date, - "path": str(source_path), - "sha256": exact_sha256(source["sha256"], f"{field}.sha256"), - } - ) - if normalized != sorted(normalized, key=lambda item: item["source_id"]): - raise ScreenBlockedError("source_hashes must be sorted by source_id") - return normalized - - -def normalize_exclusions(raw: Any) -> list[dict[str, Any]]: - if not isinstance(raw, list): - raise ScreenBlockedError("exclusions must be an array") - normalized: list[dict[str, Any]] = [] - for index, exclusion in enumerate(raw): - field = f"exclusions[{index}]" - if not isinstance(exclusion, dict) or set(exclusion) != EXCLUSION_FIELDS: - raise ScreenBlockedError( - f"{field} fields must be exactly {sorted(EXCLUSION_FIELDS)}" - ) - exchange = exact_upper_string(exclusion["exchange"], f"{field}.exchange") - if exchange not in SUPPORTED_EXCHANGES: - raise ScreenBlockedError( - f"{field}.exchange must be one of {sorted(SUPPORTED_EXCHANGES)}" - ) - reasons = exclusion["reasons"] - if ( - not isinstance(reasons, list) - or not reasons - or any(not isinstance(reason, str) or not reason for reason in reasons) - or reasons != sorted(set(reasons)) - ): - raise ScreenBlockedError( - f"{field}.reasons must be a non-empty sorted unique string array" - ) - broker_symbol = exclusion["broker_symbol"] - if broker_symbol is not None: - broker_symbol = as_nonempty_string(broker_symbol, f"{field}.broker_symbol") - broker_source_id = exclusion["broker_source_id"] - if broker_source_id is not None: - broker_source_id = as_nonempty_string( - broker_source_id, f"{field}.broker_source_id" - ) - normalized.append( - { - "exchange": exchange, - "canonical_symbol": as_nonempty_string( - exclusion["canonical_symbol"], f"{field}.canonical_symbol" - ), - "broker_symbol": broker_symbol, - "name": as_string(exclusion["name"], f"{field}.name"), - "reasons": reasons, - "official_source_id": as_nonempty_string( - exclusion["official_source_id"], - f"{field}.official_source_id", - ), - "broker_source_id": broker_source_id, - } - ) - if normalized != sorted( - normalized, - key=lambda item: ( - EXCHANGE_ORDER.index(item["exchange"]), - item["canonical_symbol"], - item["reasons"], - ), - ): - raise ScreenBlockedError( - "exclusions must follow KOSPI, KOSDAQ, NYSE, NASDAQ order and then " - "canonical_symbol, reasons" - ) - return normalized - - -def normalize_counts(raw: Any) -> dict[str, Any]: - if not isinstance(raw, dict) or set(raw) != COUNT_FIELDS: - raise ScreenBlockedError( - f"counts fields must be exactly {sorted(COUNT_FIELDS)}" - ) - by_exchange = raw["by_exchange"] - if not isinstance(by_exchange, dict) or set(by_exchange) != SUPPORTED_EXCHANGES: - raise ScreenBlockedError( - "counts.by_exchange must contain exactly KOSPI, KOSDAQ, NYSE, NASDAQ" - ) - normalized_by_exchange: dict[str, dict[str, int]] = {} - for exchange in sorted(SUPPORTED_EXCHANGES): - counts = by_exchange[exchange] - field = f"counts.by_exchange.{exchange}" - if not isinstance(counts, dict) or set(counts) != EXCHANGE_COUNT_FIELDS: - raise ScreenBlockedError( - f"{field} fields must be exactly {sorted(EXCHANGE_COUNT_FIELDS)}" - ) - normalized_by_exchange[exchange] = { - key: nonnegative_integer(counts[key], f"{field}.{key}") - for key in sorted(EXCHANGE_COUNT_FIELDS) - } - normalized: dict[str, Any] = { - field: nonnegative_integer(raw[field], f"counts.{field}") - for field in ( - "official_rows", - "broker_rows", - "catalog_mapped_rows", - "included", - "excluded", - ) - } - normalized["by_exchange"] = normalized_by_exchange - total_mappings = { - "official_rows": "official", - "broker_rows": "broker", - "catalog_mapped_rows": "catalog_mapped", - "included": "included", - "excluded": "excluded", - } - for total_field, exchange_field in total_mappings.items(): - expected = sum( - counts[exchange_field] for counts in normalized_by_exchange.values() - ) - if normalized[total_field] != expected: - raise ScreenBlockedError( - f"counts.{total_field} must equal the by_exchange sum" - ) - return normalized - - -def normalize_instruments_v2( - raw: Any, - analysis_date: str, - source_ids: set[str], -) -> list[dict[str, Any]]: - if not isinstance(raw, list) or not raw: - raise ScreenBlockedError("instruments must be a non-empty array") - normalized: list[dict[str, Any]] = [] - seen: set[tuple[str, str]] = set() - for index, instrument in enumerate(raw): - field = f"instruments[{index}]" - if not isinstance(instrument, dict) or set(instrument) != INSTRUMENT_V2_FIELDS: - raise ScreenBlockedError( - f"{field} fields must be exactly {sorted(INSTRUMENT_V2_FIELDS)}" - ) - exchange = exact_upper_string(instrument["exchange"], f"{field}.exchange") - if exchange not in SUPPORTED_EXCHANGES: - raise ScreenBlockedError( - f"{field}.exchange must be one of {sorted(SUPPORTED_EXCHANGES)}" - ) - exchange_contract = EXCHANGE_CONTRACTS[exchange] - market = exact_upper_string(instrument["market"], f"{field}.market") - if market != exchange_contract["market"]: - raise ScreenBlockedError( - f"{field}.market must be {exchange_contract['market']} for {exchange}" - ) - canonical_symbol = as_nonempty_string( - instrument["canonical_symbol"], f"{field}.canonical_symbol" - ) - key = (market, canonical_symbol) - if key in seen: - raise ScreenBlockedError( - f"duplicate canonical instrument: {market}:{canonical_symbol}" - ) - seen.add(key) - instrument_type = exact_upper_string( - instrument["instrument_type"], f"{field}.instrument_type" - ) - if instrument_type not in SUPPORTED_INSTRUMENT_TYPES: - raise ScreenBlockedError( - f"{field}.instrument_type must be one of " - f"{sorted(SUPPORTED_INSTRUMENT_TYPES)}" - ) - if market == "KR" and instrument_type == "ADR": - raise ScreenBlockedError(f"{field}.instrument_type ADR requires market US") - for contract_field in ("benchmark_id", "currency", "venue"): - actual = exact_upper_string( - instrument[contract_field], f"{field}.{contract_field}" - ) - if actual != exchange_contract[contract_field]: - raise ScreenBlockedError( - f"{field}.{contract_field} must be " - f"{exchange_contract[contract_field]} for {exchange}" - ) - ticker_path = Path( - as_nonempty_string(instrument["ticker_csv"], f"{field}.ticker_csv") - ) - benchmark_path = Path( - as_nonempty_string(instrument["benchmark_csv"], f"{field}.benchmark_csv") - ) - if not ticker_path.is_absolute() or not benchmark_path.is_absolute(): - raise ScreenBlockedError( - f"{field}.ticker_csv and benchmark_csv must be absolute" - ) - if instrument["broker_tradability_verified"] is not True: - raise ScreenBlockedError( - f"{field}.broker_tradability_verified must be true" - ) - official_source_id = as_nonempty_string( - instrument["official_source_id"], f"{field}.official_source_id" - ) - broker_source_id = as_nonempty_string( - instrument["broker_source_id"], f"{field}.broker_source_id" - ) - if official_source_id not in source_ids or broker_source_id not in source_ids: - raise ScreenBlockedError( - f"{field} references an unknown official_source_id or broker_source_id" - ) - normalized.append( - { - "market": market, - "exchange": exchange, - "canonical_symbol": canonical_symbol, - "data_symbol": as_nonempty_string( - instrument["data_symbol"], f"{field}.data_symbol" - ), - "broker_symbol": as_nonempty_string( - instrument["broker_symbol"], f"{field}.broker_symbol" - ), - "instrument_type": instrument_type, - "benchmark_id": exchange_contract["benchmark_id"], - "currency": exchange_contract["currency"], - "venue": exchange_contract["venue"], - "ticker_csv": str(ticker_path), - "benchmark_csv": str(benchmark_path), - "tick_contract": normalize_tick_contract( - instrument["tick_contract"], - f"{field}.tick_contract", - analysis_date, - ), - "source_name": as_nonempty_string( - instrument["source_name"], f"{field}.source_name" - ), - "ticker_csv_sha256": exact_sha256( - instrument["ticker_csv_sha256"], - f"{field}.ticker_csv_sha256", - ), - "benchmark_csv_sha256": exact_sha256( - instrument["benchmark_csv_sha256"], - f"{field}.benchmark_csv_sha256", - ), - "broker_tradability_verified": True, - "official_source_id": official_source_id, - "broker_source_id": broker_source_id, - } - ) - return normalized - - -def normalize_manifest_v2(manifest: dict[str, Any]) -> dict[str, Any]: - if set(manifest) != MANIFEST_V2_FIELDS: - raise ScreenBlockedError( - f"manifest fields must be exactly {sorted(MANIFEST_V2_FIELDS)}" - ) - if manifest["schema"] != MANIFEST_SCHEMA_V2: - raise ScreenBlockedError(f"unsupported manifest schema: {manifest['schema']!r}") - supplied_hash = exact_sha256(manifest["manifest_hash"], "manifest_hash") - hash_input = { - key: value for key, value in manifest.items() if key != "manifest_hash" - } - if sha256_json(hash_input) != supplied_hash: - raise ScreenBlockedError( - "manifest_hash does not match canonical manifest content" - ) - as_of = as_iso_date(manifest["as_of"], "as_of") - analysis_date = as_iso_date(manifest["analysis_date"], "analysis_date") - if analysis_date > as_of: - raise ScreenBlockedError("analysis_date must not be after as_of") - if manifest["build_status"] not in {"READY", "BLOCKED"}: - raise ScreenBlockedError("build_status must be READY or BLOCKED") - if manifest["build_status"] == "BLOCKED": - raise ScreenBlockedError("manifest build_status is BLOCKED") - coverage_contract = normalize_catalog_coverage_contract( - manifest["catalog_coverage_contract"] - ) - source_hashes = normalize_source_hashes(manifest["source_hashes"], as_of) - source_ids = {source["source_id"] for source in source_hashes} - instruments = normalize_instruments_v2( - manifest["instruments"], - analysis_date, - source_ids, - ) - exclusions = normalize_exclusions(manifest["exclusions"]) - for exclusion in exclusions: - if exclusion["official_source_id"] not in source_ids: - raise ScreenBlockedError( - "exclusion references an unknown official_source_id" - ) - if ( - exclusion["broker_source_id"] is not None - and exclusion["broker_source_id"] not in source_ids - ): - raise ScreenBlockedError("exclusion references an unknown broker_source_id") - counts = normalize_counts(manifest["counts"]) - if counts["included"] != len(instruments): - raise ScreenBlockedError("counts.included must equal instruments length") - if counts["excluded"] != len(exclusions): - raise ScreenBlockedError("counts.excluded must equal exclusions length") - for exchange in sorted(SUPPORTED_EXCHANGES): - included = sum(instrument["exchange"] == exchange for instrument in instruments) - excluded = sum(exclusion["exchange"] == exchange for exclusion in exclusions) - missing_catalog = sum( - exclusion["exchange"] == exchange - and "missing_eod_mapping" in exclusion["reasons"] - for exclusion in exclusions - ) - official = counts["by_exchange"][exchange]["official"] - catalog_mapped = counts["by_exchange"][exchange]["catalog_mapped"] - if catalog_mapped > official: - raise ScreenBlockedError( - f"counts.by_exchange.{exchange}.catalog_mapped cannot exceed official" - ) - if catalog_mapped != official - missing_catalog: - raise ScreenBlockedError( - f"counts.by_exchange.{exchange}.catalog_mapped does not match " - "official rows without missing_eod_mapping" - ) - if counts["by_exchange"][exchange]["included"] > official: - raise ScreenBlockedError( - f"counts.by_exchange.{exchange}.included cannot exceed official" - ) - if counts["by_exchange"][exchange]["included"] != included: - raise ScreenBlockedError( - f"counts.by_exchange.{exchange}.included does not match instruments" - ) - if counts["by_exchange"][exchange]["excluded"] != excluded: - raise ScreenBlockedError( - f"counts.by_exchange.{exchange}.excluded does not match exclusions" - ) - exchange_scope = frozenset( - exchange - for exchange in SUPPORTED_EXCHANGES - if counts["by_exchange"][exchange]["official"] > 0 - ) - if exchange_scope not in SUPPORTED_EXCHANGE_SCOPES: - raise ScreenBlockedError( - "manifest must cover exactly KOSPI/KOSDAQ, NYSE/NASDAQ, " - "or all four exchanges" - ) - inactive_exchanges = SUPPORTED_EXCHANGES - exchange_scope - for exchange in inactive_exchanges: - if any(counts["by_exchange"][exchange].values()): - raise ScreenBlockedError( - f"inactive exchange {exchange} must have zero counts" - ) - expected_status = ( - "READY" - if all( - counts["by_exchange"][exchange]["included"] > 0 - and meets_minimum_coverage_ratio( - counts["by_exchange"][exchange]["catalog_mapped"], - counts["by_exchange"][exchange]["official"], - coverage_contract["minimum_ratio_by_exchange"][exchange], - ) - and meets_minimum_coverage_ratio( - counts["by_exchange"][exchange]["included"], - counts["by_exchange"][exchange]["official"], - coverage_contract["minimum_screenable_ratio_by_exchange"][exchange], - ) - for exchange in exchange_scope - ) - else "BLOCKED" - ) - if manifest["build_status"] != expected_status: - raise ScreenBlockedError( - "build_status must be READY only when every active exchange has " - "included rows and meets its catalog mapping and screenable " - "coverage contracts" - ) - normalized = { - "schema": MANIFEST_SCHEMA_V2, - "build_status": "READY", - "as_of": as_of, - "analysis_date": analysis_date, - "catalog_coverage_contract": coverage_contract, - "source_hashes": source_hashes, - "instruments": instruments, - "exclusions": exclusions, - "counts": counts, - } - normalized["manifest_hash"] = sha256_json(normalized) - if normalized["manifest_hash"] != supplied_hash: - raise ScreenBlockedError("manifest content is not in normalized canonical form") - return normalized - - -def normalize_manifest(manifest: dict[str, Any]) -> dict[str, Any]: - schema = manifest.get("schema") - if schema == MANIFEST_SCHEMA: - return normalize_manifest_v1(manifest) - if schema == MANIFEST_SCHEMA_V2: - return normalize_manifest_v2(manifest) - raise ScreenBlockedError(f"unsupported manifest schema: {schema!r}") - - -def normalize_selector_v1(selector: dict[str, Any]) -> dict[str, Any]: - required = { - "selector_version", - "min_total_score", - "eligible_setup_statuses", - "top_k_by_market", - "max_blocked_fraction", - } - if set(selector) != required: - raise ScreenBlockedError(f"selector fields must be exactly {sorted(required)}") - if selector["selector_version"] != SELECTOR_VERSION: - raise ScreenBlockedError( - f"unsupported selector version: {selector['selector_version']!r}" - ) - minimum = as_decimal(selector["min_total_score"], "min_total_score") - if minimum < 0 or minimum > 100: - raise ScreenBlockedError("min_total_score must be between 0 and 100") - maximum_blocked = as_decimal( - selector["max_blocked_fraction"], "max_blocked_fraction" - ) - if maximum_blocked < 0 or maximum_blocked > 1: - raise ScreenBlockedError("max_blocked_fraction must be between 0 and 1") - - normalized_statuses = normalize_eligible_setup_statuses( - selector["eligible_setup_statuses"] - ) - - top_k = selector["top_k_by_market"] - if not isinstance(top_k, dict) or set(top_k) != SUPPORTED_MARKETS: - raise ScreenBlockedError("top_k_by_market must contain exactly KR and US") - normalized_top_k: dict[str, int] = {} - for market in sorted(SUPPORTED_MARKETS): - value = top_k[market] - if isinstance(value, bool) or not isinstance(value, int) or value < 0: - raise ScreenBlockedError( - f"top_k_by_market.{market} must be an integer >= 0" - ) - normalized_top_k[market] = value - - return { - "selector_version": SELECTOR_VERSION, - "min_total_score": format(minimum, "f"), - "eligible_setup_statuses": normalized_statuses, - "top_k_by_market": normalized_top_k, - "max_blocked_fraction": format(maximum_blocked, "f"), - } - - -def normalize_selector_v2(selector: dict[str, Any]) -> dict[str, Any]: - required = { - "selector_version", - "min_total_score", - "eligible_setup_statuses", - "top_k_by_exchange", - "min_selected_by_exchange", - "max_blocked_fraction", - } - if set(selector) != required: - raise ScreenBlockedError(f"selector fields must be exactly {sorted(required)}") - if selector["selector_version"] != SELECTOR_VERSION_V2: - raise ScreenBlockedError( - f"unsupported selector version: {selector['selector_version']!r}" - ) - minimum = as_decimal(selector["min_total_score"], "min_total_score") - if minimum < 0 or minimum > 100: - raise ScreenBlockedError("min_total_score must be between 0 and 100") - maximum_blocked = as_decimal( - selector["max_blocked_fraction"], "max_blocked_fraction" - ) - if maximum_blocked < 0 or maximum_blocked > 1: - raise ScreenBlockedError("max_blocked_fraction must be between 0 and 1") - normalized_statuses = normalize_eligible_setup_statuses( - selector["eligible_setup_statuses"] - ) - top_k = selector["top_k_by_exchange"] - if not isinstance(top_k, dict) or set(top_k) != SUPPORTED_EXCHANGES: - raise ScreenBlockedError( - "top_k_by_exchange must contain exactly KOSPI, KOSDAQ, NYSE, NASDAQ" - ) - normalized_top_k: dict[str, int] = {} - for exchange in sorted(SUPPORTED_EXCHANGES): - normalized_top_k[exchange] = nonnegative_integer( - top_k[exchange], - f"top_k_by_exchange.{exchange}", - ) - minimum_selected = selector["min_selected_by_exchange"] - if ( - not isinstance(minimum_selected, dict) - or set(minimum_selected) != SUPPORTED_EXCHANGES - ): - raise ScreenBlockedError( - "min_selected_by_exchange must contain exactly KOSPI, KOSDAQ, NYSE, NASDAQ" - ) - normalized_minimum_selected: dict[str, int] = {} - for exchange in sorted(SUPPORTED_EXCHANGES): - value = nonnegative_integer( - minimum_selected[exchange], - f"min_selected_by_exchange.{exchange}", - ) - if value > normalized_top_k[exchange]: - raise ScreenBlockedError( - f"min_selected_by_exchange.{exchange} must be <= " - f"top_k_by_exchange.{exchange}" - ) - normalized_minimum_selected[exchange] = value - return { - "selector_version": SELECTOR_VERSION_V2, - "min_total_score": format(minimum, "f"), - "eligible_setup_statuses": normalized_statuses, - "top_k_by_exchange": normalized_top_k, - "min_selected_by_exchange": normalized_minimum_selected, - "max_blocked_fraction": format(maximum_blocked, "f"), - } - - -def normalize_selector(selector: dict[str, Any]) -> dict[str, Any]: - version = selector.get("selector_version") - if version == SELECTOR_VERSION: - return normalize_selector_v1(selector) - if version == SELECTOR_VERSION_V2: - return normalize_selector_v2(selector) - raise ScreenBlockedError(f"unsupported selector version: {version!r}") - - -def instrument_market(instrument: dict[str, Any]) -> str: - return str(instrument.get("market")) - - -def instrument_ticker(instrument: dict[str, Any]) -> str: - if "ticker" in instrument: - return str(instrument["ticker"]) - return str(instrument["canonical_symbol"]) - - -def instrument_tick_size(instrument: dict[str, Any]) -> str: - if "tick_size" in instrument: - return str(instrument["tick_size"]) - return str(instrument["tick_contract"]["resolved_tick_size"]) - - -def blocked_payload( - instrument: dict[str, Any], - reason: str, - *, - method_version: str, -) -> dict[str, Any]: - return { - "source_skill": "quant-stock-technical", - "result_schema": "quant-stock-technical/v1", - "calculation_status": "BLOCKED", - "reason": reason, - "method_version": method_version, - "market": instrument_market(instrument), - "ticker": instrument_ticker(instrument), - } - - -def analyze_instrument( - instrument: dict[str, Any], - analysis_date: date, - manifest_directory: Path, - *, - calculator: Any = qta, -) -> dict[str, Any]: - ticker_path = Path(instrument["ticker_csv"]) - benchmark_path = Path(instrument["benchmark_csv"]) - if not ticker_path.is_absolute(): - ticker_path = manifest_directory / ticker_path - if not benchmark_path.is_absolute(): - benchmark_path = manifest_directory / benchmark_path - try: - if "ticker_csv_sha256" in instrument: - if sha256_file(ticker_path) != instrument["ticker_csv_sha256"]: - raise ScreenBlockedError("ticker_csv_sha256 does not match ticker_csv") - if sha256_file(benchmark_path) != instrument["benchmark_csv_sha256"]: - raise ScreenBlockedError( - "benchmark_csv_sha256 does not match benchmark_csv" - ) - ticker_rows, benchmark_rows = qta.align( - qta.read_csv(str(ticker_path)), - qta.read_csv(str(benchmark_path)), - analysis_date, - ) - return calculator.calculate( - ticker_rows, - benchmark_rows, - instrument_market(instrument), - instrument_ticker(instrument), - float(Decimal(instrument_tick_size(instrument))), - instrument["source_name"], - ) - except (qta.BlockedError, OSError, ValueError) as exc: - return blocked_payload( - instrument, - str(exc), - method_version=calculator.METHOD_VERSION, - ) - - -def analyze_instrument_task( - task: tuple[dict[str, Any], str, str, str], -) -> dict[str, Any]: - instrument, analysis_date_text, manifest_directory_text, method_version = task - calculators = { - qta.METHOD_VERSION: qta, - qta2.METHOD_VERSION: qta2, - } - calculator = calculators.get(method_version) - if calculator is None: - raise ScreenBlockedError( - f"unsupported QTA method version: {method_version!r}" - ) - return analyze_instrument( - instrument, - date.fromisoformat(analysis_date_text), - Path(manifest_directory_text), - calculator=calculator, - ) - - -def score(payload: dict[str, Any], path: str) -> Decimal: - value: Any = payload - for part in path.split("."): - if not isinstance(value, dict) or part not in value: - raise ScreenBlockedError( - f"{payload.get('market')}:{payload.get('ticker')} missing {path}" - ) - value = value[part] - return as_decimal(value, path) - - -def ranking_key(payload: dict[str, Any]) -> tuple[Any, ...]: - if payload.get("method_version") == qta2.METHOD_VERSION: - return ( - -score(payload, "total_score"), - -score(payload, "short.score"), - -score(payload, "medium.score"), - score(payload, "risk.score"), - -score(payload, "long.score"), - str(payload["market"]), - str(payload["ticker"]), - ) - return ( - -score(payload, "total_score"), - -score(payload, "medium.score"), - -score(payload, "long.score"), - -score(payload, "short.score"), - score(payload, "risk.score"), - str(payload["market"]), - str(payload["ticker"]), - ) - - -def finalize_screen( - manifest: dict[str, Any], - selector: dict[str, Any], - results: list[dict[str, Any]], -) -> dict[str, Any]: - instrument_by_key = { - (item["market"], item["ticker"]): { - "market": item["market"], - "ticker": item["ticker"], - "tick_size": item["tick_size"], - } - for item in manifest["instruments"] - } - result_by_key: dict[tuple[str, str], dict[str, Any]] = {} - for result in results: - key = (str(result.get("market")), str(result.get("ticker"))) - if key in result_by_key: - raise ScreenBlockedError(f"duplicate calculation result: {key}") - result_by_key[key] = result - - ordered_results: list[dict[str, Any]] = [] - for instrument in manifest["instruments"]: - key = (instrument["market"], instrument["ticker"]) - if key not in result_by_key: - raise ScreenBlockedError(f"missing calculation result: {key}") - ordered_results.append(result_by_key[key]) - - blocked_count = sum( - result.get("calculation_status") != "READY" for result in ordered_results - ) - blocked_fraction = Decimal(blocked_count) / Decimal(len(ordered_results)) - maximum_blocked = Decimal(selector["max_blocked_fraction"]) - screen_status = "READY" if blocked_fraction <= maximum_blocked else "BLOCKED" - - decisions: list[dict[str, Any]] = [] - eligible_by_market: dict[str, list[dict[str, Any]]] = { - market: [] for market in SUPPORTED_MARKETS - } - statuses = set(selector["eligible_setup_statuses"]) - minimum = Decimal(selector["min_total_score"]) - for payload in ordered_results: - reasons: list[str] = [] - if payload.get("calculation_status") != "READY": - reasons.append("calculation_not_ready") - else: - if payload.get("setup_status") not in statuses: - reasons.append("setup_status_ineligible") - if score(payload, "total_score") < minimum: - reasons.append("score_below_minimum") - eligible = not reasons - if eligible: - market = str(payload["market"]) - if market not in eligible_by_market: - raise ScreenBlockedError(f"result market is unsupported: {market}") - eligible_by_market[market].append(payload) - decisions.append( - { - "market": payload.get("market"), - "ticker": payload.get("ticker"), - "instrument": instrument_by_key[ - (str(payload.get("market")), str(payload.get("ticker"))) - ], - "eligible": eligible, - "reasons": reasons, - "qta": payload, - } - ) - - selected: dict[str, list[dict[str, Any]]] = {} - selected_keys: set[tuple[str, str]] = set() - for market in sorted(SUPPORTED_MARKETS): - ranked = sorted(eligible_by_market[market], key=ranking_key) - chosen = ranked[: selector["top_k_by_market"][market]] - selected[market] = [ - { - "rank": index, - "instrument": instrument_by_key[(market, str(payload["ticker"]))], - "qta": payload, - } - for index, payload in enumerate(chosen, start=1) - ] - selected_keys.update((market, str(payload["ticker"])) for payload in chosen) - - for decision in decisions: - decision["selected"] = ( - decision["market"], - decision["ticker"], - ) in selected_keys - - method_versions = { - str(result.get("method_version")) for result in ordered_results - } - if len(method_versions) != 1: - raise ScreenBlockedError("screen results contain multiple QTA methods") - output = { - "source_skill": "quant-stock-technical", - "schema": SCREEN_SCHEMA, - "screen_status": screen_status, - "method_version": method_versions.pop(), - "selector_version": SELECTOR_VERSION, - "analysis_date": manifest["analysis_date"], - "manifest_hash": sha256_json(manifest), - "selector_hash": sha256_json(selector), - "blocked_count": blocked_count, - "blocked_fraction": format(blocked_fraction, "f"), - "instrument_count": len(ordered_results), - "selector": selector, - "selected": selected if screen_status == "READY" else {"KR": [], "US": []}, - "decisions": decisions, - } - output["screen_hash"] = sha256_json(output) - return output - - -def finalize_screen_v2( - manifest: dict[str, Any], - selector: dict[str, Any], - results: list[dict[str, Any]], -) -> dict[str, Any]: - instrument_by_key = { - (instrument["market"], instrument["canonical_symbol"]): instrument - for instrument in manifest["instruments"] - } - result_by_key: dict[tuple[str, str], dict[str, Any]] = {} - for result in results: - key = (str(result.get("market")), str(result.get("ticker"))) - if key in result_by_key: - raise ScreenBlockedError(f"duplicate calculation result: {key}") - result_by_key[key] = result - - ordered_results: list[dict[str, Any]] = [] - for instrument in manifest["instruments"]: - key = (instrument["market"], instrument["canonical_symbol"]) - if key not in result_by_key: - raise ScreenBlockedError(f"missing calculation result: {key}") - ordered_results.append(result_by_key[key]) - if len(result_by_key) != len(ordered_results): - raise ScreenBlockedError("calculation results include unknown instruments") - - blocked_count = sum( - result.get("calculation_status") != "READY" for result in ordered_results - ) - blocked_fraction = Decimal(blocked_count) / Decimal(len(ordered_results)) - maximum_blocked = Decimal(selector["max_blocked_fraction"]) - - statuses = set(selector["eligible_setup_statuses"]) - minimum = Decimal(selector["min_total_score"]) - eligible_by_exchange: dict[str, list[dict[str, Any]]] = { - exchange: [] for exchange in SUPPORTED_EXCHANGES - } - decision_by_key: dict[tuple[str, str], dict[str, Any]] = {} - decisions: list[dict[str, Any]] = [] - for payload in ordered_results: - key = (str(payload.get("market")), str(payload.get("ticker"))) - instrument = instrument_by_key[key] - reasons: list[str] = [] - if payload.get("calculation_status") != "READY": - reasons.append("calculation_not_ready") - else: - if payload.get("setup_status") not in statuses: - reasons.append("setup_status_ineligible") - if score(payload, "total_score") < minimum: - reasons.append("score_below_minimum") - eligible = not reasons - if eligible: - eligible_by_exchange[instrument["exchange"]].append(payload) - decision = { - "market": instrument["market"], - "exchange": instrument["exchange"], - "canonical_symbol": instrument["canonical_symbol"], - "instrument": instrument, - "eligible": eligible, - "reasons": reasons, - "exchange_rank": None, - "selected": False, - "qta": payload, - } - decisions.append(decision) - decision_by_key[key] = decision - - selected: dict[str, list[dict[str, Any]]] = { - exchange: [] for exchange in sorted(SUPPORTED_EXCHANGES) - } - ranked_by_exchange: dict[str, list[dict[str, Any]]] = {} - for exchange in sorted(SUPPORTED_EXCHANGES): - ranked = sorted(eligible_by_exchange[exchange], key=ranking_key) - ranked_by_exchange[exchange] = ranked - for exchange_rank, payload in enumerate(ranked, start=1): - key = (str(payload["market"]), str(payload["ticker"])) - decision_by_key[key]["exchange_rank"] = exchange_rank - minimum_selection_met = all( - len(ranked_by_exchange[exchange]) - >= selector["min_selected_by_exchange"][exchange] - for exchange in SUPPORTED_EXCHANGES - ) - screen_status = ( - "READY" - if blocked_fraction <= maximum_blocked and minimum_selection_met - else "BLOCKED" - ) - - selected_keys: set[tuple[str, str]] = set() - if screen_status == "READY": - for exchange in sorted(SUPPORTED_EXCHANGES): - ranked = ranked_by_exchange[exchange] - chosen = ranked[: selector["top_k_by_exchange"][exchange]] - selected[exchange] = [ - { - "exchange_rank": exchange_rank, - "instrument": instrument_by_key[ - (str(payload["market"]), str(payload["ticker"])) - ], - "qta": payload, - } - for exchange_rank, payload in enumerate(chosen, start=1) - ] - selected_keys.update( - (str(payload["market"]), str(payload["ticker"])) for payload in chosen - ) - for key in selected_keys: - decision_by_key[key]["selected"] = True - - method_versions = { - str(result.get("method_version")) for result in ordered_results - } - if len(method_versions) != 1: - raise ScreenBlockedError("screen results contain multiple QTA methods") - output = { - "source_skill": "quant-stock-technical", - "schema": SCREEN_SCHEMA_V2, - "screen_status": screen_status, - "method_version": method_versions.pop(), - "selector_version": SELECTOR_VERSION_V2, - "analysis_date": manifest["analysis_date"], - "manifest_hash": manifest["manifest_hash"], - "selector_hash": sha256_json(selector), - "blocked_count": blocked_count, - "blocked_fraction": format(blocked_fraction, "f"), - "instrument_count": len(ordered_results), - "selector": selector, - "selected": selected, - "decisions": decisions, - } - output["screen_hash"] = sha256_json(output) - return output - - -def build_screen( - manifest: dict[str, Any], - selector: dict[str, Any], - manifest_directory: Path, - *, - method_version: str = qta.METHOD_VERSION, - workers: int = 1, -) -> dict[str, Any]: - normalized_manifest = normalize_manifest(manifest) - normalized_selector = normalize_selector(selector) - if ( - normalized_manifest["schema"] == MANIFEST_SCHEMA - and normalized_selector["selector_version"] != SELECTOR_VERSION - ): - raise ScreenBlockedError("qta-universe-manifest/v1 requires qta-screen-1.0.0") - if ( - normalized_manifest["schema"] == MANIFEST_SCHEMA_V2 - and normalized_selector["selector_version"] != SELECTOR_VERSION_V2 - ): - raise ScreenBlockedError("qta-universe-manifest/v2 requires qta-screen-1.1.0") - analysis_date = date.fromisoformat(normalized_manifest["analysis_date"]) - calculators = { - qta.METHOD_VERSION: qta, - qta2.METHOD_VERSION: qta2, - } - calculator = calculators.get(method_version) - if calculator is None: - raise ScreenBlockedError( - f"unsupported QTA method version: {method_version!r}" - ) - if isinstance(workers, bool) or workers <= 0 or workers > 32: - raise ScreenBlockedError("workers must be an integer between 1 and 32") - tasks = [ - ( - instrument, - analysis_date.isoformat(), - str(manifest_directory), - calculator.METHOD_VERSION, - ) - for instrument in normalized_manifest["instruments"] - ] - if workers == 1: - results = [analyze_instrument_task(task) for task in tasks] - else: - with concurrent.futures.ProcessPoolExecutor( - max_workers=workers - ) as executor: - results = list( - executor.map( - analyze_instrument_task, - tasks, - chunksize=8, - ) - ) - if normalized_manifest["schema"] == MANIFEST_SCHEMA: - return finalize_screen(normalized_manifest, normalized_selector, results) - return finalize_screen_v2(normalized_manifest, normalized_selector, results) - - -def synthetic_result( - market: str, - ticker: str, - total: str, - medium: str, - long: str, - short: str, - risk: str, -) -> dict[str, Any]: - return { - "source_skill": "quant-stock-technical", - "result_schema": "quant-stock-technical/v1", - "calculation_status": "READY", - "setup_status": "READY", - "method_version": qta.METHOD_VERSION, - "analysis_date": "2026-07-25", - "market": market, - "ticker": ticker, - "source_name": "screen-self-test", - "shared_sessions": 1000, - "score_basis": "ticker-relative historical percentile; not probability of profit", - "short": {"opinion": "긍정", "score": float(short)}, - "medium": {"opinion": "긍정", "score": float(medium)}, - "long": {"opinion": "긍정", "score": float(long)}, - "risk": {"score": float(risk), "counterpoint": "self-test"}, - "entry_price": 100.0, - "stop_price": 90.0, - "take_profit_price": 120.0, - "total_score": float(total), - } - - -def self_test() -> None: - raw_instruments = [ - { - "market": "US", - "ticker": "BBB", - "ticker_csv": "bbb.csv", - "benchmark_csv": "bench.csv", - "tick_size": "0.01", - "source_name": "self-test", - }, - { - "market": "US", - "ticker": "AAA", - "ticker_csv": "aaa.csv", - "benchmark_csv": "bench.csv", - "tick_size": "0.01", - "source_name": "self-test", - }, - ] - selector = normalize_selector( - { - "selector_version": SELECTOR_VERSION, - "min_total_score": "60", - "eligible_setup_statuses": ["READY"], - "top_k_by_market": {"KR": 0, "US": 2}, - "max_blocked_fraction": "0", - } - ) - manifest_a = normalize_manifest( - { - "schema": MANIFEST_SCHEMA, - "analysis_date": "2026-07-25", - "instruments": raw_instruments, - } - ) - manifest_b = normalize_manifest( - { - "schema": MANIFEST_SCHEMA, - "analysis_date": "2026-07-25", - "instruments": list(reversed(raw_instruments)), - } - ) - results = [ - synthetic_result("US", "BBB", "70", "70", "70", "70", "30"), - synthetic_result("US", "AAA", "70", "70", "70", "70", "30"), - ] - first = finalize_screen(manifest_a, selector, results) - second = finalize_screen(manifest_b, selector, list(reversed(results))) - assert first == second - assert [item["qta"]["ticker"] for item in first["selected"]["US"]] == [ - "AAA", - "BBB", - ] - assert first["screen_status"] == "READY" - assert len(first["screen_hash"]) == 64 - assert normalize_eligible_setup_statuses(["READY", "CONDITIONAL", "READY"]) == [ - "CONDITIONAL", - "READY", - ] - try: - normalize_selector({**selector, "eligible_setup_statuses": ["BLOCKED"]}) - except ScreenBlockedError as exc: - assert "subset of READY and CONDITIONAL" in str(exc) - else: - raise AssertionError("qta-screen-1.0.0 accepted an unknown setup status") - - exchanges = ["KOSDAQ", "KOSPI", "NASDAQ", "NYSE"] - source_hashes = [] - for source_id in sorted( - f"{role}-{exchange}" - for exchange in exchanges - for role in ("broker", "official") - ): - role, exchange = source_id.split("-", maxsplit=1) - source_hashes.append( - { - "source_id": source_id, - "role": role, - "provider": "self-test", - "exchange": exchange, - "as_of": "2026-07-26", - "path": f"/tmp/{source_id}.txt", - "sha256": "0" * 64, - } - ) - instrument_specs = [ - ("KOSDAQ", "KR", "100002", "1", "KOSDAQ_COMPOSITE", "KRW", "KRX"), - ("KOSPI", "KR", "100001", "1", "KOSPI_COMPOSITE", "KRW", "KRX"), - ("NASDAQ", "US", "AAA", "0.01", "NASDAQ_COMPOSITE", "USD", "NASD"), - ("NASDAQ", "US", "CCC", "0.01", "NASDAQ_COMPOSITE", "USD", "NASD"), - ("NYSE", "US", "BBB", "0.01", "NYSE_COMPOSITE", "USD", "NYSE"), - ] - instruments_v2 = [] - for ( - exchange, - market, - symbol, - tick_size, - benchmark_id, - currency, - venue, - ) in instrument_specs: - instruments_v2.append( - { - "market": market, - "exchange": exchange, - "canonical_symbol": symbol, - "data_symbol": symbol, - "broker_symbol": symbol, - "instrument_type": "COMMON", - "benchmark_id": benchmark_id, - "currency": currency, - "venue": venue, - "ticker_csv": f"/tmp/{exchange}-{symbol}.csv", - "benchmark_csv": f"/tmp/{exchange}-benchmark.csv", - "tick_contract": { - "schema": "qta-tick-contract/v1", - "kind": "RESOLVED_PRICE_LADDER", - "rule_id": f"{exchange}_SELF_TEST", - "effective_date": "2026-07-25", - "reference_price": "100", - "resolved_tick_size": tick_size, - }, - "source_name": "screen-self-test", - "ticker_csv_sha256": "1" * 64, - "benchmark_csv_sha256": "2" * 64, - "broker_tradability_verified": True, - "official_source_id": f"official-{exchange}", - "broker_source_id": f"broker-{exchange}", - } - ) - by_exchange = {} - for exchange in exchanges: - included = sum(spec[0] == exchange for spec in instrument_specs) - by_exchange[exchange] = { - "official": included, - "broker": included, - "catalog_mapped": included, - "included": included, - "excluded": 0, - } - manifest_v2_without_hash = { - "schema": MANIFEST_SCHEMA_V2, - "build_status": "READY", - "as_of": "2026-07-26", - "analysis_date": "2026-07-25", - "catalog_coverage_contract": { - "schema": CATALOG_COVERAGE_SCHEMA, - "minimum_ratio_by_exchange": {exchange: "1" for exchange in EXCHANGE_ORDER}, - "minimum_screenable_ratio_by_exchange": { - exchange: "1" for exchange in EXCHANGE_ORDER - }, - }, - "source_hashes": source_hashes, - "instruments": instruments_v2, - "exclusions": [], - "counts": { - "official_rows": len(instrument_specs), - "broker_rows": len(instrument_specs), - "catalog_mapped_rows": len(instrument_specs), - "included": len(instrument_specs), - "excluded": 0, - "by_exchange": by_exchange, - }, - } - normalized_manifest_v2 = normalize_manifest( - { - **manifest_v2_without_hash, - "manifest_hash": sha256_json(manifest_v2_without_hash), - } - ) - selector_v2 = normalize_selector( - { - "selector_version": SELECTOR_VERSION_V2, - "min_total_score": "60", - "eligible_setup_statuses": ["READY"], - "top_k_by_exchange": { - "KOSDAQ": 1, - "KOSPI": 1, - "NASDAQ": 1, - "NYSE": 1, - }, - "min_selected_by_exchange": { - "KOSDAQ": 1, - "KOSPI": 1, - "NASDAQ": 1, - "NYSE": 1, - }, - "max_blocked_fraction": "0", - } - ) - try: - normalize_selector( - {**selector_v2, "eligible_setup_statuses": ["READY", "UNKNOWN"]} - ) - except ScreenBlockedError as exc: - assert "subset of READY and CONDITIONAL" in str(exc) - else: - raise AssertionError("qta-screen-1.1.0 accepted an unknown setup status") - results_v2 = [ - synthetic_result(market, symbol, "70", "70", "70", "70", "30") - for _, market, symbol, _, _, _, _ in reversed(instrument_specs) - ] - screen_v2_a = finalize_screen_v2( - normalized_manifest_v2, - selector_v2, - results_v2, - ) - screen_v2_b = finalize_screen_v2( - normalized_manifest_v2, - selector_v2, - list(reversed(results_v2)), - ) - assert screen_v2_a == screen_v2_b - assert screen_v2_a["schema"] == SCREEN_SCHEMA_V2 - assert set(screen_v2_a["selected"]) == SUPPORTED_EXCHANGES - assert all( - selected[0]["exchange_rank"] == 1 - for selected in screen_v2_a["selected"].values() - ) - assert all( - set(selected[0]["instrument"]) == INSTRUMENT_V2_FIELDS - for selected in screen_v2_a["selected"].values() - ) - decision_v2_by_symbol = { - decision["canonical_symbol"]: decision for decision in screen_v2_a["decisions"] - } - assert decision_v2_by_symbol["AAA"]["exchange_rank"] == 1 - assert decision_v2_by_symbol["AAA"]["selected"] - assert decision_v2_by_symbol["CCC"]["exchange_rank"] == 2 - assert not decision_v2_by_symbol["CCC"]["selected"] - kr_exchanges = {"KOSPI", "KOSDAQ"} - kr_instruments = [ - item for item in instruments_v2 if item["exchange"] in kr_exchanges - ] - kr_by_exchange = { - exchange: ( - dict(by_exchange[exchange]) - if exchange in kr_exchanges - else { - "official": 0, - "broker": 0, - "catalog_mapped": 0, - "included": 0, - "excluded": 0, - } - ) - for exchange in EXCHANGE_ORDER - } - kr_manifest_without_hash = { - **manifest_v2_without_hash, - "source_hashes": [ - item - for item in source_hashes - if item["exchange"] in kr_exchanges | {"GLOBAL"} - ], - "instruments": kr_instruments, - "counts": { - "official_rows": len(kr_instruments), - "broker_rows": len(kr_instruments), - "catalog_mapped_rows": len(kr_instruments), - "included": len(kr_instruments), - "excluded": 0, - "by_exchange": kr_by_exchange, - }, - } - kr_manifest = normalize_manifest( - { - **kr_manifest_without_hash, - "manifest_hash": sha256_json(kr_manifest_without_hash), - } - ) - kr_selector = normalize_selector( - { - **selector_v2, - "min_selected_by_exchange": { - "KOSPI": 1, - "KOSDAQ": 1, - "NYSE": 0, - "NASDAQ": 0, - }, - } - ) - kr_results = [ - item for item in results_v2 if item["market"] == "KR" - ] - kr_screen = finalize_screen_v2(kr_manifest, kr_selector, kr_results) - assert kr_screen["screen_status"] == "READY" - assert not kr_screen["selected"]["NYSE"] - assert not kr_screen["selected"]["NASDAQ"] - coverage_selector_v2 = normalize_selector( - { - "selector_version": SELECTOR_VERSION_V2, - "min_total_score": "60", - "eligible_setup_statuses": ["READY"], - "top_k_by_exchange": { - "KOSDAQ": 1, - "KOSPI": 1, - "NASDAQ": 1, - "NYSE": 2, - }, - "min_selected_by_exchange": { - "KOSDAQ": 1, - "KOSPI": 1, - "NASDAQ": 1, - "NYSE": 2, - }, - "max_blocked_fraction": "1", - } - ) - coverage_blocked_v2 = finalize_screen_v2( - normalized_manifest_v2, - coverage_selector_v2, - results_v2, - ) - assert coverage_blocked_v2["blocked_fraction"] == "0" - assert coverage_blocked_v2["screen_status"] == "BLOCKED" - assert all(not selected for selected in coverage_blocked_v2["selected"].values()) - tampered_manifest_v2 = { - **manifest_v2_without_hash, - "analysis_date": "2026-07-24", - "manifest_hash": sha256_json(manifest_v2_without_hash), - } - try: - normalize_manifest(tampered_manifest_v2) - except ScreenBlockedError: - pass - else: - raise AssertionError("tampered qta-universe-manifest/v2 was accepted") - - sparse_by_exchange = { - exchange: dict(values) for exchange, values in by_exchange.items() - } - sparse_by_exchange["KOSPI"]["official"] += 1 - sparse_by_exchange["KOSPI"]["excluded"] += 1 - sparse_without_hash = { - **manifest_v2_without_hash, - "catalog_coverage_contract": { - "schema": CATALOG_COVERAGE_SCHEMA, - "minimum_ratio_by_exchange": {exchange: "1" for exchange in EXCHANGE_ORDER}, - "minimum_screenable_ratio_by_exchange": { - exchange: "0.5" for exchange in EXCHANGE_ORDER - }, - }, - "exclusions": [ - { - "exchange": "KOSPI", - "canonical_symbol": "999999", - "broker_symbol": None, - "name": "Missing Catalog", - "reasons": ["missing_eod_mapping"], - "official_source_id": "official-KOSPI", - "broker_source_id": "broker-KOSPI", - } - ], - "counts": { - **manifest_v2_without_hash["counts"], - "official_rows": len(instrument_specs) + 1, - "excluded": 1, - "by_exchange": sparse_by_exchange, - }, - } - try: - normalize_manifest( - { - **sparse_without_hash, - "manifest_hash": sha256_json(sparse_without_hash), - } - ) - except ScreenBlockedError as exc: - assert "catalog mapping and screenable coverage contracts" in str(exc) - else: - raise AssertionError("sparse full-coverage manifest was accepted as READY") - - unscreenable_by_exchange = { - exchange: dict(values) for exchange, values in by_exchange.items() - } - unscreenable_by_exchange["KOSPI"]["official"] += 1 - unscreenable_by_exchange["KOSPI"]["catalog_mapped"] += 1 - unscreenable_by_exchange["KOSPI"]["excluded"] += 1 - unscreenable_without_hash = { - **manifest_v2_without_hash, - "exclusions": [ - { - "exchange": "KOSPI", - "canonical_symbol": "999998", - "broker_symbol": None, - "name": "Not Tradable", - "reasons": ["not_broker_tradable"], - "official_source_id": "official-KOSPI", - "broker_source_id": "broker-KOSPI", - } - ], - "counts": { - **manifest_v2_without_hash["counts"], - "official_rows": len(instrument_specs) + 1, - "catalog_mapped_rows": len(instrument_specs) + 1, - "excluded": 1, - "by_exchange": unscreenable_by_exchange, - }, - } - try: - normalize_manifest( - { - **unscreenable_without_hash, - "manifest_hash": sha256_json(unscreenable_without_hash), - } - ) - except ScreenBlockedError as exc: - assert "catalog mapping and screenable coverage contracts" in str(exc) - else: - raise AssertionError("under-screenable manifest was accepted as READY") - assert ( - normalize_exclusions( - [ - { - "exchange": "KOSPI", - "canonical_symbol": "999999", - "broker_symbol": None, - "name": "", - "reasons": ["not_broker_tradable"], - "official_source_id": "official-KOSPI", - "broker_source_id": None, - } - ] - )[0]["name"] - == "" - ) - print( - canonical_json( - { - "self_test": "PASS", - "selector_version": SELECTOR_VERSION, - "screen_hash": first["screen_hash"], - } - ) - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--manifest") - parser.add_argument("--selector") - parser.add_argument("--output") - parser.add_argument( - "--method-version", - choices=(qta.METHOD_VERSION, qta2.METHOD_VERSION), - default=qta.METHOD_VERSION, - ) - parser.add_argument( - "--workers", - type=int, - default=0, - help="worker processes; 0 selects up to 8 local CPUs", - ) - parser.add_argument("--self-test", action="store_true") - return parser.parse_args() - - -def emit(value: dict[str, Any], output: str | None) -> None: - rendered = json.dumps( - value, ensure_ascii=False, sort_keys=True, indent=2, allow_nan=False - ) - if output: - output_path = Path(output) - output_path.write_text(rendered + "\n", encoding="utf-8") - summary = { - "source_skill": value.get( - "source_skill", - "quant-stock-technical", - ), - "schema": value.get("schema"), - "screen_status": value.get("screen_status"), - "method_version": value.get("method_version"), - "analysis_date": value.get("analysis_date"), - "blocked_count": value.get("blocked_count"), - "screen_hash": value.get("screen_hash"), - "reason": value.get("reason"), - "output": str(output_path.resolve()), - } - print( - json.dumps( - summary, - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - allow_nan=False, - ) - ) - else: - print(rendered) - - -def main() -> int: - args = parse_args() - if args.self_test: - self_test() - return 0 - if not args.manifest or not args.selector: - emit( - { - "source_skill": "quant-stock-technical", - "schema": SCREEN_SCHEMA, - "screen_status": "BLOCKED", - "reason": "--manifest and --selector are required", - }, - args.output, - ) - return 2 - manifest_path = Path(args.manifest).resolve() - failure_schema = SCREEN_SCHEMA - try: - manifest = load_object(manifest_path) - if manifest.get("schema") == MANIFEST_SCHEMA_V2: - failure_schema = SCREEN_SCHEMA_V2 - output = build_screen( - manifest, - load_object(Path(args.selector).resolve()), - manifest_path.parent, - method_version=args.method_version, - workers=( - min(8, max(1, os.cpu_count() or 1)) - if args.workers == 0 - else args.workers - ), - ) - except (ScreenBlockedError, OSError, ValueError, json.JSONDecodeError) as exc: - emit( - { - "source_skill": "quant-stock-technical", - "schema": failure_schema, - "screen_status": "BLOCKED", - "reason": str(exc), - }, - args.output, - ) - return 2 - emit(output, args.output) - return 0 if output["screen_status"] == "READY" else 2 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/plugins/quant-stock-technical/skills/quant-stock-technical/scripts/selected_open1h_outcomes.py b/plugins/quant-stock-technical/skills/quant-stock-technical/scripts/selected_open1h_outcomes.py deleted file mode 100644 index 49317fb..0000000 --- a/plugins/quant-stock-technical/skills/quant-stock-technical/scripts/selected_open1h_outcomes.py +++ /dev/null @@ -1,535 +0,0 @@ -#!/usr/bin/env python3 -"""Collect and compare selected candidates from frozen point-in-time screens.""" - -from __future__ import annotations - -import argparse -import json -import sys -from datetime import date -from decimal import Decimal -from pathlib import Path -from typing import Any - -import analyze_stock as qta1 -import collect_open1h_research as intraday -import evaluate_open1h_research as evaluation -import fetch_kis_kr_eod as shared -import screen_universe as screen -import walk_forward_open1h as walk_forward - - -SCHEMA = "qta-selected-open1h-outcomes/v1" - - -class SelectedOutcomesBlockedError(ValueError): - """Raised when selected point-in-time outcomes cannot be verified.""" - - -def absolute_file(value: str, field: str) -> Path: - path = Path(value).expanduser() - if not path.is_absolute() or not path.is_file() or path.is_symlink(): - raise SelectedOutcomesBlockedError( - f"{field} must be an absolute regular non-symlink file" - ) - return path - - -def absolute_directory(value: str, field: str) -> Path: - path = Path(value).expanduser() - if not path.is_absolute(): - raise SelectedOutcomesBlockedError(f"{field} must be absolute") - if path.exists() and (not path.is_dir() or path.is_symlink()): - raise SelectedOutcomesBlockedError( - f"{field} must be a non-symlink directory" - ) - return path - - -def load_manifest(path: Path) -> dict[str, Any]: - try: - raw = json.loads(path.read_text(encoding="utf-8")) - manifest = screen.normalize_manifest_v2(raw) - except (OSError, json.JSONDecodeError, screen.ScreenBlockedError) as exc: - raise SelectedOutcomesBlockedError( - f"manifest validation failed: {exc}" - ) from exc - supplied_hash = raw.get("manifest_hash") - if not isinstance(supplied_hash, str): - raise SelectedOutcomesBlockedError("manifest_hash is required") - return manifest - - -def selected_rows( - screen_value: dict[str, Any], - *, - market: str, -) -> list[dict[str, Any]]: - selected = screen_value.get("selected") - if not isinstance(selected, dict): - raise SelectedOutcomesBlockedError( - "screen.selected must be an exchange object" - ) - exchanges = intraday.MARKET_CONTRACTS[market]["exchanges"] - rows: list[dict[str, Any]] = [] - for exchange in exchanges: - exchange_rows = selected.get(exchange, []) - if not isinstance(exchange_rows, list): - raise SelectedOutcomesBlockedError( - f"screen.selected.{exchange} must be an array" - ) - for row in exchange_rows: - if ( - not isinstance(row, dict) - or not isinstance(row.get("instrument"), dict) - or not isinstance(row.get("qta"), dict) - ): - raise SelectedOutcomesBlockedError( - "selected row is missing instrument or QTA" - ) - if row["instrument"]["market"] != market: - continue - rows.append(row) - rows.sort( - key=lambda item: ( - exchanges.index(item["instrument"]["exchange"]), - int(item["exchange_rank"]), - item["instrument"]["canonical_symbol"], - ) - ) - return rows - - -def cache_path( - output_directory: Path, - *, - session_date: str, - instrument: dict[str, Any], -) -> Path: - return ( - output_directory - / "bars" - / session_date - / instrument["exchange"] - / f"{instrument['canonical_symbol']}.json" - ) - - -def load_or_fetch( - client: shared.KisReadClient, - output_directory: Path, - *, - market: str, - session_date: str, - instrument: dict[str, Any], -) -> tuple[dict[str, Any], bool]: - path = cache_path( - output_directory, - session_date=session_date, - instrument=instrument, - ) - if path.exists(): - try: - artifact = intraday.normalize_bars_artifact( - json.loads(path.read_text(encoding="utf-8")) - ) - except (OSError, json.JSONDecodeError, intraday.ResearchBlockedError) as exc: - raise SelectedOutcomesBlockedError( - f"cached bars are invalid: {path}: {exc}" - ) from exc - if ( - artifact["market"] != market - or artifact["session_date"] != session_date - or artifact["exchange"] != instrument["exchange"] - or artifact["canonical_symbol"] != instrument["canonical_symbol"] - or artifact["broker_symbol"] != instrument["broker_symbol"] - ): - raise SelectedOutcomesBlockedError( - f"cached bars identity mismatch: {path}" - ) - return artifact, True - raw_rows = intraday.fetch_raw_rows( - client, - market=market, - exchange=instrument["exchange"], - broker_symbol=instrument["broker_symbol"], - session_date=session_date, - ) - artifact = intraday.bars_artifact( - instrument=instrument, - market=market, - session_date=session_date, - raw_rows=raw_rows, - ) - shared.atomic_write_json(path, artifact) - return artifact, False - - -def benchmark_open_regime_bps( - instrument: dict[str, Any], - *, - analysis_date: str, - session_date: str, -) -> Decimal | None: - rows = { - row.day: row - for row in qta1.read_csv(instrument["benchmark_csv"]) - } - previous = rows.get(date.fromisoformat(analysis_date)) - current = rows.get(date.fromisoformat(session_date)) - if previous is None or current is None: - return None - return ( - Decimal(str(current.open)) / Decimal(str(previous.close)) - Decimal(1) - ) * Decimal(10000) - - -def compact( - *, - method: str, - analysis_date: str, - session_date: str, - selected: dict[str, Any], - artifact: dict[str, Any] | None, - bars_reason: str | None, -) -> dict[str, Any]: - instrument = selected["instrument"] - qta = selected["qta"] - regime = benchmark_open_regime_bps( - instrument, - analysis_date=analysis_date, - session_date=session_date, - ) - base = { - "method_version": method, - "analysis_date": analysis_date, - "session_date": session_date, - "exchange": instrument["exchange"], - "canonical_symbol": instrument["canonical_symbol"], - "broker_symbol": instrument["broker_symbol"], - "exchange_rank": selected["exchange_rank"], - "qta_total_score": walk_forward.decimal_text( - walk_forward.decimal_value(qta["total_score"], "total_score") - ), - "entry_price": format( - walk_forward.decimal_value(qta["entry_price"], "entry_price"), - "f", - ), - "stop_price": format( - walk_forward.decimal_value(qta["stop_price"], "stop_price"), - "f", - ), - "take_profit_price": format( - walk_forward.decimal_value( - qta["take_profit_price"], "take_profit_price" - ), - "f", - ), - "market_open_regime_bps": ( - walk_forward.decimal_text(regime) if regime is not None else None - ), - "market_open_regime_pass": ( - regime >= 0 if regime is not None else None - ), - "bars_status": "READY" if artifact is not None else "BLOCKED", - "bars_reason": bars_reason, - } - if artifact is None: - return { - **base, - "coverage_status": None, - "bar_count": 0, - "return_60m_bps": None, - "entry_touched_by_minute_high": None, - "entry_crossed_by_bar_close": None, - "entry_cross_bar_time": None, - "end_return_from_entry_bps": None, - "bars_hash": None, - } - bars = artifact["bars"] - summary = artifact["summary"] - entry = walk_forward.decimal_value(qta["entry_price"], "entry_price") - cross_time = evaluation.first_bar_close_cross(bars, entry) - return { - **base, - "coverage_status": intraday.coverage_summary( - bars, market=artifact["market"] - )["coverage_status"], - "bar_count": len(bars), - "return_60m_bps": summary["return_60m_bps"], - "entry_touched_by_minute_high": ( - Decimal(summary["window_high"]) >= entry - ), - "entry_crossed_by_bar_close": cross_time is not None, - "entry_cross_bar_time": cross_time, - "end_return_from_entry_bps": ( - intraday.bps(Decimal(summary["end_close"]), entry) - if cross_time is not None - else None - ), - "bars_hash": artifact["bars_hash"], - } - - -def run(args: argparse.Namespace) -> dict[str, Any]: - manifest_path = absolute_file(args.manifest, "manifest") - output_directory = absolute_directory( - args.output_directory, "output_directory" - ) - output_directory.mkdir(parents=True, exist_ok=True) - manifest = load_manifest(manifest_path) - if manifest["as_of"] != args.session_date: - raise SelectedOutcomesBlockedError( - "manifest.as_of must equal session_date" - ) - screens: dict[str, dict[str, Any]] = {} - selections: dict[str, list[dict[str, Any]]] = {} - for path_text in args.screen: - path = absolute_file(path_text, "screen") - value = evaluation.normalized_screen(path) - method = value["method_version"] - if method in screens: - raise SelectedOutcomesBlockedError( - f"duplicate method screen: {method}" - ) - if value["manifest_hash"] != manifest["manifest_hash"]: - raise SelectedOutcomesBlockedError( - f"{method} screen manifest hash mismatch" - ) - if value["analysis_date"] != manifest["analysis_date"]: - raise SelectedOutcomesBlockedError( - f"{method} screen analysis date mismatch" - ) - screens[method] = value - selections[method] = selected_rows(value, market=args.market) - if not screens: - raise SelectedOutcomesBlockedError("at least one screen is required") - selected_by_key: dict[tuple[str, str], dict[str, Any]] = {} - for method_rows in selections.values(): - for row in method_rows: - instrument = row["instrument"] - selected_by_key[ - (instrument["exchange"], instrument["canonical_symbol"]) - ] = instrument - key, secret, token = shared.load_api_credentials(args.environment) - client = shared.KisReadClient( - environment=args.environment, - app_key=key, - app_secret=secret, - access_token=token, - interval_ms=args.request_interval_ms, - ) - artifacts: dict[tuple[str, str], dict[str, Any] | None] = {} - errors: dict[tuple[str, str], str | None] = {} - cache_hits = 0 - for key_value, instrument in sorted(selected_by_key.items()): - try: - artifact, cache_hit = load_or_fetch( - client, - output_directory, - market=args.market, - session_date=args.session_date, - instrument=instrument, - ) - artifacts[key_value] = artifact - errors[key_value] = None - cache_hits += int(cache_hit) - except ( - shared.EodBlockedError, - intraday.ResearchBlockedError, - SelectedOutcomesBlockedError, - ) as exc: - artifacts[key_value] = None - errors[key_value] = str(exc) - rows: list[dict[str, Any]] = [] - for method, method_rows in selections.items(): - for selected in method_rows: - instrument = selected["instrument"] - key_value = ( - instrument["exchange"], - instrument["canonical_symbol"], - ) - rows.append( - compact( - method=method, - analysis_date=manifest["analysis_date"], - session_date=args.session_date, - selected=selected, - artifact=artifacts[key_value], - bars_reason=errors[key_value], - ) - ) - methods = sorted({row["method_version"] for row in rows}) - exchanges = intraday.MARKET_CONTRACTS[args.market]["exchanges"] - rows.sort( - key=lambda item: ( - methods.index(item["method_version"]), - exchanges.index(item["exchange"]), - int(item["exchange_rank"]), - item["canonical_symbol"], - ) - ) - summary = { - method: { - exchange: walk_forward.summarize( - [ - row - for row in rows - if row["method_version"] == method - and row["exchange"] == exchange - ] - ) - for exchange in exchanges - } - for method in methods - } - without_hash = { - "schema": SCHEMA, - "source_skill": "quant-stock-technical", - "manifest_path": str(manifest_path), - "manifest_hash": manifest["manifest_hash"], - "analysis_date": manifest["analysis_date"], - "session_date": args.session_date, - "market": args.market, - "methods": methods, - "screen_hashes": { - method: screens[method]["screen_hash"] for method in methods - }, - "selected_union_count": len(selected_by_key), - "summary_by_method_and_exchange": summary, - "candidate_rows": rows, - "request_count": client.request_count, - "retry_count": client.retry_count, - "cache_hit_count": cache_hits, - "api_mutation_count": 0, - "live_enabled": False, - "validation_status": "RESEARCH_ONLY", - } - output = { - **without_hash, - "outcomes_hash": shared.sha256_bytes( - shared.canonical_json(without_hash).encode("utf-8") - ), - } - shared.atomic_write_json(output_directory / "selected-outcomes.json", output) - return output - - -def self_test() -> None: - rows = [ - { - "method_version": "qta-2.0.0", - "session_date": "2026-07-29", - "exchange": "NASDAQ", - "bars_status": "READY", - "return_60m_bps": "50", - "entry_crossed_by_bar_close": True, - "end_return_from_entry_bps": "10", - "market_open_regime_pass": True, - } - ] - summary = walk_forward.summarize(rows) - assert summary["mean_return_60m_bps"] == "50.00" - print( - json.dumps( - { - "schema": SCHEMA, - "self_test": "PASS", - "api_mutation_count": 0, - "live_enabled": False, - }, - sort_keys=True, - ) - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--self-test", action="store_true") - parser.add_argument("--manifest") - parser.add_argument("--screen", action="append", default=[]) - parser.add_argument("--market", choices=("KR", "US")) - parser.add_argument("--session-date") - parser.add_argument("--output-directory") - parser.add_argument("--environment", choices=("live", "paper"), default="live") - parser.add_argument("--request-interval-ms", type=int, default=120) - args = parser.parse_args() - if not args.self_test: - missing = [ - name - for name in ( - "manifest", - "market", - "session_date", - "output_directory", - ) - if getattr(args, name) is None - ] - if missing: - parser.error(f"missing required arguments: {', '.join(missing)}") - if not args.screen: - parser.error("--screen is required at least once") - try: - date.fromisoformat(args.session_date) - except ValueError: - parser.error("--session-date must be YYYY-MM-DD") - floor = 1000 if args.environment == "paper" else 100 - if args.request_interval_ms < floor: - parser.error( - f"--request-interval-ms must be at least {floor}" - ) - return args - - -def main() -> int: - args = parse_args() - if args.self_test: - self_test() - return 0 - try: - output = run(args) - except ( - SelectedOutcomesBlockedError, - walk_forward.WalkForwardBlockedError, - shared.EodBlockedError, - intraday.ResearchBlockedError, - OSError, - ValueError, - ) as exc: - print( - json.dumps( - { - "schema": SCHEMA, - "status": "BLOCKED", - "reason": str(exc), - "api_mutation_count": 0, - "live_enabled": False, - }, - sort_keys=True, - ), - file=sys.stderr, - ) - return 2 - print( - json.dumps( - { - "schema": SCHEMA, - "status": "READY", - "outcomes_hash": output["outcomes_hash"], - "selected_union_count": output["selected_union_count"], - "request_count": output["request_count"], - "retry_count": output["retry_count"], - "api_mutation_count": 0, - "live_enabled": False, - "output": str( - Path(args.output_directory) / "selected-outcomes.json" - ), - }, - sort_keys=True, - ) - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/plugins/quant-stock-technical/skills/quant-stock-technical/scripts/walk_forward_open1h.py b/plugins/quant-stock-technical/skills/quant-stock-technical/scripts/walk_forward_open1h.py deleted file mode 100644 index b311ecc..0000000 --- a/plugins/quant-stock-technical/skills/quant-stock-technical/scripts/walk_forward_open1h.py +++ /dev/null @@ -1,854 +0,0 @@ -#!/usr/bin/env python3 -"""Run a deterministic selected-candidate opening-hour walk-forward study.""" - -from __future__ import annotations - -import argparse -import json -import statistics -import sys -from concurrent.futures import ProcessPoolExecutor -from datetime import date -from decimal import Decimal, InvalidOperation, ROUND_HALF_EVEN -from pathlib import Path -from typing import Any, Iterable - -import analyze_stock as qta1 -import analyze_stock_v2 as qta2 -import collect_open1h_research as intraday -import evaluate_open1h_research as evaluation -import fetch_kis_kr_eod as shared -import screen_universe as screen - - -SCHEMA = "qta-selected-open1h-walk-forward/v1" -UNIVERSE_MODE = "STATIC_SOURCE_MANIFEST_SURVIVOR_BIASED" -METHODS = (qta1.METHOD_VERSION, qta2.METHOD_VERSION) -EXCHANGES = ("KOSPI", "KOSDAQ") - - -class WalkForwardBlockedError(ValueError): - """Raised when a walk-forward study cannot be completed causally.""" - - -def decimal_value(value: Any, field: str) -> Decimal: - if isinstance(value, bool): - raise WalkForwardBlockedError(f"{field} must be decimal") - try: - parsed = Decimal(str(value)) - except (InvalidOperation, ValueError) as exc: - raise WalkForwardBlockedError(f"{field} must be decimal") from exc - if not parsed.is_finite(): - raise WalkForwardBlockedError(f"{field} must be finite") - return parsed - - -def decimal_text(value: Decimal, places: str = "0.01") -> str: - return format( - value.quantize(Decimal(places), rounding=ROUND_HALF_EVEN), - "f", - ) - - -def absolute_file(value: str, field: str) -> Path: - path = Path(value).expanduser() - if not path.is_absolute() or not path.is_file() or path.is_symlink(): - raise WalkForwardBlockedError( - f"{field} must be an absolute regular non-symlink file" - ) - return path - - -def absolute_directory(value: str, field: str) -> Path: - path = Path(value).expanduser() - if not path.is_absolute(): - raise WalkForwardBlockedError(f"{field} must be absolute") - if path.exists() and (not path.is_dir() or path.is_symlink()): - raise WalkForwardBlockedError( - f"{field} must be a non-symlink directory" - ) - return path - - -def load_manifest(path: Path) -> tuple[dict[str, Any], list[dict[str, Any]]]: - try: - raw = json.loads(path.read_text(encoding="utf-8")) - manifest = screen.normalize_manifest_v2(raw) - except (OSError, json.JSONDecodeError, screen.ScreenBlockedError) as exc: - raise WalkForwardBlockedError( - f"source manifest validation failed: {exc}" - ) from exc - instruments = [ - instrument - for instrument in manifest["instruments"] - if instrument["market"] == "KR" - and instrument["exchange"] in EXCHANGES - ] - if not instruments: - raise WalkForwardBlockedError( - "source manifest has no KOSPI/KOSDAQ instruments" - ) - instruments.sort( - key=lambda item: ( - EXCHANGES.index(item["exchange"]), - item["canonical_symbol"], - ) - ) - for instrument in instruments: - ticker_path = absolute_file( - instrument["ticker_csv"], "instrument.ticker_csv" - ) - benchmark_path = absolute_file( - instrument["benchmark_csv"], "instrument.benchmark_csv" - ) - if shared.sha256_file(ticker_path) != instrument["ticker_csv_sha256"]: - raise WalkForwardBlockedError( - f"ticker hash mismatch: {instrument['canonical_symbol']}" - ) - if ( - shared.sha256_file(benchmark_path) - != instrument["benchmark_csv_sha256"] - ): - raise WalkForwardBlockedError( - f"benchmark hash mismatch: {instrument['benchmark_id']}" - ) - return manifest, instruments - - -def shared_session_pairs( - instruments: list[dict[str, Any]], - *, - end_session: date, - sessions: int, -) -> list[tuple[date, date]]: - benchmark_paths = { - exchange: next( - Path(item["benchmark_csv"]) - for item in instruments - if item["exchange"] == exchange - ) - for exchange in EXCHANGES - } - date_sets = [ - {row.day for row in qta1.read_csv(str(path))} - for path in benchmark_paths.values() - ] - common = sorted( - day for day in set.intersection(*date_sets) if day <= end_session - ) - if len(common) < sessions + 1: - raise WalkForwardBlockedError( - f"benchmarks expose only {max(0, len(common) - 1)} usable sessions" - ) - targets = common[-sessions:] - positions = {day: index for index, day in enumerate(common)} - return [(common[positions[target] - 1], target) for target in targets] - - -def blocked_payload( - instrument: dict[str, Any], - reason: str, - method_version: str, -) -> dict[str, Any]: - return { - "source_skill": "quant-stock-technical", - "result_schema": "quant-stock-technical/v1", - "calculation_status": "BLOCKED", - "reason": reason, - "method_version": method_version, - "market": "KR", - "ticker": instrument["canonical_symbol"], - } - - -def analyze_pair_task( - task: tuple[dict[str, Any], str], -) -> dict[str, Any]: - instrument, analysis_date_text = task - analysis_date = date.fromisoformat(analysis_date_text) - try: - ticker_rows, benchmark_rows = qta1.align( - qta1.read_csv(instrument["ticker_csv"]), - qta1.read_csv(instrument["benchmark_csv"]), - analysis_date, - ) - historical_tick = shared.tick_size( - instrument["exchange"], - Decimal(str(ticker_rows[-1].close)), - ) - common = { - "ticker_bars": ticker_rows, - "benchmark_bars": benchmark_rows, - "market": "KR", - "ticker": instrument["canonical_symbol"], - "tick_size": float(historical_tick), - "source_name": instrument["source_name"], - } - payloads = { - qta1.METHOD_VERSION: qta1.calculate(**common), - qta2.METHOD_VERSION: qta2.calculate(**common), - } - previous_close = Decimal(str(ticker_rows[-1].close)) - except (qta1.BlockedError, OSError, ValueError) as exc: - payloads = { - method: blocked_payload(instrument, str(exc), method) - for method in METHODS - } - previous_close = None - return { - "instrument": instrument, - "previous_close": ( - format(previous_close, "f") if previous_close is not None else None - ), - "payloads": payloads, - } - - -def eligible(payload: dict[str, Any]) -> bool: - return ( - payload.get("calculation_status") == "READY" - and payload.get("setup_status") == "READY" - and decimal_value(payload.get("total_score"), "total_score") - >= Decimal("60") - ) - - -def select_candidates( - analyses: list[dict[str, Any]], - *, - method: str, - top_k: int, -) -> list[dict[str, Any]]: - selected: list[dict[str, Any]] = [] - for exchange in EXCHANGES: - ready = [ - item - for item in analyses - if item["instrument"]["exchange"] == exchange - and eligible(item["payloads"][method]) - ] - ready.sort(key=lambda item: screen.ranking_key(item["payloads"][method])) - for rank, item in enumerate(ready[:top_k], start=1): - selected.append( - { - **item, - "exchange_rank": rank, - } - ) - return selected - - -def bars_cache_path( - output_directory: Path, - *, - session_date: str, - instrument: dict[str, Any], -) -> Path: - return ( - output_directory - / "bars" - / session_date - / instrument["exchange"] - / f"{instrument['canonical_symbol']}.json" - ) - - -def load_or_fetch_bars( - client: shared.KisReadClient, - output_directory: Path, - *, - session_date: str, - instrument: dict[str, Any], -) -> tuple[dict[str, Any], bool]: - path = bars_cache_path( - output_directory, - session_date=session_date, - instrument=instrument, - ) - if path.exists(): - try: - artifact = intraday.normalize_bars_artifact( - json.loads(path.read_text(encoding="utf-8")) - ) - except (OSError, json.JSONDecodeError, intraday.ResearchBlockedError) as exc: - raise WalkForwardBlockedError( - f"invalid cached bars: {path}: {exc}" - ) from exc - if ( - artifact["session_date"] != session_date - or artifact["exchange"] != instrument["exchange"] - or artifact["canonical_symbol"] != instrument["canonical_symbol"] - or artifact["broker_symbol"] != instrument["broker_symbol"] - ): - raise WalkForwardBlockedError( - f"cached bars identity mismatch: {path}" - ) - return artifact, True - raw_rows = intraday.fetch_raw_rows( - client, - market="KR", - exchange=instrument["exchange"], - broker_symbol=instrument["broker_symbol"], - session_date=session_date, - ) - artifact = intraday.bars_artifact( - instrument=instrument, - market="KR", - session_date=session_date, - raw_rows=raw_rows, - ) - shared.atomic_write_json(path, artifact) - return artifact, False - - -def benchmark_rows( - instruments: list[dict[str, Any]], -) -> dict[str, dict[date, qta1.RawBar]]: - output: dict[str, dict[date, qta1.RawBar]] = {} - for exchange in EXCHANGES: - path = next( - item["benchmark_csv"] - for item in instruments - if item["exchange"] == exchange - ) - output[exchange] = {row.day: row for row in qta1.read_csv(path)} - return output - - -def open_regime_bps( - rows: dict[date, qta1.RawBar], - *, - analysis_date: date, - session_date: date, -) -> Decimal: - previous = rows.get(analysis_date) - current = rows.get(session_date) - if previous is None or current is None: - raise WalkForwardBlockedError( - "benchmark is missing an analysis/session pair" - ) - return ( - Decimal(str(current.open)) / Decimal(str(previous.close)) - Decimal(1) - ) * Decimal(10000) - - -def compact_candidate( - *, - method: str, - analysis_date: date, - session_date: date, - selected: dict[str, Any], - artifact: dict[str, Any] | None, - bars_error: str | None, - regime_bps: Decimal, -) -> dict[str, Any]: - instrument = selected["instrument"] - qta = selected["payloads"][method] - base = { - "method_version": method, - "analysis_date": analysis_date.isoformat(), - "session_date": session_date.isoformat(), - "exchange": instrument["exchange"], - "canonical_symbol": instrument["canonical_symbol"], - "broker_symbol": instrument["broker_symbol"], - "exchange_rank": selected["exchange_rank"], - "qta_total_score": decimal_text( - decimal_value(qta["total_score"], "qta_total_score") - ), - "qta_short_score": decimal_text( - decimal_value(qta["short"]["score"], "qta_short_score") - ), - "qta_medium_score": decimal_text( - decimal_value(qta["medium"]["score"], "qta_medium_score") - ), - "qta_long_score": decimal_text( - decimal_value(qta["long"]["score"], "qta_long_score") - ), - "qta_risk_score": decimal_text( - decimal_value(qta["risk"]["score"], "qta_risk_score") - ), - "entry_price": format(decimal_value(qta["entry_price"], "entry"), "f"), - "stop_price": format(decimal_value(qta["stop_price"], "stop"), "f"), - "take_profit_price": format( - decimal_value(qta["take_profit_price"], "take_profit"), "f" - ), - "previous_close": selected["previous_close"], - "market_open_regime_bps": decimal_text(regime_bps), - "market_open_regime_pass": regime_bps >= 0, - "bars_status": "READY" if artifact is not None else "BLOCKED", - "bars_reason": bars_error, - } - if artifact is None: - return { - **base, - "coverage_status": None, - "bar_count": 0, - "return_60m_bps": None, - "maximum_excursion_bps": None, - "minimum_excursion_bps": None, - "entry_touched_by_minute_high": None, - "entry_crossed_by_bar_close": None, - "entry_cross_bar_time": None, - "end_return_from_entry_bps": None, - "bars_hash": None, - } - bars = artifact["bars"] - summary = artifact["summary"] - entry = decimal_value(qta["entry_price"], "entry") - cross_time = evaluation.first_bar_close_cross(bars, entry) - end_close = Decimal(summary["end_close"]) - return { - **base, - "coverage_status": intraday.coverage_summary( - bars, market="KR" - )["coverage_status"], - "bar_count": len(bars), - "return_60m_bps": summary["return_60m_bps"], - "maximum_excursion_bps": summary["maximum_excursion_bps"], - "minimum_excursion_bps": summary["minimum_excursion_bps"], - "entry_touched_by_minute_high": ( - Decimal(summary["window_high"]) >= entry - ), - "entry_crossed_by_bar_close": cross_time is not None, - "entry_cross_bar_time": cross_time, - "end_return_from_entry_bps": ( - intraday.bps(end_close, entry) if cross_time is not None else None - ), - "bars_hash": artifact["bars_hash"], - } - - -def mean(values: list[Decimal]) -> str | None: - if not values: - return None - return decimal_text(sum(values, Decimal(0)) / Decimal(len(values))) - - -def median(values: list[Decimal]) -> str | None: - if not values: - return None - return decimal_text(statistics.median(values)) - - -def fraction(numerator: int, denominator: int) -> str | None: - if denominator == 0: - return None - return decimal_text( - Decimal(numerator) / Decimal(denominator), - "0.000001", - ) - - -def summarize(rows: list[dict[str, Any]]) -> dict[str, Any]: - ready = [row for row in rows if row["bars_status"] == "READY"] - returns = [ - decimal_value(row["return_60m_bps"], "return_60m_bps") - for row in ready - ] - crossed = [ - row for row in ready if row["entry_crossed_by_bar_close"] is True - ] - entry_returns = [ - decimal_value(row["end_return_from_entry_bps"], "entry return") - for row in crossed - ] - regime_evaluated = [ - row - for row in ready - if isinstance(row["market_open_regime_pass"], bool) - ] - regime_ready = [ - row for row in ready if row["market_open_regime_pass"] is True - ] - regime_returns = [ - decimal_value(row["return_60m_bps"], "regime return") - for row in regime_ready - ] - regime_crossed = [ - row - for row in crossed - if row["market_open_regime_pass"] is True - ] - regime_entry_returns = [ - decimal_value(row["end_return_from_entry_bps"], "regime entry return") - for row in regime_crossed - ] - session_means: list[Decimal] = [] - for session_date in sorted({row["session_date"] for row in ready}): - values = [ - decimal_value(row["return_60m_bps"], "session return") - for row in ready - if row["session_date"] == session_date - ] - if values: - session_means.append( - sum(values, Decimal(0)) / Decimal(len(values)) - ) - return { - "selected_count": len(rows), - "bars_ready_count": len(ready), - "bars_blocked_count": len(rows) - len(ready), - "mean_return_60m_bps": mean(returns), - "median_return_60m_bps": median(returns), - "positive_fraction": fraction( - sum(value > 0 for value in returns), len(returns) - ), - "entry_cross_count": len(crossed), - "entry_cross_fraction": fraction(len(crossed), len(ready)), - "mean_end_return_from_entry_bps": mean(entry_returns), - "median_end_return_from_entry_bps": median(entry_returns), - "positive_end_return_from_entry_fraction": fraction( - sum(value > 0 for value in entry_returns), - len(entry_returns), - ), - "market_open_regime_evaluated_count": len(regime_evaluated), - "market_open_regime_pass_count": len(regime_ready), - "market_open_regime_pass_fraction": fraction( - len(regime_ready), len(regime_evaluated) - ), - "market_open_regime_pass_mean_return_60m_bps": mean(regime_returns), - "market_open_regime_entry_cross_count": len(regime_crossed), - "market_open_regime_cross_mean_end_return_from_entry_bps": mean( - regime_entry_returns - ), - "mean_equal_weight_session_return_60m_bps": mean(session_means), - } - - -def grouped_summary(rows: list[dict[str, Any]]) -> dict[str, Any]: - return { - method: { - exchange: summarize( - [ - row - for row in rows - if row["method_version"] == method - and row["exchange"] == exchange - ] - ) - for exchange in EXCHANGES - } - for method in METHODS - } - - -def run(args: argparse.Namespace) -> dict[str, Any]: - manifest_path = absolute_file(args.manifest, "manifest") - output_directory = absolute_directory( - args.output_directory, "output_directory" - ) - output_directory.mkdir(parents=True, exist_ok=True) - manifest, instruments = load_manifest(manifest_path) - pairs = shared_session_pairs( - instruments, - end_session=date.fromisoformat(args.end_session), - sessions=args.sessions, - ) - benchmark = benchmark_rows(instruments) - key, secret, token = shared.load_api_credentials(args.environment) - client = shared.KisReadClient( - environment=args.environment, - app_key=key, - app_secret=secret, - access_token=token, - interval_ms=args.request_interval_ms, - ) - workers = min(8, max(1, args.workers)) - candidate_rows: list[dict[str, Any]] = [] - session_receipts: list[dict[str, Any]] = [] - cache_hits = 0 - for analysis_date, session_date in pairs: - tasks = [ - (instrument, analysis_date.isoformat()) - for instrument in instruments - ] - with ProcessPoolExecutor(max_workers=workers) as executor: - analyses = list( - executor.map(analyze_pair_task, tasks, chunksize=16) - ) - selections = { - method: select_candidates( - analyses, - method=method, - top_k=args.top_k, - ) - for method in METHODS - } - selected_by_key: dict[ - tuple[str, str], dict[str, Any] - ] = {} - for method_rows in selections.values(): - for item in method_rows: - instrument = item["instrument"] - selected_by_key[ - (instrument["exchange"], instrument["canonical_symbol"]) - ] = item - artifacts: dict[tuple[str, str], dict[str, Any] | None] = {} - errors: dict[tuple[str, str], str | None] = {} - for key_value, item in sorted(selected_by_key.items()): - try: - artifact, cache_hit = load_or_fetch_bars( - client, - output_directory, - session_date=session_date.isoformat(), - instrument=item["instrument"], - ) - artifacts[key_value] = artifact - errors[key_value] = None - cache_hits += int(cache_hit) - except ( - shared.EodBlockedError, - intraday.ResearchBlockedError, - WalkForwardBlockedError, - ) as exc: - artifacts[key_value] = None - errors[key_value] = str(exc) - for method, method_rows in selections.items(): - for item in method_rows: - instrument = item["instrument"] - key_value = ( - instrument["exchange"], - instrument["canonical_symbol"], - ) - regime = open_regime_bps( - benchmark[instrument["exchange"]], - analysis_date=analysis_date, - session_date=session_date, - ) - candidate_rows.append( - compact_candidate( - method=method, - analysis_date=analysis_date, - session_date=session_date, - selected=item, - artifact=artifacts[key_value], - bars_error=errors[key_value], - regime_bps=regime, - ) - ) - session_receipts.append( - { - "analysis_date": analysis_date.isoformat(), - "session_date": session_date.isoformat(), - "analyzed_instrument_count": len(analyses), - "qta1_blocked_count": sum( - item["payloads"][qta1.METHOD_VERSION].get( - "calculation_status" - ) - != "READY" - for item in analyses - ), - "qta2_blocked_count": sum( - item["payloads"][qta2.METHOD_VERSION].get( - "calculation_status" - ) - != "READY" - for item in analyses - ), - "qta1_selected_count": len( - selections[qta1.METHOD_VERSION] - ), - "qta2_selected_count": len( - selections[qta2.METHOD_VERSION] - ), - "unique_bar_request_candidates": len(selected_by_key), - "bars_blocked_count": sum( - artifact is None for artifact in artifacts.values() - ), - } - ) - print( - json.dumps(session_receipts[-1], sort_keys=True), - file=sys.stderr, - flush=True, - ) - candidate_rows.sort( - key=lambda item: ( - item["session_date"], - METHODS.index(item["method_version"]), - EXCHANGES.index(item["exchange"]), - item["exchange_rank"], - item["canonical_symbol"], - ) - ) - without_hash = { - "schema": SCHEMA, - "source_skill": "quant-stock-technical", - "source_manifest_path": str(manifest_path), - "source_manifest_hash": manifest["manifest_hash"], - "source_manifest_as_of": manifest["as_of"], - "universe_mode": UNIVERSE_MODE, - "universe_limitations": [ - ( - "the source manifest membership is held static across earlier " - "sessions, creating survivorship and membership bias" - ), - ( - "historical bid-ask spreads and exact quote-cycle market-regime " - "states are unavailable" - ), - ( - "market_open_regime_bps is the exact benchmark opening gap, " - "not the runner's later intraday regime observation" - ), - ], - "market": "KR", - "methods": list(METHODS), - "top_k_per_exchange": args.top_k, - "minimum_total_score": "60", - "session_count": len(pairs), - "first_session_date": pairs[0][1].isoformat(), - "last_session_date": pairs[-1][1].isoformat(), - "session_receipts": session_receipts, - "summary_by_method_and_exchange": grouped_summary(candidate_rows), - "candidate_rows": candidate_rows, - "request_count": client.request_count, - "retry_count": client.retry_count, - "cache_hit_count": cache_hits, - "api_mutation_count": 0, - "live_enabled": False, - "validation_status": "RESEARCH_ONLY", - } - output = { - **without_hash, - "walk_forward_hash": shared.sha256_bytes( - shared.canonical_json(without_hash).encode("utf-8") - ), - } - output_path = output_directory / "walk-forward.json" - shared.atomic_write_json(output_path, output) - return output - - -def self_test() -> None: - rows = [ - { - "method_version": qta1.METHOD_VERSION, - "session_date": "2026-01-02", - "exchange": "KOSPI", - "bars_status": "READY", - "return_60m_bps": "100", - "entry_crossed_by_bar_close": True, - "end_return_from_entry_bps": "25", - "market_open_regime_pass": True, - }, - { - "method_version": qta1.METHOD_VERSION, - "session_date": "2026-01-03", - "exchange": "KOSPI", - "bars_status": "READY", - "return_60m_bps": "-50", - "entry_crossed_by_bar_close": False, - "end_return_from_entry_bps": None, - "market_open_regime_pass": False, - }, - ] - summary = summarize(rows) - assert summary["mean_return_60m_bps"] == "25.00" - assert summary["positive_fraction"] == "0.500000" - assert summary["entry_cross_fraction"] == "0.500000" - assert summary["positive_end_return_from_entry_fraction"] == "1.000000" - assert summary["market_open_regime_pass_mean_return_60m_bps"] == "100.00" - assert summary["market_open_regime_entry_cross_count"] == 1 - print( - json.dumps( - { - "schema": SCHEMA, - "self_test": "PASS", - "api_mutation_count": 0, - "live_enabled": False, - }, - sort_keys=True, - ) - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--self-test", action="store_true") - parser.add_argument("--manifest") - parser.add_argument("--output-directory") - parser.add_argument("--end-session") - parser.add_argument("--sessions", type=int, default=20) - parser.add_argument("--top-k", type=int, default=2) - parser.add_argument("--environment", choices=("live", "paper"), default="live") - parser.add_argument("--request-interval-ms", type=int, default=120) - parser.add_argument("--workers", type=int, default=8) - args = parser.parse_args() - if not args.self_test: - missing = [ - name - for name in ("manifest", "output_directory", "end_session") - if getattr(args, name) is None - ] - if missing: - parser.error(f"missing required arguments: {', '.join(missing)}") - try: - date.fromisoformat(args.end_session) - except ValueError as exc: - parser.error("--end-session must be YYYY-MM-DD") - if args.sessions < 2 or args.sessions > 252: - parser.error("--sessions must be between 2 and 252") - if args.top_k < 1 or args.top_k > 20: - parser.error("--top-k must be between 1 and 20") - floor = 1000 if args.environment == "paper" else 100 - if args.request_interval_ms < floor: - parser.error( - f"--request-interval-ms must be at least {floor}" - ) - if args.workers < 1 or args.workers > 8: - parser.error("--workers must be between 1 and 8") - return args - - -def main() -> int: - args = parse_args() - if args.self_test: - self_test() - return 0 - try: - output = run(args) - except ( - WalkForwardBlockedError, - shared.EodBlockedError, - intraday.ResearchBlockedError, - OSError, - ValueError, - ) as exc: - print( - json.dumps( - { - "schema": SCHEMA, - "status": "BLOCKED", - "reason": str(exc), - "api_mutation_count": 0, - "live_enabled": False, - }, - sort_keys=True, - ), - file=sys.stderr, - ) - return 2 - print( - json.dumps( - { - "schema": SCHEMA, - "status": "READY", - "session_count": output["session_count"], - "walk_forward_hash": output["walk_forward_hash"], - "output": str( - Path(args.output_directory) / "walk-forward.json" - ), - "request_count": output["request_count"], - "retry_count": output["retry_count"], - "api_mutation_count": 0, - "live_enabled": False, - }, - sort_keys=True, - ) - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/plugins/stock-scenario-story/.claude-plugin/plugin.json b/plugins/stock-scenario-story/.claude-plugin/plugin.json index 3a4ba27..41d1fa6 100644 --- a/plugins/stock-scenario-story/.claude-plugin/plugin.json +++ b/plugins/stock-scenario-story/.claude-plugin/plugin.json @@ -2,7 +2,7 @@ "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "stock-scenario-story", "displayName": "Stock Scenario Story", - "version": "0.1.0", + "version": "0.1.1", "description": "Research and narrate a sourced, numbers-free market story after a validated quant handoff.", "author": { "name": "mrcha033", diff --git a/plugins/stock-scenario-story/.codex-plugin/plugin.json b/plugins/stock-scenario-story/.codex-plugin/plugin.json index 502bf74..a3e98e5 100644 --- a/plugins/stock-scenario-story/.codex-plugin/plugin.json +++ b/plugins/stock-scenario-story/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "stock-scenario-story", - "version": "0.1.0", + "version": "0.1.1", "description": "Research and narrate a sourced, numbers-free market story after a validated quant handoff.", "author": { "name": "mrcha033", diff --git a/plugins/stock-scenario-story/skills/stock-scenario-story/SKILL.md b/plugins/stock-scenario-story/skills/stock-scenario-story/SKILL.md index 86d57e0..08a335c 100644 --- a/plugins/stock-scenario-story/skills/stock-scenario-story/SKILL.md +++ b/plugins/stock-scenario-story/skills/stock-scenario-story/SKILL.md @@ -1,6 +1,6 @@ --- name: stock-scenario-story -description: Research and perform a vivid, numbers-free, evidence-grounded stock story only after a complete quant-stock-technical JSON result has passed the bundled handoff validator. Use when the user wants a flamboyant showman-style qualitative tale about a company's products, characters, competition, customers, catalysts, culture, policy pressures, dreams, and dangers as pure entertainment. Never run standalone, reveal or restate upstream numbers, reconstruct missing quant inputs, or present the performance as a forecast or personalized investment advice. +description: Research and perform a vivid, numbers-free, evidence-grounded stock story only after a complete external QTA JSON result matching quant-stock-technical/v1 has passed the bundled handoff validator. Use when the user wants a flamboyant showman-style qualitative tale about a company's products, characters, competition, customers, catalysts, culture, policy pressures, dreams, and dangers as pure entertainment. Never run standalone, reveal or restate upstream numbers, reconstruct missing quant inputs, or present the performance as a forecast or personalized investment advice. --- # Stock Scenario Story @@ -9,7 +9,10 @@ Turn a validated deterministic technical result into an entertaining but auditab ## Mandatory invocation gate -Require the complete JSON output from `$quant-stock-technical`. A summary, screenshot, Markdown table, manually typed score, or output from another calculator is insufficient. +Require the complete JSON output from the standalone QTA system. It must match +the legacy `quant-stock-technical/v1` contract; the producer is not a required +marketplace skill. A summary, screenshot, Markdown table, manually typed score, +or output from another calculator is insufficient. Run: @@ -17,7 +20,9 @@ Run: python3 scripts/validate_quant_handoff.py /absolute/path/to/quant-result.json --output /absolute/path/to/quant-receipt.json ``` -Continue only when the receipt has `gate_status: PASSED`. If it returns `BLOCKED`, stop and request a fresh JSON run from `$quant-stock-technical`. Never repair, infer, or hand-create the missing upstream fields. +Continue only when the receipt has `gate_status: PASSED`. If it returns +`BLOCKED`, stop and request a fresh JSON export from the standalone QTA system. +Never repair, infer, or hand-create the missing upstream fields. Treat the receipt as an invisible backstage pass. Use it only to prove that the deterministic analysis ran first. Never quote, paraphrase, compare, hint at, or display its numeric contents in the qualitative output. diff --git a/plugins/stock-scenario-story/skills/stock-scenario-story/agents/openai.yaml b/plugins/stock-scenario-story/skills/stock-scenario-story/agents/openai.yaml index bacba26..f448aad 100644 --- a/plugins/stock-scenario-story/skills/stock-scenario-story/agents/openai.yaml +++ b/plugins/stock-scenario-story/skills/stock-scenario-story/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Stock Scenario Story" short_description: "Research-driven stories from validated quant results" - default_prompt: "Use $stock-scenario-story after $quant-stock-technical to research and narrate evidence-backed market scenarios." + default_prompt: "Use $stock-scenario-story with a complete external QTA JSON result matching quant-stock-technical/v1 to research and narrate evidence-backed market scenarios." diff --git a/release/catalog.json b/release/catalog.json index 400b2bf..ddc2983 100644 --- a/release/catalog.json +++ b/release/catalog.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "distributionVersion": "2.4.5", + "distributionVersion": "2.5.0", "releaseTagPrefix": "skills-v", "guaranteeScope": "Package compatibility guarantees archive layout, manifest placement, and byte integrity. Runtime compatibility remains conditional on each skill's declared tools, authentication, network, and data requirements.", "formats": { @@ -94,45 +94,9 @@ ], "notes": "The skill deliberately depends on an existing local browser login and does not accept copied credentials or cookies." }, - { - "name": "quant-stock-polling-trader", - "version": "0.5.4", - "runtime": { - "codex": "supported", - "claudeCode": "supported", - "chatgptWeb": "unsupported", - "claudeWeb": "unsupported" - }, - "requirements": [ - "Python 3", - "local shell execution", - "KIS Open API credentials for the selected paper or live scope", - "outbound access to the selected broker API", - "secure local secret injection or a permission-0600 dotenv fallback", - "frozen screen, account, exposure, calendar, risk, and execution-policy inputs", - "systemd for generated Linux user units" - ], - "notes": "The daily KIS live/shadow pipeline now accepts research-only QTA2 screens, performs a fresh same-session benchmark-regime check before entry quotes, and persists strategy fill ownership in an account/market position ledger across session dates. Daily preparation remains market-scoped, ntfy stays best-effort, and API mutations remain fixed to zero." - }, - { - "name": "quant-stock-technical", - "version": "0.4.6", - "runtime": { - "codex": "supported", - "claudeCode": "supported", - "chatgptWeb": "conditional", - "claudeWeb": "conditional" - }, - "requirements": [ - "Python 3", - "user-provided finalized OHLCV and benchmark files, or KIS Open API credentials for read-only Korean and U.S. EOD acquisition", - "outbound KRX, Nasdaq Trader, KIS, and benchmark source access" - ], - "notes": "Adds read-only selected-candidate point-in-time outcomes and an explicitly survivor-biased 2-252-session Korean opening-hour walk-forward diagnostic. A flat initial true-range seed now initializes ADX at zero instead of making it permanently unavailable, and feature failures identify the exact field. QTA2 remains research-only." - }, { "name": "stock-scenario-story", - "version": "0.1.0", + "version": "0.1.1", "runtime": { "codex": "conditional", "claudeCode": "conditional", @@ -140,11 +104,11 @@ "claudeWeb": "conditional" }, "requirements": [ - "complete quant-stock-technical JSON handoff", + "complete external QTA JSON handoff matching quant-stock-technical/v1", "Python 3", "web research with point-of-use citations" ], - "notes": "The skill is never standalone at runtime even though its upload archive is standalone." + "notes": "The skill consumes the legacy quant-stock-technical/v1 handoff contract from an external QTA system; the producer is no longer distributed in this marketplace." }, { "name": "yonsei-central-student-governance-counsel", diff --git a/release/release-notes.md b/release/release-notes.md index e4a2088..8c8485a 100644 --- a/release/release-notes.md +++ b/release/release-notes.md @@ -1,6 +1,13 @@ This release contains the versioned distribution packages declared in `release/catalog.json`. +- Distribution 2.5.0 removes `quant-stock-technical` and + `quant-stock-polling-trader` from the skill marketplace. Their deterministic + source, tests, and active shadow automation were ported byte-for-byte into a + dedicated standalone QTA project before removal. `stock-scenario-story` + is updated to 0.1.1, remains available, and continues to validate the legacy + `quant-stock-technical/v1` external handoff contract. + - Distribution 2.4.5 updates `quant-stock-technical` to 0.4.6. It adds read-only selected-candidate point-in-time outcomes and an explicitly survivor-biased 2-252-session Korean opening-hour walk-forward diagnostic. diff --git a/skills/quant-stock-polling-trader/SKILL.md b/skills/quant-stock-polling-trader/SKILL.md deleted file mode 100644 index 9674a9c..0000000 --- a/skills/quant-stock-polling-trader/SKILL.md +++ /dev/null @@ -1,192 +0,0 @@ ---- -name: quant-stock-polling-trader -description: Prepare and supervise deterministic Korean and U.S. KIS shadow sessions from a frozen quant-stock-technical screen. Use when the user wants secure KIS credential setup, pre-open screening, settled-cash sizing, first-hour quote polling, reconciliation, or recurring systemd automation. The daily path is shadow-only and sends no order, correction, or cancellation mutations. ---- - -# Quant Stock Polling Trader - -Run a small, direct pipeline: - -```text -prepare -> snapshot and plan -> first-hour shadow session -> reconcile -``` - -The default daily workflow does not use approved source digests, provenance -signatures, account-identity HMACs, trading-arm files, or mandatory Toss/NH -exports. Do not add those gates back unless the user explicitly asks for a -separate audited deployment profile. - -## Credentials - -Keep credentials out of chat, repositories, command arguments, plans, and -status files. Resolve them from the process environment first and -`~/.config/mrcha-skills/secrets.env` second. The fallback file must be a -regular user-owned `0600` file. - -Use separate KIS paper and live values: - -```text -QTA_KIS_PAPER_APP_KEY -QTA_KIS_PAPER_APP_SECRET -QTA_KIS_PAPER_ACCOUNT_PREFIX -QTA_KIS_PAPER_ACCOUNT_PRODUCT -QTA_KIS_LIVE_APP_KEY -QTA_KIS_LIVE_APP_SECRET -QTA_KIS_LIVE_ACCOUNT_PREFIX -QTA_KIS_LIVE_ACCOUNT_PRODUCT -``` - -For the daily workflow use `live` credentials only to read the live account -and market data while keeping `mode=shadow`. - -Check presence first: - -```bash -python3 scripts/broker_credentials.py status --environment live -``` - -If configuration is missing, ask the user to run this in their own terminal: - -```bash -python3 scripts/broker_credentials.py configure --environment live -``` - -Never ask the user to paste a credential into chat. Report missing variable -names only. Wait for the user to confirm that interactive configuration has -finished, then check presence again and authenticate: - -```bash -python3 scripts/broker_credentials.py status --environment live -python3 scripts/broker_credentials.py auth-check --environment live -``` - -Authentication is proven only when the result says `AUTHENTICATED`. An -authenticated token does not enable live trading. - -## Daily workflow - -Read `references/daily-pipeline-contract.md`, then: - -1. Use one stable `qta-daily-shadow-config/v2` file. -2. Run `scripts/daily_pipeline.py prepare` before the selected market opens. - It freezes the official calendar, stops normally on a closed market, - refreshes only that market's exchange pair and adjusted EOD cache, and - creates the deterministic screen. -3. Run `scripts/daily_pipeline.py snapshot` shortly before the open. It reads - KIS settled cash, positions, and open orders, optionally merges any - explicitly configured external exposure files, and creates the plan. -4. Run `scripts/daily_pipeline.py entry` before the 30-second authentication - warm-up. The Python runner polls the first hour mechanically and then - reconciles the local intent ledger. -5. Use `scripts/systemd_units.py generate` for recurring user services and - timers. When the user asks to automate, generation is intermediate work: - verify, install, enable, start, and inspect the requested units. - -Daily preparation starts at `01:00` in each market timezone. This leaves eight -hours before the Korean open and eight and a half hours before the U.S. open, -without making a Korean session wait for U.S. EOD or vice versa. -Finish an initial multi-year bootstrap before relying on a daily timer; the -timer is intended to resume and increment an existing cache. - -Set optional `qta_method_version` to `qta-2.0.0` for the research-only -first-hour model. The runner then performs one read-only benchmark-regime call -per active exchange and cycle before stock quotes. A negative, stale, -wrong-session, or unavailable regime observation blocks new entries for that -cycle. QTA2 cannot run in paper or live mode. - -To receive optional ntfy completion/block notifications, place these values in -the same mode-`0600` environment file used by the generated services: - -```text -QTA_NTFY_TOPIC_URL=https://ntfy.example/a-long-private-topic -QTA_NTFY_TOKEN=optional-access-token -``` - -The full topic URL and token are never written to a bundle, status file, or -journal. Notification delivery is best-effort and cannot turn a completed -shadow stage into a trading failure. A journal receipt records only -`SENT`, `FAILED`, or `DISABLED`. - -The state path is: - -```text -RUNTIME_ROOT/state/// -``` - -The account alias is nonsecret and configured by the user. A current session -is never entered twice. An unresolved broker mutation from a paper-mode run -still requires reconciliation before another entry. - -Filled strategy quantities are stored in -`RUNTIME_ROOT/state///positions.sqlite3`, outside the -date directory. Before a later session reads quotes, the runner reconciles -that ledger against the frozen KIS account snapshot. Matching positions roll -forward as `CARRY_EXIT_ONLY`; quantity differences, working orders, duplicate -ownership, or a prior `MANUAL_BLOCK` stop new entries. Holdings not created by -this strategy remain visible as unmanaged broker positions and are never -silently adopted or sold. - -## Direct planning and polling - -For a non-recurring session, read `references/execution-contract.md`, -`references/account-snapshot-contract.md`, and -`references/broker-contracts.md`. - -```bash -python3 scripts/account_snapshot.py collect --job /absolute/account-job.json -python3 scripts/plan_orders.py \ - --screen /absolute/screen.json \ - --account /absolute/account.json \ - --exposure /absolute/exposure.json \ - --risk /absolute/risk.json \ - --execution /absolute/execution.json \ - --output /absolute/order-plan.json -python3 scripts/run_session.py run \ - --plan /absolute/order-plan.json \ - --broker kis-live \ - --mode shadow \ - --state-dir /absolute/state \ - --output /absolute/session-status.json -python3 scripts/reconcile.py \ - --plan /absolute/order-plan.json \ - --broker kis-live \ - --state-dir /absolute/state \ - --output /absolute/reconciliation-status.json -``` - -Additional Toss or NH exposure files are optional. If the user configures one, -validate and merge it; if none is configured, continue with the KIS account -view without fabricating an empty external-broker file. - -## Fixed safety boundary - -- The daily path accepts only `broker=kis-live`, `mode=shadow`. -- `live_enabled` stays `false` and `api_mutation_count` stays `0`. -- Do not submit, amend, or cancel an order in shadow mode. -- Use settled cash only. Exclude borrowed buying power. -- Keep margin, shorting, automatic FX, and additions to existing positions - disabled. -- Use the official calendar snapshot; do not guess holidays, early closes, - timezones, or DST. -- Use complete prior-session data; do not fill gaps with internet quotes or - estimates. -- Keep ranking, sizing, quote checks, and cycle timing deterministic. -- Do not put an LLM in the polling loop. -- Never claim profit, guaranteed returns, or a recommendation. - -Run all bundled self-tests after changing the implementation: - -```bash -python3 scripts/execution_core.py --self-test -python3 scripts/freeze_market_session.py --self-test -python3 scripts/broker_adapters.py --self-test -python3 scripts/broker_credentials.py self-test -python3 scripts/account_snapshot.py self-test -python3 scripts/market_calendar.py self-test -python3 scripts/daily_pipeline.py self-test -python3 scripts/plan_orders.py --self-test -python3 scripts/position_lifecycle.py --self-test -python3 scripts/run_session.py self-test -python3 scripts/reconcile.py --self-test -python3 scripts/systemd_units.py self-test -``` diff --git a/skills/quant-stock-polling-trader/agents/openai.yaml b/skills/quant-stock-polling-trader/agents/openai.yaml deleted file mode 100644 index f3a6248..0000000 --- a/skills/quant-stock-polling-trader/agents/openai.yaml +++ /dev/null @@ -1,4 +0,0 @@ -interface: - display_name: "Quant Stock Polling Trader" - short_description: "Simple KIS shadow polling and systemd automation" - default_prompt: "Use $quant-stock-polling-trader to configure a direct KIS shadow workflow, prepare the market, snapshot and plan, run first-hour polling, reconcile, and automate it with systemd without approval-digest or trading-arm gates." diff --git a/skills/quant-stock-polling-trader/references/account-snapshot-contract.md b/skills/quant-stock-polling-trader/references/account-snapshot-contract.md deleted file mode 100644 index d9e9973..0000000 --- a/skills/quant-stock-polling-trader/references/account-snapshot-contract.md +++ /dev/null @@ -1,117 +0,0 @@ -# Account snapshot collector contract - -Use `scripts/account_snapshot.py` only for KIS live credentials in shadow mode. -It sends read-only requests and emits one `qta-account-snapshot/v2`, one -`qta-exposure-snapshot/v2`, and one readable status with -`api_mutation_count: 0`. - -## Inputs - -The job is nonsecret JSON. Every path must be absolute: - -```json -{ - "schema": "qta-account-snapshot-job/v2", - "broker": "kis-live", - "mode": "shadow", - "market": "KR", - "account_alias": "kis-live-kr", - "universe_manifest": "/secure/runtime/universe-manifest.json", - "fx_to_krw": "1", - "manual_exposure_components": [ - "/secure/runtime/exposure/toss.json", - "/secure/runtime/exposure/nh.json" - ], - "manual_component_max_age_seconds": 1800, - "output_account_path": "/secure/runtime/account-kr.json", - "output_exposure_path": "/secure/runtime/exposure-kr.json", - "output_status_path": "/secure/runtime/account-kr-status.json" -} -``` - -For a U.S. job, set `fx_to_krw` to `KIS_PRESENT_BALANCE` so the same -`CTRP6504R` snapshot supplies its positive `frst_bltn_exrt` value. A caller may -instead bind an exact observed positive numeric value, in which case the -receipt records `fx_source: JOB_INPUT`. KIS legitimately omits the USD cash -row when the account has neither settled USD nor a U.S. position. In that -case the collector freezes `settled_cash: "0"` and `fx_to_krw: null`, records -`fx_source: NOT_APPLICABLE:NO_SETTLED_USD_OR_POSITION`, and the planner -deterministically produces no positive quantity. It does not use -auto-exchange buying power or invent an exchange rate. If a position exists -without a USD cash row, the collector uses the position's positive -`bass_exrt`; conflicting rates or cash rows are `BLOCKED`. - -Never insert a placeholder or silently reuse an old rate. The collector -validates the manifest hash and uses its `broker_symbol -> exchange` mapping. -Anything held or working outside KOSPI, KOSDAQ, NYSE, and NASDAQ is `BLOCKED`; -extend the exposure schema instead of assigning a false exchange. - -KIS credentials follow the resolution rules in `SKILL.md`. Account routing -values and secrets are never written. The nonsecret `account_alias` identifies -the local account view. - -## Optional additional exposure - -Toss and NH files are optional. Use them only when the user wants those -holdings included and has a fresh export or verified account view: - -```bash -python3 scripts/account_snapshot.py manual-template \ - --broker toss \ - --source-as-of 2026-07-27T08:45:00+09:00 \ - --output /secure/runtime/exposure/toss.json -``` - -Each configured component has exactly: - -```json -{ - "schema": "qta-exposure-component/v1", - "broker": "toss", - "source_kind": "verified-manual", - "source_as_of": "2026-07-27T08:45:00+09:00", - "positions": [ - { - "market": "US", - "exchange": "NASDAQ", - "symbol": "AAPL", - "quantity": "0.182408", - "market_value_krw": "89096" - } - ] -} -``` - -`quantity` may be `null` only when the source genuinely omits it. Include -fractional holdings. The collector rejects invalid configured components, but -an empty `manual_exposure_components` list is valid and means “use KIS only.” -It does not scrape a browser, infer a symbol from a Korean display name, or -fabricate a source time. - -## Cash derivation - -The account contract requires settled, unborrowed cash: - -- KR uses the minimum nonnegative value among KIS `dnca_tot_amt`, - `nxdy_excc_amt`, and `prvs_rcdl_excc_amt`. -- US uses the minimum nonnegative USD value among `frcr_dncl_amt_2`, - `frcr_drwg_psbl_amt_1`, and `nxdy_frcr_drwg_psbl_amt`. -- An absent USD row means zero settled USD; it never authorizes KRW - auto-exchange buying power. -- Loan amounts are preserved separately as `borrowed_buying_power` and never - added to `settled_cash`. - -This is a settlement-safe derivation. A new broker field or account product -must be contract-probed before changing it. - -## Collection - -```bash -python3 scripts/account_snapshot.py collect \ - --job /secure/runtime/account-snapshot-kr-job.json -``` - -The collector freezes one common `as_of` after the KIS reads and any configured -component validation complete. The account and exposure objects therefore -meet the planner's exact timestamp equality requirement. Missing or stale -configured inputs return `BLOCKED`; unconfigured external brokers do not. diff --git a/skills/quant-stock-polling-trader/references/broker-contracts.md b/skills/quant-stock-polling-trader/references/broker-contracts.md deleted file mode 100644 index 98723fe..0000000 --- a/skills/quant-stock-polling-trader/references/broker-contracts.md +++ /dev/null @@ -1,209 +0,0 @@ -# Broker contracts - -Checked against official sources on 2026-07-26. Re-run contract probes whenever -the broker changes an API version or order rule. - -## Toss Securities Open API 1.2.4 - -Source of truth: -`https://openapi.tossinvest.com/openapi-docs/latest/openapi.json`. - -Toss currently exposes one production REST server and no documented paper -server. Keep mutations disabled until the account's staged Open API access is -active. - -Core paths: - -```text -POST /oauth2/token -GET /api/v1/accounts -GET /api/v1/prices -GET /api/v1/orderbook -GET /api/v1/stocks -GET /api/v1/holdings -GET /api/v1/buying-power -GET /api/v1/orders -POST /api/v1/orders -GET /api/v1/orders/{orderId} -POST /api/v1/orders/{orderId}/modify -POST /api/v1/orders/{orderId}/cancel -POST /api/v1/conditional-orders -``` - -`GET /api/v1/stocks` accepts one to 200 requested symbols and returns market, -security type, common-share flag, listing status, currency, and Korean -liquidation/suspension details. It does not enumerate the full catalog. Once -this account has API access, query the frozen official-universe symbols in -deterministic batches, persist the complete responses as a dated snapshot, and -add a separate Toss-qualified universe contract. Until that snapshot path is -implemented and verified, never reuse the KIS-qualified v2 `broker_symbol` for -Toss. - -Send `Authorization: Bearer ...`; account operations also require -`X-Tossinvest-Account: accountSeq`. Centralize token refresh because issuing a -new token invalidates the prior token for that client. If -`QTA_TOSS_ACCESS_TOKEN` supplies an external token, require its timezone-aware -ISO-8601 expiry in `QTA_TOSS_ACCESS_TOKEN_EXPIRES_AT`; block when exactly one -of the pair is present. - -Keep `accountSeq` in the runtime credential environment. Do not copy it into a -plan, status file, or error message. - -Bind every Toss mutation request hash to the uppercase HTTP method, exact API -path, account sequence header value, and canonical request body. An identical -body for another account or path must never share the same hash. - -Pace `MARKET_DATA`, `ORDER`, and `ORDER_HISTORY` independently. The initial -client limits are 10, 3, and 5 TPS respectively; the conservative 3 TPS order -limit remains safe during the documented 09:00–09:10 KST peak. Only lower a -client limit when `X-RateLimit-Limit` reports a smaller current value. Do not -blindly retry an order mutation after a 429 or timeout. - -After a mutation has been sent, an HTTP 5xx, malformed success body, or -success body without an order ID is ambiguous and must enter `UNKNOWN`. -Only an explicit structured Toss 4xx rejection is terminal `REJECTED`. - -Both price and order-book source timestamps are nullable in the schema. Require -both during Toss polling, parse each as timezone-aware ISO-8601, and block the -entry when either is missing, invalid, future-dated beyond tolerance, or older -than the frozen quote-age limit. Select the lowest positive ask and highest -positive bid rather than trusting array position. - -`clientOrderId` is optional on creates, limited to 36 safe characters, and -idempotent for only 10 minutes. A same-key/same-body retry returns the original -result; a changed body conflicts. Order queries cannot search or return that -client ID. Modify and cancel return new operation order IDs and have no -idempotency key. Never blindly retry them after a timeout. - -Use decimal strings. KR orders are whole-share. U.S. fractional buys are -amount-based market orders and fractional sells are quantity-based market -orders during regular hours. Keep the deterministic price-capped path -whole-share and limit-only. - -The current schema has a documented contradiction around -`GET /orders?status=CLOSED`. Treat both a successful page and -`closed-not-supported` as valid contract-probe outcomes; do not depend on -CLOSED history until the user's account confirms it. - -Toss live status remains `BLOCKED`. Do not promote it until all three -mutation-lifecycle gaps are closed and fault-injected: - -- recover an accepted-then-timeout create with the same `clientOrderId` and - byte-equivalent body inside the 10-minute window, then reconcile it; -- verify the original order reaches a terminal state after a cancel operation, - including cancel/fill races and rate limiting; -- register and continuously supervise the intended stop/take-profit protection - after an entry fill. - -## Korea Investment & Securities Open API - -Official examples: -`https://github.com/koreainvestment/open-trading-api`. - -```text -live https://openapi.koreainvestment.com:9443 -paper https://openapivts.koreainvestment.com:29443 -POST /oauth2/tokenP -``` - -KIS issues separate app keys and secrets for live and paper use. Resolve -process environment first, then an OS secret-store injection, then the local -`0600` `~/.config/mrcha-skills/secrets.env` fallback. Use the scoped -`QTA_KIS_PAPER_*` and `QTA_KIS_LIVE_*` names documented in `SKILL.md`; never -persist access tokens. The credential helper may prove token issuance, but it -must not submit an order or claim account/order readiness from OAuth alone. - -Use `authorization`, `appkey`, `appsecret`, `tr_id`, and `custtype: P`. -Account requests contain `CANO` and `ACNT_PRDT_CD`. - -Keep `CANO` and `ACNT_PRDT_CD` in the runtime credential environment. Do not -copy them into a plan, status file, or error message. - -Core paths and current official TR IDs: - -```text -KR quote GET /uapi/domestic-stock/v1/quotations/inquire-price - FHKST01010100 -KR order book GET /uapi/domestic-stock/v1/quotations/inquire-asking-price-exp-ccn - FHKST01010200 -KR balance GET /uapi/domestic-stock/v1/trading/inquire-balance - live TTTC8434R / paper VTTC8434R -KR open orders GET /uapi/domestic-stock/v1/trading/inquire-psbl-rvsecncl - live TTTC0084R -KR fills GET /uapi/domestic-stock/v1/trading/inquire-daily-ccld - live TTTC0081R / paper VTTC0081R -KR buy POST /uapi/domestic-stock/v1/trading/order-cash - live TTTC0012U / paper VTTC0012U -KR sell POST same path - live TTTC0011U / paper VTTC0011U -KR modify/cancel POST /uapi/domestic-stock/v1/trading/order-rvsecncl - live TTTC0013U / paper VTTC0013U - -US quote GET /uapi/overseas-price/v1/quotations/price - HHDFS00000300 -US order book GET /uapi/overseas-price/v1/quotations/inquire-asking-price - live HHDFS76200100 / paper unsupported -US balance GET /uapi/overseas-stock/v1/trading/inquire-balance - live TTTS3012R / paper VTTS3012R -US present balance GET /uapi/overseas-stock/v1/trading/inquire-present-balance - live CTRP6504R / paper VTRP6504R -US fills GET /uapi/overseas-stock/v1/trading/inquire-ccnl - live TTTS3035R / paper VTTS3035R -US open orders GET /uapi/overseas-stock/v1/trading/inquire-nccs - live TTTS3018R -US buy POST /uapi/overseas-stock/v1/trading/order - live TTTT1002U / paper VTTT1002U -US sell POST same path - live TTTT1006U / paper VTTT1001U -US modify/cancel POST /uapi/overseas-stock/v1/trading/order-rvsecncl - live TTTT1004U / paper VTTT1004U -``` - -KIS does not expose the local deterministic intent ID as a broker search key in -these order calls. Persist the request first. After an ambiguous submit, -reconcile orders, fills, and balances; do not automatically resubmit. -An HTTP 5xx, malformed success body, or successful response without `ODNO` is -ambiguous. Only a structured nonzero `rt_cd` with a broker `msg_cd` is an -authoritative rejection. - -The two documented KIS domestic REST quote responses do not currently provide -a verified independent trading date, trade timestamp, and order-book -timestamp contract. The adapter refuses to synthesize the date from local -receipt time or reuse `aspr_acpt_hour` for both observations. Domestic KIS -polling therefore remains fail-closed unless a contract-probed response -supplies a consistent `stck_bsop_date`, `stck_cntg_hour`, and -`aspr_acpt_hour`, or a separately verified timestamp-bearing source is added. - -Paper REST limits are lower than production limits and supported order types -can be narrower. The official U.S. paper examples support limit orders; use -whole-share limit orders for the initial contract. - -The U.S. order-book TR `HHDFS76200100` is production-only. Do not combine a -production quote with a paper mutation in one automated session. Validate U.S. -trigger decisions through a mutation-free production shadow session, and test -the paper order request and response serialization separately with explicit -fixtures or a bounded contract probe. A U.S. KIS paper polling session must -return `BLOCKED` before fetching a quote. - -For live U.S. shadow polling, use the one `HHDFS76200100` response only when it -contains last, bid, ask, `dymd`, and `dhms`. Assign its one source timestamp to -those values because they come from that same response. Do not fall back to the -separate price endpoint and label its value with the order-book timestamp. - -Keep KIS live promotion disabled until all of these blockers are implemented -and fault-injected: - -- paginate domestic and overseas order history using `tr_cont` and the - `CTX_AREA_FK/NK100|200` continuation keys; -- recover accepted-then-timeout mutations by reconciling uniquely matched - orders, fills, open orders, and balances without automatic resubmission; -- normalize domestic and overseas accepted, partial, filled, cancelled, and - rejected records using the broker's full status fields; -- enforce a process-shared app-key/account rate limiter, including the - temporary lower limit for newly registered production applications. - -The read-only account collector currently supports only KIS live credentials -with shadow execution. It uses `TTTC8434R + TTTC0084R` for Korea and -`CTRP6504R + TTTS3018R` for the U.S. It follows documented continuation -headers and keys, freezes only after all reads complete, and never sends an -order, modify, or cancel request. diff --git a/skills/quant-stock-polling-trader/references/daily-pipeline-contract.md b/skills/quant-stock-polling-trader/references/daily-pipeline-contract.md deleted file mode 100644 index a8638d6..0000000 --- a/skills/quant-stock-polling-trader/references/daily-pipeline-contract.md +++ /dev/null @@ -1,155 +0,0 @@ -# Daily shadow pipeline - -`daily_pipeline.py` turns one stable configuration into the current market -date's artifacts. It supports only: - -```text -broker=kis-live -mode=shadow -live_enabled=false -api_mutation_count=0 -``` - -The stages are: - -1. `prepare`: freeze the official market calendar, stop normally on - `MARKET_CLOSED`, refresh official membership/KIS masters/adjusted EOD and - benchmarks for KOSPI/KOSDAQ or NYSE/NASDAQ, build that market's universe, - and screen it. -2. `snapshot`: read KIS settled cash, positions, and open orders; merge any - explicitly configured additional exposure files; create the execution - policy and deterministic plan; and preview request serialization. -3. `entry`: start the deterministic first-hour shadow runner, then perform - bounded read-only reconciliation. - -There is no source-approval digest, provenance receipt, account-identity HMAC, -or trading-arm artifact in this workflow. Those artifacts are not needed for -a no-mutation shadow session and must not be made prerequisites. - -## Stable configuration - -Use `qta-daily-shadow-config/v2`: - -```json -{ - "schema": "qta-daily-shadow-config/v2", - "runtime_root": "/secure/qta", - "broker": "kis-live", - "mode": "shadow", - "history_start_date": "2021-01-01", - "minimum_sessions": 756, - "qta_method_version": "qta-2.0.0", - "request_interval_ms": 120, - "catalog_coverage_contract": { - "schema": "qta-catalog-coverage-contract/v1", - "minimum_ratio_by_exchange": { - "KOSPI": "1", - "KOSDAQ": "1", - "NYSE": "1", - "NASDAQ": "1" - }, - "minimum_screenable_ratio_by_exchange": { - "KOSPI": "0.5", - "KOSDAQ": "0.5", - "NYSE": "0.5", - "NASDAQ": "0.5" - } - }, - "selector": { - "selector_version": "qta-screen-1.1.0", - "min_total_score": "60", - "eligible_setup_statuses": ["READY"], - "top_k_by_exchange": { - "KOSPI": 2, - "KOSDAQ": 2, - "NYSE": 2, - "NASDAQ": 2 - }, - "min_selected_by_exchange": { - "KOSPI": 0, - "KOSDAQ": 0, - "NYSE": 0, - "NASDAQ": 0 - }, - "max_blocked_fraction": "0.02" - }, - "risk_policy_path": "/secure/qta/config/risk-policy.json", - "execution_policy": { - "snapshot_max_age_seconds": 1800, - "poll_interval_seconds": 2, - "quote_max_age_seconds": 5, - "max_spread_bps": "25", - "max_gap_bps": "20", - "trigger_mode": "CROSS_FROM_BELOW", - "order_ttl_seconds": 30, - "order_type": "LIMIT", - "time_in_force": "DAY", - "allow_partial_fill": true, - "cancel_remainder_at_window_end": true - }, - "account_aliases": { - "KR": "kis-live-kr", - "US": "kis-live-us" - }, - "prepare_complete_seconds_before_open": 600 -} -``` - -`qta_method_version` is optional and defaults to `qta-1.0.0`. Select -`qta-2.0.0` only for shadow research; the screen and runner then enforce its -liquidity and same-session market-regime contracts. - -Three other optional fields may be added: - -```json -{ - "exposure_component_paths": [ - "/secure/qta/manual/{session_date}/toss.json", - "/secure/qta/manual/{session_date}/nh.json" - ], - "exposure_component_max_age_seconds": 86400 -} -``` - -If a path is configured, it must exist and pass validation. If no path is -configured, the pipeline uses the KIS account view and continues. It never -creates fake empty exposure files. - -## State and restart behavior - -The intent ledger is: - -```text -RUNTIME_ROOT/state/// -``` - -The strategy-position ledger persists across dates at: - -```text -RUNTIME_ROOT/state///positions.sqlite3 -``` - -The workflow descriptor is -`RUNTIME_ROOT/current/.json`, and date-specific artifacts are under -`RUNTIME_ROOT/workflows//`. - -The stages move through `PREPARED`, `PLANNED`, and `READY`. A same-date stage -may reuse its completed output. Entry writes an attempt marker before polling -and refuses a second entry for the same session. - -The runner still keeps a single-writer ledger, treats ambiguous paper -mutations as `UNKNOWN`, and reconciles nonterminal states. These rules protect -paper-mode compatibility; daily shadow mode itself sends zero mutations. - -## Status - -Each stage writes one readable JSON status containing its stage, status, -market, session date, reason, details, timestamp, `live_enabled`, and -`api_mutation_count`. Status files are operational records, not signatures; -they do not contain receipt hashes. - -When a generated systemd service has `QTA_NTFY_TOPIC_URL` in its protected -environment file, the outer systemd wrapper sends a best-effort completion or -block notification after this status is written. Notification configuration is -not part of the deterministic plan inputs, and delivery failure never retries -the stage or changes its status. diff --git a/skills/quant-stock-polling-trader/references/execution-contract.md b/skills/quant-stock-polling-trader/references/execution-contract.md deleted file mode 100644 index edbd101..0000000 --- a/skills/quant-stock-polling-trader/references/execution-contract.md +++ /dev/null @@ -1,279 +0,0 @@ -# Open-one-hour execution contract - -## Version boundary - -Keep three independent contracts: - -```text -qta-1.0.0/2.0.0 completed T-1 adjusted daily data -> technical payload -qta-screen-1.0.0 legacy KR/US manifest -> deterministic candidates -qta-screen-1.1.0 selected-market manifest -> exchange-ranked candidates -open1h-exec-2.0.0 broker/account snapshots -> intents and order lifecycle -``` - -The entry window is `09:00–10:00 Asia/Seoul` for Korea and -`09:30–10:30 America/New_York` for the U.S. The planner validates those exact -local clocks with the IANA timezone database and rejects a supplied UTC offset -that disagrees with `ZoneInfo`, including an incorrect U.S. DST offset. It also -requires a frozen `qta-market-session/v1` JSON snapshot whose local file -SHA-256 and canonical `session_hash` match the policy. The snapshot binds the -scheduled open session, regular open/close, and previous session date. A -weekday check is only a second defense; holiday and early-close truth comes -from the frozen calendar snapshot. - -## Required frozen inputs - -- `qta-screen/v1` or `qta-screen/v2` artifact and canonical hash; -- broker, environment, nonsecret account alias, market, and trading date; -- broker account snapshot including settled cash, positions, open orders, and - snapshot time; -- cross-broker exposure snapshot, including fractional holdings even when - exact quantity is unavailable; -- numeric risk and execution policies; -- fee, tax, FX, tick, lot, fractional, and settlement semantics; -- hashed local market-calendar snapshot and entry-window timestamps; -- code, configuration, and universe hashes. - -A v2 screen requires `qta-exposure-snapshot/v2`. Every exposure position must -name one of `KOSPI`, `KOSDAQ`, `NYSE`, or `NASDAQ`; match positions by -`exchange:broker_symbol`. Never collapse a v2 position back to a market-only -key. Order intents use the manifest's verified `broker_symbol`, while -`canonical_symbol` and `data_symbol` remain provenance fields. - -For v2, broker symbols are qualified by the KIS stock-master snapshots used by -the universe builder. The planner therefore requires account `broker: "kis"`. -It rejects Toss planning even if a caller supplies a self-consistent screen -hash. Toss remains unavailable for v2 until a separate Toss-qualified -all-universe snapshot contract exists; a symbol lookup endpoint is not evidence -that every selected symbol was qualified for that broker. - -The v2 screen must match the complete `finalize_screen_v2` top-level contract: -source/schema/status/method and selector versions, analysis date, manifest and -selector hashes, counts and blocked fraction, normalized selector, all four -selected arrays, decisions, and screen hash. The planner validates the exact -field set, selector hash, counts, eligibility reasons, decision-to-selection -identity, instrument provenance, and canonical screen hash. It validates the -exact emitted READY or BLOCKED QTA schema, binds every READY payload's -`analysis_date` to the screen date, and recomputes the documented -total/medium/long/short/risk/ticker ranking before accepting exchange ranks or -selection order. It also checks the qta-1.0.0 score formula within the -two-decimal rounding envelope, score/opinion and score/setup relationships, -resolved-tick alignment, and the emitted 2R take-profit geometry. A smaller, -reranked, or semantically forged object cannot make itself acceptable merely -by hashing its own contents. - -For `qta-2.0.0`, the validator instead recomputes its first-hour total, -verifies the market-specific turnover floor, and requires the exact -research-only market-regime policy. QTA2 can run only in shadow mode. Before -stock quotes in each cycle, the runner reads KOSPI Composite, KOSDAQ -Composite, Nasdaq Composite, or the documented S&P 500 broad-market proxy for -NYSE candidates. It verifies the current-session timestamp, a maximum age of -90 seconds, and return from previous close of at least zero basis points. A -failed check skips new entries for that exchange and cycle. - -Each account position must include `market` and `symbol`, plus `exchange` for a -v2 screen. Each item in `open_orders` must represent a currently working order -and include `side, market, symbol`, plus `exchange` for v2. Existing positions -and working buy orders occupy exposure keys and concurrent-position slots -before any new intent is sized. - -For v2, `analysis_date` must equal the calendar artifact's -`previous_session_date`. For both legacy and v2 screens, account and -cross-broker exposure snapshots must be timezone-aware, share one exact frozen -`as_of` instant, fall on the entry session's local date, be no earlier than the -calendar `source_as_of`, and be no later than the window start. Their age at -the window start must not exceed the policy's `snapshot_max_age_seconds`; that -field is capped at 3,600 seconds. - -The exact market-session object contains: - -```text -schema, provider, source_id, source_as_of, source_path, source_sha256, -market, timezone, session_date, previous_session_date, scheduled_status, -regular_open, regular_close, session_hash -``` - -`source_path` must be absolute and contain a JSON object exactly matching the -same fields except `source_path, source_sha256, session_hash`. -`source_sha256` is recomputed from that file. `session_hash` is canonical -SHA-256 over the full market-session object excluding only `session_hash`. -The complete object and hash are copied into the plan context and revalidated -from disk by the session runner, not only by the planner. - -`scheduled_status: OPEN` means the calendar scheduled a regular session. It is -not evidence that the exchange is currently healthy. Emergency closures, -market-wide halts, broker incidents, and stale quotes remain runtime -`ENTRY_FREEZE` conditions. The current market-level contract treats KOSPI and -KOSDAQ as sharing the KR session, and NYSE and NASDAQ as sharing the U.S. -session. If venue calendars diverge, migrate to exchange-specific artifacts -instead of guessing. - -Lending availability is not settled cash. Keep it in a distinct field and -exclude it from quantity calculations. - -## Deterministic planning - -Calculate every quantity from the same pre-session snapshot: - -```text -effective_loss_per_share = - entry_limit - stop - + entry_limit * round_trip_cost_bps / 10000 - -quantity = floor_to_lot(min( - per_trade_risk_native / effective_loss_per_share, - max_symbol_notional_native / entry_limit, - settled_unborrowed_cash_native / worst_case_all_in_price -)) -``` - -Subtract existing and working exposure before creating an intent. If additions -to existing cross-broker exposure are disabled, reject the symbol even when it -is held at another broker or only as a fractional share. - -Do not reallocate unused cash after a skipped or rejected order. Keep ranking, -tie-breaking, cash reservation, and candidate count fixed for the session. - -Every plan embeds the complete normalized `screen, account, exposure, risk, -execution` objects under exact `frozen_inputs`. Before preview, polling, or -reconciliation, the runner validates the exact plan/context/window/policy and -intent schemas, verifies each deterministic intent ID and intent hash, then -runs the planner again from those frozen inputs and requires the rebuilt plan -to equal the supplied plan. Recomputing only `plan_hash`, or even making a -self-consistent quantity/cash/intent rehash, cannot change the result without -also changing the frozen inputs. - -These hashes prove deterministic internal consistency only. They are not -signatures or authorization gates. Keep account numbers and secrets out of -frozen inputs. Use the nonsecret account alias for the local state path; broker -routing values go directly from the credential loader to the adapter. - -## Trigger and polling - -Poll at absolute times `window_start + n * interval`. Skip late cycles rather -than replaying them. Run all slow work—universe construction, daily QTA, -ranking, sizing, account/exposure freezing, calendar validation, and request -serialization—before the opening bell. The hot path contains only a -deterministic rank-ordered quote, trigger evaluation, persisted transition, -and broker request. - -KIS KR and Toss need two REST calls per waiting intent: price and order book. -KIS-live US requires one same-source order-book call that contains last, bid, -ask, date, and time; its timestamp-less price fallback is forbidden. The -adapters cannot otherwise provide a safely joined bid/ask batch, so do not -claim batch latency. Before accepting a session, compute a broker- and -market-specific pacing budget from candidate count, quote-call count, and one -possible submit per intent in paper mode. Reserve one pacing interval for each -request, including response/persistence headroom: - -```text -KIS paper: ceil(intent_count * 3 * 1.05s) -KIS live KR shadow: ceil(intent_count * 2 * 0.12s) -KIS live US shadow: ceil(intent_count * 1 * 0.12s) -Toss shadow: ceil(intent_count * 2 * 0.10s) -``` - -QTA2 adds one KIS benchmark request per active exchange to the same cycle -budget. - -Reject a configured interval below that floor. This is only an API-pacing -feasibility bound, not proof of network latency. Production promotion also -requires observed shadow percentiles. - -Start the runner before the open. It acquires/refreshes the broker token -30 seconds before the scheduled open and blocks the session if authentication -fails or finishes late. Before warm-up, resolve every venue and serialize every -potential submit request; a missing venue or invalid request must fail before -the opening bell. Every cycle records scheduled/start/end times, lateness, -duration, quote/status/submit latency, skipped-cycle counts, and expected REST -request counts. Non-mutation quote telemetry is committed once per cycle; -`RESERVED`, `SUBMITTING`, acknowledgements, and other mutation states retain -individual `synchronous=FULL` commits. If a quote consumes the cycle deadline, -freeze new entries instead of evaluating later ranks from a delayed cycle. - -Do not start a submission inside its 10-second HTTP timeout plus broker pacing -margin before the window end. Pass the absolute window end into the broker -limiter, refuse a paced send that would cross it, and record both submit start -and acknowledgement time. If an already-sent request acknowledges after the -window, persist the acknowledgement, freeze entries, then immediately enter -the normal cancel/reconciliation phase. - -Require a fresh quote and a price-capped limit order: - -```text -last_price >= qta_entry -best_ask <= entry_limit -quote_age <= max_quote_age -spread_bps <= max_spread_bps -``` - -Freeze whether the trigger accepts a price already above entry at the first -poll (`AT_OR_ABOVE`) or requires a below-to-above transition -(`CROSS_FROM_BELOW`). Never infer this choice at runtime. -For `CROSS_FROM_BELOW`, update the prior-price baseline only after quote -status, instrument identity, positive bid/ask/last, receipt freshness, and both -source timestamps pass. A stale, missing-source, mismatched, or malformed -quote cannot seed a later crossing. - -If an adapter does not provide bid/ask, do not substitute a market order. -Either obtain the order book through the broker-specific endpoint or block the -candidate. - -At the window end, freeze new entries, cancel outstanding paper-mode entry -orders, and reconcile. Continue managing already filled paper positions. -Automatic market flattening is not implemented. - -The account/market strategy-position ledger is stored one directory above the -date-specific intent ledger. Authoritative cumulative entry fills create or -update a position with its original stop, target, tick, FX, and plan identity. -Before the next session, exact broker quantities are reconciled and valid -residuals roll to `CARRY_EXIT_ONLY`; they are excluded from new additions. -The bundled lifecycle module deterministically calculates stop, gap-stop, -target, close-liquidation, partial-fill, and daily-loss decisions, but the -current polling service does not yet send live exit mutations. Do not describe -that decision module as active automatic liquidation. - -## State machine - -```text -PLANNED -> WAIT_TRIGGER -> RESERVED -> SUBMITTING -SUBMITTING -> ACKNOWLEDGED | UNKNOWN -ACKNOWLEDGED -> PARTIALLY_FILLED | FILLED | CANCEL_PENDING | REJECTED -PARTIALLY_FILLED -> FILLED | CANCEL_PENDING -CANCEL_PENDING -> CANCELLED | FILLED | UNKNOWN -UNKNOWN -> RECONCILING -> ACKNOWLEDGED | PARTIALLY_FILLED | FILLED -UNKNOWN -> RECONCILING -> MANUAL_BLOCK -``` - -Persist the intent and exact request hash before `SUBMITTING`. Store broker -status values without discarding unknown values. - -One state directory may never silently bridge plans. Immediately after opening -the ledger, reject any nonterminal intent—or any unresolved `MANUAL_BLOCK`— -whose `plan_hash` or intent ID is outside the current plan. Also reject -`RESERVED`, `SUBMITTING`, `UNKNOWN`, `RECONCILING`, `CANCEL_PENDING`, and -`MANUAL_BLOCK` even when they belong to the same plan. Require broker evidence -and explicit clearance before warming authentication, reading quotes, or -sending a new mutation. - -When reconciliation observes `ACKNOWLEDGED` or `PARTIALLY_FILLED` after a -cancel request, keep the local state at `CANCEL_PENDING`, return `BLOCKED`, and -monitor again. A cancel acknowledgement is not terminal proof that the -original order can no longer fill. - -## Live trading - -Live trading is not supported. `kis-live/shadow` may read live account and -market data, but `live_enabled` remains false and the adapter sends no order, -correction, or cancellation mutation. Enabling live mutations requires a -separate implementation and a separate user request; it is not a hidden -promotion state in this skill. - -An offline request preview uses a placeholder account identifier. Its request -hash proves deterministic serialization for that placeholder only. A preview -lists potential requests for all planned intents and does not claim that an -intraday trigger was observed. - -Default kill behavior is `ENTRY_FREEZE`: stop mutations, cancel known open -entry orders when safe, preserve protective orders, and reconcile. Never infer -permission to liquidate existing user positions. diff --git a/skills/quant-stock-polling-trader/references/fixtures/aggressive-2m-risk-policy.json b/skills/quant-stock-polling-trader/references/fixtures/aggressive-2m-risk-policy.json deleted file mode 100644 index 70acb21..0000000 --- a/skills/quant-stock-polling-trader/references/fixtures/aggressive-2m-risk-policy.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "allow_auto_fx": false, - "allow_borrowed_cash": false, - "allow_existing_additions": false, - "allow_margin": false, - "allow_short": false, - "base_currency": "KRW", - "cash_buffer_bps": "0", - "max_concurrent_positions": 10000, - "max_daily_loss_krw": "2000000", - "max_symbol_notional_krw": "2000000", - "per_trade_risk_krw": "2000000", - "round_trip_cost_bps": "30", - "schema": "qta-risk-policy/v1", - "whole_shares_only": true -} diff --git a/skills/quant-stock-polling-trader/references/fixtures/kr-market-session-2026-07-27.json b/skills/quant-stock-polling-trader/references/fixtures/kr-market-session-2026-07-27.json deleted file mode 100644 index 1701817..0000000 --- a/skills/quant-stock-polling-trader/references/fixtures/kr-market-session-2026-07-27.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "schema": "qta-market-session/v1", - "provider": "DETERMINISTIC_TEST_FIXTURE", - "source_id": "fixture-kr-2026-07-27", - "source_as_of": "2026-07-27T08:00:00+09:00", - "market": "KR", - "timezone": "Asia/Seoul", - "session_date": "2026-07-27", - "previous_session_date": "2026-07-24", - "scheduled_status": "OPEN", - "regular_open": "2026-07-27T09:00:00+09:00", - "regular_close": "2026-07-27T15:30:00+09:00" -} diff --git a/skills/quant-stock-polling-trader/references/fixtures/us-market-session-2026-07-26-invalid.json b/skills/quant-stock-polling-trader/references/fixtures/us-market-session-2026-07-26-invalid.json deleted file mode 100644 index 7f04e37..0000000 --- a/skills/quant-stock-polling-trader/references/fixtures/us-market-session-2026-07-26-invalid.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "schema": "qta-market-session/v1", - "provider": "DETERMINISTIC_TEST_FIXTURE", - "source_id": "fixture-us-invalid-sunday-2026-07-26", - "source_as_of": "2026-07-26T08:00:00-04:00", - "market": "US", - "timezone": "America/New_York", - "session_date": "2026-07-26", - "previous_session_date": "2026-07-24", - "scheduled_status": "OPEN", - "regular_open": "2026-07-26T09:30:00-04:00", - "regular_close": "2026-07-26T16:00:00-04:00" -} diff --git a/skills/quant-stock-polling-trader/references/fixtures/us-market-session-2026-07-27.json b/skills/quant-stock-polling-trader/references/fixtures/us-market-session-2026-07-27.json deleted file mode 100644 index b1508cc..0000000 --- a/skills/quant-stock-polling-trader/references/fixtures/us-market-session-2026-07-27.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "schema": "qta-market-session/v1", - "provider": "DETERMINISTIC_TEST_FIXTURE", - "source_id": "fixture-us-2026-07-27", - "source_as_of": "2026-07-27T08:00:00-04:00", - "market": "US", - "timezone": "America/New_York", - "session_date": "2026-07-27", - "previous_session_date": "2026-07-24", - "scheduled_status": "OPEN", - "regular_open": "2026-07-27T09:30:00-04:00", - "regular_close": "2026-07-27T16:00:00-04:00" -} diff --git a/skills/quant-stock-polling-trader/references/fixtures/us-market-session-2026-12-07.json b/skills/quant-stock-polling-trader/references/fixtures/us-market-session-2026-12-07.json deleted file mode 100644 index fa2381f..0000000 --- a/skills/quant-stock-polling-trader/references/fixtures/us-market-session-2026-12-07.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "schema": "qta-market-session/v1", - "provider": "DETERMINISTIC_TEST_FIXTURE", - "source_id": "fixture-us-2026-12-07", - "source_as_of": "2026-12-07T08:00:00-05:00", - "market": "US", - "timezone": "America/New_York", - "session_date": "2026-12-07", - "previous_session_date": "2026-12-04", - "scheduled_status": "OPEN", - "regular_open": "2026-12-07T09:30:00-05:00", - "regular_close": "2026-12-07T16:00:00-05:00" -} diff --git a/skills/quant-stock-polling-trader/references/policy-examples.md b/skills/quant-stock-polling-trader/references/policy-examples.md deleted file mode 100644 index 0623d95..0000000 --- a/skills/quant-stock-polling-trader/references/policy-examples.md +++ /dev/null @@ -1,192 +0,0 @@ -# Frozen policy examples - -These values define a software canary, not an expected-return claim. Replace -dates and account-derived values only after a broker snapshot and calendar -preflight. - -## Risk policy - -```json -{ - "schema": "qta-risk-policy/v1", - "base_currency": "KRW", - "per_trade_risk_krw": "10000", - "max_symbol_notional_krw": "200000", - "max_concurrent_positions": 1, - "max_daily_loss_krw": "20000", - "round_trip_cost_bps": "30", - "cash_buffer_bps": "100", - "allow_existing_additions": false, - "allow_borrowed_cash": false, - "allow_margin": false, - "allow_short": false, - "allow_auto_fx": false, - "whole_shares_only": true -} -``` - -`borrowed_buying_power` may be present in an account snapshot for audit, but -this policy always excludes it. A `qta-screen/v2` plan requires -`"broker": "kis"` because its `broker_symbol` values are qualified against KIS -stock-master snapshots. Do not change this field to `toss`; build and validate -a separate Toss-qualified universe contract first. - -For the user's explicitly requested effectively uncapped 2,000,000 KRW shadow -allocation, use -`references/fixtures/aggressive-2m-risk-policy.json`. Its numeric fields are -set to the full operating capital and its concurrent-position cap is above any -feasible screen size, so they do not impose a smaller portfolio limit. -`round_trip_cost_bps: 30` remains part of the fill-cost calculation and -`cash_buffer_bps: 0` reserves no extra cash. This does not authorize borrowed -cash, margin, shorting, or live mutation. - -## Account snapshot - -```json -{ - "schema": "qta-account-snapshot/v2", - "broker": "kis", - "environment": "paper", - "account_alias": "kis-paper-kr", - "market": "KR", - "currency": "KRW", - "as_of": "2026-07-27T08:50:00+09:00", - "settled_cash": "200000", - "borrowed_buying_power": "5000000", - "fx_to_krw": "1", - "positions": [], - "open_orders": [] -} -``` - -Never put a real account number, account product code, Toss account sequence, -app key, or secret in this file. `account_alias` is a nonsecret local label; -the adapter reads the actual routing values from the credential environment. - -After planning, run and reconcile directly: - -```bash -python3 scripts/run_session.py run \ - --plan /secure/runtime/order-plan.json \ - --broker kis-paper \ - --mode paper \ - --state-dir /secure/runtime/state - -python3 scripts/reconcile.py \ - --plan /secure/runtime/order-plan.json \ - --broker kis-paper \ - --state-dir /secure/runtime/state -``` - -Both commands use the broker-routing environment. Store those values in an OS -secret store or process environment; do not write them into these JSON files. - -## Execution policy - -Korea: - -```json -{ - "schema": "qta-execution-policy/v1", - "market": "KR", - "timezone": "Asia/Seoul", - "entry_window_start": "2026-07-27T09:00:00+09:00", - "entry_window_end": "2026-07-27T10:00:00+09:00", - "market_session": { - "schema": "qta-market-session/v1", - "provider": "KRX", - "source_id": "replace-with-frozen-calendar-source-id", - "source_as_of": "2026-07-27T08:30:00+09:00", - "source_path": "/absolute/path/to/kr-market-session-2026-07-27.json", - "source_sha256": "replace-with-lowercase-sha256", - "market": "KR", - "timezone": "Asia/Seoul", - "session_date": "2026-07-27", - "previous_session_date": "2026-07-24", - "scheduled_status": "OPEN", - "regular_open": "2026-07-27T09:00:00+09:00", - "regular_close": "2026-07-27T15:30:00+09:00", - "session_hash": "replace-with-canonical-session-sha256" - }, - "snapshot_max_age_seconds": 1800, - "poll_interval_seconds": 4, - "quote_max_age_seconds": 5, - "max_spread_bps": "25", - "max_gap_bps": "20", - "trigger_mode": "AT_OR_ABOVE", - "order_ttl_seconds": 30, - "order_type": "LIMIT", - "time_in_force": "DAY", - "allow_partial_fill": true, - "cancel_remainder_at_window_end": true -} -``` - -The four-second example is the minimum admission value for one KIS-paper -intent: two quote calls plus one possible submit at the adapter's conservative -1.05-second pacing. Recompute it from the final number of planned intents; -two KIS-paper intents require at least seven seconds. KIS-live and Toss -shadow sessions have different pacing floors, but live mutation remains -disabled until the promotion gates pass. Start the runner at least 30 seconds -before the regular open so authentication is not on the first quote cycle. - -For the U.S., use `America/New_York` and `09:30–10:30` with the correct offset -for that session. Do not derive it by subtracting a fixed number of hours from -KST. The planner checks the supplied offset against `ZoneInfo`, so use `-04:00` -only when New York is actually on daylight time and `-05:00` when it is on -standard time. - -For a v2 run, stamp the account snapshot and -`qta-exposure-snapshot/v2` with the same timezone-aware `as_of` instant after -both reads are frozen. That instant must be on the local session date, at or -after `market_session.source_as_of`, no later than the entry-window start, and -no older than `snapshot_max_age_seconds`. The screen `analysis_date` must equal -the hashed calendar snapshot's `previous_session_date`. - -The file at `market_session.source_path` contains exactly the market-session -fields shown above except `source_path`, `source_sha256`, and `session_hash`. -Generate the two hashes with the execution helper after freezing that file; -the placeholder values above are intentionally not runnable. - -```bash -python3 scripts/freeze_market_session.py \ - --source /absolute/path/to/kr-market-session-2026-07-27.json \ - --output /absolute/path/to/kr-market-session-bound-2026-07-27.json -``` - -Copy the complete bound object into the execution policy without editing it. - -## Venue map - -Legacy `qta-screen/v1` plans require: - -```json -{ - "schema": "qta-venue-map/v1", - "venues": { - "KR:005930": "KRX", - "US:AAPL": "NASD" - } -} -``` - -An exchange-aware v2 plan carries the verified venue inside each hashed intent, -so `run_session.py` does not require `--venue-map`. If a v2 map is supplied for -an independent cross-check, prefer exchange-qualified keys: - -```json -{ - "schema": "qta-venue-map/v1", - "venues": { - "KOSPI:005930": "KRX", - "NASDAQ:AAPL": "NASD", - "NYSE:IBM": "NYSE" - } -} -``` - -A supplied map must match the hashed intent or the session blocks. KIS paper -uses KRX for Korea. KIS U.S. order venues are `NASD`, `NYSE`, and `AMEX`; quote -endpoints use different codes internally, which the adapter maps. Toss ignores -the venue map for legacy v1 previews, but v2 planning blocks Toss because the -universe has not been Toss-qualified. diff --git a/skills/quant-stock-polling-trader/references/systemd-contract.md b/skills/quant-stock-polling-trader/references/systemd-contract.md deleted file mode 100644 index aa5336f..0000000 --- a/skills/quant-stock-polling-trader/references/systemd-contract.md +++ /dev/null @@ -1,105 +0,0 @@ -# Deterministic systemd supervision contract - -`systemd_units.py generate` creates user-level service and timer files without -changing the user manager. This is a safety property of the command, not a -reason for an agent to stop early. When the user explicitly requests actual -installation or automation, generation is an intermediate step: review and -verify the exact generated units, then install, enable, start, and inspect only -the requested scope. - -## Bundle - -The input is one exact `qta-systemd-bundle/v2` JSON object. It binds: - -- the exact Python executable and installed technical/trader skill roots; -- a user-owned, regular, non-symlink mode-0600 environment file; -- one absolute runtime directory containing all mutable jobs, plans, state, - venue maps, and status files; -- unique static EOD/account-snapshot/entry jobs or daily - prepare/snapshot/entry jobs for KR and US. - -Daily preparation starts at 01:00 in each market timezone on weekdays. That -provides eight hours before the Korean open and eight and a half hours before -the U.S. open for the measured multi-hour market refresh. An initial -multi-year bootstrap can take longer than the daily window and must be -completed before relying on the timer; subsequent runs resume and increment -the completed cache. Entry services start at 08:59 Asia/Seoul and 09:29 -America/New_York so the -runner can complete its 30-second token warm-up before the exchange open. -Account snapshot services run at 08:50 Asia/Seoul and 09:20 -America/New_York, use KIS live credentials in shadow mode, and must complete -before their market's entry service. -Entry timers are never persistent: a sleeping or offline host must not replay a -missed opening session. Account snapshot timers are also never persistent. -Daily preparation and legacy EOD timers are persistent; late daily preparation -still fails its pre-open deadline rather than replaying an entry. Entry and -snapshot timers are never persistent. - -For recurring automation, use `daily-prepare`, `daily-snapshot`, and -`daily-entry` jobs whose `input_path` is the stable -`qta-daily-shadow-config/v2` file described in -`daily-pipeline-contract.md`. The daily runner calculates the session date and -atomically publishes current descriptors. A timer bound to a date-stamped job -file remains session-specific and must not be reported as future-date -automation. Never reuse a prior session's plan, calendar, state directory, or -output path on a later date. - -The generator accepts only KIS paper/paper or KIS live/shadow entry pairs. -It refuses live mode. The generated service executes Python directly without a -shell, removes `PYTHONPATH` and `PYTHONHOME` before the wrapper starts and -again before the worker process, disables bytecode writes, applies systemd -filesystem/process hardening, and grants writes only to the bound runtime -directory. - -## Optional ntfy notification - -The mode-`0600` environment file may define `QTA_NTFY_TOPIC_URL` and, -for an authenticated topic, `QTA_NTFY_TOKEN`. The topic must be a plain -HTTP(S) topic URL without embedded credentials, a query, or a fragment. The -wrapper publishes one completion notification after each job and reports only -the channel result (`SENT`, `FAILED`, or `DISABLED`) to the journal. It never -prints the topic URL, token, or server response body. - -Notification failure is deliberately best-effort: it does not change the -worker's return code and cannot cause a successful shadow stage to be retried. -`BLOCKED` and `MANUAL_BLOCK` use urgent priority; entry completion uses high -priority. The notification is an operational side channel, not a broker API -mutation, and it does not change `live_enabled=false` or -`api_mutation_count=0`. - -## Single writer - -Every generated execution acquires a nonblocking, non-symlink regular-file -market lock below a non-symlink `runtime_directory/locks`. Preparation, -snapshot, and entry acquire only their selected market lock. Failure -to acquire a lock is `BLOCKED`, not a retry or a second writer. The wrapper -sets every opened lock file to mode `0600`, including a regular lock file that -predates the current bundle, and verifies the resulting mode before acquiring -it. - -Generate into a new or empty output directory so stale service/timer files -cannot survive from an older bundle. The generated `systemd-bundle.json` is a -normalized, hashed, exact `qta-systemd-bundle/v2` object that the `execute` -subcommand must be able to read without projection or manual editing. -Derived `schedule` and `persistent` values belong only in generated timer -files; they are not serialized as job input fields. - -Unit files and their SHA-256 values are recorded in -`generation-status.json` with: - -```text -activation_performed=false -live_enabled=false -api_mutation_count=0 -``` - -Run `systemd-analyze --user verify` against the generated files on the target -Linux host before copying them to `~/.config/systemd/user/`. Resolve the -intended filenames explicitly; do not install a wildcard that could include a -stale or unreviewed unit. After an explicitly requested activation, run -`systemctl --user daemon-reload`, enable/start only the approved timers or -services, and verify `is-enabled`, `is-active`, `list-timers`, and the service -journal. Entry or snapshot activation must not occur until the daily config, -credentials, runtime ownership, and installed skill roots are valid. The daily -producer itself creates the date-bound calendar, plan, and state path before -entry; a missing stage descriptor is `BLOCKED`. diff --git a/skills/quant-stock-polling-trader/scripts/account_snapshot.py b/skills/quant-stock-polling-trader/scripts/account_snapshot.py deleted file mode 100644 index 4b96fb2..0000000 --- a/skills/quant-stock-polling-trader/scripts/account_snapshot.py +++ /dev/null @@ -1,1199 +0,0 @@ -#!/usr/bin/env python3 -"""Freeze KIS account state and merge verified cross-broker exposures.""" - -from __future__ import annotations - -import argparse -import json -import os -import tempfile -from datetime import datetime, timezone -from decimal import Decimal, InvalidOperation -from pathlib import Path -from typing import Any - -from broker_adapters import HttpResponse, KisBroker, QueueTransport, kis_result -from broker_credentials import ( - load_kis_credentials, - require_kis_runtime_credentials, -) -from execution_core import ( - BlockedError, - canonical_json, - normalized_account_snapshot, - normalized_exposure_snapshot, - parse_aware_datetime, - sha256_file, - sha256_json, -) - -JOB_SCHEMA = "qta-account-snapshot-job/v2" -COMPONENT_SCHEMA = "qta-exposure-component/v1" -STATUS_SCHEMA = "qta-account-snapshot-status/v2" -MANIFEST_SCHEMA = "qta-universe-manifest/v2" -ACCOUNT_SCHEMA = "qta-account-snapshot/v2" -EXPOSURE_SCHEMA = "qta-exposure-snapshot/v2" - -EXCHANGE_MARKET = { - "KOSPI": "KR", - "KOSDAQ": "KR", - "NYSE": "US", - "NASDAQ": "US", -} -US_EXCHANGE_CODES = { - "NAS": "NASDAQ", - "NASD": "NASDAQ", - "NASDAQ": "NASDAQ", - "NYS": "NYSE", - "NYSE": "NYSE", -} - - -def exact_fields(value: dict[str, Any], required: set[str], label: str) -> None: - missing = required - set(value) - extra = set(value) - required - if missing or extra: - raise BlockedError( - f"{label} fields mismatch: missing={sorted(missing)}, extra={sorted(extra)}" - ) - - -def decimal_value(value: Any, label: str) -> Decimal: - if isinstance(value, bool): - raise BlockedError(f"{label} must be a decimal") - try: - result = Decimal(str(value).strip()) - except (InvalidOperation, ValueError, AttributeError) as exc: - raise BlockedError(f"{label} must be a decimal") from exc - if not result.is_finite(): - raise BlockedError(f"{label} must be finite") - return result - - -def nonnegative_decimal(value: Any, label: str) -> Decimal: - result = decimal_value(value, label) - if result < 0: - raise BlockedError(f"{label} must be nonnegative") - return result - - -def positive_decimal(value: Any, label: str) -> Decimal: - result = decimal_value(value, label) - if result <= 0: - raise BlockedError(f"{label} must be positive") - return result - - -def read_json_object(path: Path, label: str) -> dict[str, Any]: - try: - value = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as exc: - raise BlockedError(f"{label} must be readable JSON: {path}") from exc - if not isinstance(value, dict): - raise BlockedError(f"{label} must be a JSON object") - return value - - -def atomic_write_json(path: Path, value: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) - temporary = Path(temporary_name) - try: - with os.fdopen(descriptor, "w", encoding="utf-8") as handle: - handle.write(json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2)) - handle.write("\n") - handle.flush() - os.fsync(handle.fileno()) - os.replace(temporary, path) - except BaseException: - temporary.unlink(missing_ok=True) - raise - - -def validate_absolute_path(value: Any, label: str) -> Path: - if not isinstance(value, str) or not value.strip(): - raise BlockedError(f"{label} must be a non-empty path") - path = Path(value) - if not path.is_absolute(): - raise BlockedError(f"{label} must be absolute") - return path - - -def load_job(path: Path) -> dict[str, Any]: - value = read_json_object(path, "account snapshot job") - required = { - "schema", - "broker", - "mode", - "market", - "account_alias", - "universe_manifest", - "fx_to_krw", - "manual_exposure_components", - "manual_component_max_age_seconds", - "output_account_path", - "output_exposure_path", - "output_status_path", - } - exact_fields(value, required, "account snapshot job") - if value["schema"] != JOB_SCHEMA: - raise BlockedError(f"unsupported account snapshot job schema: {value['schema']!r}") - if value["broker"] != "kis-live" or value["mode"] != "shadow": - raise BlockedError("account collector currently requires kis-live/shadow") - market = str(value["market"]).upper() - if market not in {"KR", "US"}: - raise BlockedError("job market must be KR or US") - alias = value["account_alias"] - if not isinstance(alias, str) or not alias.strip() or alias != alias.strip(): - raise BlockedError("account_alias must be a non-empty trimmed string") - manifest = validate_absolute_path(value["universe_manifest"], "universe_manifest") - raw_fx = value["fx_to_krw"] - if market == "US" and raw_fx == "KIS_PRESENT_BALANCE": - fx: Decimal | None = None - else: - fx = positive_decimal(raw_fx, "fx_to_krw") - if market == "KR" and fx != 1: - raise BlockedError("KR fx_to_krw must be 1") - components = value["manual_exposure_components"] - if not isinstance(components, list): - raise BlockedError("manual_exposure_components must be an array") - component_paths = [ - validate_absolute_path(item, f"manual_exposure_components[{index}]") - for index, item in enumerate(components) - ] - max_age = value["manual_component_max_age_seconds"] - if isinstance(max_age, bool) or not isinstance(max_age, int) or max_age <= 0: - raise BlockedError("manual_component_max_age_seconds must be a positive integer") - if max_age > 86400: - raise BlockedError("manual_component_max_age_seconds must not exceed 86400") - output_paths = { - key: validate_absolute_path(value[key], key) - for key in ( - "output_account_path", - "output_exposure_path", - "output_status_path", - ) - } - if len(set(output_paths.values())) != len(output_paths): - raise BlockedError("account, exposure, and status output paths must differ") - return { - **value, - "market": market, - "universe_manifest": manifest, - "fx_to_krw": fx, - "manual_exposure_components": component_paths, - "manual_component_max_age_seconds": max_age, - **output_paths, - } - - -def load_manifest_map(path: Path) -> tuple[dict[str, str], str]: - manifest = read_json_object(path, "universe manifest") - if manifest.get("schema") != MANIFEST_SCHEMA: - raise BlockedError(f"universe manifest must use {MANIFEST_SCHEMA}") - if manifest.get("build_status") != "READY": - raise BlockedError("universe manifest build_status must be READY") - supplied_hash = manifest.get("manifest_hash") - if ( - not isinstance(supplied_hash, str) - or len(supplied_hash) != 64 - or supplied_hash != sha256_json( - {key: item for key, item in manifest.items() if key != "manifest_hash"} - ) - ): - raise BlockedError("universe manifest hash is invalid") - instruments = manifest.get("instruments") - if not isinstance(instruments, list): - raise BlockedError("universe manifest instruments must be an array") - mapping: dict[str, str] = {} - for index, instrument in enumerate(instruments): - if not isinstance(instrument, dict): - raise BlockedError(f"manifest instruments[{index}] must be an object") - symbol = str(instrument.get("broker_symbol", "")).strip().upper() - exchange = str(instrument.get("exchange", "")).strip().upper() - market = str(instrument.get("market", "")).strip().upper() - if not symbol or exchange not in EXCHANGE_MARKET: - raise BlockedError(f"manifest instruments[{index}] has invalid identity") - if EXCHANGE_MARKET[exchange] != market: - raise BlockedError(f"manifest instruments[{index}] exchange/market mismatch") - key = f"{market}:{symbol}" - previous = mapping.setdefault(key, exchange) - if previous != exchange: - raise BlockedError(f"broker symbol maps to multiple exchanges: {key}") - return mapping, supplied_hash - - -def normalize_component( - value: dict[str, Any], - *, - frozen_at: datetime, - max_age_seconds: int, - path: Path, -) -> tuple[list[dict[str, Any]], dict[str, Any]]: - required = {"schema", "broker", "source_kind", "source_as_of", "positions"} - exact_fields(value, required, f"exposure component {path}") - if value["schema"] != COMPONENT_SCHEMA: - raise BlockedError(f"unsupported exposure component schema in {path}") - broker = value["broker"] - if broker not in {"toss", "nh"}: - raise BlockedError(f"manual exposure component broker must be toss or nh: {path}") - if value["source_kind"] not in {"broker-export", "verified-manual"}: - raise BlockedError(f"unsupported manual source_kind in {path}") - source_as_of = parse_aware_datetime(value["source_as_of"], f"{path} source_as_of") - age = (frozen_at.astimezone(timezone.utc) - source_as_of.astimezone(timezone.utc)).total_seconds() - if age < 0: - raise BlockedError(f"manual exposure component is future-dated: {path}") - if age > max_age_seconds: - raise BlockedError(f"manual exposure component is stale: {path}") - raw_positions = value["positions"] - if not isinstance(raw_positions, list): - raise BlockedError(f"manual exposure component positions must be an array: {path}") - positions: list[dict[str, Any]] = [] - for index, item in enumerate(raw_positions): - if not isinstance(item, dict): - raise BlockedError(f"{path} positions[{index}] must be an object") - exact_fields( - item, - {"market", "exchange", "symbol", "quantity", "market_value_krw"}, - f"{path} positions[{index}]", - ) - exchange = str(item["exchange"]).upper() - market = str(item["market"]).upper() - if exchange not in EXCHANGE_MARKET or EXCHANGE_MARKET[exchange] != market: - raise BlockedError(f"{path} positions[{index}] exchange/market mismatch") - symbol = str(item["symbol"]).strip().upper() - if not symbol: - raise BlockedError(f"{path} positions[{index}] symbol is empty") - quantity = item["quantity"] - if quantity is not None: - quantity = format( - nonnegative_decimal(quantity, f"{path} positions[{index}].quantity"), - "f", - ) - market_value = format( - nonnegative_decimal( - item["market_value_krw"], - f"{path} positions[{index}].market_value_krw", - ), - "f", - ) - positions.append( - { - "broker": broker, - "market": market, - "exchange": exchange, - "symbol": symbol, - "quantity": quantity, - "market_value_krw": market_value, - } - ) - return positions, { - "broker": broker, - "path": str(path), - "sha256": sha256_file(path), - "source_as_of": source_as_of.isoformat(), - "position_count": len(positions), - } - - -def response_list(body: dict[str, Any], field: str, label: str) -> list[dict[str, Any]]: - raw = body.get(field, []) - if raw in (None, ""): - return [] - if isinstance(raw, dict): - raw = [raw] - if not isinstance(raw, list) or any(not isinstance(item, dict) for item in raw): - raise BlockedError(f"KIS {label} {field} must be an object array") - return raw - - -def request_pages( - broker: KisBroker, - *, - path: str, - tr_id: str, - query: dict[str, str], - output_fields: tuple[str, ...], - context_suffix: str | None, - label: str, -) -> dict[str, list[dict[str, Any]]]: - collected = {field: [] for field in output_fields} - current_query = dict(query) - continuation = "" - for page in range(1, 51): - headers = broker.headers(tr_id) - if continuation: - headers["tr_cont"] = "N" - response = broker.transport.request( - "GET", - f"{broker.base_url}{path}", - headers=headers, - query=current_query, - timeout_seconds=10, - ) - body = kis_result(response, label) - for field in output_fields: - collected[field].extend(response_list(body, field, label)) - continuation = str(response.headers.get("tr_cont", "")).strip().upper() - if continuation not in {"M", "F"}: - return collected - if context_suffix is None: - raise BlockedError(f"KIS {label} requested unsupported continuation") - if context_suffix == "": - continue - fk = str(body.get(f"ctx_area_fk{context_suffix}", "")).strip() - nk = str(body.get(f"ctx_area_nk{context_suffix}", "")).strip() - if not fk or not nk: - raise BlockedError(f"KIS {label} continuation keys are missing") - current_query[f"CTX_AREA_FK{context_suffix}"] = fk - current_query[f"CTX_AREA_NK{context_suffix}"] = nk - raise BlockedError(f"KIS {label} exceeded 50 continuation pages") - - -def manifest_exchange( - mapping: dict[str, str], - market: str, - symbol: str, - *, - broker_exchange: Any = None, -) -> str: - normalized_symbol = symbol.strip().upper() - exchange = mapping.get(f"{market}:{normalized_symbol}") - if exchange is None and market == "US": - exchange = US_EXCHANGE_CODES.get(str(broker_exchange or "").strip().upper()) - if exchange is None: - raise BlockedError( - f"held or working symbol is absent from the selected-market map: " - f"{market}:{normalized_symbol}" - ) - return exchange - - -def side_from_kis(value: Any, label: str) -> str: - code = str(value or "").strip() - if code == "01": - return "SELL" - if code == "02": - return "BUY" - raise BlockedError(f"KIS {label} has unsupported side code: {code!r}") - - -def aggregate_positions(positions: list[dict[str, Any]]) -> list[dict[str, Any]]: - totals: dict[tuple[str, str, str], tuple[Decimal, Decimal]] = {} - for item in positions: - key = (item["market"], item["exchange"], item["symbol"]) - quantity, market_value = totals.get(key, (Decimal(0), Decimal(0))) - totals[key] = ( - quantity + nonnegative_decimal(item["quantity"], "position quantity"), - market_value - + nonnegative_decimal(item["market_value_krw"], "position market value"), - ) - return [ - { - "market": market, - "exchange": exchange, - "symbol": symbol, - "quantity": format(totals[(market, exchange, symbol)][0], "f"), - "market_value_krw": format(totals[(market, exchange, symbol)][1], "f"), - } - for market, exchange, symbol in sorted(totals) - ] - - -def minimum_nonnegative(rows: list[tuple[Any, str]]) -> Decimal: - return min(max(decimal_value(value, label), Decimal(0)) for value, label in rows) - - -def collect_kr( - broker: KisBroker, - mapping: dict[str, str], -) -> tuple[Decimal, Decimal, list[dict[str, Any]], list[dict[str, Any]]]: - balance = request_pages( - broker, - path="/uapi/domestic-stock/v1/trading/inquire-balance", - tr_id="TTTC8434R", - query={ - "CANO": broker.account_prefix, - "ACNT_PRDT_CD": broker.account_product, - "AFHR_FLPR_YN": "N", - "OFL_YN": "", - "INQR_DVSN": "02", - "UNPR_DVSN": "01", - "FUND_STTL_ICLD_YN": "N", - "FNCG_AMT_AUTO_RDPT_YN": "N", - "PRCS_DVSN": "00", - "CTX_AREA_FK100": "", - "CTX_AREA_NK100": "", - }, - output_fields=("output1", "output2"), - context_suffix="100", - label="KR balance", - ) - if not balance["output2"]: - raise BlockedError("KIS KR balance summary is missing") - summary = balance["output2"][-1] - settled_cash = minimum_nonnegative( - [ - (summary.get("dnca_tot_amt"), "KR dnca_tot_amt"), - (summary.get("nxdy_excc_amt"), "KR nxdy_excc_amt"), - (summary.get("prvs_rcdl_excc_amt"), "KR prvs_rcdl_excc_amt"), - ] - ) - borrowed = nonnegative_decimal(summary.get("tot_loan_amt", "0"), "KR tot_loan_amt") - positions: list[dict[str, Any]] = [] - for row in balance["output1"]: - quantity = nonnegative_decimal(row.get("hldg_qty"), "KR hldg_qty") - if quantity == 0: - continue - symbol = str(row.get("pdno", "")).strip().upper() - exchange = manifest_exchange(mapping, "KR", symbol) - positions.append( - { - "market": "KR", - "exchange": exchange, - "symbol": symbol, - "quantity": format(quantity, "f"), - "market_value_krw": format( - nonnegative_decimal(row.get("evlu_amt"), "KR evlu_amt"), "f" - ), - } - ) - orders = request_pages( - broker, - path="/uapi/domestic-stock/v1/trading/inquire-psbl-rvsecncl", - tr_id="TTTC0084R", - query={ - "CANO": broker.account_prefix, - "ACNT_PRDT_CD": broker.account_product, - "INQR_DVSN_1": "1", - "INQR_DVSN_2": "0", - "CTX_AREA_FK100": "", - "CTX_AREA_NK100": "", - }, - output_fields=("output",), - context_suffix="100", - label="KR open orders", - ) - open_orders: list[dict[str, Any]] = [] - for row in orders["output"]: - remaining = nonnegative_decimal(row.get("psbl_qty"), "KR psbl_qty") - if remaining == 0: - continue - symbol = str(row.get("pdno", "")).strip().upper() - open_orders.append( - { - "side": side_from_kis(row.get("sll_buy_dvsn_cd"), "KR open order"), - "market": "KR", - "exchange": manifest_exchange(mapping, "KR", symbol), - "symbol": symbol, - } - ) - return settled_cash, borrowed, positions, open_orders - - -def collect_us( - broker: KisBroker, - mapping: dict[str, str], - fx_to_krw: Decimal | None, -) -> tuple[ - Decimal, - Decimal, - list[dict[str, Any]], - list[dict[str, Any]], - Decimal | None, - str, -]: - present = request_pages( - broker, - path="/uapi/overseas-stock/v1/trading/inquire-present-balance", - tr_id="CTRP6504R", - query={ - "CANO": broker.account_prefix, - "ACNT_PRDT_CD": broker.account_product, - "WCRC_FRCR_DVSN_CD": "02", - "NATN_CD": "840", - "TR_MKET_CD": "00", - "INQR_DVSN_CD": "00", - }, - output_fields=("output1", "output2", "output3"), - context_suffix="", - label="US present balance", - ) - usd_cash_rows = [ - row - for row in present["output2"] - if str(row.get("crcy_cd", "")).strip().upper() == "USD" - ] - cash_snapshots: set[tuple[Decimal, Decimal, Decimal]] = set() - observed_cash_fx: set[Decimal] = set() - for index, cash in enumerate(usd_cash_rows): - cash_snapshots.add( - ( - nonnegative_decimal( - cash.get("frcr_dncl_amt_2"), - f"US output2[{index}].frcr_dncl_amt_2", - ), - nonnegative_decimal( - cash.get("frcr_drwg_psbl_amt_1"), - f"US output2[{index}].frcr_drwg_psbl_amt_1", - ), - nonnegative_decimal( - cash.get("nxdy_frcr_drwg_psbl_amt"), - f"US output2[{index}].nxdy_frcr_drwg_psbl_amt", - ), - ) - ) - if fx_to_krw is None: - observed_cash_fx.add( - positive_decimal( - cash.get("frst_bltn_exrt"), - f"US output2[{index}].frst_bltn_exrt", - ) - ) - if len(cash_snapshots) > 1: - raise BlockedError("KIS US present balance returned conflicting USD cash rows") - if len(observed_cash_fx) > 1: - raise BlockedError("KIS US present balance returned conflicting USD FX rates") - settled_cash = min(next(iter(cash_snapshots))) if cash_snapshots else Decimal(0) - - position_rows: list[tuple[dict[str, Any], Decimal]] = [] - observed_position_fx: set[Decimal] = set() - borrowed = Decimal(0) - for index, row in enumerate(present["output1"]): - quantity = nonnegative_decimal( - row.get("cblc_qty13"), f"US output1[{index}].cblc_qty13" - ) - borrowed += nonnegative_decimal( - row.get("loan_rmnd", "0"), f"US output1[{index}].loan_rmnd" - ) - if quantity == 0: - continue - position_rows.append((row, quantity)) - if fx_to_krw is None and not observed_cash_fx: - observed_position_fx.add( - positive_decimal( - row.get("bass_exrt"), - f"US output1[{index}].bass_exrt", - ) - ) - if len(observed_position_fx) > 1: - raise BlockedError( - "KIS US present balance returned conflicting position FX rates" - ) - - if fx_to_krw is not None: - effective_fx: Decimal | None = fx_to_krw - fx_source = "JOB_INPUT" - elif observed_cash_fx: - effective_fx = next(iter(observed_cash_fx)) - fx_source = "KIS_PRESENT_BALANCE:frst_bltn_exrt" - elif observed_position_fx: - effective_fx = next(iter(observed_position_fx)) - fx_source = "KIS_PRESENT_BALANCE:bass_exrt" - else: - effective_fx = None - fx_source = "NOT_APPLICABLE:NO_SETTLED_USD_OR_POSITION" - - positions: list[dict[str, Any]] = [] - for row, quantity in position_rows: - if effective_fx is None: - raise BlockedError("KIS US positions require a positive FX rate") - symbol = str(row.get("pdno", "")).strip().upper() - exchange = manifest_exchange( - mapping, - "US", - symbol, - broker_exchange=row.get("ovrs_excg_cd"), - ) - market_value_usd = nonnegative_decimal( - row.get("frcr_evlu_amt2"), "US frcr_evlu_amt2" - ) - positions.append( - { - "market": "US", - "exchange": exchange, - "symbol": symbol, - "quantity": format(quantity, "f"), - "market_value_krw": format(market_value_usd * effective_fx, "f"), - } - ) - orders = request_pages( - broker, - path="/uapi/overseas-stock/v1/trading/inquire-nccs", - tr_id="TTTS3018R", - query={ - "CANO": broker.account_prefix, - "ACNT_PRDT_CD": broker.account_product, - "OVRS_EXCG_CD": "NASD", - "SORT_SQN": "DS", - "CTX_AREA_FK200": "", - "CTX_AREA_NK200": "", - }, - output_fields=("output",), - context_suffix="200", - label="US open orders", - ) - open_orders: list[dict[str, Any]] = [] - for row in orders["output"]: - remaining = nonnegative_decimal(row.get("nccs_qty"), "US nccs_qty") - if remaining == 0: - continue - symbol = str(row.get("pdno", "")).strip().upper() - open_orders.append( - { - "side": side_from_kis(row.get("sll_buy_dvsn_cd"), "US open order"), - "market": "US", - "exchange": manifest_exchange( - mapping, - "US", - symbol, - broker_exchange=row.get("ovrs_excg_cd"), - ), - "symbol": symbol, - } - ) - return ( - settled_cash, - borrowed, - positions, - open_orders, - effective_fx, - fx_source, - ) - - -def collect( - job: dict[str, Any], - *, - transport: Any | None = None, - frozen_at: datetime | None = None, -) -> dict[str, Any]: - credential_status = load_kis_credentials("live") - require_kis_runtime_credentials(credential_status) - mapping, manifest_hash = load_manifest_map(job["universe_manifest"]) - broker = KisBroker( - app_key=os.environ["QTA_KIS_APP_KEY"], - app_secret=os.environ["QTA_KIS_APP_SECRET"], - account_prefix=os.environ["QTA_KIS_ACCOUNT_PREFIX"], - account_product=os.environ["QTA_KIS_ACCOUNT_PRODUCT"], - environment="live", - transport=transport, - ) - if job["market"] == "KR": - settled, borrowed, positions, open_orders = collect_kr(broker, mapping) - effective_fx = Decimal(1) - fx_source = "FIXED_KRW" - else: - ( - settled, - borrowed, - positions, - open_orders, - effective_fx, - fx_source, - ) = collect_us( - broker, - mapping, - job["fx_to_krw"], - ) - positions = aggregate_positions(positions) - freeze_time = frozen_at or datetime.now(timezone.utc) - if freeze_time.tzinfo is None: - raise BlockedError("frozen_at must include a timezone") - freeze_time = freeze_time.astimezone(timezone.utc) - as_of = freeze_time.isoformat() - account_positions = [ - { - "market": item["market"], - "exchange": item["exchange"], - "symbol": item["symbol"], - "quantity": item["quantity"], - "market_value_krw": item["market_value_krw"], - } - for item in positions - ] - account = normalized_account_snapshot( - { - "schema": ACCOUNT_SCHEMA, - "broker": "kis", - "environment": "shadow", - "account_alias": job["account_alias"], - "market": job["market"], - "currency": "KRW" if job["market"] == "KR" else "USD", - "as_of": as_of, - "settled_cash": format(settled, "f"), - "borrowed_buying_power": format(borrowed, "f"), - "fx_to_krw": ( - format(effective_fx, "f") if effective_fx is not None else None - ), - "positions": account_positions, - "open_orders": open_orders, - }, - screen_schema="qta-screen/v2", - ) - exposure_positions = [ - { - "broker": "kis", - "market": item["market"], - "exchange": item["exchange"], - "symbol": item["symbol"], - "quantity": item["quantity"], - "market_value_krw": item["market_value_krw"], - } - for item in positions - ] - component_receipts: list[dict[str, Any]] = [] - component_brokers: set[str] = set() - for path in job["manual_exposure_components"]: - component = read_json_object(path, "manual exposure component") - component_positions, component_receipt = normalize_component( - component, - frozen_at=freeze_time, - max_age_seconds=job["manual_component_max_age_seconds"], - path=path, - ) - component_broker = component_receipt["broker"] - if component_broker in component_brokers: - raise BlockedError( - f"manual exposure broker appears more than once: {component_broker}" - ) - component_brokers.add(component_broker) - exposure_positions.extend(component_positions) - component_receipts.append(component_receipt) - exposure_keys: set[tuple[str, str, str]] = set() - for item in exposure_positions: - key = (item["broker"], item["exchange"], item["symbol"]) - if key in exposure_keys: - raise BlockedError( - "duplicate cross-broker exposure position: " + ":".join(key) - ) - exposure_keys.add(key) - exposure = normalized_exposure_snapshot( - { - "schema": EXPOSURE_SCHEMA, - "as_of": as_of, - "positions": exposure_positions, - } - ) - atomic_write_json(job["output_account_path"], account) - atomic_write_json(job["output_exposure_path"], exposure) - receipt = { - "schema": STATUS_SCHEMA, - "status": "READY", - "broker": "kis", - "mode": "shadow", - "market": job["market"], - "as_of": as_of, - "universe_manifest_hash": manifest_hash, - "account_path": str(job["output_account_path"]), - "account_sha256": sha256_file(job["output_account_path"]), - "exposure_path": str(job["output_exposure_path"]), - "exposure_sha256": sha256_file(job["output_exposure_path"]), - "manual_components": component_receipts, - "fx_to_krw": account["fx_to_krw"], - "fx_source": fx_source, - "settled_cash": account["settled_cash"], - "borrowed_buying_power_excluded": True, - "position_count": len(account["positions"]), - "open_order_count": len(account["open_orders"]), - "cross_broker_position_count": len(exposure["positions"]), - "api_mutation_count": 0, - } - atomic_write_json(job["output_status_path"], receipt) - return receipt - - -def manual_template(broker: str, source_as_of: str) -> dict[str, Any]: - if broker not in {"toss", "nh"}: - raise BlockedError("manual template broker must be toss or nh") - parsed = parse_aware_datetime(source_as_of, "source_as_of") - return { - "schema": COMPONENT_SCHEMA, - "broker": broker, - "source_kind": "verified-manual", - "source_as_of": parsed.isoformat(), - "positions": [], - } - - -def self_test() -> dict[str, Any]: - with tempfile.TemporaryDirectory() as directory_name: - directory = Path(directory_name) - now = datetime(2026, 7, 27, 0, 50, tzinfo=timezone.utc) - manifest_without_hash = { - "schema": MANIFEST_SCHEMA, - "build_status": "READY", - "instruments": [ - {"market": "KR", "exchange": "KOSPI", "broker_symbol": "005930"}, - {"market": "KR", "exchange": "KOSPI", "broker_symbol": "000660"}, - {"market": "US", "exchange": "NASDAQ", "broker_symbol": "AAPL"}, - {"market": "US", "exchange": "NYSE", "broker_symbol": "IBM"}, - ], - } - manifest = { - **manifest_without_hash, - "manifest_hash": sha256_json(manifest_without_hash), - } - manifest_path = directory / "manifest.json" - atomic_write_json(manifest_path, manifest) - manual_path = directory / "toss.json" - atomic_write_json( - manual_path, - { - "schema": COMPONENT_SCHEMA, - "broker": "toss", - "source_kind": "verified-manual", - "source_as_of": "2026-07-27T09:45:00+09:00", - "positions": [ - { - "market": "US", - "exchange": "NASDAQ", - "symbol": "AAPL", - "quantity": "0.5", - "market_value_krw": "100000", - } - ], - }, - ) - job_raw = { - "schema": JOB_SCHEMA, - "broker": "kis-live", - "mode": "shadow", - "market": "KR", - "account_alias": "kis-live-kr", - "universe_manifest": str(manifest_path), - "fx_to_krw": "1", - "manual_exposure_components": [str(manual_path)], - "manual_component_max_age_seconds": 3600, - "output_account_path": str(directory / "account.json"), - "output_exposure_path": str(directory / "exposure.json"), - "output_status_path": str(directory / "status.json"), - } - job_path = directory / "job.json" - atomic_write_json(job_path, job_raw) - job = load_job(job_path) - responses = [ - HttpResponse( - 200, - {}, - { - "access_token": "fixture-token", - "token_type": "Bearer", - "expires_in": 86400, - }, - ), - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "output1": [ - { - "pdno": "005930", - "hldg_qty": "2", - "evlu_amt": "150000", - } - ], - "output2": [ - { - "dnca_tot_amt": "2500000", - "nxdy_excc_amt": "2100000", - "prvs_rcdl_excc_amt": "2000000", - "tot_loan_amt": "5000000", - } - ], - }, - ), - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "output": [ - { - "pdno": "000660", - "psbl_qty": "1", - "sll_buy_dvsn_cd": "02", - } - ], - }, - ), - ] - previous = { - name: os.environ.get(name) - for name in ( - "QTA_KIS_APP_KEY", - "QTA_KIS_APP_SECRET", - "QTA_KIS_ACCOUNT_PREFIX", - "QTA_KIS_ACCOUNT_PRODUCT", - ) - } - os.environ.update( - { - "QTA_KIS_APP_KEY": "test-key", - "QTA_KIS_APP_SECRET": "test-secret", - "QTA_KIS_ACCOUNT_PREFIX": "12345678", - "QTA_KIS_ACCOUNT_PRODUCT": "01", - } - ) - try: - receipt = collect( - job, - transport=QueueTransport(responses), - frozen_at=now, - ) - finally: - for name, value in previous.items(): - if value is None: - os.environ.pop(name, None) - else: - os.environ[name] = value - account = read_json_object(job["output_account_path"], "test account") - exposure = read_json_object(job["output_exposure_path"], "test exposure") - assert receipt["status"] == "READY" - assert receipt["api_mutation_count"] == 0 - assert account["settled_cash"] == "2000000" - assert account["borrowed_buying_power"] == "5000000" - assert account["positions"][0]["exchange"] == "KOSPI" - assert account["open_orders"][0]["side"] == "BUY" - assert account["as_of"] == exposure["as_of"] - assert len(exposure["positions"]) == 2 - - us_job_raw = { - **job_raw, - "market": "US", - "account_alias": "kis-live-us", - "fx_to_krw": "KIS_PRESENT_BALANCE", - "output_account_path": str(directory / "account-us.json"), - "output_exposure_path": str(directory / "exposure-us.json"), - "output_status_path": str(directory / "status-us.json"), - } - us_job_path = directory / "job-us.json" - atomic_write_json(us_job_path, us_job_raw) - us_responses = [ - HttpResponse( - 200, - {}, - { - "access_token": "fixture-token", - "token_type": "Bearer", - "expires_in": 86400, - }, - ), - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "output1": [ - { - "pdno": "AAPL", - "ovrs_excg_cd": "NASD", - "cblc_qty13": "1.25", - "frcr_evlu_amt2": "250", - "loan_rmnd": "0", - } - ], - "output2": [ - { - "crcy_cd": "USD", - "frcr_dncl_amt_2": "1500", - "frcr_drwg_psbl_amt_1": "1200", - "nxdy_frcr_drwg_psbl_amt": "1100", - "frst_bltn_exrt": "1400", - } - ], - "output3": [{}], - }, - ), - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "output": [ - { - "pdno": "IBM", - "ovrs_excg_cd": "NYSE", - "nccs_qty": "2", - "sll_buy_dvsn_cd": "01", - } - ], - }, - ), - ] - previous_us = { - name: os.environ.get(name) - for name in ( - "QTA_KIS_APP_KEY", - "QTA_KIS_APP_SECRET", - "QTA_KIS_ACCOUNT_PREFIX", - "QTA_KIS_ACCOUNT_PRODUCT", - ) - } - os.environ.update( - { - "QTA_KIS_APP_KEY": "test-key", - "QTA_KIS_APP_SECRET": "test-secret", - "QTA_KIS_ACCOUNT_PREFIX": "12345678", - "QTA_KIS_ACCOUNT_PRODUCT": "01", - } - ) - try: - us_receipt = collect( - load_job(us_job_path), - transport=QueueTransport(us_responses), - frozen_at=now, - ) - finally: - for name, value in previous_us.items(): - if value is None: - os.environ.pop(name, None) - else: - os.environ[name] = value - us_account = read_json_object(Path(us_job_raw["output_account_path"]), "US account") - assert us_receipt["status"] == "READY" - assert us_account["settled_cash"] == "1100" - assert us_account["positions"][0]["market_value_krw"] == "350000" - assert us_account["fx_to_krw"] == "1400" - assert us_receipt["fx_source"] == "KIS_PRESENT_BALANCE:frst_bltn_exrt" - assert us_account["open_orders"][0]["exchange"] == "NYSE" - assert us_account["open_orders"][0]["side"] == "SELL" - - empty_us_job_raw = { - **us_job_raw, - "manual_exposure_components": [], - "output_account_path": str(directory / "account-us-empty.json"), - "output_exposure_path": str(directory / "exposure-us-empty.json"), - "output_status_path": str(directory / "status-us-empty.json"), - } - empty_us_job_path = directory / "job-us-empty.json" - atomic_write_json(empty_us_job_path, empty_us_job_raw) - empty_us_responses = [ - HttpResponse( - 200, - {}, - { - "access_token": "fixture-token", - "token_type": "Bearer", - "expires_in": 86400, - }, - ), - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "output1": [], - "output2": [], - "output3": [{"tot_frcr_cblc_smtl": "0"}], - }, - ), - HttpResponse(200, {}, {"rt_cd": "0", "output": []}), - ] - previous_empty_us = { - name: os.environ.get(name) - for name in ( - "QTA_KIS_APP_KEY", - "QTA_KIS_APP_SECRET", - "QTA_KIS_ACCOUNT_PREFIX", - "QTA_KIS_ACCOUNT_PRODUCT", - ) - } - os.environ.update( - { - "QTA_KIS_APP_KEY": "test-key", - "QTA_KIS_APP_SECRET": "test-secret", - "QTA_KIS_ACCOUNT_PREFIX": "12345678", - "QTA_KIS_ACCOUNT_PRODUCT": "01", - } - ) - try: - empty_us_receipt = collect( - load_job(empty_us_job_path), - transport=QueueTransport(empty_us_responses), - frozen_at=now, - ) - finally: - for name, value in previous_empty_us.items(): - if value is None: - os.environ.pop(name, None) - else: - os.environ[name] = value - empty_us_account = read_json_object( - Path(empty_us_job_raw["output_account_path"]), - "empty US account", - ) - assert empty_us_receipt["status"] == "READY" - assert empty_us_receipt["fx_source"] == ( - "NOT_APPLICABLE:NO_SETTLED_USD_OR_POSITION" - ) - assert empty_us_account["settled_cash"] == "0" - assert empty_us_account["fx_to_krw"] is None - assert empty_us_account["positions"] == [] - assert empty_us_account["open_orders"] == [] - - stale = read_json_object(manual_path, "manual") - stale["source_as_of"] = "2026-07-25T09:00:00+09:00" - try: - normalize_component( - stale, - frozen_at=now, - max_age_seconds=3600, - path=manual_path, - ) - except BlockedError: - pass - else: - raise AssertionError("stale manual exposure component was accepted") - return { - "schema": "qta-account-snapshot-self-test/v1", - "status": "PASS", - "self_test": "PASS", - "api_mutation_count": 0, - } - - -def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description=__doc__) - subparsers = parser.add_subparsers(dest="command", required=True) - collect_parser = subparsers.add_parser("collect") - collect_parser.add_argument("--job", type=Path, required=True) - template_parser = subparsers.add_parser("manual-template") - template_parser.add_argument("--broker", choices=("toss", "nh"), required=True) - template_parser.add_argument("--source-as-of", required=True) - template_parser.add_argument("--output", type=Path, required=True) - subparsers.add_parser("self-test") - return parser - - -def main() -> int: - args = build_parser().parse_args() - try: - if args.command == "collect": - output = collect(load_job(args.job)) - elif args.command == "manual-template": - if not args.output.is_absolute(): - raise BlockedError("--output must be absolute") - output = manual_template(args.broker, args.source_as_of) - atomic_write_json(args.output, output) - else: - output = self_test() - except BlockedError as exc: - print( - canonical_json( - { - "schema": "qta-account-snapshot-error/v1", - "status": "BLOCKED", - "reason": str(exc), - "api_mutation_count": 0, - } - ) - ) - return 2 - print(canonical_json(output)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/skills/quant-stock-polling-trader/scripts/broker_adapters.py b/skills/quant-stock-polling-trader/scripts/broker_adapters.py deleted file mode 100755 index 8062df2..0000000 --- a/skills/quant-stock-polling-trader/scripts/broker_adapters.py +++ /dev/null @@ -1,2379 +0,0 @@ -#!/usr/bin/env python3 -"""Toss and KIS REST adapters with explicit request contracts.""" - -from __future__ import annotations - -import argparse -import json -import sys -import time -from collections.abc import Callable -from dataclasses import dataclass, replace -from datetime import date, datetime, timedelta, timezone -from decimal import Decimal, ROUND_HALF_UP -from typing import Any, ClassVar -from urllib.error import HTTPError, URLError -from urllib.parse import urlencode -from urllib.request import Request, urlopen -from zoneinfo import ZoneInfo - -from execution_core import ( - BlockedError, - canonical_json, - decimal_value, - normalize_symbol, - redact, - sha256_json, -) - - -class TransportFailure(RuntimeError): - """Network failure with no authoritative broker response.""" - - -class AmbiguousMutationError(TransportFailure): - """Mutation may have reached the broker; caller must reconcile.""" - - -class AuthoritativeMutationRejection(BlockedError): - """Broker explicitly rejected a mutation before returning an order ID.""" - - -@dataclass(frozen=True) -class HttpResponse: - status: int - headers: dict[str, str] - body: Any - request_started_at: str | None = None - received_at: str | None = None - - -class UrlLibTransport: - def request( - self, - method: str, - url: str, - *, - headers: dict[str, str] | None = None, - query: dict[str, Any] | None = None, - json_body: dict[str, Any] | None = None, - form_body: dict[str, Any] | None = None, - timeout_seconds: float = 10.0, - mutation: bool = False, - ) -> HttpResponse: - if query: - url = f"{url}?{urlencode(query, doseq=True)}" - request_headers = dict(headers or {}) - data: bytes | None = None - if json_body is not None and form_body is not None: - raise ValueError("json_body and form_body are mutually exclusive") - if json_body is not None: - data = canonical_json(json_body).encode("utf-8") - request_headers.setdefault("Content-Type", "application/json") - elif form_body is not None: - data = urlencode(form_body).encode("utf-8") - request_headers.setdefault( - "Content-Type", "application/x-www-form-urlencoded" - ) - request = Request(url, data=data, headers=request_headers, method=method) - request_started_at = datetime.now(timezone.utc).isoformat() - try: - with urlopen(request, timeout=timeout_seconds) as response: - raw = response.read() - body = json.loads(raw) if raw else None - return HttpResponse( - status=response.status, - headers={ - key.lower(): value for key, value in response.headers.items() - }, - body=body, - request_started_at=request_started_at, - received_at=datetime.now(timezone.utc).isoformat(), - ) - except HTTPError as exc: - raw = exc.read() - try: - body = json.loads(raw) if raw else None - except json.JSONDecodeError: - body = {"raw": raw.decode("utf-8", errors="replace")} - return HttpResponse( - status=exc.code, - headers={key.lower(): value for key, value in exc.headers.items()}, - body=body, - request_started_at=request_started_at, - received_at=datetime.now(timezone.utc).isoformat(), - ) - except (URLError, TimeoutError) as exc: - if mutation: - raise AmbiguousMutationError(str(exc)) from exc - raise TransportFailure(str(exc)) from exc - - -class QueueTransport: - """Deterministic fixture transport used only by self-tests.""" - - def __init__(self, responses: list[HttpResponse | BaseException]): - self.responses = list(responses) - self.requests: list[dict[str, Any]] = [] - - def request(self, method: str, url: str, **kwargs: Any) -> HttpResponse: - self.requests.append({"method": method, "url": url, **kwargs}) - if not self.responses: - raise AssertionError("fixture transport exhausted") - request_started_at = datetime.now(timezone.utc).isoformat() - response = self.responses.pop(0) - if isinstance(response, BaseException): - raise response - return replace( - response, - request_started_at=response.request_started_at or request_started_at, - received_at=response.received_at or datetime.now(timezone.utc).isoformat(), - ) - - -class PacedTransport: - """Serialize requests with a deterministic minimum interval.""" - - def __init__(self, delegate: Any, minimum_interval_seconds: float): - self.delegate = delegate - self.minimum_interval_seconds = minimum_interval_seconds - self.last_started: float | None = None - - def request(self, method: str, url: str, **kwargs: Any) -> HttpResponse: - deadline_at = kwargs.pop("deadline_at", None) - deadline = normalize_deadline(deadline_at) - now = time.monotonic() - remaining = 0.0 - if self.last_started is not None: - remaining = self.minimum_interval_seconds - (now - self.last_started) - if deadline is not None: - projected_start = datetime.now(timezone.utc) + timedelta( - seconds=max(remaining, 0.0) - ) - if projected_start >= deadline: - raise BlockedError( - "mutation pacing would cross the absolute submit deadline" - ) - if remaining > 0: - time.sleep(remaining) - if deadline is not None and datetime.now(timezone.utc) >= deadline: - raise BlockedError("absolute submit deadline elapsed before HTTP send") - self.last_started = time.monotonic() - request_started_at = datetime.now(timezone.utc).isoformat() - response = self.delegate.request(method, url, **kwargs) - return replace( - response, - request_started_at=response.request_started_at or request_started_at, - received_at=response.received_at or datetime.now(timezone.utc).isoformat(), - ) - - -class MinimumIntervalRateLimiter: - """Conservative single-process pacing for one broker rate-limit group.""" - - def __init__( - self, - requests_per_second: float, - *, - monotonic: Callable[[], float] = time.monotonic, - sleeper: Callable[[float], None] = time.sleep, - ): - if isinstance(requests_per_second, bool) or requests_per_second <= 0: - raise ValueError("requests_per_second must be positive") - self.minimum_interval_seconds = 1.0 / float(requests_per_second) - self.monotonic = monotonic - self.sleeper = sleeper - self.last_started: float | None = None - - def acquire(self, *, deadline_at: datetime | None = None) -> None: - deadline = normalize_deadline(deadline_at) - now = self.monotonic() - remaining = 0.0 - if self.last_started is not None: - remaining = self.minimum_interval_seconds - (now - self.last_started) - if deadline is not None: - projected_start = datetime.now(timezone.utc) + timedelta( - seconds=max(remaining, 0.0) - ) - if projected_start >= deadline: - raise BlockedError( - "mutation pacing would cross the absolute submit deadline" - ) - if remaining > 0: - self.sleeper(remaining) - if deadline is not None and datetime.now(timezone.utc) >= deadline: - raise BlockedError("absolute submit deadline elapsed before HTTP send") - self.last_started = self.monotonic() - - def observe_headers(self, headers: dict[str, str]) -> None: - normalized = {str(key).lower(): value for key, value in headers.items()} - raw_limit = normalized.get("x-ratelimit-limit") - if raw_limit is None: - return - try: - observed_limit = float(raw_limit) - except (TypeError, ValueError): - return - if observed_limit > 0: - self.minimum_interval_seconds = max( - self.minimum_interval_seconds, - 1.0 / observed_limit, - ) - - -def require_mapping(value: Any, label: str) -> dict[str, Any]: - if not isinstance(value, dict): - raise BlockedError(f"{label} must be an object") - return value - - -def normalize_deadline(value: datetime | None) -> datetime | None: - if value is None: - return None - if not isinstance(value, datetime) or value.tzinfo is None: - raise BlockedError("absolute submit deadline must be timezone-aware") - return value.astimezone(timezone.utc) - - -def toss_result(response: HttpResponse, operation: str) -> Any: - body = require_mapping(response.body, f"Toss {operation} response") - if response.status < 200 or response.status >= 300: - error = body.get("error") - code = error.get("code") if isinstance(error, dict) else "unknown" - raise BlockedError(f"Toss {operation} failed: HTTP {response.status} {code}") - if "result" not in body: - raise BlockedError(f"Toss {operation} response missing result") - return body["result"] - - -def toss_mutation_result( - response: HttpResponse, - operation: str, -) -> dict[str, Any]: - """Return an acknowledged Toss mutation or classify it fail-closed.""" - if response.status >= 500: - raise AmbiguousMutationError( - f"Toss {operation} may have been accepted: HTTP {response.status}" - ) - if not isinstance(response.body, dict): - raise AmbiguousMutationError( - f"Toss {operation} returned a non-object mutation response" - ) - body = response.body - if response.status < 200 or response.status >= 300: - error = body.get("error") - code = error.get("code") if isinstance(error, dict) else None - if 400 <= response.status < 500 and str(code or "").strip(): - raise AuthoritativeMutationRejection( - f"Toss {operation} rejected: HTTP {response.status} {code}" - ) - raise AmbiguousMutationError( - f"Toss {operation} returned no authoritative rejection: " - f"HTTP {response.status}" - ) - result = body.get("result") - if not isinstance(result, dict): - raise AmbiguousMutationError( - f"Toss {operation} success response missing result object" - ) - return result - - -def kis_result(response: HttpResponse, operation: str) -> dict[str, Any]: - body = require_mapping(response.body, f"KIS {operation} response") - if response.status != 200: - raise BlockedError(f"KIS {operation} failed: HTTP {response.status}") - if str(body.get("rt_cd")) != "0": - raise BlockedError( - f"KIS {operation} failed: {body.get('msg_cd', 'unknown')} " - f"{body.get('msg1', '')}".strip() - ) - return body - - -def kis_mutation_result( - response: HttpResponse, - operation: str, -) -> dict[str, Any]: - """Return an acknowledged KIS mutation or classify it fail-closed.""" - if response.status >= 500: - raise AmbiguousMutationError( - f"KIS {operation} may have been accepted: HTTP {response.status}" - ) - if not isinstance(response.body, dict): - raise AmbiguousMutationError( - f"KIS {operation} returned a non-object mutation response" - ) - body = response.body - if response.status != 200: - code = body.get("error_code", body.get("msg_cd")) - if 400 <= response.status < 500 and str(code or "").strip(): - raise AuthoritativeMutationRejection( - f"KIS {operation} rejected: HTTP {response.status} {code}" - ) - raise AmbiguousMutationError( - f"KIS {operation} returned no authoritative rejection: " - f"HTTP {response.status}" - ) - result_code = str(body.get("rt_cd", "")).strip() - if result_code != "0": - message_code = str(body.get("msg_cd", "")).strip() - if result_code and message_code: - raise AuthoritativeMutationRejection( - f"KIS {operation} rejected: {message_code} " - f"{body.get('msg1', '')}".strip() - ) - raise AmbiguousMutationError( - f"KIS {operation} response missing an authoritative result code" - ) - output = body.get("output") - if not isinstance(output, dict): - raise AmbiguousMutationError( - f"KIS {operation} success response missing output object" - ) - return output - - -def mutation_order_id(value: Any, label: str) -> str: - """Require the acknowledgement identifier needed for reconciliation.""" - if isinstance(value, bool): - raise AmbiguousMutationError(f"{label} is missing") - order_id = str(value or "").strip() - if not order_id: - raise AmbiguousMutationError(f"{label} is missing") - return order_id - - -def decimal_string(value: Any, field: str) -> str: - return format(decimal_value(value, field), "f") - - -def parse_expiration_timestamp(value: Any, field: str) -> datetime: - if isinstance(value, datetime): - parsed = value - elif isinstance(value, str): - try: - parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) - except ValueError as exc: - raise BlockedError(f"{field} must be an ISO-8601 timestamp") from exc - else: - raise BlockedError(f"{field} must be an ISO-8601 timestamp") - if parsed.tzinfo is None: - raise BlockedError(f"{field} must include a timezone") - return parsed.astimezone(timezone.utc) - - -def positive_level_prices(value: Any, label: str) -> list[Any]: - if not isinstance(value, list) or not value: - raise BlockedError(f"{label} must be a non-empty array") - prices = [] - for index, level in enumerate(value): - if not isinstance(level, dict) or "price" not in level: - raise BlockedError(f"{label}[{index}] must contain price") - price = decimal_value(level["price"], f"{label}[{index}].price") - if price > 0: - prices.append(price) - if not prices: - raise BlockedError(f"{label} has no positive price level") - return prices - - -def kis_market_timestamp( - *, - date_value: Any, - time_value: Any, - timezone_name: str, -) -> str | None: - """Parse a KIS market-local YYYYMMDD/HHMMSS timestamp, or fail closed.""" - compact_time = str(time_value or "").strip() - if len(compact_time) != 6 or not compact_time.isdigit(): - return None - compact_date = str(date_value or "").strip() - zone = ZoneInfo(timezone_name) - if len(compact_date) != 8 or not compact_date.isdigit(): - return None - try: - parsed = datetime.strptime(compact_date + compact_time, "%Y%m%d%H%M%S").replace( - tzinfo=zone - ) - except ValueError: - return None - return parsed.isoformat() - - -class TossBroker: - LIVE_BASE_URL = "https://openapi.tossinvest.com" - CREATE_ORDER_PATH = "/api/v1/orders" - MARKET_DATA_TPS = 10 - ORDER_TPS = 3 - ORDER_HISTORY_TPS = 5 - - def __init__( - self, - *, - client_id: str, - client_secret: str, - account_seq: int, - transport: Any | None = None, - access_token: str | None = None, - access_token_expires_at: datetime | str | None = None, - market_data_limiter: MinimumIntervalRateLimiter | None = None, - order_limiter: MinimumIntervalRateLimiter | None = None, - order_history_limiter: MinimumIntervalRateLimiter | None = None, - ): - if not client_id or not client_secret: - raise BlockedError("Toss client_id and client_secret are required") - if isinstance(account_seq, bool) or not isinstance(account_seq, int): - raise BlockedError("Toss account_seq must be an integer") - self.client_id = client_id - self.client_secret = client_secret - self.account_seq = account_seq - self.transport = transport or UrlLibTransport() - if access_token is not None and ( - not isinstance(access_token, str) or not access_token - ): - raise BlockedError("Toss access_token must be a non-empty string") - if (access_token is None) != (access_token_expires_at is None): - raise BlockedError( - "Toss external access token and expiration timestamp " - "must be provided together" - ) - self.access_token = access_token - self.token_expires_at = ( - None - if access_token_expires_at is None - else parse_expiration_timestamp( - access_token_expires_at, - "Toss access token expiration", - ) - ) - self.market_data_limiter = market_data_limiter or MinimumIntervalRateLimiter( - self.MARKET_DATA_TPS - ) - self.order_limiter = order_limiter or MinimumIntervalRateLimiter(self.ORDER_TPS) - self.order_history_limiter = ( - order_history_limiter or MinimumIntervalRateLimiter(self.ORDER_HISTORY_TPS) - ) - - @staticmethod - def capabilities() -> dict[str, Any]: - return { - "broker": "toss", - "supported_markets": ["KR", "US"], - "paper": False, - "quote_batch_max": 200, - "batch_bid_ask": False, - "order_types": ["LIMIT", "MARKET"], - "initial_contract_order_types": ["LIMIT"], - "client_order_id": True, - "client_order_id_ttl_seconds": 600, - "query_by_client_order_id": False, - "modify_returns_new_id": True, - "cancel_returns_new_id": True, - "individual_fills": False, - "conditional_types": ["SINGLE", "OCO", "OTO"], - "client_rate_limits_tps": { - "MARKET_DATA": TossBroker.MARKET_DATA_TPS, - "ORDER": TossBroker.ORDER_TPS, - "ORDER_HISTORY": TossBroker.ORDER_HISTORY_TPS, - }, - "order_rate_limit_policy": "peak_safe", - } - - def token(self) -> str: - if ( - self.access_token - and self.token_expires_at is not None - and datetime.now(timezone.utc) + timedelta(seconds=60) - < self.token_expires_at - ): - return self.access_token - response = self.transport.request( - "POST", - f"{self.LIVE_BASE_URL}/oauth2/token", - form_body={ - "grant_type": "client_credentials", - "client_id": self.client_id, - "client_secret": self.client_secret, - }, - timeout_seconds=10, - mutation=False, - ) - body = require_mapping(response.body, "Toss token response") - if response.status != 200 or "access_token" not in body: - raise BlockedError( - f"Toss token request failed: HTTP {response.status} " - f"{body.get('error', 'unknown')}" - ) - expires_in = int(body.get("expires_in", 0)) - if expires_in <= 0: - raise BlockedError("Toss token response has invalid expires_in") - self.access_token = str(body["access_token"]) - self.token_expires_at = datetime.now(timezone.utc) + timedelta( - seconds=expires_in - ) - return self.access_token - - def headers(self, *, account: bool) -> dict[str, str]: - headers = {"Authorization": f"Bearer {self.token()}"} - if account: - headers["X-Tossinvest-Account"] = str(self.account_seq) - return headers - - def limited_request( - self, - limiter: MinimumIntervalRateLimiter, - method: str, - url: str, - *, - deadline_at: datetime | None = None, - **kwargs: Any, - ) -> HttpResponse: - limiter.acquire(deadline_at=deadline_at) - request_started_at = datetime.now(timezone.utc).isoformat() - response = self.transport.request(method, url, **kwargs) - limiter.observe_headers(response.headers) - return replace( - response, - request_started_at=response.request_started_at or request_started_at, - received_at=response.received_at or datetime.now(timezone.utc).isoformat(), - ) - - def mutation_request_hash( - self, - method: str, - path: str, - body: dict[str, Any], - ) -> str: - return sha256_json( - { - "method": method.upper(), - "path": path, - "account_seq": str(self.account_seq), - "body": body, - } - ) - - @staticmethod - def build_order_body(intent: dict[str, Any]) -> dict[str, Any]: - if intent.get("side") not in {"BUY", "SELL"}: - raise BlockedError("Toss intent side must be BUY or SELL") - if intent.get("order_type") != "LIMIT": - raise BlockedError("initial Toss contract requires LIMIT") - quantity = decimal_value(intent.get("quantity"), "quantity") - if quantity <= 0 or quantity != quantity.to_integral_value(): - raise BlockedError("initial Toss contract requires positive whole quantity") - client_order_id = str(intent.get("client_order_id", "")) - if ( - not client_order_id - or len(client_order_id) > 36 - or any( - character - not in "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" - for character in client_order_id - ) - ): - raise BlockedError("invalid Toss client_order_id") - return { - "clientOrderId": client_order_id, - "symbol": normalize_symbol(str(intent["market"]), str(intent["symbol"])), - "side": intent["side"], - "orderType": "LIMIT", - "timeInForce": "DAY", - "quantity": format(quantity, "f"), - "price": decimal_string(intent["limit_price"], "limit_price"), - "confirmHighValueOrder": False, - } - - @staticmethod - def build_oco_body(intent: dict[str, Any], expire_date: str) -> dict[str, Any]: - quantity = decimal_value(intent["quantity"], "quantity") - if quantity != quantity.to_integral_value(): - raise BlockedError("Toss OCO quantity must be a whole number") - take_profit = decimal_string(intent["take_profit_price"], "take_profit_price") - stop = decimal_string(intent["stop_price"], "stop_price") - return { - "symbol": normalize_symbol(str(intent["market"]), str(intent["symbol"])), - "type": "OCO", - "quantity": format(quantity, "f"), - "orderType": "LIMIT", - "clientOrderId": f"oco-{str(intent['intent_id'])[:28]}", - "expireDate": expire_date, - "first": { - "orderSide": "SELL", - "triggerPrice": take_profit, - "orderPrice": take_profit, - }, - "second": { - "orderSide": "SELL", - "triggerPrice": stop, - "orderPrice": stop, - }, - "confirmHighValueOrder": False, - } - - def preview_submit(self, intent: dict[str, Any]) -> dict[str, Any]: - body = self.build_order_body(intent) - request_hash = self.mutation_request_hash( - "POST", - self.CREATE_ORDER_PATH, - body, - ) - return { - "method": "POST", - "path": self.CREATE_ORDER_PATH, - "headers": { - "Authorization": "[REDACTED]", - "X-Tossinvest-Account": "[REDACTED]", - }, - "body": body, - "request_hash": request_hash, - "timeout_policy": "retry same clientOrderId and byte-equivalent body only within 600 seconds", - } - - def submit( - self, - intent: dict[str, Any], - *, - deadline_at: datetime | None = None, - ) -> dict[str, Any]: - body = self.build_order_body(intent) - request_hash = self.mutation_request_hash( - "POST", - self.CREATE_ORDER_PATH, - body, - ) - response = self.limited_request( - self.order_limiter, - "POST", - f"{self.LIVE_BASE_URL}{self.CREATE_ORDER_PATH}", - headers=self.headers(account=True), - json_body=body, - timeout_seconds=10, - mutation=True, - deadline_at=deadline_at, - ) - result = toss_mutation_result(response, "submit") - order_id = mutation_order_id( - result.get("orderId"), - "Toss submit result orderId", - ) - return { - "broker_order_id": order_id, - "client_order_id": result.get("clientOrderId"), - "request_hash": request_hash, - "submit_started_at": response.request_started_at, - "ack_received_at": response.received_at, - "raw": redact(result), - } - - def quote(self, market: str, symbol: str) -> dict[str, Any]: - normalized = normalize_symbol(market, symbol) - headers = self.headers(account=False) - price_response = self.limited_request( - self.market_data_limiter, - "GET", - f"{self.LIVE_BASE_URL}/api/v1/prices", - headers=headers, - query={"symbols": normalized}, - timeout_seconds=5, - ) - prices = toss_result(price_response, "price") - if not isinstance(prices, list): - raise BlockedError("Toss price result must be an array") - matching = [ - item - for item in prices - if isinstance(item, dict) and item.get("symbol") == normalized - ] - if len(matching) != 1: - raise BlockedError(f"Toss price missing or duplicated: {normalized}") - orderbook_response = self.limited_request( - self.market_data_limiter, - "GET", - f"{self.LIVE_BASE_URL}/api/v1/orderbook", - headers=headers, - query={"symbol": normalized}, - timeout_seconds=5, - ) - orderbook = require_mapping( - toss_result(orderbook_response, "orderbook"), "Toss orderbook result" - ) - asks = positive_level_prices(orderbook.get("asks"), "Toss asks") - bids = positive_level_prices(orderbook.get("bids"), "Toss bids") - return { - "market": market, - "symbol": normalized, - "currency": matching[0].get("currency"), - "last_price": decimal_string(matching[0].get("lastPrice"), "lastPrice"), - "best_ask": format(min(asks), "f"), - "best_bid": format(max(bids), "f"), - "trade_timestamp": matching[0].get("timestamp"), - "book_timestamp": orderbook.get("timestamp"), - "received_at": datetime.now(timezone.utc).isoformat(), - "raw_status": "OK", - } - - def get_order(self, broker_order_id: str) -> dict[str, Any]: - response = self.limited_request( - self.order_history_limiter, - "GET", - f"{self.LIVE_BASE_URL}/api/v1/orders/{broker_order_id}", - headers=self.headers(account=True), - timeout_seconds=10, - ) - return require_mapping(toss_result(response, "get order"), "Toss order result") - - def cancel(self, broker_order_id: str) -> dict[str, Any]: - path = f"/api/v1/orders/{broker_order_id}/cancel" - body: dict[str, Any] = {} - request_hash = self.mutation_request_hash("POST", path, body) - response = self.limited_request( - self.order_limiter, - "POST", - f"{self.LIVE_BASE_URL}{path}", - headers=self.headers(account=True), - json_body=body, - timeout_seconds=10, - mutation=True, - ) - result = toss_mutation_result(response, "cancel") - operation_order_id = mutation_order_id( - result.get("orderId"), - "Toss cancel result operation orderId", - ) - return { - "original_order_id": broker_order_id, - "operation_order_id": operation_order_id, - "request_hash": request_hash, - "raw": redact(result), - } - - -class KisBroker: - BASE_URLS: ClassVar[dict[str, str]] = { - "live": "https://openapi.koreainvestment.com:9443", - "paper": "https://openapivts.koreainvestment.com:29443", - } - KR_QUOTE_VENUE: ClassVar[dict[str, str]] = { - "KRX": "J", - "NXT": "NX", - "SOR": "UN", - } - US_ORDER_VENUES: ClassVar[set[str]] = {"NASD", "NYSE", "AMEX"} - US_QUOTE_VENUE: ClassVar[dict[str, str]] = { - "NASD": "NAS", - "NYSE": "NYS", - "AMEX": "AMS", - } - REGIME_BENCHMARKS: ClassVar[dict[str, dict[str, str]]] = { - "KOSPI": { - "market": "KR", - "benchmark_id": "KOSPI_COMPOSITE", - "provider_symbol": "0001", - "regime_proxy_id": "KOSPI_COMPOSITE", - "timezone": "Asia/Seoul", - }, - "KOSDAQ": { - "market": "KR", - "benchmark_id": "KOSDAQ_COMPOSITE", - "provider_symbol": "1001", - "regime_proxy_id": "KOSDAQ_COMPOSITE", - "timezone": "Asia/Seoul", - }, - "NYSE": { - "market": "US", - "benchmark_id": "NYSE_COMPOSITE", - "provider_symbol": "SPX", - "regime_proxy_id": "S&P_500", - "timezone": "America/New_York", - }, - "NASDAQ": { - "market": "US", - "benchmark_id": "NASDAQ_COMPOSITE", - "provider_symbol": "COMP", - "regime_proxy_id": "NASDAQ_COMPOSITE", - "timezone": "America/New_York", - }, - } - - def __init__( - self, - *, - app_key: str, - app_secret: str, - account_prefix: str, - account_product: str, - environment: str, - transport: Any | None = None, - access_token: str | None = None, - ): - if environment not in self.BASE_URLS: - raise BlockedError("KIS environment must be live or paper") - if not app_key or not app_secret: - raise BlockedError("KIS app_key and app_secret are required") - if len(account_prefix) != 8 or not account_prefix.isdigit(): - raise BlockedError("KIS account_prefix must be 8 digits") - if len(account_product) != 2 or not account_product.isdigit(): - raise BlockedError("KIS account_product must be 2 digits") - self.app_key = app_key - self.app_secret = app_secret - self.account_prefix = account_prefix - self.account_product = account_product - self.environment = environment - raw_transport = transport or UrlLibTransport() - interval = ( - 0.0 if transport is not None else (1.05 if environment == "paper" else 0.12) - ) - self.transport = PacedTransport(raw_transport, interval) - self.access_token = access_token - - @staticmethod - def capabilities(environment: str) -> dict[str, Any]: - if environment not in {"live", "paper"}: - raise BlockedError("KIS environment must be live or paper") - return { - "broker": "kis", - "environment": environment, - "supported_markets": ["KR", "US"], - "paper": True, - "polling_order_book_markets": ( - ["KR"] if environment == "paper" else ["KR", "US"] - ), - "us_paper_polling_session": False, - "quote_batch_max": 1, - "batch_bid_ask": False, - "order_types": ["LIMIT"], - "client_order_id": False, - "query_by_client_order_id": False, - "modify_returns_new_id": False, - "cancel_returns_new_id": False, - "individual_fills": False, - "server_oco": False, - "whole_shares_only": True, - "minimum_request_interval_ms": 1000 if environment == "paper" else 100, - } - - @property - def base_url(self) -> str: - return self.BASE_URLS[self.environment] - - def token(self) -> str: - if self.access_token: - return self.access_token - response = self.transport.request( - "POST", - f"{self.base_url}/oauth2/tokenP", - json_body={ - "grant_type": "client_credentials", - "appkey": self.app_key, - "appsecret": self.app_secret, - }, - timeout_seconds=10, - ) - body = require_mapping(response.body, "KIS token response") - if response.status != 200 or not body.get("access_token"): - raise BlockedError( - f"KIS token request failed: HTTP {response.status} " - f"{body.get('error_code', body.get('msg_cd', 'unknown'))}" - ) - self.access_token = str(body["access_token"]) - return self.access_token - - def headers(self, tr_id: str) -> dict[str, str]: - return { - "authorization": f"Bearer {self.token()}", - "appkey": self.app_key, - "appsecret": self.app_secret, - "tr_id": tr_id, - "custtype": "P", - "Content-Type": "application/json", - } - - def order_tr_id(self, market: str, side: str) -> str: - market = market.upper() - side = side.upper() - if market == "KR": - live = "TTTC0012U" if side == "BUY" else "TTTC0011U" - return live if self.environment == "live" else f"V{live[1:]}" - if market == "US": - if side == "BUY": - return "TTTT1002U" if self.environment == "live" else "VTTT1002U" - if side == "SELL": - return "TTTT1006U" if self.environment == "live" else "VTTT1001U" - raise BlockedError(f"unsupported KIS order market/side: {market}/{side}") - - def kr_quote_venue(self, venue: str) -> str: - venue = venue.upper() - if venue not in self.KR_QUOTE_VENUE: - raise BlockedError("KR venue must be KRX, NXT, or SOR") - if self.environment == "paper" and venue != "KRX": - raise BlockedError("KIS paper KR quotes require KRX") - return self.KR_QUOTE_VENUE[venue] - - def build_order_request( - self, intent: dict[str, Any], *, venue: str - ) -> dict[str, Any]: - market = str(intent["market"]).upper() - side = str(intent["side"]).upper() - if side not in {"BUY", "SELL"}: - raise BlockedError("KIS side must be BUY or SELL") - if intent.get("order_type") != "LIMIT": - raise BlockedError("initial KIS contract requires LIMIT") - quantity = decimal_value(intent["quantity"], "quantity") - if quantity <= 0 or quantity != quantity.to_integral_value(): - raise BlockedError("KIS stock quantity must be a positive whole number") - symbol = normalize_symbol(market, str(intent["symbol"])) - price = decimal_string(intent["limit_price"], "limit_price") - tr_id = self.order_tr_id(market, side) - if market == "KR": - if venue not in {"KRX", "NXT", "SOR"}: - raise BlockedError("KR venue must be KRX, NXT, or SOR") - if self.environment == "paper" and venue != "KRX": - raise BlockedError("KIS paper KR orders require KRX") - body = { - "CANO": self.account_prefix, - "ACNT_PRDT_CD": self.account_product, - "PDNO": symbol, - "ORD_DVSN": "00", - "ORD_QTY": format(quantity, "f"), - "ORD_UNPR": price, - "EXCG_ID_DVSN_CD": venue, - "SLL_TYPE": "01" if side == "SELL" else "", - "CNDT_PRIC": "", - } - path = "/uapi/domestic-stock/v1/trading/order-cash" - else: - if venue not in self.US_ORDER_VENUES: - raise BlockedError("US venue must be NASD, NYSE, or AMEX") - body = { - "CANO": self.account_prefix, - "ACNT_PRDT_CD": self.account_product, - "OVRS_EXCG_CD": venue, - "PDNO": symbol, - "ORD_QTY": format(quantity, "f"), - "OVRS_ORD_UNPR": price, - "CTAC_TLNO": "", - "MGCO_APTM_ODNO": "", - "SLL_TYPE": "00" if side == "SELL" else "", - "ORD_SVR_DVSN_CD": "0", - "ORD_DVSN": "00", - } - path = "/uapi/overseas-stock/v1/trading/order" - return { - "method": "POST", - "path": path, - "tr_id": tr_id, - "body": body, - "request_hash": sha256_json({"tr_id": tr_id, "body": body}), - } - - def preview_submit(self, intent: dict[str, Any], *, venue: str) -> dict[str, Any]: - request = self.build_order_request(intent, venue=venue) - return { - **request, - "headers": { - "authorization": "[REDACTED]", - "appkey": "[REDACTED]", - "appsecret": "[REDACTED]", - "tr_id": request["tr_id"], - "custtype": "P", - }, - "body": redact(request["body"]), - "timeout_policy": "never resubmit automatically; reconcile orders, fills, and balances", - } - - def submit( - self, - intent: dict[str, Any], - *, - venue: str, - deadline_at: datetime | None = None, - ) -> dict[str, Any]: - request = self.build_order_request(intent, venue=venue) - response = self.transport.request( - "POST", - f"{self.base_url}{request['path']}", - headers=self.headers(request["tr_id"]), - json_body=request["body"], - timeout_seconds=10, - mutation=True, - deadline_at=deadline_at, - ) - output = kis_mutation_result(response, "submit") - order_id = mutation_order_id( - output.get("ODNO"), - "KIS submit output ODNO", - ) - return { - "broker_order_id": order_id, - "request_hash": request["request_hash"], - "broker_time": output.get("ORD_TMD"), - "organization": output.get("KRX_FWDG_ORD_ORGNO"), - "submit_started_at": response.request_started_at, - "ack_received_at": response.received_at, - "raw": redact(output), - } - - def quote(self, market: str, symbol: str, *, venue: str) -> dict[str, Any]: - market = market.upper() - symbol = normalize_symbol(market, symbol) - if market == "KR": - quote_venue = self.kr_quote_venue(venue) - tr_id = "FHKST01010100" - price_response = self.transport.request( - "GET", - f"{self.base_url}/uapi/domestic-stock/v1/quotations/inquire-price", - headers=self.headers(tr_id), - query={ - "FID_COND_MRKT_DIV_CODE": quote_venue, - "FID_INPUT_ISCD": symbol, - }, - timeout_seconds=5, - ) - price_body = kis_result(price_response, "KR price") - output = require_mapping(price_body.get("output"), "KIS KR price output") - book_tr = "FHKST01010200" - book_response = self.transport.request( - "GET", - f"{self.base_url}/uapi/domestic-stock/v1/quotations/inquire-asking-price-exp-ccn", - headers=self.headers(book_tr), - query={ - "FID_COND_MRKT_DIV_CODE": quote_venue, - "FID_INPUT_ISCD": symbol, - }, - timeout_seconds=5, - ) - book_body = kis_result(book_response, "KR orderbook") - book = require_mapping(book_body.get("output1"), "KIS KR orderbook output") - received_at = datetime.now(timezone.utc) - price_date = str(output.get("stck_bsop_date") or "").strip() - book_date = str(book.get("stck_bsop_date") or "").strip() - if any( - len(value) != 8 or not value.isdigit() - for value in (price_date, book_date) - ): - raise BlockedError( - "KIS KR price and orderbook responses each require a valid " - "broker-supplied stck_bsop_date" - ) - if price_date != book_date: - raise BlockedError( - "KIS KR price and orderbook stck_bsop_date values must match" - ) - trade_timestamp = kis_market_timestamp( - date_value=price_date, - time_value=output.get("stck_cntg_hour"), - timezone_name="Asia/Seoul", - ) - book_timestamp = kis_market_timestamp( - date_value=book_date, - time_value=book.get("aspr_acpt_hour"), - timezone_name="Asia/Seoul", - ) - if trade_timestamp is None or book_timestamp is None: - raise BlockedError( - "KIS KR quote response requires independent broker-supplied " - "stck_cntg_hour and aspr_acpt_hour timestamps" - ) - last = output.get("stck_prpr") - if last is None: - raise BlockedError("KIS KR price response missing last price") - return { - "market": "KR", - "symbol": symbol, - "currency": "KRW", - "last_price": decimal_string(last, "stck_prpr"), - "best_ask": decimal_string(book.get("askp1"), "askp1"), - "best_bid": decimal_string(book.get("bidp1"), "bidp1"), - "trade_timestamp": trade_timestamp, - "book_timestamp": book_timestamp, - "received_at": received_at.isoformat(), - "source_timestamp_raw": { - "trade": { - "stck_bsop_date": price_date, - "stck_cntg_hour": output.get("stck_cntg_hour"), - }, - "book": { - "stck_bsop_date": book_date, - "aspr_acpt_hour": book.get("aspr_acpt_hour"), - }, - }, - "raw_status": "OK", - } - if self.environment == "paper": - raise BlockedError( - "KIS U.S. paper order book HHDFS76200100 is unsupported; " - "use production shadow quotes and separate paper serialization tests" - ) - if venue not in self.US_ORDER_VENUES: - raise BlockedError("US venue must be NASD, NYSE, or AMEX") - quote_venue = self.US_QUOTE_VENUE[venue] - book_tr = "HHDFS76200100" - book_response = self.transport.request( - "GET", - f"{self.base_url}/uapi/overseas-price/v1/quotations/inquire-asking-price", - headers=self.headers(book_tr), - query={"AUTH": "", "EXCD": quote_venue, "SYMB": symbol}, - timeout_seconds=5, - ) - book_body = kis_result(book_response, "US orderbook") - book = require_mapping(book_body.get("output1"), "KIS US orderbook output") - last = book.get("last") - ask = book.get("pask1") or book.get("pask") - bid = book.get("pbid1") or book.get("pbid") - if last is None or ask is None or bid is None: - raise BlockedError( - "KIS US orderbook response missing same-source last/bid/ask" - ) - received_at = datetime.now(timezone.utc) - source_timestamp = kis_market_timestamp( - date_value=book.get("dymd"), - time_value=book.get("dhms"), - timezone_name="America/New_York", - ) - if source_timestamp is None: - raise BlockedError( - "KIS US orderbook response requires broker-supplied dymd/dhms" - ) - return { - "market": "US", - "symbol": symbol, - "currency": "USD", - "last_price": decimal_string(last, "last"), - "best_ask": decimal_string(ask, "best ask"), - "best_bid": decimal_string(bid, "best bid"), - "trade_timestamp": source_timestamp, - "book_timestamp": source_timestamp, - "received_at": received_at.isoformat(), - "source_timestamp_raw": { - "dymd": book.get("dymd"), - "dhms": book.get("dhms"), - }, - "raw_status": "OK", - } - - def benchmark_quote( - self, - *, - exchange: str, - session_date: str, - ) -> dict[str, Any]: - """Return one same-session, read-only benchmark regime observation.""" - exchange = str(exchange).upper() - contract = self.REGIME_BENCHMARKS.get(exchange) - if contract is None: - raise BlockedError(f"unsupported benchmark exchange: {exchange}") - try: - parsed_session_date = date.fromisoformat(session_date) - except (TypeError, ValueError) as exc: - raise BlockedError("benchmark session_date must be YYYY-MM-DD") from exc - if parsed_session_date.isoformat() != session_date: - raise BlockedError("benchmark session_date must be YYYY-MM-DD") - compact_date = session_date.replace("-", "") - received_at = datetime.now(timezone.utc) - - if contract["market"] == "KR": - tr_id = "FHPUP02110100" - response = self.transport.request( - "GET", - ( - f"{self.base_url}/uapi/domestic-stock/v1/quotations/" - "inquire-index-tickprice" - ), - headers=self.headers(tr_id), - query={ - "FID_INPUT_ISCD": contract["provider_symbol"], - "FID_COND_MRKT_DIV_CODE": "U", - }, - timeout_seconds=5, - ) - body = kis_result(response, f"{exchange} benchmark") - rows = body.get("output") - if not isinstance(rows, list) or not rows: - raise BlockedError( - f"KIS {exchange} benchmark output must be a non-empty array" - ) - valid_rows = [ - row - for row in rows - if isinstance(row, dict) - and kis_market_timestamp( - date_value=compact_date, - time_value=row.get("stck_cntg_hour"), - timezone_name=contract["timezone"], - ) - is not None - ] - if not valid_rows: - raise BlockedError( - f"KIS {exchange} benchmark has no broker timestamp" - ) - latest = max( - valid_rows, - key=lambda row: str(row["stck_cntg_hour"]), - ) - source_timestamp = kis_market_timestamp( - date_value=compact_date, - time_value=latest["stck_cntg_hour"], - timezone_name=contract["timezone"], - ) - current = decimal_value( - latest.get("bstp_nmix_prpr"), - f"{exchange} benchmark current", - ) - difference = abs( - decimal_value( - latest.get("bstp_nmix_prdy_vrss"), - f"{exchange} benchmark previous difference", - ) - ) - sign = str(latest.get("prdy_vrss_sign") or "").strip() - if sign in {"1", "2"}: - signed_difference = difference - elif sign == "3": - signed_difference = Decimal(0) - elif sign in {"4", "5"}: - signed_difference = -difference - else: - raise BlockedError( - f"KIS {exchange} benchmark has an unknown previous-close sign" - ) - previous_close = current - signed_difference - if current <= 0 or previous_close <= 0: - raise BlockedError( - f"KIS {exchange} benchmark prices must be positive" - ) - else: - if self.environment == "paper": - raise BlockedError( - "KIS U.S. paper benchmark feed is unsupported; use " - "kis-live/shadow" - ) - tr_id = "FHKST03030200" - response = self.transport.request( - "GET", - ( - f"{self.base_url}/uapi/overseas-price/v1/quotations/" - "inquire-time-indexchartprice" - ), - headers=self.headers(tr_id), - query={ - "FID_COND_MRKT_DIV_CODE": "N", - "FID_INPUT_ISCD": contract["provider_symbol"], - "FID_HOUR_CLS_CODE": "0", - "FID_PW_DATA_INCU_YN": "N", - }, - timeout_seconds=5, - ) - body = kis_result(response, f"{exchange} benchmark") - summary = require_mapping( - body.get("output1"), - f"KIS {exchange} benchmark output1", - ) - rows = body.get("output2") - if not isinstance(rows, list) or not rows: - raise BlockedError( - f"KIS {exchange} benchmark output2 must be a non-empty array" - ) - valid_rows = [ - row - for row in rows - if isinstance(row, dict) - and str(row.get("stck_bsop_date") or "").strip() == compact_date - and kis_market_timestamp( - date_value=row.get("stck_bsop_date"), - time_value=row.get("stck_cntg_hour"), - timezone_name=contract["timezone"], - ) - is not None - ] - if not valid_rows: - raise BlockedError( - f"KIS {exchange} benchmark has no bar for {session_date}" - ) - latest = max( - valid_rows, - key=lambda row: str(row["stck_cntg_hour"]), - ) - source_timestamp = kis_market_timestamp( - date_value=latest["stck_bsop_date"], - time_value=latest["stck_cntg_hour"], - timezone_name=contract["timezone"], - ) - current = decimal_value( - latest.get("optn_prpr"), - f"{exchange} benchmark current", - ) - previous_close = decimal_value( - summary.get("ovrs_nmix_prdy_clpr"), - f"{exchange} benchmark previous close", - ) - if current <= 0 or previous_close <= 0: - raise BlockedError( - f"KIS {exchange} benchmark prices must be positive" - ) - - if source_timestamp is None: - raise BlockedError( - f"KIS {exchange} benchmark source timestamp is invalid" - ) - change_bps = ( - (current / previous_close - Decimal(1)) * Decimal(10000) - ).quantize(Decimal("0.01"), rounding=ROUND_HALF_UP) - return { - "schema": "qta-market-regime-quote/v1", - "broker": "kis", - "exchange": exchange, - "benchmark_id": contract["benchmark_id"], - "regime_proxy_id": contract["regime_proxy_id"], - "provider_symbol": contract["provider_symbol"], - "current": format(current, "f"), - "previous_close": format(previous_close, "f"), - "change_bps": format(change_bps, "f"), - "source_timestamp": source_timestamp, - "received_at": received_at.isoformat(), - "raw_status": "OK", - } - - def build_cancel_request( - self, - *, - market: str, - broker_order_id: str, - symbol: str, - quantity: str, - price: str, - venue: str, - organization: str = "", - ) -> dict[str, Any]: - market = market.upper() - quantity = decimal_string(quantity, "cancel quantity") - price = decimal_string(price, "cancel price") - if market == "KR": - if venue not in {"KRX", "NXT", "SOR"}: - raise BlockedError("KR venue must be KRX, NXT, or SOR") - tr_id = "TTTC0013U" if self.environment == "live" else "VTTC0013U" - path = "/uapi/domestic-stock/v1/trading/order-rvsecncl" - body = { - "CANO": self.account_prefix, - "ACNT_PRDT_CD": self.account_product, - "KRX_FWDG_ORD_ORGNO": organization, - "ORGN_ODNO": broker_order_id, - "ORD_DVSN": "00", - "RVSE_CNCL_DVSN_CD": "02", - "ORD_QTY": quantity, - "ORD_UNPR": price, - "QTY_ALL_ORD_YN": "Y", - "EXCG_ID_DVSN_CD": venue, - } - elif market == "US": - if venue not in self.US_ORDER_VENUES: - raise BlockedError("US venue must be NASD, NYSE, or AMEX") - tr_id = "TTTT1004U" if self.environment == "live" else "VTTT1004U" - path = "/uapi/overseas-stock/v1/trading/order-rvsecncl" - body = { - "CANO": self.account_prefix, - "ACNT_PRDT_CD": self.account_product, - "OVRS_EXCG_CD": venue, - "PDNO": normalize_symbol("US", symbol), - "ORGN_ODNO": broker_order_id, - "RVSE_CNCL_DVSN_CD": "02", - "ORD_QTY": quantity, - "OVRS_ORD_UNPR": "0", - "MGCO_APTM_ODNO": "", - "ORD_SVR_DVSN_CD": "0", - } - else: - raise BlockedError(f"unsupported market: {market}") - return { - "method": "POST", - "path": path, - "tr_id": tr_id, - "body": body, - "request_hash": sha256_json({"tr_id": tr_id, "body": body}), - } - - def cancel(self, **kwargs: Any) -> dict[str, Any]: - request = self.build_cancel_request(**kwargs) - response = self.transport.request( - "POST", - f"{self.base_url}{request['path']}", - headers=self.headers(request["tr_id"]), - json_body=request["body"], - timeout_seconds=10, - mutation=True, - ) - output = kis_mutation_result(response, "cancel") - operation_order_id = mutation_order_id( - output.get("ODNO"), - "KIS cancel output ODNO", - ) - return { - "original_order_id": kwargs["broker_order_id"], - "operation_order_id": operation_order_id, - "request_hash": request["request_hash"], - "raw": redact(output), - } - - @staticmethod - def _case_value(record: dict[str, Any], *names: str) -> Any: - lowered = {str(key).lower(): value for key, value in record.items()} - for name in names: - if name.lower() in lowered: - return lowered[name.lower()] - return None - - @staticmethod - def _same_order_id(left: Any, right: Any) -> bool: - a = str(left or "").strip() - b = str(right or "").strip() - if not a or not b: - return False - return a == b or a.lstrip("0") == b.lstrip("0") - - def order_history( - self, - *, - market: str, - trading_date: str, - symbol: str, - venue: str, - ) -> list[dict[str, Any]]: - market = market.upper() - compact_date = trading_date.replace("-", "") - if len(compact_date) != 8 or not compact_date.isdigit(): - raise BlockedError("trading_date must be YYYY-MM-DD") - if market == "KR": - tr_id = "TTTC0081R" if self.environment == "live" else "VTTC0081R" - response = self.transport.request( - "GET", - f"{self.base_url}/uapi/domestic-stock/v1/trading/inquire-daily-ccld", - headers=self.headers(tr_id), - query={ - "CANO": self.account_prefix, - "ACNT_PRDT_CD": self.account_product, - "INQR_STRT_DT": compact_date, - "INQR_END_DT": compact_date, - "SLL_BUY_DVSN_CD": "00", - "PDNO": normalize_symbol("KR", symbol), - "CCLD_DVSN": "00", - "INQR_DVSN": "00", - "INQR_DVSN_3": "00", - "ORD_GNO_BRNO": "", - "ODNO": "", - "INQR_DVSN_1": "", - "CTX_AREA_FK100": "", - "CTX_AREA_NK100": "", - "EXCG_ID_DVSN_CD": venue, - }, - timeout_seconds=10, - ) - body = kis_result(response, "KR order history") - records = body.get("output1", []) - elif market == "US": - tr_id = "TTTS3035R" if self.environment == "live" else "VTTS3035R" - response = self.transport.request( - "GET", - f"{self.base_url}/uapi/overseas-stock/v1/trading/inquire-ccnl", - headers=self.headers(tr_id), - query={ - "CANO": self.account_prefix, - "ACNT_PRDT_CD": self.account_product, - "PDNO": "" if self.environment == "paper" else "%", - "ORD_STRT_DT": compact_date, - "ORD_END_DT": compact_date, - "SLL_BUY_DVSN": "00", - "CCLD_NCCS_DVSN": "00", - "OVRS_EXCG_CD": "" if self.environment == "paper" else "NASD", - "SORT_SQN": "DS", - "ORD_DT": "", - "ORD_GNO_BRNO": "", - "ODNO": "", - "CTX_AREA_NK200": "", - "CTX_AREA_FK200": "", - }, - timeout_seconds=10, - ) - body = kis_result(response, "US order history") - records = body.get("output", []) - else: - raise BlockedError(f"unsupported KIS history market: {market}") - if isinstance(records, dict): - records = [records] - if not isinstance(records, list) or any( - not isinstance(item, dict) for item in records - ): - raise BlockedError("KIS order history output must be an object array") - return records - - def get_order( - self, - *, - market: str, - trading_date: str, - symbol: str, - venue: str, - broker_order_id: str, - ) -> dict[str, Any]: - records = self.order_history( - market=market, - trading_date=trading_date, - symbol=symbol, - venue=venue, - ) - matches = [ - record - for record in records - if self._same_order_id( - self._case_value(record, "ODNO", "odno"), broker_order_id - ) - ] - if len(matches) != 1: - raise BlockedError( - f"KIS order {broker_order_id} match count is {len(matches)}" - ) - record = matches[0] - ordered = self._case_value( - record, "ORD_QTY", "ord_qty", "FT_ORD_QTY", "ft_ord_qty" - ) - filled = self._case_value( - record, - "TOT_CCLD_QTY", - "tot_ccld_qty", - "FT_CCLD_QTY", - "ft_ccld_qty", - ) - average_fill_price = self._case_value( - record, - "AVG_PRVS", - "avg_prvs", - "FT_CCLD_UNPR3", - "ft_ccld_unpr3", - ) - remaining = self._case_value( - record, "RMN_QTY", "rmn_qty", "NCCS_QTY", "nccs_qty" - ) - ordered_decimal = ( - decimal_value(ordered, "ordered quantity") - if ordered not in (None, "") - else None - ) - filled_decimal = ( - decimal_value(filled, "filled quantity") - if filled not in (None, "") - else None - ) - remaining_decimal = ( - decimal_value(remaining, "remaining quantity") - if remaining not in (None, "") - else None - ) - average_fill_decimal = ( - decimal_value(average_fill_price, "average fill price") - if average_fill_price not in (None, "") - else None - ) - if average_fill_decimal is not None and average_fill_decimal <= 0: - if filled_decimal is not None and filled_decimal > 0: - raise BlockedError( - "KIS filled order has a nonpositive average fill price" - ) - average_fill_decimal = None - cancel_value = str( - self._case_value(record, "CNCL_YN", "cncl_yn", "CANCEL_YN") or "" - ).upper() - if ( - ordered_decimal is not None - and filled_decimal is not None - and ordered_decimal > 0 - and filled_decimal >= ordered_decimal - ): - normalized_status = "FILLED" - elif ( - filled_decimal is not None - and filled_decimal > 0 - and (remaining_decimal is None or remaining_decimal > 0) - ): - normalized_status = "PARTIALLY_FILLED" - elif cancel_value == "Y": - normalized_status = "CANCELLED" - elif remaining_decimal is not None and remaining_decimal > 0: - normalized_status = "ACKNOWLEDGED" - else: - normalized_status = "UNKNOWN" - return { - "broker_order_id": broker_order_id, - "normalized_status": normalized_status, - "ordered_quantity": None - if ordered_decimal is None - else format(ordered_decimal, "f"), - "filled_quantity": None - if filled_decimal is None - else format(filled_decimal, "f"), - "average_fill_price": None - if average_fill_decimal is None - else format(average_fill_decimal, "f"), - "remaining_quantity": None - if remaining_decimal is None - else format(remaining_decimal, "f"), - "raw": record, - } - - -def self_test() -> None: - intent = { - "intent_id": "1" * 32, - "client_order_id": "qta-" + "1" * 28, - "market": "US", - "symbol": "AAPL", - "side": "BUY", - "order_type": "LIMIT", - "time_in_force": "DAY", - "quantity": "1", - "limit_price": "200.00", - "stop_price": "180.00", - "take_profit_price": "240.00", - } - now = datetime.now(timezone.utc) - fake_clock = [0.0] - - def fake_monotonic() -> float: - return fake_clock[0] - - def fake_sleep(seconds: float) -> None: - fake_clock[0] += seconds - - market_data_limiter = MinimumIntervalRateLimiter( - 10, - monotonic=fake_monotonic, - sleeper=fake_sleep, - ) - toss_transport = QueueTransport( - [ - HttpResponse( - 200, - {}, - { - "result": { - "orderId": "toss-order-1", - "clientOrderId": intent["client_order_id"], - } - }, - ), - HttpResponse( - 200, - {"X-RateLimit-Limit": "10"}, - { - "result": [ - { - "symbol": "AAPL", - "timestamp": now.isoformat(), - "lastPrice": "200.50", - "currency": "USD", - } - ] - }, - ), - HttpResponse( - 200, - {"x-ratelimit-limit": "5"}, - { - "result": { - "timestamp": now.isoformat(), - "asks": [ - {"price": "202.00"}, - {"price": "0"}, - {"price": "201.00"}, - ], - "bids": [ - {"price": "199.00"}, - {"price": "0"}, - {"price": "200.00"}, - ], - } - }, - ), - ] - ) - toss = TossBroker( - client_id="test-client", - client_secret="test-secret", - account_seq=1, - transport=toss_transport, - access_token="test-token", - access_token_expires_at=now + timedelta(minutes=5), - market_data_limiter=market_data_limiter, - ) - preview = toss.preview_submit(intent) - assert preview["body"]["quantity"] == "1" - assert preview["body"]["price"] == "200.00" - assert preview["request_hash"] == toss.mutation_request_hash( - "POST", - "/api/v1/orders", - preview["body"], - ) - account_two = TossBroker( - client_id="test-client", - client_secret="test-secret", - account_seq=2, - transport=QueueTransport([]), - ) - assert account_two.preview_submit(intent)["request_hash"] != preview["request_hash"] - assert ( - toss.mutation_request_hash("PUT", "/api/v1/orders", preview["body"]) - != preview["request_hash"] - ) - assert ( - toss.mutation_request_hash("POST", "/api/v1/orders/other", preview["body"]) - != preview["request_hash"] - ) - changed_body = {**preview["body"], "price": "201.00"} - assert ( - toss.mutation_request_hash("POST", "/api/v1/orders", changed_body) - != preview["request_hash"] - ) - ack = toss.submit(intent) - assert ack["broker_order_id"] == "toss-order-1" - assert ack["request_hash"] == preview["request_hash"] - assert ack["submit_started_at"] - assert ack["ack_received_at"] - quote = toss.quote("US", "AAPL") - assert quote["best_ask"] == "201.00" - assert quote["best_bid"] == "200.00" - assert fake_clock[0] == 0.1 - assert market_data_limiter.minimum_interval_seconds == 0.2 - assert toss.capabilities()["client_order_id_ttl_seconds"] == 600 - assert toss.capabilities()["client_rate_limits_tps"]["ORDER"] == 3 - - try: - TossBroker( - client_id="test-client", - client_secret="test-secret", - account_seq=1, - access_token="test-token", - ) - except BlockedError: - pass - else: - raise AssertionError("Toss external token without expiration must be blocked") - - def toss_with_response(response: HttpResponse) -> TossBroker: - return TossBroker( - client_id="test-client", - client_secret="test-secret", - account_seq=1, - transport=QueueTransport([response]), - access_token="test-token", - access_token_expires_at=now + timedelta(minutes=5), - ) - - for ambiguous_response in ( - HttpResponse(503, {}, None), - HttpResponse(200, {}, {"result": {}}), - ): - try: - toss_with_response(ambiguous_response).submit(intent) - except AmbiguousMutationError: - pass - else: - raise AssertionError("Toss 5xx/missing-order-id mutation must be ambiguous") - try: - toss_with_response( - HttpResponse( - 400, - {}, - {"error": {"code": "INVALID_ORDER", "message": "rejected"}}, - ) - ).submit(intent) - except AuthoritativeMutationRejection: - pass - else: - raise AssertionError("explicit Toss rejection must be authoritative") - try: - toss_with_response(HttpResponse(200, {}, {"result": {}})).cancel( - "existing-order" - ) - except AmbiguousMutationError: - pass - else: - raise AssertionError("Toss cancel missing operation ID must be ambiguous") - malformed_toss_quote = TossBroker( - client_id="test-client", - client_secret="test-secret", - account_seq=1, - transport=QueueTransport( - [ - HttpResponse( - 200, - {}, - { - "result": [ - { - "symbol": "AAPL", - "timestamp": now.isoformat(), - "currency": "USD", - } - ] - }, - ), - HttpResponse( - 200, - {}, - { - "result": { - "timestamp": now.isoformat(), - "asks": [{"price": "201"}], - "bids": [{"price": "200"}], - } - }, - ), - ] - ), - access_token="test-token", - access_token_expires_at=now + timedelta(minutes=5), - ) - try: - malformed_toss_quote.quote("US", "AAPL") - except BlockedError as exc: - assert "lastPrice must be" in str(exc) - else: - raise AssertionError("missing Toss lastPrice must fail as BlockedError") - - kis_transport = QueueTransport( - [ - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "msg_cd": "0", - "msg1": "OK", - "output": {"ODNO": "12345", "ORD_TMD": "093001"}, - }, - ) - ] - ) - kis = KisBroker( - app_key="test-key", - app_secret="test-secret", - account_prefix="12345678", - account_product="01", - environment="paper", - transport=kis_transport, - access_token="test-token", - ) - kis_preview = kis.preview_submit(intent, venue="NASD") - assert kis_preview["tr_id"] == "VTTT1002U" - assert kis_preview["body"]["CANO"] == "[REDACTED]" - assert kis_preview["body"]["ACNT_PRDT_CD"] == "[REDACTED]" - kis_ack = kis.submit(intent, venue="NASD") - assert kis_ack["broker_order_id"] == "12345" - assert kis_ack["submit_started_at"] - assert kis_ack["ack_received_at"] - - def kis_with_response(response: HttpResponse) -> KisBroker: - return KisBroker( - app_key="test-key", - app_secret="test-secret", - account_prefix="12345678", - account_product="01", - environment="paper", - transport=QueueTransport([response]), - access_token="test-token", - ) - - for ambiguous_response in ( - HttpResponse(503, {}, {"rt_cd": "1", "msg_cd": "SERVER_ERROR"}), - HttpResponse( - 200, - {}, - {"rt_cd": "0", "msg_cd": "0", "msg1": "OK", "output": {}}, - ), - ): - try: - kis_with_response(ambiguous_response).submit(intent, venue="NASD") - except AmbiguousMutationError: - pass - else: - raise AssertionError("KIS 5xx/missing-order-id mutation must be ambiguous") - try: - kis_with_response( - HttpResponse( - 200, - {}, - { - "rt_cd": "1", - "msg_cd": "ORDER_REJECTED", - "msg1": "rejected", - }, - ) - ).submit(intent, venue="NASD") - except AuthoritativeMutationRejection: - pass - else: - raise AssertionError("explicit KIS rejection must be authoritative") - try: - kis_with_response( - HttpResponse( - 200, - {}, - {"rt_cd": "0", "msg_cd": "0", "msg1": "OK", "output": {}}, - ) - ).cancel( - market="US", - broker_order_id="12345", - symbol="AAPL", - quantity="1", - price="200", - venue="NASD", - ) - except AmbiguousMutationError: - pass - else: - raise AssertionError("KIS cancel missing operation ID must be ambiguous") - - deadline_delegate = QueueTransport( - [ - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "msg_cd": "0", - "msg1": "OK", - "output": {"ODNO": "late"}, - }, - ) - ] - ) - paced = PacedTransport(deadline_delegate, minimum_interval_seconds=10) - paced.last_started = time.monotonic() - try: - paced.request( - "POST", - "https://example.invalid/order", - deadline_at=datetime.now(timezone.utc) + timedelta(seconds=1), - ) - except BlockedError as exc: - assert "deadline" in str(exc) - else: - raise AssertionError("pacing past the submit deadline must be blocked") - assert deadline_delegate.requests == [] - - sell_intent = {**intent, "side": "SELL"} - assert kis.build_order_request(sell_intent, venue="NASD")["tr_id"] == "VTTT1001U" - kr_intent = { - **intent, - "market": "KR", - "symbol": "5930", - "limit_price": "70000", - } - kr_request = kis.build_order_request(kr_intent, venue="KRX") - assert kr_request["tr_id"] == "VTTC0012U" - assert kr_request["body"]["PDNO"] == "005930" - assert kis.kr_quote_venue("KRX") == "J" - try: - kis.kr_quote_venue("NXT") - except BlockedError: - pass - else: - raise AssertionError("KIS paper NXT quote must be blocked") - - live_kis = KisBroker( - app_key="test-key", - app_secret="test-secret", - account_prefix="12345678", - account_product="01", - environment="live", - transport=QueueTransport([]), - access_token="test-token", - ) - assert live_kis.kr_quote_venue("KRX") == "J" - assert live_kis.kr_quote_venue("NXT") == "NX" - assert live_kis.kr_quote_venue("SOR") == "UN" - - quote_transport = QueueTransport( - [ - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "msg_cd": "0", - "msg1": "OK", - "output": { - "stck_prpr": "70000", - "stck_bsop_date": "20260727", - "stck_cntg_hour": "093000", - }, - }, - ), - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "msg_cd": "0", - "msg1": "OK", - "output1": { - "askp1": "70100", - "bidp1": "69900", - "stck_bsop_date": "20260727", - "aspr_acpt_hour": "093001", - }, - "output2": {"stck_prpr": "70000"}, - }, - ), - ] - ) - quote_kis = KisBroker( - app_key="test-key", - app_secret="test-secret", - account_prefix="12345678", - account_product="01", - environment="live", - transport=quote_transport, - access_token="test-token", - ) - kr_quote = quote_kis.quote("KR", "005930", venue="NXT") - assert kr_quote["trade_timestamp"].endswith("T09:30:00+09:00") - assert kr_quote["book_timestamp"].endswith("T09:30:01+09:00") - assert kr_quote["trade_timestamp"] != kr_quote["book_timestamp"] - assert all( - request["query"]["FID_COND_MRKT_DIV_CODE"] == "NX" - for request in quote_transport.requests - ) - - malformed_kr_quote = KisBroker( - app_key="test-key", - app_secret="test-secret", - account_prefix="12345678", - account_product="01", - environment="live", - transport=QueueTransport( - [ - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "msg_cd": "0", - "msg1": "OK", - "output": { - "stck_prpr": "70000", - "stck_bsop_date": "20260727", - "stck_cntg_hour": "093000", - }, - }, - ), - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "msg_cd": "0", - "msg1": "OK", - "output1": { - "bidp1": "69900", - "stck_bsop_date": "20260727", - "aspr_acpt_hour": "093001", - }, - }, - ), - ] - ), - access_token="test-token", - ) - try: - malformed_kr_quote.quote("KR", "005930", venue="KRX") - except BlockedError as exc: - assert "askp1 must be" in str(exc) - else: - raise AssertionError("missing KIS askp1 must fail as BlockedError") - - undated_quote_kis = KisBroker( - app_key="test-key", - app_secret="test-secret", - account_prefix="12345678", - account_product="01", - environment="live", - transport=QueueTransport( - [ - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "msg_cd": "0", - "msg1": "OK", - "output": { - "stck_prpr": "70000", - "stck_cntg_hour": "093000", - }, - }, - ), - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "msg_cd": "0", - "msg1": "OK", - "output1": { - "askp1": "70100", - "bidp1": "69900", - "stck_bsop_date": "20260727", - "aspr_acpt_hour": "093001", - }, - "output2": {"stck_prpr": "70000"}, - }, - ), - ] - ), - access_token="test-token", - ) - try: - undated_quote_kis.quote("KR", "005930", venue="KRX") - except BlockedError as exc: - assert "each require a valid" in str(exc) - else: - raise AssertionError("KIS KR quote must not synthesize a trading date") - - single_timestamp_kis = KisBroker( - app_key="test-key", - app_secret="test-secret", - account_prefix="12345678", - account_product="01", - environment="live", - transport=QueueTransport( - [ - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "msg_cd": "0", - "msg1": "OK", - "output": { - "stck_prpr": "70000", - "stck_bsop_date": "20260727", - }, - }, - ), - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "msg_cd": "0", - "msg1": "OK", - "output1": { - "askp1": "70100", - "bidp1": "69900", - "stck_bsop_date": "20260727", - "aspr_acpt_hour": "093001", - }, - "output2": {"stck_prpr": "70000"}, - }, - ), - ] - ), - access_token="test-token", - ) - try: - single_timestamp_kis.quote("KR", "005930", venue="KRX") - except BlockedError as exc: - assert "independent broker-supplied" in str(exc) - else: - raise AssertionError("KIS KR price and book timestamps must be independent") - - us_cancel = kis.build_cancel_request( - market="US", - broker_order_id="12345", - symbol="AAPL", - quantity="1", - price="200.00", - venue="NASD", - ) - assert us_cancel["tr_id"] == "VTTT1004U" - assert us_cancel["body"]["PDNO"] == "AAPL" - assert us_cancel["body"]["OVRS_ORD_UNPR"] == "0" - assert us_cancel["body"]["ORD_SVR_DVSN_CD"] == "0" - - assert ( - kis_market_timestamp( - date_value="20260727", - time_value="093001", - timezone_name="America/New_York", - ) - == "2026-07-27T09:30:01-04:00" - ) - assert ( - kis_market_timestamp( - date_value="", - time_value="093001", - timezone_name="America/New_York", - ) - is None - ) - - for venue, expected_exchange_code, symbol in ( - ("NASD", "NAS", "AAPL"), - ("NYSE", "NYS", "IBM"), - ): - us_quote_transport = QueueTransport( - [ - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "msg_cd": "0", - "msg1": "OK", - "output1": { - "last": "200.25", - "pask1": "200.30", - "pbid1": "200.20", - "dymd": "20260727", - "dhms": "093001", - }, - }, - ) - ] - ) - us_quote_kis = KisBroker( - app_key="test-key", - app_secret="test-secret", - account_prefix="12345678", - account_product="01", - environment="live", - transport=us_quote_transport, - access_token="test-token", - ) - us_quote = us_quote_kis.quote("US", symbol, venue=venue) - assert us_quote["last_price"] == "200.25" - assert us_quote["trade_timestamp"] == us_quote["book_timestamp"] - assert us_quote["trade_timestamp"].endswith("T09:30:01-04:00") - assert len(us_quote_transport.requests) == 1 - assert us_quote_transport.requests[0]["query"]["EXCD"] == expected_exchange_code - assert us_quote_transport.requests[0]["url"].endswith( - "/uapi/overseas-price/v1/quotations/inquire-asking-price" - ) - - no_last_transport = QueueTransport( - [ - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "msg_cd": "0", - "msg1": "OK", - "output1": { - "pask1": "200.30", - "pbid1": "200.20", - "dymd": "20260727", - "dhms": "093001", - }, - }, - ), - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "msg_cd": "0", - "msg1": "OK", - "output": {"last": "199.00"}, - }, - ), - ] - ) - no_last_kis = KisBroker( - app_key="test-key", - app_secret="test-secret", - account_prefix="12345678", - account_product="01", - environment="live", - transport=no_last_transport, - access_token="test-token", - ) - try: - no_last_kis.quote("US", "AAPL", venue="NASD") - except BlockedError as exc: - assert "same-source last/bid/ask" in str(exc) - else: - raise AssertionError("KIS US quote must not borrow last from another endpoint") - assert len(no_last_transport.requests) == 1 - assert len(no_last_transport.responses) == 1 - - try: - kis.quote("US", "AAPL", venue="NASD") - except BlockedError as exc: - assert "paper order book" in str(exc) - else: - raise AssertionError("KIS U.S. paper quote must be blocked") - - ambiguous = QueueTransport([AmbiguousMutationError("accepted then timeout")]) - ambiguous_broker = KisBroker( - app_key="test-key", - app_secret="test-secret", - account_prefix="12345678", - account_product="01", - environment="paper", - transport=ambiguous, - access_token="test-token", - ) - try: - ambiguous_broker.submit(intent, venue="NASD") - except AmbiguousMutationError: - pass - else: - raise AssertionError("ambiguous KIS mutation did not raise") - - print( - canonical_json( - { - "self_test": "PASS", - "toss_request_hash": preview["request_hash"], - "kis_request_hash": kis_preview["request_hash"], - } - ) - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--self-test", action="store_true") - parser.add_argument("--capabilities", choices=("toss", "kis-paper", "kis-live")) - return parser.parse_args() - - -def main() -> int: - args = parse_args() - if args.self_test: - self_test() - return 0 - if args.capabilities == "toss": - print(canonical_json(TossBroker.capabilities())) - return 0 - if args.capabilities == "kis-paper": - print(canonical_json(KisBroker.capabilities("paper"))) - return 0 - if args.capabilities == "kis-live": - print(canonical_json(KisBroker.capabilities("live"))) - return 0 - print( - canonical_json( - { - "status": "BLOCKED", - "reason": "use --self-test or --capabilities", - } - ) - ) - return 2 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/skills/quant-stock-polling-trader/scripts/broker_credentials.py b/skills/quant-stock-polling-trader/scripts/broker_credentials.py deleted file mode 100644 index 4a7eef0..0000000 --- a/skills/quant-stock-polling-trader/scripts/broker_credentials.py +++ /dev/null @@ -1,469 +0,0 @@ -#!/usr/bin/env python3 -"""Load and configure KIS credentials without exposing their values.""" - -from __future__ import annotations - -import argparse -import getpass -import json -import os -import re -import stat -import sys -import tempfile -from datetime import datetime, timezone -from pathlib import Path -from typing import Any -from unittest.mock import patch - -from broker_adapters import KisBroker, TransportFailure -from execution_core import BlockedError, canonical_json - -STATUS_SCHEMA = "qta-broker-credential-status/v1" -DEFAULT_SECRETS_PATH = Path("~/.config/mrcha-skills/secrets.env").expanduser() -SECRETS_PATH_ENV = "QTA_SECRETS_FILE" -FIELD_SUFFIXES = { - "app_key": "APP_KEY", - "app_secret": "APP_SECRET", - "account_prefix": "ACCOUNT_PREFIX", - "account_product": "ACCOUNT_PRODUCT", -} -GENERIC_NAMES = { - field: f"QTA_KIS_{suffix}" for field, suffix in FIELD_SUFFIXES.items() -} -DOTENV_KEY = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") -_INJECTED_ENVIRONMENT: dict[str, str] = {} - - -def credential_scope(environment: str) -> str: - normalized = environment.strip().lower() - if normalized == "paper": - return "PAPER" - if normalized in {"shadow", "live"}: - return "LIVE" - raise BlockedError("KIS credential environment must be paper, shadow, or live") - - -def resolve_secrets_path(path: str | Path | None = None) -> Path: - candidate = path if path is not None else os.environ.get(SECRETS_PATH_ENV) - return Path(candidate).expanduser() if candidate else DEFAULT_SECRETS_PATH - - -def inspect_secrets_file(path: Path) -> dict[str, Any]: - try: - metadata = path.lstat() - except FileNotFoundError: - return { - "path": str(path), - "exists": False, - "secure": None, - "mode": None, - } - if stat.S_ISLNK(metadata.st_mode): - raise BlockedError(f"credential file must not be a symlink: {path}") - if not stat.S_ISREG(metadata.st_mode): - raise BlockedError(f"credential path must be a regular file: {path}") - if metadata.st_uid != os.getuid(): - raise BlockedError(f"credential file must be owned by the current user: {path}") - mode = stat.S_IMODE(metadata.st_mode) - if mode != 0o600: - raise BlockedError( - f"credential file permissions must be 0600, found {mode:04o}: {path}" - ) - return { - "path": str(path), - "exists": True, - "secure": True, - "mode": "0600", - } - - -def parse_dotenv_value(raw: str, *, line_number: int) -> str: - value = raw.strip() - if not value: - return "" - if value.startswith('"'): - try: - parsed = json.loads(value) - except json.JSONDecodeError as exc: - raise BlockedError( - f"invalid double-quoted dotenv value on line {line_number}" - ) from exc - if not isinstance(parsed, str): - raise BlockedError(f"dotenv value on line {line_number} must be a string") - return parsed - if value.startswith("'"): - if len(value) < 2 or not value.endswith("'"): - raise BlockedError( - f"invalid single-quoted dotenv value on line {line_number}" - ) - return value[1:-1].replace("\\'", "'").replace("\\\\", "\\") - return value - - -def parse_dotenv(path: Path) -> dict[str, str]: - inspection = inspect_secrets_file(path) - if not inspection["exists"]: - return {} - values: dict[str, str] = {} - for line_number, raw_line in enumerate( - path.read_text(encoding="utf-8").splitlines(), start=1 - ): - stripped = raw_line.strip() - if not stripped or stripped.startswith("#"): - continue - if stripped.startswith("export "): - stripped = stripped[7:].lstrip() - if "=" not in stripped: - raise BlockedError(f"invalid dotenv assignment on line {line_number}") - key, raw_value = stripped.split("=", 1) - key = key.strip() - if not DOTENV_KEY.fullmatch(key): - raise BlockedError(f"invalid dotenv key on line {line_number}") - if key in values: - raise BlockedError(f"duplicate dotenv key on line {line_number}: {key}") - values[key] = parse_dotenv_value(raw_value, line_number=line_number) - return values - - -def scoped_name(scope: str, field: str) -> str: - return f"QTA_KIS_{scope}_{FIELD_SUFFIXES[field]}" - - -def choose_value( - *, - generic_name: str, - scoped_variable: str, - dotenv_values: dict[str, str], -) -> tuple[str | None, str | None]: - candidates = ( - (os.environ.get(generic_name), f"environment:{generic_name}"), - (os.environ.get(scoped_variable), f"environment:{scoped_variable}"), - (dotenv_values.get(scoped_variable), f"dotenv:{scoped_variable}"), - (dotenv_values.get(generic_name), f"dotenv:{generic_name}"), - ) - for value, source in candidates: - if value: - return value, source - return None, None - - -def clear_injected_environment() -> None: - for name, injected_value in tuple(_INJECTED_ENVIRONMENT.items()): - if os.environ.get(name) == injected_value: - os.environ.pop(name, None) - _INJECTED_ENVIRONMENT.pop(name, None) - - -def load_kis_credentials( - environment: str, - *, - secrets_path: str | Path | None = None, -) -> dict[str, Any]: - clear_injected_environment() - scope = credential_scope(environment) - path = resolve_secrets_path(secrets_path) - inspection = inspect_secrets_file(path) - dotenv_values = parse_dotenv(path) if inspection["exists"] else {} - sources: dict[str, str] = {} - present: list[str] = [] - missing: list[str] = [] - - for field, generic_name in GENERIC_NAMES.items(): - value, source = choose_value( - generic_name=generic_name, - scoped_variable=scoped_name(scope, field), - dotenv_values=dotenv_values, - ) - if value is None or source is None: - missing.append(generic_name) - continue - os.environ[generic_name] = value - if source != f"environment:{generic_name}": - _INJECTED_ENVIRONMENT[generic_name] = value - present.append(generic_name) - sources[generic_name] = source - - return { - "schema": STATUS_SCHEMA, - "broker": "kis", - "credential_scope": scope.lower(), - "status": "READY" if not missing else "BLOCKED", - "secrets_file": inspection, - "present": sorted(present), - "missing": sorted(missing), - "sources": dict(sorted(sources.items())), - } - - -def require_kis_runtime_credentials(report: dict[str, Any]) -> None: - if report["missing"]: - raise BlockedError( - "missing KIS runtime credentials: " + ", ".join(report["missing"]) - ) - - -def dotenv_quote(value: str) -> str: - return json.dumps(value, ensure_ascii=False) - - -def assignment_key(raw_line: str) -> str | None: - stripped = raw_line.strip() - if not stripped or stripped.startswith("#"): - return None - if stripped.startswith("export "): - stripped = stripped[7:].lstrip() - if "=" not in stripped: - return None - key = stripped.split("=", 1)[0].strip() - return key if DOTENV_KEY.fullmatch(key) else None - - -def write_dotenv_updates(path: Path, updates: dict[str, str]) -> None: - inspection = inspect_secrets_file(path) - existing_lines = ( - path.read_text(encoding="utf-8").splitlines() if inspection["exists"] else [] - ) - output_lines: list[str] = [] - replaced: set[str] = set() - for raw_line in existing_lines: - key = assignment_key(raw_line) - if key not in updates: - output_lines.append(raw_line) - continue - if key not in replaced: - output_lines.append(f"{key}={dotenv_quote(updates[key])}") - replaced.add(key) - if output_lines and output_lines[-1] != "": - output_lines.append("") - for key in updates: - if key not in replaced: - output_lines.append(f"{key}={dotenv_quote(updates[key])}") - - parent_existed = path.parent.exists() - path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) - if not parent_existed: - path.parent.chmod(0o700) - file_descriptor, temporary_name = tempfile.mkstemp( - prefix=".secrets-", dir=path.parent - ) - temporary_path = Path(temporary_name) - try: - os.fchmod(file_descriptor, 0o600) - with os.fdopen(file_descriptor, "w", encoding="utf-8") as handle: - handle.write("\n".join(output_lines) + "\n") - handle.flush() - os.fsync(handle.fileno()) - os.replace(temporary_path, path) - path.chmod(0o600) - except BaseException: - temporary_path.unlink(missing_ok=True) - raise - - -def prompt_secret(label: str, *, existing: str | None = None) -> str: - suffix = " (Enter keeps the stored value)" if existing else "" - value = getpass.getpass(f"{label}{suffix}: ") - if value: - return value - if existing: - return existing - raise BlockedError(f"{label} is required") - - -def configure_kis(environment: str, path: Path) -> dict[str, Any]: - scope = credential_scope(environment) - inspection = inspect_secrets_file(path) - existing = parse_dotenv(path) if inspection["exists"] else {} - updates: dict[str, str] = {} - labels = { - "app_key": "KIS app key", - "app_secret": "KIS app secret", - "account_prefix": "KIS account number (first 8 digits)", - } - for field in ("app_key", "app_secret", "account_prefix"): - name = scoped_name(scope, field) - updates[name] = prompt_secret(labels[field], existing=existing.get(name)) - - product_name = scoped_name(scope, "account_product") - stored_product = existing.get(product_name) - product = getpass.getpass( - "KIS account product (2 digits" - + (", Enter keeps the stored value" if stored_product else ", Enter uses 01") - + "): " - ) - updates[product_name] = product or stored_product or "01" - if len(updates[scoped_name(scope, "account_prefix")]) != 8 or not updates[ - scoped_name(scope, "account_prefix") - ].isdigit(): - raise BlockedError("KIS account number prefix must be 8 digits") - if len(updates[product_name]) != 2 or not updates[product_name].isdigit(): - raise BlockedError("KIS account product must be 2 digits") - write_dotenv_updates(path, updates) - report = load_kis_credentials(environment, secrets_path=path) - require_kis_runtime_credentials(report) - return { - **report, - "configured": True, - "values_printed": False, - } - - -def auth_check(environment: str, path: Path) -> dict[str, Any]: - report = load_kis_credentials(environment, secrets_path=path) - require_kis_runtime_credentials(report) - broker = KisBroker( - app_key=os.environ["QTA_KIS_APP_KEY"], - app_secret=os.environ["QTA_KIS_APP_SECRET"], - account_prefix=os.environ["QTA_KIS_ACCOUNT_PREFIX"], - account_product=os.environ["QTA_KIS_ACCOUNT_PRODUCT"], - environment="paper" if credential_scope(environment) == "PAPER" else "live", - access_token=None, - ) - token = broker.token() - if not token: - raise BlockedError("KIS authentication returned an empty access token") - return { - "schema": STATUS_SCHEMA, - "broker": "kis", - "credential_scope": report["credential_scope"], - "status": "AUTHENTICATED", - "checked_at": datetime.now(timezone.utc).isoformat(), - "token_received": True, - "token_persisted": False, - "secrets_file": report["secrets_file"], - "sources": report["sources"], - } - - -def self_test() -> None: - with tempfile.TemporaryDirectory(prefix="qta-credentials-") as directory: - path = Path(directory) / "secrets.env" - path.write_text( - "\n".join( - ( - 'UNRELATED_VALUE="preserved"', - 'QTA_KIS_PAPER_APP_KEY="fixture-app"', - 'QTA_KIS_PAPER_APP_SECRET="fixture-secret"', - 'QTA_KIS_PAPER_ACCOUNT_PREFIX="00000000"', - 'QTA_KIS_PAPER_ACCOUNT_PRODUCT="01"', - 'QTA_KIS_LIVE_APP_KEY="fixture-live-app"', - 'QTA_KIS_LIVE_APP_SECRET="fixture-live-secret"', - 'QTA_KIS_LIVE_ACCOUNT_PREFIX="11111111"', - 'QTA_KIS_LIVE_ACCOUNT_PRODUCT="01"', - "", - ) - ), - encoding="utf-8", - ) - path.chmod(0o600) - with patch.dict(os.environ, {}, clear=True): - report = load_kis_credentials("paper", secrets_path=path) - assert report["status"] == "READY" - assert os.environ["QTA_KIS_APP_KEY"] == "fixture-app" - rendered = canonical_json(report) - assert "fixture-app" not in rendered - assert "fixture-secret" not in rendered - live_report = load_kis_credentials("live", secrets_path=path) - assert live_report["status"] == "READY" - assert os.environ["QTA_KIS_APP_KEY"] == "fixture-live-app" - assert os.environ["QTA_KIS_ACCOUNT_PREFIX"] == "11111111" - with patch.dict( - os.environ, - {"QTA_KIS_APP_KEY": "environment-app"}, - clear=True, - ): - report = load_kis_credentials("paper", secrets_path=path) - assert report["sources"]["QTA_KIS_APP_KEY"].startswith("environment:") - assert os.environ["QTA_KIS_APP_KEY"] == "environment-app" - write_dotenv_updates( - path, - {"QTA_KIS_PAPER_ACCOUNT_PRODUCT": "02"}, - ) - assert parse_dotenv(path)["UNRELATED_VALUE"] == "preserved" - assert parse_dotenv(path)["QTA_KIS_PAPER_ACCOUNT_PRODUCT"] == "02" - path.chmod(0o644) - try: - load_kis_credentials("paper", secrets_path=path) - except BlockedError as exc: - assert "0600" in str(exc) - else: - raise AssertionError("insecure credential file permissions must be blocked") - - configured_path = Path(directory) / "configured.env" - with ( - patch.dict(os.environ, {}, clear=True), - patch.object( - getpass, - "getpass", - side_effect=("new-app", "new-secret", "22222222", "01"), - ), - ): - configured = configure_kis("paper", configured_path) - assert configured["status"] == "READY" - assert configured_path.stat().st_mode & 0o777 == 0o600 - configured_rendered = canonical_json(configured) - assert "new-app" not in configured_rendered - assert "new-secret" not in configured_rendered - with patch.object(KisBroker, "token", return_value="sensitive-token"): - authenticated = auth_check("paper", configured_path) - authenticated_rendered = canonical_json(authenticated) - assert authenticated["status"] == "AUTHENTICATED" - assert "sensitive-token" not in authenticated_rendered - print(canonical_json({"self_test": "PASS", "secrets_exposed": False})) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - subparsers = parser.add_subparsers(dest="command") - for command in ("status", "configure", "auth-check"): - subparser = subparsers.add_parser(command) - subparser.add_argument( - "--environment", - required=True, - choices=("paper", "shadow", "live"), - ) - subparser.add_argument("--secrets-file") - subparsers.add_parser("self-test") - return parser.parse_args() - - -def main() -> int: - args = parse_args() - if args.command == "self-test": - self_test() - return 0 - if args.command not in {"status", "configure", "auth-check"}: - print( - canonical_json( - { - "schema": STATUS_SCHEMA, - "status": "BLOCKED", - "reason": "choose status, configure, auth-check, or self-test", - } - ) - ) - return 2 - path = resolve_secrets_path(args.secrets_file) - try: - if args.command == "configure": - output = configure_kis(args.environment, path) - elif args.command == "auth-check": - output = auth_check(args.environment, path) - else: - output = load_kis_credentials(args.environment, secrets_path=path) - except (BlockedError, OSError, TransportFailure, ValueError) as exc: - output = { - "schema": STATUS_SCHEMA, - "broker": "kis", - "status": "BLOCKED", - "reason": str(exc), - } - print(canonical_json(output)) - return 2 - print(canonical_json(output)) - return 0 if output["status"] in {"READY", "AUTHENTICATED"} else 2 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/skills/quant-stock-polling-trader/scripts/daily_pipeline.py b/skills/quant-stock-polling-trader/scripts/daily_pipeline.py deleted file mode 100644 index 3794470..0000000 --- a/skills/quant-stock-polling-trader/scripts/daily_pipeline.py +++ /dev/null @@ -1,1565 +0,0 @@ -#!/usr/bin/env python3 -"""Run the deterministic daily pre-open, snapshot, and shadow-entry pipeline.""" - -from __future__ import annotations - -import argparse -import json -import os -import shutil -import sqlite3 -import stat -import subprocess -import sys -import tempfile -import time -from datetime import date, datetime, timedelta, timezone -from decimal import Decimal -from pathlib import Path -from typing import Any -from zoneinfo import ZoneInfo - -import market_calendar -from execution_core import ( - ENTRY_WINDOWS, - TERMINAL_STATES, - BlockedError, - normalized_execution_policy, - normalized_risk_policy, - sha256_file, -) - -CONFIG_SCHEMA = "qta-daily-shadow-config/v2" -STATUS_SCHEMA = "qta-daily-shadow-status/v2" -DESCRIPTOR_SCHEMA = "qta-daily-shadow-descriptor/v2" -REQUIRED_CONFIG_FIELDS = { - "schema", - "runtime_root", - "broker", - "mode", - "history_start_date", - "minimum_sessions", - "request_interval_ms", - "catalog_coverage_contract", - "selector", - "risk_policy_path", - "execution_policy", - "account_aliases", - "prepare_complete_seconds_before_open", -} -OPTIONAL_CONFIG_FIELDS = { - "exposure_component_paths", - "exposure_component_max_age_seconds", - "qta_method_version", -} -EXECUTION_DEFAULT_FIELDS = { - "snapshot_max_age_seconds", - "poll_interval_seconds", - "quote_max_age_seconds", - "max_spread_bps", - "max_gap_bps", - "trigger_mode", - "order_ttl_seconds", - "order_type", - "time_in_force", - "allow_partial_fill", - "cancel_remainder_at_window_end", -} -TERMINAL_OR_MANUAL = TERMINAL_STATES -RECONCILE_ATTEMPTS = 20 -RECONCILE_INTERVAL_SECONDS = 15 - - -class PipelineBlockedError(BlockedError): - """Raised when a daily gate cannot be completed without guessing.""" - - -def atomic_write(path: Path, payload: bytes, mode: int = 0o600) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - descriptor, temporary_name = tempfile.mkstemp( - prefix=f".{path.name}.", suffix=".tmp", dir=str(path.parent) - ) - temporary = Path(temporary_name) - try: - with os.fdopen(descriptor, "wb") as handle: - handle.write(payload) - handle.flush() - os.fsync(handle.fileno()) - os.replace(temporary, path) - os.chmod(path, mode) - finally: - if temporary.exists(): - temporary.unlink() - - -def atomic_write_json(path: Path, value: Any) -> None: - atomic_write( - path, - (json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode( - "utf-8" - ), - ) - - -def load_json(path: Path, label: str) -> dict[str, Any]: - try: - value = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as exc: - raise PipelineBlockedError(f"{label} must be readable JSON: {path}") from exc - if not isinstance(value, dict): - raise PipelineBlockedError(f"{label} must be one JSON object") - return value - - -def exact_fields(value: dict[str, Any], expected: set[str], label: str) -> None: - if set(value) != expected: - raise PipelineBlockedError( - f"{label} fields mismatch; missing={sorted(expected - set(value))}, " - f"extra={sorted(set(value) - expected)}" - ) - - -def absolute_path(value: Any, label: str) -> Path: - path = Path(str(value)).expanduser() - if not path.is_absolute(): - raise PipelineBlockedError(f"{label} must be absolute") - if any(character in str(path) for character in ("\n", "\r", "\0")): - raise PipelineBlockedError(f"{label} contains an invalid character") - return path - - -def regular_nonsymlink(path: Path, label: str) -> None: - try: - metadata = path.lstat() - except FileNotFoundError as exc: - raise PipelineBlockedError(f"{label} does not exist: {path}") from exc - if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode): - raise PipelineBlockedError(f"{label} must be a regular non-symlink file") - - -def validate_sha256(value: Any, label: str) -> str: - text = str(value) - if len(text) != 64 or any(character not in "0123456789abcdef" for character in text): - raise PipelineBlockedError(f"{label} must be a lowercase SHA-256") - return text - - -def parse_iso_date(value: Any, label: str) -> date: - if not isinstance(value, str): - raise PipelineBlockedError(f"{label} must be YYYY-MM-DD") - try: - parsed = date.fromisoformat(value) - except ValueError as exc: - raise PipelineBlockedError(f"{label} must be YYYY-MM-DD") from exc - if parsed.isoformat() != value: - raise PipelineBlockedError(f"{label} must be YYYY-MM-DD") - return parsed - - -def normalize_config(raw: Any, path: Path) -> dict[str, Any]: - if not isinstance(raw, dict): - raise PipelineBlockedError("daily config must be one JSON object") - missing = REQUIRED_CONFIG_FIELDS - set(raw) - extra = set(raw) - REQUIRED_CONFIG_FIELDS - OPTIONAL_CONFIG_FIELDS - if missing or extra: - raise PipelineBlockedError( - "daily config fields mismatch; " - f"missing={sorted(missing)}, extra={sorted(extra)}" - ) - if raw["schema"] != CONFIG_SCHEMA: - raise PipelineBlockedError(f"daily config.schema must be {CONFIG_SCHEMA}") - runtime = absolute_path(raw["runtime_root"], "runtime_root") - if runtime.exists() and (runtime.is_symlink() or not runtime.is_dir()): - raise PipelineBlockedError("runtime_root must be a non-symlink directory") - if raw["broker"] != "kis-live" or raw["mode"] != "shadow": - raise PipelineBlockedError("daily pipeline requires kis-live/shadow") - history_start = parse_iso_date(raw["history_start_date"], "history_start_date") - minimum_sessions = raw["minimum_sessions"] - if ( - isinstance(minimum_sessions, bool) - or not isinstance(minimum_sessions, int) - or minimum_sessions < 756 - ): - raise PipelineBlockedError("minimum_sessions must be an integer >= 756") - interval = raw["request_interval_ms"] - if isinstance(interval, bool) or not isinstance(interval, int) or interval < 100: - raise PipelineBlockedError("request_interval_ms must be an integer >= 100") - coverage = raw["catalog_coverage_contract"] - if not isinstance(coverage, dict): - raise PipelineBlockedError("catalog_coverage_contract must be an object") - selector = raw["selector"] - if not isinstance(selector, dict) or selector.get("selector_version") != ( - "qta-screen-1.1.0" - ): - raise PipelineBlockedError("selector must use qta-screen-1.1.0") - risk_path = absolute_path(raw["risk_policy_path"], "risk_policy_path") - regular_nonsymlink(risk_path, "risk_policy_path") - risk = normalized_risk_policy(load_json(risk_path, "risk policy")) - for field in ( - "allow_existing_additions", - "allow_borrowed_cash", - "allow_margin", - "allow_short", - "allow_auto_fx", - ): - if risk[field] is not False: - raise PipelineBlockedError(f"risk policy {field} must be false") - if risk["whole_shares_only"] is not True: - raise PipelineBlockedError("risk policy whole_shares_only must be true") - execution = raw["execution_policy"] - if not isinstance(execution, dict): - raise PipelineBlockedError("execution_policy must be an object") - exact_fields(execution, EXECUTION_DEFAULT_FIELDS, "execution_policy") - qta_method_version = raw.get("qta_method_version", "qta-1.0.0") - if qta_method_version not in {"qta-1.0.0", "qta-2.0.0"}: - raise PipelineBlockedError( - "qta_method_version must be qta-1.0.0 or qta-2.0.0" - ) - component_paths = raw.get("exposure_component_paths", []) - if not isinstance(component_paths, list): - raise PipelineBlockedError("exposure_component_paths must be an array") - normalized_component_paths: list[str] = [] - for index, item in enumerate(component_paths): - if not isinstance(item, str) or not item: - raise PipelineBlockedError( - f"exposure_component_paths[{index}] must be a string" - ) - text = item - substituted = text.replace("{session_date}", "2000-01-01") - absolute_path( - substituted, - f"exposure_component_paths[{index}]", - ) - if text.count("{session_date}") > 1 or "{" in text.replace( - "{session_date}", "" - ): - raise PipelineBlockedError( - f"exposure_component_paths[{index}] has an unknown template" - ) - normalized_component_paths.append(text) - max_age = raw.get("exposure_component_max_age_seconds", 86400) - if ( - isinstance(max_age, bool) - or not isinstance(max_age, int) - or max_age <= 0 - or max_age > 86400 - ): - raise PipelineBlockedError( - "exposure_component_max_age_seconds must be 1..86400" - ) - aliases = raw["account_aliases"] - if not isinstance(aliases, dict) or set(aliases) != {"KR", "US"}: - raise PipelineBlockedError("account_aliases must contain exactly KR and US") - if any( - not isinstance(value, str) - or not value.strip() - or value != value.strip() - or any(character not in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_" for character in value) - for value in aliases.values() - ): - raise PipelineBlockedError( - "account aliases must use only letters, digits, hyphen, and underscore" - ) - deadline = raw["prepare_complete_seconds_before_open"] - if ( - isinstance(deadline, bool) - or not isinstance(deadline, int) - or deadline < 600 - or deadline > 7200 - ): - raise PipelineBlockedError( - "prepare_complete_seconds_before_open must be 600..7200" - ) - return { - **raw, - "runtime_root": runtime.resolve(strict=False), - "history_start_date": history_start.isoformat(), - "risk_policy_path": risk_path.resolve(), - "risk_policy": risk, - "exposure_component_paths": normalized_component_paths, - "exposure_component_max_age_seconds": max_age, - "qta_method_version": qta_method_version, - "config_path": path.resolve(), - } - - -def load_config(path: Path) -> dict[str, Any]: - regular_nonsymlink(path, "daily config") - return normalize_config(load_json(path, "daily config"), path) - - -def subprocess_environment() -> dict[str, str]: - environment = dict(os.environ) - environment["PYTHONDONTWRITEBYTECODE"] = "1" - environment.pop("PYTHONPATH", None) - environment.pop("PYTHONHOME", None) - return environment - - -def run_command(command: list[str], label: str) -> None: - result = subprocess.run( - command, - check=False, - capture_output=True, - text=True, - env=subprocess_environment(), - ) - if result.returncode != 0: - reason = result.stderr.strip() or result.stdout.strip() or ( - f"exit status {result.returncode}" - ) - if len(reason) > 2000: - reason = reason[-2000:] - raise PipelineBlockedError(f"{label} failed: {reason}") - - -def python_command(script: Path, *arguments: str) -> list[str]: - regular_nonsymlink(script, "pipeline script") - return [sys.executable, "-B", "-s", str(script), *arguments] - - -def workflow_directory(config: dict[str, Any], market: str, session: str) -> Path: - return config["runtime_root"] / "workflows" / market.lower() / session - - -def current_descriptor_path(config: dict[str, Any], market: str) -> Path: - return config["runtime_root"] / "current" / f"{market.lower()}.json" - - -def stage_status( - *, - stage: str, - status: str, - market: str, - session_date: str, - reason: str = "", - details: dict[str, Any] | None = None, -) -> dict[str, Any]: - return { - "schema": STATUS_SCHEMA, - "stage": stage, - "status": status, - "market": market, - "session_date": session_date, - "reason": reason, - "details": details or {}, - "recorded_at": datetime.now(timezone.utc).isoformat(), - "live_enabled": False, - "api_mutation_count": 0, - } - - -def current_market_date(market: str) -> str: - return datetime.now( - ZoneInfo(ENTRY_WINDOWS[market]["timezone"]) - ).date().isoformat() - - -def read_source_snapshot(path: Path, as_of: str, label: str) -> dict[str, Any]: - receipt = load_json(path, label) - if receipt.get("as_of") != as_of: - raise PipelineBlockedError(f"{label} as_of mismatch") - if not isinstance(receipt.get("official_sources"), list) or not isinstance( - receipt.get("broker_sources"), list - ): - raise PipelineBlockedError(f"{label} source arrays are missing") - return receipt - - -def ensure_source_snapshot( - *, - market: str, - as_of: str, - directory: Path, - technical_root: Path, -) -> dict[str, Any]: - receipt_path = directory / "source-snapshot.json" - if receipt_path.is_file() and not receipt_path.is_symlink(): - return read_source_snapshot(receipt_path, as_of, f"{market} source snapshot") - script = technical_root / "scripts" / ( - "fetch_kis_kr_eod.py" if market == "KR" else "fetch_kis_us_eod.py" - ) - run_command( - python_command( - script, - "snapshot-sources", - "--as-of", - as_of, - "--output-directory", - str(directory), - ), - f"{market} official source snapshot", - ) - return read_source_snapshot(receipt_path, as_of, f"{market} source snapshot") - - -def copy_cache_tree(source: Path, target: Path) -> None: - if not source.is_dir() or source.is_symlink() or target.exists(): - return - - def link_or_copy(from_path: str, to_path: str) -> str: - try: - os.link(from_path, to_path) - return to_path - except OSError: - return shutil.copy2(from_path, to_path) - - shutil.copytree(source, target, copy_function=link_or_copy) - - -def seed_eod_cache( - runtime_root: Path, - target_root: Path, - analysis_date: str, - component: str, -) -> None: - if target_root.exists() and any(target_root.iterdir()): - return - candidates: list[tuple[date, Path]] = [] - eod_root = runtime_root / "eod" - if not eod_root.is_dir(): - return - target_date = date.fromisoformat(analysis_date) - for catalog in eod_root.rglob("eod-catalog.csv"): - if catalog.parent.name != component: - continue - try: - relative_parts = catalog.relative_to(eod_root).parts - candidate_date = date.fromisoformat(relative_parts[0]) - except (ValueError, IndexError): - continue - candidate_root = catalog.parent - if candidate_date > target_date or candidate_root.resolve() == target_root.resolve(): - continue - receipt_path = candidate_root / "eod-bundle-receipt.json" - if ( - receipt_path.is_symlink() - or not receipt_path.is_file() - or catalog.is_symlink() - ): - continue - try: - receipt = json.loads(receipt_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - continue - if ( - not isinstance(receipt, dict) - or receipt.get("status") != "READY" - or receipt.get("analysis_date") != candidate_date.isoformat() - or receipt.get("api_mutation_count") != 0 - ): - continue - candidates.append((candidate_date, candidate_root)) - if not candidates: - return - source_root = max(candidates, key=lambda item: (item[0], str(item[1])))[1] - for name in ("stocks", "benchmarks"): - copy_cache_tree(source_root / name, target_root / name) - - -def ensure_eod( - *, - config: dict[str, Any], - technical_root: Path, - workflow: Path, - as_of: str, - analysis_date: str, - market: str, - sources: dict[str, Any], - cache_namespace: str, -) -> Path: - eod_base = config["runtime_root"] / "eod" / analysis_date / cache_namespace - component = market.lower() - output = eod_base / component - receipt_path = output / "eod-bundle-receipt.json" - jobs_directory = workflow / "jobs" - jobs_directory.mkdir(parents=True, exist_ok=True) - - if not receipt_path.is_file(): - seed_eod_cache(config["runtime_root"], output, analysis_date, component) - job = { - "schema": ( - "qta-kis-kr-eod-job/v1" - if market == "KR" - else "qta-kis-us-eod-job/v1" - ), - "as_of": as_of, - "analysis_date": analysis_date, - "environment": "live", - "output_directory": str(output.resolve()), - "history_start_date": config["history_start_date"], - "minimum_sessions": config["minimum_sessions"], - "request_interval_ms": config["request_interval_ms"], - "official_sources": sources["official_sources"], - "broker_sources": sources["broker_sources"], - "catalog_coverage_contract": config["catalog_coverage_contract"], - "base_eod_catalog": "", - } - job_path = jobs_directory / f"{component}-eod.json" - atomic_write_json(job_path, job) - script = technical_root / "scripts" / ( - "fetch_kis_kr_eod.py" - if market == "KR" - else "fetch_kis_us_eod.py" - ) - run_command( - python_command( - script, - "collect", - "--job", - str(job_path), - ), - f"KIS {market} adjusted EOD", - ) - receipt = load_json(receipt_path, f"{market} EOD receipt") - if ( - receipt.get("status") != "READY" - or receipt.get("analysis_date") != analysis_date - or receipt.get("api_mutation_count") != 0 - ): - raise PipelineBlockedError(f"{market} EOD receipt is not READY and read-only") - return output - - -def derive_universe_build_spec( - *, - config: dict[str, Any], - eod_output: Path, - workflow: Path, - market: str, -) -> Path: - if market not in {"KR", "US"}: - raise PipelineBlockedError("market must be KR or US") - receipt_path = eod_output / "eod-bundle-receipt.json" - regular_nonsymlink(receipt_path, "EOD receipt") - receipt = load_json(receipt_path, "EOD receipt") - binding = receipt.get("build_spec") - if not isinstance(binding, dict): - raise PipelineBlockedError("EOD receipt build_spec must be an object") - exact_fields(binding, {"path", "sha256"}, "EOD receipt build_spec") - source_path = absolute_path(binding["path"], "EOD build spec path") - expected_path = (eod_output / "universe-build-spec.json").resolve() - regular_nonsymlink(source_path, "EOD build spec") - if source_path.resolve() != expected_path: - raise PipelineBlockedError( - "EOD receipt build_spec path does not match its bundle" - ) - expected_sha256 = validate_sha256( - binding["sha256"], - "EOD receipt build_spec.sha256", - ) - if sha256_file(source_path) != expected_sha256: - raise PipelineBlockedError("EOD build spec hash mismatch") - derived = load_json(source_path, "EOD build spec") - expected_exchanges = ( - {"KOSPI", "KOSDAQ"} if market == "KR" else {"NYSE", "NASDAQ"} - ) - for field in ("official_sources", "broker_sources"): - sources = derived.get(field) - if not isinstance(sources, list) or any( - not isinstance(item, dict) or not isinstance(item.get("exchange"), str) - for item in sources - ): - raise PipelineBlockedError(f"EOD build spec {field} must be an array") - exchanges = {item["exchange"] for item in sources} - if len(sources) != len(exchanges) or exchanges != expected_exchanges: - raise PipelineBlockedError( - f"EOD build spec {field} must contain exactly " - f"{sorted(expected_exchanges)} for {market}" - ) - derived["catalog_coverage_contract"] = config["catalog_coverage_contract"] - output_path = workflow / "universe-build-spec.json" - atomic_write_json(output_path, derived) - return output_path - - -def prepare( - config: dict[str, Any], - market: str, - technical_root: Path, - trader_root: Path, -) -> dict[str, Any]: - session = current_market_date(market) - workflow = workflow_directory(config, market, session) - workflow.mkdir(parents=True, exist_ok=True) - current_path = current_descriptor_path(config, market) - if current_path.is_file() and not current_path.is_symlink(): - current = load_json(current_path, "current daily descriptor") - if ( - current.get("schema") == DESCRIPTOR_SCHEMA - and current.get("market") == market - and current.get("session_date") == session - ): - current_status = current.get("status") - if current_status in {"BLOCKED", "MANUAL_BLOCK"}: - raise PipelineBlockedError( - str( - current.get("blocked_reason") - or "current market session is terminally blocked" - ) - ) - if current_status in { - "MARKET_CLOSED", - "PREPARED", - "PLANNED", - "READY", - }: - return load_json( - workflow / "prepare-status.json", - "existing prepare status", - ) - calendar = market_calendar.snapshot( - market=market, - session_date=session, - output_directory=workflow / "calendar", - ) - if calendar["status"] == "MARKET_CLOSED": - receipt = stage_status( - stage="prepare", - status="MARKET_CLOSED", - market=market, - session_date=session, - details={"calendar_receipt": calendar}, - ) - descriptor = { - "schema": DESCRIPTOR_SCHEMA, - "status": "MARKET_CLOSED", - "market": market, - "session_date": session, - "workflow_directory": str(workflow.resolve()), - "prepare_status": str((workflow / "prepare-status.json").resolve()), - "live_enabled": False, - "api_mutation_count": 0, - } - atomic_write_json(workflow / "prepare-status.json", receipt) - atomic_write_json(current_path, descriptor) - return receipt - if calendar["status"] != "READY": - raise PipelineBlockedError("calendar snapshot is neither READY nor closed") - market_session_path = Path(calendar["market_session_path"]) - market_session = load_json(market_session_path, "market session") - regular_open = datetime.fromisoformat(market_session["regular_open"]) - deadline = regular_open - timedelta( - seconds=config["prepare_complete_seconds_before_open"] - ) - if datetime.now(timezone.utc) >= deadline.astimezone(timezone.utc): - raise PipelineBlockedError( - "prepare started too late to complete before the account snapshot gate" - ) - analysis_date = market_session["previous_session_date"] - if parse_iso_date(config["history_start_date"], "history_start_date") >= ( - parse_iso_date(analysis_date, "analysis_date") - ): - raise PipelineBlockedError("history_start_date must precede analysis_date") - source_root = config["runtime_root"] / "sources" / session - sources = ensure_source_snapshot( - market=market, - as_of=session, - directory=source_root / market.lower(), - technical_root=technical_root, - ) - eod_output = ensure_eod( - config=config, - technical_root=technical_root, - workflow=workflow, - as_of=session, - analysis_date=analysis_date, - market=market, - sources=sources, - cache_namespace="official-market-v1", - ) - if datetime.now(timezone.utc) >= deadline.astimezone(timezone.utc): - raise PipelineBlockedError( - "EOD preparation finished after the account snapshot deadline" - ) - build_spec = derive_universe_build_spec( - config=config, - eod_output=eod_output, - workflow=workflow, - market=market, - ) - manifest_path = workflow / "universe-manifest.json" - run_command( - python_command( - technical_root / "scripts" / "build_universe_manifest.py", - "--build-spec", - str(build_spec), - "--output", - str(manifest_path), - ), - f"{market} universe manifest", - ) - manifest = load_json(manifest_path, "universe manifest") - if manifest.get("build_status") != "READY": - raise PipelineBlockedError("universe manifest build_status is not READY") - selector_path = workflow / "selector.json" - atomic_write_json(selector_path, config["selector"]) - screen_path = workflow / "screen.json" - run_command( - python_command( - technical_root / "scripts" / "screen_universe.py", - "--manifest", - str(manifest_path), - "--selector", - str(selector_path), - "--output", - str(screen_path), - "--method-version", - config["qta_method_version"], - ), - f"{market} screen", - ) - screen = load_json(screen_path, "screen") - if screen.get("screen_status") != "READY": - raise PipelineBlockedError("qta-screen/v2 is not READY") - receipt = stage_status( - stage="prepare", - status="READY", - market=market, - session_date=session, - details={ - "analysis_date": analysis_date, - "manifest_hash": manifest["manifest_hash"], - "screen_hash": screen["screen_hash"], - "calendar_session_hash": market_session["session_hash"], - }, - ) - descriptor = { - "schema": DESCRIPTOR_SCHEMA, - "status": "PREPARED", - "market": market, - "session_date": session, - "analysis_date": analysis_date, - "workflow_directory": str(workflow.resolve()), - "market_session_path": str(market_session_path.resolve()), - "manifest_path": str(manifest_path.resolve()), - "screen_path": str(screen_path.resolve()), - "manifest_hash": manifest["manifest_hash"], - "screen_hash": screen["screen_hash"], - "live_enabled": False, - "api_mutation_count": 0, - } - atomic_write_json(workflow / "prepare-status.json", receipt) - atomic_write_json(current_path, descriptor) - return receipt - - -def resolve_exposure_components( - config: dict[str, Any], session_date: str -) -> list[Path]: - paths: list[Path] = [] - for value in config["exposure_component_paths"]: - path = Path(value.replace("{session_date}", session_date)) - regular_nonsymlink(path, "configured exposure component") - paths.append(path.resolve()) - brokers: set[str] = set() - for path in paths: - component = load_json(path, "configured exposure component") - broker = component.get("broker") - if broker not in {"toss", "nh"}: - raise PipelineBlockedError( - f"configured exposure component broker is invalid: {path}" - ) - if broker in brokers: - raise PipelineBlockedError( - f"configured exposure broker appears more than once: {broker}" - ) - brokers.add(broker) - return paths - - -def read_ledger_states(ledger_path: Path) -> list[dict[str, str]]: - if not ledger_path.is_file() or ledger_path.is_symlink(): - return [] - uri = f"file:{ledger_path.as_posix()}?mode=ro" - try: - connection = sqlite3.connect(uri, uri=True) - rows = connection.execute( - "SELECT intent_id, plan_hash, state FROM intents ORDER BY intent_id" - ).fetchall() - except sqlite3.Error as exc: - raise PipelineBlockedError(f"cannot inspect prior ledger: {ledger_path}") from exc - finally: - if "connection" in locals(): - connection.close() - return [ - {"intent_id": str(row[0]), "plan_hash": str(row[1]), "state": str(row[2])} - for row in rows - ] - - -def state_market_directory( - runtime_root: Path, account_alias: str, market: str -) -> Path: - account_root = runtime_root / "state" / account_alias - if account_root.exists() and ( - account_root.is_symlink() or not account_root.is_dir() - ): - raise PipelineBlockedError( - "account state root must be a non-symlink directory" - ) - market_root = account_root / market.lower() - if market_root.exists() and ( - market_root.is_symlink() or not market_root.is_dir() - ): - raise PipelineBlockedError( - "market state root must be a non-symlink directory" - ) - return market_root - - -def unresolved_state_directories( - market_root: Path, - *, - exclude_date: str | None = None, -) -> list[tuple[Path, list[dict[str, str]]]]: - if not market_root.exists(): - return [] - if market_root.is_symlink() or not market_root.is_dir(): - raise PipelineBlockedError("market state root must be a non-symlink directory") - output: list[tuple[Path, list[dict[str, str]]]] = [] - for directory in sorted(market_root.iterdir(), key=lambda item: item.name): - if ( - not directory.is_dir() - or directory.is_symlink() - or (exclude_date is not None and directory.name == exclude_date) - ): - continue - states = read_ledger_states(directory / "ledger.sqlite3") - unresolved = [ - item - for item in states - if item["state"] not in TERMINAL_OR_MANUAL - or item["state"] == "MANUAL_BLOCK" - ] - if unresolved: - output.append((directory, unresolved)) - return output - - -def prior_descriptor_for_state( - config: dict[str, Any], market: str, state_directory: Path -) -> dict[str, Any]: - session = state_directory.name - descriptor_path = ( - config["runtime_root"] - / "workflows" - / market.lower() - / session - / "descriptor.json" - ) - return load_json(descriptor_path, "prior workflow descriptor") - - -def reconcile_one( - *, - trader_root: Path, - descriptor: dict[str, Any], - output_path: Path, -) -> None: - command = python_command( - trader_root / "scripts" / "reconcile.py", - "--plan", - descriptor["plan_path"], - "--broker", - "kis-live", - "--state-dir", - descriptor["state_directory"], - "--output", - str(output_path), - ) - result = subprocess.run( - command, - check=False, - capture_output=True, - text=True, - env=subprocess_environment(), - ) - if result.returncode not in {0, 2}: - raise PipelineBlockedError( - "reconciliation process failed outside its receipt contract" - ) - - -def mark_manual_block(state_directory: Path, reason: str) -> None: - ledger_path = state_directory / "ledger.sqlite3" - states = read_ledger_states(ledger_path) - if not states: - return - from execution_core import Ledger - - ledger = Ledger(ledger_path) - try: - for item in states: - state = ledger.get(item["intent_id"]).state - if state in TERMINAL_OR_MANUAL: - continue - if state in {"ACKNOWLEDGED", "PARTIALLY_FILLED", "CANCEL_PENDING"}: - ledger.transition(item["intent_id"], "UNKNOWN", {"reason": reason}) - state = "UNKNOWN" - if state == "UNKNOWN": - ledger.transition(item["intent_id"], "RECONCILING", {}) - state = "RECONCILING" - if state in {"PLANNED", "WAIT_TRIGGER", "RESERVED", "RECONCILING"}: - ledger.transition(item["intent_id"], "MANUAL_BLOCK", {"reason": reason}) - elif state == "SUBMITTING": - ledger.transition(item["intent_id"], "UNKNOWN", {"reason": reason}) - ledger.transition(item["intent_id"], "RECONCILING", {}) - ledger.transition(item["intent_id"], "MANUAL_BLOCK", {"reason": reason}) - finally: - ledger.close() - - -def reconcile_prior_states( - *, - config: dict[str, Any], - trader_root: Path, - market: str, - account_alias: str, - current_session: str, - receipt_directory: Path, -) -> None: - market_root = state_market_directory( - config["runtime_root"], account_alias, market - ) - unresolved = unresolved_state_directories( - market_root, exclude_date=current_session - ) - for state_directory, states in unresolved: - if any(item["state"] == "MANUAL_BLOCK" for item in states): - raise PipelineBlockedError( - f"prior MANUAL_BLOCK requires explicit clearance: {state_directory}" - ) - descriptor = prior_descriptor_for_state(config, market, state_directory) - for attempt in range(1, RECONCILE_ATTEMPTS + 1): - reconcile_one( - trader_root=trader_root, - descriptor=descriptor, - output_path=( - receipt_directory - / f"prior-{state_directory.name}-reconcile-{attempt:02d}.json" - ), - ) - observed = read_ledger_states(state_directory / "ledger.sqlite3") - if any(item["state"] == "MANUAL_BLOCK" for item in observed): - raise PipelineBlockedError( - f"prior reconciliation produced MANUAL_BLOCK: {state_directory}" - ) - remaining = [ - item for item in observed if item["state"] not in TERMINAL_OR_MANUAL - ] - if not remaining: - break - if attempt < RECONCILE_ATTEMPTS: - time.sleep(RECONCILE_INTERVAL_SECONDS) - else: - mark_manual_block( - state_directory, - "read-only reconciliation exhausted 20 attempts", - ) - raise PipelineBlockedError( - f"prior state became MANUAL_BLOCK after reconciliation: " - f"{state_directory}" - ) - - -def snapshot_and_plan( - config: dict[str, Any], - market: str, - technical_root: Path, - trader_root: Path, -) -> dict[str, Any]: - session = current_market_date(market) - descriptor_path = current_descriptor_path(config, market) - descriptor = load_json(descriptor_path, "current daily descriptor") - if descriptor.get("session_date") != session or descriptor.get("market") != market: - raise PipelineBlockedError("current descriptor is not for this market session") - if descriptor.get("status") in {"BLOCKED", "MANUAL_BLOCK"}: - raise PipelineBlockedError( - str( - descriptor.get("blocked_reason") - or "current market session is terminally blocked" - ) - ) - if descriptor.get("status") == "MARKET_CLOSED": - receipt = stage_status( - stage="snapshot", - status="MARKET_CLOSED", - market=market, - session_date=session, - ) - atomic_write_json( - Path(descriptor["workflow_directory"]) / "snapshot-status.json", - receipt, - ) - return receipt - if descriptor.get("status") == "PLANNED": - return load_json( - Path(descriptor["workflow_directory"]) / "snapshot-status.json", - "existing snapshot status", - ) - if descriptor.get("status") != "PREPARED": - raise PipelineBlockedError("daily prepare stage is not PREPARED") - workflow = Path(descriptor["workflow_directory"]) - market_session = load_json( - Path(descriptor["market_session_path"]), "market session" - ) - regular_open = datetime.fromisoformat(market_session["regular_open"]) - now = datetime.now(timezone.utc) - age_until_open = (regular_open.astimezone(timezone.utc) - now).total_seconds() - snapshot_max_age = config["execution_policy"]["snapshot_max_age_seconds"] - if age_until_open <= 0: - raise PipelineBlockedError("account snapshot started after regular open") - if age_until_open > snapshot_max_age: - raise PipelineBlockedError( - "account snapshot started too early for snapshot_max_age_seconds" - ) - components = resolve_exposure_components(config, session) - account_directory = workflow / "account" - account_directory.mkdir(parents=True, exist_ok=True) - account_alias = config["account_aliases"][market] - reconcile_prior_states( - config=config, - trader_root=trader_root, - market=market, - account_alias=account_alias, - current_session=session, - receipt_directory=workflow / "reconciliation", - ) - state_market_root = state_market_directory( - config["runtime_root"], account_alias, market - ) - state_directory = state_market_root / session - account_path = account_directory / "account.json" - exposure_path = account_directory / "exposure.json" - account_status_path = account_directory / "status.json" - account_job = { - "schema": "qta-account-snapshot-job/v2", - "broker": "kis-live", - "mode": "shadow", - "market": market, - "account_alias": account_alias, - "universe_manifest": descriptor["manifest_path"], - "fx_to_krw": "1" if market == "KR" else "KIS_PRESENT_BALANCE", - "manual_exposure_components": [str(path) for path in components], - "manual_component_max_age_seconds": config[ - "exposure_component_max_age_seconds" - ], - "output_account_path": str(account_path.resolve()), - "output_exposure_path": str(exposure_path.resolve()), - "output_status_path": str(account_status_path.resolve()), - } - account_job_path = workflow / "jobs" / "account-snapshot.json" - atomic_write_json(account_job_path, account_job) - run_command( - python_command( - trader_root / "scripts" / "account_snapshot.py", - "collect", - "--job", - str(account_job_path), - ), - "KIS and cross-broker account snapshot", - ) - account_status = load_json(account_status_path, "account snapshot status") - if ( - account_status.get("status") != "READY" - or account_status.get("api_mutation_count") != 0 - ): - raise PipelineBlockedError("account snapshot status is not READY/read-only") - execution = { - "schema": "qta-execution-policy/v1", - "market": market, - "timezone": market_session["timezone"], - "entry_window_start": market_session["regular_open"], - "entry_window_end": ( - datetime.fromisoformat(market_session["regular_open"]) - + timedelta(hours=1) - ).isoformat(), - "market_session": market_session, - **config["execution_policy"], - } - execution = normalized_execution_policy(execution) - execution_path = workflow / "execution-policy.json" - atomic_write_json(execution_path, execution) - plan_path = workflow / "order-plan.json" - run_command( - python_command( - trader_root / "scripts" / "plan_orders.py", - "--screen", - descriptor["screen_path"], - "--account", - str(account_path), - "--exposure", - str(exposure_path), - "--risk", - str(config["risk_policy_path"]), - "--execution", - str(execution_path), - "--output", - str(plan_path), - ), - "deterministic order plan", - ) - plan = load_json(plan_path, "order plan") - if plan.get("plan_status") not in {"READY", "NO_ORDERS"}: - raise PipelineBlockedError("order plan is not READY or NO_ORDERS") - preview_path = workflow / "submit-preview.json" - run_command( - python_command( - trader_root / "scripts" / "run_session.py", - "preview", - "--plan", - str(plan_path), - "--broker", - "kis-live", - "--output", - str(preview_path), - ), - "venue and submit serialization preflight", - ) - receipt = stage_status( - stage="snapshot", - status="READY", - market=market, - session_date=session, - details={ - "plan_hash": plan["plan_hash"], - "settled_cash": account_status["settled_cash"], - "borrowed_buying_power_excluded": True, - "fx_to_krw": account_status["fx_to_krw"], - "additional_exposure_components": len(components), - "state_directory": str(state_directory.resolve()), - }, - ) - full_descriptor = { - **descriptor, - "status": "PLANNED", - "account_alias": account_alias, - "account_path": str(account_path.resolve()), - "exposure_path": str(exposure_path.resolve()), - "account_status_path": str(account_status_path.resolve()), - "execution_policy_path": str(execution_path.resolve()), - "plan_path": str(plan_path.resolve()), - "plan_hash": plan["plan_hash"], - "state_directory": str(state_directory.resolve()), - "session_status_path": str((workflow / "session-status.json").resolve()), - "reconciliation_status_path": str( - (workflow / "reconciliation-status.json").resolve() - ), - "live_enabled": False, - "api_mutation_count": 0, - } - atomic_write_json(workflow / "snapshot-status.json", receipt) - atomic_write_json(workflow / "descriptor.json", full_descriptor) - atomic_write_json(descriptor_path, full_descriptor) - return receipt - - -def reconcile_current( - *, - descriptor: dict[str, Any], - trader_root: Path, - output_path: Path, -) -> dict[str, Any]: - state_directory = Path(descriptor["state_directory"]) - for attempt in range(1, RECONCILE_ATTEMPTS + 1): - attempt_path = output_path.with_name( - f"{output_path.stem}-attempt-{attempt:02d}.json" - ) - reconcile_one( - trader_root=trader_root, - descriptor=descriptor, - output_path=attempt_path, - ) - states = read_ledger_states(state_directory / "ledger.sqlite3") - remaining = [ - item for item in states if item["state"] not in TERMINAL_OR_MANUAL - ] - if not remaining: - final = load_json(attempt_path, "reconciliation receipt") - atomic_write_json(output_path, final) - return final - if attempt < RECONCILE_ATTEMPTS: - time.sleep(RECONCILE_INTERVAL_SECONDS) - mark_manual_block( - state_directory, - "read-only reconciliation exhausted 20 attempts", - ) - states = read_ledger_states(state_directory / "ledger.sqlite3") - manual = stage_status( - stage="reconcile", - status="MANUAL_BLOCK", - market=descriptor["market"], - session_date=descriptor["session_date"], - reason="UNKNOWN or CANCEL_PENDING remained after 20 read-only attempts", - details={"intents": states, "emergency_alert": True}, - ) - atomic_write_json(output_path, manual) - return manual - - -def build_final_report( - *, - config: dict[str, Any], - descriptor: dict[str, Any], - session_receipt: dict[str, Any], - reconciliation: dict[str, Any], -) -> dict[str, Any]: - plan = load_json(Path(descriptor["plan_path"]), "order plan") - screen = load_json(Path(descriptor["screen_path"]), "screen") - account_status = load_json( - Path(descriptor["account_status_path"]), "account status" - ) - states = read_ledger_states( - Path(descriptor["state_directory"]) / "ledger.sqlite3" - ) - selected = sum( - len(screen["selected"][exchange]) - for exchange in ( - ("KOSPI", "KOSDAQ") if descriptor["market"] == "KR" else ("NYSE", "NASDAQ") - ) - ) - state_counts = { - state: sum(1 for item in states if item["state"] == state) - for state in sorted({item["state"] for item in states}) - } - metrics = session_receipt.get("polling_metrics", {}) - first_unmet = "" - final_status = "READY" - if state_counts.get("MANUAL_BLOCK", 0): - final_status = "MANUAL_BLOCK" - first_unmet = reconciliation.get( - "reason", "one or more intents require manual clearance" - ) - elif session_receipt.get("status") == "BLOCKED": - final_status = "BLOCKED" - first_unmet = session_receipt.get("reason", "session blocked") - elif reconciliation.get("status") not in {"READY", None}: - final_status = "BLOCKED" - first_unmet = "reconciliation did not reach READY" - submitted = int(metrics.get("submits_started", 0)) - if submitted != 0 or session_receipt.get("mutation_sent") is True: - raise PipelineBlockedError("shadow session reported an API mutation") - return { - "schema": "qta-daily-shadow-final-report/v1", - "status": final_status, - "session_date": descriptor["session_date"], - "market": descriptor["market"], - "broker": "kis-live", - "mode": "shadow", - "manifest_hash": descriptor["manifest_hash"], - "screen_hash": descriptor["screen_hash"], - "plan_hash": descriptor["plan_hash"], - "selected_count": selected, - "planned_count": len(plan.get("intents", [])), - "submitted_count": 0, - "filled_count": state_counts.get("FILLED", 0), - "cancelled_count": state_counts.get("CANCELLED", 0), - "blocked_count": state_counts.get("MANUAL_BLOCK", 0), - "cycles": int(metrics.get("cycles_completed", 0)), - "skipped_cycles": int(metrics.get("cycles_skipped", 0)), - "max_schedule_latency_ms": int( - metrics.get("max_schedule_lateness_ms", 0) - ), - "max_quote_latency_ms": int(metrics.get("max_quote_latency_ms", 0)), - "max_submit_latency_ms": int(metrics.get("max_submit_latency_ms", 0)), - "settled_cash_used": format( - Decimal(plan["settled_cash_start"]) - - Decimal(plan["settled_cash_unreserved"]), - "f", - ), - "settled_cash_available": account_status["settled_cash"], - "borrowed_cash_excluded": load_json( - Path(descriptor["account_path"]), "account" - )["borrowed_buying_power"], - "intent_final_states": states, - "first_unmet_gate": first_unmet, - "api_mutation_count": 0, - "live_enabled": False, - } - - -def entry( - config: dict[str, Any], - market: str, - technical_root: Path, - trader_root: Path, - max_cycles: int | None, -) -> dict[str, Any]: - session = current_market_date(market) - descriptor_path = current_descriptor_path(config, market) - descriptor = load_json(descriptor_path, "current daily descriptor") - if descriptor.get("session_date") != session or descriptor.get("market") != market: - raise PipelineBlockedError("current descriptor is not for this market session") - if descriptor.get("status") in {"BLOCKED", "MANUAL_BLOCK"}: - raise PipelineBlockedError( - str( - descriptor.get("blocked_reason") - or "current market session is terminally blocked" - ) - ) - if descriptor.get("status") == "MARKET_CLOSED": - receipt = stage_status( - stage="entry", - status="MARKET_CLOSED", - market=market, - session_date=session, - ) - atomic_write_json( - Path(descriptor["workflow_directory"]) / "entry-status.json", - receipt, - ) - return receipt - if descriptor.get("status") != "PLANNED": - raise PipelineBlockedError("daily snapshot and plan stage is not ready") - account_alias = config["account_aliases"][market] - if descriptor.get("account_alias") != account_alias: - raise PipelineBlockedError("current plan account alias does not match config") - expected_state = ( - state_market_directory(config["runtime_root"], account_alias, market) - / session - ).resolve() - if Path(descriptor["state_directory"]).resolve() != expected_state: - raise PipelineBlockedError( - "state directory does not follow the account/market/session rule" - ) - ledger_path = expected_state / "ledger.sqlite3" - current_states = read_ledger_states(ledger_path) - if ledger_path.exists(): - if any(item["state"] not in TERMINAL_OR_MANUAL for item in current_states): - reconciliation = reconcile_current( - descriptor=descriptor, - trader_root=trader_root, - output_path=Path(descriptor["reconciliation_status_path"]), - ) - if reconciliation.get("status") != "READY": - raise PipelineBlockedError( - "existing current-session ledger did not reconcile to terminal" - ) - raise PipelineBlockedError( - "current session was already started; automatic re-entry is forbidden" - ) - attempt_path = Path(descriptor["workflow_directory"]) / "entry-attempt.json" - if attempt_path.exists(): - raise PipelineBlockedError( - "current session already has an entry-attempt marker" - ) - atomic_write_json( - attempt_path, - { - "schema": "qta-shadow-entry-attempt/v1", - "market": market, - "session_date": session, - "plan_hash": descriptor["plan_hash"], - "started_at": datetime.now(timezone.utc).isoformat(), - "live_enabled": False, - "api_mutation_count": 0, - }, - ) - command = python_command( - trader_root / "scripts" / "run_session.py", - "run", - "--plan", - descriptor["plan_path"], - "--broker", - "kis-live", - "--mode", - "shadow", - "--state-dir", - descriptor["state_directory"], - "--output", - descriptor["session_status_path"], - ) - if max_cycles is not None: - if max_cycles <= 0: - raise PipelineBlockedError("max_cycles must be positive when supplied") - command.extend(["--max-cycles", str(max_cycles)]) - result = subprocess.run( - command, - check=False, - capture_output=True, - text=True, - env=subprocess_environment(), - ) - session_receipt = load_json( - Path(descriptor["session_status_path"]), "session status" - ) - reconciliation = reconcile_current( - descriptor=descriptor, - trader_root=trader_root, - output_path=Path(descriptor["reconciliation_status_path"]), - ) - report = build_final_report( - config=config, - descriptor=descriptor, - session_receipt=session_receipt, - reconciliation=reconciliation, - ) - workflow = Path(descriptor["workflow_directory"]) - atomic_write_json(workflow / "final-report.json", report) - atomic_write_json( - descriptor_path, - { - **descriptor, - "status": report["status"], - "final_report_path": str((workflow / "final-report.json").resolve()), - }, - ) - if result.returncode != 0 or report["status"] not in {"READY"}: - raise PipelineBlockedError( - report["first_unmet_gate"] or "shadow session did not finish READY" - ) - return report - - -def self_test() -> None: - with tempfile.TemporaryDirectory(prefix="qta-daily-pipeline-") as temporary: - root = Path(temporary) - config = { - "runtime_root": root, - "exposure_component_paths": [], - } - if resolve_exposure_components(config, "2026-07-28") != []: - raise AssertionError("additional exposure components must be optional") - receipt = stage_status( - stage="self-test", - status="PASS", - market="US", - session_date="2026-07-28", - ) - if receipt["api_mutation_count"] != 0 or receipt["live_enabled"]: - raise AssertionError("shadow safety status failed") - if "receipt_hash" in receipt: - raise AssertionError("daily status must not require a receipt signature") - print( - json.dumps( - { - "self_test": "PASS", - "config_schema": CONFIG_SCHEMA, - "external_exposure_required": False, - "arm_required": False, - "provenance_required": False, - "live_enabled": False, - "api_mutation_count": 0, - }, - sort_keys=True, - ) - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - subparsers = parser.add_subparsers(dest="command", required=True) - for name in ("prepare", "snapshot", "entry"): - stage = subparsers.add_parser(name) - stage.add_argument("--config", required=True) - stage.add_argument("--market", choices=("KR", "US"), required=True) - stage.add_argument("--technical-skill-root", required=True) - stage.add_argument("--trader-skill-root", required=True) - if name == "entry": - stage.add_argument("--max-cycles", type=int) - subparsers.add_parser("self-test") - return parser.parse_args() - - -def main() -> int: - args = parse_args() - if args.command == "self-test": - self_test() - return 0 - config_path = absolute_path(args.config, "config") - technical_root = absolute_path( - args.technical_skill_root, "technical_skill_root" - ) - trader_root = absolute_path(args.trader_skill_root, "trader_skill_root") - market = args.market - session = current_market_date(market) - try: - config = load_config(config_path) - config["runtime_root"].mkdir(parents=True, exist_ok=True, mode=0o700) - if args.command == "prepare": - output = prepare(config, market, technical_root, trader_root) - elif args.command == "snapshot": - output = snapshot_and_plan( - config, market, technical_root, trader_root - ) - else: - output = entry( - config, - market, - technical_root, - trader_root, - args.max_cycles, - ) - print(json.dumps(output, ensure_ascii=False, sort_keys=True)) - return 0 - except ( - PipelineBlockedError, - market_calendar.CalendarBlockedError, - OSError, - ValueError, - json.JSONDecodeError, - ) as exc: - receipt = stage_status( - stage=args.command, - status="BLOCKED", - market=market, - session_date=session, - reason=str(exc), - ) - try: - workflow = workflow_directory( - {"runtime_root": absolute_path( - load_json(config_path, "daily config")["runtime_root"], - "runtime_root", - )}, - market, - session, - ) - blocked_path = workflow / f"{args.command}-blocked.json" - stage_status_path = workflow / f"{args.command}-status.json" - atomic_write_json(blocked_path, receipt) - atomic_write_json(stage_status_path, receipt) - descriptor_path = current_descriptor_path( - {"runtime_root": workflow.parents[2]}, - market, - ) - existing: dict[str, Any] = {} - if descriptor_path.is_file() and not descriptor_path.is_symlink(): - candidate = load_json(descriptor_path, "current daily descriptor") - if ( - candidate.get("market") == market - and candidate.get("session_date") == session - ): - existing = candidate - final_status = ( - "MANUAL_BLOCK" - if existing.get("status") == "MANUAL_BLOCK" - else "BLOCKED" - ) - blocked_descriptor = { - **existing, - "schema": DESCRIPTOR_SCHEMA, - "status": final_status, - "market": market, - "session_date": session, - "workflow_directory": str(workflow.resolve()), - "blocked_stage": args.command, - "blocked_reason": str(exc), - "blocked_status_path": str(blocked_path.resolve()), - "live_enabled": False, - "api_mutation_count": 0, - } - atomic_write_json( - workflow / "descriptor.json", - blocked_descriptor, - ) - atomic_write_json(descriptor_path, blocked_descriptor) - except (OSError, ValueError, KeyError): - pass - print(json.dumps(receipt, ensure_ascii=False, sort_keys=True), file=sys.stderr) - return 2 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/skills/quant-stock-polling-trader/scripts/execution_core.py b/skills/quant-stock-polling-trader/scripts/execution_core.py deleted file mode 100755 index 2bd45fd..0000000 --- a/skills/quant-stock-polling-trader/scripts/execution_core.py +++ /dev/null @@ -1,2838 +0,0 @@ -#!/usr/bin/env python3 -"""Deterministic planning primitives and durable order-intent ledger.""" - -from __future__ import annotations - -import argparse -import fcntl -import hashlib -import json -import sqlite3 -import sys -import tempfile -from collections.abc import Iterator -from contextlib import contextmanager -from dataclasses import dataclass -from datetime import date, datetime, timedelta, timezone -from decimal import ROUND_DOWN, Decimal, InvalidOperation -from pathlib import Path -from typing import Any -from zoneinfo import ZoneInfo - -EXECUTION_VERSION = "open1h-exec-2.0.0" -PLAN_SCHEMA = "qta-order-plan/v2" -RISK_SCHEMA = "qta-risk-policy/v1" -EXECUTION_POLICY_SCHEMA = "qta-execution-policy/v1" -MARKET_SESSION_SCHEMA = "qta-market-session/v1" -ACCOUNT_SCHEMA = "qta-account-snapshot/v2" -EXPOSURE_SCHEMA = "qta-exposure-snapshot/v1" -EXPOSURE_SCHEMA_V2 = "qta-exposure-snapshot/v2" -SCREEN_SCHEMA = "qta-screen/v1" -SCREEN_SCHEMA_V2 = "qta-screen/v2" -MARKET_CURRENCY = {"KR": "KRW", "US": "USD"} -V2_SELECTOR_VERSION = "qta-screen-1.1.0" -QTA_METHOD_V1 = "qta-1.0.0" -QTA_METHOD_V2 = "qta-2.0.0" -SUPPORTED_QTA_METHODS = {QTA_METHOD_V1, QTA_METHOD_V2} -V2_EXCHANGES_BY_MARKET = { - "KR": ("KOSPI", "KOSDAQ"), - "US": ("NYSE", "NASDAQ"), -} -V2_EXCHANGE_CONTRACTS = { - "KOSPI": { - "market": "KR", - "benchmark_id": "KOSPI_COMPOSITE", - "currency": "KRW", - "venue": "KRX", - }, - "KOSDAQ": { - "market": "KR", - "benchmark_id": "KOSDAQ_COMPOSITE", - "currency": "KRW", - "venue": "KRX", - }, - "NYSE": { - "market": "US", - "benchmark_id": "NYSE_COMPOSITE", - "currency": "USD", - "venue": "NYSE", - }, - "NASDAQ": { - "market": "US", - "benchmark_id": "NASDAQ_COMPOSITE", - "currency": "USD", - "venue": "NASD", - }, -} -V2_INSTRUMENT_FIELDS = { - "market", - "exchange", - "canonical_symbol", - "data_symbol", - "broker_symbol", - "instrument_type", - "benchmark_id", - "currency", - "venue", - "ticker_csv", - "benchmark_csv", - "tick_contract", - "source_name", - "ticker_csv_sha256", - "benchmark_csv_sha256", - "broker_tradability_verified", - "official_source_id", - "broker_source_id", -} -V2_TICK_CONTRACT_FIELDS = { - "schema", - "kind", - "rule_id", - "effective_date", - "reference_price", - "resolved_tick_size", -} -V2_INSTRUMENT_TYPES = {"COMMON", "ADR", "REIT"} -V2_SCREEN_FIELDS = { - "source_skill", - "schema", - "screen_status", - "method_version", - "selector_version", - "analysis_date", - "manifest_hash", - "selector_hash", - "blocked_count", - "blocked_fraction", - "instrument_count", - "selector", - "selected", - "decisions", - "screen_hash", -} -V2_SELECTOR_FIELDS = { - "selector_version", - "min_total_score", - "eligible_setup_statuses", - "top_k_by_exchange", - "min_selected_by_exchange", - "max_blocked_fraction", -} -V2_DECISION_FIELDS = { - "market", - "exchange", - "canonical_symbol", - "instrument", - "eligible", - "reasons", - "exchange_rank", - "selected", - "qta", -} -QTA_READY_FIELDS = { - "source_skill", - "result_schema", - "calculation_status", - "setup_status", - "method_version", - "analysis_date", - "market", - "ticker", - "source_name", - "shared_sessions", - "score_basis", - "short", - "medium", - "long", - "risk", - "entry_price", - "stop_price", - "take_profit_price", - "total_score", - "reference_observations", - "assumptions", -} -QTA2_READY_FIELDS = QTA_READY_FIELDS | { - "validation_status", - "liquidity", - "market_regime", -} -QTA2_LIQUIDITY_FIELDS = { - "median_20_session_turnover", - "minimum_turnover", - "currency", - "status", -} -QTA2_MARKET_REGIME_FIELDS = { - "metric", - "minimum_change_bps", - "max_age_seconds", - "status", -} -QTA_BLOCKED_FIELDS = { - "source_skill", - "result_schema", - "calculation_status", - "reason", - "method_version", - "market", - "ticker", -} -QTA_HORIZON_FIELDS = {"opinion", "score"} -QTA_RISK_FIELDS = {"score", "counterpoint"} -QTA_REFERENCE_FIELDS = { - "return_5", - "return_20", - "close_over_sma_20", - "sma_5_over_sma_20", - "signed_log_volume_ratio_20", - "rsi_14", - "return_63", - "close_over_sma_50", - "sma_20_over_sma_50", - "relative_return_63", - "adx_direction_14", - "return_126", - "return_252", - "close_over_sma_200", - "sma_50_over_sma_200", - "relative_return_252", - "ATR14/close", - "20d annualized volatility", - "252d maximum drawdown", - "60d gap p95", -} -QTA_SCORE_BASIS = "ticker-relative historical percentile; not probability of profit" -QTA2_SCORE_BASIS = ( - "ticker-relative historical percentiles aligned to first-hour " - "continuation; not probability of profit" -) -QTA_ASSUMPTIONS = [ - "input rows are finalized completed daily sessions", - "prices are corporate-action adjusted and aligned to analysis-date raw price scale", - "fees, tax, FX, slippage, position size, and execution are excluded", -] -QTA2_ASSUMPTIONS = [ - *QTA_ASSUMPTIONS, - "QTA 2.0 is research-only until multi-session walk-forward validation", - "same-session benchmark regime admission is required downstream", -] -QTA2_LIQUIDITY_FLOOR = { - "KR": Decimal("1000000000"), - "US": Decimal("1000000"), -} -QTA2_REGIME_METRIC = "same_session_previous_close_return_bps" -QTA2_REGIME_MINIMUM_CHANGE_BPS = Decimal("0") -QTA2_REGIME_MAX_AGE_SECONDS = 90 -QTA_SETUP_STATUSES = {"READY", "CONDITIONAL"} -QTA_OPINIONS = {"강한 긍정", "긍정", "중립", "부정", "강한 부정"} -QTA_MIN_SHARED_SESSIONS = 756 -QTA_MIN_REFERENCE_OBSERVATIONS = 252 -QTA_MAX_REFERENCE_OBSERVATIONS = 756 -ENTRY_WINDOWS = { - "KR": { - "timezone": "Asia/Seoul", - "start": (9, 0), - "end": (10, 0), - }, - "US": { - "timezone": "America/New_York", - "start": (9, 30), - "end": (10, 30), - }, -} -MARKET_SESSION_SOURCE_FIELDS = { - "schema", - "provider", - "source_id", - "source_as_of", - "market", - "timezone", - "session_date", - "previous_session_date", - "scheduled_status", - "regular_open", - "regular_close", -} -MARKET_SESSION_FIELDS = MARKET_SESSION_SOURCE_FIELDS | { - "source_path", - "source_sha256", - "session_hash", -} -MAX_SNAPSHOT_AGE_SECONDS = 3600 - -TERMINAL_STATES = { - "FILLED", - "CANCELLED", - "REJECTED", - "MANUAL_BLOCK", -} -TRANSITIONS = { - "PLANNED": {"WAIT_TRIGGER", "MANUAL_BLOCK"}, - "WAIT_TRIGGER": {"RESERVED", "CANCELLED", "MANUAL_BLOCK"}, - "RESERVED": {"SUBMITTING", "CANCELLED", "MANUAL_BLOCK"}, - "SUBMITTING": {"ACKNOWLEDGED", "UNKNOWN", "REJECTED"}, - "ACKNOWLEDGED": { - "PARTIALLY_FILLED", - "FILLED", - "CANCEL_PENDING", - "CANCELLED", - "REJECTED", - "UNKNOWN", - }, - "PARTIALLY_FILLED": { - "PARTIALLY_FILLED", - "FILLED", - "CANCEL_PENDING", - "CANCELLED", - "UNKNOWN", - }, - "CANCEL_PENDING": { - "CANCELLED", - "FILLED", - "UNKNOWN", - }, - "UNKNOWN": {"RECONCILING", "MANUAL_BLOCK"}, - "RECONCILING": { - "ACKNOWLEDGED", - "PARTIALLY_FILLED", - "FILLED", - "CANCELLED", - "REJECTED", - "MANUAL_BLOCK", - }, -} - - -class BlockedError(ValueError): - """Raised when an execution input is incomplete or unsafe.""" - - -def canonical_json(value: Any) -> str: - return json.dumps( - value, - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - allow_nan=False, - ) - - -def sha256_json(value: Any) -> str: - return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest() - - -def sha256_file(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def decimal_value(value: Any, field: str) -> Decimal: - if isinstance(value, bool): - raise BlockedError(f"{field} must be a decimal string or number") - try: - result = Decimal(str(value)) - except (InvalidOperation, ValueError) as exc: - raise BlockedError(f"{field} must be a decimal string or number") from exc - if not result.is_finite(): - raise BlockedError(f"{field} must be finite") - return result - - -def nonnegative_decimal(value: Any, field: str) -> Decimal: - result = decimal_value(value, field) - if result < 0: - raise BlockedError(f"{field} must be >= 0") - return result - - -def positive_decimal(value: Any, field: str) -> Decimal: - result = decimal_value(value, field) - if result <= 0: - raise BlockedError(f"{field} must be > 0") - return result - - -def exact_fields(value: dict[str, Any], required: set[str], label: str) -> None: - if set(value) != required: - missing = sorted(required - set(value)) - extra = sorted(set(value) - required) - raise BlockedError(f"{label} fields mismatch; missing={missing}, extra={extra}") - - -def parse_aware_datetime(value: Any, field: str) -> datetime: - try: - parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) - except ValueError as exc: - raise BlockedError(f"{field} must be an ISO-8601 datetime") from exc - if parsed.tzinfo is None or parsed.utcoffset() is None: - raise BlockedError(f"{field} must include a UTC offset") - return parsed - - -def parse_iso_date(value: Any, field: str) -> date: - if not isinstance(value, str): - raise BlockedError(f"{field} must be YYYY-MM-DD") - try: - parsed = date.fromisoformat(value) - except ValueError as exc: - raise BlockedError(f"{field} must be YYYY-MM-DD") from exc - if parsed.isoformat() != value: - raise BlockedError(f"{field} must be YYYY-MM-DD") - return parsed - - -def load_json_object(path: str | Path) -> dict[str, Any]: - source = Path(path) - value = json.loads(source.read_text(encoding="utf-8")) - if not isinstance(value, dict): - raise BlockedError(f"{source} must contain a JSON object") - return value - - -def emit_json(value: dict[str, Any], output: str | None = None) -> None: - rendered = json.dumps( - value, ensure_ascii=False, sort_keys=True, indent=2, allow_nan=False - ) - if output: - Path(output).write_text(rendered + "\n", encoding="utf-8") - print(rendered) - - -def normalize_symbol(market: str, symbol: str) -> str: - normalized_market = market.upper() - normalized = symbol.strip().upper() - if normalized_market == "KR": - if normalized.isdigit() and len(normalized) <= 6: - return normalized.zfill(6) - if len(normalized) == 6 and all( - character in "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" - for character in normalized - ): - return normalized - raise BlockedError(f"invalid KR symbol: {symbol!r}") - if normalized_market == "US": - if not normalized or any( - character not in "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.-" - for character in normalized - ): - raise BlockedError(f"invalid US symbol: {symbol!r}") - return normalized - raise BlockedError(f"unsupported market: {market!r}") - - -def normalized_risk_policy(value: dict[str, Any]) -> dict[str, Any]: - required = { - "schema", - "base_currency", - "per_trade_risk_krw", - "max_symbol_notional_krw", - "max_concurrent_positions", - "max_daily_loss_krw", - "round_trip_cost_bps", - "cash_buffer_bps", - "allow_existing_additions", - "allow_borrowed_cash", - "allow_margin", - "allow_short", - "allow_auto_fx", - "whole_shares_only", - } - exact_fields(value, required, "risk policy") - if value["schema"] != RISK_SCHEMA: - raise BlockedError(f"unsupported risk policy schema: {value['schema']!r}") - if value["base_currency"] != "KRW": - raise BlockedError("base_currency must be KRW") - for field in ( - "allow_existing_additions", - "allow_borrowed_cash", - "allow_margin", - "allow_short", - "allow_auto_fx", - "whole_shares_only", - ): - if not isinstance(value[field], bool): - raise BlockedError(f"{field} must be boolean") - if value["allow_borrowed_cash"]: - raise BlockedError("borrowed cash is unsupported by this execution contract") - if value["allow_margin"] or value["allow_short"] or value["allow_auto_fx"]: - raise BlockedError("margin, shorting, and automatic FX must remain disabled") - if not value["whole_shares_only"]: - raise BlockedError("initial execution contract requires whole shares") - count = value["max_concurrent_positions"] - if isinstance(count, bool) or not isinstance(count, int) or count <= 0: - raise BlockedError("max_concurrent_positions must be a positive integer") - for field in ( - "per_trade_risk_krw", - "max_symbol_notional_krw", - "max_daily_loss_krw", - ): - positive_decimal(value[field], field) - for field in ("round_trip_cost_bps", "cash_buffer_bps"): - amount = nonnegative_decimal(value[field], field) - if amount > Decimal(10000): - raise BlockedError(f"{field} must be <= 10000") - return { - **value, - "per_trade_risk_krw": format( - positive_decimal(value["per_trade_risk_krw"], "per_trade_risk_krw"), - "f", - ), - "max_symbol_notional_krw": format( - positive_decimal( - value["max_symbol_notional_krw"], "max_symbol_notional_krw" - ), - "f", - ), - "max_daily_loss_krw": format( - positive_decimal(value["max_daily_loss_krw"], "max_daily_loss_krw"), - "f", - ), - "round_trip_cost_bps": format( - nonnegative_decimal(value["round_trip_cost_bps"], "round_trip_cost_bps"), - "f", - ), - "cash_buffer_bps": format( - nonnegative_decimal(value["cash_buffer_bps"], "cash_buffer_bps"), - "f", - ), - } - - -def canonical_market_session_hash(value: dict[str, Any]) -> str: - """Hash a market-session artifact without its self-referential hash.""" - try: - return sha256_json( - {key: item for key, item in value.items() if key != "session_hash"} - ) - except (TypeError, ValueError) as exc: - raise BlockedError("market_session must contain canonical JSON values") from exc - - -def market_session_from_source(path: str | Path) -> dict[str, Any]: - source_path = Path(path).resolve() - try: - source_payload = load_json_object(source_path) - source_sha256 = sha256_file(source_path) - except (OSError, json.JSONDecodeError) as exc: - raise BlockedError( - "market-session source must be a readable JSON snapshot" - ) from exc - exact_fields( - source_payload, - MARKET_SESSION_SOURCE_FIELDS, - "market-session source snapshot", - ) - result = { - **source_payload, - "source_path": str(source_path), - "source_sha256": source_sha256, - } - result["session_hash"] = canonical_market_session_hash(result) - return result - - -def normalized_market_session( - value: Any, - *, - expected_market: str, - expected_timezone: str, -) -> dict[str, Any]: - if not isinstance(value, dict): - raise BlockedError("market_session must be an object") - exact_fields(value, MARKET_SESSION_FIELDS, "market_session") - if value["schema"] != MARKET_SESSION_SCHEMA: - raise BlockedError(f"market_session.schema must be {MARKET_SESSION_SCHEMA}") - provider = validate_nonempty_string(value["provider"], "market_session.provider") - source_id = validate_nonempty_string(value["source_id"], "market_session.source_id") - if value["market"] != expected_market: - raise BlockedError(f"market_session.market must be {expected_market}") - if value["timezone"] != expected_timezone: - raise BlockedError(f"market_session.timezone must be {expected_timezone}") - if value["scheduled_status"] != "OPEN": - raise BlockedError("market_session.scheduled_status must be OPEN") - - session_date = parse_iso_date(value["session_date"], "market_session.session_date") - previous_session_date = parse_iso_date( - value["previous_session_date"], - "market_session.previous_session_date", - ) - if session_date.weekday() >= 5: - raise BlockedError("market_session.session_date must be a weekday") - if previous_session_date >= session_date: - raise BlockedError( - "market_session.previous_session_date must precede session_date" - ) - - zone = ZoneInfo(expected_timezone) - regular_open = parse_aware_datetime( - value["regular_open"], "market_session.regular_open" - ) - regular_close = parse_aware_datetime( - value["regular_close"], "market_session.regular_close" - ) - source_as_of = parse_aware_datetime( - value["source_as_of"], "market_session.source_as_of" - ) - for supplied, localized, field in ( - (regular_open, regular_open.astimezone(zone), "regular_open"), - (regular_close, regular_close.astimezone(zone), "regular_close"), - ): - if ( - supplied.replace(tzinfo=None) != localized.replace(tzinfo=None) - or supplied.utcoffset() != localized.utcoffset() - ): - raise BlockedError( - "market_session." - f"{field} local clock and UTC offset must match {expected_timezone}" - ) - if supplied.isoformat() != value[field]: - raise BlockedError( - f"market_session.{field} must be a canonical ISO-8601 datetime" - ) - if localized.date() != session_date: - raise BlockedError(f"market_session.{field} must fall on session_date") - if regular_close <= regular_open: - raise BlockedError("market_session.regular_close must be after regular_open") - if source_as_of.isoformat() != value["source_as_of"]: - raise BlockedError( - "market_session.source_as_of must be a canonical ISO-8601 datetime" - ) - if source_as_of > regular_open: - raise BlockedError("market_session.source_as_of must not be after regular_open") - - source_path_text = validate_nonempty_string( - value["source_path"], "market_session.source_path" - ) - source_path = Path(source_path_text) - if not source_path.is_absolute(): - raise BlockedError("market_session.source_path must be absolute") - validate_sha256(value["source_sha256"], "market_session.source_sha256") - try: - actual_source_sha256 = sha256_file(source_path) - source_payload = load_json_object(source_path) - except (OSError, json.JSONDecodeError) as exc: - raise BlockedError( - "market_session.source_path must be a readable JSON snapshot" - ) from exc - if actual_source_sha256 != value["source_sha256"]: - raise BlockedError("market_session.source_sha256 does not match source_path") - exact_fields( - source_payload, - MARKET_SESSION_SOURCE_FIELDS, - "market_session source snapshot", - ) - expected_source_payload = {key: value[key] for key in MARKET_SESSION_SOURCE_FIELDS} - if source_payload != expected_source_payload: - raise BlockedError( - "market_session fields do not match the hashed source snapshot" - ) - - validate_sha256(value["session_hash"], "market_session.session_hash") - if value["session_hash"] != canonical_market_session_hash(value): - raise BlockedError( - "market_session.session_hash does not match canonical contents" - ) - return { - **value, - "provider": provider, - "source_id": source_id, - "source_as_of": source_as_of.isoformat(), - "regular_open": regular_open.isoformat(), - "regular_close": regular_close.isoformat(), - } - - -def normalized_execution_policy(value: dict[str, Any]) -> dict[str, Any]: - required = { - "schema", - "market", - "timezone", - "entry_window_start", - "entry_window_end", - "market_session", - "snapshot_max_age_seconds", - "poll_interval_seconds", - "quote_max_age_seconds", - "max_spread_bps", - "max_gap_bps", - "trigger_mode", - "order_ttl_seconds", - "order_type", - "time_in_force", - "allow_partial_fill", - "cancel_remainder_at_window_end", - } - exact_fields(value, required, "execution policy") - if value["schema"] != EXECUTION_POLICY_SCHEMA: - raise BlockedError(f"unsupported execution policy schema: {value['schema']!r}") - market = str(value["market"]).upper() - if market not in MARKET_CURRENCY: - raise BlockedError("market must be KR or US") - window_contract = ENTRY_WINDOWS[market] - expected_timezone = window_contract["timezone"] - if value["timezone"] != expected_timezone: - raise BlockedError(f"{market} timezone must be {expected_timezone}") - start = parse_aware_datetime(value["entry_window_start"], "entry_window_start") - end = parse_aware_datetime(value["entry_window_end"], "entry_window_end") - if start >= end: - raise BlockedError("entry window start must be before end") - zone = ZoneInfo(expected_timezone) - start_local = start.astimezone(zone) - end_local = end.astimezone(zone) - for supplied, localized, field in ( - (start, start_local, "entry_window_start"), - (end, end_local, "entry_window_end"), - ): - if ( - supplied.replace(tzinfo=None) != localized.replace(tzinfo=None) - or supplied.utcoffset() != localized.utcoffset() - ): - raise BlockedError( - f"{field} local clock and UTC offset must match {expected_timezone}" - ) - expected_start = window_contract["start"] - expected_end = window_contract["end"] - if ( - (start_local.hour, start_local.minute) != expected_start - or start_local.second != 0 - or start_local.microsecond != 0 - or (end_local.hour, end_local.minute) != expected_end - or end_local.second != 0 - or end_local.microsecond != 0 - or start_local.date() != end_local.date() - ): - raise BlockedError( - f"{market} entry window must be " - f"{expected_start[0]:02d}:{expected_start[1]:02d}-" - f"{expected_end[0]:02d}:{expected_end[1]:02d} {expected_timezone}" - ) - if (end - start).total_seconds() != 3600: - raise BlockedError("entry window must be exactly one hour") - market_session = normalized_market_session( - value["market_session"], - expected_market=market, - expected_timezone=expected_timezone, - ) - regular_open = parse_aware_datetime( - market_session["regular_open"], "market_session.regular_open" - ) - regular_close = parse_aware_datetime( - market_session["regular_close"], "market_session.regular_close" - ) - if start != regular_open: - raise BlockedError("entry_window_start must equal market_session.regular_open") - if end != regular_open + timedelta(hours=1): - raise BlockedError( - "entry_window_end must equal one hour after market_session.regular_open" - ) - if end > regular_close: - raise BlockedError( - "entry_window_end must not exceed market_session.regular_close" - ) - for field in ( - "snapshot_max_age_seconds", - "poll_interval_seconds", - "quote_max_age_seconds", - "order_ttl_seconds", - ): - amount = value[field] - if isinstance(amount, bool) or not isinstance(amount, int) or amount <= 0: - raise BlockedError(f"{field} must be a positive integer") - if value["snapshot_max_age_seconds"] > MAX_SNAPSHOT_AGE_SECONDS: - raise BlockedError( - f"snapshot_max_age_seconds must be <= {MAX_SNAPSHOT_AGE_SECONDS}" - ) - if value["order_type"] != "LIMIT" or value["time_in_force"] != "DAY": - raise BlockedError("initial execution contract requires LIMIT + DAY") - if value["trigger_mode"] not in {"AT_OR_ABOVE", "CROSS_FROM_BELOW"}: - raise BlockedError("trigger_mode must be AT_OR_ABOVE or CROSS_FROM_BELOW") - for field in ("allow_partial_fill", "cancel_remainder_at_window_end"): - if not isinstance(value[field], bool): - raise BlockedError(f"{field} must be boolean") - for field in ("max_spread_bps", "max_gap_bps"): - amount = nonnegative_decimal(value[field], field) - if amount > Decimal(10000): - raise BlockedError(f"{field} must be <= 10000") - return { - **value, - "market": market, - "entry_window_start": start_local.isoformat(), - "entry_window_end": end_local.isoformat(), - "market_session": market_session, - "max_spread_bps": format( - nonnegative_decimal(value["max_spread_bps"], "max_spread_bps"), "f" - ), - "max_gap_bps": format( - nonnegative_decimal(value["max_gap_bps"], "max_gap_bps"), "f" - ), - } - - -def normalized_account_snapshot( - value: dict[str, Any], - *, - screen_schema: str = SCREEN_SCHEMA, -) -> dict[str, Any]: - required = { - "schema", - "broker", - "environment", - "account_alias", - "market", - "currency", - "as_of", - "settled_cash", - "borrowed_buying_power", - "fx_to_krw", - "positions", - "open_orders", - } - exact_fields(value, required, "account snapshot") - if value["schema"] != ACCOUNT_SCHEMA: - raise BlockedError(f"unsupported account snapshot schema: {value['schema']!r}") - market = str(value["market"]).upper() - if market not in MARKET_CURRENCY: - raise BlockedError("account market must be KR or US") - if value["currency"] != MARKET_CURRENCY[market]: - raise BlockedError( - f"{market} account currency must be {MARKET_CURRENCY[market]}" - ) - if value["environment"] not in {"paper", "shadow", "live"}: - raise BlockedError("account environment must be paper, shadow, or live") - broker = validate_nonempty_string(value["broker"], "broker") - account_alias = validate_nonempty_string(value["account_alias"], "account_alias") - as_of = parse_aware_datetime(value["as_of"], "account as_of") - settled = nonnegative_decimal(value["settled_cash"], "settled_cash") - borrowed = nonnegative_decimal( - value["borrowed_buying_power"], "borrowed_buying_power" - ) - if not isinstance(value["positions"], list) or not isinstance( - value["open_orders"], list - ): - raise BlockedError("positions and open_orders must be arrays") - raw_fx = value["fx_to_krw"] - if raw_fx is None: - if market != "US" or settled != 0 or value["positions"]: - raise BlockedError( - "fx_to_krw may be null only for an empty US account with " - "zero settled cash" - ) - fx: Decimal | None = None - else: - fx = positive_decimal(raw_fx, "fx_to_krw") - if screen_schema not in {SCREEN_SCHEMA, SCREEN_SCHEMA_V2}: - raise BlockedError(f"unsupported screen schema: {screen_schema!r}") - - positions: list[dict[str, Any]] = [] - for index, position in enumerate(value["positions"]): - label = f"account positions[{index}]" - if not isinstance(position, dict): - raise BlockedError(f"{label} must be an object") - if "market" not in position or "symbol" not in position: - raise BlockedError(f"{label} requires market and symbol") - position_market = str(position["market"]).upper() - if position_market != market: - raise BlockedError(f"{label}.market must match account market {market}") - symbol = normalize_symbol(position_market, str(position["symbol"])) - normalized_position = { - **position, - "market": position_market, - "symbol": symbol, - } - if screen_schema == SCREEN_SCHEMA_V2: - if "exchange" not in position: - raise BlockedError(f"{label} requires exchange for qta-screen/v2") - exchange = str(position["exchange"]).upper() - contract = V2_EXCHANGE_CONTRACTS.get(exchange) - if contract is None or contract["market"] != position_market: - raise BlockedError(f"{label}.exchange must match its market") - normalized_position["exchange"] = exchange - positions.append(normalized_position) - - open_orders: list[dict[str, Any]] = [] - for index, open_order in enumerate(value["open_orders"]): - label = f"account open_orders[{index}]" - if not isinstance(open_order, dict): - raise BlockedError(f"{label} must be an object") - required_order_fields = {"side", "market", "symbol"} - missing_order_fields = required_order_fields - set(open_order) - if missing_order_fields: - raise BlockedError( - f"{label} missing fields needed for exposure key: " - f"{sorted(missing_order_fields)}" - ) - side = str(open_order["side"]).upper() - if side not in {"BUY", "SELL"}: - raise BlockedError(f"{label}.side must be BUY or SELL") - order_market = str(open_order["market"]).upper() - if order_market != market: - raise BlockedError(f"{label}.market must match account market {market}") - symbol = normalize_symbol(order_market, str(open_order["symbol"])) - normalized_open_order = { - **open_order, - "side": side, - "market": order_market, - "symbol": symbol, - } - if screen_schema == SCREEN_SCHEMA_V2: - if "exchange" not in open_order: - raise BlockedError(f"{label} requires exchange for qta-screen/v2") - exchange = str(open_order["exchange"]).upper() - contract = V2_EXCHANGE_CONTRACTS.get(exchange) - if contract is None or contract["market"] != order_market: - raise BlockedError(f"{label}.exchange must match its market") - normalized_open_order["exchange"] = exchange - open_orders.append(normalized_open_order) - - positions.sort( - key=lambda item: ( - item["market"], - item.get("exchange", ""), - item["symbol"], - ) - ) - open_orders.sort( - key=lambda item: ( - item["market"], - item.get("exchange", ""), - item["symbol"], - item["side"], - ) - ) - return { - **value, - "broker": broker, - "account_alias": account_alias, - "market": market, - "as_of": as_of.isoformat(), - "settled_cash": format(settled, "f"), - "borrowed_buying_power": format(borrowed, "f"), - "fx_to_krw": format(fx, "f") if fx is not None else None, - "positions": positions, - "open_orders": open_orders, - } - - -def normalized_exposure_snapshot(value: dict[str, Any]) -> dict[str, Any]: - required = {"schema", "as_of", "positions"} - exact_fields(value, required, "exposure snapshot") - schema = value["schema"] - if schema not in {EXPOSURE_SCHEMA, EXPOSURE_SCHEMA_V2}: - raise BlockedError(f"unsupported exposure snapshot schema: {value['schema']!r}") - as_of = parse_aware_datetime(value["as_of"], "exposure as_of") - if not isinstance(value["positions"], list): - raise BlockedError("exposure positions must be an array") - positions: list[dict[str, Any]] = [] - required_position = { - "broker", - "market", - "symbol", - "quantity", - "market_value_krw", - } - if schema == EXPOSURE_SCHEMA_V2: - required_position = required_position | {"exchange"} - for index, position in enumerate(value["positions"]): - if not isinstance(position, dict): - raise BlockedError(f"positions[{index}] must be an object") - exact_fields(position, required_position, f"positions[{index}]") - market = str(position["market"]).upper() - symbol = normalize_symbol(market, str(position["symbol"])) - normalized_position = {**position, "market": market, "symbol": symbol} - if schema == EXPOSURE_SCHEMA_V2: - exchange = str(position["exchange"]).upper() - contract = V2_EXCHANGE_CONTRACTS.get(exchange) - if contract is None: - raise BlockedError( - f"positions[{index}].exchange is unsupported: {exchange!r}" - ) - if contract["market"] != market: - raise BlockedError( - f"positions[{index}] exchange and market are inconsistent" - ) - normalized_position["exchange"] = exchange - if position["quantity"] is not None: - nonnegative_decimal(position["quantity"], f"positions[{index}].quantity") - nonnegative_decimal( - position["market_value_krw"], - f"positions[{index}].market_value_krw", - ) - positions.append(normalized_position) - positions.sort( - key=lambda item: ( - item["market"], - item.get("exchange", ""), - item["symbol"], - item["broker"], - ) - ) - return {**value, "as_of": as_of.isoformat(), "positions": positions} - - -def validate_screen(screen: dict[str, Any]) -> None: - schema = screen.get("schema") - if schema not in {SCREEN_SCHEMA, SCREEN_SCHEMA_V2}: - raise BlockedError( - f"screen schema must be {SCREEN_SCHEMA} or {SCREEN_SCHEMA_V2}" - ) - if schema == SCREEN_SCHEMA_V2: - validate_v2_screen(screen) - return - if screen.get("screen_status") != "READY": - raise BlockedError("screen_status must be READY") - if screen.get("method_version") != "qta-1.0.0": - raise BlockedError("method_version must be qta-1.0.0") - if screen.get("selector_version") != "qta-screen-1.0.0": - raise BlockedError("selector_version must be qta-screen-1.0.0") - if not isinstance(screen.get("selected"), dict): - raise BlockedError("screen.selected must be an object") - if "screen_hash" not in screen: - raise BlockedError("screen_hash is required") - supplied_hash = screen["screen_hash"] - validate_sha256(supplied_hash, "screen_hash") - expected_hash = canonical_screen_hash(screen) - if supplied_hash != expected_hash: - raise BlockedError("screen_hash does not match canonical screen payload") - - -def validate_sha256(value: Any, field: str) -> None: - if ( - not isinstance(value, str) - or len(value) != 64 - or any(character not in "0123456789abcdef" for character in value) - ): - raise BlockedError(f"{field} must be a lowercase SHA-256 hex digest") - - -def canonical_screen_hash(screen: dict[str, Any]) -> str: - """Hash a screen's canonical content without its self-referential hash.""" - try: - return sha256_json( - {key: item for key, item in screen.items() if key != "screen_hash"} - ) - except (TypeError, ValueError) as exc: - raise BlockedError("screen must contain canonical JSON values") from exc - - -def validate_nonempty_string(value: Any, field: str) -> str: - if not isinstance(value, str) or not value or value != value.strip(): - raise BlockedError(f"{field} must be a non-empty trimmed string") - return value - - -def validate_positive_decimal_string(value: Any, field: str) -> Decimal: - if not isinstance(value, str) or value != value.strip(): - raise BlockedError(f"{field} must be a positive decimal string") - return positive_decimal(value, field) - - -def validate_decimal_string( - value: Any, - field: str, - *, - minimum: Decimal, - maximum: Decimal, -) -> Decimal: - if not isinstance(value, str) or value != value.strip(): - raise BlockedError(f"{field} must be a decimal string") - result = decimal_value(value, field) - if result < minimum or result > maximum: - raise BlockedError(f"{field} must be between {minimum} and {maximum}") - return result - - -def validate_nonnegative_integer(value: Any, field: str) -> int: - if isinstance(value, bool) or not isinstance(value, int) or value < 0: - raise BlockedError(f"{field} must be an integer >= 0") - return value - - -def validate_v2_selector(selector: Any) -> dict[str, Any]: - if not isinstance(selector, dict): - raise BlockedError("screen.selector must be an object") - exact_fields(selector, V2_SELECTOR_FIELDS, "screen.selector") - if selector["selector_version"] != V2_SELECTOR_VERSION: - raise BlockedError( - f"screen.selector.selector_version must be {V2_SELECTOR_VERSION}" - ) - validate_decimal_string( - selector["min_total_score"], - "screen.selector.min_total_score", - minimum=Decimal(0), - maximum=Decimal(100), - ) - validate_decimal_string( - selector["max_blocked_fraction"], - "screen.selector.max_blocked_fraction", - minimum=Decimal(0), - maximum=Decimal(1), - ) - statuses = selector["eligible_setup_statuses"] - if ( - not isinstance(statuses, list) - or not statuses - or any( - not isinstance(status, str) or not status or status != status.strip() - for status in statuses - ) - ): - raise BlockedError( - "screen.selector.eligible_setup_statuses must be a non-empty " - "trimmed string array" - ) - if statuses != sorted(set(statuses)): - raise BlockedError( - "screen.selector.eligible_setup_statuses must be sorted and unique" - ) - if not set(statuses).issubset(QTA_SETUP_STATUSES): - raise BlockedError( - "screen.selector.eligible_setup_statuses must be a non-empty " - "subset of READY and CONDITIONAL" - ) - exchanges = set(V2_EXCHANGE_CONTRACTS) - for field in ("top_k_by_exchange", "min_selected_by_exchange"): - mapping = selector[field] - if not isinstance(mapping, dict) or set(mapping) != exchanges: - raise BlockedError( - f"screen.selector.{field} must contain exactly " - + ", ".join(sorted(exchanges)) - ) - for exchange in sorted(exchanges): - top_k = validate_nonnegative_integer( - selector["top_k_by_exchange"][exchange], - f"screen.selector.top_k_by_exchange.{exchange}", - ) - minimum_selected = validate_nonnegative_integer( - selector["min_selected_by_exchange"][exchange], - f"screen.selector.min_selected_by_exchange.{exchange}", - ) - if minimum_selected > top_k: - raise BlockedError( - "screen.selector.min_selected_by_exchange." - f"{exchange} must be <= top_k_by_exchange.{exchange}" - ) - return selector - - -def validate_json_number( - value: Any, - field: str, - *, - minimum: Decimal, - maximum: Decimal | None = None, -) -> Decimal: - if isinstance(value, bool) or not isinstance(value, (int, float)): - raise BlockedError(f"{field} must be a JSON number") - parsed = decimal_value(value, field) - if parsed < minimum or (maximum is not None and parsed > maximum): - upper = "" if maximum is None else f" and <= {maximum}" - raise BlockedError(f"{field} must be >= {minimum}{upper}") - return parsed - - -def qta_allowed_opinions(score: Decimal) -> set[str]: - boundary_labels = { - Decimal(20): {"강한 부정", "부정"}, - Decimal(40): {"부정", "중립"}, - Decimal(60): {"중립", "긍정"}, - Decimal(80): {"긍정", "강한 긍정"}, - } - if score in boundary_labels: - return boundary_labels[score] - if score >= Decimal(80): - return {"강한 긍정"} - if score >= Decimal(60): - return {"긍정"} - if score >= Decimal(40): - return {"중립"} - if score >= Decimal(20): - return {"부정"} - return {"강한 부정"} - - -def validate_v2_qta_identity( - qta: dict[str, Any], - instrument: dict[str, Any], - label: str, -) -> None: - if ( - qta.get("source_skill") != "quant-stock-technical" - or qta.get("result_schema") != "quant-stock-technical/v1" - or qta.get("method_version") not in SUPPORTED_QTA_METHODS - ): - raise BlockedError(f"{label} provenance is invalid") - if qta.get("market") != instrument["market"]: - raise BlockedError(f"{label}.market must equal instrument.market") - if qta.get("ticker") != instrument["canonical_symbol"]: - raise BlockedError(f"{label}.ticker must equal instrument.canonical_symbol") - - -def validate_v2_ready_qta( - qta: dict[str, Any], - instrument: dict[str, Any], - label: str, - *, - screen_analysis_date: date, -) -> None: - method_version = qta.get("method_version") - exact_fields( - qta, - QTA2_READY_FIELDS - if method_version == QTA_METHOD_V2 - else QTA_READY_FIELDS, - label, - ) - validate_v2_qta_identity(qta, instrument, label) - if qta["calculation_status"] != "READY": - raise BlockedError(f"{label}.calculation_status must be READY") - qta_analysis_date = parse_iso_date(qta["analysis_date"], f"{label}.analysis_date") - if qta_analysis_date != screen_analysis_date: - raise BlockedError(f"{label}.analysis_date must equal screen.analysis_date") - source_name = validate_nonempty_string(qta["source_name"], f"{label}.source_name") - if source_name != instrument["source_name"]: - raise BlockedError(f"{label}.source_name must equal instrument.source_name") - shared_sessions = qta["shared_sessions"] - if ( - isinstance(shared_sessions, bool) - or not isinstance(shared_sessions, int) - or shared_sessions < QTA_MIN_SHARED_SESSIONS - ): - raise BlockedError( - f"{label}.shared_sessions must be an integer >= {QTA_MIN_SHARED_SESSIONS}" - ) - expected_score_basis = ( - QTA2_SCORE_BASIS - if method_version == QTA_METHOD_V2 - else QTA_SCORE_BASIS - ) - if qta["score_basis"] != expected_score_basis: - raise BlockedError(f"{label}.score_basis is invalid") - - horizon_scores: dict[str, Decimal] = {} - for horizon in ("short", "medium", "long"): - payload = qta[horizon] - horizon_label = f"{label}.{horizon}" - if not isinstance(payload, dict): - raise BlockedError(f"{horizon_label} must be an object") - exact_fields(payload, QTA_HORIZON_FIELDS, horizon_label) - horizon_score = validate_json_number( - payload["score"], - f"{horizon_label}.score", - minimum=Decimal(0), - maximum=Decimal(100), - ) - horizon_scores[horizon] = horizon_score - if payload["opinion"] not in qta_allowed_opinions(horizon_score): - raise BlockedError(f"{horizon_label}.opinion does not match its score") - - risk = qta["risk"] - if not isinstance(risk, dict): - raise BlockedError(f"{label}.risk must be an object") - exact_fields(risk, QTA_RISK_FIELDS, f"{label}.risk") - risk_score = validate_json_number( - risk["score"], - f"{label}.risk.score", - minimum=Decimal(0), - maximum=Decimal(100), - ) - validate_nonempty_string(risk["counterpoint"], f"{label}.risk.counterpoint") - - entry = validate_json_number( - qta["entry_price"], - f"{label}.entry_price", - minimum=Decimal(0), - ) - stop = validate_json_number( - qta["stop_price"], - f"{label}.stop_price", - minimum=Decimal(0), - ) - take_profit = validate_json_number( - qta["take_profit_price"], - f"{label}.take_profit_price", - minimum=Decimal(0), - ) - if stop <= 0 or not stop < entry < take_profit: - raise BlockedError( - f"{label} prices must satisfy stop_price < entry_price < take_profit_price" - ) - tick = validate_positive_decimal_string( - instrument["tick_contract"]["resolved_tick_size"], - f"{label}.instrument_tick_size", - ) - for price, field in ( - (entry, "entry_price"), - (stop, "stop_price"), - (take_profit, "take_profit_price"), - ): - if price % tick != 0: - raise BlockedError(f"{label}.{field} must align to the resolved tick") - expected_take_profit = entry + Decimal(2) * (entry - stop) - if take_profit != expected_take_profit: - raise BlockedError( - f"{label}.take_profit_price must equal the " - f"{method_version} 2R target" - ) - total_score = validate_json_number( - qta["total_score"], - f"{label}.total_score", - minimum=Decimal(0), - maximum=Decimal(100), - ) - if method_version == QTA_METHOD_V2: - recomputed_total = ( - Decimal("0.50") * horizon_scores["short"] - + Decimal("0.30") * horizon_scores["medium"] - + Decimal("0.20") * (Decimal(100) - risk_score) - ) - else: - recomputed_total = ( - Decimal("0.25") * horizon_scores["short"] - + Decimal("0.35") * horizon_scores["medium"] - + Decimal("0.40") * horizon_scores["long"] - - Decimal("0.20") * max(risk_score - Decimal(50), Decimal(0)) - ) - recomputed_total = min(Decimal(100), max(Decimal(0), recomputed_total)) - if abs(total_score - recomputed_total) > Decimal("0.02"): - raise BlockedError( - f"{label}.total_score does not match the {method_version} score formula" - ) - liquidity_ready = True - if method_version == QTA_METHOD_V2: - if qta["validation_status"] != "RESEARCH_ONLY": - raise BlockedError( - f"{label}.validation_status must be RESEARCH_ONLY" - ) - liquidity = qta["liquidity"] - if not isinstance(liquidity, dict): - raise BlockedError(f"{label}.liquidity must be an object") - exact_fields( - liquidity, - QTA2_LIQUIDITY_FIELDS, - f"{label}.liquidity", - ) - median_turnover = validate_json_number( - liquidity["median_20_session_turnover"], - f"{label}.liquidity.median_20_session_turnover", - minimum=Decimal(0), - ) - minimum_turnover = validate_json_number( - liquidity["minimum_turnover"], - f"{label}.liquidity.minimum_turnover", - minimum=Decimal(0), - ) - expected_currency = MARKET_CURRENCY[instrument["market"]] - if liquidity["currency"] != expected_currency: - raise BlockedError( - f"{label}.liquidity.currency must be {expected_currency}" - ) - expected_floor = QTA2_LIQUIDITY_FLOOR[instrument["market"]] - if minimum_turnover != expected_floor: - raise BlockedError( - f"{label}.liquidity.minimum_turnover does not match QTA 2.0" - ) - liquidity_ready = median_turnover >= minimum_turnover - expected_liquidity_status = "READY" if liquidity_ready else "BLOCKED" - if liquidity["status"] != expected_liquidity_status: - raise BlockedError( - f"{label}.liquidity.status does not match turnover" - ) - market_regime = qta["market_regime"] - if not isinstance(market_regime, dict): - raise BlockedError(f"{label}.market_regime must be an object") - exact_fields( - market_regime, - QTA2_MARKET_REGIME_FIELDS, - f"{label}.market_regime", - ) - if market_regime["metric"] != QTA2_REGIME_METRIC: - raise BlockedError(f"{label}.market_regime.metric is invalid") - minimum_change = validate_json_number( - market_regime["minimum_change_bps"], - f"{label}.market_regime.minimum_change_bps", - minimum=Decimal("-10000"), - maximum=Decimal("10000"), - ) - if minimum_change != QTA2_REGIME_MINIMUM_CHANGE_BPS: - raise BlockedError( - f"{label}.market_regime.minimum_change_bps does not match " - "QTA 2.0" - ) - if ( - isinstance(market_regime["max_age_seconds"], bool) - or market_regime["max_age_seconds"] - != QTA2_REGIME_MAX_AGE_SECONDS - ): - raise BlockedError( - f"{label}.market_regime.max_age_seconds does not match QTA 2.0" - ) - if market_regime["status"] != "REQUIRED": - raise BlockedError( - f"{label}.market_regime.status must be REQUIRED" - ) - allowed_setup_statuses = ( - {"READY", "CONDITIONAL"} - if total_score == Decimal(60) - else {"READY"} - if total_score > Decimal(60) - else {"CONDITIONAL"} - ) - if not liquidity_ready: - allowed_setup_statuses = {"CONDITIONAL"} - if qta["setup_status"] not in allowed_setup_statuses: - raise BlockedError(f"{label}.setup_status does not match total_score") - - references = qta["reference_observations"] - if not isinstance(references, dict): - raise BlockedError(f"{label}.reference_observations must be an object") - exact_fields( - references, - QTA_REFERENCE_FIELDS, - f"{label}.reference_observations", - ) - maximum_reference_count = min( - QTA_MAX_REFERENCE_OBSERVATIONS, - shared_sessions - 1, - ) - for feature in sorted(QTA_REFERENCE_FIELDS): - count = references[feature] - if ( - isinstance(count, bool) - or not isinstance(count, int) - or count < QTA_MIN_REFERENCE_OBSERVATIONS - or count > maximum_reference_count - ): - raise BlockedError( - f"{label}.reference_observations.{feature} must be an integer " - f"between {QTA_MIN_REFERENCE_OBSERVATIONS} and " - f"{maximum_reference_count}" - ) - expected_assumptions = ( - QTA2_ASSUMPTIONS - if method_version == QTA_METHOD_V2 - else QTA_ASSUMPTIONS - ) - if qta["assumptions"] != expected_assumptions: - raise BlockedError( - f"{label}.assumptions do not match {method_version}" - ) - - -def validate_v2_blocked_qta( - qta: dict[str, Any], - instrument: dict[str, Any], - label: str, -) -> None: - exact_fields(qta, QTA_BLOCKED_FIELDS, label) - validate_v2_qta_identity(qta, instrument, label) - if qta["calculation_status"] != "BLOCKED": - raise BlockedError(f"{label}.calculation_status must be BLOCKED") - validate_nonempty_string(qta["reason"], f"{label}.reason") - - -def validate_v2_qta( - qta: dict[str, Any], - instrument: dict[str, Any], - label: str, - *, - screen_analysis_date: date, - require_ready: bool, -) -> None: - calculation_status = qta.get("calculation_status") - if calculation_status == "READY": - validate_v2_ready_qta( - qta, - instrument, - label, - screen_analysis_date=screen_analysis_date, - ) - return - if calculation_status == "BLOCKED" and not require_ready: - validate_v2_blocked_qta(qta, instrument, label) - return - expected = "READY" if require_ready else "READY or BLOCKED" - raise BlockedError(f"{label}.calculation_status must be {expected}") - - -def qta_ranking_key(qta: dict[str, Any]) -> tuple[Any, ...]: - if qta.get("method_version") == QTA_METHOD_V2: - return ( - -decimal_value(qta["total_score"], "qta.total_score"), - -decimal_value(qta["short"]["score"], "qta.short.score"), - -decimal_value(qta["medium"]["score"], "qta.medium.score"), - decimal_value(qta["risk"]["score"], "qta.risk.score"), - -decimal_value(qta["long"]["score"], "qta.long.score"), - str(qta["market"]), - str(qta["ticker"]), - ) - return ( - -decimal_value(qta["total_score"], "qta.total_score"), - -decimal_value(qta["medium"]["score"], "qta.medium.score"), - -decimal_value(qta["long"]["score"], "qta.long.score"), - -decimal_value(qta["short"]["score"], "qta.short.score"), - decimal_value(qta["risk"]["score"], "qta.risk.score"), - str(qta["market"]), - str(qta["ticker"]), - ) - - -def validate_v2_instrument( - instrument: dict[str, Any], - exchange: str, - label: str, - *, - analysis_date: date | None = None, -) -> dict[str, Any]: - exact_fields(instrument, V2_INSTRUMENT_FIELDS, label) - contract = V2_EXCHANGE_CONTRACTS[exchange] - for field in ("market", "benchmark_id", "currency", "venue"): - if instrument[field] != contract[field]: - raise BlockedError( - f"{label}.{field} must be {contract[field]} for {exchange}" - ) - if instrument["exchange"] != exchange: - raise BlockedError(f"{label}.exchange must be {exchange}") - if instrument["instrument_type"] not in V2_INSTRUMENT_TYPES: - raise BlockedError(f"{label}.instrument_type must be COMMON, ADR, or REIT") - market = contract["market"] - canonical_symbol = normalize_symbol(market, str(instrument["canonical_symbol"])) - broker_symbol = normalize_symbol(market, str(instrument["broker_symbol"])) - if canonical_symbol != instrument["canonical_symbol"]: - raise BlockedError(f"{label}.canonical_symbol must be normalized") - if broker_symbol != instrument["broker_symbol"]: - raise BlockedError(f"{label}.broker_symbol must be normalized") - validate_nonempty_string(instrument["data_symbol"], f"{label}.data_symbol") - for field in ( - "source_name", - "official_source_id", - "broker_source_id", - ): - validate_nonempty_string(instrument[field], f"{label}.{field}") - for field in ("ticker_csv", "benchmark_csv"): - path = validate_nonempty_string(instrument[field], f"{label}.{field}") - if not Path(path).is_absolute(): - raise BlockedError(f"{label}.{field} must be an absolute path") - if not isinstance(instrument["broker_tradability_verified"], bool): - raise BlockedError(f"{label}.broker_tradability_verified must be boolean") - if not instrument["broker_tradability_verified"]: - raise BlockedError(f"{label} is not broker-tradability verified") - validate_sha256(instrument["ticker_csv_sha256"], f"{label}.ticker_csv_sha256") - validate_sha256( - instrument["benchmark_csv_sha256"], - f"{label}.benchmark_csv_sha256", - ) - tick_contract = instrument["tick_contract"] - if not isinstance(tick_contract, dict): - raise BlockedError(f"{label}.tick_contract must be an object") - exact_fields(tick_contract, V2_TICK_CONTRACT_FIELDS, f"{label}.tick_contract") - if tick_contract["schema"] != "qta-tick-contract/v1": - raise BlockedError(f"{label}.tick_contract.schema must be qta-tick-contract/v1") - if tick_contract["kind"] != "RESOLVED_PRICE_LADDER": - raise BlockedError(f"{label}.tick_contract.kind must be RESOLVED_PRICE_LADDER") - validate_nonempty_string( - tick_contract["rule_id"], - f"{label}.tick_contract.rule_id", - ) - effective_date = validate_nonempty_string( - tick_contract["effective_date"], - f"{label}.tick_contract.effective_date", - ) - try: - parsed_effective_date = date.fromisoformat(effective_date) - except ValueError as exc: - raise BlockedError( - f"{label}.tick_contract.effective_date must be YYYY-MM-DD" - ) from exc - if parsed_effective_date.isoformat() != effective_date: - raise BlockedError(f"{label}.tick_contract.effective_date must be YYYY-MM-DD") - if analysis_date is not None and parsed_effective_date > analysis_date: - raise BlockedError( - f"{label}.tick_contract.effective_date must not be after analysis_date" - ) - validate_positive_decimal_string( - tick_contract["reference_price"], - f"{label}.tick_contract.reference_price", - ) - validate_positive_decimal_string( - tick_contract["resolved_tick_size"], - f"{label}.tick_contract.resolved_tick_size", - ) - return instrument - - -def validate_v2_selected( - selected: dict[str, Any], - *, - analysis_date: date, - method_version: str, -) -> dict[tuple[str, str], dict[str, Any]]: - expected_exchanges = set(V2_EXCHANGE_CONTRACTS) - if set(selected) != expected_exchanges: - raise BlockedError( - "qta-screen/v2 selected must contain exactly " - + ", ".join(sorted(expected_exchanges)) - ) - seen: dict[tuple[str, str], dict[str, Any]] = {} - for exchange in sorted(expected_exchanges): - candidates = selected[exchange] - if not isinstance(candidates, list): - raise BlockedError(f"screen.selected.{exchange} must be an array") - ranks: list[int] = [] - for index, selected_item in enumerate(candidates): - label = f"screen.selected.{exchange}[{index}]" - if not isinstance(selected_item, dict): - raise BlockedError(f"{label} must be an object") - exact_fields( - selected_item, - {"exchange_rank", "instrument", "qta"}, - label, - ) - exchange_rank = selected_item["exchange_rank"] - if ( - isinstance(exchange_rank, bool) - or not isinstance(exchange_rank, int) - or exchange_rank <= 0 - ): - raise BlockedError(f"{label}.exchange_rank must be a positive integer") - ranks.append(exchange_rank) - instrument = selected_item["instrument"] - qta = selected_item["qta"] - if not isinstance(instrument, dict) or not isinstance(qta, dict): - raise BlockedError(f"{label}.instrument and qta must be objects") - validate_v2_instrument( - instrument, - exchange, - f"{label}.instrument", - analysis_date=analysis_date, - ) - expected_market = V2_EXCHANGE_CONTRACTS[exchange]["market"] - if instrument["market"] != expected_market: - raise BlockedError( - f"{label}.instrument.market must be {expected_market}" - ) - validate_v2_qta( - qta, - instrument, - f"{label}.qta", - screen_analysis_date=analysis_date, - require_ready=True, - ) - if qta["method_version"] != method_version: - raise BlockedError( - f"{label}.qta method differs from screen.method_version" - ) - key = (exchange, instrument["broker_symbol"]) - if key in seen: - raise BlockedError( - f"duplicate selected broker symbol: {exchange}:{key[1]}" - ) - seen[key] = selected_item - if ranks != list(range(1, len(ranks) + 1)): - raise BlockedError( - f"screen.selected.{exchange} exchange_rank values must be " - "ordered and contiguous" - ) - return seen - - -def validate_v2_screen(screen: dict[str, Any]) -> None: - exact_fields(screen, V2_SCREEN_FIELDS, "qta-screen/v2") - if screen["source_skill"] != "quant-stock-technical": - raise BlockedError("qta-screen/v2 source_skill must be quant-stock-technical") - if screen["schema"] != SCREEN_SCHEMA_V2: - raise BlockedError(f"qta-screen/v2 schema must be {SCREEN_SCHEMA_V2}") - if screen["screen_status"] != "READY": - raise BlockedError("screen_status must be READY") - if screen["method_version"] not in SUPPORTED_QTA_METHODS: - raise BlockedError( - "method_version must be qta-1.0.0 or qta-2.0.0" - ) - if screen["selector_version"] != V2_SELECTOR_VERSION: - raise BlockedError(f"selector_version must be {V2_SELECTOR_VERSION}") - validate_sha256(screen["screen_hash"], "screen_hash") - if screen["screen_hash"] != canonical_screen_hash(screen): - raise BlockedError("screen_hash does not match canonical screen payload") - - analysis_date = parse_iso_date(screen["analysis_date"], "screen.analysis_date") - validate_sha256(screen["manifest_hash"], "screen.manifest_hash") - validate_sha256(screen["selector_hash"], "screen.selector_hash") - selector = validate_v2_selector(screen["selector"]) - if selector["selector_version"] != screen["selector_version"]: - raise BlockedError("screen selector versions are inconsistent") - if screen["selector_hash"] != sha256_json(selector): - raise BlockedError("selector_hash does not match screen.selector") - - blocked_count = validate_nonnegative_integer( - screen["blocked_count"], "screen.blocked_count" - ) - instrument_count = validate_nonnegative_integer( - screen["instrument_count"], "screen.instrument_count" - ) - if instrument_count <= 0: - raise BlockedError("screen.instrument_count must be positive") - if blocked_count > instrument_count: - raise BlockedError("screen.blocked_count must not exceed instrument_count") - blocked_fraction = validate_decimal_string( - screen["blocked_fraction"], - "screen.blocked_fraction", - minimum=Decimal(0), - maximum=Decimal(1), - ) - expected_blocked_fraction = Decimal(blocked_count) / Decimal(instrument_count) - if screen["blocked_fraction"] != format(expected_blocked_fraction, "f"): - raise BlockedError( - "screen.blocked_fraction does not match blocked_count/instrument_count" - ) - if blocked_fraction > Decimal(selector["max_blocked_fraction"]): - raise BlockedError("READY screen exceeds selector.max_blocked_fraction") - - selected = screen["selected"] - if not isinstance(selected, dict): - raise BlockedError("screen.selected must be an object") - selected_by_key = validate_v2_selected( - selected, - analysis_date=analysis_date, - method_version=screen["method_version"], - ) - decisions = screen["decisions"] - if not isinstance(decisions, list): - raise BlockedError("screen.decisions must be an array") - if len(decisions) != instrument_count: - raise BlockedError("screen.decisions length must equal instrument_count") - - statuses = set(selector["eligible_setup_statuses"]) - minimum_score = Decimal(selector["min_total_score"]) - decision_by_key: dict[tuple[str, str], dict[str, Any]] = {} - eligible_decisions: dict[str, list[dict[str, Any]]] = { - exchange: [] for exchange in V2_EXCHANGE_CONTRACTS - } - computed_blocked_count = 0 - for index, decision in enumerate(decisions): - label = f"screen.decisions[{index}]" - if not isinstance(decision, dict): - raise BlockedError(f"{label} must be an object") - exact_fields(decision, V2_DECISION_FIELDS, label) - exchange = decision["exchange"] - if exchange not in V2_EXCHANGE_CONTRACTS: - raise BlockedError(f"{label}.exchange is unsupported") - instrument = decision["instrument"] - qta = decision["qta"] - if not isinstance(instrument, dict) or not isinstance(qta, dict): - raise BlockedError(f"{label}.instrument and qta must be objects") - validate_v2_instrument( - instrument, - exchange, - f"{label}.instrument", - analysis_date=analysis_date, - ) - if decision["market"] != instrument["market"]: - raise BlockedError(f"{label}.market must equal instrument.market") - if decision["canonical_symbol"] != instrument["canonical_symbol"]: - raise BlockedError( - f"{label}.canonical_symbol must equal instrument.canonical_symbol" - ) - validate_v2_qta( - qta, - instrument, - f"{label}.qta", - screen_analysis_date=analysis_date, - require_ready=False, - ) - if qta["method_version"] != screen["method_version"]: - raise BlockedError( - f"{label}.qta method differs from screen.method_version" - ) - calculation_ready = qta["calculation_status"] == "READY" - if not calculation_ready: - computed_blocked_count += 1 - expected_reasons: list[str] = [] - if not calculation_ready: - expected_reasons.append("calculation_not_ready") - else: - if qta.get("setup_status") not in statuses: - expected_reasons.append("setup_status_ineligible") - if ( - decimal_value(qta.get("total_score"), f"{label}.qta.total_score") - < minimum_score - ): - expected_reasons.append("score_below_minimum") - reasons = decision["reasons"] - if ( - not isinstance(reasons, list) - or any(not isinstance(reason, str) for reason in reasons) - or reasons != expected_reasons - ): - raise BlockedError(f"{label}.reasons do not match the selector decision") - eligible = decision["eligible"] - selected_flag = decision["selected"] - if not isinstance(eligible, bool) or not isinstance(selected_flag, bool): - raise BlockedError(f"{label}.eligible and selected must be boolean") - if eligible != (not expected_reasons): - raise BlockedError(f"{label}.eligible does not match reasons") - exchange_rank = decision["exchange_rank"] - if eligible: - if ( - isinstance(exchange_rank, bool) - or not isinstance(exchange_rank, int) - or exchange_rank <= 0 - ): - raise BlockedError( - f"{label}.exchange_rank must be positive for an eligible decision" - ) - eligible_decisions[exchange].append(decision) - elif exchange_rank is not None: - raise BlockedError( - f"{label}.exchange_rank must be null for an ineligible decision" - ) - - key = (exchange, instrument["broker_symbol"]) - if key in decision_by_key: - raise BlockedError(f"duplicate decision broker symbol: {exchange}:{key[1]}") - decision_by_key[key] = decision - selected_item = selected_by_key.get(key) - if selected_flag != (selected_item is not None): - raise BlockedError(f"{label}.selected disagrees with screen.selected") - if selected_item is not None and ( - selected_item["exchange_rank"] != exchange_rank - or selected_item["instrument"] != instrument - or selected_item["qta"] != qta - ): - raise BlockedError( - f"{label} does not exactly match its screen.selected item" - ) - - if computed_blocked_count != blocked_count: - raise BlockedError("screen.blocked_count does not match calculation decisions") - if set(selected_by_key) - set(decision_by_key): - raise BlockedError("screen.selected contains an item absent from decisions") - for exchange in sorted(V2_EXCHANGE_CONTRACTS): - ranked = sorted( - eligible_decisions[exchange], - key=lambda decision: qta_ranking_key(decision["qta"]), - ) - for expected_rank, decision in enumerate(ranked, start=1): - if decision["exchange_rank"] != expected_rank: - raise BlockedError( - f"screen.decisions {exchange} exchange_rank does not match " - "qta ranking_key" - ) - minimum_selected = selector["min_selected_by_exchange"][exchange] - if len(ranked) < minimum_selected: - raise BlockedError( - f"READY screen does not meet minimum selection for {exchange}" - ) - expected_selected_count = min( - selector["top_k_by_exchange"][exchange], - len(ranked), - ) - if len(selected[exchange]) != expected_selected_count: - raise BlockedError( - f"screen.selected.{exchange} count does not match selector" - ) - expected_selected = ranked[:expected_selected_count] - for selected_item, expected_decision in zip( - selected[exchange], - expected_selected, - ): - if ( - selected_item["exchange_rank"] != expected_decision["exchange_rank"] - or selected_item["instrument"] != expected_decision["instrument"] - or selected_item["qta"] != expected_decision["qta"] - ): - raise BlockedError( - f"screen.selected.{exchange} order does not match qta ranking_key" - ) - - -def selected_for_execution( - screen: dict[str, Any], - market: str, -) -> list[dict[str, Any]]: - if screen["schema"] == SCREEN_SCHEMA: - selected = screen["selected"].get(market) - if not isinstance(selected, list): - raise BlockedError(f"screen.selected.{market} must be an array") - return sorted(selected, key=lambda item: int(item["rank"])) - - candidates: list[dict[str, Any]] = [] - for exchange_order, exchange in enumerate(V2_EXCHANGES_BY_MARKET[market], start=1): - for selected_item in screen["selected"][exchange]: - candidates.append( - { - **selected_item, - "_exchange": exchange, - "_exchange_order": exchange_order, - } - ) - candidates.sort( - key=lambda item: ( - item["_exchange_order"], - int(item["exchange_rank"]), - item["instrument"]["broker_symbol"], - ) - ) - return [ - {**item, "_execution_rank": index} - for index, item in enumerate(candidates, start=1) - ] - - -def round_down_to_tick(value: Decimal, tick: Decimal) -> Decimal: - if tick <= 0: - raise BlockedError("tick size must be positive") - return (value / tick).to_integral_value(rounding=ROUND_DOWN) * tick - - -def whole_quantity(value: Decimal) -> int: - return int(value.to_integral_value(rounding=ROUND_DOWN)) - - -def existing_exposure_keys( - exposure: dict[str, Any], -) -> set[tuple[str, str]]: - if exposure["schema"] == EXPOSURE_SCHEMA: - return { - (position["market"], position["symbol"]) - for position in exposure["positions"] - if decimal_value(position["market_value_krw"], "market_value_krw") > 0 - or ( - position["quantity"] is not None - and decimal_value(position["quantity"], "quantity") > 0 - ) - } - return { - (position["exchange"], position["symbol"]) - for position in exposure["positions"] - if decimal_value(position["market_value_krw"], "market_value_krw") > 0 - or ( - position["quantity"] is not None - and decimal_value(position["quantity"], "quantity") > 0 - ) - } - - -def account_position_keys( - account: dict[str, Any], - screen_schema: str, -) -> set[tuple[str, str]]: - if screen_schema == SCREEN_SCHEMA: - return { - (position["market"], position["symbol"]) - for position in account["positions"] - } - return { - (position["exchange"], position["symbol"]) for position in account["positions"] - } - - -def account_open_buy_keys( - account: dict[str, Any], - screen_schema: str, -) -> set[tuple[str, str]]: - open_buys = [ - open_order - for open_order in account["open_orders"] - if open_order["side"] == "BUY" - ] - if screen_schema == SCREEN_SCHEMA: - return { - (open_order["market"], open_order["symbol"]) for open_order in open_buys - } - return {(open_order["exchange"], open_order["symbol"]) for open_order in open_buys} - - -def validate_temporal_snapshots( - account: dict[str, Any], - exposure: dict[str, Any], - execution: dict[str, Any], -) -> None: - zone = ZoneInfo(execution["timezone"]) - window_start = parse_aware_datetime( - execution["entry_window_start"], "entry_window_start" - ) - market_session = execution["market_session"] - session_date = parse_iso_date( - market_session["session_date"], - "market_session.session_date", - ) - if window_start.astimezone(zone).date() != session_date: - raise BlockedError( - "entry_window_start must fall on market_session.session_date" - ) - account_as_of = parse_aware_datetime(account["as_of"], "account as_of") - exposure_as_of = parse_aware_datetime(exposure["as_of"], "exposure as_of") - if account_as_of != exposure_as_of: - raise BlockedError( - "account and exposure snapshots must share one frozen as_of instant" - ) - snapshot_local = account_as_of.astimezone(zone) - if snapshot_local.date() != session_date: - raise BlockedError( - "account and exposure snapshots must be from the entry session local date" - ) - if account_as_of > window_start: - raise BlockedError( - "account and exposure snapshots must be frozen no later than " - "entry_window_start" - ) - source_as_of = parse_aware_datetime( - market_session["source_as_of"], - "market_session.source_as_of", - ) - if source_as_of > account_as_of: - raise BlockedError( - "market_session.source_as_of must not be after the frozen snapshots" - ) - snapshot_age_seconds = (window_start - account_as_of).total_seconds() - if snapshot_age_seconds > execution["snapshot_max_age_seconds"]: - raise BlockedError("account snapshots exceed snapshot_max_age_seconds") - - -def validate_v2_analysis_date( - screen: dict[str, Any], - execution: dict[str, Any], -) -> None: - market_session = execution["market_session"] - analysis_date = parse_iso_date(screen["analysis_date"], "screen.analysis_date") - previous_session_date = parse_iso_date( - market_session["previous_session_date"], - "market_session.previous_session_date", - ) - if analysis_date != previous_session_date: - raise BlockedError( - "qta-screen/v2 analysis_date must equal " - "market_session.previous_session_date" - ) - - -def plan_orders( - screen_raw: dict[str, Any], - account_raw: dict[str, Any], - exposure_raw: dict[str, Any], - risk_raw: dict[str, Any], - execution_raw: dict[str, Any], -) -> dict[str, Any]: - validate_screen(screen_raw) - screen_schema = screen_raw["schema"] - account = normalized_account_snapshot( - account_raw, - screen_schema=screen_schema, - ) - exposure = normalized_exposure_snapshot(exposure_raw) - risk = normalized_risk_policy(risk_raw) - execution = normalized_execution_policy(execution_raw) - market = execution["market"] - if account["market"] != market: - raise BlockedError("account and execution markets differ") - expected_exposure_schema = ( - EXPOSURE_SCHEMA if screen_schema == SCREEN_SCHEMA else EXPOSURE_SCHEMA_V2 - ) - if exposure["schema"] != expected_exposure_schema: - raise BlockedError( - f"{screen_schema} requires exposure schema {expected_exposure_schema}" - ) - validate_temporal_snapshots(account, exposure, execution) - if screen_schema == SCREEN_SCHEMA_V2: - if account["broker"] != "kis": - raise BlockedError( - "qta-screen/v2 broker_symbol is KIS-qualified and requires " - "account broker kis; Toss requires a separate Toss-qualified " - "universe snapshot contract" - ) - validate_v2_analysis_date(screen_raw, execution) - selected = selected_for_execution(screen_raw, market) - - settled_cash = Decimal(account["settled_cash"]) - raw_fx = account["fx_to_krw"] - if raw_fx is None: - if settled_cash != 0: - raise BlockedError("a positive settled cash balance requires fx_to_krw") - per_trade_risk_native = Decimal(0) - max_notional_native = Decimal(0) - else: - fx = Decimal(raw_fx) - per_trade_risk_native = Decimal(risk["per_trade_risk_krw"]) / fx - max_notional_native = Decimal(risk["max_symbol_notional_krw"]) / fx - cost_bps = Decimal(risk["round_trip_cost_bps"]) - cash_buffer_bps = Decimal(risk["cash_buffer_bps"]) - gap_bps = Decimal(execution["max_gap_bps"]) - position_keys = existing_exposure_keys(exposure) - position_keys.update(account_position_keys(account, screen_schema)) - open_buy_keys = account_open_buy_keys(account, screen_schema) - occupied_keys = position_keys | open_buy_keys - - context = { - "execution_version": EXECUTION_VERSION, - "screen_hash": canonical_screen_hash(screen_raw), - "account_hash": sha256_json(account), - "exposure_hash": sha256_json(exposure), - "risk_hash": sha256_json(risk), - "execution_policy_hash": sha256_json(execution), - "execution_policy": execution, - "market_session": execution["market_session"], - "market_session_hash": execution["market_session"]["session_hash"], - "snapshot_max_age_seconds": execution["snapshot_max_age_seconds"], - "broker": account["broker"], - "environment": account["environment"], - "account_alias": account["account_alias"], - "market": market, - } - if screen_schema == SCREEN_SCHEMA_V2: - context.update( - { - "screen_schema": SCREEN_SCHEMA_V2, - "candidate_order_contract": ( - "exchange_contract_order_then_exchange_rank_then_broker_symbol" - ), - "exchange_order": list(V2_EXCHANGES_BY_MARKET[market]), - "broker_symbol_qualification": "KIS", - "analysis_date": screen_raw["analysis_date"], - "snapshot_as_of": account["as_of"], - } - ) - plan_seed = sha256_json(context) - intents: list[dict[str, Any]] = [] - skipped: list[dict[str, Any]] = [] - planned_keys: set[tuple[str, str]] = set() - cash_remaining = settled_cash - - for selected_item in selected: - is_v2 = screen_schema == SCREEN_SCHEMA_V2 - qta = selected_item.get("qta") - instrument = selected_item.get("instrument") - if not isinstance(qta, dict) or not isinstance(instrument, dict): - raise BlockedError("selected item must contain qta and instrument objects") - exchange = selected_item["_exchange"] if is_v2 else None - rank = ( - int(selected_item["_execution_rank"]) - if is_v2 - else int(selected_item["rank"]) - ) - symbol = normalize_symbol( - market, - str(instrument["broker_symbol"] if is_v2 else qta["ticker"]), - ) - if ( - qta.get("source_skill") != "quant-stock-technical" - or qta.get("result_schema") != "quant-stock-technical/v1" - or qta.get("calculation_status") != "READY" - or qta.get("method_version") != screen_raw["method_version"] - ): - raise BlockedError("selected QTA payload contract is invalid") - if is_v2: - if qta.get("setup_status") not in set( - screen_raw["selector"]["eligible_setup_statuses"] - ): - raise BlockedError( - "selected QTA setup_status is not allowed by the v2 selector" - ) - if qta.get("market") != market: - raise BlockedError("selected QTA market differs from execution market") - if qta.get("ticker") != instrument["canonical_symbol"]: - raise BlockedError("selected QTA ticker differs from canonical_symbol") - key = (str(exchange), symbol) - else: - if qta.get("setup_status") != "READY": - raise BlockedError("legacy selected QTA setup_status must be READY") - symbol = normalize_symbol(market, str(qta["ticker"])) - key = (market, symbol) - if key in planned_keys: - skipped_item = { - "market": market, - "symbol": symbol, - "reason": "duplicate_planned_exposure", - } - if is_v2: - skipped_item.update( - { - "exchange": exchange, - "exchange_rank": int(selected_item["exchange_rank"]), - "canonical_symbol": instrument["canonical_symbol"], - } - ) - skipped.append(skipped_item) - continue - if key in open_buy_keys: - skipped_item = { - "market": market, - "symbol": symbol, - "reason": "existing_open_buy_order", - } - if is_v2: - skipped_item.update( - { - "exchange": exchange, - "exchange_rank": int(selected_item["exchange_rank"]), - "canonical_symbol": instrument["canonical_symbol"], - } - ) - skipped.append(skipped_item) - continue - if key in position_keys and not risk["allow_existing_additions"]: - skipped_item = { - "market": market, - "symbol": symbol, - "reason": "existing_exposure", - } - if is_v2: - skipped_item.update( - { - "exchange": exchange, - "exchange_rank": int(selected_item["exchange_rank"]), - "canonical_symbol": instrument["canonical_symbol"], - } - ) - skipped.append(skipped_item) - continue - if ( - key not in occupied_keys - and len(occupied_keys | planned_keys) >= risk["max_concurrent_positions"] - ): - skipped_item = { - "market": market, - "symbol": symbol, - "reason": "max_concurrent_positions", - } - if is_v2: - skipped_item.update( - { - "exchange": exchange, - "exchange_rank": int(selected_item["exchange_rank"]), - "canonical_symbol": instrument["canonical_symbol"], - } - ) - skipped.append(skipped_item) - continue - entry = positive_decimal(qta["entry_price"], f"{symbol}.entry_price") - stop = positive_decimal(qta["stop_price"], f"{symbol}.stop_price") - if stop >= entry: - raise BlockedError(f"{symbol} stop must be below entry") - tick_value = ( - instrument["tick_contract"]["resolved_tick_size"] - if is_v2 - else instrument["tick_size"] - ) - tick = positive_decimal(tick_value, f"{symbol}.tick_size") - limit_price = round_down_to_tick( - entry * (Decimal(1) + gap_bps / Decimal(10000)), tick - ) - limit_price = max(limit_price, entry) - effective_loss = limit_price - stop + limit_price * cost_bps / Decimal(10000) - if effective_loss <= 0: - raise BlockedError(f"{symbol} effective loss per share is non-positive") - risk_quantity = whole_quantity(per_trade_risk_native / effective_loss) - notional_quantity = whole_quantity(max_notional_native / limit_price) - worst_case_unit = limit_price * (Decimal(1) + cash_buffer_bps / Decimal(10000)) - cash_quantity = whole_quantity(cash_remaining / worst_case_unit) - quantity = min(risk_quantity, notional_quantity, cash_quantity) - if quantity <= 0: - skipped.append( - {"market": market, "symbol": symbol, "reason": "quantity_below_one"} - ) - continue - reserved_cash = worst_case_unit * Decimal(quantity) - intent_seed = { - "plan_seed": plan_seed, - "rank": rank, - "market": market, - "symbol": symbol, - "side": "BUY", - "order_type": "LIMIT", - "time_in_force": "DAY", - "quantity": str(quantity), - "limit_price": format(limit_price, "f"), - } - if is_v2: - intent_seed.update( - { - "exchange": exchange, - "exchange_rank": int(selected_item["exchange_rank"]), - "canonical_symbol": instrument["canonical_symbol"], - "data_symbol": instrument["data_symbol"], - "broker_symbol": instrument["broker_symbol"], - "venue": instrument["venue"], - } - ) - intent_id = hashlib.sha256( - canonical_json(intent_seed).encode("utf-8") - ).hexdigest()[:32] - client_order_id = f"qta-{intent_id[:28]}" - intent = { - "intent_id": intent_id, - "client_order_id": client_order_id, - "rank": rank, - "market": market, - "symbol": symbol, - "currency": account["currency"], - "side": "BUY", - "order_type": "LIMIT", - "time_in_force": "DAY", - "quantity": str(quantity), - "entry_trigger": format(entry, "f"), - "limit_price": format(limit_price, "f"), - "stop_price": format(stop, "f"), - "take_profit_price": format( - positive_decimal( - qta["take_profit_price"], f"{symbol}.take_profit_price" - ), - "f", - ), - "effective_loss_per_share": format(effective_loss, "f"), - "reserved_cash": format(reserved_cash, "f"), - "qta_payload_hash": sha256_json(qta), - "initial_state": "PLANNED", - } - if is_v2: - intent.update( - { - "exchange": exchange, - "exchange_rank": int(selected_item["exchange_rank"]), - "canonical_symbol": instrument["canonical_symbol"], - "data_symbol": instrument["data_symbol"], - "broker_symbol": instrument["broker_symbol"], - "instrument_type": instrument["instrument_type"], - "benchmark_id": instrument["benchmark_id"], - "venue": instrument["venue"], - "resolved_tick_size": format(tick, "f"), - "tick_contract_hash": sha256_json(instrument["tick_contract"]), - } - ) - intent["intent_hash"] = sha256_json(intent) - intents.append(intent) - planned_keys.add(key) - cash_remaining -= reserved_cash - - output = { - "schema": PLAN_SCHEMA, - "plan_status": "READY" if intents else "NO_ORDERS", - "execution_version": EXECUTION_VERSION, - "context": context, - "entry_window": { - "timezone": execution["timezone"], - "start": execution["entry_window_start"], - "end": execution["entry_window_end"], - "poll_interval_seconds": execution["poll_interval_seconds"], - }, - "quote_policy": { - "max_age_seconds": execution["quote_max_age_seconds"], - "max_spread_bps": execution["max_spread_bps"], - "max_gap_bps": execution["max_gap_bps"], - "trigger_mode": execution["trigger_mode"], - }, - "order_policy": { - "ttl_seconds": execution["order_ttl_seconds"], - "allow_partial_fill": execution["allow_partial_fill"], - "cancel_remainder_at_window_end": execution[ - "cancel_remainder_at_window_end" - ], - }, - "settled_cash_start": format(settled_cash, "f"), - "borrowed_buying_power_excluded": account["borrowed_buying_power"], - "settled_cash_unreserved": format(cash_remaining, "f"), - "intents": intents, - "skipped": skipped, - "frozen_inputs": { - "screen": screen_raw, - "account": account, - "exposure": exposure, - "risk": risk, - "execution": execution, - }, - } - output["plan_hash"] = sha256_json(output) - return output - - -@dataclass(frozen=True) -class IntentRecord: - intent_id: str - state: str - broker_order_id: str | None - request_hash: str | None - payload: dict[str, Any] - - -class Ledger: - """SQLite ledger that persists intent state before broker mutation.""" - - def __init__(self, path: str | Path): - self.path = Path(path) - self.path.parent.mkdir(parents=True, exist_ok=True) - self.connection = sqlite3.connect(str(self.path)) - self.connection.execute("PRAGMA journal_mode=WAL") - self.connection.execute("PRAGMA synchronous=FULL") - self.connection.executescript( - """ - CREATE TABLE IF NOT EXISTS intents ( - intent_id TEXT PRIMARY KEY, - plan_hash TEXT NOT NULL, - state TEXT NOT NULL, - broker_order_id TEXT, - request_hash TEXT, - payload_json TEXT NOT NULL, - updated_at TEXT NOT NULL - ); - CREATE TABLE IF NOT EXISTS events ( - sequence INTEGER PRIMARY KEY AUTOINCREMENT, - intent_id TEXT NOT NULL, - event_type TEXT NOT NULL, - payload_json TEXT NOT NULL, - created_at TEXT NOT NULL - ); - """ - ) - self.connection.commit() - - def close(self) -> None: - self.connection.close() - - @contextmanager - def transaction(self) -> Iterator[sqlite3.Connection]: - self.connection.execute("BEGIN IMMEDIATE") - try: - yield self.connection - except BaseException: - self.connection.rollback() - raise - else: - self.connection.commit() - - def create_intent(self, plan_hash: str, intent: dict[str, Any]) -> None: - intent_id = str(intent["intent_id"]) - now = datetime.now(timezone.utc).isoformat() - payload_json = canonical_json(intent) - with self.transaction() as connection: - existing = connection.execute( - "SELECT plan_hash, payload_json FROM intents WHERE intent_id = ?", - (intent_id,), - ).fetchone() - if existing: - if existing != (plan_hash, payload_json): - raise BlockedError(f"intent collision: {intent_id}") - return - connection.execute( - """ - INSERT INTO intents - (intent_id, plan_hash, state, payload_json, updated_at) - VALUES (?, ?, 'PLANNED', ?, ?) - """, - (intent_id, plan_hash, payload_json, now), - ) - connection.execute( - """ - INSERT INTO events (intent_id, event_type, payload_json, created_at) - VALUES (?, 'INTENT_CREATED', ?, ?) - """, - (intent_id, canonical_json({"plan_hash": plan_hash}), now), - ) - - def get(self, intent_id: str) -> IntentRecord: - row = self.connection.execute( - """ - SELECT intent_id, state, broker_order_id, request_hash, payload_json - FROM intents WHERE intent_id = ? - """, - (intent_id,), - ).fetchone() - if row is None: - raise BlockedError(f"unknown intent: {intent_id}") - return IntentRecord( - intent_id=row[0], - state=row[1], - broker_order_id=row[2], - request_hash=row[3], - payload=json.loads(row[4]), - ) - - def transition( - self, - intent_id: str, - new_state: str, - event_payload: dict[str, Any], - *, - broker_order_id: str | None = None, - request_hash: str | None = None, - ) -> None: - current = self.get(intent_id) - allowed = TRANSITIONS.get(current.state, set()) - if new_state not in allowed: - raise BlockedError( - f"illegal intent transition: {current.state} -> {new_state}" - ) - now = datetime.now(timezone.utc).isoformat() - next_broker_id = broker_order_id or current.broker_order_id - next_request_hash = request_hash or current.request_hash - with self.transaction() as connection: - changed = connection.execute( - """ - UPDATE intents - SET state = ?, broker_order_id = ?, request_hash = ?, updated_at = ? - WHERE intent_id = ? AND state = ? - """, - ( - new_state, - next_broker_id, - next_request_hash, - now, - intent_id, - current.state, - ), - ).rowcount - if changed != 1: - raise BlockedError(f"concurrent intent mutation: {intent_id}") - connection.execute( - """ - INSERT INTO events (intent_id, event_type, payload_json, created_at) - VALUES (?, ?, ?, ?) - """, - ( - intent_id, - f"STATE_{new_state}", - canonical_json(event_payload), - now, - ), - ) - - def append_event( - self, intent_id: str, event_type: str, event_payload: dict[str, Any] - ) -> None: - self.append_events([(intent_id, event_type, event_payload)]) - - def append_events( - self, - events: list[tuple[str, str, dict[str, Any]]], - ) -> None: - """Persist non-mutation telemetry in one FULL-synchronous transaction.""" - if not events: - return - for intent_id, event_type, event_payload in events: - self.get(intent_id) - if not isinstance(event_type, str) or not event_type: - raise BlockedError("event_type must be a non-empty string") - if not isinstance(event_payload, dict): - raise BlockedError("event payload must be an object") - now = datetime.now(timezone.utc).isoformat() - with self.transaction() as connection: - connection.executemany( - """ - INSERT INTO events (intent_id, event_type, payload_json, created_at) - VALUES (?, ?, ?, ?) - """, - [ - ( - intent_id, - event_type, - canonical_json(event_payload), - now, - ) - for intent_id, event_type, event_payload in events - ], - ) - - def nonterminal(self) -> list[IntentRecord]: - rows = self.connection.execute( - """ - SELECT intent_id, state, broker_order_id, request_hash, payload_json - FROM intents ORDER BY intent_id - """ - ).fetchall() - return [ - IntentRecord(row[0], row[1], row[2], row[3], json.loads(row[4])) - for row in rows - if row[1] not in TERMINAL_STATES - ] - - def nonterminal_bindings(self) -> list[dict[str, str]]: - rows = self.connection.execute( - """ - SELECT intent_id, plan_hash, state - FROM intents ORDER BY intent_id - """ - ).fetchall() - return [ - { - "intent_id": str(row[0]), - "plan_hash": str(row[1]), - "state": str(row[2]), - } - for row in rows - if row[2] not in TERMINAL_STATES or row[2] == "MANUAL_BLOCK" - ] - - def events(self, intent_id: str) -> list[dict[str, Any]]: - rows = self.connection.execute( - """ - SELECT sequence, event_type, payload_json, created_at - FROM events WHERE intent_id = ? ORDER BY sequence - """, - (intent_id,), - ).fetchall() - return [ - { - "sequence": row[0], - "event_type": row[1], - "payload": json.loads(row[2]), - "created_at": row[3], - } - for row in rows - ] - - -class SingleWriterLock: - """Non-blocking process lock for one account/session state directory.""" - - def __init__(self, path: str | Path): - self.path = Path(path) - self.handle: Any = None - - def __enter__(self): - self.path.parent.mkdir(parents=True, exist_ok=True) - self.handle = self.path.open("a+", encoding="utf-8") - try: - fcntl.flock(self.handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) - except BlockingIOError as exc: - self.handle.close() - self.handle = None - raise BlockedError(f"another writer holds {self.path}") from exc - return self - - def __exit__( - self, - exc_type: object, - exc: object, - traceback: object, - ) -> None: - if self.handle is not None: - fcntl.flock(self.handle.fileno(), fcntl.LOCK_UN) - self.handle.close() - self.handle = None - - -def redact(value: Any) -> Any: - secret_fragments = ( - "secret", - "token", - "authorization", - "appkey", - "app_key", - "password", - "account_number", - "accountnumber", - "accountno", - "account_prefix", - "accountprefix", - "account_product", - "accountproduct", - "account_seq", - "accountseq", - "cano", - "acnt_no", - "acnt_prdt_cd", - "x-tossinvest-account", - ) - if isinstance(value, dict): - result: dict[str, Any] = {} - for key, item in value.items(): - lowered = key.lower() - result[key] = ( - "[REDACTED]" - if any(fragment in lowered for fragment in secret_fragments) - else redact(item) - ) - return result - if isinstance(value, list): - return [redact(item) for item in value] - return value - - -def self_test() -> None: - qta_payload = { - "source_skill": "quant-stock-technical", - "result_schema": "quant-stock-technical/v1", - "calculation_status": "READY", - "setup_status": "READY", - "method_version": "qta-1.0.0", - "ticker": "NEW", - "entry_price": 100, - "stop_price": 90, - "take_profit_price": 120, - } - screen = { - "schema": SCREEN_SCHEMA, - "screen_status": "READY", - "method_version": "qta-1.0.0", - "selector_version": "qta-screen-1.0.0", - "selected": { - "KR": [], - "US": [ - { - "rank": 1, - "instrument": { - "market": "US", - "ticker": "NEW", - "tick_size": "0.01", - }, - "qta": qta_payload, - } - ], - }, - } - screen["screen_hash"] = canonical_screen_hash(screen) - account = { - "schema": ACCOUNT_SCHEMA, - "broker": "kis", - "environment": "paper", - "account_alias": "paper-us", - "market": "US", - "currency": "USD", - "as_of": "2026-07-27T09:00:00-04:00", - "settled_cash": "200", - "borrowed_buying_power": "5000", - "fx_to_krw": "1400", - "positions": [], - "open_orders": [], - } - exposure = { - "schema": EXPOSURE_SCHEMA, - "as_of": "2026-07-27T09:00:00-04:00", - "positions": [], - } - risk = { - "schema": RISK_SCHEMA, - "base_currency": "KRW", - "per_trade_risk_krw": "15000", - "max_symbol_notional_krw": "280000", - "max_concurrent_positions": 1, - "max_daily_loss_krw": "28000", - "round_trip_cost_bps": "10", - "cash_buffer_bps": "100", - "allow_existing_additions": False, - "allow_borrowed_cash": False, - "allow_margin": False, - "allow_short": False, - "allow_auto_fx": False, - "whole_shares_only": True, - } - execution = { - "schema": EXECUTION_POLICY_SCHEMA, - "market": "US", - "timezone": "America/New_York", - "entry_window_start": "2026-07-27T09:30:00-04:00", - "entry_window_end": "2026-07-27T10:30:00-04:00", - "market_session": market_session_from_source( - Path(__file__).resolve().parents[1] - / "references" - / "fixtures" - / "us-market-session-2026-07-27.json" - ), - "snapshot_max_age_seconds": 1800, - "poll_interval_seconds": 2, - "quote_max_age_seconds": 5, - "max_spread_bps": "25", - "max_gap_bps": "20", - "trigger_mode": "AT_OR_ABOVE", - "order_ttl_seconds": 30, - "order_type": "LIMIT", - "time_in_force": "DAY", - "allow_partial_fill": True, - "cancel_remainder_at_window_end": True, - } - first = plan_orders(screen, account, exposure, risk, execution) - second = plan_orders(screen, account, exposure, risk, execution) - assert first == second - assert first["plan_status"] == "READY" - assert first["borrowed_buying_power_excluded"] == "5000" - assert len(first["intents"]) == 1 - assert first["intents"][0]["quantity"] == "1" - - empty_account = { - **account, - "settled_cash": "0", - "fx_to_krw": None, - } - empty_plan = plan_orders(screen, empty_account, exposure, risk, execution) - assert empty_plan["plan_status"] == "NO_ORDERS" - assert empty_plan["frozen_inputs"]["account"]["fx_to_krw"] is None - assert empty_plan["skipped"][0]["reason"] == "quantity_below_one" - - held = json.loads(canonical_json(exposure)) - held["positions"] = [ - { - "broker": "toss", - "market": "US", - "symbol": "NEW", - "quantity": "0.5", - "market_value_krw": "70000", - } - ] - no_orders = plan_orders(screen, account, held, risk, execution) - assert no_orders["plan_status"] == "NO_ORDERS" - assert no_orders["skipped"][0]["reason"] == "existing_exposure" - - with tempfile.TemporaryDirectory(prefix="qta-ledger-") as directory: - ledger = Ledger(Path(directory) / "ledger.sqlite3") - intent = first["intents"][0] - ledger.create_intent(first["plan_hash"], intent) - ledger.create_intent(first["plan_hash"], intent) - ledger.transition(intent["intent_id"], "WAIT_TRIGGER", {}) - ledger.transition(intent["intent_id"], "RESERVED", {}) - ledger.transition( - intent["intent_id"], - "SUBMITTING", - {"request_hash": "b" * 64}, - request_hash="b" * 64, - ) - ledger.transition(intent["intent_id"], "UNKNOWN", {"reason": "timeout"}) - ledger.transition(intent["intent_id"], "RECONCILING", {}) - ledger.transition(intent["intent_id"], "MANUAL_BLOCK", {"reason": "ambiguous"}) - assert ledger.get(intent["intent_id"]).state == "MANUAL_BLOCK" - assert len(ledger.events(intent["intent_id"])) == 7 - ledger.close() - - redacted = redact( - { - "authorization": "Bearer secret", - "nested": {"appsecret": "secret", "safe": "ok"}, - } - ) - assert redacted["authorization"] == "[REDACTED]" - assert redacted["nested"]["appsecret"] == "[REDACTED]" - assert redacted["nested"]["safe"] == "ok" - print( - canonical_json( - { - "self_test": "PASS", - "execution_version": EXECUTION_VERSION, - "plan_hash": first["plan_hash"], - } - ) - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--self-test", action="store_true") - return parser.parse_args() - - -def main() -> int: - args = parse_args() - if args.self_test: - self_test() - return 0 - emit_json( - { - "schema": PLAN_SCHEMA, - "plan_status": "BLOCKED", - "reason": "use plan_orders.py or pass --self-test", - } - ) - return 2 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/skills/quant-stock-polling-trader/scripts/freeze_market_session.py b/skills/quant-stock-polling-trader/scripts/freeze_market_session.py deleted file mode 100644 index b3116e7..0000000 --- a/skills/quant-stock-polling-trader/scripts/freeze_market_session.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python3 -"""Hash and validate one frozen market-session source JSON file.""" - -from __future__ import annotations - -import argparse -import sys - -from execution_core import ( - ENTRY_WINDOWS, - BlockedError, - emit_json, - market_session_from_source, - normalized_market_session, -) - - -def build(source: str) -> dict[str, object]: - session = market_session_from_source(source) - market = session.get("market") - if market not in ENTRY_WINDOWS: - raise BlockedError("market-session source market must be KR or US") - return normalized_market_session( - session, - expected_market=market, - expected_timezone=ENTRY_WINDOWS[market]["timezone"], - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--source") - parser.add_argument("--output") - parser.add_argument("--self-test", action="store_true") - return parser.parse_args() - - -def main() -> int: - args = parse_args() - source = args.source - if args.self_test: - from pathlib import Path - - source = str( - Path(__file__).resolve().parents[1] - / "references" - / "fixtures" - / "kr-market-session-2026-07-27.json" - ) - if source is None: - raise BlockedError("--source is required") - result = build(source) - if args.self_test: - result = { - "self_test": "PASS", - "schema": result["schema"], - "session_hash": result["session_hash"], - } - emit_json(result, args.output) - return 0 - - -if __name__ == "__main__": - try: - sys.exit(main()) - except (BlockedError, OSError, ValueError) as exc: - emit_json({"status": "BLOCKED", "reason": str(exc)}) - sys.exit(2) diff --git a/skills/quant-stock-polling-trader/scripts/market_calendar.py b/skills/quant-stock-polling-trader/scripts/market_calendar.py deleted file mode 100644 index 0b25f50..0000000 --- a/skills/quant-stock-polling-trader/scripts/market_calendar.py +++ /dev/null @@ -1,670 +0,0 @@ -#!/usr/bin/env python3 -"""Freeze an official KR or US equity-market calendar snapshot.""" - -from __future__ import annotations - -import argparse -import hashlib -import html -import json -import os -import re -import sys -import tempfile -from collections.abc import Callable -from datetime import date, datetime, time, timedelta -from html.parser import HTMLParser -from pathlib import Path -from typing import Any -from urllib.parse import urlencode -from urllib.request import Request, urlopen -from zoneinfo import ZoneInfo - -from execution_core import ( - ENTRY_WINDOWS, - BlockedError, - market_session_from_source, - normalized_market_session, -) - -RECEIPT_SCHEMA = "qta-market-calendar-receipt/v1" -KRX_HOLIDAY_PAGE = ( - "https://global.krx.co.kr/contents/GLB/05/0501/0501110000/" - "GLB0501110000.jsp" -) -KRX_HOURS_PAGE = ( - "https://global.krx.co.kr/contents/GLB/06/0602/0602020204/" - "GLB0602020204T1.jsp" -) -KRX_OTP_URL = "https://global.krx.co.kr/contents/COM/GenerateOTP.jspx" -KRX_DATA_URL = "https://global.krx.co.kr/contents/GLB/99/GLB99000001.jspx" -KRX_BLD = "GLB/05/0501/0501110000/glb0501110000_01" -NYSE_CALENDAR_URL = "https://www.nyse.com/markets/hours-calendars" -NASDAQ_CALENDAR_URL = "https://www.nasdaqtrader.com/Trader.aspx?id=calendar" -USER_AGENT = "qta-market-calendar/1.0 (+official read-only snapshot)" -MONTH_PATTERN = ( - r"January|February|March|April|May|June|July|August|" - r"September|October|November|December" -) -FULL_DATE = re.compile( - rf"\b({MONTH_PATTERN})\s+(\d{{1,2}}),\s+(\d{{4}})\b", - re.IGNORECASE, -) -MONTH_DAY = re.compile( - rf"\b({MONTH_PATTERN})\s+(\d{{1,2}})\b", - re.IGNORECASE, -) - - -class CalendarBlockedError(BlockedError): - """Raised when official calendar evidence cannot support one session.""" - - -class CalendarHtmlParser(HTMLParser): - """Collect text rows and block text without executing page scripts.""" - - def __init__(self) -> None: - super().__init__(convert_charrefs=True) - self.rows: list[list[str]] = [] - self.blocks: list[str] = [] - self._row: list[str] | None = None - self._cell: list[str] | None = None - self._block: list[str] | None = None - self._block_depth = 0 - - def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: - del attrs - tag = tag.lower() - if tag == "tr": - self._row = [] - elif tag in {"th", "td"} and self._row is not None: - self._cell = [] - if tag in {"p", "li"}: - if self._block is None: - self._block = [] - self._block_depth = 1 - else: - self._block_depth += 1 - - def handle_endtag(self, tag: str) -> None: - tag = tag.lower() - if tag in {"th", "td"} and self._row is not None and self._cell is not None: - self._row.append(normalize_text(" ".join(self._cell))) - self._cell = None - elif tag == "tr" and self._row is not None: - if any(self._row): - self.rows.append(self._row) - self._row = None - self._cell = None - if tag in {"p", "li"} and self._block is not None: - self._block_depth -= 1 - if self._block_depth == 0: - text = normalize_text(" ".join(self._block)) - if text: - self.blocks.append(text) - self._block = None - - def handle_data(self, data: str) -> None: - if self._cell is not None: - self._cell.append(data) - if self._block is not None: - self._block.append(data) - - -def normalize_text(value: str) -> str: - return " ".join(html.unescape(value).replace("\xa0", " ").split()) - - -def canonical_json(value: Any) -> str: - return json.dumps( - value, - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - allow_nan=False, - ) - - -def sha256_bytes(payload: bytes) -> str: - return hashlib.sha256(payload).hexdigest() - - -def sha256_file(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def atomic_write(path: Path, payload: bytes) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - descriptor, temporary = tempfile.mkstemp( - prefix=f".{path.name}.", suffix=".tmp", dir=str(path.parent) - ) - temporary_path = Path(temporary) - try: - with os.fdopen(descriptor, "wb") as handle: - handle.write(payload) - handle.flush() - os.fsync(handle.fileno()) - os.replace(temporary_path, path) - os.chmod(path, 0o600) - finally: - if temporary_path.exists(): - temporary_path.unlink() - - -def atomic_write_json(path: Path, value: Any) -> None: - atomic_write( - path, - (json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode( - "utf-8" - ), - ) - - -def fetch_bytes(url: str, data: bytes | None = None) -> bytes: - headers = { - "User-Agent": USER_AGENT, - "Accept": "application/json,text/html,application/xhtml+xml", - } - if data is not None: - headers["Content-Type"] = "application/x-www-form-urlencoded" - request = Request(url, data=data, headers=headers, method="POST" if data else "GET") - with urlopen(request, timeout=30) as response: - status = getattr(response, "status", 200) - if status != 200: - raise CalendarBlockedError(f"official calendar HTTP status {status}: {url}") - payload = response.read() - if not payload: - raise CalendarBlockedError(f"official calendar response is empty: {url}") - return payload - - -def parse_iso_date(value: str, label: str) -> date: - try: - return date.fromisoformat(value) - except (TypeError, ValueError) as exc: - raise CalendarBlockedError(f"{label} must be YYYY-MM-DD") from exc - - -def parsed_html(payload: bytes, label: str) -> CalendarHtmlParser: - try: - text = payload.decode("utf-8") - except UnicodeDecodeError as exc: - raise CalendarBlockedError(f"{label} is not UTF-8") from exc - parser = CalendarHtmlParser() - parser.feed(text) - parser.close() - if not parser.rows: - raise CalendarBlockedError(f"{label} contains no calendar table rows") - return parser - - -def full_date_from_match(match: re.Match[str]) -> date: - return datetime.strptime( - f"{match.group(1)} {match.group(2)}, {match.group(3)}", - "%B %d, %Y", - ).date() - - -def parse_nasdaq_calendar(payload: bytes, year: int) -> tuple[set[date], set[date]]: - parser = parsed_html(payload, "Nasdaq calendar") - closed: set[date] = set() - early: set[date] = set() - for row in parser.rows: - if len(row) < 3: - continue - match = FULL_DATE.search(row[0]) - if match is None: - continue - session = full_date_from_match(match) - if session.year != year: - continue - status = row[-1].lower() - if status == "closed": - closed.add(session) - elif re.fullmatch(r"1:00\s*p\.?m\.?", status): - early.add(session) - else: - raise CalendarBlockedError( - f"Nasdaq calendar has unknown status for {session}: {row[-1]}" - ) - if not closed: - raise CalendarBlockedError(f"Nasdaq calendar has no closed dates for {year}") - return closed, early - - -def parse_nyse_calendar(payload: bytes, year: int) -> tuple[set[date], set[date]]: - parser = parsed_html(payload, "NYSE calendar") - header: list[str] | None = None - closed: set[date] = set() - for row in parser.rows: - if "Holiday" in row and str(year) in row: - header = row - continue - if header is None or len(row) < len(header): - continue - year_index = header.index(str(year)) - value = row[year_index] - match = MONTH_DAY.search(value) - if match is None: - continue - closed.add( - datetime.strptime( - f"{match.group(1)} {match.group(2)}, {year}", - "%B %d, %Y", - ).date() - ) - early: set[date] = set() - for block in parser.blocks: - lowered = block.lower() - if "close early at 1:00 p.m." not in lowered: - continue - for match in FULL_DATE.finditer(block): - parsed = full_date_from_match(match) - if parsed.year == year: - early.add(parsed) - if not closed: - raise CalendarBlockedError(f"NYSE calendar has no closed dates for {year}") - if "Core Trading Session: 9:30 a.m. to 4:00 p.m. ET" not in normalize_text( - payload.decode("utf-8") - ): - raise CalendarBlockedError("NYSE core trading-hour contract is missing") - return closed, early - - -def parse_krx_holidays(payload: bytes, year: int) -> set[date]: - try: - decoded = json.loads(payload) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise CalendarBlockedError("KRX holiday payload is not valid JSON") from exc - if not isinstance(decoded, dict) or set(decoded) != {"block1"}: - raise CalendarBlockedError("KRX holiday payload fields changed") - rows = decoded["block1"] - if not isinstance(rows, list) or not rows: - raise CalendarBlockedError(f"KRX holiday payload is empty for {year}") - holidays: set[date] = set() - for index, row in enumerate(rows): - if not isinstance(row, dict) or "calnd_dd" not in row: - raise CalendarBlockedError(f"KRX holiday row {index} is malformed") - parsed = parse_iso_date(row["calnd_dd"], f"KRX holiday row {index}.calnd_dd") - if parsed.year != year: - raise CalendarBlockedError("KRX holiday payload contains another year") - holidays.add(parsed) - return holidays - - -def validate_krx_hours(payload: bytes) -> None: - text = normalize_text(payload.decode("utf-8")) - if "Regular market session" not in text or "09:00 - 15:30" not in text: - raise CalendarBlockedError("KRX regular trading-hour contract is missing") - - -def validate_krx_holiday_page(payload: bytes) -> None: - try: - text = payload.decode("utf-8") - except UnicodeDecodeError as exc: - raise CalendarBlockedError("KRX holiday page is not UTF-8") from exc - if KRX_BLD.lower() not in text.lower() or "search_bas_yy" not in text: - raise CalendarBlockedError("KRX holiday-page request contract is missing") - - -def previous_open_session(session: date, closed: set[date]) -> date: - candidate = session - timedelta(days=1) - for _ in range(14): - if candidate.weekday() < 5 and candidate not in closed: - return candidate - candidate -= timedelta(days=1) - raise CalendarBlockedError("official calendar cannot resolve previous session") - - -def source_record(path: Path, source_id: str) -> dict[str, Any]: - return { - "source_id": source_id, - "path": str(path.resolve()), - "sha256": sha256_file(path), - "bytes": path.stat().st_size, - } - - -def krx_year_payload( - year: int, - fetcher: Callable[[str, bytes | None], bytes], -) -> bytes: - otp = fetcher(KRX_OTP_URL, urlencode({"bld": KRX_BLD}).encode("ascii")) - try: - otp_text = otp.decode("ascii").strip() - except UnicodeDecodeError as exc: - raise CalendarBlockedError("KRX OTP is not ASCII") from exc - if not otp_text or not re.fullmatch(r"[A-Za-z0-9+/=_-]+", otp_text): - raise CalendarBlockedError("KRX OTP format changed") - return fetcher( - KRX_DATA_URL, - urlencode( - {"search_bas_yy": str(year), "gridTp": "KRX", "code": otp_text} - ).encode("ascii"), - ) - - -def build_kr_snapshot( - *, - session: date, - now: datetime, - output_directory: Path, - fetcher: Callable[[str, bytes | None], bytes], -) -> tuple[dict[str, Any] | None, dict[str, Any]]: - holiday_page = fetcher(KRX_HOLIDAY_PAGE, None) - hours_page = fetcher(KRX_HOURS_PAGE, None) - validate_krx_holiday_page(holiday_page) - validate_krx_hours(hours_page) - years = {session.year, (session - timedelta(days=14)).year} - holiday_payloads = {year: krx_year_payload(year, fetcher) for year in years} - closed = set().union( - *(parse_krx_holidays(payload, year) for year, payload in holiday_payloads.items()) - ) - - raw_directory = output_directory / "official" - records: list[dict[str, Any]] = [] - for path, payload, source_id in ( - (raw_directory / "krx-holiday-page.html", holiday_page, KRX_HOLIDAY_PAGE), - (raw_directory / "krx-trading-hours.html", hours_page, KRX_HOURS_PAGE), - ): - atomic_write(path, payload) - records.append(source_record(path, source_id)) - for year, payload in sorted(holiday_payloads.items()): - path = raw_directory / f"krx-holidays-{year}.json" - atomic_write(path, payload) - records.append(source_record(path, f"{KRX_DATA_URL}#KRX-{year}")) - - is_open = session.weekday() < 5 and session not in closed - previous = previous_open_session(session, closed) - if not is_open: - return None, { - "market": "KR", - "session_date": session.isoformat(), - "previous_session_date": previous.isoformat(), - "scheduled_status": "MARKET_CLOSED", - "official_sources": sorted(records, key=lambda item: item["source_id"]), - } - zone = ZoneInfo("Asia/Seoul") - regular_open = datetime.combine(session, time(9, 0), zone) - regular_close = datetime.combine(session, time(15, 30), zone) - if now > regular_open: - raise CalendarBlockedError("KR calendar snapshot completed after regular open") - source = { - "schema": "qta-market-session/v1", - "provider": "KRX", - "source_id": KRX_HOLIDAY_PAGE, - "source_as_of": now.isoformat(), - "market": "KR", - "timezone": "Asia/Seoul", - "session_date": session.isoformat(), - "previous_session_date": previous.isoformat(), - "scheduled_status": "OPEN", - "regular_open": regular_open.isoformat(), - "regular_close": regular_close.isoformat(), - } - return source, { - "market": "KR", - "session_date": session.isoformat(), - "previous_session_date": previous.isoformat(), - "scheduled_status": "OPEN", - "official_sources": sorted(records, key=lambda item: item["source_id"]), - } - - -def build_us_snapshot( - *, - session: date, - now: datetime, - output_directory: Path, - fetcher: Callable[[str, bytes | None], bytes], -) -> tuple[dict[str, Any] | None, dict[str, Any]]: - nyse_payload = fetcher(NYSE_CALENDAR_URL, None) - nasdaq_payload = fetcher(NASDAQ_CALENDAR_URL, None) - years = {session.year, (session - timedelta(days=14)).year} - nyse_closed: set[date] = set() - nyse_early: set[date] = set() - for year in sorted(years): - year_nyse_closed, year_nyse_early = parse_nyse_calendar( - nyse_payload, year - ) - year_nasdaq_closed, year_nasdaq_early = parse_nasdaq_calendar( - nasdaq_payload, year - ) - if year_nyse_closed != year_nasdaq_closed: - raise CalendarBlockedError( - f"NYSE and Nasdaq official closed-date sets do not match for {year}" - ) - if year_nyse_early != year_nasdaq_early: - raise CalendarBlockedError( - f"NYSE and Nasdaq official early-close date sets do not match for {year}" - ) - nyse_closed.update(year_nyse_closed) - nyse_early.update(year_nyse_early) - raw_directory = output_directory / "official" - records: list[dict[str, Any]] = [] - for path, payload, source_id in ( - (raw_directory / "nyse-hours-calendar.html", nyse_payload, NYSE_CALENDAR_URL), - ( - raw_directory / "nasdaq-trader-calendar.html", - nasdaq_payload, - NASDAQ_CALENDAR_URL, - ), - ): - atomic_write(path, payload) - records.append(source_record(path, source_id)) - - is_open = session.weekday() < 5 and session not in nyse_closed - previous = previous_open_session(session, nyse_closed) - if not is_open: - return None, { - "market": "US", - "session_date": session.isoformat(), - "previous_session_date": previous.isoformat(), - "scheduled_status": "MARKET_CLOSED", - "official_sources": sorted(records, key=lambda item: item["source_id"]), - } - zone = ZoneInfo("America/New_York") - regular_open = datetime.combine(session, time(9, 30), zone) - close_clock = time(13, 0) if session in nyse_early else time(16, 0) - regular_close = datetime.combine(session, close_clock, zone) - if now > regular_open: - raise CalendarBlockedError("US calendar snapshot completed after regular open") - source = { - "schema": "qta-market-session/v1", - "provider": "NYSE+NASDAQ", - "source_id": f"{NYSE_CALENDAR_URL}|{NASDAQ_CALENDAR_URL}", - "source_as_of": now.isoformat(), - "market": "US", - "timezone": "America/New_York", - "session_date": session.isoformat(), - "previous_session_date": previous.isoformat(), - "scheduled_status": "OPEN", - "regular_open": regular_open.isoformat(), - "regular_close": regular_close.isoformat(), - } - return source, { - "market": "US", - "session_date": session.isoformat(), - "previous_session_date": previous.isoformat(), - "scheduled_status": "OPEN", - "official_sources": sorted(records, key=lambda item: item["source_id"]), - } - - -def snapshot( - *, - market: str, - session_date: str, - output_directory: Path, - now: datetime | None = None, - fetcher: Callable[[str, bytes | None], bytes] = fetch_bytes, -) -> dict[str, Any]: - normalized_market = market.upper() - if normalized_market not in ENTRY_WINDOWS: - raise CalendarBlockedError("market must be KR or US") - session = parse_iso_date(session_date, "session_date") - zone = ZoneInfo(ENTRY_WINDOWS[normalized_market]["timezone"]) - observed_now = (now or datetime.now(zone)).astimezone(zone).replace(microsecond=0) - if observed_now.date() != session: - raise CalendarBlockedError( - "session_date must equal the current date in the market timezone" - ) - if output_directory.exists() and ( - output_directory.is_symlink() or not output_directory.is_dir() - ): - raise CalendarBlockedError("output_directory must be a non-symlink directory") - output_directory.mkdir(parents=True, exist_ok=True) - - builder = build_kr_snapshot if normalized_market == "KR" else build_us_snapshot - source, evidence = builder( - session=session, - now=observed_now, - output_directory=output_directory, - fetcher=fetcher, - ) - base_receipt = { - "schema": RECEIPT_SCHEMA, - **evidence, - "source_as_of": observed_now.isoformat(), - "live_enabled": False, - "api_mutation_count": 0, - } - if source is None: - receipt_without_hash = { - **base_receipt, - "status": "MARKET_CLOSED", - "market_session_source_path": "", - "market_session_source_sha256": "", - "market_session_path": "", - "session_hash": "", - } - else: - source_path = output_directory / "market-session-source.json" - atomic_write_json(source_path, source) - bound = market_session_from_source(source_path) - bound = normalized_market_session( - bound, - expected_market=normalized_market, - expected_timezone=ENTRY_WINDOWS[normalized_market]["timezone"], - ) - bound_path = output_directory / "market-session.json" - atomic_write_json(bound_path, bound) - receipt_without_hash = { - **base_receipt, - "status": "READY", - "market_session_source_path": str(source_path.resolve()), - "market_session_source_sha256": sha256_file(source_path), - "market_session_path": str(bound_path.resolve()), - "session_hash": bound["session_hash"], - } - receipt = { - **receipt_without_hash, - "receipt_hash": sha256_bytes( - canonical_json(receipt_without_hash).encode("utf-8") - ), - } - atomic_write_json(output_directory / "calendar-receipt.json", receipt) - return receipt - - -def self_test() -> None: - nasdaq = b""" - - - -
2026HolidayStatus
January 1, 2026New YearClosed
November 27, 2026Early Close1:00 p.m.
- """ - nyse = b""" -

Core Trading Session: 9:30 a.m. to 4:00 p.m. ET

- -
Holiday2026
New YearThursday, January 1
-

Each market will close early at 1:00 p.m. on Friday, - November 27, 2026.

- """ - if parse_nasdaq_calendar(nasdaq, 2026) != parse_nyse_calendar(nyse, 2026): - raise AssertionError("US calendar parser parity failed") - krx = json.dumps( - { - "block1": [ - { - "calnd_dd": "2026-01-01", - "dy_tp_cd": "THU", - "calnd_dd_dy": "2026-01-01", - "kr_dy_tp": "Thursday", - "holdy_eng_nm": "New Year's Day", - } - ] - } - ).encode() - if parse_krx_holidays(krx, 2026) != {date(2026, 1, 1)}: - raise AssertionError("KRX calendar parser failed") - print( - json.dumps( - { - "self_test": "PASS", - "live_enabled": False, - "api_mutation_count": 0, - }, - sort_keys=True, - ) - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - subparsers = parser.add_subparsers(dest="command", required=True) - snapshot_parser = subparsers.add_parser("snapshot") - snapshot_parser.add_argument("--market", choices=("KR", "US"), required=True) - snapshot_parser.add_argument("--session-date", required=True) - snapshot_parser.add_argument("--output-directory", required=True) - subparsers.add_parser("self-test") - return parser.parse_args() - - -def main() -> int: - args = parse_args() - if args.command == "self-test": - self_test() - return 0 - output_directory = Path(args.output_directory).expanduser() - if not output_directory.is_absolute(): - print( - json.dumps( - {"status": "BLOCKED", "reason": "output_directory must be absolute"}, - sort_keys=True, - ), - file=sys.stderr, - ) - return 2 - try: - receipt = snapshot( - market=args.market, - session_date=args.session_date, - output_directory=output_directory, - ) - print(json.dumps(receipt, sort_keys=True)) - return 0 - except (CalendarBlockedError, OSError, ValueError) as exc: - blocked = { - "schema": RECEIPT_SCHEMA, - "status": "BLOCKED", - "market": args.market, - "session_date": args.session_date, - "reason": str(exc), - "live_enabled": False, - "api_mutation_count": 0, - } - try: - atomic_write_json(output_directory / "calendar-receipt.json", blocked) - except OSError: - pass - print(json.dumps(blocked, sort_keys=True), file=sys.stderr) - return 2 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/skills/quant-stock-polling-trader/scripts/plan_orders.py b/skills/quant-stock-polling-trader/scripts/plan_orders.py deleted file mode 100755 index 9ad8e56..0000000 --- a/skills/quant-stock-polling-trader/scripts/plan_orders.py +++ /dev/null @@ -1,217 +0,0 @@ -#!/usr/bin/env python3 -"""Create a side-effect-free deterministic order plan from frozen JSON inputs.""" - -from __future__ import annotations - -import argparse -import json -import sys -import tempfile -from pathlib import Path -from typing import Any - -from execution_core import ( - ACCOUNT_SCHEMA, - EXECUTION_POLICY_SCHEMA, - EXPOSURE_SCHEMA, - PLAN_SCHEMA, - RISK_SCHEMA, - SCREEN_SCHEMA, - BlockedError, - canonical_json, - canonical_screen_hash, - emit_json, - load_json_object, - market_session_from_source, - plan_orders, -) - - -def build_from_paths(args: argparse.Namespace) -> dict[str, Any]: - return plan_orders( - load_json_object(args.screen), - load_json_object(args.account), - load_json_object(args.exposure), - load_json_object(args.risk), - load_json_object(args.execution), - ) - - -def fixture_inputs() -> tuple[dict[str, Any], ...]: - qta = { - "source_skill": "quant-stock-technical", - "result_schema": "quant-stock-technical/v1", - "calculation_status": "READY", - "setup_status": "READY", - "method_version": "qta-1.0.0", - "ticker": "005930", - "entry_price": 70000, - "stop_price": 65000, - "take_profit_price": 80000, - } - screen = { - "schema": SCREEN_SCHEMA, - "screen_status": "READY", - "method_version": "qta-1.0.0", - "selector_version": "qta-screen-1.0.0", - "selected": { - "KR": [ - { - "rank": 1, - "instrument": { - "market": "KR", - "ticker": "005930", - "tick_size": "100", - }, - "qta": qta, - } - ], - "US": [], - }, - } - screen["screen_hash"] = canonical_screen_hash(screen) - account = { - "schema": ACCOUNT_SCHEMA, - "broker": "kis", - "environment": "paper", - "account_alias": "paper-kr", - "market": "KR", - "currency": "KRW", - "as_of": "2026-07-27T08:50:00+09:00", - "settled_cash": "200000", - "borrowed_buying_power": "5000000", - "fx_to_krw": "1", - "positions": [], - "open_orders": [], - } - exposure = { - "schema": EXPOSURE_SCHEMA, - "as_of": "2026-07-27T08:50:00+09:00", - "positions": [], - } - risk = { - "schema": RISK_SCHEMA, - "base_currency": "KRW", - "per_trade_risk_krw": "10000", - "max_symbol_notional_krw": "200000", - "max_concurrent_positions": 1, - "max_daily_loss_krw": "20000", - "round_trip_cost_bps": "20", - "cash_buffer_bps": "100", - "allow_existing_additions": False, - "allow_borrowed_cash": False, - "allow_margin": False, - "allow_short": False, - "allow_auto_fx": False, - "whole_shares_only": True, - } - execution = { - "schema": EXECUTION_POLICY_SCHEMA, - "market": "KR", - "timezone": "Asia/Seoul", - "entry_window_start": "2026-07-27T09:00:00+09:00", - "entry_window_end": "2026-07-27T10:00:00+09:00", - "market_session": market_session_from_source( - Path(__file__).resolve().parents[1] - / "references" - / "fixtures" - / "kr-market-session-2026-07-27.json" - ), - "snapshot_max_age_seconds": 1800, - "poll_interval_seconds": 2, - "quote_max_age_seconds": 5, - "max_spread_bps": "25", - "max_gap_bps": "20", - "trigger_mode": "AT_OR_ABOVE", - "order_ttl_seconds": 30, - "order_type": "LIMIT", - "time_in_force": "DAY", - "allow_partial_fill": True, - "cancel_remainder_at_window_end": True, - } - return screen, account, exposure, risk, execution - - -def self_test() -> None: - values = fixture_inputs() - direct = plan_orders(*values) - with tempfile.TemporaryDirectory(prefix="qta-plan-cli-") as directory: - names = ("screen", "account", "exposure", "risk", "execution") - paths: list[str] = [] - for name, value in zip(names, values): - path = Path(directory) / f"{name}.json" - path.write_text(canonical_json(value), encoding="utf-8") - paths.append(str(path)) - args = argparse.Namespace( - screen=paths[0], - account=paths[1], - exposure=paths[2], - risk=paths[3], - execution=paths[4], - ) - loaded = build_from_paths(args) - assert direct == loaded - assert direct["plan_status"] == "READY" - assert direct["borrowed_buying_power_excluded"] == "5000000" - assert direct["intents"][0]["symbol"] == "005930" - print( - canonical_json( - { - "self_test": "PASS", - "schema": PLAN_SCHEMA, - "plan_hash": direct["plan_hash"], - } - ) - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--screen") - parser.add_argument("--account") - parser.add_argument("--exposure") - parser.add_argument("--risk") - parser.add_argument("--execution") - parser.add_argument("--output") - parser.add_argument("--self-test", action="store_true") - return parser.parse_args() - - -def main() -> int: - args = parse_args() - if args.self_test: - self_test() - return 0 - missing = [ - name - for name in ("screen", "account", "exposure", "risk", "execution") - if not getattr(args, name) - ] - if missing: - emit_json( - { - "schema": PLAN_SCHEMA, - "plan_status": "BLOCKED", - "reason": "missing arguments: " + ", ".join(missing), - }, - args.output, - ) - return 2 - try: - plan = build_from_paths(args) - except (BlockedError, OSError, ValueError, json.JSONDecodeError) as exc: - emit_json( - { - "schema": PLAN_SCHEMA, - "plan_status": "BLOCKED", - "reason": str(exc), - }, - args.output, - ) - return 2 - emit_json(plan, args.output) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/skills/quant-stock-polling-trader/scripts/position_lifecycle.py b/skills/quant-stock-polling-trader/scripts/position_lifecycle.py deleted file mode 100644 index e60c987..0000000 --- a/skills/quant-stock-polling-trader/scripts/position_lifecycle.py +++ /dev/null @@ -1,1236 +0,0 @@ -#!/usr/bin/env python3 -"""Deterministic strategy-position lifecycle and exit decisions.""" - -from __future__ import annotations - -import argparse -import json -import os -import sqlite3 -import sys -import tempfile -from dataclasses import dataclass -from datetime import date, datetime, timedelta, timezone -from decimal import Decimal -from pathlib import Path -from typing import Any - -from execution_core import ( - SCREEN_SCHEMA_V2, - BlockedError, - canonical_json, - decimal_value, - exact_fields, - normalize_symbol, - normalized_account_snapshot, - parse_aware_datetime, - positive_decimal, - round_down_to_tick, - sha256_json, -) - -POSITION_POLICY_SCHEMA = "qta-position-policy/v1" -STRATEGY_POSITION_SCHEMA = "qta-strategy-position/v1" -EXIT_DECISION_SCHEMA = "qta-exit-decision/v1" -POSITION_RECONCILIATION_SCHEMA = "qta-position-reconciliation/v1" -POSITION_POLICY_FIELDS = { - "schema", - "close_liquidation_seconds_before_close", - "exit_order_ttl_seconds", - "max_exit_replacements", - "quote_max_age_seconds", - "partial_entry_action", - "gap_down_action", - "daily_loss_action", - "overnight_residual_action", -} -STRATEGY_POSITION_FIELDS = { - "schema", - "position_id", - "plan_hash", - "entry_intent_id", - "market", - "exchange", - "symbol", - "venue", - "currency", - "entry_session_date", - "last_managed_session_date", - "planned_quantity", - "acquired_quantity", - "exited_quantity", - "average_entry_price", - "average_exit_price", - "stop_price", - "take_profit_price", - "tick_size", - "fx_to_krw", - "daily_baseline_session_date", - "daily_baseline_price", - "realized_pnl_native", - "daily_realized_pnl_native", - "status", -} -POSITION_STATUSES = { - "PENDING_ENTRY", - "OPEN", - "EXIT_PENDING", - "EXIT_PARTIAL", - "CARRY_EXIT_ONLY", - "CLOSED", - "MANUAL_BLOCK", -} -MARKET_EXCHANGES = { - "KR": {"KOSPI", "KOSDAQ"}, - "US": {"NYSE", "NASDAQ"}, -} -MARKET_CURRENCY = {"KR": "KRW", "US": "USD"} - - -def require_iso_date(value: Any, field: str) -> str: - if not isinstance(value, str): - raise BlockedError(f"{field} must be YYYY-MM-DD") - try: - parsed = date.fromisoformat(value) - except ValueError as exc: - raise BlockedError(f"{field} must be YYYY-MM-DD") from exc - if parsed.isoformat() != value: - raise BlockedError(f"{field} must be YYYY-MM-DD") - return value - - -def require_nonnegative_integer(value: Any, field: str) -> int: - if isinstance(value, bool) or not isinstance(value, int) or value < 0: - raise BlockedError(f"{field} must be a nonnegative integer") - return value - - -def require_positive_integer(value: Any, field: str) -> int: - amount = require_nonnegative_integer(value, field) - if amount == 0: - raise BlockedError(f"{field} must be a positive integer") - return amount - - -def normalized_position_policy(value: dict[str, Any]) -> dict[str, Any]: - exact_fields(value, POSITION_POLICY_FIELDS, "position policy") - if value["schema"] != POSITION_POLICY_SCHEMA: - raise BlockedError( - f"position policy schema must be {POSITION_POLICY_SCHEMA}" - ) - close_seconds = require_positive_integer( - value["close_liquidation_seconds_before_close"], - "close_liquidation_seconds_before_close", - ) - if close_seconds > 3600: - raise BlockedError( - "close_liquidation_seconds_before_close must be <= 3600" - ) - ttl = require_positive_integer( - value["exit_order_ttl_seconds"], "exit_order_ttl_seconds" - ) - if ttl > 300: - raise BlockedError("exit_order_ttl_seconds must be <= 300") - replacements = require_nonnegative_integer( - value["max_exit_replacements"], "max_exit_replacements" - ) - if replacements > 20: - raise BlockedError("max_exit_replacements must be <= 20") - quote_age = require_positive_integer( - value["quote_max_age_seconds"], "quote_max_age_seconds" - ) - if quote_age > 60: - raise BlockedError("quote_max_age_seconds must be <= 60") - expected_enums = { - "partial_entry_action": "CANCEL_REMAINDER_MANAGE_FILLED", - "gap_down_action": "MARKETABLE_LIMIT", - "daily_loss_action": "ENTRY_FREEZE_LIQUIDATE", - "overnight_residual_action": "EXIT_ONLY_NEXT_SESSION", - } - for field, expected in expected_enums.items(): - if value[field] != expected: - raise BlockedError(f"{field} must be {expected}") - return { - **value, - "close_liquidation_seconds_before_close": close_seconds, - "exit_order_ttl_seconds": ttl, - "max_exit_replacements": replacements, - "quote_max_age_seconds": quote_age, - } - - -def strategy_position_id(plan_hash: str, intent: dict[str, Any]) -> str: - if len(plan_hash) != 64 or any( - character not in "0123456789abcdef" for character in plan_hash - ): - raise BlockedError("plan_hash must be lowercase SHA-256") - seed = { - "plan_hash": plan_hash, - "entry_intent_id": intent.get("intent_id"), - "market": intent.get("market"), - "exchange": intent.get("exchange"), - "symbol": intent.get("symbol"), - "venue": intent.get("venue"), - } - return sha256_json(seed)[:32] - - -def position_from_intent( - *, - plan_hash: str, - intent: dict[str, Any], - session_date: str, - fx_to_krw: Any, -) -> dict[str, Any]: - session = require_iso_date(session_date, "session_date") - market = str(intent.get("market", "")).upper() - if market not in MARKET_EXCHANGES: - raise BlockedError("intent market must be KR or US") - exchange = str(intent.get("exchange", "")).upper() - if exchange not in MARKET_EXCHANGES[market]: - raise BlockedError("intent exchange does not match market") - symbol = normalize_symbol(market, str(intent.get("symbol", ""))) - venue = str(intent.get("venue", "")).upper() - if not venue: - raise BlockedError("intent venue is required") - quantity = positive_decimal(intent.get("quantity"), "intent quantity") - if quantity != quantity.to_integral_value(): - raise BlockedError("strategy positions require whole-share quantity") - currency = str(intent.get("currency", "")).upper() - if currency != MARKET_CURRENCY[market]: - raise BlockedError("intent currency does not match market") - fx = positive_decimal(fx_to_krw, "fx_to_krw") - stop = positive_decimal(intent.get("stop_price"), "stop_price") - take_profit = positive_decimal( - intent.get("take_profit_price"), "take_profit_price" - ) - tick_source = intent.get("resolved_tick_size") - if tick_source is None: - tick_source = intent.get("tick_size") - if tick_source is None: - raise BlockedError("intent resolved tick size is required") - tick = positive_decimal(tick_source, "tick_size") - entry_intent_id = str(intent.get("intent_id", "")) - if len(entry_intent_id) != 32: - raise BlockedError("entry intent ID must contain 32 characters") - output = { - "schema": STRATEGY_POSITION_SCHEMA, - "position_id": strategy_position_id(plan_hash, intent), - "plan_hash": plan_hash, - "entry_intent_id": entry_intent_id, - "market": market, - "exchange": exchange, - "symbol": symbol, - "venue": venue, - "currency": currency, - "entry_session_date": session, - "last_managed_session_date": session, - "planned_quantity": format(quantity, "f"), - "acquired_quantity": "0", - "exited_quantity": "0", - "average_entry_price": None, - "average_exit_price": None, - "stop_price": format(stop, "f"), - "take_profit_price": format(take_profit, "f"), - "tick_size": format(tick, "f"), - "fx_to_krw": format(fx, "f"), - "daily_baseline_session_date": None, - "daily_baseline_price": None, - "realized_pnl_native": "0", - "daily_realized_pnl_native": "0", - "status": "PENDING_ENTRY", - } - return normalize_strategy_position(output) - - -def normalize_strategy_position(value: dict[str, Any]) -> dict[str, Any]: - exact_fields(value, STRATEGY_POSITION_FIELDS, "strategy position") - if value["schema"] != STRATEGY_POSITION_SCHEMA: - raise BlockedError( - f"strategy position schema must be {STRATEGY_POSITION_SCHEMA}" - ) - position_id = str(value["position_id"]) - if len(position_id) != 32 or any( - character not in "0123456789abcdef" for character in position_id - ): - raise BlockedError("position_id must be 32 lowercase hex characters") - plan_hash = str(value["plan_hash"]) - if len(plan_hash) != 64 or any( - character not in "0123456789abcdef" for character in plan_hash - ): - raise BlockedError("plan_hash must be lowercase SHA-256") - market = str(value["market"]).upper() - exchange = str(value["exchange"]).upper() - if market not in MARKET_EXCHANGES or exchange not in MARKET_EXCHANGES[market]: - raise BlockedError("strategy position exchange does not match market") - symbol = normalize_symbol(market, str(value["symbol"])) - currency = str(value["currency"]).upper() - if currency != MARKET_CURRENCY[market]: - raise BlockedError("strategy position currency does not match market") - entry_date = require_iso_date( - value["entry_session_date"], "entry_session_date" - ) - managed_date = require_iso_date( - value["last_managed_session_date"], "last_managed_session_date" - ) - if managed_date < entry_date: - raise BlockedError("last managed session cannot precede entry session") - planned = positive_decimal(value["planned_quantity"], "planned_quantity") - acquired = decimal_value(value["acquired_quantity"], "acquired_quantity") - exited = decimal_value(value["exited_quantity"], "exited_quantity") - for amount, field in ( - (planned, "planned_quantity"), - (acquired, "acquired_quantity"), - (exited, "exited_quantity"), - ): - if amount < 0 or amount != amount.to_integral_value(): - raise BlockedError(f"{field} must be a nonnegative whole number") - if acquired > planned: - raise BlockedError("acquired quantity exceeds planned quantity") - if exited > acquired: - raise BlockedError("exited quantity exceeds acquired quantity") - average_entry = value["average_entry_price"] - if acquired > 0: - average_entry = format( - positive_decimal(average_entry, "average_entry_price"), "f" - ) - elif average_entry is not None: - raise BlockedError("zero acquired quantity cannot have an entry price") - average_exit = value["average_exit_price"] - if exited > 0: - average_exit = format( - positive_decimal(average_exit, "average_exit_price"), "f" - ) - elif average_exit is not None: - raise BlockedError("zero exited quantity cannot have an exit price") - baseline_date = value["daily_baseline_session_date"] - baseline_price = value["daily_baseline_price"] - if baseline_date is None: - if baseline_price is not None: - raise BlockedError("daily baseline date and price must both be null") - else: - baseline_date = require_iso_date( - baseline_date, "daily_baseline_session_date" - ) - if baseline_date != managed_date: - raise BlockedError("daily baseline must match last managed session") - baseline_price = format( - positive_decimal(baseline_price, "daily_baseline_price"), "f" - ) - stop = positive_decimal(value["stop_price"], "stop_price") - take_profit = positive_decimal(value["take_profit_price"], "take_profit_price") - if stop >= take_profit: - raise BlockedError("stop price must be below take-profit price") - tick = positive_decimal(value["tick_size"], "tick_size") - fx = positive_decimal(value["fx_to_krw"], "fx_to_krw") - realized = decimal_value(value["realized_pnl_native"], "realized_pnl_native") - daily_realized = decimal_value( - value["daily_realized_pnl_native"], "daily_realized_pnl_native" - ) - status = str(value["status"]) - if status not in POSITION_STATUSES: - raise BlockedError(f"unsupported strategy position status: {status}") - if acquired == 0 and status != "PENDING_ENTRY": - raise BlockedError("zero acquired quantity must be PENDING_ENTRY") - if acquired > 0 and exited == acquired and status != "CLOSED": - raise BlockedError("fully exited position must be CLOSED") - if acquired > exited and status in {"PENDING_ENTRY", "CLOSED"}: - raise BlockedError("open quantity requires an open position status") - return { - **value, - "market": market, - "exchange": exchange, - "symbol": symbol, - "currency": currency, - "entry_session_date": entry_date, - "last_managed_session_date": managed_date, - "planned_quantity": format(planned, "f"), - "acquired_quantity": format(acquired, "f"), - "exited_quantity": format(exited, "f"), - "average_entry_price": average_entry, - "average_exit_price": average_exit, - "stop_price": format(stop, "f"), - "take_profit_price": format(take_profit, "f"), - "tick_size": format(tick, "f"), - "fx_to_krw": format(fx, "f"), - "daily_baseline_session_date": baseline_date, - "daily_baseline_price": baseline_price, - "realized_pnl_native": format(realized, "f"), - "daily_realized_pnl_native": format(daily_realized, "f"), - "status": status, - } - - -def open_quantity(position: dict[str, Any]) -> Decimal: - normalized = normalize_strategy_position(position) - return Decimal(normalized["acquired_quantity"]) - Decimal( - normalized["exited_quantity"] - ) - - -def record_entry_fill( - position: dict[str, Any], - *, - cumulative_filled_quantity: Any, - average_fill_price: Any, - session_date: str, -) -> dict[str, Any]: - normalized = normalize_strategy_position(position) - session = require_iso_date(session_date, "session_date") - cumulative = decimal_value( - cumulative_filled_quantity, "cumulative_filled_quantity" - ) - prior = Decimal(normalized["acquired_quantity"]) - planned = Decimal(normalized["planned_quantity"]) - if ( - cumulative < prior - or cumulative > planned - or cumulative != cumulative.to_integral_value() - ): - raise BlockedError("entry cumulative fill is non-monotonic or out of bounds") - if cumulative == 0: - return normalized - average = positive_decimal(average_fill_price, "average_fill_price") - if session < normalized["entry_session_date"]: - raise BlockedError("entry fill session precedes the planned entry session") - exited = Decimal(normalized["exited_quantity"]) - next_status = "CLOSED" if cumulative == exited else "OPEN" - output = { - **normalized, - "last_managed_session_date": session, - "acquired_quantity": format(cumulative, "f"), - "average_entry_price": format(average, "f"), - "daily_baseline_session_date": session, - "daily_baseline_price": format(average, "f"), - "status": next_status, - } - return normalize_strategy_position(output) - - -def roll_position_session( - position: dict[str, Any], - *, - session_date: str, -) -> dict[str, Any]: - normalized = normalize_strategy_position(position) - session = require_iso_date(session_date, "session_date") - if session < normalized["last_managed_session_date"]: - raise BlockedError("cannot roll a position backward") - if session == normalized["last_managed_session_date"]: - return normalized - status = ( - "CLOSED" - if open_quantity(normalized) == 0 - else "CARRY_EXIT_ONLY" - ) - return normalize_strategy_position( - { - **normalized, - "last_managed_session_date": session, - "daily_baseline_session_date": None, - "daily_baseline_price": None, - "daily_realized_pnl_native": "0", - "status": status, - } - ) - - -def set_daily_baseline( - position: dict[str, Any], - *, - session_date: str, - price: Any, -) -> dict[str, Any]: - normalized = normalize_strategy_position(position) - session = require_iso_date(session_date, "session_date") - if session != normalized["last_managed_session_date"]: - raise BlockedError("daily baseline session must be the managed session") - baseline = positive_decimal(price, "daily baseline price") - existing_date = normalized["daily_baseline_session_date"] - if existing_date is not None: - if ( - existing_date != session - or Decimal(str(normalized["daily_baseline_price"])) != baseline - ): - raise BlockedError("daily baseline is immutable within one session") - return normalized - return normalize_strategy_position( - { - **normalized, - "daily_baseline_session_date": session, - "daily_baseline_price": format(baseline, "f"), - } - ) - - -def record_exit_fill( - position: dict[str, Any], - *, - cumulative_exited_quantity: Any, - average_exit_price: Any, -) -> dict[str, Any]: - normalized = normalize_strategy_position(position) - cumulative = decimal_value( - cumulative_exited_quantity, "cumulative_exited_quantity" - ) - prior = Decimal(normalized["exited_quantity"]) - acquired = Decimal(normalized["acquired_quantity"]) - if ( - cumulative < prior - or cumulative > acquired - or cumulative != cumulative.to_integral_value() - ): - raise BlockedError("exit cumulative fill is non-monotonic or out of bounds") - if cumulative == 0: - return normalized - average_exit = positive_decimal(average_exit_price, "average_exit_price") - entry = positive_decimal( - normalized["average_entry_price"], "average_entry_price" - ) - prior_average_exit = ( - Decimal(str(normalized["average_exit_price"])) - if prior > 0 - else Decimal(0) - ) - prior_exit_proceeds = prior_average_exit * prior - cumulative_exit_proceeds = average_exit * cumulative - incremental_quantity = cumulative - prior - incremental_exit_proceeds = cumulative_exit_proceeds - prior_exit_proceeds - realized = ( - Decimal(normalized["realized_pnl_native"]) - + incremental_exit_proceeds - - entry * incremental_quantity - ) - baseline_value = normalized["daily_baseline_price"] - if baseline_value is None: - raise BlockedError("daily baseline is required before recording an exit") - baseline = positive_decimal(baseline_value, "daily_baseline_price") - daily_realized = ( - Decimal(normalized["daily_realized_pnl_native"]) - + incremental_exit_proceeds - - baseline * incremental_quantity - ) - status = "CLOSED" if cumulative == acquired else "EXIT_PARTIAL" - return normalize_strategy_position( - { - **normalized, - "exited_quantity": format(cumulative, "f"), - "average_exit_price": format(average_exit, "f"), - "realized_pnl_native": format(realized, "f"), - "daily_realized_pnl_native": format(daily_realized, "f"), - "status": status, - } - ) - - -def partial_entry_requires_cancel( - position: dict[str, Any], - *, - broker_remaining_quantity: Any, -) -> bool: - normalized = normalize_strategy_position(position) - remaining = decimal_value( - broker_remaining_quantity, "broker_remaining_quantity" - ) - return Decimal(normalized["acquired_quantity"]) > 0 and remaining > 0 - - -def normalize_exit_quote( - quote: dict[str, Any], - *, - position: dict[str, Any], - observed_at: datetime, - max_age_seconds: int, -) -> dict[str, Any]: - normalized = normalize_strategy_position(position) - if str(quote.get("market", "")).upper() != normalized["market"]: - raise BlockedError("exit quote market does not match position") - if normalize_symbol( - normalized["market"], str(quote.get("symbol", "")) - ) != normalized["symbol"]: - raise BlockedError("exit quote symbol does not match position") - last = positive_decimal(quote.get("last_price"), "exit quote last_price") - bid = positive_decimal(quote.get("best_bid"), "exit quote best_bid") - ask = positive_decimal(quote.get("best_ask"), "exit quote best_ask") - if bid > ask: - raise BlockedError("exit quote best bid exceeds best ask") - trade_time = parse_aware_datetime( - quote.get("trade_timestamp"), "exit quote trade_timestamp" - ) - book_time = parse_aware_datetime( - quote.get("book_timestamp"), "exit quote book_timestamp" - ) - received = parse_aware_datetime( - quote.get("received_at"), "exit quote received_at" - ) - now = observed_at.astimezone(timezone.utc) - for supplied, field in ( - (trade_time, "trade_timestamp"), - (book_time, "book_timestamp"), - (received, "received_at"), - ): - age = (now - supplied.astimezone(timezone.utc)).total_seconds() - if age < -1: - raise BlockedError(f"exit quote {field} is future-dated") - if age > max_age_seconds: - raise BlockedError(f"exit quote {field} is stale") - tick = Decimal(normalized["tick_size"]) - return { - "last_price": format(last, "f"), - "best_bid": format(bid, "f"), - "best_ask": format(ask, "f"), - "limit_price": format(round_down_to_tick(bid, tick), "f"), - "trade_timestamp": trade_time.isoformat(), - "book_timestamp": book_time.isoformat(), - "received_at": received.isoformat(), - } - - -def exit_decision( - position: dict[str, Any], - quote: dict[str, Any], - policy: dict[str, Any], - *, - observed_at: datetime, - regular_close: datetime, - first_valid_quote_of_session: bool, - daily_loss_breached: bool, -) -> dict[str, Any]: - normalized = normalize_strategy_position(position) - normalized_policy = normalized_position_policy(policy) - quantity = open_quantity(normalized) - if quantity <= 0: - action = "CLOSED" - reason = "NO_OPEN_QUANTITY" - normalized_quote = None - limit_price = None - else: - if observed_at.tzinfo is None or observed_at.utcoffset() is None: - raise BlockedError("observed_at must be timezone-aware") - if regular_close.tzinfo is None or regular_close.utcoffset() is None: - raise BlockedError("regular_close must be timezone-aware") - if observed_at >= regular_close: - action = "MANUAL_BLOCK" - reason = "MARKET_CLOSED_WITH_RESIDUAL" - normalized_quote = None - limit_price = None - else: - normalized_quote = normalize_exit_quote( - quote, - position=normalized, - observed_at=observed_at, - max_age_seconds=normalized_policy["quote_max_age_seconds"], - ) - last = Decimal(normalized_quote["last_price"]) - stop = Decimal(normalized["stop_price"]) - target = Decimal(normalized["take_profit_price"]) - close_trigger = regular_close - timedelta( - seconds=normalized_policy[ - "close_liquidation_seconds_before_close" - ] - ) - action = "HOLD" - reason = "NO_EXIT_TRIGGER" - if daily_loss_breached: - action = "SUBMIT_EXIT" - reason = "DAILY_LOSS_LIMIT" - elif last <= stop: - action = "SUBMIT_EXIT" - reason = ( - "STOP_GAP" - if first_valid_quote_of_session and last < stop - else "STOP" - ) - elif last >= target: - action = "SUBMIT_EXIT" - reason = "TAKE_PROFIT" - elif observed_at >= close_trigger: - action = "SUBMIT_EXIT" - reason = "MARKET_CLOSE" - limit_price = ( - normalized_quote["limit_price"] - if action == "SUBMIT_EXIT" - else None - ) - without_hash = { - "schema": EXIT_DECISION_SCHEMA, - "position_id": normalized["position_id"], - "session_date": normalized["last_managed_session_date"], - "observed_at": observed_at.isoformat(), - "action": action, - "reason": reason, - "quantity": format(max(quantity, Decimal(0)), "f"), - "order_type": "LIMIT" if action == "SUBMIT_EXIT" else None, - "limit_price": limit_price, - "quote": normalized_quote, - "daily_loss_breached": bool(daily_loss_breached), - } - return {**without_hash, "decision_hash": sha256_json(without_hash)} - - -def marked_daily_pnl_krw( - position: dict[str, Any], - *, - best_bid: Any, - estimated_exit_cost_bps: Any, -) -> Decimal: - normalized = normalize_strategy_position(position) - if normalized["daily_baseline_price"] is None: - raise BlockedError("daily baseline is required for daily P&L") - bid = positive_decimal(best_bid, "best_bid") - cost_bps = decimal_value(estimated_exit_cost_bps, "estimated_exit_cost_bps") - if cost_bps < 0 or cost_bps > 10000: - raise BlockedError("estimated_exit_cost_bps must be between 0 and 10000") - quantity = open_quantity(normalized) - baseline = Decimal(str(normalized["daily_baseline_price"])) - daily_realized = Decimal(normalized["daily_realized_pnl_native"]) - exit_cost = bid * quantity * cost_bps / Decimal(10000) - pnl_native = daily_realized + (bid - baseline) * quantity - exit_cost - return pnl_native * Decimal(normalized["fx_to_krw"]) - - -def portfolio_daily_loss_krw( - marked_pnl_by_position: dict[str, Decimal], -) -> Decimal: - pnl = sum(marked_pnl_by_position.values(), Decimal(0)) - return max(-pnl, Decimal(0)) - - -def daily_loss_limit_breached(loss_krw: Any, maximum_loss_krw: Any) -> bool: - loss = decimal_value(loss_krw, "loss_krw") - maximum = positive_decimal(maximum_loss_krw, "maximum_loss_krw") - if loss < 0: - raise BlockedError("loss_krw must be nonnegative") - return loss >= maximum - - -def exit_order_id( - decision: dict[str, Any], - *, - attempt: int, -) -> str: - attempt_number = require_nonnegative_integer(attempt, "exit attempt") - if decision.get("schema") != EXIT_DECISION_SCHEMA: - raise BlockedError("exit decision schema is invalid") - if decision.get("action") != "SUBMIT_EXIT": - raise BlockedError("exit order requires a SUBMIT_EXIT decision") - seed = { - "position_id": decision["position_id"], - "session_date": decision["session_date"], - "reason": decision["reason"], - "quantity": decision["quantity"], - "limit_price": decision["limit_price"], - "attempt": attempt_number, - } - return sha256_json(seed)[:32] - - -@dataclass(frozen=True) -class StoredPosition: - position_id: str - status: str - payload: dict[str, Any] - - -class PositionLedger: - """Account/market-level position registry that survives session dates.""" - - def __init__(self, path: str | Path): - self.path = Path(path) - self.path.parent.mkdir(parents=True, exist_ok=True) - self.connection = sqlite3.connect(str(self.path)) - self.connection.execute("PRAGMA journal_mode=WAL") - self.connection.execute("PRAGMA synchronous=FULL") - self.connection.executescript( - """ - CREATE TABLE IF NOT EXISTS strategy_positions ( - position_id TEXT PRIMARY KEY, - status TEXT NOT NULL, - payload_json TEXT NOT NULL, - updated_at TEXT NOT NULL - ); - CREATE TABLE IF NOT EXISTS position_events ( - sequence INTEGER PRIMARY KEY AUTOINCREMENT, - position_id TEXT NOT NULL, - event_type TEXT NOT NULL, - payload_json TEXT NOT NULL, - created_at TEXT NOT NULL - ); - """ - ) - self.connection.commit() - - def close(self) -> None: - self.connection.close() - - def put( - self, - position: dict[str, Any], - *, - event_type: str, - event_payload: dict[str, Any], - ) -> None: - normalized = normalize_strategy_position(position) - if not event_type: - raise BlockedError("position event type is required") - if not isinstance(event_payload, dict): - raise BlockedError("position event payload must be an object") - now = datetime.now(timezone.utc).isoformat() - rendered = canonical_json(normalized) - self.connection.execute("BEGIN IMMEDIATE") - try: - existing = self.connection.execute( - """ - SELECT payload_json FROM strategy_positions - WHERE position_id = ? - """, - (normalized["position_id"],), - ).fetchone() - if existing is None: - self.connection.execute( - """ - INSERT INTO strategy_positions - (position_id, status, payload_json, updated_at) - VALUES (?, ?, ?, ?) - """, - ( - normalized["position_id"], - normalized["status"], - rendered, - now, - ), - ) - else: - self.connection.execute( - """ - UPDATE strategy_positions - SET status = ?, payload_json = ?, updated_at = ? - WHERE position_id = ? - """, - ( - normalized["status"], - rendered, - now, - normalized["position_id"], - ), - ) - self.connection.execute( - """ - INSERT INTO position_events - (position_id, event_type, payload_json, created_at) - VALUES (?, ?, ?, ?) - """, - ( - normalized["position_id"], - event_type, - canonical_json(event_payload), - now, - ), - ) - except BaseException: - self.connection.rollback() - raise - else: - self.connection.commit() - - def get(self, position_id: str) -> StoredPosition: - row = self.connection.execute( - """ - SELECT position_id, status, payload_json - FROM strategy_positions WHERE position_id = ? - """, - (position_id,), - ).fetchone() - if row is None: - raise BlockedError(f"unknown strategy position: {position_id}") - return StoredPosition( - position_id=str(row[0]), - status=str(row[1]), - payload=normalize_strategy_position(json.loads(row[2])), - ) - - def get_optional(self, position_id: str) -> StoredPosition | None: - row = self.connection.execute( - """ - SELECT position_id, status, payload_json - FROM strategy_positions WHERE position_id = ? - """, - (position_id,), - ).fetchone() - if row is None: - return None - return StoredPosition( - position_id=str(row[0]), - status=str(row[1]), - payload=normalize_strategy_position(json.loads(row[2])), - ) - - def open_positions(self) -> list[StoredPosition]: - rows = self.connection.execute( - """ - SELECT position_id, status, payload_json - FROM strategy_positions - WHERE status != 'CLOSED' - ORDER BY position_id - """ - ).fetchall() - return [ - StoredPosition( - position_id=str(row[0]), - status=str(row[1]), - payload=normalize_strategy_position(json.loads(row[2])), - ) - for row in rows - ] - - def events(self, position_id: str) -> list[dict[str, Any]]: - rows = self.connection.execute( - """ - SELECT sequence, event_type, payload_json, created_at - FROM position_events - WHERE position_id = ? - ORDER BY sequence - """, - (position_id,), - ).fetchall() - return [ - { - "sequence": int(row[0]), - "event_type": str(row[1]), - "payload": json.loads(row[2]), - "created_at": str(row[3]), - } - for row in rows - ] - - -def reconcile_account_positions( - ledger: PositionLedger, - *, - account_snapshot: dict[str, Any], - session_date: str, -) -> dict[str, Any]: - """Match bot-owned positions to the broker snapshot and roll valid carries.""" - session = require_iso_date(session_date, "session_date") - account = normalized_account_snapshot( - account_snapshot, - screen_schema=SCREEN_SCHEMA_V2, - ) - market = account["market"] - account_quantities: dict[tuple[str, str], Decimal] = {} - for item in account["positions"]: - key = (str(item["exchange"]), str(item["symbol"])) - quantity = decimal_value(item.get("quantity"), "account position quantity") - if quantity < 0: - raise BlockedError("account position quantity must be nonnegative") - account_quantities[key] = account_quantities.get(key, Decimal(0)) + quantity - open_order_keys = { - (str(item["exchange"]), str(item["symbol"])) - for item in account["open_orders"] - } - - stored_positions = [ - item for item in ledger.open_positions() if item.payload["market"] == market - ] - by_key: dict[tuple[str, str], list[StoredPosition]] = {} - for stored in stored_positions: - key = (stored.payload["exchange"], stored.payload["symbol"]) - by_key.setdefault(key, []).append(stored) - - results: list[dict[str, Any]] = [] - managed_keys: set[tuple[str, str]] = set() - for key in sorted(by_key): - positions = by_key[key] - managed_keys.add(key) - expected = sum( - (open_quantity(item.payload) for item in positions), Decimal(0) - ) - observed = account_quantities.get(key, Decimal(0)) - reasons: list[str] = [] - if len(positions) != 1: - reasons.append("duplicate_strategy_positions_for_symbol") - if observed != expected: - reasons.append("broker_quantity_differs_from_strategy_ledger") - if key in open_order_keys: - reasons.append("broker_has_open_order_for_strategy_symbol") - if any(item.status == "MANUAL_BLOCK" for item in positions): - reasons.append("prior_manual_block_requires_explicit_repair") - - if reasons: - for stored in positions: - if stored.status != "MANUAL_BLOCK": - blocked_position = normalize_strategy_position( - {**stored.payload, "status": "MANUAL_BLOCK"} - ) - ledger.put( - blocked_position, - event_type="ACCOUNT_RECONCILIATION_BLOCKED", - event_payload={ - "session_date": session, - "expected_quantity": format(expected, "f"), - "observed_quantity": format(observed, "f"), - "reasons": reasons, - }, - ) - state = "MANUAL_BLOCK" - else: - stored = positions[0] - rolled = roll_position_session( - stored.payload, - session_date=session, - ) - if rolled != stored.payload: - ledger.put( - rolled, - event_type="POSITION_CARRIED", - event_payload={ - "from_session_date": stored.payload[ - "last_managed_session_date" - ], - "to_session_date": session, - "quantity": format(expected, "f"), - }, - ) - state = rolled["status"] - results.append( - { - "exchange": key[0], - "symbol": key[1], - "strategy_position_ids": [ - item.position_id for item in positions - ], - "expected_quantity": format(expected, "f"), - "observed_quantity": format(observed, "f"), - "state": state, - "reasons": reasons, - } - ) - - unmanaged = [ - { - "exchange": key[0], - "symbol": key[1], - "quantity": format(quantity, "f"), - } - for key, quantity in sorted(account_quantities.items()) - if key not in managed_keys and quantity > 0 - ] - without_hash = { - "schema": POSITION_RECONCILIATION_SCHEMA, - "account_as_of": account["as_of"], - "account_alias": account["account_alias"], - "market": market, - "session_date": session, - "status": ( - "MANUAL_BLOCK" - if any(item["state"] == "MANUAL_BLOCK" for item in results) - else "READY" - ), - "managed_positions": results, - "unmanaged_broker_positions": unmanaged, - "api_mutation_count": 0, - "live_enabled": False, - } - return { - **without_hash, - "reconciliation_hash": sha256_json(without_hash), - } - - -def atomic_write_json(path: Path, value: dict[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - descriptor, temporary_name = tempfile.mkstemp( - prefix=f".{path.name}.", dir=path.parent - ) - temporary = Path(temporary_name) - try: - with os.fdopen(descriptor, "w", encoding="utf-8") as handle: - handle.write(json.dumps(value, sort_keys=True, separators=(",", ":"))) - handle.write("\n") - handle.flush() - os.fsync(handle.fileno()) - os.replace(temporary, path) - except BaseException: - temporary.unlink(missing_ok=True) - raise - - -def self_test() -> None: - plan_hash = "a" * 64 - intent = { - "intent_id": "b" * 32, - "market": "KR", - "exchange": "KOSPI", - "symbol": "005930", - "venue": "KRX", - "currency": "KRW", - "quantity": "10", - "stop_price": "65000", - "take_profit_price": "80000", - "resolved_tick_size": "100", - } - policy = normalized_position_policy( - { - "schema": POSITION_POLICY_SCHEMA, - "close_liquidation_seconds_before_close": 600, - "exit_order_ttl_seconds": 15, - "max_exit_replacements": 3, - "quote_max_age_seconds": 5, - "partial_entry_action": "CANCEL_REMAINDER_MANAGE_FILLED", - "gap_down_action": "MARKETABLE_LIMIT", - "daily_loss_action": "ENTRY_FREEZE_LIQUIDATE", - "overnight_residual_action": "EXIT_ONLY_NEXT_SESSION", - } - ) - position = position_from_intent( - plan_hash=plan_hash, - intent=intent, - session_date="2026-07-29", - fx_to_krw="1", - ) - position = record_entry_fill( - position, - cumulative_filled_quantity="4", - average_fill_price="70000", - session_date="2026-07-29", - ) - assert partial_entry_requires_cancel( - position, broker_remaining_quantity="6" - ) - now = datetime(2026, 7, 29, 9, 1, tzinfo=timezone(timedelta(hours=9))) - quote = { - "market": "KR", - "symbol": "005930", - "last_price": "64000", - "best_bid": "63900", - "best_ask": "64000", - "trade_timestamp": now.isoformat(), - "book_timestamp": now.isoformat(), - "received_at": now.isoformat(), - } - decision = exit_decision( - position, - quote, - policy, - observed_at=now, - regular_close=datetime( - 2026, 7, 29, 15, 30, tzinfo=timezone(timedelta(hours=9)) - ), - first_valid_quote_of_session=True, - daily_loss_breached=False, - ) - assert decision["reason"] == "STOP_GAP" - assert decision["quantity"] == "4" - assert decision["limit_price"] == "63900" - assert len(exit_order_id(decision, attempt=0)) == 32 - assert marked_daily_pnl_krw( - position, - best_bid="64000", - estimated_exit_cost_bps="30", - ) < 0 - position = record_exit_fill( - position, - cumulative_exited_quantity="2", - average_exit_price="64000", - ) - assert position["status"] == "EXIT_PARTIAL" - assert open_quantity(position) == Decimal(2) - carried = roll_position_session(position, session_date="2026-07-30") - assert carried["status"] == "CARRY_EXIT_ONLY" - with tempfile.TemporaryDirectory(prefix="qta-position-ledger-") as temporary: - ledger = PositionLedger(Path(temporary) / "positions.sqlite3") - try: - ledger.put( - carried, - event_type="POSITION_CARRIED", - event_payload={"session_date": "2026-07-30"}, - ) - assert ledger.open_positions()[0].status == "CARRY_EXIT_ONLY" - assert ledger.events(carried["position_id"])[0]["event_type"] == ( - "POSITION_CARRIED" - ) - finally: - ledger.close() - print( - json.dumps( - { - "schema": STRATEGY_POSITION_SCHEMA, - "self_test": "PASS", - "decision_hash": decision["decision_hash"], - "live_enabled": False, - "api_mutation_count": 0, - }, - sort_keys=True, - ) - ) - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--self-test", action="store_true") - subparsers = parser.add_subparsers(dest="command") - reconcile_parser = subparsers.add_parser("reconcile-account") - reconcile_parser.add_argument("--ledger", required=True) - reconcile_parser.add_argument("--account-snapshot", required=True) - reconcile_parser.add_argument("--session-date", required=True) - reconcile_parser.add_argument("--output", required=True) - args = parser.parse_args(argv) - try: - if args.self_test: - self_test() - return 0 - if args.command != "reconcile-account": - parser.error("use --self-test or reconcile-account") - paths = { - name: Path(getattr(args, name)) - for name in ("ledger", "account_snapshot", "output") - } - if any(not path.is_absolute() for path in paths.values()): - raise BlockedError("ledger, account snapshot, and output paths must be absolute") - if ( - not paths["account_snapshot"].is_file() - or paths["account_snapshot"].is_symlink() - ): - raise BlockedError( - "account snapshot must be a regular non-symlink file" - ) - account_value = json.loads( - paths["account_snapshot"].read_text(encoding="utf-8") - ) - if not isinstance(account_value, dict): - raise BlockedError("account snapshot must contain one JSON object") - ledger = PositionLedger(paths["ledger"]) - try: - receipt = reconcile_account_positions( - ledger, - account_snapshot=account_value, - session_date=args.session_date, - ) - finally: - ledger.close() - atomic_write_json(paths["output"], receipt) - print(json.dumps(receipt, sort_keys=True)) - return 0 - except ( - AssertionError, - BlockedError, - ValueError, - OSError, - json.JSONDecodeError, - ) as exc: - print( - json.dumps( - { - "status": "BLOCKED", - "reason": str(exc), - "api_mutation_count": 0, - "live_enabled": False, - }, - sort_keys=True, - ) - ) - return 2 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/skills/quant-stock-polling-trader/scripts/reconcile.py b/skills/quant-stock-polling-trader/scripts/reconcile.py deleted file mode 100755 index 7251d7d..0000000 --- a/skills/quant-stock-polling-trader/scripts/reconcile.py +++ /dev/null @@ -1,390 +0,0 @@ -#!/usr/bin/env python3 -"""Reconcile local nonterminal intents against broker order records.""" - -from __future__ import annotations - -import argparse -import json -import sys -import tempfile -from datetime import datetime, timezone -from pathlib import Path -from typing import Any -from unittest.mock import patch - -from broker_adapters import TransportFailure -from execution_core import ( - BlockedError, - Ledger, - SingleWriterLock, - canonical_json, - emit_json, - load_json_object, - plan_orders as build_order_plan, -) -from broker_credentials import load_kis_credentials, require_kis_runtime_credentials -from plan_orders import fixture_inputs -from run_session import ( - create_broker, - current_order, - normalize_venue_map, - parse_timestamp, - validate_broker_binding, - validate_plan, - venue_for, -) - -RECONCILIATION_SCHEMA = "qta-reconciliation-status/v2" - - -def latest_ack(ledger: Ledger, intent_id: str) -> dict[str, Any] | None: - for event in reversed(ledger.events(intent_id)): - if event["event_type"] == "STATE_ACKNOWLEDGED": - ack = event["payload"].get("ack") - return ack if isinstance(ack, dict) else None - return None - - -def reconcile( - plan: dict[str, Any], - broker_name: str, - venues: dict[str, str], - state_directory: Path, - *, - broker: Any | None = None, -) -> dict[str, Any]: - validate_plan(plan) - validate_broker_binding(plan, broker_name) - for intent in plan["intents"]: - venue_for(venues, intent) - state_directory.mkdir(parents=True, exist_ok=True) - trading_date = ( - parse_timestamp(plan["entry_window"]["start"], "entry window start") - .date() - .isoformat() - ) - broker = broker or create_broker(broker_name) - changes: list[dict[str, Any]] = [] - blocked: list[dict[str, Any]] = [] - - with SingleWriterLock(state_directory / "writer.lock"): - ledger = Ledger(state_directory / "ledger.sqlite3") - try: - intent_by_id = {intent["intent_id"]: intent for intent in plan["intents"]} - for record in ledger.nonterminal(): - intent = intent_by_id.get(record.intent_id) - if intent is None: - raise BlockedError( - f"ledger intent is absent from plan: {record.intent_id}" - ) - if record.state in { - "PLANNED", - "WAIT_TRIGGER", - "RESERVED", - "SUBMITTING", - }: - if record.state == "SUBMITTING": - ledger.transition( - record.intent_id, - "UNKNOWN", - {"reason": "restart_during_submitting"}, - ) - ledger.transition(record.intent_id, "RECONCILING", {}) - ledger.transition( - record.intent_id, - "MANUAL_BLOCK", - {"reason": "restart_during_submitting"}, - ) - else: - ledger.transition( - record.intent_id, - "MANUAL_BLOCK", - {"reason": f"restart_from_{record.state.lower()}"}, - ) - blocked.append( - { - "intent_id": record.intent_id, - "reason": f"unsafe restart state {record.state}", - } - ) - continue - if record.state == "UNKNOWN" and not record.broker_order_id: - ledger.transition(record.intent_id, "RECONCILING", {}) - ledger.transition( - record.intent_id, - "MANUAL_BLOCK", - {"reason": "unknown mutation without broker order id"}, - ) - blocked.append( - { - "intent_id": record.intent_id, - "reason": "unknown mutation without broker order id", - } - ) - continue - ack = latest_ack(ledger, record.intent_id) - if ack is None and record.broker_order_id: - ack = {"broker_order_id": record.broker_order_id} - if not isinstance(ack, dict): - if record.state == "UNKNOWN": - ledger.transition(record.intent_id, "RECONCILING", {}) - ledger.transition( - record.intent_id, - "MANUAL_BLOCK", - {"reason": "broker acknowledgement unavailable"}, - ) - blocked.append( - { - "intent_id": record.intent_id, - "reason": "broker acknowledgement unavailable", - } - ) - continue - if record.state == "UNKNOWN": - ledger.transition(record.intent_id, "RECONCILING", {}) - record = ledger.get(record.intent_id) - try: - snapshot = current_order( - broker, - broker_name, - intent=intent, - ack=ack, - venue=venue_for(venues, intent), - trading_date=trading_date, - ) - except (BlockedError, TransportFailure) as exc: - ledger.append_event( - record.intent_id, - "RECONCILE_BLOCKED", - {"reason": str(exc)}, - ) - blocked.append({"intent_id": record.intent_id, "reason": str(exc)}) - continue - next_state = snapshot["normalized_status"] - if next_state == "UNKNOWN": - if record.state != "RECONCILING": - ledger.transition(record.intent_id, "UNKNOWN", snapshot) - ledger.transition(record.intent_id, "RECONCILING", {}) - ledger.transition( - record.intent_id, - "MANUAL_BLOCK", - {"snapshot": snapshot}, - ) - blocked.append( - { - "intent_id": record.intent_id, - "reason": "broker status is unknown", - } - ) - continue - if record.state == "CANCEL_PENDING" and next_state in { - "ACKNOWLEDGED", - "CANCEL_PENDING", - "PARTIALLY_FILLED", - }: - ledger.append_event( - record.intent_id, "CANCEL_STILL_PENDING", snapshot - ) - blocked.append( - { - "intent_id": record.intent_id, - "reason": ( - "cancel remains pending while broker order is " - f"{next_state.lower()}" - ), - } - ) - continue - if next_state == record.state: - ledger.append_event( - record.intent_id, "RECONCILE_NO_CHANGE", snapshot - ) - continue - ledger.transition(record.intent_id, next_state, {"snapshot": snapshot}) - changes.append( - { - "intent_id": record.intent_id, - "from": record.state, - "to": next_state, - } - ) - final = [ - { - "intent_id": intent["intent_id"], - "state": ledger.get(intent["intent_id"]).state, - "broker_order_id": ledger.get(intent["intent_id"]).broker_order_id, - } - for intent in plan["intents"] - ] - finally: - ledger.close() - output = { - "schema": RECONCILIATION_SCHEMA, - "plan_hash": plan["plan_hash"], - "broker": broker_name, - "reconciled_at": datetime.now(timezone.utc).isoformat(), - "status": "BLOCKED" if blocked else "READY", - "changes": changes, - "blocked": blocked, - "intents": final, - } - return output - - -def self_test() -> None: - plan = build_order_plan(*fixture_inputs()) - intent = plan["intents"][0] - venues = {"KR:005930": "KRX"} - try: - reconcile( - plan, - "toss", - venues, - Path("/unused/reconcile-state"), - broker=object(), - ) - except BlockedError as exc: - assert "plan broker does not match selected adapter" in str(exc) - else: - raise AssertionError("reconcile must bind the adapter to the plan broker") - with patch.dict("os.environ", {}, clear=False): - with tempfile.TemporaryDirectory(prefix="qta-reconcile-") as directory: - ledger = Ledger(Path(directory) / "ledger.sqlite3") - ledger.create_intent(plan["plan_hash"], intent) - ledger.transition(intent["intent_id"], "WAIT_TRIGGER", {}) - ledger.transition(intent["intent_id"], "RESERVED", {}) - ledger.transition( - intent["intent_id"], - "SUBMITTING", - {"request_hash": "a" * 64}, - request_hash="a" * 64, - ) - ledger.transition( - intent["intent_id"], "UNKNOWN", {"reason": "accepted then timeout"} - ) - ledger.close() - - class NeverCalledBroker: - pass - - receipt = reconcile( - plan, - "kis-paper", - venues, - Path(directory), - broker=NeverCalledBroker(), - ) - assert receipt["status"] == "BLOCKED" - assert receipt["intents"][0]["state"] == "MANUAL_BLOCK" - - for pending_status in ( - "ACKNOWLEDGED", - "CANCEL_PENDING", - "PARTIALLY_FILLED", - ): - with tempfile.TemporaryDirectory(prefix="qta-cancel-pending-") as directory: - ledger = Ledger(Path(directory) / "ledger.sqlite3") - ledger.create_intent(plan["plan_hash"], intent) - ledger.transition(intent["intent_id"], "WAIT_TRIGGER", {}) - ledger.transition(intent["intent_id"], "RESERVED", {}) - ledger.transition( - intent["intent_id"], - "SUBMITTING", - {"request_hash": "b" * 64}, - request_hash="b" * 64, - ) - ledger.transition( - intent["intent_id"], - "ACKNOWLEDGED", - {"ack": {"broker_order_id": "fixture-order"}}, - broker_order_id="fixture-order", - ) - ledger.transition(intent["intent_id"], "CANCEL_PENDING", {}) - ledger.close() - - class PendingCancelBroker: - def get_order(self, **_: Any) -> dict[str, Any]: - return {"normalized_status": pending_status} - - receipt = reconcile( - plan, - "kis-paper", - venues, - Path(directory), - broker=PendingCancelBroker(), - ) - assert receipt["status"] == "BLOCKED" - assert receipt["intents"][0]["state"] == "CANCEL_PENDING" - print( - canonical_json( - { - "self_test": "PASS", - "schema": RECONCILIATION_SCHEMA, - "unknown_without_order_id": "MANUAL_BLOCK", - } - ) - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--plan") - parser.add_argument("--broker", choices=("toss", "kis-paper", "kis-live")) - parser.add_argument("--venue-map") - parser.add_argument("--state-dir") - parser.add_argument("--output") - parser.add_argument("--self-test", action="store_true") - return parser.parse_args() - - -def main() -> int: - args = parse_args() - if args.self_test: - self_test() - return 0 - missing = [ - name - for name in ("plan", "broker", "state_dir") - if not getattr(args, name) - ] - if missing: - emit_json( - { - "schema": RECONCILIATION_SCHEMA, - "status": "BLOCKED", - "reason": "missing arguments: " + ", ".join(missing), - }, - args.output, - ) - return 2 - try: - if args.broker in {"kis-paper", "kis-live"}: - require_kis_runtime_credentials( - load_kis_credentials(args.broker.split("-", 1)[1]) - ) - receipt = reconcile( - load_json_object(args.plan), - args.broker, - ( - normalize_venue_map(load_json_object(args.venue_map)) - if args.venue_map - else {} - ), - Path(args.state_dir).resolve(), - ) - except (BlockedError, OSError, ValueError, json.JSONDecodeError) as exc: - emit_json( - { - "schema": RECONCILIATION_SCHEMA, - "status": "BLOCKED", - "reason": str(exc), - }, - args.output, - ) - return 2 - emit_json(receipt, args.output) - return 0 if receipt["status"] == "READY" else 2 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/skills/quant-stock-polling-trader/scripts/run_session.py b/skills/quant-stock-polling-trader/scripts/run_session.py deleted file mode 100755 index 1a500b3..0000000 --- a/skills/quant-stock-polling-trader/scripts/run_session.py +++ /dev/null @@ -1,2655 +0,0 @@ -#!/usr/bin/env python3 -"""Preview or run a deterministic first-hour shadow/paper polling session.""" - -from __future__ import annotations - -import argparse -import json -import os -import sys -import time -from datetime import datetime, timedelta, timezone -from decimal import ROUND_CEILING, Decimal -from pathlib import Path -from typing import Any -from zoneinfo import ZoneInfo - -from broker_adapters import ( - AmbiguousMutationError, - AuthoritativeMutationRejection, - KisBroker, - TossBroker, - TransportFailure, -) -from broker_credentials import load_kis_credentials, require_kis_runtime_credentials -from execution_core import ( - ENTRY_WINDOWS, - EXECUTION_VERSION, - MARKET_CURRENCY, - MAX_SNAPSHOT_AGE_SECONDS, - PLAN_SCHEMA, - QTA2_REGIME_MAX_AGE_SECONDS, - QTA2_REGIME_MINIMUM_CHANGE_BPS, - QTA2_REGIME_METRIC, - QTA_METHOD_V2, - SCREEN_SCHEMA_V2, - V2_EXCHANGE_CONTRACTS, - V2_EXCHANGES_BY_MARKET, - BlockedError, - Ledger, - SingleWriterLock, - canonical_json, - decimal_value, - emit_json, - exact_fields, - load_json_object, - nonnegative_decimal, - normalize_symbol, - normalized_execution_policy, - normalized_market_session, - parse_aware_datetime, - parse_iso_date, - plan_orders as rebuild_order_plan, - positive_decimal, - redact, - sha256_json, - validate_nonempty_string, - validate_sha256, -) -from position_lifecycle import ( - PositionLedger, - position_from_intent, - reconcile_account_positions, - record_entry_fill, - strategy_position_id, -) - -VENUE_SCHEMA = "qta-venue-map/v1" -SESSION_STATUS_SCHEMA = "qta-session-status/v2" -PREOPEN_WARMUP_SECONDS = 30 -SUBMIT_HTTP_TIMEOUT_SECONDS = 10 -DEFAULT_QUOTE_HTTP_REQUESTS_PER_INTENT = 2 -BROKER_PACING_SECONDS = { - "toss": Decimal("0.1"), - "kis-paper": Decimal("1.05"), - "kis-live": Decimal("0.12"), -} -PLAN_FIELDS = { - "schema", - "plan_status", - "execution_version", - "context", - "entry_window", - "quote_policy", - "order_policy", - "settled_cash_start", - "borrowed_buying_power_excluded", - "settled_cash_unreserved", - "intents", - "skipped", - "frozen_inputs", - "plan_hash", -} -FROZEN_INPUT_FIELDS = { - "screen", - "account", - "exposure", - "risk", - "execution", -} -BASE_CONTEXT_FIELDS = { - "execution_version", - "screen_hash", - "account_hash", - "exposure_hash", - "risk_hash", - "execution_policy_hash", - "execution_policy", - "market_session", - "market_session_hash", - "snapshot_max_age_seconds", - "broker", - "environment", - "account_alias", - "market", -} -V2_CONTEXT_FIELDS = BASE_CONTEXT_FIELDS | { - "screen_schema", - "candidate_order_contract", - "exchange_order", - "broker_symbol_qualification", - "analysis_date", - "snapshot_as_of", -} -ENTRY_WINDOW_FIELDS = { - "timezone", - "start", - "end", - "poll_interval_seconds", -} -QUOTE_POLICY_FIELDS = { - "max_age_seconds", - "max_spread_bps", - "max_gap_bps", - "trigger_mode", -} -ORDER_POLICY_FIELDS = { - "ttl_seconds", - "allow_partial_fill", - "cancel_remainder_at_window_end", -} -BASE_INTENT_FIELDS = { - "intent_id", - "client_order_id", - "rank", - "market", - "symbol", - "currency", - "side", - "order_type", - "time_in_force", - "quantity", - "entry_trigger", - "limit_price", - "stop_price", - "take_profit_price", - "effective_loss_per_share", - "reserved_cash", - "qta_payload_hash", - "initial_state", - "intent_hash", -} -V2_INTENT_FIELDS = BASE_INTENT_FIELDS | { - "exchange", - "exchange_rank", - "canonical_symbol", - "data_symbol", - "broker_symbol", - "instrument_type", - "benchmark_id", - "venue", - "resolved_tick_size", - "tick_contract_hash", -} -SKIP_REASONS = { - "duplicate_planned_exposure", - "existing_open_buy_order", - "existing_exposure", - "max_concurrent_positions", - "quantity_below_one", -} -MARKET_REGIME_QUOTE_FIELDS = { - "schema", - "broker", - "exchange", - "benchmark_id", - "regime_proxy_id", - "provider_symbol", - "current", - "previous_close", - "change_bps", - "source_timestamp", - "received_at", - "raw_status", -} -REGIME_PROXY_BY_EXCHANGE = { - "KOSPI": "KOSPI_COMPOSITE", - "KOSDAQ": "KOSDAQ_COMPOSITE", - "NYSE": "S&P_500", - "NASDAQ": "NASDAQ_COMPOSITE", -} - - -def validate_positive_integer(value: Any, field: str) -> int: - if isinstance(value, bool) or not isinstance(value, int) or value <= 0: - raise BlockedError(f"{field} must be a positive integer") - return value - - -def validate_decimal_text( - value: Any, - field: str, - *, - positive: bool, -) -> Decimal: - if not isinstance(value, str) or not value or value != value.strip(): - raise BlockedError(f"{field} must be a decimal string") - result = ( - positive_decimal(value, field) - if positive - else nonnegative_decimal(value, field) - ) - if format(result, "f") != value: - raise BlockedError(f"{field} must be a canonical decimal string") - return result - - -def validate_intent( - intent: Any, - *, - index: int, - context: dict[str, Any], - is_v2: bool, -) -> tuple[int, Decimal, tuple[str, str]]: - label = f"plan.intents[{index}]" - if not isinstance(intent, dict): - raise BlockedError(f"{label} must be an object") - exact_fields( - intent, - V2_INTENT_FIELDS if is_v2 else BASE_INTENT_FIELDS, - label, - ) - validate_sha256(intent["intent_hash"], f"{label}.intent_hash") - expected_intent_hash = sha256_json( - {key: value for key, value in intent.items() if key != "intent_hash"} - ) - if intent["intent_hash"] != expected_intent_hash: - raise BlockedError(f"{label}.intent_hash does not match intent contents") - validate_sha256(intent["qta_payload_hash"], f"{label}.qta_payload_hash") - - market = context["market"] - if intent["market"] != market: - raise BlockedError(f"{label}.market must equal plan.context.market") - symbol = normalize_symbol(market, str(intent["symbol"])) - if symbol != intent["symbol"]: - raise BlockedError(f"{label}.symbol must be normalized") - if intent["currency"] != MARKET_CURRENCY[market]: - raise BlockedError(f"{label}.currency does not match market") - if ( - intent["side"] != "BUY" - or intent["order_type"] != "LIMIT" - or intent["time_in_force"] != "DAY" - or intent["initial_state"] != "PLANNED" - ): - raise BlockedError(f"{label} must be BUY, LIMIT, DAY, and initially PLANNED") - rank = validate_positive_integer(intent["rank"], f"{label}.rank") - quantity_text = intent["quantity"] - if ( - not isinstance(quantity_text, str) - or not quantity_text.isdigit() - or str(int(quantity_text)) != quantity_text - or int(quantity_text) <= 0 - ): - raise BlockedError(f"{label}.quantity must be a canonical positive integer") - quantity = Decimal(quantity_text) - entry = validate_decimal_text( - intent["entry_trigger"], - f"{label}.entry_trigger", - positive=True, - ) - limit_price = validate_decimal_text( - intent["limit_price"], - f"{label}.limit_price", - positive=True, - ) - stop = validate_decimal_text( - intent["stop_price"], - f"{label}.stop_price", - positive=True, - ) - take_profit = validate_decimal_text( - intent["take_profit_price"], - f"{label}.take_profit_price", - positive=True, - ) - effective_loss = validate_decimal_text( - intent["effective_loss_per_share"], - f"{label}.effective_loss_per_share", - positive=True, - ) - reserved_cash = validate_decimal_text( - intent["reserved_cash"], - f"{label}.reserved_cash", - positive=True, - ) - if not stop < entry <= limit_price < take_profit: - raise BlockedError( - f"{label} prices must satisfy stop < entry <= limit < take_profit" - ) - if effective_loss < limit_price - stop: - raise BlockedError(f"{label}.effective_loss_per_share is below price loss") - if reserved_cash < limit_price * quantity: - raise BlockedError(f"{label}.reserved_cash is below limit notional") - - intent_seed = { - "plan_seed": sha256_json(context), - "rank": rank, - "market": market, - "symbol": symbol, - "side": "BUY", - "order_type": "LIMIT", - "time_in_force": "DAY", - "quantity": quantity_text, - "limit_price": intent["limit_price"], - } - exposure_namespace = market - if is_v2: - exchange = intent["exchange"] - contract = V2_EXCHANGE_CONTRACTS.get(exchange) - if contract is None or contract["market"] != market: - raise BlockedError(f"{label}.exchange does not match market") - exchange_rank = validate_positive_integer( - intent["exchange_rank"], - f"{label}.exchange_rank", - ) - canonical_symbol = normalize_symbol( - market, - validate_nonempty_string( - intent["canonical_symbol"], - f"{label}.canonical_symbol", - ), - ) - broker_symbol = normalize_symbol( - market, - validate_nonempty_string( - intent["broker_symbol"], - f"{label}.broker_symbol", - ), - ) - if canonical_symbol != intent["canonical_symbol"]: - raise BlockedError(f"{label}.canonical_symbol must be normalized") - if broker_symbol != intent["broker_symbol"] or symbol != broker_symbol: - raise BlockedError( - f"{label}.symbol and broker_symbol must match and be normalized" - ) - validate_nonempty_string(intent["data_symbol"], f"{label}.data_symbol") - if intent["instrument_type"] not in {"COMMON", "ADR", "REIT"}: - raise BlockedError(f"{label}.instrument_type is unsupported") - if ( - intent["benchmark_id"] != contract["benchmark_id"] - or intent["venue"] != contract["venue"] - ): - raise BlockedError(f"{label}.benchmark_id and venue must match exchange") - validate_sha256( - intent["tick_contract_hash"], - f"{label}.tick_contract_hash", - ) - tick_size = validate_decimal_text( - intent["resolved_tick_size"], - f"{label}.resolved_tick_size", - positive=True, - ) - for price, field in ( - (entry, "entry_trigger"), - (limit_price, "limit_price"), - (stop, "stop_price"), - (take_profit, "take_profit_price"), - ): - if price % tick_size != 0: - raise BlockedError( - f"{label}.{field} must align to resolved_tick_size" - ) - intent_seed.update( - { - "exchange": exchange, - "exchange_rank": exchange_rank, - "canonical_symbol": canonical_symbol, - "data_symbol": intent["data_symbol"], - "broker_symbol": broker_symbol, - "venue": intent["venue"], - } - ) - exposure_namespace = exchange - expected_intent_id = sha256_json(intent_seed)[:32] - if intent["intent_id"] != expected_intent_id: - raise BlockedError(f"{label}.intent_id does not match deterministic seed") - if intent["client_order_id"] != f"qta-{expected_intent_id[:28]}": - raise BlockedError( - f"{label}.client_order_id does not match deterministic intent_id" - ) - return rank, reserved_cash, (exposure_namespace, symbol) - - -def validate_skipped( - skipped: Any, - *, - market: str, -) -> None: - if not isinstance(skipped, list): - raise BlockedError("plan.skipped must be an array") - for index, item in enumerate(skipped): - label = f"plan.skipped[{index}]" - if not isinstance(item, dict): - raise BlockedError(f"{label} must be an object") - base_fields = {"market", "symbol", "reason"} - extended_fields = base_fields | { - "exchange", - "exchange_rank", - "canonical_symbol", - } - item_fields = frozenset(item) - if item_fields not in { - frozenset(base_fields), - frozenset(extended_fields), - }: - raise BlockedError(f"{label} fields mismatch") - if item["market"] != market: - raise BlockedError(f"{label}.market must equal plan market") - if normalize_symbol(market, str(item["symbol"])) != item["symbol"]: - raise BlockedError(f"{label}.symbol must be normalized") - if item["reason"] not in SKIP_REASONS: - raise BlockedError(f"{label}.reason is unsupported") - if item_fields == frozenset(extended_fields): - exchange = item["exchange"] - contract = V2_EXCHANGE_CONTRACTS.get(exchange) - if contract is None or contract["market"] != market: - raise BlockedError(f"{label}.exchange does not match market") - validate_positive_integer( - item["exchange_rank"], - f"{label}.exchange_rank", - ) - canonical_symbol = normalize_symbol( - market, - str(item["canonical_symbol"]), - ) - if canonical_symbol != item["canonical_symbol"]: - raise BlockedError(f"{label}.canonical_symbol must be normalized") - - -def validate_plan(plan: dict[str, Any]) -> None: - if not isinstance(plan, dict): - raise BlockedError("plan must be an object") - exact_fields(plan, PLAN_FIELDS, "plan") - if plan.get("schema") != PLAN_SCHEMA: - raise BlockedError(f"plan schema must be {PLAN_SCHEMA}") - if plan.get("plan_status") not in {"READY", "NO_ORDERS"}: - raise BlockedError("plan_status must be READY or NO_ORDERS") - if plan["execution_version"] != EXECUTION_VERSION: - raise BlockedError(f"execution_version must be {EXECUTION_VERSION}") - claimed = plan.get("plan_hash") - validate_sha256(claimed, "plan_hash") - unhashed = dict(plan) - unhashed.pop("plan_hash") - if sha256_json(unhashed) != claimed: - raise BlockedError("plan_hash does not match plan contents") - - context = plan.get("context") - if not isinstance(context, dict): - raise BlockedError("plan.context must be an object") - is_v2 = context.get("screen_schema") == SCREEN_SCHEMA_V2 - exact_fields( - context, - V2_CONTEXT_FIELDS if is_v2 else BASE_CONTEXT_FIELDS, - "plan.context", - ) - if context["execution_version"] != plan["execution_version"]: - raise BlockedError( - "plan.context.execution_version must equal plan.execution_version" - ) - for field in ( - "screen_hash", - "account_hash", - "exposure_hash", - "risk_hash", - "execution_policy_hash", - ): - validate_sha256(context[field], f"plan.context.{field}") - validate_nonempty_string(context["account_alias"], "plan.context.account_alias") - if context["broker"] not in {"kis", "toss"}: - raise BlockedError("plan.context.broker must be kis or toss") - if context["environment"] not in {"paper", "shadow", "live"}: - raise BlockedError("plan.context.environment must be paper, shadow, or live") - market = context.get("market") - if market not in ENTRY_WINDOWS: - raise BlockedError("plan.context.market is unsupported") - snapshot_max_age = validate_positive_integer( - context["snapshot_max_age_seconds"], - "plan.context.snapshot_max_age_seconds", - ) - if snapshot_max_age > MAX_SNAPSHOT_AGE_SECONDS: - raise BlockedError( - "plan.context.snapshot_max_age_seconds exceeds the hard maximum" - ) - market_session = context.get("market_session") - normalized_session = normalized_market_session( - market_session, - expected_market=market, - expected_timezone=ENTRY_WINDOWS[market]["timezone"], - ) - if normalized_session != market_session: - raise BlockedError("plan.context.market_session is not normalized") - if context.get("market_session_hash") != normalized_session["session_hash"]: - raise BlockedError( - "plan.context.market_session_hash does not match market_session" - ) - execution_policy = context["execution_policy"] - if not isinstance(execution_policy, dict): - raise BlockedError("plan.context.execution_policy must be an object") - normalized_policy = normalized_execution_policy(execution_policy) - if normalized_policy != execution_policy: - raise BlockedError("plan.context.execution_policy is not normalized") - if sha256_json(normalized_policy) != context["execution_policy_hash"]: - raise BlockedError( - "plan.context.execution_policy_hash does not match execution_policy" - ) - if ( - normalized_policy["market"] != market - or normalized_policy["market_session"] != normalized_session - or normalized_policy["snapshot_max_age_seconds"] != snapshot_max_age - ): - raise BlockedError("plan.context execution policy duplicates are inconsistent") - if is_v2: - if ( - context["candidate_order_contract"] - != "exchange_contract_order_then_exchange_rank_then_broker_symbol" - or context["exchange_order"] != list(V2_EXCHANGES_BY_MARKET[market]) - or context["broker_symbol_qualification"] != "KIS" - or context["broker"] != "kis" - ): - raise BlockedError("qta-screen/v2 plan context contract is invalid") - analysis_date = parse_iso_date( - context["analysis_date"], - "plan.context.analysis_date", - ) - if analysis_date.isoformat() != normalized_session["previous_session_date"]: - raise BlockedError( - "plan.context.analysis_date must equal previous_session_date" - ) - snapshot_as_of = parse_aware_datetime( - context["snapshot_as_of"], - "plan.context.snapshot_as_of", - ) - session_zone = ZoneInfo(normalized_session["timezone"]) - if ( - snapshot_as_of.astimezone(session_zone).date().isoformat() - != normalized_session["session_date"] - ): - raise BlockedError("plan.context.snapshot_as_of must fall on session_date") - source_as_of = parse_aware_datetime( - normalized_session["source_as_of"], - "market_session.source_as_of", - ) - regular_open = parse_aware_datetime( - normalized_session["regular_open"], - "market_session.regular_open", - ) - if not source_as_of <= snapshot_as_of <= regular_open: - raise BlockedError( - "plan.context.snapshot_as_of must be between source and open" - ) - if (regular_open - snapshot_as_of).total_seconds() > snapshot_max_age: - raise BlockedError("plan.context.snapshot_as_of exceeds snapshot freshness") - - entry_window = plan["entry_window"] - if not isinstance(entry_window, dict): - raise BlockedError("plan.entry_window must be an object") - exact_fields(entry_window, ENTRY_WINDOW_FIELDS, "plan.entry_window") - if entry_window["timezone"] != ENTRY_WINDOWS[market]["timezone"]: - raise BlockedError("plan.entry_window.timezone does not match market") - window_start = parse_aware_datetime( - entry_window["start"], "plan.entry_window.start" - ) - window_end = parse_aware_datetime(entry_window["end"], "plan.entry_window.end") - regular_open = parse_aware_datetime( - normalized_session["regular_open"], - "market_session.regular_open", - ) - regular_close = parse_aware_datetime( - normalized_session["regular_close"], - "market_session.regular_close", - ) - if ( - window_start != regular_open - or window_end != regular_open + timedelta(hours=1) - or window_end > regular_close - ): - raise BlockedError( - "plan.entry_window must be the first hour of the regular session" - ) - validate_positive_integer( - entry_window["poll_interval_seconds"], - "plan.entry_window.poll_interval_seconds", - ) - if entry_window != { - "timezone": normalized_policy["timezone"], - "start": normalized_policy["entry_window_start"], - "end": normalized_policy["entry_window_end"], - "poll_interval_seconds": normalized_policy["poll_interval_seconds"], - }: - raise BlockedError( - "plan.entry_window does not match the embedded execution policy" - ) - - quote_policy = plan["quote_policy"] - if not isinstance(quote_policy, dict): - raise BlockedError("plan.quote_policy must be an object") - exact_fields(quote_policy, QUOTE_POLICY_FIELDS, "plan.quote_policy") - validate_positive_integer( - quote_policy["max_age_seconds"], - "plan.quote_policy.max_age_seconds", - ) - for field in ("max_spread_bps", "max_gap_bps"): - value = validate_decimal_text( - quote_policy[field], - f"plan.quote_policy.{field}", - positive=False, - ) - if value > Decimal(10000): - raise BlockedError(f"plan.quote_policy.{field} must be <= 10000") - if quote_policy["trigger_mode"] not in {"AT_OR_ABOVE", "CROSS_FROM_BELOW"}: - raise BlockedError("plan.quote_policy.trigger_mode is unsupported") - if quote_policy != { - "max_age_seconds": normalized_policy["quote_max_age_seconds"], - "max_spread_bps": normalized_policy["max_spread_bps"], - "max_gap_bps": normalized_policy["max_gap_bps"], - "trigger_mode": normalized_policy["trigger_mode"], - }: - raise BlockedError( - "plan.quote_policy does not match the embedded execution policy" - ) - - order_policy = plan["order_policy"] - if not isinstance(order_policy, dict): - raise BlockedError("plan.order_policy must be an object") - exact_fields(order_policy, ORDER_POLICY_FIELDS, "plan.order_policy") - validate_positive_integer( - order_policy["ttl_seconds"], - "plan.order_policy.ttl_seconds", - ) - for field in ("allow_partial_fill", "cancel_remainder_at_window_end"): - if not isinstance(order_policy[field], bool): - raise BlockedError(f"plan.order_policy.{field} must be boolean") - if order_policy != { - "ttl_seconds": normalized_policy["order_ttl_seconds"], - "allow_partial_fill": normalized_policy["allow_partial_fill"], - "cancel_remainder_at_window_end": normalized_policy[ - "cancel_remainder_at_window_end" - ], - }: - raise BlockedError( - "plan.order_policy does not match the embedded execution policy" - ) - - settled_cash_start = validate_decimal_text( - plan["settled_cash_start"], - "plan.settled_cash_start", - positive=False, - ) - validate_decimal_text( - plan["borrowed_buying_power_excluded"], - "plan.borrowed_buying_power_excluded", - positive=False, - ) - settled_cash_unreserved = validate_decimal_text( - plan["settled_cash_unreserved"], - "plan.settled_cash_unreserved", - positive=False, - ) - if settled_cash_unreserved > settled_cash_start: - raise BlockedError( - "plan.settled_cash_unreserved must not exceed settled_cash_start" - ) - intents = plan["intents"] - if not isinstance(intents, list): - raise BlockedError("plan.intents must be an array") - if plan["plan_status"] != ("READY" if intents else "NO_ORDERS"): - raise BlockedError("plan_status does not match intents") - intent_ranks: list[int] = [] - intent_keys: set[tuple[str, str]] = set() - reserved_total = Decimal(0) - for index, intent in enumerate(intents): - rank, reserved_cash, exposure_key = validate_intent( - intent, - index=index, - context=context, - is_v2=is_v2, - ) - if exposure_key in intent_keys: - raise BlockedError("plan.intents contain duplicate exposure keys") - intent_keys.add(exposure_key) - intent_ranks.append(rank) - reserved_total += reserved_cash - if intent_ranks != sorted(set(intent_ranks)): - raise BlockedError("plan.intent ranks must be strictly increasing and unique") - if settled_cash_start - reserved_total != settled_cash_unreserved: - raise BlockedError("plan cash totals do not match reserved_cash across intents") - validate_skipped(plan["skipped"], market=market) - frozen_inputs = plan["frozen_inputs"] - if not isinstance(frozen_inputs, dict): - raise BlockedError("plan.frozen_inputs must be an object") - exact_fields(frozen_inputs, FROZEN_INPUT_FIELDS, "plan.frozen_inputs") - if any(not isinstance(frozen_inputs[field], dict) for field in FROZEN_INPUT_FIELDS): - raise BlockedError("every plan.frozen_inputs value must be an object") - rebuilt_plan = rebuild_order_plan( - frozen_inputs["screen"], - frozen_inputs["account"], - frozen_inputs["exposure"], - frozen_inputs["risk"], - frozen_inputs["execution"], - ) - if rebuilt_plan != plan: - raise BlockedError( - "plan does not match deterministic rebuild from frozen_inputs" - ) - - -def validate_broker_binding(plan: dict[str, Any], broker_name: str) -> None: - context = plan["context"] - expected_broker = "toss" if broker_name == "toss" else "kis" - if context["broker"] != expected_broker: - raise BlockedError("plan broker does not match selected adapter") - environment = context["environment"] - if broker_name == "kis-paper": - if environment != "paper": - raise BlockedError("KIS paper adapter requires a paper plan") - elif broker_name == "kis-live": - if environment not in {"shadow", "live"}: - raise BlockedError("KIS live adapter requires a shadow/live plan") - elif broker_name == "toss": - if environment not in {"shadow", "live"}: - raise BlockedError("Toss adapter requires a shadow/live plan") - else: - raise BlockedError(f"unsupported broker: {broker_name}") - - -def normalize_venue_map(value: dict[str, Any]) -> dict[str, str]: - if set(value) != {"schema", "venues"} or value.get("schema") != VENUE_SCHEMA: - raise BlockedError(f"venue map schema must be {VENUE_SCHEMA}") - venues = value["venues"] - if not isinstance(venues, dict): - raise BlockedError("venue map venues must be an object") - normalized: dict[str, str] = {} - for key, venue in venues.items(): - if not isinstance(key, str) or ":" not in key or not isinstance(venue, str): - raise BlockedError( - "venue keys must be MARKET:SYMBOL or EXCHANGE:SYMBOL strings" - ) - namespace, symbol = key.split(":", 1) - normalized_key = f"{namespace.strip().upper()}:{symbol.strip().upper()}" - normalized_venue = venue.strip().upper() - if not namespace.strip() or not symbol.strip() or not normalized_venue: - raise BlockedError("venue keys and values must be non-empty") - if ( - normalized_key in normalized - and normalized[normalized_key] != normalized_venue - ): - raise BlockedError(f"conflicting venue map entry for {normalized_key}") - normalized[normalized_key] = normalized_venue - return dict(sorted(normalized.items())) - - -def venue_for(venues: dict[str, str], intent: dict[str, Any]) -> str: - market = str(intent["market"]).upper() - symbol = str(intent["symbol"]).upper() - market_key = f"{market}:{symbol}" - intent_venue = intent.get("venue") - exchange = intent.get("exchange") - if intent_venue is None: - if market_key not in venues: - raise BlockedError(f"venue map missing {market_key}") - return venues[market_key] - - if not isinstance(intent_venue, str) or not intent_venue.strip(): - raise BlockedError("intent venue must be a non-empty string") - venue = intent_venue.strip().upper() - exchange_contracts = { - "KOSPI": ("KR", "KRX"), - "KOSDAQ": ("KR", "KRX"), - "NYSE": ("US", "NYSE"), - "NASDAQ": ("US", "NASD"), - } - if not isinstance(exchange, str) or exchange.upper() not in exchange_contracts: - raise BlockedError("intent venue requires a supported exchange") - normalized_exchange = exchange.upper() - expected_market, expected_venue = exchange_contracts[normalized_exchange] - if market != expected_market or venue != expected_venue: - raise BlockedError( - f"{normalized_exchange} intent requires market {expected_market} " - f"and venue {expected_venue}" - ) - - exchange_key = f"{normalized_exchange}:{symbol}" - mapped = venues.get(exchange_key, venues.get(market_key)) - if mapped is not None and mapped != venue: - raise BlockedError( - f"venue map {mapped} conflicts with hashed intent venue {venue} " - f"for {exchange_key}" - ) - return venue - - -def parse_timestamp(value: Any, field: str) -> datetime: - if not isinstance(value, str): - raise BlockedError(f"{field} must be an ISO-8601 string") - try: - parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) - except ValueError as exc: - raise BlockedError(f"{field} must be an ISO-8601 string") from exc - if parsed.tzinfo is None: - raise BlockedError(f"{field} must include a timezone") - return parsed - - -def evaluate_quote( - intent: dict[str, Any], - quote: dict[str, Any], - quote_policy: dict[str, Any], - now: datetime, - previous_last: Decimal | None, - *, - require_source_timestamp: bool, -) -> dict[str, Any]: - quality_reasons: list[str] = [] - if quote.get("raw_status") != "OK": - quality_reasons.append("quote_status_not_ok") - if quote.get("market") != intent.get("market") or quote.get("symbol") != intent.get( - "symbol" - ): - quality_reasons.append("quote_instrument_mismatch") - last = decimal_value(quote.get("last_price"), "last_price") - ask = decimal_value(quote.get("best_ask"), "best_ask") - bid = decimal_value(quote.get("best_bid"), "best_bid") - if bid <= 0 or ask <= 0 or last <= 0 or ask < bid: - quality_reasons.append("malformed_bid_ask") - received = parse_timestamp(quote.get("received_at"), "received_at") - age = ( - now.astimezone(timezone.utc) - received.astimezone(timezone.utc) - ).total_seconds() - if age < -1 or age > int(quote_policy["max_age_seconds"]): - quality_reasons.append("stale_quote") - source_timestamp_present = all( - bool(quote.get(field)) for field in ("trade_timestamp", "book_timestamp") - ) - if require_source_timestamp and not source_timestamp_present: - quality_reasons.append("source_timestamp_missing") - source_ages: list[float] = [] - for field in ("trade_timestamp", "book_timestamp"): - value = quote.get(field) - if not value: - continue - try: - source_time = parse_timestamp(value, field) - except BlockedError: - quality_reasons.append(f"{field}_invalid") - continue - source_ages.append( - ( - now.astimezone(timezone.utc) - source_time.astimezone(timezone.utc) - ).total_seconds() - ) - source_age = max(source_ages) if source_ages else None - if any( - item < -1 or item > int(quote_policy["max_age_seconds"]) for item in source_ages - ): - quality_reasons.append("stale_source_quote") - reasons = list(quality_reasons) - midpoint = (ask + bid) / Decimal(2) if ask >= bid > 0 else Decimal(0) - spread_bps = ( - Decimal(10000) * (ask - bid) / midpoint if midpoint > 0 else Decimal("Infinity") - ) - if spread_bps > decimal_value(quote_policy["max_spread_bps"], "max_spread_bps"): - reasons.append("spread_too_wide") - entry = decimal_value(intent["entry_trigger"], "entry_trigger") - limit_price = decimal_value(intent["limit_price"], "limit_price") - if ask > limit_price: - reasons.append("ask_above_limit") - trigger_mode = quote_policy["trigger_mode"] - if trigger_mode == "AT_OR_ABOVE": - crossed = last >= entry - elif trigger_mode == "CROSS_FROM_BELOW": - crossed = previous_last is not None and previous_last < entry <= last - else: - raise BlockedError(f"unsupported trigger_mode: {trigger_mode}") - if not crossed: - reasons.append("entry_not_triggered") - return { - "triggered": not reasons, - "quality_valid": not quality_reasons, - "reasons": reasons, - "last_price": format(last, "f"), - "best_ask": format(ask, "f"), - "best_bid": format(bid, "f"), - "spread_bps": format(spread_bps, "f"), - "quote_age_seconds": f"{age:.6f}", - "source_timestamp_present": source_timestamp_present, - "source_age_seconds": None if source_age is None else f"{source_age:.6f}", - } - - -def normalize_toss_order(raw: dict[str, Any]) -> dict[str, Any]: - raw_status = str(raw.get("status", "UNKNOWN")) - status_map = { - "PENDING": "ACKNOWLEDGED", - "PENDING_CANCEL": "CANCEL_PENDING", - "PENDING_REPLACE": "UNKNOWN", - "PARTIAL_FILLED": "PARTIALLY_FILLED", - "FILLED": "FILLED", - "CANCELED": "CANCELLED", - "REJECTED": "REJECTED", - "CANCEL_REJECTED": "UNKNOWN", - "REPLACE_REJECTED": "UNKNOWN", - "REPLACED": "UNKNOWN", - } - execution = raw.get("execution") - if not isinstance(execution, dict): - execution = {} - return { - "normalized_status": status_map.get(raw_status, "UNKNOWN"), - "raw_status": raw_status, - "filled_quantity": execution.get("filledQuantity"), - "average_fill_price": execution.get("averageFilledPrice"), - "raw": redact(raw), - } - - -def preview_requests( - plan: dict[str, Any], broker_name: str, venues: dict[str, str] -) -> dict[str, Any]: - validate_plan(plan) - previews: list[dict[str, Any]] = [] - if broker_name == "toss": - broker: Any = TossBroker( - client_id="preview", - client_secret="preview", - account_seq=1, - ) - for intent in plan["intents"]: - previews.append( - { - "intent_id": intent["intent_id"], - "request": broker.preview_submit(intent), - } - ) - elif broker_name in {"kis-paper", "kis-live"}: - environment = broker_name.split("-", 1)[1] - broker = KisBroker( - app_key="preview", - app_secret="preview", - account_prefix="00000000", - account_product="01", - environment=environment, - access_token="preview", - ) - for intent in plan["intents"]: - previews.append( - { - "intent_id": intent["intent_id"], - "request": broker.preview_submit( - intent, venue=venue_for(venues, intent) - ), - } - ) - else: - raise BlockedError(f"unsupported broker: {broker_name}") - output = { - "schema": "qta-request-preview/v1", - "status": "READY", - "plan_hash": plan["plan_hash"], - "broker": broker_name, - "account_bound": False, - "request_hash_scope": "placeholder_account", - "trigger_evaluated": False, - "mutation_sent": False, - "previews": previews, - } - output["preview_hash"] = sha256_json(output) - return output - - -def require_environment(name: str) -> str: - value = os.environ.get(name) - if not value: - raise BlockedError(f"required environment variable is absent: {name}") - return value - - -def runtime_toss_account_seq() -> int: - raw = require_environment("QTA_TOSS_ACCOUNT_SEQ") - try: - account_seq = int(raw) - except ValueError as exc: - raise BlockedError( - "QTA_TOSS_ACCOUNT_SEQ must be a canonical positive integer" - ) from exc - if account_seq <= 0 or str(account_seq) != raw: - raise BlockedError("QTA_TOSS_ACCOUNT_SEQ must be a canonical positive integer") - return account_seq - - -def create_broker(broker_name: str) -> Any: - if broker_name == "toss": - return TossBroker( - client_id=require_environment("QTA_TOSS_CLIENT_ID"), - client_secret=require_environment("QTA_TOSS_CLIENT_SECRET"), - account_seq=runtime_toss_account_seq(), - access_token=os.environ.get("QTA_TOSS_ACCESS_TOKEN"), - access_token_expires_at=os.environ.get("QTA_TOSS_ACCESS_TOKEN_EXPIRES_AT"), - ) - if broker_name in {"kis-paper", "kis-live"}: - environment = broker_name.split("-", 1)[1] - return KisBroker( - app_key=require_environment("QTA_KIS_APP_KEY"), - app_secret=require_environment("QTA_KIS_APP_SECRET"), - account_prefix=require_environment("QTA_KIS_ACCOUNT_PREFIX"), - account_product=require_environment("QTA_KIS_ACCOUNT_PRODUCT"), - environment=environment, - access_token=os.environ.get("QTA_KIS_ACCESS_TOKEN"), - ) - raise BlockedError(f"unsupported broker: {broker_name}") - - -def validate_mode(plan: dict[str, Any], broker_name: str, mode: str) -> None: - validate_broker_binding(plan, broker_name) - context = plan["context"] - screen_method = plan["frozen_inputs"]["screen"].get("method_version") - if screen_method == "qta-2.0.0" and mode != "shadow": - raise BlockedError( - "qta-2.0.0 is RESEARCH_ONLY and may run only in shadow mode" - ) - if mode == "live": - raise BlockedError( - "live promotion is intentionally disabled until accepted-timeout " - "recovery, terminal cancel verification, and OCO protection are complete" - ) - if mode == "paper": - if broker_name != "kis-paper" or context.get("environment") != "paper": - raise BlockedError("paper mode requires a KIS paper plan and adapter") - elif mode == "shadow": - if context.get("environment") not in {"shadow", "live"}: - raise BlockedError("shadow mode requires a shadow/live account snapshot") - else: - raise BlockedError("mode must be paper, shadow, or live") - - -def validate_runtime_capabilities(plan: dict[str, Any], broker_name: str) -> None: - if broker_name == "kis-paper" and any( - str(intent.get("market", "")).upper() == "US" - for intent in plan.get("intents", []) - if isinstance(intent, dict) - ): - raise BlockedError( - "KIS U.S. paper sessions are unsupported because " - "HHDFS76200100 order-book quotes have no paper TR; " - "use KIS live production shadow quotes and test paper order " - "serialization separately" - ) - - -def preflight_venues( - plan: dict[str, Any], - venues: dict[str, str], -) -> dict[str, str]: - return { - str(intent["intent_id"]): venue_for(venues, intent) - for intent in plan["intents"] - } - - -def preflight_submit_requests( - broker: Any, - broker_name: str, - plan: dict[str, Any], - venue_by_intent: dict[str, str], -) -> dict[str, dict[str, Any]]: - return { - str(intent["intent_id"]): ( - broker.preview_submit(intent) - if broker_name == "toss" - else broker.preview_submit( - intent, - venue=venue_by_intent[str(intent["intent_id"])], - ) - ) - for intent in plan["intents"] - } - - -def quote_http_request_count( - broker_name: str, - intent: dict[str, Any], -) -> int: - if broker_name == "kis-live" and str(intent.get("market", "")).upper() == "US": - return 1 - return DEFAULT_QUOTE_HTTP_REQUESTS_PER_INTENT - - -def plan_uses_qta2(plan: dict[str, Any]) -> bool: - frozen_inputs = plan.get("frozen_inputs") - if not isinstance(frozen_inputs, dict): - return False - screen = frozen_inputs.get("screen") - return ( - isinstance(screen, dict) - and screen.get("method_version") == QTA_METHOD_V2 - ) - - -def active_qta2_exchanges(plan: dict[str, Any]) -> list[str]: - if not plan_uses_qta2(plan): - return [] - present = { - str(intent.get("exchange") or "").upper() - for intent in plan.get("intents", []) - if isinstance(intent, dict) - } - market = str(plan.get("context", {}).get("market") or "").upper() - return [ - exchange - for exchange in V2_EXCHANGES_BY_MARKET.get(market, ()) - if exchange in present - ] - - -def polling_admission( - plan: dict[str, Any], - broker_name: str, - mode: str, -) -> dict[str, Any]: - """Fail closed when a full deterministic cycle cannot fit broker pacing.""" - if broker_name not in BROKER_PACING_SECONDS: - raise BlockedError(f"unsupported broker: {broker_name}") - intent_count = len(plan["intents"]) - quote_requests = sum( - quote_http_request_count(broker_name, intent) for intent in plan["intents"] - ) - regime_requests = len(active_qta2_exchanges(plan)) - mutation_requests = intent_count if mode == "paper" else 0 - request_budget = quote_requests + regime_requests + mutation_requests - pacing_seconds = BROKER_PACING_SECONDS[broker_name] * request_budget - minimum_interval = max( - 1, - int(pacing_seconds.to_integral_value(rounding=ROUND_CEILING)), - ) - configured_interval = int(plan["entry_window"]["poll_interval_seconds"]) - if configured_interval < minimum_interval: - raise BlockedError( - f"{broker_name} {mode} cycle with {intent_count} intents requires " - f"poll_interval_seconds >= {minimum_interval} for " - f"{request_budget} paced requests; got {configured_interval}" - ) - return { - "intent_count": intent_count, - "quote_request_budget": quote_requests, - "quote_request_contract": ( - "KIS-live US=1; KIS KR and Toss=2 per waiting intent" - ), - "market_regime_request_budget": regime_requests, - "worst_case_mutation_requests": mutation_requests, - "worst_case_request_budget": request_budget, - "minimum_request_interval_ms": int(BROKER_PACING_SECONDS[broker_name] * 1000), - "minimum_poll_interval_seconds": minimum_interval, - "configured_poll_interval_seconds": configured_interval, - "network_and_persistence_headroom_seconds": format( - Decimal(configured_interval) - pacing_seconds, - "f", - ), - } - - -def evaluate_market_regime( - *, - exchange: str, - quote: dict[str, Any], - observed_at: datetime, - session_date: str, -) -> dict[str, Any]: - """Validate and deterministically admit one QTA2 exchange regime quote.""" - exchange = str(exchange).upper() - if exchange not in V2_EXCHANGE_CONTRACTS: - raise BlockedError(f"unsupported market regime exchange: {exchange}") - if observed_at.tzinfo is None or observed_at.utcoffset() is None: - raise BlockedError("market regime observed_at must be timezone-aware") - exact_fields(quote, MARKET_REGIME_QUOTE_FIELDS, "market regime quote") - reasons: list[str] = [] - expected_benchmark = V2_EXCHANGE_CONTRACTS[exchange]["benchmark_id"] - if quote["schema"] != "qta-market-regime-quote/v1": - reasons.append("invalid_schema") - if quote["broker"] != "kis": - reasons.append("invalid_broker") - if quote["exchange"] != exchange: - reasons.append("exchange_mismatch") - if quote["benchmark_id"] != expected_benchmark: - reasons.append("benchmark_mismatch") - if quote["regime_proxy_id"] != REGIME_PROXY_BY_EXCHANGE[exchange]: - reasons.append("proxy_mismatch") - if quote["raw_status"] != "OK": - reasons.append("raw_status_not_ok") - - current = positive_decimal(quote["current"], "market regime current") - previous_close = positive_decimal( - quote["previous_close"], - "market regime previous_close", - ) - change_bps = decimal_value( - quote["change_bps"], - "market regime change_bps", - ) - recomputed_change = ( - (current / previous_close - Decimal(1)) * Decimal(10000) - ) - if abs(change_bps - recomputed_change) > Decimal("0.02"): - reasons.append("change_bps_mismatch") - - zone_name = ( - "Asia/Seoul" - if exchange in {"KOSPI", "KOSDAQ"} - else "America/New_York" - ) - for timestamp_field in ("source_timestamp", "received_at"): - try: - timestamp = parse_aware_datetime( - quote[timestamp_field], - f"market regime {timestamp_field}", - ) - except BlockedError: - reasons.append(f"{timestamp_field}_invalid") - continue - age_seconds = ( - observed_at.astimezone(timezone.utc) - - timestamp.astimezone(timezone.utc) - ).total_seconds() - if age_seconds < -1 or age_seconds > QTA2_REGIME_MAX_AGE_SECONDS: - reasons.append(f"{timestamp_field}_stale") - if ( - timestamp_field == "source_timestamp" - and timestamp.astimezone(ZoneInfo(zone_name)).date().isoformat() - != session_date - ): - reasons.append("source_session_date_mismatch") - - quality_valid = not reasons - admitted = ( - quality_valid - and change_bps >= QTA2_REGIME_MINIMUM_CHANGE_BPS - ) - if quality_valid and not admitted: - reasons.append("benchmark_below_minimum") - without_hash = { - "metric": QTA2_REGIME_METRIC, - "exchange": exchange, - "benchmark_id": expected_benchmark, - "regime_proxy_id": quote["regime_proxy_id"], - "minimum_change_bps": format( - QTA2_REGIME_MINIMUM_CHANGE_BPS, - "f", - ), - "observed_change_bps": format(change_bps, "f"), - "max_age_seconds": QTA2_REGIME_MAX_AGE_SECONDS, - "quality_valid": quality_valid, - "admitted": admitted, - "reasons": reasons, - "quote": quote, - "observed_at": observed_at.isoformat(), - } - return {**without_hash, "decision_hash": sha256_json(without_hash)} - - -def wait_until(target: datetime) -> None: - while True: - remaining = target.timestamp() - datetime.now(timezone.utc).timestamp() - if remaining <= 0: - return - time.sleep(min(remaining, 60)) - - -def warm_broker_before_open(broker: Any, start: datetime) -> dict[str, Any]: - warmup_at = start - timedelta(seconds=PREOPEN_WARMUP_SECONDS) - now = datetime.now(timezone.utc) - if now >= start: - raise BlockedError( - "session runner must start before the regular open for pre-open " - "authentication warm-up" - ) - wait_until(warmup_at) - started_at = datetime.now(timezone.utc) - started_monotonic = time.monotonic() - try: - broker.token() - except (BlockedError, TransportFailure) as exc: - raise BlockedError(f"pre-open broker authentication failed: {exc}") from exc - completed_at = datetime.now(timezone.utc) - if completed_at >= start: - raise BlockedError( - "broker authentication did not complete before the regular open" - ) - return { - "scheduled_at": warmup_at.isoformat(), - "started_at": started_at.isoformat(), - "completed_at": completed_at.isoformat(), - "latency_ms": int((time.monotonic() - started_monotonic) * 1000), - "token_material_logged": False, - } - - -def entry_window_open(end: datetime, now: datetime | None = None) -> bool: - observed = now or datetime.now(timezone.utc) - return observed.timestamp() < end.timestamp() - - -def require_remaining_quantity(snapshot: dict[str, Any]) -> str: - remaining = decimal_value(snapshot.get("remaining_quantity"), "remaining_quantity") - if remaining <= 0 or remaining != remaining.to_integral_value(): - raise BlockedError( - "KIS cancel requires a positive whole current remaining quantity" - ) - return format(remaining, "f") - - -def current_order( - broker: Any, - broker_name: str, - *, - intent: dict[str, Any], - ack: dict[str, Any], - venue: str, - trading_date: str, -) -> dict[str, Any]: - if broker_name == "toss": - return normalize_toss_order(broker.get_order(ack["broker_order_id"])) - return broker.get_order( - market=intent["market"], - trading_date=trading_date, - symbol=intent["symbol"], - venue=venue, - broker_order_id=ack["broker_order_id"], - ) - - -def cancel_order( - broker: Any, - broker_name: str, - *, - intent: dict[str, Any], - ack: dict[str, Any], - venue: str, - quantity: str | None = None, -) -> dict[str, Any]: - if broker_name == "toss": - return broker.cancel(ack["broker_order_id"]) - if quantity is None: - raise BlockedError("KIS cancel requires current remaining quantity") - return broker.cancel( - market=intent["market"], - broker_order_id=ack["broker_order_id"], - symbol=intent["symbol"], - quantity=quantity, - price=intent["limit_price"], - venue=venue, - organization=str(ack.get("organization") or ""), - ) - - -def session_status( - plan: dict[str, Any], - broker_name: str, - mode: str, - ledger: Ledger, - started_at: datetime, - stopped_at: datetime, - freeze_reason: str | None, - warmup: dict[str, Any], - polling_metrics: dict[str, Any], -) -> dict[str, Any]: - intents = [] - for intent in plan["intents"]: - record = ledger.get(intent["intent_id"]) - intents.append( - { - "intent_id": record.intent_id, - "state": record.state, - "broker_order_id": record.broker_order_id, - "request_hash": record.request_hash, - "event_count": len(ledger.events(record.intent_id)), - } - ) - output = { - "schema": SESSION_STATUS_SCHEMA, - "plan_hash": plan["plan_hash"], - "broker": broker_name, - "mode": mode, - "started_at": started_at.isoformat(), - "stopped_at": stopped_at.isoformat(), - "entry_freeze": freeze_reason is not None, - "freeze_reason": freeze_reason, - "preopen_warmup": warmup, - "polling_metrics": polling_metrics, - "intents": intents, - } - return output - - -def validate_ledger_scope(ledger: Ledger, plan: dict[str, Any]) -> None: - current_intent_ids = { - str(intent["intent_id"]) - for intent in plan["intents"] - if isinstance(intent, dict) and "intent_id" in intent - } - foreign = [ - binding - for binding in ledger.nonterminal_bindings() - if binding["plan_hash"] != plan["plan_hash"] - or binding["intent_id"] not in current_intent_ids - ] - if foreign: - states = sorted({binding["state"] for binding in foreign}) - raise BlockedError( - "state directory contains nonterminal intents outside the current " - f"plan (states={states}); reconcile the prior plan before running " - "a new one" - ) - unresolved_states = { - "RESERVED", - "SUBMITTING", - "UNKNOWN", - "RECONCILING", - "CANCEL_PENDING", - "MANUAL_BLOCK", - } - unresolved = [ - binding - for binding in ledger.nonterminal_bindings() - if binding["state"] in unresolved_states - ] - if unresolved: - states = sorted({binding["state"] for binding in unresolved}) - raise BlockedError( - "state directory contains unresolved mutation states " - f"(states={states}); broker evidence and explicit clearance are " - "required before new entries" - ) - - -def position_ledger_path(state_directory: Path) -> Path: - """Return the account/market ledger shared by all session-date directories.""" - if not state_directory.is_absolute(): - raise BlockedError("state_directory must be absolute") - return state_directory.parent / "positions.sqlite3" - - -def persist_entry_fill( - position_ledger: PositionLedger, - *, - plan: dict[str, Any], - intent: dict[str, Any], - snapshot: dict[str, Any], - session_date: str, -) -> dict[str, Any] | None: - """Persist an authoritative cumulative entry fill exactly once per change.""" - raw_filled = snapshot.get("filled_quantity") - if raw_filled in (None, ""): - return None - cumulative = decimal_value(raw_filled, "broker filled_quantity") - if cumulative < 0 or cumulative != cumulative.to_integral_value(): - raise BlockedError( - "broker filled_quantity must be a nonnegative whole number" - ) - if cumulative == 0: - return None - raw_average = snapshot.get("average_fill_price") - if raw_average in (None, ""): - raise BlockedError( - "broker reported an entry fill without average_fill_price" - ) - average = positive_decimal(raw_average, "broker average_fill_price") - account = plan["frozen_inputs"]["account"] - fx_to_krw = account.get("fx_to_krw") - if intent["market"] == "KR": - fx_to_krw = "1" - elif fx_to_krw in (None, ""): - raise BlockedError( - "a U.S. strategy position requires frozen fx_to_krw" - ) - - position_id = strategy_position_id(plan["plan_hash"], intent) - stored = position_ledger.get_optional(position_id) - if stored is None: - position = position_from_intent( - plan_hash=plan["plan_hash"], - intent=intent, - session_date=session_date, - fx_to_krw=fx_to_krw, - ) - event_type = "ENTRY_FILL_CREATED" - else: - position = stored.payload - if ( - position["plan_hash"] != plan["plan_hash"] - or position["entry_intent_id"] != intent["intent_id"] - ): - raise BlockedError( - "strategy position identity differs from plan entry intent" - ) - prior_quantity = Decimal(position["acquired_quantity"]) - prior_average = ( - None - if position["average_entry_price"] is None - else Decimal(position["average_entry_price"]) - ) - if cumulative == prior_quantity and average == prior_average: - return position - if Decimal(position["exited_quantity"]) > 0: - raise BlockedError( - "entry fill changed after exit fills were recorded" - ) - event_type = "ENTRY_FILL_UPDATED" - - updated = record_entry_fill( - position, - cumulative_filled_quantity=format(cumulative, "f"), - average_fill_price=format(average, "f"), - session_date=session_date, - ) - position_ledger.put( - updated, - event_type=event_type, - event_payload={ - "broker_order_id": snapshot.get("broker_order_id"), - "normalized_status": snapshot.get("normalized_status"), - "ordered_quantity": snapshot.get("ordered_quantity"), - "filled_quantity": format(cumulative, "f"), - "average_fill_price": format(average, "f"), - "remaining_quantity": snapshot.get("remaining_quantity"), - "session_date": session_date, - }, - ) - return updated - - -def run_session( - plan: dict[str, Any], - broker_name: str, - mode: str, - venues: dict[str, str], - state_directory: Path, - *, - max_cycles: int | None = None, -) -> dict[str, Any]: - validate_plan(plan) - validate_mode(plan, broker_name, mode) - if plan["plan_status"] == "NO_ORDERS": - state_directory.mkdir(parents=True, exist_ok=True) - with SingleWriterLock(state_directory / "writer.lock"): - ledger = Ledger(state_directory / "ledger.sqlite3") - try: - validate_ledger_scope(ledger, plan) - finally: - ledger.close() - return { - "schema": SESSION_STATUS_SCHEMA, - "plan_hash": plan["plan_hash"], - "broker": broker_name, - "mode": mode, - "status": "NO_ORDERS", - "mutation_sent": False, - } - validate_runtime_capabilities(plan, broker_name) - venue_by_intent = preflight_venues(plan, venues) - start = parse_timestamp(plan["entry_window"]["start"], "entry window start") - end = parse_timestamp(plan["entry_window"]["end"], "entry window end") - interval = int(plan["entry_window"]["poll_interval_seconds"]) - admission = polling_admission(plan, broker_name, mode) - broker = create_broker(broker_name) - preview_by_intent = preflight_submit_requests( - broker, - broker_name, - plan, - venue_by_intent, - ) - trading_date = start.date().isoformat() - state_directory.mkdir(parents=True, exist_ok=True) - started_at = datetime.now(timezone.utc) - freeze_reason: str | None = None - previous_last: dict[str, Decimal] = {} - acknowledgements: dict[str, dict[str, Any]] = {} - warmup: dict[str, Any] = {} - polling_metrics: dict[str, Any] = { - "admission": admission, - "telemetry_anchor_intent_id": plan["intents"][0]["intent_id"], - "cycles_started": 0, - "cycles_completed": 0, - "cycles_skipped": 0, - "quotes_evaluated": 0, - "market_regime_queries": 0, - "market_regime_passes": 0, - "market_regime_blocks": 0, - "order_status_queries": 0, - "submits_started": 0, - "expected_http_requests_started": 0, - "max_schedule_lateness_ms": 0, - "max_cycle_duration_ms": 0, - "max_quote_latency_ms": 0, - "max_market_regime_latency_ms": 0, - "max_order_status_latency_ms": 0, - "max_submit_latency_ms": 0, - "position_reconciliation": None, - } - - with SingleWriterLock(state_directory / "writer.lock"): - ledger = Ledger(state_directory / "ledger.sqlite3") - position_ledger = PositionLedger(position_ledger_path(state_directory)) - try: - validate_ledger_scope(ledger, plan) - position_reconciliation = reconcile_account_positions( - position_ledger, - account_snapshot=plan["frozen_inputs"]["account"], - session_date=trading_date, - ) - polling_metrics["position_reconciliation"] = { - "status": position_reconciliation["status"], - "reconciliation_hash": position_reconciliation[ - "reconciliation_hash" - ], - "managed_position_count": len( - position_reconciliation["managed_positions"] - ), - "unmanaged_broker_position_count": len( - position_reconciliation["unmanaged_broker_positions"] - ), - } - if position_reconciliation["status"] != "READY": - raise BlockedError( - "strategy position ledger differs from the frozen broker " - "account snapshot; reconcile MANUAL_BLOCK before new entries" - ) - warmup = warm_broker_before_open(broker, start) - for intent in plan["intents"]: - ledger.create_intent(plan["plan_hash"], intent) - if ledger.get(intent["intent_id"]).state == "PLANNED": - ledger.transition(intent["intent_id"], "WAIT_TRIGGER", {}) - - cycle = 0 - while True: - target = start.timestamp() + cycle * interval - now = datetime.now(timezone.utc) - if now.timestamp() >= end.timestamp(): - freeze_reason = "entry_window_closed" - break - if max_cycles is not None and cycle >= max_cycles: - freeze_reason = "max_cycles_reached" - break - if now.timestamp() < target: - time.sleep(min(target - now.timestamp(), 60)) - continue - if now.timestamp() >= target + interval: - skipped_count = max( - 1, - int((now.timestamp() - target) // interval), - ) - if max_cycles is not None: - skipped_count = min( - skipped_count, - max_cycles - cycle, - ) - lateness_ms = int((now.timestamp() - target) * 1000) - polling_metrics["cycles_skipped"] += skipped_count - polling_metrics["max_schedule_lateness_ms"] = max( - polling_metrics["max_schedule_lateness_ms"], - lateness_ms, - ) - ledger.append_event( - plan["intents"][0]["intent_id"], - "CYCLE_SKIPPED", - { - "first_cycle": cycle, - "last_cycle": cycle + skipped_count - 1, - "skipped_count": skipped_count, - "scheduled_at": datetime.fromtimestamp( - target, timezone.utc - ).isoformat(), - "observed_at": now.isoformat(), - "lateness_ms": lateness_ms, - "reason": "scheduled_cycle_expired", - }, - ) - cycle += skipped_count - continue - - cycle_started_at = now - cycle_started_monotonic = time.monotonic() - cycle_deadline = min(target + interval, end.timestamp()) - lateness_ms = max(0, int((now.timestamp() - target) * 1000)) - polling_metrics["cycles_started"] += 1 - polling_metrics["max_schedule_lateness_ms"] = max( - polling_metrics["max_schedule_lateness_ms"], - lateness_ms, - ) - cycle_events: list[tuple[str, str, dict[str, Any]]] = [ - ( - plan["intents"][0]["intent_id"], - "CYCLE_START", - { - "cycle": cycle, - "scheduled_at": datetime.fromtimestamp( - target, timezone.utc - ).isoformat(), - "started_at": cycle_started_at.isoformat(), - "deadline_at": datetime.fromtimestamp( - cycle_deadline, timezone.utc - ).isoformat(), - "lateness_ms": lateness_ms, - }, - ) - ] - regime_by_exchange: dict[str, dict[str, Any]] = {} - if plan_uses_qta2(plan): - waiting_by_exchange: dict[str, list[dict[str, Any]]] = {} - for candidate in plan["intents"]: - if ledger.get(candidate["intent_id"]).state == "WAIT_TRIGGER": - waiting_by_exchange.setdefault( - candidate["exchange"], - [], - ).append(candidate) - for exchange in active_qta2_exchanges(plan): - waiting = waiting_by_exchange.get(exchange, []) - if not waiting: - continue - if datetime.now(timezone.utc).timestamp() >= cycle_deadline: - freeze_reason = "cycle_latency_budget_exceeded" - break - regime_started_at = datetime.now(timezone.utc) - regime_started_monotonic = time.monotonic() - polling_metrics["market_regime_queries"] += 1 - polling_metrics["expected_http_requests_started"] += 1 - try: - regime_quote = broker.benchmark_quote( - exchange=exchange, - session_date=trading_date, - ) - regime_observed_at = datetime.now(timezone.utc) - regime_decision = evaluate_market_regime( - exchange=exchange, - quote=regime_quote, - observed_at=regime_observed_at, - session_date=trading_date, - ) - except (BlockedError, TransportFailure) as exc: - regime_decision = { - "metric": QTA2_REGIME_METRIC, - "exchange": exchange, - "benchmark_id": V2_EXCHANGE_CONTRACTS[exchange][ - "benchmark_id" - ], - "minimum_change_bps": format( - QTA2_REGIME_MINIMUM_CHANGE_BPS, - "f", - ), - "max_age_seconds": QTA2_REGIME_MAX_AGE_SECONDS, - "quality_valid": False, - "admitted": False, - "reasons": ["benchmark_quote_unavailable"], - "error": str(exc), - "observed_at": datetime.now( - timezone.utc - ).isoformat(), - } - regime_latency_ms = int( - (time.monotonic() - regime_started_monotonic) * 1000 - ) - regime_decision["latency_ms"] = regime_latency_ms - regime_decision["started_at"] = ( - regime_started_at.isoformat() - ) - regime_by_exchange[exchange] = regime_decision - polling_metrics["max_market_regime_latency_ms"] = max( - polling_metrics["max_market_regime_latency_ms"], - regime_latency_ms, - ) - metric = ( - "market_regime_passes" - if regime_decision["admitted"] - else "market_regime_blocks" - ) - polling_metrics[metric] += 1 - cycle_events.append( - ( - waiting[0]["intent_id"], - "MARKET_REGIME_EVALUATED", - regime_decision, - ) - ) - if freeze_reason == "cycle_latency_budget_exceeded": - ledger.append_events(cycle_events) - break - for intent in plan["intents"]: - if datetime.now(timezone.utc).timestamp() >= cycle_deadline: - freeze_reason = "cycle_latency_budget_exceeded" - break - record = ledger.get(intent["intent_id"]) - venue = venue_by_intent[intent["intent_id"]] - if record.state == "WAIT_TRIGGER": - if plan_uses_qta2(plan): - regime_decision = regime_by_exchange.get( - intent["exchange"] - ) - if ( - regime_decision is None - or not regime_decision["admitted"] - ): - cycle_events.append( - ( - intent["intent_id"], - "ENTRY_BLOCKED_BY_MARKET_REGIME", - { - "exchange": intent["exchange"], - "decision": regime_decision, - }, - ) - ) - continue - if not entry_window_open(end): - freeze_reason = "entry_window_closed" - break - quote_started_at = datetime.now(timezone.utc) - quote_started_monotonic = time.monotonic() - polling_metrics["expected_http_requests_started"] += ( - quote_http_request_count(broker_name, intent) - ) - try: - quote = ( - broker.quote( - intent["market"], intent["symbol"], venue=venue - ) - if broker_name != "toss" - else broker.quote(intent["market"], intent["symbol"]) - ) - except (BlockedError, TransportFailure) as exc: - quote_latency_ms = int( - (time.monotonic() - quote_started_monotonic) * 1000 - ) - polling_metrics["max_quote_latency_ms"] = max( - polling_metrics["max_quote_latency_ms"], - quote_latency_ms, - ) - cycle_events.append( - ( - intent["intent_id"], - "QUOTE_BLOCKED", - { - "reason": str(exc), - "started_at": quote_started_at.isoformat(), - "completed_at": datetime.now( - timezone.utc - ).isoformat(), - "latency_ms": quote_latency_ms, - }, - ) - ) - freeze_reason = "quote_failure" - break - quote_completed_at = datetime.now(timezone.utc) - quote_latency_ms = int( - (time.monotonic() - quote_started_monotonic) * 1000 - ) - polling_metrics["quotes_evaluated"] += 1 - polling_metrics["max_quote_latency_ms"] = max( - polling_metrics["max_quote_latency_ms"], - quote_latency_ms, - ) - decision = evaluate_quote( - intent, - quote, - plan["quote_policy"], - quote_completed_at, - previous_last.get(intent["intent_id"]), - require_source_timestamp=True, - ) - decision.update( - { - "quote_started_at": quote_started_at.isoformat(), - "quote_completed_at": quote_completed_at.isoformat(), - "quote_latency_ms": quote_latency_ms, - } - ) - if quote_completed_at.timestamp() >= cycle_deadline: - decision["triggered"] = False - decision["reasons"].append("cycle_latency_budget_exceeded") - freeze_reason = "cycle_latency_budget_exceeded" - if decision["quality_valid"]: - previous_last[intent["intent_id"]] = decimal_value( - quote["last_price"], "last_price" - ) - cycle_events.append( - ( - intent["intent_id"], - "QUOTE_EVALUATED", - decision, - ) - ) - if freeze_reason == "cycle_latency_budget_exceeded": - break - if not decision["triggered"]: - continue - if not entry_window_open(end): - freeze_reason = "entry_window_closed" - break - if mode == "shadow": - shadow_request = preview_by_intent[intent["intent_id"]] - cycle_events.append( - ( - intent["intent_id"], - "SHADOW_WOULD_SUBMIT", - {"request": shadow_request}, - ) - ) - ledger.append_events(cycle_events) - cycle_events = [] - ledger.transition( - intent["intent_id"], - "CANCELLED", - {"reason": "shadow_only"}, - ) - continue - submit_safety_seconds = SUBMIT_HTTP_TIMEOUT_SECONDS + float( - BROKER_PACING_SECONDS[broker_name] - ) - submit_cutoff = end - timedelta(seconds=submit_safety_seconds) - if not entry_window_open(submit_cutoff): - ledger.append_events(cycle_events) - cycle_events = [] - ledger.transition( - intent["intent_id"], - "CANCELLED", - {"reason": "submit_safety_cutoff_reached"}, - ) - freeze_reason = "submit_safety_cutoff_reached" - break - ledger.append_events(cycle_events) - cycle_events = [] - ledger.transition(intent["intent_id"], "RESERVED", decision) - preview = preview_by_intent[intent["intent_id"]] - if not entry_window_open(end): - ledger.transition( - intent["intent_id"], - "CANCELLED", - {"reason": "entry_window_closed_before_submit"}, - ) - freeze_reason = "entry_window_closed" - break - if datetime.now(timezone.utc).timestamp() >= cycle_deadline: - ledger.transition( - intent["intent_id"], - "CANCELLED", - {"reason": "cycle_latency_budget_exceeded"}, - ) - freeze_reason = "cycle_latency_budget_exceeded" - break - ledger.transition( - intent["intent_id"], - "SUBMITTING", - {"request_hash": preview["request_hash"]}, - request_hash=preview["request_hash"], - ) - submit_started_at = datetime.now(timezone.utc) - submit_started_monotonic = time.monotonic() - polling_metrics["submits_started"] += 1 - polling_metrics["expected_http_requests_started"] += 1 - try: - ack = ( - broker.submit(intent, deadline_at=end) - if broker_name == "toss" - else broker.submit( - intent, - venue=venue, - deadline_at=end, - ) - ) - except AmbiguousMutationError as exc: - polling_metrics["max_submit_latency_ms"] = max( - polling_metrics["max_submit_latency_ms"], - int( - (time.monotonic() - submit_started_monotonic) * 1000 - ), - ) - ledger.transition( - intent["intent_id"], - "UNKNOWN", - { - "reason": str(exc), - "submit_started_at": ( - submit_started_at.isoformat() - ), - "completed_at": datetime.now( - timezone.utc - ).isoformat(), - }, - ) - freeze_reason = "ambiguous_mutation" - break - except AuthoritativeMutationRejection as exc: - ledger.transition( - intent["intent_id"], - "REJECTED", - { - "reason": str(exc), - "submit_started_at": ( - submit_started_at.isoformat() - ), - "completed_at": datetime.now( - timezone.utc - ).isoformat(), - }, - ) - freeze_reason = "submit_rejected" - break - except (BlockedError, TransportFailure) as exc: - ledger.transition( - intent["intent_id"], - "UNKNOWN", - { - "reason": str(exc), - "submit_started_at": ( - submit_started_at.isoformat() - ), - "completed_at": datetime.now( - timezone.utc - ).isoformat(), - }, - ) - freeze_reason = "ambiguous_mutation" - break - submit_latency_ms = int( - (time.monotonic() - submit_started_monotonic) * 1000 - ) - polling_metrics["max_submit_latency_ms"] = max( - polling_metrics["max_submit_latency_ms"], - submit_latency_ms, - ) - acknowledgements[intent["intent_id"]] = ack - ledger.transition( - intent["intent_id"], - "ACKNOWLEDGED", - { - "ack": ack, - "runner_submit_started_at": ( - submit_started_at.isoformat() - ), - "runner_ack_received_at": datetime.now( - timezone.utc - ).isoformat(), - "latency_ms": submit_latency_ms, - }, - broker_order_id=ack["broker_order_id"], - ) - if not entry_window_open(end): - freeze_reason = "ack_after_entry_window" - break - elif record.state in {"ACKNOWLEDGED", "PARTIALLY_FILLED"}: - ack = acknowledgements.get(intent["intent_id"]) - if ack is None: - for event in reversed(ledger.events(intent["intent_id"])): - payload = event["payload"] - if event["event_type"] == "STATE_ACKNOWLEDGED": - ack = payload.get("ack") - break - if not isinstance(ack, dict): - ledger.transition( - intent["intent_id"], - "UNKNOWN", - {"reason": "ack_missing_after_restart"}, - ) - freeze_reason = "ack_missing" - break - status_started_at = datetime.now(timezone.utc) - status_started_monotonic = time.monotonic() - polling_metrics["order_status_queries"] += 1 - polling_metrics["expected_http_requests_started"] += 1 - try: - snapshot = current_order( - broker, - broker_name, - intent=intent, - ack=ack, - venue=venue, - trading_date=trading_date, - ) - except (BlockedError, TransportFailure) as exc: - status_latency_ms = int( - (time.monotonic() - status_started_monotonic) * 1000 - ) - polling_metrics["max_order_status_latency_ms"] = max( - polling_metrics["max_order_status_latency_ms"], - status_latency_ms, - ) - cycle_events.append( - ( - intent["intent_id"], - "RECONCILE_RETRY", - { - "reason": str(exc), - "started_at": status_started_at.isoformat(), - "completed_at": datetime.now( - timezone.utc - ).isoformat(), - "latency_ms": status_latency_ms, - }, - ) - ) - continue - status_latency_ms = int( - (time.monotonic() - status_started_monotonic) * 1000 - ) - polling_metrics["max_order_status_latency_ms"] = max( - polling_metrics["max_order_status_latency_ms"], - status_latency_ms, - ) - next_state = snapshot["normalized_status"] - try: - persist_entry_fill( - position_ledger, - plan=plan, - intent=intent, - snapshot=snapshot, - session_date=trading_date, - ) - except BlockedError as exc: - ledger.transition( - intent["intent_id"], - "UNKNOWN", - { - "reason": ( - "strategy position ledger sync failed: " - f"{exc}" - ), - "snapshot": snapshot, - }, - ) - freeze_reason = "position_ledger_sync_failed" - break - if next_state != record.state: - if next_state not in { - "ACKNOWLEDGED", - "PARTIALLY_FILLED", - "FILLED", - "CANCELLED", - "REJECTED", - }: - ledger.transition( - intent["intent_id"], - "UNKNOWN", - {"snapshot": snapshot}, - ) - freeze_reason = "unknown_broker_status" - break - ledger.transition( - intent["intent_id"], next_state, {"snapshot": snapshot} - ) - cycle_duration_ms = int( - (time.monotonic() - cycle_started_monotonic) * 1000 - ) - polling_metrics["cycles_completed"] += 1 - polling_metrics["max_cycle_duration_ms"] = max( - polling_metrics["max_cycle_duration_ms"], - cycle_duration_ms, - ) - if ( - not freeze_reason - and datetime.now(timezone.utc).timestamp() >= cycle_deadline - ): - freeze_reason = "cycle_latency_budget_exceeded" - cycle_events.append( - ( - plan["intents"][0]["intent_id"], - "CYCLE_END", - { - "cycle": cycle, - "completed_at": datetime.now(timezone.utc).isoformat(), - "duration_ms": cycle_duration_ms, - "freeze_reason": freeze_reason, - }, - ) - ) - ledger.append_events(cycle_events) - if freeze_reason and freeze_reason != "entry_window_closed": - break - cycle += 1 - - for intent in plan["intents"]: - record = ledger.get(intent["intent_id"]) - if record.state == "WAIT_TRIGGER": - ledger.transition( - intent["intent_id"], - "CANCELLED", - { - "reason": ( - "entry_window_closed" - if freeze_reason == "entry_window_closed" - else str(freeze_reason or "entry_window_ended") - ) - }, - ) - - if plan["order_policy"]["cancel_remainder_at_window_end"]: - for intent in plan["intents"]: - record = ledger.get(intent["intent_id"]) - if record.state not in {"ACKNOWLEDGED", "PARTIALLY_FILLED"}: - continue - ack = acknowledgements.get(intent["intent_id"]) - if ack is None: - for event in reversed(ledger.events(intent["intent_id"])): - if event["event_type"] == "STATE_ACKNOWLEDGED": - ack = event["payload"].get("ack") - break - if not isinstance(ack, dict): - ledger.transition( - intent["intent_id"], - "UNKNOWN", - {"reason": "cannot_cancel_without_ack"}, - ) - continue - cancel_quantity: str | None = None - if broker_name in {"kis-paper", "kis-live"}: - try: - cancel_snapshot = current_order( - broker, - broker_name, - intent=intent, - ack=ack, - venue=venue_by_intent[intent["intent_id"]], - trading_date=trading_date, - ) - persist_entry_fill( - position_ledger, - plan=plan, - intent=intent, - snapshot=cancel_snapshot, - session_date=trading_date, - ) - next_state = cancel_snapshot["normalized_status"] - if next_state in {"FILLED", "CANCELLED"} or ( - next_state == "REJECTED" - and record.state == "ACKNOWLEDGED" - ): - if next_state != record.state: - ledger.transition( - intent["intent_id"], - next_state, - {"snapshot": cancel_snapshot}, - ) - continue - if next_state == "REJECTED": - raise BlockedError( - "KIS partially filled order cannot regress to rejected" - ) - if next_state not in { - "ACKNOWLEDGED", - "PARTIALLY_FILLED", - }: - raise BlockedError( - "KIS cancel preflight returned an unknown state" - ) - if ( - next_state == "PARTIALLY_FILLED" - and record.state == "ACKNOWLEDGED" - ): - ledger.transition( - intent["intent_id"], - "PARTIALLY_FILLED", - {"snapshot": cancel_snapshot}, - ) - cancel_quantity = require_remaining_quantity( - cancel_snapshot - ) - ledger.append_event( - intent["intent_id"], - "CANCEL_PREFLIGHT", - {"snapshot": cancel_snapshot}, - ) - except (BlockedError, TransportFailure) as exc: - ledger.transition( - intent["intent_id"], - "UNKNOWN", - {"reason": f"cancel preflight failed: {exc}"}, - ) - continue - ledger.transition(intent["intent_id"], "CANCEL_PENDING", {}) - try: - result = cancel_order( - broker, - broker_name, - intent=intent, - ack=ack, - venue=venue_by_intent[intent["intent_id"]], - quantity=cancel_quantity, - ) - except (AmbiguousMutationError, TransportFailure) as exc: - ledger.transition( - intent["intent_id"], - "UNKNOWN", - {"reason": str(exc)}, - ) - except BlockedError as exc: - ledger.transition( - intent["intent_id"], - "UNKNOWN", - {"reason": str(exc)}, - ) - else: - ledger.append_event( - intent["intent_id"], "CANCEL_ACCEPTED", result - ) - return session_status( - plan, - broker_name, - mode, - ledger, - started_at, - datetime.now(timezone.utc), - freeze_reason, - warmup, - polling_metrics, - ) - finally: - position_ledger.close() - ledger.close() - - -def self_test() -> None: - now = datetime.now(timezone.utc) - intent = { - "market": "US", - "symbol": "AAPL", - "entry_trigger": "100", - "limit_price": "101", - } - policy = { - "max_age_seconds": 5, - "max_spread_bps": "25", - "max_gap_bps": "100", - "trigger_mode": "AT_OR_ABOVE", - } - quote = { - "market": "US", - "symbol": "AAPL", - "last_price": "100.5", - "best_ask": "100.6", - "best_bid": "100.4", - "received_at": now.isoformat(), - "trade_timestamp": now.isoformat(), - "book_timestamp": now.isoformat(), - "raw_status": "OK", - } - decision = evaluate_quote( - intent, quote, policy, now, None, require_source_timestamp=True - ) - assert decision["triggered"] - crossed_policy = {**policy, "trigger_mode": "CROSS_FROM_BELOW"} - assert not evaluate_quote( - intent, quote, crossed_policy, now, None, require_source_timestamp=True - )["triggered"] - assert evaluate_quote( - intent, - quote, - crossed_policy, - now, - Decimal(99), - require_source_timestamp=True, - )["triggered"] - stale_source = { - **quote, - "trade_timestamp": ( - now.replace(microsecond=0) - timedelta(seconds=10) - ).isoformat(), - "book_timestamp": ( - now.replace(microsecond=0) - timedelta(seconds=10) - ).isoformat(), - } - stale_decision = evaluate_quote( - intent, - stale_source, - policy, - now, - None, - require_source_timestamp=True, - ) - assert "stale_source_quote" in stale_decision["reasons"] - assert not stale_decision["quality_valid"] - assert not evaluate_quote( - intent, - quote, - crossed_policy, - now, - ( - Decimal(stale_decision["last_price"]) - if stale_decision["quality_valid"] - else None - ), - require_source_timestamp=True, - )["triggered"] - mixed_future_source = { - **quote, - "trade_timestamp": (now + timedelta(seconds=10)).isoformat(), - } - future_decision = evaluate_quote( - intent, - mixed_future_source, - policy, - now, - None, - require_source_timestamp=True, - ) - assert "stale_source_quote" in future_decision["reasons"] - missing_source = { - **quote, - "trade_timestamp": None, - "book_timestamp": None, - } - missing_decision = evaluate_quote( - intent, - missing_source, - policy, - now, - None, - require_source_timestamp=True, - ) - assert "source_timestamp_missing" in missing_decision["reasons"] - one_missing_source = { - **quote, - "book_timestamp": None, - } - one_missing_decision = evaluate_quote( - intent, - one_missing_source, - policy, - now, - None, - require_source_timestamp=True, - ) - assert "source_timestamp_missing" in one_missing_decision["reasons"] - invalid_source = { - **quote, - "trade_timestamp": "not-a-timestamp", - } - invalid_decision = evaluate_quote( - intent, - invalid_source, - policy, - now, - None, - require_source_timestamp=True, - ) - assert "trade_timestamp_invalid" in invalid_decision["reasons"] - for raw_status in ( - "PENDING_REPLACE", - "REPLACED", - "CANCEL_REJECTED", - "REPLACE_REJECTED", - ): - assert ( - normalize_toss_order({"status": raw_status})["normalized_status"] - == "UNKNOWN" - ) - try: - validate_runtime_capabilities({"intents": [intent]}, "kis-paper") - except BlockedError as exc: - assert "U.S. paper sessions" in str(exc) - else: - raise AssertionError("KIS U.S. paper session must be blocked") - one_intent_plan = { - "intents": [intent], - "entry_window": {"poll_interval_seconds": 4}, - } - assert ( - polling_admission(one_intent_plan, "kis-paper", "paper")[ - "minimum_poll_interval_seconds" - ] - == 4 - ) - too_fast_plan = { - "intents": [intent], - "entry_window": {"poll_interval_seconds": 3}, - } - try: - polling_admission(too_fast_plan, "kis-paper", "paper") - except BlockedError as exc: - assert "poll_interval_seconds >= 4" in str(exc) - else: - raise AssertionError("KIS paper pacing must reject an infeasible cycle") - four_intent_shadow_plan = { - "intents": [intent, intent, intent, intent], - "entry_window": {"poll_interval_seconds": 1}, - } - assert quote_http_request_count("kis-live", intent) == 1 - assert quote_http_request_count("toss", intent) == 2 - assert ( - polling_admission(four_intent_shadow_plan, "kis-live", "shadow")[ - "minimum_poll_interval_seconds" - ] - == 1 - ) - - class FailingWarmupBroker: - def token(self) -> str: - raise TransportFailure("fixture authentication outage") - - try: - warm_broker_before_open( - FailingWarmupBroker(), - datetime.now(timezone.utc) + timedelta(seconds=1), - ) - except BlockedError as exc: - assert "pre-open broker authentication failed" in str(exc) - else: - raise AssertionError("pre-open transport failure must fail closed") - assert entry_window_open(now + timedelta(seconds=1), now) - assert not entry_window_open(now, now) - assert require_remaining_quantity({"remaining_quantity": "2"}) == "2" - v2_intent = { - **intent, - "exchange": "NASDAQ", - "venue": "NASD", - } - assert venue_for({}, v2_intent) == "NASD" - assert venue_for({"NASDAQ:AAPL": "NASD"}, v2_intent) == "NASD" - try: - venue_for({"US:AAPL": "NYSE"}, v2_intent) - except BlockedError as exc: - assert "conflicts with hashed intent venue" in str(exc) - else: - raise AssertionError("a venue map must not override the hashed v2 venue") - try: - venue_for({}, intent) - except BlockedError as exc: - assert "venue map missing US:AAPL" in str(exc) - else: - raise AssertionError("legacy intents still require an external venue map") - - class CaptureCancelBroker: - def __init__(self) -> None: - self.kwargs: dict[str, Any] | None = None - - def cancel(self, **kwargs: Any) -> dict[str, Any]: - self.kwargs = kwargs - return {"accepted": True} - - capture = CaptureCancelBroker() - cancel_order( - capture, - "kis-paper", - intent=intent, - ack={"broker_order_id": "12345", "organization": "06000"}, - venue="NASD", - quantity="2", - ) - assert capture.kwargs is not None - assert capture.kwargs["symbol"] == "AAPL" - assert capture.kwargs["quantity"] == "2" - print( - canonical_json( - { - "self_test": "PASS", - "spread_bps": decision["spread_bps"], - "live_gate": "BLOCKED", - } - ) - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - subparsers = parser.add_subparsers(dest="command") - - preview = subparsers.add_parser("preview") - preview.add_argument("--plan", required=True) - preview.add_argument( - "--broker", required=True, choices=("toss", "kis-paper", "kis-live") - ) - preview.add_argument("--venue-map") - preview.add_argument("--output") - - run = subparsers.add_parser("run") - run.add_argument("--plan", required=True) - run.add_argument( - "--broker", required=True, choices=("toss", "kis-paper", "kis-live") - ) - run.add_argument("--mode", required=True, choices=("paper", "shadow", "live")) - run.add_argument("--venue-map") - run.add_argument("--state-dir", required=True) - run.add_argument("--output") - run.add_argument("--max-cycles", type=int) - - subparsers.add_parser("self-test") - return parser.parse_args() - - -def main() -> int: - args = parse_args() - if args.command == "self-test": - self_test() - return 0 - if args.command not in {"preview", "run"}: - emit_json( - { - "schema": SESSION_STATUS_SCHEMA, - "status": "BLOCKED", - "reason": "choose preview, run, or self-test", - } - ) - return 2 - try: - if args.command == "run" and args.broker in {"kis-paper", "kis-live"}: - require_kis_runtime_credentials( - load_kis_credentials(args.broker.split("-", 1)[1]) - ) - plan = load_json_object(args.plan) - venues = ( - normalize_venue_map(load_json_object(args.venue_map)) - if args.venue_map - else {} - ) - if args.command == "preview": - output = preview_requests(plan, args.broker, venues) - else: - if args.max_cycles is not None and args.max_cycles <= 0: - raise BlockedError("--max-cycles must be positive") - output = run_session( - plan, - args.broker, - args.mode, - venues, - Path(args.state_dir).resolve(), - max_cycles=args.max_cycles, - ) - except (BlockedError, OSError, ValueError, json.JSONDecodeError) as exc: - output = { - "schema": SESSION_STATUS_SCHEMA, - "status": "BLOCKED", - "reason": str(exc), - } - emit_json(output, args.output) - return 2 - emit_json(output, args.output) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/skills/quant-stock-polling-trader/scripts/systemd_units.py b/skills/quant-stock-polling-trader/scripts/systemd_units.py deleted file mode 100644 index 3b16cfb..0000000 --- a/skills/quant-stock-polling-trader/scripts/systemd_units.py +++ /dev/null @@ -1,1026 +0,0 @@ -#!/usr/bin/env python3 -"""Generate and execute deterministic user-level systemd jobs for QTA.""" - -from __future__ import annotations - -import argparse -import fcntl -import hashlib -import json -import os -import re -import stat -import subprocess -import sys -import time -import urllib.error -import urllib.parse -import urllib.request -from pathlib import Path -from typing import Any, Mapping - - -BUNDLE_SCHEMA = "qta-systemd-bundle/v2" -STATUS_SCHEMA = "qta-systemd-generation-status/v2" -TOP_FIELDS = { - "schema", - "unit_prefix", - "python_executable", - "technical_skill_root", - "trader_skill_root", - "environment_file", - "runtime_directory", - "jobs", -} -JOB_FIELDS = { - "name", - "kind", - "market", - "input_path", - "plan_path", - "state_directory", - "output_path", - "broker", - "mode", - "venue_map", - "max_cycles", - "timeout_start_seconds", -} -SCHEDULES = { - ("eod", "KR"): "Mon..Fri *-*-* 07:00:00 Asia/Seoul", - ("eod", "US"): "Mon..Fri *-*-* 18:00:00 America/New_York", - ("snapshot", "KR"): "Mon..Fri *-*-* 08:50:00 Asia/Seoul", - ("snapshot", "US"): "Mon..Fri *-*-* 09:20:00 America/New_York", - ("entry", "KR"): "Mon..Fri *-*-* 08:59:00 Asia/Seoul", - ("entry", "US"): "Mon..Fri *-*-* 09:29:00 America/New_York", - ("daily-prepare", "KR"): "Mon..Fri *-*-* 01:00:00 Asia/Seoul", - ("daily-prepare", "US"): "Mon..Fri *-*-* 01:00:00 America/New_York", - ("daily-snapshot", "KR"): "Mon..Fri *-*-* 08:50:00 Asia/Seoul", - ("daily-snapshot", "US"): "Mon..Fri *-*-* 09:20:00 America/New_York", - ("daily-entry", "KR"): "Mon..Fri *-*-* 08:59:00 Asia/Seoul", - ("daily-entry", "US"): "Mon..Fri *-*-* 09:29:00 America/New_York", -} -UNIT_NAME = re.compile(r"^[a-z0-9][a-z0-9-]{0,47}$") -NTFY_TOPIC_URL_ENV = "QTA_NTFY_TOPIC_URL" -NTFY_TOKEN_ENV = "QTA_NTFY_TOKEN" -NTFY_TIMEOUT_SECONDS = 10 - - -class UnitBlockedError(ValueError): - """Raised when a unit bundle would rely on an unsafe or ambiguous value.""" - - -def sha256_file(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def atomic_write(path: Path, payload: bytes, mode: int = 0o600) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") - descriptor = os.open( - temporary, - os.O_WRONLY | os.O_CREAT | os.O_EXCL, - mode, - ) - try: - with os.fdopen(descriptor, "wb") as handle: - handle.write(payload) - handle.flush() - os.fsync(handle.fileno()) - os.replace(temporary, path) - os.chmod(path, mode) - finally: - if temporary.exists(): - temporary.unlink() - - -def exact_fields(value: Mapping[str, Any], expected: set[str], label: str) -> None: - if set(value) != expected: - raise UnitBlockedError( - f"{label} fields mismatch; missing={sorted(expected - set(value))}, " - f"extra={sorted(set(value) - expected)}" - ) - - -def absolute_path(value: Any, label: str) -> Path: - path = Path(str(value)).expanduser() - if not path.is_absolute(): - raise UnitBlockedError(f"{label} must be absolute") - if any(character in str(path) for character in ("\n", "\r", "\0")): - raise UnitBlockedError(f"{label} contains an invalid character") - return path - - -def regular_nonsymlink(path: Path, label: str, mode: int | None = None) -> None: - try: - metadata = path.lstat() - except FileNotFoundError as exc: - raise UnitBlockedError(f"{label} does not exist: {path}") from exc - if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode): - raise UnitBlockedError(f"{label} must be a regular non-symlink file") - if mode is not None and stat.S_IMODE(metadata.st_mode) != mode: - raise UnitBlockedError(f"{label} must have mode {mode:04o}") - - -def within(path: Path, root: Path, label: str) -> Path: - resolved = path.resolve(strict=False) - try: - resolved.relative_to(root.resolve(strict=False)) - except ValueError as exc: - raise UnitBlockedError(f"{label} must be inside runtime_directory") from exc - return resolved - - -def normalize_bundle(raw: Any) -> dict[str, Any]: - if not isinstance(raw, dict): - raise UnitBlockedError("bundle must be one JSON object") - exact_fields(raw, TOP_FIELDS, "bundle") - if raw["schema"] != BUNDLE_SCHEMA: - raise UnitBlockedError(f"bundle.schema must be {BUNDLE_SCHEMA}") - prefix = str(raw["unit_prefix"]) - if not UNIT_NAME.fullmatch(prefix): - raise UnitBlockedError("unit_prefix must be lowercase letters, digits, hyphens") - python_path = absolute_path(raw["python_executable"], "python_executable") - try: - python = python_path.resolve(strict=True) - except FileNotFoundError as exc: - raise UnitBlockedError("python_executable does not exist") from exc - regular_nonsymlink(python, "resolved python_executable") - technical_root = absolute_path(raw["technical_skill_root"], "technical_skill_root") - trader_root = absolute_path(raw["trader_skill_root"], "trader_skill_root") - for root, label in ( - (technical_root, "technical_skill_root"), - (trader_root, "trader_skill_root"), - ): - if not root.is_dir() or root.is_symlink(): - raise UnitBlockedError(f"{label} must be a non-symlink directory") - required_scripts = ( - technical_root / "scripts" / "fetch_kis_kr_eod.py", - technical_root / "scripts" / "fetch_kis_us_eod.py", - trader_root / "scripts" / "account_snapshot.py", - trader_root / "scripts" / "daily_pipeline.py", - trader_root / "scripts" / "market_calendar.py", - trader_root / "scripts" / "run_session.py", - trader_root / "scripts" / "systemd_units.py", - ) - for script in required_scripts: - regular_nonsymlink(script, "required skill script") - environment = absolute_path(raw["environment_file"], "environment_file") - regular_nonsymlink(environment, "environment_file", 0o600) - runtime = absolute_path(raw["runtime_directory"], "runtime_directory") - if runtime.exists() and (not runtime.is_dir() or runtime.is_symlink()): - raise UnitBlockedError("runtime_directory must be a non-symlink directory") - jobs = raw["jobs"] - if not isinstance(jobs, list) or not jobs: - raise UnitBlockedError("jobs must be a non-empty array") - normalized_jobs: list[dict[str, Any]] = [] - names: set[str] = set() - pairs: set[tuple[str, str]] = set() - for index, job in enumerate(jobs): - label = f"jobs[{index}]" - if not isinstance(job, dict): - raise UnitBlockedError(f"{label} must be an object") - exact_fields(job, JOB_FIELDS, label) - name = str(job["name"]) - if not UNIT_NAME.fullmatch(name) or name in names: - raise UnitBlockedError(f"{label}.name is invalid or duplicate") - names.add(name) - kind = str(job["kind"]).lower() - market = str(job["market"]).upper() - if (kind, market) not in SCHEDULES or (kind, market) in pairs: - raise UnitBlockedError(f"{label} has an unsupported or duplicate job scope") - pairs.add((kind, market)) - timeout = job["timeout_start_seconds"] - if ( - isinstance(timeout, bool) - or not isinstance(timeout, int) - or timeout < 60 - or timeout > 86400 - ): - raise UnitBlockedError(f"{label}.timeout_start_seconds must be 60..86400") - max_cycles = job["max_cycles"] - if ( - isinstance(max_cycles, bool) - or not isinstance(max_cycles, int) - or max_cycles < 0 - ): - raise UnitBlockedError(f"{label}.max_cycles must be an integer >= 0") - path_values = { - field: ( - within(absolute_path(job[field], f"{label}.{field}"), runtime, label) - if str(job[field]) - else None - ) - for field in ( - "input_path", - "plan_path", - "state_directory", - "output_path", - "venue_map", - ) - } - broker = str(job["broker"]) - mode = str(job["mode"]) - if kind.startswith("daily-"): - if path_values["input_path"] is None: - raise UnitBlockedError( - f"{label}.input_path is required for a daily pipeline job" - ) - if any( - path_values[field] is not None - for field in ( - "plan_path", - "state_directory", - "output_path", - "venue_map", - ) - ): - raise UnitBlockedError( - f"{label} has static paths on a daily pipeline job" - ) - if broker != "kis-live" or mode != "shadow": - raise UnitBlockedError( - f"{label} daily pipeline requires kis-live/shadow" - ) - if kind != "daily-entry" and max_cycles: - raise UnitBlockedError( - f"{label}.max_cycles is allowed only for daily-entry" - ) - elif kind == "eod": - if path_values["input_path"] is None: - raise UnitBlockedError(f"{label}.input_path is required for EOD") - if any( - path_values[field] is not None - for field in ( - "plan_path", - "state_directory", - "output_path", - "venue_map", - ) - ): - raise UnitBlockedError(f"{label} has entry-only paths on an EOD job") - if broker or mode or max_cycles: - raise UnitBlockedError(f"{label} has entry-only values on an EOD job") - elif kind == "snapshot": - if path_values["input_path"] is None: - raise UnitBlockedError( - f"{label}.input_path is required for an account snapshot" - ) - if any( - path_values[field] is not None - for field in ( - "plan_path", - "state_directory", - "output_path", - "venue_map", - ) - ): - raise UnitBlockedError( - f"{label} has entry-only paths on an account snapshot job" - ) - if broker != "kis-live" or mode != "shadow": - raise UnitBlockedError( - f"{label} account snapshot requires kis-live/shadow" - ) - if max_cycles: - raise UnitBlockedError( - f"{label}.max_cycles must be zero for an account snapshot" - ) - else: - required = ("plan_path", "state_directory", "output_path") - if any(path_values[field] is None for field in required): - raise UnitBlockedError(f"{label} is missing an entry path") - if path_values["input_path"] is not None: - raise UnitBlockedError(f"{label}.input_path must be empty for entry") - if broker not in {"kis-paper", "kis-live"}: - raise UnitBlockedError(f"{label}.broker must be kis-paper or kis-live") - if mode not in {"paper", "shadow"}: - raise UnitBlockedError( - f"{label}.mode must be paper or shadow; live is not generated" - ) - if broker == "kis-paper" and mode != "paper": - raise UnitBlockedError(f"{label} kis-paper requires paper mode") - if broker == "kis-live" and mode != "shadow": - raise UnitBlockedError(f"{label} kis-live requires shadow mode") - normalized_jobs.append( - { - "name": name, - "kind": kind, - "market": market, - "schedule": SCHEDULES[(kind, market)], - "persistent": kind in {"eod", "daily-prepare"}, - "input_path": str(path_values["input_path"] or ""), - "plan_path": str(path_values["plan_path"] or ""), - "state_directory": str(path_values["state_directory"] or ""), - "output_path": str(path_values["output_path"] or ""), - "broker": broker, - "mode": mode, - "venue_map": str(path_values["venue_map"] or ""), - "max_cycles": max_cycles, - "timeout_start_seconds": timeout, - } - ) - normalized_jobs.sort(key=lambda item: item["name"]) - return { - "schema": BUNDLE_SCHEMA, - "unit_prefix": prefix, - "python_executable": str(python.resolve()), - "technical_skill_root": str(technical_root.resolve()), - "trader_skill_root": str(trader_root.resolve()), - "environment_file": str(environment.resolve()), - "runtime_directory": str(runtime.resolve(strict=False)), - "jobs": normalized_jobs, - } - - -def execution_bundle(bundle: dict[str, Any]) -> dict[str, Any]: - """Project a normalized bundle back to the exact executable v1 schema.""" - return { - key: ( - [ - {field: job[field] for field in JOB_FIELDS} - for job in bundle["jobs"] - ] - if key == "jobs" - else bundle[key] - ) - for key in TOP_FIELDS - } - - -def systemd_quote(value: str) -> str: - escaped = ( - value.replace("\\", "\\\\") - .replace('"', '\\"') - .replace("$", "\\$") - .replace("%", "%%") - ) - return f'"{escaped}"' - - -def systemd_path(value: str) -> str: - """Escape one absolute path for directives that do not accept quoting.""" - escaped: list[str] = [] - for octet in value.encode("utf-8"): - character = chr(octet) - if character == "%": - escaped.append("%%") - elif 0x21 <= octet <= 0x7E and character not in {'"', "'", "\\"}: - escaped.append(character) - else: - escaped.append(f"\\x{octet:02x}") - return "".join(escaped) - - -def service_text(bundle: dict[str, Any], job: dict[str, Any], bundle_path: Path) -> str: - root = Path(bundle["runtime_directory"]) - conditions = [] - for field in ("input_path", "plan_path", "venue_map"): - if job[field]: - conditions.append(f"ConditionPathExists={systemd_path(job[field])}") - command = " ".join( - systemd_quote(item) - for item in ( - bundle["python_executable"], - "-B", - "-s", - str(Path(bundle["trader_skill_root"]) / "scripts" / "systemd_units.py"), - "execute", - "--bundle", - str(bundle_path), - "--name", - job["name"], - ) - ) - dependencies: list[str] = ["network-online.target"] - predecessor_kind = { - "daily-snapshot": "daily-prepare", - "daily-entry": "daily-snapshot", - }.get(job["kind"]) - if predecessor_kind is not None: - predecessor = next( - ( - item - for item in bundle["jobs"] - if item["kind"] == predecessor_kind - and item["market"] == job["market"] - ), - None, - ) - if predecessor is not None: - dependencies.append( - f"{bundle['unit_prefix']}-{predecessor['name']}.service" - ) - return ( - "[Unit]\n" - f"Description=QTA {job['kind']} {job['market']} ({job['name']})\n" - f"After={' '.join(dependencies)}\n" - "Wants=network-online.target\n" - + ("\n".join(conditions) + "\n" if conditions else "") - + "\n[Service]\n" - "Type=oneshot\n" - f"WorkingDirectory={systemd_path(str(root))}\n" - f"EnvironmentFile={systemd_path(bundle['environment_file'])}\n" - 'Environment="PYTHONDONTWRITEBYTECODE=1"\n' - "UnsetEnvironment=PYTHONPATH PYTHONHOME\n" - f"ExecStart={command}\n" - f"TimeoutStartSec={job['timeout_start_seconds']}s\n" - "UMask=0077\n" - "NoNewPrivileges=yes\n" - "PrivateTmp=yes\n" - "PrivateDevices=yes\n" - "ProtectSystem=strict\n" - "ProtectHome=read-only\n" - "ProtectKernelTunables=yes\n" - "ProtectKernelModules=yes\n" - "ProtectControlGroups=yes\n" - "RestrictSUIDSGID=yes\n" - "LockPersonality=yes\n" - "MemoryDenyWriteExecute=yes\n" - "RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\n" - f"ReadWritePaths={systemd_path(str(root))}\n" - f"ReadOnlyPaths={systemd_path(bundle['technical_skill_root'])} " - f"{systemd_path(bundle['trader_skill_root'])} " - f"{systemd_path(bundle['environment_file'])}\n" - ) - - -def timer_text(prefix: str, job: dict[str, Any]) -> str: - return ( - "[Unit]\n" - f"Description=Schedule QTA {job['kind']} {job['market']} ({job['name']})\n" - "\n[Timer]\n" - f"OnCalendar={job['schedule']}\n" - "AccuracySec=1s\n" - "RandomizedDelaySec=0\n" - f"Persistent={'yes' if job['persistent'] else 'no'}\n" - f"Unit={prefix}-{job['name']}.service\n" - "\n[Install]\n" - "WantedBy=timers.target\n" - ) - - -def generate(bundle: dict[str, Any], output_directory: Path) -> dict[str, Any]: - if output_directory.exists(): - if output_directory.is_symlink() or not output_directory.is_dir(): - raise UnitBlockedError( - "output_directory must be a non-symlink directory" - ) - if any(output_directory.iterdir()): - raise UnitBlockedError( - "output_directory must be empty to prevent stale unit files" - ) - else: - output_directory.mkdir(parents=True) - bundle_path = output_directory / "systemd-bundle.json" - runnable_bundle = execution_bundle(bundle) - atomic_write( - bundle_path, - (json.dumps(runnable_bundle, indent=2, sort_keys=True) + "\n").encode( - "utf-8" - ), - ) - files: list[dict[str, Any]] = [] - prefix = bundle["unit_prefix"] - for job in bundle["jobs"]: - service_path = output_directory / f"{prefix}-{job['name']}.service" - timer_path = output_directory / f"{prefix}-{job['name']}.timer" - atomic_write(service_path, service_text(bundle, job, bundle_path).encode()) - atomic_write(timer_path, timer_text(prefix, job).encode()) - for path in (service_path, timer_path): - files.append( - { - "path": str(path.resolve()), - "sha256": sha256_file(path), - "bytes": path.stat().st_size, - } - ) - files.sort(key=lambda item: item["path"]) - status = { - "schema": STATUS_SCHEMA, - "status": "READY", - "bundle_path": str(bundle_path.resolve()), - "bundle_sha256": sha256_file(bundle_path), - "unit_files": files, - "activation_performed": False, - "live_enabled": False, - "api_mutation_count": 0, - } - atomic_write( - output_directory / "generation-status.json", - (json.dumps(status, indent=2, sort_keys=True) + "\n").encode(), - ) - return status - - -def command_for(bundle: dict[str, Any], job: dict[str, Any]) -> list[str]: - python = bundle["python_executable"] - if job["kind"].startswith("daily-"): - stage = job["kind"].removeprefix("daily-") - command = [ - python, - "-B", - "-s", - str( - Path(bundle["trader_skill_root"]) - / "scripts" - / "daily_pipeline.py" - ), - stage, - "--config", - job["input_path"], - "--market", - job["market"], - "--technical-skill-root", - bundle["technical_skill_root"], - "--trader-skill-root", - bundle["trader_skill_root"], - ] - if job["kind"] == "daily-entry" and job["max_cycles"]: - command.extend(["--max-cycles", str(job["max_cycles"])]) - return command - if job["kind"] == "eod": - script_name = ( - "fetch_kis_kr_eod.py" if job["market"] == "KR" else "fetch_kis_us_eod.py" - ) - return [ - python, - "-B", - "-s", - str(Path(bundle["technical_skill_root"]) / "scripts" / script_name), - "collect", - "--job", - job["input_path"], - ] - if job["kind"] == "snapshot": - return [ - python, - "-B", - "-s", - str( - Path(bundle["trader_skill_root"]) - / "scripts" - / "account_snapshot.py" - ), - "collect", - "--job", - job["input_path"], - ] - command = [ - python, - "-B", - "-s", - str(Path(bundle["trader_skill_root"]) / "scripts" / "run_session.py"), - "run", - "--plan", - job["plan_path"], - "--broker", - job["broker"], - "--mode", - job["mode"], - "--state-dir", - job["state_directory"], - "--output", - job["output_path"], - ] - if job["venue_map"]: - command.extend(["--venue-map", job["venue_map"]]) - if job["max_cycles"]: - command.extend(["--max-cycles", str(job["max_cycles"])]) - return command - - -def acquire_market_locks( - runtime: Path, markets: list[str] -) -> list[int]: - lock_directory = runtime / "locks" - try: - lock_directory.mkdir(parents=True, exist_ok=True, mode=0o700) - lock_metadata = lock_directory.lstat() - except OSError as exc: - raise UnitBlockedError("cannot create or inspect the lock directory") from exc - if stat.S_ISLNK(lock_metadata.st_mode) or not stat.S_ISDIR( - lock_metadata.st_mode - ): - raise UnitBlockedError("lock directory must be a non-symlink directory") - descriptors: list[int] = [] - try: - for market in sorted(set(markets)): - lock_path = lock_directory / f"{market.lower()}.lock" - lock_flags = os.O_RDWR | os.O_CREAT - if hasattr(os, "O_NOFOLLOW"): - lock_flags |= os.O_NOFOLLOW - try: - descriptor = os.open(lock_path, lock_flags, 0o600) - except OSError as exc: - raise UnitBlockedError( - f"cannot securely open the {market} market lock" - ) from exc - if not stat.S_ISREG(os.fstat(descriptor).st_mode): - os.close(descriptor) - raise UnitBlockedError("market lock must be a regular file") - try: - os.fchmod(descriptor, 0o600) - except OSError as exc: - os.close(descriptor) - raise UnitBlockedError( - f"cannot restrict the {market} market lock permissions" - ) from exc - if stat.S_IMODE(os.fstat(descriptor).st_mode) != 0o600: - os.close(descriptor) - raise UnitBlockedError( - f"{market} market lock permissions must be 0600" - ) - try: - fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) - except BlockingIOError as exc: - os.close(descriptor) - raise UnitBlockedError( - f"another {market} QTA process owns the single-writer lock" - ) from exc - os.set_inheritable(descriptor, True) - descriptors.append(descriptor) - except BaseException: - for descriptor in descriptors: - os.close(descriptor) - raise - return descriptors - - -def ntfy_topic_url(environment: Mapping[str, str]) -> str | None: - value = environment.get(NTFY_TOPIC_URL_ENV, "").strip() - if not value: - return None - try: - parsed = urllib.parse.urlsplit(value) - except ValueError as exc: - raise UnitBlockedError( - f"{NTFY_TOPIC_URL_ENV} must be a plain HTTP(S) topic URL" - ) from exc - if ( - parsed.scheme not in {"http", "https"} - or not parsed.netloc - or parsed.username is not None - or parsed.password is not None - or parsed.query - or parsed.fragment - or not parsed.path.strip("/") - or any(ord(character) < 0x20 for character in value) - ): - raise UnitBlockedError( - f"{NTFY_TOPIC_URL_ENV} must be a plain HTTP(S) topic URL" - ) - return value - - -def read_daily_outcome( - runtime: Path, - job: Mapping[str, Any], - returncode: int, -) -> tuple[str, list[str]]: - outcome = "PASS" if returncode == 0 else "BLOCKED" - details: list[str] = [] - if not str(job["kind"]).startswith("daily-"): - return outcome, details - descriptor_path = runtime / "current" / f"{str(job['market']).lower()}.json" - try: - descriptor = json.loads(descriptor_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - return outcome, details - if not isinstance(descriptor, dict) or descriptor.get("market") != job["market"]: - return outcome, details - status = descriptor.get("status") - if ( - isinstance(status, str) - and status - and ( - returncode == 0 - or status in {"BLOCKED", "MANUAL_BLOCK"} - ) - ): - outcome = status - session_date = descriptor.get("session_date") - if isinstance(session_date, str) and session_date: - details.append(f"session={session_date}") - if job["kind"] == "daily-snapshot": - plan_hash = descriptor.get("plan_hash") - if isinstance(plan_hash, str) and plan_hash: - details.append(f"plan={plan_hash[:12]}") - if job["kind"] == "daily-entry": - report_path = descriptor.get("final_report_path") - if isinstance(report_path, str) and report_path: - try: - report = json.loads(Path(report_path).read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - report = {} - if isinstance(report, dict): - counts = report.get("counts") - if isinstance(counts, dict): - for field in ( - "planned", - "submitted", - "filled", - "cancelled", - "blocked", - ): - value = counts.get(field) - if isinstance(value, int) and not isinstance(value, bool): - details.append(f"{field}={value}") - return outcome, details - - -def publish_ntfy( - *, - environment: Mapping[str, str], - job: Mapping[str, Any], - outcome: str, - duration_seconds: int, - details: list[str], -) -> dict[str, str]: - try: - topic_url = ntfy_topic_url(environment) - except UnitBlockedError: - return { - "channel": "ntfy", - "status": "FAILED", - "reason": "INVALID_TOPIC_URL", - } - if topic_url is None: - return {"channel": "ntfy", "status": "DISABLED"} - blocked = outcome in {"BLOCKED", "MANUAL_BLOCK"} - title = f"QTA {job['market']} {job['kind']} {outcome}" - message_parts = [ - f"market={job['market']}", - f"stage={job['kind']}", - f"status={outcome}", - f"duration={duration_seconds}s", - *details, - "live_enabled=false", - "broker_api_mutations=0", - ] - headers = { - "Content-Type": "text/plain; charset=utf-8", - "Title": title, - "Priority": "urgent" if blocked else ( - "high" if job["kind"] == "daily-entry" else "default" - ), - "Tags": "warning" if blocked else "white_check_mark", - } - token = environment.get(NTFY_TOKEN_ENV, "").strip() - if token: - if any(ord(character) < 0x21 or ord(character) > 0x7E for character in token): - return { - "channel": "ntfy", - "status": "FAILED", - "reason": "INVALID_TOKEN", - } - headers["Authorization"] = f"Bearer {token}" - try: - request = urllib.request.Request( - topic_url, - data=" ".join(message_parts).encode("utf-8"), - headers=headers, - method="POST", - ) - with urllib.request.urlopen( - request, - timeout=NTFY_TIMEOUT_SECONDS, - ) as response: - status = getattr(response, "status", 200) - if not 200 <= status < 300: - return { - "channel": "ntfy", - "status": "FAILED", - "reason": f"HTTP_{status}", - } - except urllib.error.HTTPError as exc: - return { - "channel": "ntfy", - "status": "FAILED", - "reason": f"HTTP_{exc.code}", - } - except (urllib.error.URLError, TimeoutError, OSError, ValueError): - return { - "channel": "ntfy", - "status": "FAILED", - "reason": "TRANSPORT_ERROR", - } - return {"channel": "ntfy", "status": "SENT"} - - -def execute(bundle: dict[str, Any], name: str) -> int: - job = next((item for item in bundle["jobs"] if item["name"] == name), None) - if job is None: - raise UnitBlockedError(f"unknown job: {name}") - runtime = Path(bundle["runtime_directory"]) - lock_markets = [job["market"]] - acquire_market_locks(runtime, lock_markets) - environment = dict(os.environ) - environment["PYTHONDONTWRITEBYTECODE"] = "1" - environment.pop("PYTHONPATH", None) - environment.pop("PYTHONHOME", None) - started = time.monotonic() - result = subprocess.run( - command_for(bundle, job), - check=False, - env=environment, - ) - duration = max(0, round(time.monotonic() - started)) - outcome, details = read_daily_outcome(runtime, job, result.returncode) - notification = publish_ntfy( - environment=environment, - job=job, - outcome=outcome, - duration_seconds=duration, - details=details, - ) - print(json.dumps(notification, sort_keys=True)) - return result.returncode - - -def self_test() -> None: - import tempfile - - with tempfile.TemporaryDirectory(prefix="qta-systemd-") as temporary: - root = Path(temporary) - technical = root / "technical" - trader = root / "trader" - runtime = root / "runtime" - output = root / "units" - for path in ( - technical / "scripts", - trader / "scripts", - runtime, - ): - path.mkdir(parents=True) - for path in ( - technical / "scripts" / "fetch_kis_kr_eod.py", - technical / "scripts" / "fetch_kis_us_eod.py", - trader / "scripts" / "run_session.py", - trader / "scripts" / "account_snapshot.py", - trader / "scripts" / "daily_pipeline.py", - trader / "scripts" / "market_calendar.py", - trader / "scripts" / "systemd_units.py", - runtime / "kr-eod.json", - runtime / "kr-account-snapshot.json", - runtime / "us-plan.json", - ): - path.write_text("{}\n", encoding="utf-8") - environment = root / "secrets.env" - environment.write_text("QTA_SELF_TEST=1\n", encoding="utf-8") - environment.chmod(0o600) - raw = { - "schema": BUNDLE_SCHEMA, - "unit_prefix": "qta", - "python_executable": str(Path(sys.executable).resolve()), - "technical_skill_root": str(technical), - "trader_skill_root": str(trader), - "environment_file": str(environment), - "runtime_directory": str(runtime), - "jobs": [ - { - "name": "eod-kr", - "kind": "eod", - "market": "KR", - "input_path": str(runtime / "kr-eod.json"), - "plan_path": "", - "state_directory": "", - "output_path": "", - "broker": "", - "mode": "", - "venue_map": "", - "max_cycles": 0, - "timeout_start_seconds": 7200, - }, - { - "name": "snapshot-kr", - "kind": "snapshot", - "market": "KR", - "input_path": str(runtime / "kr-account-snapshot.json"), - "plan_path": "", - "state_directory": "", - "output_path": "", - "broker": "kis-live", - "mode": "shadow", - "venue_map": "", - "max_cycles": 0, - "timeout_start_seconds": 300, - }, - { - "name": "entry-us", - "kind": "entry", - "market": "US", - "input_path": "", - "plan_path": str(runtime / "us-plan.json"), - "state_directory": str(runtime / "state-us"), - "output_path": str(runtime / "us-receipt.json"), - "broker": "kis-live", - "mode": "shadow", - "venue_map": "", - "max_cycles": 1, - "timeout_start_seconds": 3900, - }, - ], - } - bundle = normalize_bundle(raw) - receipt = generate(bundle, output) - if len(receipt["unit_files"]) != 6: - raise AssertionError("systemd self-test unit count mismatch") - service = (output / "qta-entry-us.service").read_text() - timer = (output / "qta-entry-us.timer").read_text() - if "ProtectSystem=strict" not in service or "live" in service: - raise AssertionError("systemd self-test hardening failed") - if ( - f"WorkingDirectory={systemd_path(bundle['runtime_directory'])}\n" - not in service - or f"EnvironmentFile={systemd_path(bundle['environment_file'])}\n" - not in service - or "UnsetEnvironment=PYTHONPATH PYTHONHOME\n" not in service - ): - raise AssertionError("systemd self-test path rendering failed") - if "09:29:00 America/New_York" not in timer or "Persistent=no" not in timer: - raise AssertionError("systemd self-test timer failed") - if read_bundle(output / "systemd-bundle.json") != bundle: - raise AssertionError("generated bundle is not executable by the reader") - permissive_lock = runtime / "locks" / "us.lock" - permissive_lock.parent.mkdir(mode=0o700) - permissive_lock.write_bytes(b"") - permissive_lock.chmod(0o644) - descriptors = acquire_market_locks(runtime, ["US"]) - try: - if stat.S_IMODE(permissive_lock.stat().st_mode) != 0o600: - raise AssertionError( - "existing market lock permissions were not restricted" - ) - finally: - for descriptor in descriptors: - os.close(descriptor) - print( - json.dumps( - { - "self_test": "PASS", - "unit_files": 6, - "activation_performed": False, - "live_enabled": False, - }, - sort_keys=True, - ) - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - subparsers = parser.add_subparsers(dest="command", required=True) - generate_parser = subparsers.add_parser("generate") - generate_parser.add_argument("--bundle", required=True) - generate_parser.add_argument("--output-directory", required=True) - execute_parser = subparsers.add_parser("execute") - execute_parser.add_argument("--bundle", required=True) - execute_parser.add_argument("--name", required=True) - subparsers.add_parser("self-test") - return parser.parse_args() - - -def read_bundle(path: Path) -> dict[str, Any]: - regular_nonsymlink(path, "bundle") - try: - raw = json.loads(path.read_text(encoding="utf-8")) - except json.JSONDecodeError as exc: - raise UnitBlockedError("bundle is not valid JSON") from exc - return normalize_bundle(raw) - - -def main() -> int: - args = parse_args() - try: - if args.command == "self-test": - self_test() - return 0 - bundle_path = absolute_path(args.bundle, "bundle") - bundle = read_bundle(bundle_path) - if args.command == "generate": - receipt = generate( - bundle, absolute_path(args.output_directory, "output_directory") - ) - print(json.dumps(receipt, sort_keys=True)) - return 0 - return execute(bundle, args.name) - except (UnitBlockedError, OSError) as exc: - print( - json.dumps( - {"status": "BLOCKED", "reason": str(exc)}, - ensure_ascii=False, - sort_keys=True, - ), - file=sys.stderr, - ) - return 2 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/skills/quant-stock-technical/SKILL.md b/skills/quant-stock-technical/SKILL.md deleted file mode 100644 index 609588b..0000000 --- a/skills/quant-stock-technical/SKILL.md +++ /dev/null @@ -1,131 +0,0 @@ ---- -name: quant-stock-technical -description: Acquire resumable read-only KIS Korean and U.S. adjusted-EOD bundles, calculate reproducible numbers-only stock technical analysis, screen deterministic KOSPI, KOSDAQ, NYSE, and NASDAQ target universes, and evaluate frozen selected candidates against opening-hour outcomes. Use when the user needs completed historical OHLCV and benchmarks, an as-of-date market/ticker report, short-, medium-, and long-horizon opinions, a quantitative risk counterpoint, entry/stop/take-profit levels, a 0-100 technical-strength score, a frozen exchange-aware screen, or a read-only point-in-time or explicitly survivor-biased walk-forward diagnostic. Do not use news, fundamentals, analyst views, narrative judgment, intraday execution, or live-order claims. ---- - -# Quant Stock Technical - -Produce technical-analysis outputs only through the bundled deterministic calculator. Do not estimate indicators, prices, labels, or scores in prose. - -## Required inputs - -Obtain two finalized daily CSV files, or build the Korean files with the -read-only KIS EOD workflow below: - -- ticker OHLCV with `date,open,high,low,close,adjusted_close,volume` -- benchmark OHLCV with the same columns - -Also obtain `market`, `ticker`, and the market's valid `tick_size`. Treat the last shared CSV date as the default analysis date. Use `--analysis-date` for historical cutoffs. Require at least 756 shared completed sessions; recommend 1,260. - -Never treat an unfinished daily candle as completed. Never silently substitute unadjusted data, another ticker, another benchmark, a guessed tick size, or synthetic observations. - -## Korean EOD acquisition - -When completed Korean history is missing, read -`references/eod-provider-contract.md`. Do not stop merely because the caller -did not prebuild thousands of CSVs. Before the entry session: - -1. Run `scripts/fetch_kis_kr_eod.py snapshot-sources` to freeze KRX KIND and - KIS KOSPI/KOSDAQ master files. -2. Bind `analysis_date` to the frozen calendar's previous completed session. -3. Run `scripts/fetch_kis_kr_eod.py collect --job ...` with at least 756 - sessions and the existing secure KIS credential environment. -4. Preserve `eod-bundle-receipt.json`, `eod-catalog.csv`, benchmarks, stock - CSVs, and `universe-build-spec.json` together. -5. Continue to universe build and screening only when the required market - scope is complete. Never lower coverage or fabricate inactive exchanges. - -The collector is resumable: the first run backfills in bounded pages and later -runs update only missing completed sessions. Run it outside the first-hour -polling path. It performs authentication and market-data reads only; -`api_mutation_count` must remain zero and no account identifier is required. -Do not issue a separate token-only auth probe immediately before starting the -collector; KIS limits token issuance to once per minute. Treat the collector's -first successful read as authentication evidence or inject one -supervisor-held token through the environment. - -## U.S. EOD acquisition - -When NYSE/NASDAQ history is missing, read -`references/us-eod-provider-contract.md`. Freeze Nasdaq Trader official symbol -directories and KIS overseas masters with -`scripts/fetch_kis_us_eod.py snapshot-sources`, then run `collect --job ...` -outside the U.S. entry window. KIS stock history must use adjusted `MODP=1`; -preserve exchange-specific membership, broker symbols, benchmarks, and tick -mappings. A U.S.-scoped build uses NYSE/NASDAQ only; combine a completed -Korean catalog only when the caller explicitly requests an all-four-exchange -research universe. If KIS supplies an interior -adjusted row with invalid OHLC geometry, preserve its hash and values in the -invalid-row audit and exclude only that real row without interpolation. An -invalid cutoff row remains blocking. - -## Workflow - -1. Read `references/methodology.md` when explaining, reviewing, or changing the model. -2. Run `scripts/analyze_stock.py` for the stable QTA1 baseline. Use - `scripts/analyze_stock_v2.py` only for first-hour research/shadow work. -3. Return the calculator output without changing values or labels. -4. State the data source and cutoff. Keep `score_basis` visible whenever a user could mistake the score for a probability. -5. If the calculator returns `BLOCKED`, report the missing or invalid input. Do not manufacture a partial recommendation. - -Example: - -```bash -python3 scripts/analyze_stock.py \ - --ticker-csv arm.csv \ - --benchmark-csv soxx.csv \ - --market NASDAQ \ - --ticker ARM \ - --tick-size 0.01 \ - --source-name "licensed-eod-export" \ - --format markdown -``` - -Use `--format json` for machine-readable output. Run `python3 scripts/analyze_stock.py --self-test` after modifying the calculator. - -For a Korean, U.S., or all-four-exchange target universe, first read -`references/universe-contract.md`, build `qta-universe-manifest/v2` with -`scripts/build_universe_manifest.py`, then read `references/screen-contract.md` -and run `scripts/screen_universe.py`. Use `qta-universe-manifest/v1` only for -legacy, explicitly curated KR/US inputs. The selector is a separate -cross-sectional strategy assumption; it does not turn either score into a -probability. `screen_universe.py --method-version qta-2.0.0` selects the -research-only first-hour model; its output additionally requires a fresh -same-session benchmark-regime admission in the execution runner. Pass the -complete frozen screen artifact to -`$quant-stock-polling-trader`; never submit orders from this skill. - -After changing universe or screening code, run: - -```bash -python3 scripts/fetch_kis_kr_eod.py --self-test -python3 scripts/fetch_kis_us_eod.py --self-test -python3 scripts/build_universe_manifest.py --self-test -python3 scripts/analyze_stock_v2.py --self-test -python3 scripts/collect_open1h_research.py --self-test -python3 scripts/evaluate_open1h_research.py --self-test -python3 scripts/selected_open1h_outcomes.py --self-test -python3 scripts/walk_forward_open1h.py --self-test -python3 scripts/screen_universe.py --self-test -``` - -For a longer selected-candidate diagnostic, use -`scripts/walk_forward_open1h.py`. It holds one source manifest membership -static across prior sessions and therefore labels the result -`STATIC_SOURCE_MANIFEST_SURVIVOR_BIASED`; it is research evidence, not a -point-in-time universe backtest or promotion evidence. - -When `$stock-scenario-story` will run next, save the unmodified JSON output to a file. That downstream skill requires `source_skill`, `result_schema`, `method_version`, and the complete calculator payload; do not summarize or reconstruct the handoff manually. - -## Output boundaries - -- Interpret the 0-100 score as historical technical strength under the emitted - method version, not as probability of profit. `qta-2.0.0` remains - `RESEARCH_ONLY` until multi-session walk-forward evidence exists. -- Treat entry, stop, and take-profit as calculated price levels, not execution or suitability advice. -- Keep brokerage fees, tax, FX, slippage, position sizing, account constraints, and order semantics outside this technical score unless a later version explicitly models them. -- Do not claim real-time monitoring. This version consumes completed end-of-day data. -- Keep account identifiers, intraday quotes, and broker mutations outside this - skill. KIS app credentials may be resolved only by the read-only EOD - collector through the existing secure environment or mode-0600 fallback; - never persist or print them. diff --git a/skills/quant-stock-technical/agents/openai.yaml b/skills/quant-stock-technical/agents/openai.yaml deleted file mode 100644 index 5602b74..0000000 --- a/skills/quant-stock-technical/agents/openai.yaml +++ /dev/null @@ -1,4 +0,0 @@ -interface: - display_name: "Quant Stock Technical" - short_description: "Four-exchange screening and opening-hour research" - default_prompt: "Use $quant-stock-technical to acquire read-only KIS EOD data, calculate reproducible technical scores, screen a frozen four-exchange universe, or evaluate selected opening-hour outcomes." diff --git a/skills/quant-stock-technical/references/eod-provider-contract.md b/skills/quant-stock-technical/references/eod-provider-contract.md deleted file mode 100644 index 3193569..0000000 --- a/skills/quant-stock-technical/references/eod-provider-contract.md +++ /dev/null @@ -1,139 +0,0 @@ -# KIS Korean adjusted-EOD provider contract - -`fetch_kis_kr_eod.py` fills the operational input gap between a frozen Korean -market session and the offline universe builder. It is a read-only, pre-open -data stage. It never belongs in the first-hour polling loop and never calls a -broker order, account, balance, fill, correction, or cancellation endpoint. - -## Source contract - -Freeze dated source masters first: - -```bash -python3 scripts/fetch_kis_kr_eod.py snapshot-sources \ - --as-of 2026-07-27 \ - --output-directory /secure/qta/sources/2026-07-27/kr -``` - -The command downloads the KRX KIND corporate-list HTML export and the official -KIS KOSPI/KOSDAQ master ZIPs, verifies their expected format, extracts only the -named `.mst` members, hashes every file, and emits `source-snapshot.json`. -Review the provider URLs when they change; never silently replace a failed -official snapshot with a scraped or synthetic list. - -## EOD job - -Create one exact `qta-kis-kr-eod-job/v1` JSON object: - -```json -{ - "schema": "qta-kis-kr-eod-job/v1", - "as_of": "2026-07-27", - "analysis_date": "2026-07-24", - "environment": "live", - "output_directory": "/secure/qta/eod/kr", - "history_start_date": "2021-01-01", - "minimum_sessions": 756, - "request_interval_ms": 120, - "official_sources": [], - "broker_sources": [], - "catalog_coverage_contract": { - "schema": "qta-catalog-coverage-contract/v1", - "minimum_ratio_by_exchange": { - "KOSPI": "1", - "KOSDAQ": "1", - "NYSE": "1", - "NASDAQ": "1" - }, - "minimum_screenable_ratio_by_exchange": { - "KOSPI": "0.5", - "KOSDAQ": "0.5", - "NYSE": "0.5", - "NASDAQ": "0.5" - } - }, - "base_eod_catalog": "" -} -``` - -Copy the exact KOSPI/KOSDAQ descriptors from `source-snapshot.json` into the -two source arrays. This produces a complete Korean-scoped v2 build spec. -Other exchange descriptors and a previously frozen cross-market catalog may -be included only for an explicit all-four-exchange research build. Relative -paths resolve from the job file. - -Run: - -```bash -python3 scripts/fetch_kis_kr_eod.py collect --job kr-eod-job.json -``` - -The KIS domestic daily endpoint is queried with -`FID_ORG_ADJ_PRC=0`, which the KIS interface defines as adjusted price. Its -adjusted OHLC is written to the calculator schema with `adjusted_close` equal -to the returned adjusted `close`. The same cutoff is enforced for KOSPI index -code `0001` and KOSDAQ index code `1001`. - -## Cache and completeness - -- The first run walks backward using the broker's bounded page size. The stock - endpoint currently returns up to 100 rows and the index endpoint may return - 50; completion is determined by the date cursor, never by assuming a short - page is final. -- Each successful symbol CSV is written atomically and survives an interrupted - bootstrap. -- Later runs reuse complete history and request only a missing cutoff. -- The exact `analysis_date` must exist in every admitted stock and benchmark. -- At least 756 sessions are required; fewer rows are a per-symbol failure, not - permission to synthesize observations. -- A conflicting duplicate date, invalid OHLC geometry, malformed response, - persistent rate limit, or missing completed candle is `BLOCKED`. -- The output catalog still records every official Korean member. Unsupported - instruments and missing KIS matches remain explicit exclusions. - -An individual new listing with fewer than 756 sessions does not make the -collector itself fail. Its catalog row remains present but has no admissible -EOD path or tick contract. The downstream hashed screenable-coverage policy, -not the collector, decides whether the complete market scope is `READY`. - -The output directory contains: - -```text -benchmarks/ -stocks/KOSPI/ -stocks/KOSDAQ/ -eod-catalog.csv -universe-build-spec.json -eod-bundle-receipt.json -``` - -If only Korean source descriptors are supplied, the emitted build spec is a -complete KOSPI/KOSDAQ input. The v2 builder emits zero counts for inactive -NYSE/NASDAQ keys and evaluates coverage only for the active Korean pair. Do -not add U.S. inputs to a Korean scheduled task or lower coverage thresholds. - -## Credentials and mutation boundary - -The collector resolves only KIS app key and secret from existing -`QTA_KIS_*` environment variables or the user-owned mode-0600 fallback already -used by the polling skill. An optional access token may be injected through -`QTA_KIS_ACCESS_TOKEN` or its scoped form; tokens are never written to disk or -receipts. When no token is injected, one token is requested for the collector -process. KIS limits token issuance to once per minute. Do not run a separate -token-only `auth-check` immediately before a collector process. Prefer using -the collector's first successful market-data response as the authentication -gate, or inject the same in-memory supervisor token through the process -environment. If KIS returns `EGW00133`, the collector waits 61 seconds and -retries token issuance exactly once. - -Every receipt fixes `api_mutation_count` to zero. Authentication and quotation -reads do not arm live trading. - -## Tick contract - -The collector resolves the current KRX equity quotation unit from the final -completed adjusted close and records an effective, source-observed rule ID, -reference price, and tick size for every admitted symbol. KOSPI and KOSDAQ -remain distinct ladders. ETF, ETN, ELW, preferred, SPAC, right, warrant, -abnormal, halted, liquidation, and administrative issues are not promoted into -the common-stock screen. diff --git a/skills/quant-stock-technical/references/methodology.md b/skills/quant-stock-technical/references/methodology.md deleted file mode 100644 index edebb15..0000000 --- a/skills/quant-stock-technical/references/methodology.md +++ /dev/null @@ -1,162 +0,0 @@ -# QTA 1.0.0 methodology - -## Scope - -`qta-1.0.0` is a long-only, daily, trend-following technical-strength model. It uses adjusted OHLCV and a same-market benchmark. It contains no news, fundamentals, sentiment, analyst estimates, sector narrative, or discretionary overrides. - -## Causal data handling - -- Use only sessions on or before the analysis date. -- Intersect ticker and benchmark dates. -- Align adjusted historical prices to the ticker's raw close scale on the analysis date. -- Exclude the current feature observation from its historical reference distribution. -- Require at least 756 shared sessions and at least 252 valid prior observations for every scored feature. - -## Percentile transform - -For each feature, compare the current value with up to 756 preceding valid values: - -```text -percentile = 100 * (count(prior < current) + 0.5 * count(prior = current)) / count(prior) -``` - -This is a ticker-relative historical percentile. It is not a forecast probability and is not a cross-sectional rank against other securities. - -## Horizon scores - -Short score: - -```text -25% return_5 -20% return_20 -20% close_over_sma_20 -15% sma_5_over_sma_20 -15% signed_log_volume_ratio_20 - 5% rsi_14 -``` - -Medium score: - -```text -25% return_63 -20% close_over_sma_50 -20% sma_20_over_sma_50 -20% relative_return_63 -15% adx_direction_14 -``` - -Long score: - -```text -25% return_126 -25% return_252 -20% close_over_sma_200 -15% sma_50_over_sma_200 -15% relative_return_252 -``` - -`signed_log_volume_ratio_20` is the sign of the one-day return multiplied by `ln(volume / median_volume_20)`. `adx_direction_14` is `(+DI - -DI) * min(ADX / 25, 1)`. - -## Risk score and total score - -Risk score: - -```text -30% percentile(ATR_14 / close) -25% percentile(20-session annualized log-return volatility) -25% percentile(252-session maximum drawdown magnitude) -20% percentile(60-session 95th-percentile absolute overnight gap) -``` - -Total score: - -```text -clip( - 0.25 * short - + 0.35 * medium - + 0.40 * long - - 0.20 * max(risk - 50, 0), - 0, - 100 -) -``` - -Opinion labels are mechanical: - -| Score | Label | -|---:|---| -| 80-100 | 강한 긍정 | -| 60-79.999 | 긍정 | -| 40-59.999 | 중립 | -| 20-39.999 | 부정 | -| 0-19.999 | 강한 부정 | - -## Price levels - -Use the market tick size supplied by the caller: - -```text -entry = round_up_to_tick(max(current_close, prior_20_session_high + tick)) -stop = round_down_to_tick(entry - 2 * ATR_14) -take_profit = round_down_to_tick(entry + 2 * (entry - stop)) -``` - -The setup is `READY` when total score is at least 60 and otherwise `CONDITIONAL`. Data or calculation failures return `BLOCKED`. - -## Quantitative counterpoint - -Select the risk component with the highest historical percentile. Render only its metric value, historical percentile, and entry-to-stop percentage using a fixed template. Do not add narrative judgment. - -## Downstream handoff contract - -Machine-readable results include: - -```text -source_skill = quant-stock-technical -result_schema = quant-stock-technical/v1 -method_version = qta-1.0.0 -calculation_status = READY -``` - -`stock-scenario-story` accepts only the complete unmodified JSON result. A Markdown rendering, hand-entered score summary, incomplete payload, or failed calculation is not a valid handoff. - -## QTA 2.0.0 first-hour research model - -`qta-2.0.0` reuses the same causal daily features, cutoff rules, price levels, -and per-ticker historical percentile transform. It changes only the horizon -aggregation for the first regular-session hour: - -```text -clip( - 0.50 * short - + 0.30 * medium - + 0.20 * (100 - risk), - 0, - 100 -) -``` - -Long-horizon strength remains visible in the payload but has zero weight in -the first-hour total. A setup also requires median 20-session turnover of at -least KRW 1,000,000,000 for Korean instruments or USD 1,000,000 for U.S. -instruments. - -The frozen payload requires a downstream same-session market-regime check: - -```text -metric = same_session_previous_close_return_bps -minimum_change_bps = 0 -max_age_seconds = 90 -``` - -The runner admits a new entry only when the exchange benchmark or documented -broad-market proxy is fresh, belongs to the current session, and is not below -the previous close. It never applies this check to exit management. - -The model was created after a full-universe diagnostic for the 2026-07-28 -opening hour. On that single session relative to QTA1, Spearman rank -correlation increased on KOSDAQ, KOSPI, NASDAQ, and NYSE. This is diagnostic -evidence, not sufficient promotion evidence. The payload therefore emits -`validation_status=RESEARCH_ONLY`, and the execution skill accepts it only in -shadow mode until multiple later sessions are evaluated without retuning the -formula. diff --git a/skills/quant-stock-technical/references/screen-contract.md b/skills/quant-stock-technical/references/screen-contract.md deleted file mode 100644 index bf70deb..0000000 --- a/skills/quant-stock-technical/references/screen-contract.md +++ /dev/null @@ -1,163 +0,0 @@ -# QTA screen contracts - -`qta-screen-1.0.0` applies a frozen cross-sectional selector to unmodified -`quant-stock-technical/v1` results. It does not modify `qta-1.0.0`. - -## Legacy v1 input - -Pass a universe manifest: - -```json -{ - "schema": "qta-universe-manifest/v1", - "analysis_date": "2026-07-25", - "instruments": [ - { - "market": "KR", - "ticker": "005930", - "ticker_csv": "data/005930.csv", - "benchmark_csv": "data/kospi.csv", - "tick_size": "100", - "source_name": "licensed-adjusted-eod" - } - ] -} -``` - -Relative CSV paths resolve from the manifest directory. Require unique -`(market, ticker)` pairs and `market` equal to `KR` or `US`. - -Pass an explicit selector: - -```json -{ - "selector_version": "qta-screen-1.0.0", - "min_total_score": "60", - "eligible_setup_statuses": ["READY"], - "top_k_by_market": {"KR": 5, "US": 5}, - "max_blocked_fraction": "0" -} -``` - -Do not hide strategy choices in defaults. Every field above is required. -`eligible_setup_statuses` must be a non-empty subset of the exact uppercase -values `READY` and `CONDITIONAL`; no other calculator or caller-defined status -is eligible. - -## Calculation and ordering - -Run the original calculator once per instrument with the manifest analysis -date. Preserve each complete calculator payload. - -Filter by calculation status, setup status, and minimum score. Sort eligible -results with this total ordering: - -```text -total_score descending -medium.score descending -long.score descending -short.score descending -risk.score ascending -market ascending -ticker ascending -``` - -The score is ticker-relative rather than a cross-sectional rank. Treat this -ordering as a new strategy assumption that needs its own historical, -walk-forward, and opening-hour execution validation. - -If the fraction of `BLOCKED` instruments exceeds `max_blocked_fraction`, return -the whole screen as `BLOCKED`. Never silently drop enough invalid instruments -to change the effective universe beyond that frozen tolerance. - -## Output - -Emit `qta-screen/v1` with: - -- selector and method versions; -- canonical hashes of the manifest and selector; -- the complete unmodified QTA payload for every instrument; -- deterministic eligibility decisions and ranks; -- selected candidates per market; -- explicit blocked reasons. - -Use the output only as an input to a separate execution skill. It is not an -order, suitability decision, or profit forecast. - -## Exchange-scoped v2 input - -Read `universe-contract.md` first. Pass the complete, unmodified -`qta-universe-manifest/v2` emitted by `build_universe_manifest.py`; the screen -recomputes its canonical hash and verifies source, CSV, tick, count, and broker -tradability metadata. The manifest may cover the Korean pair, the U.S. pair, -or all four exchanges. It rejects a `BLOCKED` manifest. - -Pass this exact selector: - -```json -{ - "selector_version": "qta-screen-1.1.0", - "min_total_score": "60", - "eligible_setup_statuses": ["READY"], - "top_k_by_exchange": { - "KOSPI": 2, - "KOSDAQ": 2, - "NYSE": 2, - "NASDAQ": 2 - }, - "min_selected_by_exchange": { - "KOSPI": 0, - "KOSDAQ": 0, - "NYSE": 0, - "NASDAQ": 0 - }, - "max_blocked_fraction": "0" -} -``` - -Every field is required. `top_k_by_exchange` must contain all four keys, -including any exchange intentionally assigned zero slots. -`min_selected_by_exchange` must also contain all four keys, and every value -must be between zero and that exchange's top K. Use zero to allow a valid -no-trade result when no setup passes; use one or more only when underfilling -that exchange should block the whole screen. The same exact -`eligible_setup_statuses` restriction from v1 applies. - -By default, run the stable `qta-1.0.0` calculator once per included -instrument. Pass `--method-version qta-2.0.0` only for the research-only -first-hour model described in `methodology.md`. Use -`canonical_symbol` as the QTA ticker, the explicit exchange benchmark CSV, and -`tick_contract.resolved_tick_size`; never use `data_symbol` or `broker_symbol` -interchangeably. - -Apply the method-specific score ordering independently inside each exchange. -QTA1 orders medium, long, short, then risk after total score. QTA2 orders -short, medium, risk, then long after total score. The final -tie-break is canonical symbol ascending. Set `exchange_rank` before taking the -configured top K. A score remains ticker-relative rather than a -cross-sectional percentile or probability. - -If the fraction of calculation-blocked included instruments exceeds -`max_blocked_fraction`, the entire v2 screen is `BLOCKED` and every exchange's -selected list is empty. The same happens when an exchange has fewer eligible -names than its explicit `min_selected_by_exchange`. Listings excluded by the -builder remain visible in the manifest and are not counted as attempted -calculations. - -Emit `qta-screen/v2` with: - -- the exact manifest hash, selector hash, and final screen hash; -- the complete unmodified QTA payload for each attempted instrument; -- explicit eligibility decisions and exchange ranks; -- exact selected keys `KOSPI`, `KOSDAQ`, `NYSE`, and `NASDAQ`; -- the complete v2 instrument metadata alongside every selected payload. - -The polling trader combines KOSPI then KOSDAQ for a Korean session, and NYSE -then NASDAQ for a U.S. session. It orders by the frozen exchange order, -`exchange_rank`, then `broker_symbol`; it does not rescore candidates. - -`screen_universe.py` uses up to eight worker processes by default. `--workers -1` provides the serial reference path. Both paths must emit byte-identical -artifacts. When `--output` is supplied, the complete screen is written only to -that file and stdout contains a compact hash/status summary rather than -thousands of instrument payloads. diff --git a/skills/quant-stock-technical/references/universe-contract.md b/skills/quant-stock-technical/references/universe-contract.md deleted file mode 100644 index 5339ebe..0000000 --- a/skills/quant-stock-technical/references/universe-contract.md +++ /dev/null @@ -1,252 +0,0 @@ -# Exchange-scoped universe contract - -`qta-universe-manifest/v2` freezes a reproducible, screen-ready intersection -for exactly one supported scope: KOSPI/KOSDAQ, NYSE/NASDAQ, or all four -exchanges. It is not a live quote feed and it does not make every listed -security eligible. - -## Source boundary - -Build each exchange from dated local snapshots: - -- an official listing or symbol-directory snapshot; -- a broker symbol-master snapshot; -- explicit symbol-to-adjusted-EOD-CSV mappings; -- an explicit same-exchange benchmark CSV mapping; -- explicit, effective-dated tick contracts. - -Hash every input file before parsing it. Keep the raw files immutable with the -result. Never query thousands of symbols one by one during the entry window. -Refresh and screen from completed T-1 data before the session, then pass only -the selected candidates to the polling trader. - -Run the offline builder with: - -```bash -python3 scripts/build_universe_manifest.py \ - --build-spec universe-build-spec.json \ - --output universe-manifest.json -``` - -The build spec has exactly: - -```text -schema, as_of, analysis_date, official_sources, broker_sources, eod_catalog, -catalog_coverage_contract -``` - -`schema` is `qta-universe-build-spec/v1`. Supply exactly one official source -and one KIS broker source for each exchange in the selected supported scope. -Do not mix a single exchange from either market. Every source descriptor has -exactly: - -```text -source_id, provider, exchange, as_of, path, format, encoding, -delimiter, skip_rows, columns, normal_status_values -``` - -All inputs are local snapshot files. The built-in direct formats are: - -| role | exchange | provider | format | -| --- | --- | --- | --- | -| official | KOSPI/KOSDAQ | `KRX` | `KRX_KIND_HTML` | -| official | NASDAQ | `NASDAQ_TRADER` | `NASDAQ_LISTED` | -| official | NYSE | `NASDAQ_TRADER` | `NASDAQ_OTHER` | -| broker | KOSPI/KOSDAQ | `KIS` | `KIS_KRX_MASTER` | -| broker | NASDAQ/NYSE | `KIS` | `KIS_OVERSEAS_MASTER` | - -`KRX_KIND_HTML` consumes the CP949 KIND corporate-list HTML-table export. -`KIS_KRX_MASTER` consumes the raw domestic `.mst` layout, including its -exchange-specific trailing flag block. `KIS_OVERSEAS_MASTER` consumes the raw, -headerless 24-column tab-separated `.COD` file. Set `columns` to `{}` for -these built-ins. Generic normalized inputs may instead use `KRX_CSV`, -`KIS_CSV`, or `KIS_FIXED_WIDTH` with explicit logical-column mappings. Never -apply decoded-character fixed offsets copied from one raw KIS market to -another. - -Acquire and timestamp snapshots outside the builder. Current official source -locations are: - -```text -KRX KIND: -https://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13 - -Nasdaq Trader: -https://www.nasdaqtrader.com/dynamic/SymDir/nasdaqlisted.txt -https://www.nasdaqtrader.com/dynamic/SymDir/otherlisted.txt - -KIS: -https://new.real.download.dws.co.kr/common/master/kospi_code.mst.zip -https://new.real.download.dws.co.kr/common/master/kosdaq_code.mst.zip -https://new.real.download.dws.co.kr/common/master/nasmst.cod.zip -https://new.real.download.dws.co.kr/common/master/nysmst.cod.zip -``` - -Check the provider's current documentation before automating downloads. The -builder deliberately performs no network access. - -`eod_catalog` has exactly `source_id, provider, as_of, path, encoding`. Its CSV -header is exactly, in this order: - -```text -exchange,canonical_symbol,data_symbol,broker_symbol,instrument_type, -benchmark_id,ticker_csv,benchmark_csv,tick_rule_id,tick_effective_date, -tick_reference_price,resolved_tick_size,source_name -``` - -The catalog is an explicit join contract, not a security master. A listed -symbol absent from it is excluded with `missing_eod_mapping`. Blank, invalid, -or nonexistent data and tick fields become per-symbol exclusion reasons rather -than guessed values. - -`catalog_coverage_contract` is a mandatory readiness gate with exactly: - -```json -{ - "schema": "qta-catalog-coverage-contract/v1", - "minimum_ratio_by_exchange": { - "KOSPI": "1", - "KOSDAQ": "1", - "NYSE": "1", - "NASDAQ": "1" - }, - "minimum_screenable_ratio_by_exchange": { - "KOSPI": "0.5", - "KOSDAQ": "0.5", - "NYSE": "0.5", - "NASDAQ": "0.5" - } -} -``` - -Each minimum is a decimal greater than zero and at most one. The builder -canonicalizes it to a decimal string. Both ratios use the frozen official-row -count as their denominator and are evaluated independently for every exchange: - -- `minimum_ratio_by_exchange` gates `catalog_mapped / official`. A value of - `1` requires an EOD-catalog row for every official member. -- `minimum_screenable_ratio_by_exchange` gates `included / official`, after - instrument classification, broker tradability, completed adjusted EOD, - benchmark, file-hash, and tick-contract checks. - -Both gates must pass. A catalog row alone never makes a symbol screenable, and -a high screenable ratio never excuses missing catalog mappings. -`screen_universe.py` independently recomputes both ratios from the hashed -manifest counts and rejects a forged `READY` manifest that misses either -minimum. - -The normalized contract is copied into the manifest before `manifest_hash` is -computed. This makes lowering or changing a threshold visible in the manifest -hash. For a full-universe build, use `1` for all four catalog-mapping -minimums. Choose screenable minimums from an explicit coverage policy that -accounts for intentionally excluded security classes, and do not lower either -set of values merely to turn a sparse proof-of-concept catalog `READY`. - -The canonical exchange mapping is: - -| exchange | market | venue | currency | benchmark_id | -| --- | --- | --- | --- | --- | -| `KOSPI` | `KR` | `KRX` | `KRW` | `KOSPI_COMPOSITE` | -| `KOSDAQ` | `KR` | `KRX` | `KRW` | `KOSDAQ_COMPOSITE` | -| `NYSE` | `US` | `NYSE` | `USD` | `NYSE_COMPOSITE` | -| `NASDAQ` | `US` | `NASD` | `USD` | `NASDAQ_COMPOSITE` | - -## Eligible instruments - -Include only `COMMON`, `ADR`, and `REIT`; `ADR` is U.S.-only. Exclude ETFs, -ETNs, preferred shares, SPACs, units, rights, warrants, test issues, abnormal -listing-status issues, and any row that cannot be classified deterministically. - -An otherwise eligible listing is still excluded when it lacks: - -- an unambiguous broker-master match; -- confirmed broker tradability; -- a ticker CSV and benchmark CSV whose last shared completed date reaches the - requested analysis date; -- file hashes and valid adjusted-OHLCV columns; -- an effective tick contract. - -Every exclusion must carry sorted, explicit reason codes. Never silently shrink -the universe. - -## Exact manifest - -The top-level object has exactly: - -```text -schema, build_status, as_of, analysis_date, source_hashes, -catalog_coverage_contract, instruments, exclusions, counts, manifest_hash -``` - -`schema` is `qta-universe-manifest/v2`. `build_status` is `READY` only when -every active exchange has at least one included instrument and meets both -hashed catalog-mapping and screenable-coverage minimums; otherwise it is -`BLOCKED`. Inactive exchanges must have zero counts. - -Each instrument has exactly: - -```text -market, exchange, canonical_symbol, data_symbol, broker_symbol, -instrument_type, benchmark_id, currency, venue, ticker_csv, benchmark_csv, -tick_contract, source_name, ticker_csv_sha256, benchmark_csv_sha256, -broker_tradability_verified, official_source_id, broker_source_id -``` - -All paths are absolute in the emitted manifest. Symbols have separate -canonical, data-provider, and broker representations; never assume they are -identical. - -The required tick object has exactly: - -```json -{ - "schema": "qta-tick-contract/v1", - "kind": "RESOLVED_PRICE_LADDER", - "rule_id": "provider-rule-identifier", - "effective_date": "2026-07-25", - "reference_price": "123.45", - "resolved_tick_size": "0.005" -} -``` - -The two numeric values are positive canonical decimal strings. Do not hard-code -one U.S. tick or one Korean tick for an entire exchange. - -`source_hashes` is sorted by `source_id`. Each entry has exactly -`source_id, role, provider, exchange, as_of, path, sha256`; `exchange` may be -`GLOBAL` for a cross-exchange mapping. Hashes are lowercase SHA-256 values. - -Each exclusion has exactly: - -```text -exchange, canonical_symbol, broker_symbol, name, reasons, -official_source_id, broker_source_id -``` - -`broker_symbol` and `broker_source_id` may be null. `reasons` is a non-empty, -sorted, unique list. - -`counts` has exact totals `official_rows, broker_rows, catalog_mapped_rows, -included, excluded` and a `by_exchange` object with all four exchanges. Each -exchange has exact `official, broker, catalog_mapped, included, excluded` -counts; totals must equal their sums. Keys remain present with all-zero values -for inactive exchanges so the v2 shape stays stable. `catalog_mapped` counts official symbols -with a catalog row, regardless of whether that row later passes EOD, tick, or -broker validation. `excluded` means exclusion-record count. It can include an -EOD-catalog row that is no longer an official member, so it is not defined as -`official - included`. - -`manifest_hash` is SHA-256 over UTF-8 canonical JSON for the entire object -excluding `manifest_hash`, using sorted keys, compact separators, and -non-ASCII characters unescaped. - -## Reproducibility and history - -Treat `as_of` as the membership/master snapshot date and `analysis_date` as the -completed EOD cutoff. Keep both dates and all hashes. Re-running from identical -snapshots must produce byte-identical JSON. - -A current listing snapshot cannot reconstruct a historical universe. For -historical or walk-forward testing, use the membership and broker-master -snapshots that were actually frozen for each historical date; otherwise label -the result as survivorship-biased and do not use it as promotion evidence. diff --git a/skills/quant-stock-technical/references/us-eod-provider-contract.md b/skills/quant-stock-technical/references/us-eod-provider-contract.md deleted file mode 100644 index 58dc9d1..0000000 --- a/skills/quant-stock-technical/references/us-eod-provider-contract.md +++ /dev/null @@ -1,60 +0,0 @@ -# KIS U.S. adjusted-EOD provider contract - -`fetch_kis_us_eod.py` produces a frozen NYSE/NASDAQ catalog before the U.S. -session. It performs authentication and quotation -reads only and fixes `api_mutation_count` to zero. - -## Frozen membership and tradability - -Run `snapshot-sources` with the source date. The command freezes and hashes: - -- Nasdaq Trader `nasdaqlisted.txt` for official NASDAQ membership; -- Nasdaq Trader `otherlisted.txt`, filtered to exchange `N`, for official NYSE - membership; -- KIS `nasmst.cod` and `nysmst.cod` for broker symbol qualification. - -The admitted universe is the deterministic intersection after the existing -common-stock, ADR, REIT, ETF, test-issue, abnormal-status, and broker -tradability rules. Never substitute AMEX, ARCA, OTC, a search result, or an -unqualified symbol. - -## Adjusted history - -KIS overseas daily history uses -`/uapi/overseas-price/v1/quotations/dailyprice`, TR `HHDFS76240000`, daily -`GUBN=0`, and adjusted-price `MODP=1`. NYSE uses `EXCD=NYS`; NASDAQ uses -`EXCD=NAS`. The cursor walks backward from the completed cutoff and writes -atomic, resumable CSV files. Every admitted ticker and benchmark requires the -exact cutoff and at least 756 sessions. - -KIS can return an interior adjusted row whose reported open, high, low, and -close do not form valid OHLC geometry. Exclude that exact provider row without -interpolation, record its exchange, symbol, date, normalized values, size, and -SHA-256 in `invalid-eod-rows.json`, and continue only if the remaining real -rows still meet the complete-session contract. An invalid row on the requested -cutoff remains `BLOCKED`; never invent the latest completed candle. The -receipt binds the audit file and reports policy -`EXCLUDED_WITHOUT_INTERPOLATION`. - -The existing v2 universe contract names `NYSE_COMPOSITE` and -`NASDAQ_COMPOSITE`. Their public benchmark symbols are `^NYA` and `^IXIC`. -Benchmark downloads are source-labeled and hashed; a malformed, incomplete, or -rate-limited response blocks the benchmark rather than silently substituting -S&P 500, an ETF, or another index. - -U.S. order-price resolution uses a conservative valid ladder: `0.0001` below -USD 1 and `0.01` at or above USD 1. A later version may consume an -effective-dated Regulation NMS variable tick table for finer price increments, -but it must not infer one from price alone. - -## Scope - -With only U.S. descriptors and an empty `base_eod_catalog`, the emitted -`universe-build-spec.json` is a complete NYSE/NASDAQ v2 input. The job may -also include all four descriptors and a completed Korean base catalog for an -explicit all-four-exchange research build. Daily U.S. execution uses the -U.S.-only form and does not refresh Korean EOD. - -The initial bootstrap belongs outside both first-hour entry windows. Completed -symbol files survive interruption; later runs fetch only the missing completed -cutoff. diff --git a/skills/quant-stock-technical/scripts/analyze_stock.py b/skills/quant-stock-technical/scripts/analyze_stock.py deleted file mode 100644 index 4a62bc1..0000000 --- a/skills/quant-stock-technical/scripts/analyze_stock.py +++ /dev/null @@ -1,693 +0,0 @@ -#!/usr/bin/env python3 -"""Deterministic daily stock technical analysis for quant-stock-technical.""" - -from __future__ import annotations - -import argparse -import csv -import json -import math -import statistics -import sys -import tempfile -from dataclasses import dataclass -from datetime import date, timedelta -from decimal import Decimal -from pathlib import Path -from typing import Any, Iterable, Optional - - -METHOD_VERSION = "qta-1.0.0" -MIN_SHARED_SESSIONS = 756 -PERCENTILE_LOOKBACK = 756 -MIN_REFERENCE_OBSERVATIONS = 252 - - -class BlockedError(ValueError): - pass - - -@dataclass(frozen=True) -class RawBar: - day: date - open: float - high: float - low: float - close: float - adjusted_close: float - volume: float - - -@dataclass(frozen=True) -class Bar: - day: date - open: float - high: float - low: float - close: float - volume: float - - -def _number(row: dict[str, str], field: str, row_number: int) -> float: - try: - value = float(row[field]) - except (KeyError, TypeError, ValueError) as exc: - raise BlockedError(f"row {row_number}: invalid or missing {field}") from exc - if not math.isfinite(value): - raise BlockedError(f"row {row_number}: non-finite {field}") - return value - - -def read_csv(path: str) -> list[RawBar]: - required = {"date", "open", "high", "low", "close", "adjusted_close", "volume"} - rows: list[RawBar] = [] - with Path(path).open("r", encoding="utf-8-sig", newline="") as handle: - reader = csv.DictReader(handle) - fields = {str(x).strip().lower() for x in (reader.fieldnames or [])} - if not required.issubset(fields): - missing = ", ".join(sorted(required - fields)) - raise BlockedError(f"CSV missing columns: {missing}") - for row_number, source_row in enumerate(reader, start=2): - row = {str(k).strip().lower(): v for k, v in source_row.items()} - try: - day = date.fromisoformat(str(row["date"]).strip()) - except (KeyError, ValueError) as exc: - raise BlockedError(f"row {row_number}: date must be YYYY-MM-DD") from exc - values = {name: _number(row, name, row_number) for name in required - {"date"}} - if min(values["open"], values["high"], values["low"], values["close"], values["adjusted_close"]) <= 0: - raise BlockedError(f"row {row_number}: prices must be positive") - if values["volume"] < 0: - raise BlockedError(f"row {row_number}: volume must be non-negative") - if values["high"] < max(values["open"], values["low"], values["close"]): - raise BlockedError(f"row {row_number}: high is inconsistent") - if values["low"] > min(values["open"], values["high"], values["close"]): - raise BlockedError(f"row {row_number}: low is inconsistent") - rows.append(RawBar(day=day, **values)) - rows.sort(key=lambda item: item.day) - if len({item.day for item in rows}) != len(rows): - raise BlockedError("CSV contains duplicate dates") - if not rows: - raise BlockedError("CSV contains no observations") - return rows - - -def normalize_to_latest_scale(rows: list[RawBar]) -> list[Bar]: - latest_factor = rows[-1].adjusted_close / rows[-1].close - if latest_factor <= 0: - raise BlockedError("invalid latest adjustment factor") - normalized: list[Bar] = [] - for row in rows: - factor = (row.adjusted_close / row.close) / latest_factor - normalized.append( - Bar( - day=row.day, - open=row.open * factor, - high=row.high * factor, - low=row.low * factor, - close=row.close * factor, - volume=row.volume, - ) - ) - return normalized - - -def cutoff(rows: list[RawBar], analysis_date: Optional[date]) -> list[RawBar]: - selected = [row for row in rows if analysis_date is None or row.day <= analysis_date] - if not selected: - raise BlockedError("no observations on or before analysis date") - if analysis_date is not None and selected[-1].day != analysis_date: - raise BlockedError("analysis date is not present in both completed-session files") - return selected - - -def align(ticker_rows: list[RawBar], benchmark_rows: list[RawBar], analysis_date: Optional[date]) -> tuple[list[Bar], list[Bar]]: - ticker_cut = cutoff(ticker_rows, analysis_date) - benchmark_cut = cutoff(benchmark_rows, analysis_date) - ticker_map = {row.day: row for row in ticker_cut} - benchmark_map = {row.day: row for row in benchmark_cut} - days = sorted(set(ticker_map) & set(benchmark_map)) - if not days: - raise BlockedError("ticker and benchmark have no shared dates") - target = analysis_date or min(ticker_cut[-1].day, benchmark_cut[-1].day) - days = [day for day in days if day <= target] - if not days or days[-1] != target: - raise BlockedError("latest requested date is not shared by ticker and benchmark") - ticker_aligned = normalize_to_latest_scale([ticker_map[day] for day in days]) - benchmark_aligned = normalize_to_latest_scale([benchmark_map[day] for day in days]) - if len(days) < MIN_SHARED_SESSIONS: - raise BlockedError(f"need at least {MIN_SHARED_SESSIONS} shared sessions; found {len(days)}") - return ticker_aligned, benchmark_aligned - - -def sma(values: list[float], period: int) -> list[Optional[float]]: - output: list[Optional[float]] = [None] * len(values) - running = 0.0 - for index, value in enumerate(values): - running += value - if index >= period: - running -= values[index - period] - if index >= period - 1: - output[index] = running / period - return output - - -def returns(values: list[float], period: int) -> list[Optional[float]]: - output: list[Optional[float]] = [None] * len(values) - for index in range(period, len(values)): - output[index] = values[index] / values[index - period] - 1.0 - return output - - -def rsi(values: list[float], period: int = 14) -> list[Optional[float]]: - output: list[Optional[float]] = [None] * len(values) - if len(values) <= period: - return output - gains = [0.0] * len(values) - losses = [0.0] * len(values) - for index in range(1, len(values)): - delta = values[index] - values[index - 1] - gains[index] = max(delta, 0.0) - losses[index] = max(-delta, 0.0) - avg_gain = sum(gains[1 : period + 1]) / period - avg_loss = sum(losses[1 : period + 1]) / period - for index in range(period, len(values)): - if index > period: - avg_gain = ((period - 1) * avg_gain + gains[index]) / period - avg_loss = ((period - 1) * avg_loss + losses[index]) / period - output[index] = 50.0 if avg_gain == avg_loss == 0 else (100.0 if avg_loss == 0 else 100.0 - 100.0 / (1.0 + avg_gain / avg_loss)) - return output - - -def atr_adx(bars: list[Bar], period: int = 14) -> tuple[list[Optional[float]], list[Optional[float]], list[Optional[float]], list[Optional[float]]]: - length = len(bars) - tr = [0.0] * length - plus_dm = [0.0] * length - minus_dm = [0.0] * length - for index, bar in enumerate(bars): - if index == 0: - tr[index] = bar.high - bar.low - continue - tr[index] = max(bar.high - bar.low, abs(bar.high - bars[index - 1].close), abs(bar.low - bars[index - 1].close)) - up = bar.high - bars[index - 1].high - down = bars[index - 1].low - bar.low - plus_dm[index] = up if up > down and up > 0 else 0.0 - minus_dm[index] = down if down > up and down > 0 else 0.0 - - atr_values: list[Optional[float]] = [None] * length - plus_di: list[Optional[float]] = [None] * length - minus_di: list[Optional[float]] = [None] * length - adx: list[Optional[float]] = [None] * length - if length <= period: - return atr_values, adx, plus_di, minus_di - - atr_value = sum(tr[1 : period + 1]) / period - plus_value = sum(plus_dm[1 : period + 1]) / period - minus_value = sum(minus_dm[1 : period + 1]) / period - dx: list[Optional[float]] = [None] * length - for index in range(period, length): - if index > period: - atr_value = ((period - 1) * atr_value + tr[index]) / period - plus_value = ((period - 1) * plus_value + plus_dm[index]) / period - minus_value = ((period - 1) * minus_value + minus_dm[index]) / period - atr_values[index] = atr_value - if atr_value > 0: - plus_di[index] = 100.0 * plus_value / atr_value - minus_di[index] = 100.0 * minus_value / atr_value - denominator = plus_di[index] + minus_di[index] - dx[index] = 0.0 if denominator == 0 else 100.0 * abs(plus_di[index] - minus_di[index]) / denominator - else: - # A newly listed or previously suspended instrument can begin with - # a flat true-range seed. Wilder's directional movement is zero - # in that case; leaving DX unavailable would prevent ADX from ever - # initializing, even after the instrument starts trading. - plus_di[index] = 0.0 - minus_di[index] = 0.0 - dx[index] = 0.0 - - first_adx = 2 * period - 1 - initial_dx = [value for value in dx[period : first_adx + 1] if value is not None] - if len(initial_dx) == period: - adx_value = sum(initial_dx) / period - adx[first_adx] = adx_value - for index in range(first_adx + 1, length): - if dx[index] is None: - continue - adx_value = ((period - 1) * adx_value + dx[index]) / period - adx[index] = adx_value - return atr_values, adx, plus_di, minus_di - - -def rolling_volatility(values: list[float], period: int = 20) -> list[Optional[float]]: - log_returns: list[Optional[float]] = [None] + [math.log(values[i] / values[i - 1]) for i in range(1, len(values))] - output: list[Optional[float]] = [None] * len(values) - for index in range(period, len(values)): - window = [value for value in log_returns[index - period + 1 : index + 1] if value is not None] - if len(window) == period: - output[index] = statistics.stdev(window) * math.sqrt(252.0) - return output - - -def rolling_max_drawdown(values: list[float], period: int = 252) -> list[Optional[float]]: - """Calculate exact sliding-window drawdown in amortized linear time.""" - if period <= 0: - raise BlockedError("maximum drawdown period must be positive") - output: list[Optional[float]] = [None] * len(values) - - # A segment stores (maximum price, minimum price, largest earlier/later - # peak-to-trough ratio). The operation is associative, so a two-stack - # aggregate queue supports an exact ordered sliding-window query. - Segment = tuple[float, float, float] - - def singleton(value: float) -> Segment: - return (value, value, 1.0) - - def combine(left: Segment, right: Segment) -> Segment: - return ( - max(left[0], right[0]), - min(left[1], right[1]), - max(left[2], right[2], left[0] / right[1]), - ) - - incoming: list[tuple[Segment, Segment]] = [] - outgoing: list[tuple[Segment, Segment]] = [] - - def push(value: float) -> None: - item = singleton(value) - aggregate = item if not incoming else combine(incoming[-1][1], item) - incoming.append((item, aggregate)) - - def pop_oldest() -> None: - if not outgoing: - while incoming: - item, _ = incoming.pop() - aggregate = ( - item - if not outgoing - else combine(item, outgoing[-1][1]) - ) - outgoing.append((item, aggregate)) - outgoing.pop() - - def aggregate() -> Segment: - if outgoing and incoming: - return combine(outgoing[-1][1], incoming[-1][1]) - if outgoing: - return outgoing[-1][1] - if incoming: - return incoming[-1][1] - raise BlockedError("cannot aggregate an empty drawdown window") - - window_size = 0 - for index, value in enumerate(values): - push(value) - window_size += 1 - if window_size > period: - pop_oldest() - window_size -= 1 - if window_size == period: - maximum_ratio = aggregate()[2] - output[index] = 1.0 - 1.0 / maximum_ratio - return output - - -def quantile(values: Iterable[float], probability: float) -> float: - ordered = sorted(values) - if not ordered: - raise BlockedError("cannot calculate quantile from empty data") - location = (len(ordered) - 1) * probability - lower = math.floor(location) - upper = math.ceil(location) - if lower == upper: - return ordered[lower] - return ordered[lower] * (upper - location) + ordered[upper] * (location - lower) - - -def rolling_gap95(bars: list[Bar], period: int = 60) -> list[Optional[float]]: - gaps: list[Optional[float]] = [None] + [abs(bars[i].open / bars[i - 1].close - 1.0) for i in range(1, len(bars))] - output: list[Optional[float]] = [None] * len(bars) - for index in range(period, len(bars)): - window = [value for value in gaps[index - period + 1 : index + 1] if value is not None] - if len(window) == period: - output[index] = quantile(window, 0.95) - return output - - -def rolling_signed_volume(bars: list[Bar], period: int = 20) -> list[Optional[float]]: - output: list[Optional[float]] = [None] * len(bars) - for index in range(period - 1, len(bars)): - median_volume = statistics.median(bar.volume for bar in bars[index - period + 1 : index + 1]) - if index == 0 or median_volume <= 0 or bars[index].volume <= 0: - continue - direction = 1.0 if bars[index].close > bars[index - 1].close else (-1.0 if bars[index].close < bars[index - 1].close else 0.0) - output[index] = direction * math.log(bars[index].volume / median_volume) - return output - - -def ratio_series(left: list[Optional[float]], right: list[Optional[float]]) -> list[Optional[float]]: - output: list[Optional[float]] = [None] * len(left) - for index, (a, b) in enumerate(zip(left, right)): - if a is not None and b not in (None, 0): - output[index] = a / b - 1.0 - return output - - -def close_over(values: list[float], average: list[Optional[float]]) -> list[Optional[float]]: - return [None if avg in (None, 0) else value / avg - 1.0 for value, avg in zip(values, average)] - - -def relative_returns(ticker: list[float], benchmark: list[float], period: int) -> list[Optional[float]]: - output: list[Optional[float]] = [None] * len(ticker) - for index in range(period, len(ticker)): - output[index] = (ticker[index] / ticker[index - period] - 1.0) - (benchmark[index] / benchmark[index - period] - 1.0) - return output - - -def percentile_rank(series: list[Optional[float]], index: int) -> tuple[float, int]: - current = series[index] - if current is None or not math.isfinite(current): - raise BlockedError("current feature value is unavailable") - start = max(0, index - PERCENTILE_LOOKBACK) - reference = [value for value in series[start:index] if value is not None and math.isfinite(value)] - if len(reference) < MIN_REFERENCE_OBSERVATIONS: - raise BlockedError(f"feature has only {len(reference)} prior observations; need {MIN_REFERENCE_OBSERVATIONS}") - less = sum(value < current for value in reference) - equal = sum(math.isclose(value, current, rel_tol=1e-12, abs_tol=1e-15) for value in reference) - return 100.0 * (less + 0.5 * equal) / len(reference), len(reference) - - -def weighted_score(percentiles: dict[str, float], weights: dict[str, float]) -> float: - return sum(percentiles[name] * weight for name, weight in weights.items()) - - -def opinion(score: float) -> str: - if score >= 80: - return "강한 긍정" - if score >= 60: - return "긍정" - if score >= 40: - return "중립" - if score >= 20: - return "부정" - return "강한 부정" - - -def decimals_for_tick(tick: float) -> int: - return max(0, -Decimal(str(tick)).normalize().as_tuple().exponent) - - -def round_up_tick(value: float, tick: float) -> float: - return math.ceil((value - 1e-12) / tick) * tick - - -def round_down_tick(value: float, tick: float) -> float: - return math.floor((value + 1e-12) / tick) * tick - - -def calculate(ticker_bars: list[Bar], benchmark_bars: list[Bar], market: str, ticker: str, tick_size: float, source_name: str) -> dict[str, Any]: - if tick_size <= 0 or not math.isfinite(tick_size): - raise BlockedError("tick size must be a positive finite number") - closes = [bar.close for bar in ticker_bars] - benchmark_closes = [bar.close for bar in benchmark_bars] - index = len(ticker_bars) - 1 - - sma5 = sma(closes, 5) - sma20 = sma(closes, 20) - sma50 = sma(closes, 50) - sma200 = sma(closes, 200) - ret5 = returns(closes, 5) - ret20 = returns(closes, 20) - ret63 = returns(closes, 63) - ret126 = returns(closes, 126) - ret252 = returns(closes, 252) - rsi14 = rsi(closes, 14) - atr14, adx14, plus_di14, minus_di14 = atr_adx(ticker_bars, 14) - signed_volume20 = rolling_signed_volume(ticker_bars, 20) - vol20 = rolling_volatility(closes, 20) - max_drawdown252 = rolling_max_drawdown(closes, 252) - gap95_60 = rolling_gap95(ticker_bars, 60) - - close_sma20 = close_over(closes, sma20) - close_sma50 = close_over(closes, sma50) - close_sma200 = close_over(closes, sma200) - sma5_sma20 = ratio_series(sma5, sma20) - sma20_sma50 = ratio_series(sma20, sma50) - sma50_sma200 = ratio_series(sma50, sma200) - relative63 = relative_returns(closes, benchmark_closes, 63) - relative252 = relative_returns(closes, benchmark_closes, 252) - atr_pct = [None if atr is None else atr / closes[i] for i, atr in enumerate(atr14)] - adx_direction: list[Optional[float]] = [None] * len(closes) - for i in range(len(closes)): - if adx14[i] is not None and plus_di14[i] is not None and minus_di14[i] is not None: - adx_direction[i] = (plus_di14[i] - minus_di14[i]) * min(adx14[i] / 25.0, 1.0) - - features = { - "return_5": ret5, - "return_20": ret20, - "close_over_sma_20": close_sma20, - "sma_5_over_sma_20": sma5_sma20, - "signed_log_volume_ratio_20": signed_volume20, - "rsi_14": rsi14, - "return_63": ret63, - "close_over_sma_50": close_sma50, - "sma_20_over_sma_50": sma20_sma50, - "relative_return_63": relative63, - "adx_direction_14": adx_direction, - "return_126": ret126, - "return_252": ret252, - "close_over_sma_200": close_sma200, - "sma_50_over_sma_200": sma50_sma200, - "relative_return_252": relative252, - } - risk_features = { - "ATR14/close": atr_pct, - "20d annualized volatility": vol20, - "252d maximum drawdown": max_drawdown252, - "60d gap p95": gap95_60, - } - percentiles: dict[str, float] = {} - reference_counts: dict[str, int] = {} - for name, series in {**features, **risk_features}.items(): - try: - percentiles[name], reference_counts[name] = percentile_rank(series, index) - except BlockedError as exc: - raise BlockedError(f"{name}: {exc}") from exc - - short_weights = { - "return_5": 0.25, - "return_20": 0.20, - "close_over_sma_20": 0.20, - "sma_5_over_sma_20": 0.15, - "signed_log_volume_ratio_20": 0.15, - "rsi_14": 0.05, - } - medium_weights = { - "return_63": 0.25, - "close_over_sma_50": 0.20, - "sma_20_over_sma_50": 0.20, - "relative_return_63": 0.20, - "adx_direction_14": 0.15, - } - long_weights = { - "return_126": 0.25, - "return_252": 0.25, - "close_over_sma_200": 0.20, - "sma_50_over_sma_200": 0.15, - "relative_return_252": 0.15, - } - risk_weights = { - "ATR14/close": 0.30, - "20d annualized volatility": 0.25, - "252d maximum drawdown": 0.25, - "60d gap p95": 0.20, - } - short_score = weighted_score(percentiles, short_weights) - medium_score = weighted_score(percentiles, medium_weights) - long_score = weighted_score(percentiles, long_weights) - risk_score = weighted_score(percentiles, risk_weights) - total_score = min(100.0, max(0.0, 0.25 * short_score + 0.35 * medium_score + 0.40 * long_score - 0.20 * max(risk_score - 50.0, 0.0))) - - prior_high = max(bar.high for bar in ticker_bars[-21:-1]) - current_close = closes[-1] - current_atr = atr14[-1] - if current_atr is None: - raise BlockedError("ATR14 is unavailable") - entry = round_up_tick(max(current_close, prior_high + tick_size), tick_size) - stop = round_down_tick(entry - 2.0 * current_atr, tick_size) - if stop <= 0: - raise BlockedError("calculated stop is non-positive") - take_profit = round_down_tick(entry + 2.0 * (entry - stop), tick_size) - decimals = decimals_for_tick(tick_size) - - dominant_risk = max(risk_features, key=lambda name: percentiles[name]) - raw_risk = risk_features[dominant_risk][-1] - if raw_risk is None: - raise BlockedError("dominant risk value is unavailable") - risk_value = f"{100.0 * raw_risk:.2f}%" - stop_distance = 100.0 * (entry - stop) / entry - risk_line = f"{dominant_risk} {risk_value} (과거 백분위 {percentiles[dominant_risk]:.1f}), 진입-손절 거리 {stop_distance:.2f}%" - - result: dict[str, Any] = { - "source_skill": "quant-stock-technical", - "result_schema": "quant-stock-technical/v1", - "calculation_status": "READY", - "setup_status": "READY" if total_score >= 60.0 else "CONDITIONAL", - "method_version": METHOD_VERSION, - "analysis_date": ticker_bars[-1].day.isoformat(), - "market": market, - "ticker": ticker, - "source_name": source_name, - "shared_sessions": len(ticker_bars), - "score_basis": "ticker-relative historical percentile; not probability of profit", - "short": {"opinion": opinion(short_score), "score": round(short_score, 2)}, - "medium": {"opinion": opinion(medium_score), "score": round(medium_score, 2)}, - "long": {"opinion": opinion(long_score), "score": round(long_score, 2)}, - "risk": {"score": round(risk_score, 2), "counterpoint": risk_line}, - "entry_price": round(entry, decimals), - "stop_price": round(stop, decimals), - "take_profit_price": round(take_profit, decimals), - "total_score": round(total_score, 2), - "reference_observations": {name: reference_counts[name] for name in sorted(reference_counts)}, - "assumptions": [ - "input rows are finalized completed daily sessions", - "prices are corporate-action adjusted and aligned to analysis-date raw price scale", - "fees, tax, FX, slippage, position size, and execution are excluded", - ], - } - return result - - -def render_markdown(result: dict[str, Any]) -> str: - if result.get("calculation_status") != "READY": - return f"**계산 상태:** BLOCKED\n\n**사유:** {result.get('reason', 'unknown')}" - return "\n".join( - [ - f"## {result['ticker']} quantitative technical analysis", - "", - f"- 분석 기준일: {result['analysis_date']}", - f"- 시장: {result['market']}", - f"- 티커: {result['ticker']}", - f"- 계산 상태: {result['calculation_status']}", - f"- 진입 상태: {result['setup_status']}", - f"- 방법론: {result['method_version']}", - "", - "| 항목 | 의견 | 점수 |", - "|---|---|---:|", - f"| 단기 | {result['short']['opinion']} | {result['short']['score']:.2f} |", - f"| 중기 | {result['medium']['opinion']} | {result['medium']['score']:.2f} |", - f"| 장기 | {result['long']['opinion']} | {result['long']['score']:.2f} |", - f"| 종합 | {opinion(result['total_score'])} | {result['total_score']:.2f} |", - "", - f"- 리스크/반박: {result['risk']['counterpoint']}", - f"- 진입가: {result['entry_price']}", - f"- 손절가: {result['stop_price']}", - f"- 익절가: {result['take_profit_price']}", - f"- 점수 기준: {result['score_basis']}", - f"- 데이터: {result['source_name']} ({result['shared_sessions']} shared sessions)", - ] - ) - - -def synthetic_bars(length: int, drift: float, phase: float) -> list[Bar]: - output: list[Bar] = [] - day = date(2021, 1, 1) - price = 100.0 - index = 0 - while len(output) < length: - if day.weekday() < 5: - cycle = 0.004 * math.sin(index / 17.0 + phase) - change = drift + cycle - previous = price - price = max(1.0, price * (1.0 + change)) - opening = previous * (1.0 + 0.001 * math.sin(index / 9.0 + phase)) - high = max(opening, price) * 1.01 - low = min(opening, price) * 0.99 - volume = 1_000_000.0 * (1.0 + 0.2 * math.sin(index / 13.0 + phase)) - output.append(Bar(day, opening, high, low, price, volume)) - index += 1 - day += timedelta(days=1) - return output - - -def self_test() -> None: - ticker_bars = synthetic_bars(1100, 0.0006, 0.0) - benchmark_bars = synthetic_bars(1100, 0.0003, 1.0) - flat_then_active = [ - Bar( - day=date(2020, 1, 1) + timedelta(days=index), - open=100.0 if index < 20 else 100.0 + index - 20, - high=100.0 if index < 20 else 101.0 + index - 20, - low=100.0 if index < 20 else 99.0 + index - 20, - close=100.0 if index < 20 else 100.5 + index - 20, - volume=0.0 if index < 20 else 1_000.0, - ) - for index in range(80) - ] - _, restarted_adx, _, _ = atr_adx(flat_then_active, 14) - assert restarted_adx[-1] is not None - assert math.isfinite(restarted_adx[-1]) - with tempfile.TemporaryDirectory(prefix="qta-self-test-") as directory: - paths = [Path(directory) / "ticker.csv", Path(directory) / "benchmark.csv"] - for path, bars in zip(paths, (ticker_bars, benchmark_bars)): - with path.open("w", encoding="utf-8", newline="") as handle: - writer = csv.DictWriter(handle, fieldnames=["date", "open", "high", "low", "close", "adjusted_close", "volume"]) - writer.writeheader() - for bar in bars: - writer.writerow( - { - "date": bar.day.isoformat(), - "open": bar.open, - "high": bar.high, - "low": bar.low, - "close": bar.close, - "adjusted_close": bar.close, - "volume": bar.volume, - } - ) - parsed_ticker, parsed_benchmark = align(read_csv(str(paths[0])), read_csv(str(paths[1])), ticker_bars[-1].day) - result = calculate(parsed_ticker, parsed_benchmark, "TEST", "SYNTH", 0.01, "deterministic-self-test") - assert result["calculation_status"] == "READY" - assert result["source_skill"] == "quant-stock-technical" - assert result["result_schema"] == "quant-stock-technical/v1" - assert 0 <= result["total_score"] <= 100 - assert result["stop_price"] < result["entry_price"] < result["take_profit_price"] - print(json.dumps({"self_test": "PASS", "method_version": METHOD_VERSION, "total_score": result["total_score"]}, ensure_ascii=False)) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--ticker-csv") - parser.add_argument("--benchmark-csv") - parser.add_argument("--market") - parser.add_argument("--ticker") - parser.add_argument("--tick-size", type=float) - parser.add_argument("--source-name", default="user-supplied-eod-csv") - parser.add_argument("--analysis-date") - parser.add_argument("--format", choices=("json", "markdown"), default="json") - parser.add_argument("--self-test", action="store_true") - return parser.parse_args() - - -def main() -> int: - args = parse_args() - if args.self_test: - self_test() - return 0 - missing = [name for name in ("ticker_csv", "benchmark_csv", "market", "ticker", "tick_size") if getattr(args, name) is None] - if missing: - result = {"source_skill": "quant-stock-technical", "result_schema": "quant-stock-technical/v1", "calculation_status": "BLOCKED", "reason": "missing arguments: " + ", ".join(missing), "method_version": METHOD_VERSION} - print(json.dumps(result, ensure_ascii=False, indent=2)) - return 2 - try: - requested_date = date.fromisoformat(args.analysis_date) if args.analysis_date else None - ticker_rows, benchmark_rows = align(read_csv(args.ticker_csv), read_csv(args.benchmark_csv), requested_date) - result = calculate(ticker_rows, benchmark_rows, args.market, args.ticker, args.tick_size, args.source_name) - except (BlockedError, OSError, ValueError) as exc: - result = {"source_skill": "quant-stock-technical", "result_schema": "quant-stock-technical/v1", "calculation_status": "BLOCKED", "reason": str(exc), "method_version": METHOD_VERSION} - print(render_markdown(result) if args.format == "markdown" else json.dumps(result, ensure_ascii=False, indent=2)) - return 2 - print(render_markdown(result) if args.format == "markdown" else json.dumps(result, ensure_ascii=False, indent=2)) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/skills/quant-stock-technical/scripts/analyze_stock_v2.py b/skills/quant-stock-technical/scripts/analyze_stock_v2.py deleted file mode 100644 index 11c92fb..0000000 --- a/skills/quant-stock-technical/scripts/analyze_stock_v2.py +++ /dev/null @@ -1,221 +0,0 @@ -#!/usr/bin/env python3 -"""First-hour-aligned QTA 2.0 daily setup score.""" - -from __future__ import annotations - -import argparse -import json -import statistics -import sys -from datetime import date -from pathlib import Path -from typing import Any - -import analyze_stock as qta1 - - -METHOD_VERSION = "qta-2.0.0" -VALIDATION_STATUS = "RESEARCH_ONLY" -LIQUIDITY_FLOOR = { - "KR": 1_000_000_000.0, - "US": 1_000_000.0, -} -MARKET_REGIME_POLICY = { - "metric": "same_session_previous_close_return_bps", - "minimum_change_bps": 0.0, - "max_age_seconds": 90, - "status": "REQUIRED", -} - - -def calculate( - ticker_bars: list[qta1.Bar], - benchmark_bars: list[qta1.Bar], - market: str, - ticker: str, - tick_size: float, - source_name: str, -) -> dict[str, Any]: - """Reuse the causal QTA1 features but score the first-hour horizon.""" - market = market.upper() - if market not in LIQUIDITY_FLOOR: - raise qta1.BlockedError("QTA 2.0 market must be KR or US") - result = qta1.calculate( - ticker_bars, - benchmark_bars, - market, - ticker, - tick_size, - source_name, - ) - short = float(result["short"]["score"]) - medium = float(result["medium"]["score"]) - risk = float(result["risk"]["score"]) - total = min( - 100.0, - max( - 0.0, - 0.50 * short - + 0.30 * medium - + 0.20 * (100.0 - risk), - ), - ) - median_turnover = statistics.median( - bar.close * bar.volume for bar in ticker_bars[-20:] - ) - minimum_turnover = LIQUIDITY_FLOOR[market] - liquidity_ready = median_turnover >= minimum_turnover - result.update( - { - "method_version": METHOD_VERSION, - "validation_status": VALIDATION_STATUS, - "score_basis": ( - "ticker-relative historical percentiles aligned to first-hour " - "continuation; not probability of profit" - ), - "liquidity": { - "median_20_session_turnover": round(median_turnover, 2), - "minimum_turnover": minimum_turnover, - "currency": "KRW" if market == "KR" else "USD", - "status": "READY" if liquidity_ready else "BLOCKED", - }, - "market_regime": dict(MARKET_REGIME_POLICY), - "total_score": round(total, 2), - "setup_status": ( - "READY" - if total >= 60.0 and liquidity_ready - else "CONDITIONAL" - ), - "assumptions": [ - *result["assumptions"], - "QTA 2.0 is research-only until multi-session walk-forward validation", - "same-session benchmark regime admission is required downstream", - ], - } - ) - return result - - -def self_test() -> None: - ticker = qta1.synthetic_bars(1100, 0.0006, 0.0) - benchmark = qta1.synthetic_bars(1100, 0.0003, 1.0) - result = calculate( - ticker, - benchmark, - "US", - "SYNTH", - 0.01, - "deterministic-self-test", - ) - expected = min( - 100.0, - max( - 0.0, - 0.50 * result["short"]["score"] - + 0.30 * result["medium"]["score"] - + 0.20 * (100.0 - result["risk"]["score"]), - ), - ) - assert abs(result["total_score"] - round(expected, 2)) <= 0.01 - assert result["method_version"] == METHOD_VERSION - assert result["validation_status"] == VALIDATION_STATUS - print( - json.dumps( - { - "self_test": "PASS", - "method_version": METHOD_VERSION, - "validation_status": VALIDATION_STATUS, - "total_score": result["total_score"], - }, - sort_keys=True, - ) - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--ticker-csv") - parser.add_argument("--benchmark-csv") - parser.add_argument("--market") - parser.add_argument("--ticker") - parser.add_argument("--tick-size", type=float) - parser.add_argument("--source-name", default="user-supplied-eod-csv") - parser.add_argument("--analysis-date") - parser.add_argument("--self-test", action="store_true") - return parser.parse_args() - - -def blocked(reason: str, market: Any, ticker: Any) -> dict[str, Any]: - return { - "source_skill": "quant-stock-technical", - "result_schema": "quant-stock-technical/v1", - "calculation_status": "BLOCKED", - "reason": reason, - "method_version": METHOD_VERSION, - "market": str(market or "").upper(), - "ticker": str(ticker or "").strip().upper(), - } - - -def main() -> int: - args = parse_args() - if args.self_test: - self_test() - return 0 - missing = [ - name - for name in ( - "ticker_csv", - "benchmark_csv", - "market", - "ticker", - "tick_size", - ) - if getattr(args, name) is None - ] - if missing: - print( - json.dumps( - blocked( - "missing arguments: " + ", ".join(missing), - args.market, - args.ticker, - ), - indent=2, - ) - ) - return 2 - try: - requested = ( - date.fromisoformat(args.analysis_date) - if args.analysis_date - else None - ) - ticker_rows, benchmark_rows = qta1.align( - qta1.read_csv(str(Path(args.ticker_csv))), - qta1.read_csv(str(Path(args.benchmark_csv))), - requested, - ) - result = calculate( - ticker_rows, - benchmark_rows, - args.market, - args.ticker, - args.tick_size, - args.source_name, - ) - except (qta1.BlockedError, OSError, ValueError) as exc: - print( - json.dumps( - blocked(str(exc), args.market, args.ticker), - ensure_ascii=False, - indent=2, - ) - ) - return 2 - print(json.dumps(result, ensure_ascii=False, indent=2)) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/skills/quant-stock-technical/scripts/build_universe_manifest.py b/skills/quant-stock-technical/scripts/build_universe_manifest.py deleted file mode 100755 index 7cd7ce7..0000000 --- a/skills/quant-stock-technical/scripts/build_universe_manifest.py +++ /dev/null @@ -1,2194 +0,0 @@ -#!/usr/bin/env python3 -"""Build a deterministic market-scoped qta-universe-manifest/v2 offline.""" - -# Python 3.9 compatibility intentionally uses Optional instead of PEP 604 unions. -# ruff: noqa: UP045 - -from __future__ import annotations - -import argparse -import csv -import hashlib -import json -import re -import sys -import tempfile -from collections.abc import Iterable, Mapping, Sequence -from dataclasses import dataclass -from datetime import date -from decimal import Decimal, InvalidOperation -from html.parser import HTMLParser -from pathlib import Path -from typing import Any, Optional - -MANIFEST_SCHEMA = "qta-universe-manifest/v2" -BUILD_SPEC_SCHEMA = "qta-universe-build-spec/v1" -TICK_CONTRACT_SCHEMA = "qta-tick-contract/v1" -TICK_CONTRACT_KIND = "RESOLVED_PRICE_LADDER" -CATALOG_COVERAGE_SCHEMA = "qta-catalog-coverage-contract/v1" - -EXCHANGES = ("KOSPI", "KOSDAQ", "NYSE", "NASDAQ") -SUPPORTED_EXCHANGE_SCOPES = { - frozenset(("KOSPI", "KOSDAQ")), - frozenset(("NYSE", "NASDAQ")), - frozenset(EXCHANGES), -} -EXCHANGE_CONTRACTS: dict[str, dict[str, str]] = { - "KOSPI": { - "market": "KR", - "benchmark_id": "KOSPI_COMPOSITE", - "currency": "KRW", - "venue": "KRX", - }, - "KOSDAQ": { - "market": "KR", - "benchmark_id": "KOSDAQ_COMPOSITE", - "currency": "KRW", - "venue": "KRX", - }, - "NYSE": { - "market": "US", - "benchmark_id": "NYSE_COMPOSITE", - "currency": "USD", - "venue": "NYSE", - }, - "NASDAQ": { - "market": "US", - "benchmark_id": "NASDAQ_COMPOSITE", - "currency": "USD", - "venue": "NASD", - }, -} - -INCLUDED_INSTRUMENT_TYPES = {"COMMON", "ADR", "REIT"} -EXCLUDED_CATALOG_TYPES = { - "ETF": "instrument_etf", - "ETN": "instrument_etn", - "PREFERRED": "instrument_preferred", - "SPAC": "instrument_spac", - "UNIT": "instrument_unit", - "RIGHT": "instrument_right", - "WARRANT": "instrument_warrant", - "TEST": "test_issue", - "ABNORMAL": "abnormal_status", -} -CATALOG_TYPES = INCLUDED_INSTRUMENT_TYPES | set(EXCLUDED_CATALOG_TYPES) - -BUILD_SPEC_FIELDS = { - "schema", - "as_of", - "analysis_date", - "official_sources", - "broker_sources", - "eod_catalog", - "catalog_coverage_contract", -} -CATALOG_COVERAGE_FIELDS = { - "schema", - "minimum_ratio_by_exchange", - "minimum_screenable_ratio_by_exchange", -} -SOURCE_FIELDS = { - "source_id", - "provider", - "exchange", - "as_of", - "path", - "format", - "encoding", - "delimiter", - "skip_rows", - "columns", - "normal_status_values", -} -EOD_CATALOG_SPEC_FIELDS = { - "source_id", - "provider", - "as_of", - "path", - "encoding", -} -EOD_CATALOG_FIELDS = ( - "exchange", - "canonical_symbol", - "data_symbol", - "broker_symbol", - "instrument_type", - "benchmark_id", - "ticker_csv", - "benchmark_csv", - "tick_rule_id", - "tick_effective_date", - "tick_reference_price", - "resolved_tick_size", - "source_name", -) -LOGICAL_COLUMNS = { - "symbol", - "name", - "instrument_type", - "status", - "etf", - "etn", - "preferred", - "spac", - "unit", - "right", - "warrant", - "test_issue", - "abnormal", - "etp", - "trading_halt", - "liquidation", - "administrative", -} -FORMATS = { - "KRX_CSV", - "KRX_KIND_HTML", - "KIS_CSV", - "KIS_FIXED_WIDTH", - "KIS_KRX_MASTER", - "KIS_OVERSEAS_MASTER", - "NASDAQ_LISTED", - "NASDAQ_OTHER", -} -TRUE_VALUES = {"1", "T", "TRUE", "Y", "YES"} -FALSE_VALUES = {"", "0", "F", "FALSE", "N", "NO"} -HEX64 = re.compile(r"^[0-9a-f]{64}$") - -# KIS official stocks_info master layouts. The downloaded domestic rows contain -# a variable-width Korean name prefix followed by these fixed ASCII tails. -KIS_KOSPI_WIDTHS = ( - 2, - 1, - 4, - 4, - 4, - *([1] * 26), - 9, - 5, - 5, - 1, - 1, - 1, - 2, - 1, - 1, - 1, - 2, - 2, - 2, - 3, - 1, - 3, - 12, - 12, - 8, - 15, - 21, - 2, - 7, - 1, - 1, - 1, - 1, - 1, - 9, - 9, - 9, - 5, - 9, - 8, - 9, - 3, - 1, - 1, - 1, -) -KIS_KOSDAQ_WIDTHS = ( - 2, - 1, - 4, - 4, - 4, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 9, - 5, - 5, - 1, - 1, - 1, - 2, - 1, - 1, - 1, - 2, - 2, - 2, - 3, - 1, - 3, - 12, - 12, - 8, - 15, - 21, - 2, - 7, - 1, - 1, - 1, - 1, - 9, - 9, - 9, - 5, - 9, - 8, - 9, - 3, - 1, - 1, - 1, -) -KIS_KRX_LAYOUTS = { - "KOSPI": { - "tail_length": 227, - "widths": KIS_KOSPI_WIDTHS, - "etp_index": 12, - "spac_index": 19, - "halt_index": 34, - "liquidation_index": 35, - "admin_index": 36, - "preferred_index": 54, - }, - "KOSDAQ": { - "tail_length": 221, - "widths": KIS_KOSDAQ_WIDTHS, - "etp_index": 8, - "spac_index": 14, - "halt_index": 29, - "liquidation_index": 30, - "admin_index": 31, - "preferred_index": 49, - }, -} - - -class UniverseBlockedError(ValueError): - """Raised when the frozen universe build contract is incomplete or invalid.""" - - -@dataclass(frozen=True) -class MasterRow: - exchange: str - symbol: str - name: str - instrument_type: str - status: str - flags: Mapping[str, str] - source_id: str - - -@dataclass(frozen=True) -class CatalogRow: - exchange: str - canonical_symbol: str - data_symbol: str - broker_symbol: str - instrument_type: str - benchmark_id: str - ticker_csv: str - benchmark_csv: str - tick_rule_id: str - tick_effective_date: str - tick_reference_price: str - resolved_tick_size: str - source_name: str - - -def canonical_json(value: Any) -> str: - return json.dumps( - value, - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - allow_nan=False, - ) - - -def sha256_json(value: Any) -> str: - return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest() - - -def sha256_file(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - while True: - chunk = handle.read(1024 * 1024) - if not chunk: - break - digest.update(chunk) - return digest.hexdigest() - - -def load_json_object(path: Path) -> dict[str, Any]: - value = json.loads(path.read_text(encoding="utf-8")) - if not isinstance(value, dict): - raise UniverseBlockedError(f"{path} must contain one JSON object") - return value - - -def require_exact_fields( - value: Mapping[str, Any], expected: Iterable[str], label: str -) -> None: - expected_set = set(expected) - actual = set(value) - if actual != expected_set: - missing = sorted(expected_set - actual) - extra = sorted(actual - expected_set) - raise UniverseBlockedError( - f"{label} fields mismatch; missing={missing}, extra={extra}" - ) - - -def require_iso_date(value: Any, label: str) -> str: - if not isinstance(value, str): - raise UniverseBlockedError(f"{label} must be a YYYY-MM-DD string") - rendered = value - try: - date.fromisoformat(rendered) - except ValueError as exc: - raise UniverseBlockedError(f"{label} must be YYYY-MM-DD") from exc - return rendered - - -def require_nonempty(value: Any, label: str) -> str: - if not isinstance(value, str): - raise UniverseBlockedError(f"{label} must be a non-empty string") - rendered = value.strip() - if not rendered: - raise UniverseBlockedError(f"{label} must be a non-empty string") - return rendered - - -def normalized_symbol(value: Any, label: str) -> str: - return require_nonempty(value, label).upper() - - -def decimal_string(value: Any, label: str, positive: bool = False) -> str: - if isinstance(value, bool): - raise UniverseBlockedError(f"{label} must be a decimal string or number") - try: - parsed = Decimal(str(value)) - except (InvalidOperation, ValueError) as exc: - raise UniverseBlockedError( - f"{label} must be a decimal string or number" - ) from exc - if not parsed.is_finite(): - raise UniverseBlockedError(f"{label} must be finite") - if positive and parsed <= 0: - raise UniverseBlockedError(f"{label} must be positive") - return format(parsed, "f") - - -def canonical_ratio_string(value: Any, label: str) -> str: - if isinstance(value, bool): - raise UniverseBlockedError(f"{label} must be a decimal string or number") - try: - parsed = Decimal(str(value)) - except (InvalidOperation, ValueError) as exc: - raise UniverseBlockedError( - f"{label} must be a decimal string or number" - ) from exc - if not parsed.is_finite() or parsed <= 0 or parsed > 1: - raise UniverseBlockedError(f"{label} must be greater than 0 and at most 1") - rendered = format(parsed.normalize(), "f") - return rendered.rstrip("0").rstrip(".") if "." in rendered else rendered - - -def meets_minimum_coverage_ratio( - numerator: int, - official_rows: int, - minimum: str, -) -> bool: - if official_rows <= 0: - return False - return Decimal(numerator) / Decimal(official_rows) >= Decimal(minimum) - - -def normalize_catalog_coverage_contract(raw: Any) -> dict[str, Any]: - label = "catalog_coverage_contract" - if not isinstance(raw, dict): - raise UniverseBlockedError(f"{label} must be an object") - require_exact_fields(raw, CATALOG_COVERAGE_FIELDS, label) - if raw["schema"] != CATALOG_COVERAGE_SCHEMA: - raise UniverseBlockedError(f"{label}.schema must be {CATALOG_COVERAGE_SCHEMA}") - normalized_ratios: dict[str, dict[str, str]] = {} - for field in ( - "minimum_ratio_by_exchange", - "minimum_screenable_ratio_by_exchange", - ): - ratios = raw[field] - if not isinstance(ratios, dict) or set(ratios) != set(EXCHANGES): - raise UniverseBlockedError( - f"{label}.{field} must contain exactly {list(EXCHANGES)}" - ) - normalized_ratios[field] = { - exchange: canonical_ratio_string( - ratios[exchange], - f"{label}.{field}.{exchange}", - ) - for exchange in EXCHANGES - } - return { - "schema": CATALOG_COVERAGE_SCHEMA, - **normalized_ratios, - } - - -def resolve_input_path(raw: Any, base_directory: Path, label: str) -> Path: - rendered = require_nonempty(raw, label) - path = Path(rendered) - if not path.is_absolute(): - path = base_directory / path - path = path.resolve() - if not path.is_file(): - raise UniverseBlockedError(f"{label} does not exist or is not a file: {path}") - return path - - -def normalize_source_descriptor( - raw: Any, - role: str, - build_as_of: str, - spec_directory: Path, - index: int, -) -> dict[str, Any]: - label = f"{role.lower()}_sources[{index}]" - if not isinstance(raw, dict): - raise UniverseBlockedError(f"{label} must be an object") - require_exact_fields(raw, SOURCE_FIELDS, label) - - source_id = require_nonempty(raw["source_id"], f"{label}.source_id") - provider = require_nonempty(raw["provider"], f"{label}.provider").upper() - exchange = str(raw["exchange"]).upper() - if exchange not in EXCHANGES: - raise UniverseBlockedError(f"{label}.exchange must be one of {EXCHANGES}") - source_as_of = require_iso_date(raw["as_of"], f"{label}.as_of") - if source_as_of != build_as_of: - raise UniverseBlockedError( - f"{label}.as_of {source_as_of} does not match build as_of {build_as_of}" - ) - source_format = str(raw["format"]).upper() - if source_format not in FORMATS: - raise UniverseBlockedError(f"{label}.format must be one of {sorted(FORMATS)}") - encoding = require_nonempty(raw["encoding"], f"{label}.encoding") - delimiter = str(raw["delimiter"]) - if len(delimiter) > 1: - raise UniverseBlockedError(f"{label}.delimiter must have at most one character") - skip_rows = raw["skip_rows"] - if isinstance(skip_rows, bool) or not isinstance(skip_rows, int) or skip_rows < 0: - raise UniverseBlockedError(f"{label}.skip_rows must be an integer >= 0") - columns = raw["columns"] - if not isinstance(columns, dict): - raise UniverseBlockedError(f"{label}.columns must be an object") - unknown_columns = set(columns) - LOGICAL_COLUMNS - if unknown_columns: - raise UniverseBlockedError( - f"{label}.columns has unknown logical fields: {sorted(unknown_columns)}" - ) - statuses = raw["normal_status_values"] - if not isinstance(statuses, list) or any( - not isinstance(item, str) for item in statuses - ): - raise UniverseBlockedError( - f"{label}.normal_status_values must be a string array" - ) - normal_status_values = sorted({item.strip().upper() for item in statuses}) - - if role == "OFFICIAL_MASTER": - expected_provider = ( - "KRX" if exchange in {"KOSPI", "KOSDAQ"} else "NASDAQ_TRADER" - ) - if provider != expected_provider: - raise UniverseBlockedError( - f"{label}.provider must be {expected_provider} for {exchange}" - ) - expected_formats = ( - {"KRX_CSV", "KRX_KIND_HTML"} - if exchange in {"KOSPI", "KOSDAQ"} - else {"NASDAQ_LISTED" if exchange == "NASDAQ" else "NASDAQ_OTHER"} - ) - if source_format not in expected_formats: - raise UniverseBlockedError( - f"{label}.format {source_format} is invalid for {exchange}" - ) - elif role == "BROKER_MASTER": - if provider != "KIS": - raise UniverseBlockedError(f"{label}.provider must be KIS") - if source_format not in { - "KIS_CSV", - "KIS_FIXED_WIDTH", - "KIS_KRX_MASTER", - "KIS_OVERSEAS_MASTER", - }: - raise UniverseBlockedError( - f"{label}.format is not a supported KIS master format" - ) - if source_format == "KIS_KRX_MASTER" and exchange not in {"KOSPI", "KOSDAQ"}: - raise UniverseBlockedError( - f"{label}.format KIS_KRX_MASTER is domestic only" - ) - if source_format == "KIS_OVERSEAS_MASTER" and exchange not in { - "NYSE", - "NASDAQ", - }: - raise UniverseBlockedError( - f"{label}.format KIS_OVERSEAS_MASTER is U.S. only" - ) - else: - raise UniverseBlockedError(f"unsupported source role: {role}") - - if source_format in { - "NASDAQ_LISTED", - "NASDAQ_OTHER", - "KRX_KIND_HTML", - "KIS_KRX_MASTER", - "KIS_OVERSEAS_MASTER", - }: - if columns: - raise UniverseBlockedError( - f"{label}.columns must be empty for built-in master formats" - ) - else: - if "symbol" not in columns: - raise UniverseBlockedError(f"{label}.columns.symbol is required") - if source_format == "KIS_FIXED_WIDTH": - for logical, bounds in columns.items(): - if ( - not isinstance(bounds, list) - or len(bounds) != 2 - or any( - isinstance(item, bool) or not isinstance(item, int) - for item in bounds - ) - or bounds[0] < 0 - or bounds[1] <= bounds[0] - ): - raise UniverseBlockedError( - f"{label}.columns.{logical} must be [start,end] " - "zero-based half-open integers" - ) - else: - for logical, header in columns.items(): - require_nonempty(header, f"{label}.columns.{logical}") - - status_is_available = source_format == "NASDAQ_LISTED" or "status" in columns - if status_is_available and not normal_status_values: - raise UniverseBlockedError( - f"{label}.normal_status_values cannot be empty when status is mapped" - ) - if not status_is_available and normal_status_values: - raise UniverseBlockedError( - f"{label}.normal_status_values must be empty without a status field" - ) - - path = resolve_input_path(raw["path"], spec_directory, f"{label}.path") - return { - "source_id": source_id, - "provider": provider, - "exchange": exchange, - "as_of": source_as_of, - "path": str(path), - "format": source_format, - "encoding": encoding, - "delimiter": delimiter, - "skip_rows": skip_rows, - "columns": columns, - "normal_status_values": normal_status_values, - "role": role, - } - - -def normalize_build_spec(raw: dict[str, Any], spec_directory: Path) -> dict[str, Any]: - require_exact_fields(raw, BUILD_SPEC_FIELDS, "build spec") - if raw["schema"] != BUILD_SPEC_SCHEMA: - raise UniverseBlockedError(f"unsupported build spec schema: {raw['schema']!r}") - as_of = require_iso_date(raw["as_of"], "build spec.as_of") - analysis_date = require_iso_date(raw["analysis_date"], "build spec.analysis_date") - if date.fromisoformat(analysis_date) > date.fromisoformat(as_of): - raise UniverseBlockedError("analysis_date cannot be later than as_of") - - official_raw = raw["official_sources"] - broker_raw = raw["broker_sources"] - if not isinstance(official_raw, list) or not isinstance(broker_raw, list): - raise UniverseBlockedError("official_sources and broker_sources must be arrays") - official = [ - normalize_source_descriptor( - item, "OFFICIAL_MASTER", as_of, spec_directory, index - ) - for index, item in enumerate(official_raw) - ] - broker = [ - normalize_source_descriptor(item, "BROKER_MASTER", as_of, spec_directory, index) - for index, item in enumerate(broker_raw) - ] - - official_by_exchange = {item["exchange"]: item for item in official} - if len(official_by_exchange) != len(official): - raise UniverseBlockedError( - "official_sources may contain only one source per exchange" - ) - exchange_scope = frozenset(official_by_exchange) - if exchange_scope not in SUPPORTED_EXCHANGE_SCOPES: - raise UniverseBlockedError( - "official_sources must contain exactly KOSPI/KOSDAQ, " - "NYSE/NASDAQ, or all four exchanges" - ) - broker_by_exchange = {item["exchange"]: item for item in broker} - if len(broker_by_exchange) != len(broker): - raise UniverseBlockedError( - "broker_sources may contain only one source per exchange" - ) - if frozenset(broker_by_exchange) != exchange_scope: - raise UniverseBlockedError( - "broker_sources must cover the same exchanges as official_sources" - ) - exchanges = tuple(exchange for exchange in EXCHANGES if exchange in exchange_scope) - - source_ids = [item["source_id"] for item in official + broker] - if len(source_ids) != len(set(source_ids)): - raise UniverseBlockedError("master source_id values must be unique") - - catalog_raw = raw["eod_catalog"] - if not isinstance(catalog_raw, dict): - raise UniverseBlockedError("eod_catalog must be an object") - require_exact_fields(catalog_raw, EOD_CATALOG_SPEC_FIELDS, "eod_catalog") - catalog_as_of = require_iso_date(catalog_raw["as_of"], "eod_catalog.as_of") - if catalog_as_of != as_of: - raise UniverseBlockedError( - f"eod_catalog.as_of {catalog_as_of} does not match build as_of {as_of}" - ) - catalog = { - "source_id": require_nonempty( - catalog_raw["source_id"], "eod_catalog.source_id" - ), - "provider": require_nonempty(catalog_raw["provider"], "eod_catalog.provider"), - "as_of": catalog_as_of, - "path": str( - resolve_input_path(catalog_raw["path"], spec_directory, "eod_catalog.path") - ), - "encoding": require_nonempty(catalog_raw["encoding"], "eod_catalog.encoding"), - } - if catalog["source_id"] in source_ids: - raise UniverseBlockedError("eod_catalog.source_id must be unique") - coverage_contract = normalize_catalog_coverage_contract( - raw["catalog_coverage_contract"] - ) - - return { - "schema": BUILD_SPEC_SCHEMA, - "as_of": as_of, - "analysis_date": analysis_date, - "official_sources": sorted( - official, key=lambda item: EXCHANGES.index(item["exchange"]) - ), - "broker_sources": sorted( - broker, key=lambda item: EXCHANGES.index(item["exchange"]) - ), - "eod_catalog": catalog, - "catalog_coverage_contract": coverage_contract, - "exchanges": exchanges, - } - - -def normalize_delimited_row( - raw: Mapping[str, Any], columns: Mapping[str, Any] -) -> dict[str, str]: - output: dict[str, str] = {} - for logical, header in columns.items(): - value = raw.get(str(header)) - output[logical] = "" if value is None else str(value).strip() - return output - - -def normalize_fixed_width_row( - line: bytes, columns: Mapping[str, Any], encoding: str -) -> dict[str, str]: - return { - logical: line[bounds[0] : bounds[1]].decode(encoding).strip() - for logical, bounds in columns.items() - } - - -def row_from_fields( - fields: Mapping[str, str], descriptor: Mapping[str, Any], row_number: int -) -> MasterRow: - label = f"{descriptor['source_id']} row {row_number}" - symbol = normalized_symbol(fields.get("symbol", ""), f"{label}.symbol") - return MasterRow( - exchange=str(descriptor["exchange"]), - symbol=symbol, - name=str(fields.get("name", "")).strip(), - instrument_type=str(fields.get("instrument_type", "")).strip(), - status=str(fields.get("status", "")).strip(), - flags={ - logical: str(fields.get(logical, "")).strip() - for logical in ( - "etf", - "etn", - "preferred", - "spac", - "unit", - "right", - "warrant", - "test_issue", - "abnormal", - "etp", - "trading_halt", - "liquidation", - "administrative", - ) - if logical in fields - }, - source_id=str(descriptor["source_id"]), - ) - - -def parse_csv_source(descriptor: Mapping[str, Any]) -> list[MasterRow]: - path = Path(str(descriptor["path"])) - delimiter = str(descriptor["delimiter"]) - if len(delimiter) != 1: - raise UniverseBlockedError( - f"{descriptor['source_id']}.delimiter must be exactly one character" - ) - with path.open("r", encoding=str(descriptor["encoding"]), newline="") as handle: - for _ in range(int(descriptor["skip_rows"])): - next(handle, None) - reader = csv.DictReader(handle, delimiter=delimiter) - if reader.fieldnames is None: - raise UniverseBlockedError(f"{descriptor['source_id']} has no header row") - required_headers = {str(header) for header in descriptor["columns"].values()} - missing_headers = required_headers - set(reader.fieldnames) - if missing_headers: - raise UniverseBlockedError( - f"{descriptor['source_id']} missing headers: {sorted(missing_headers)}" - ) - rows: list[MasterRow] = [] - for row_number, raw in enumerate( - reader, start=2 + int(descriptor["skip_rows"]) - ): - fields = normalize_delimited_row(raw, descriptor["columns"]) - if not any(fields.values()): - continue - rows.append(row_from_fields(fields, descriptor, row_number)) - return rows - - -def parse_fixed_width_source(descriptor: Mapping[str, Any]) -> list[MasterRow]: - path = Path(str(descriptor["path"])) - lines = path.read_bytes().splitlines() - rows: list[MasterRow] = [] - skip_rows = int(descriptor["skip_rows"]) - for row_number, line in enumerate(lines[skip_rows:], start=skip_rows + 1): - if not line.strip(): - continue - fields = normalize_fixed_width_row( - line, - descriptor["columns"], - str(descriptor["encoding"]), - ) - rows.append(row_from_fields(fields, descriptor, row_number)) - return rows - - -def split_fixed_fields(value: str, widths: Sequence[int]) -> list[str]: - output: list[str] = [] - offset = 0 - for width in widths: - output.append(value[offset : offset + width].strip()) - offset += width - if len(value) != offset: - raise UniverseBlockedError( - f"fixed-width tail has {len(value)} characters; expected exactly {offset}" - ) - return output - - -def parse_kis_krx_master(descriptor: Mapping[str, Any]) -> list[MasterRow]: - exchange = str(descriptor["exchange"]) - layout = KIS_KRX_LAYOUTS[exchange] - path = Path(str(descriptor["path"])) - lines = path.read_text(encoding=str(descriptor["encoding"])).splitlines() - rows: list[MasterRow] = [] - skip_rows = int(descriptor["skip_rows"]) - for row_number, line in enumerate(lines[skip_rows:], start=skip_rows + 1): - if not line.strip(): - continue - tail_length = int(layout["tail_length"]) - if len(line) <= tail_length + 21: - raise UniverseBlockedError( - f"{descriptor['source_id']} row {row_number} is shorter than the " - "KIS domestic master layout" - ) - prefix = line[:-tail_length] - tail = line[-tail_length:] - symbol = prefix[0:9].rstrip() - standard_symbol = prefix[9:21].rstrip() - name = prefix[21:].strip() - if not symbol or not standard_symbol or not name: - raise UniverseBlockedError( - f"{descriptor['source_id']} row {row_number} has an incomplete " - "KIS domestic prefix" - ) - parts = split_fixed_fields(tail, layout["widths"]) - group_code = parts[0].upper() - etp_code = parts[int(layout["etp_index"])].upper() - flags = { - "etf": "Y" if etp_code in {"1", "2", "5"} else "N", - "etn": "Y" if etp_code in {"3", "4"} else "N", - "etp": ("Y" if etp_code not in {"", "0", "1", "2", "3", "4", "5"} else "N"), - "spac": parts[int(layout["spac_index"])], - "preferred": ( - "Y" if parts[int(layout["preferred_index"])] not in {"", "0"} else "N" - ), - "trading_halt": parts[int(layout["halt_index"])], - "liquidation": parts[int(layout["liquidation_index"])], - "administrative": parts[int(layout["admin_index"])], - "warrant": "Y" if group_code in {"EW", "SW"} else "N", - "right": "Y" if group_code == "SR" else "N", - } - rows.append( - row_from_fields( - { - "symbol": symbol, - "name": name, - "instrument_type": group_code, - **flags, - }, - descriptor, - row_number, - ) - ) - return rows - - -def parse_kis_overseas_master( - descriptor: Mapping[str, Any], -) -> list[MasterRow]: - path = Path(str(descriptor["path"])) - exchange = str(descriptor["exchange"]) - expected_exchange_code = "NAS" if exchange == "NASDAQ" else "NYS" - lines = path.read_bytes().splitlines() - rows: list[MasterRow] = [] - skip_rows = int(descriptor["skip_rows"]) - for row_number, raw_line in enumerate(lines[skip_rows:], start=skip_rows + 1): - if not raw_line.strip(): - continue - fields = raw_line.decode(str(descriptor["encoding"])).split("\t") - if len(fields) != 24: - raise UniverseBlockedError( - f"{descriptor['source_id']} row {row_number} has {len(fields)} " - "columns; expected 24" - ) - if fields[2].strip().upper() != expected_exchange_code: - raise UniverseBlockedError( - f"{descriptor['source_id']} row {row_number} exchange code " - f"{fields[2]!r} does not match {expected_exchange_code}" - ) - symbol = fields[4].strip() - korean_name = fields[6].strip() - english_name = fields[7].strip() - security_type = fields[8].strip() - dr_flag = fields[17].strip().upper() - classification = fields[22].strip() - if dr_flag not in {"Y", "N"}: - raise UniverseBlockedError( - f"{descriptor['source_id']} row {row_number} has invalid DR flag " - f"{dr_flag!r}" - ) - is_etp = security_type == "3" - flags = { - "etf": ("Y" if is_etp and classification in {"001", "005"} else "N"), - "etn": ("Y" if is_etp and classification in {"002", "006"} else "N"), - "etp": ( - "Y" - if is_etp and classification not in {"001", "002", "005", "006"} - else "N" - ), - "warrant": "Y" if security_type == "4" else "N", - } - if security_type == "1": - normalized_type = "INDEX" - elif security_type == "2" and dr_flag == "Y": - normalized_type = "DR" - elif security_type == "2": - normalized_type = "STOCK" - elif security_type == "3": - normalized_type = "ETP" - elif security_type == "4": - normalized_type = "WARRANT" - else: - normalized_type = f"UNKNOWN_SECURITY_TYPE_{security_type or 'BLANK'}" - rows.append( - row_from_fields( - { - "symbol": symbol, - "name": english_name or korean_name, - "instrument_type": normalized_type, - **flags, - }, - descriptor, - row_number, - ) - ) - return rows - - -class KindTableParser(HTMLParser): - """Collect text cells from the offline KIND HTML-table Excel export.""" - - def __init__(self) -> None: - super().__init__(convert_charrefs=True) - self.rows: list[list[str]] = [] - self._row: Optional[list[str]] = None - self._cell_parts: Optional[list[str]] = None - - def handle_starttag(self, tag: str, attrs: list[tuple[str, Optional[str]]]) -> None: - del attrs - lowered = tag.lower() - if lowered == "tr": - self._row = [] - elif lowered in {"th", "td"} and self._row is not None: - self._cell_parts = [] - - def handle_data(self, data: str) -> None: - if self._cell_parts is not None: - self._cell_parts.append(data) - - def handle_endtag(self, tag: str) -> None: - lowered = tag.lower() - if lowered in {"th", "td"} and self._cell_parts is not None: - if self._row is not None: - self._row.append(" ".join("".join(self._cell_parts).split())) - self._cell_parts = None - elif lowered == "tr" and self._row is not None: - if self._row: - self.rows.append(self._row) - self._row = None - self._cell_parts = None - - -def parse_kind_html_source(descriptor: Mapping[str, Any]) -> list[MasterRow]: - path = Path(str(descriptor["path"])) - parser = KindTableParser() - parser.feed(path.read_text(encoding=str(descriptor["encoding"]))) - required_headers = {"회사명", "시장구분", "종목코드"} - header_index: Optional[int] = None - headers: list[str] = [] - for index, cells in enumerate(parser.rows): - if required_headers.issubset(set(cells)): - header_index = index - headers = cells - break - if header_index is None: - raise UniverseBlockedError( - f"{descriptor['source_id']} KIND table is missing headers " - f"{sorted(required_headers)}" - ) - if len(headers) != len(set(headers)): - raise UniverseBlockedError( - f"{descriptor['source_id']} KIND table has duplicate headers" - ) - market_values = { - "KOSPI": {"유가", "유가증권"}, - "KOSDAQ": {"코스닥"}, - } - exchange = str(descriptor["exchange"]) - expected_values = market_values[exchange] - rows: list[MasterRow] = [] - for row_number, cells in enumerate( - parser.rows[header_index + 1 :], start=header_index + 2 - ): - if len(cells) != len(headers): - raise UniverseBlockedError( - f"{descriptor['source_id']} KIND row {row_number} has " - f"{len(cells)} cells; expected {len(headers)}" - ) - raw = dict(zip(headers, cells)) - if raw["시장구분"].strip() not in expected_values: - continue - fields = { - "symbol": raw["종목코드"].strip(), - "name": raw["회사명"].strip(), - } - rows.append(row_from_fields(fields, descriptor, row_number)) - return rows - - -def parse_nasdaq_source(descriptor: Mapping[str, Any]) -> list[MasterRow]: - path = Path(str(descriptor["path"])) - source_format = str(descriptor["format"]) - with path.open("r", encoding=str(descriptor["encoding"]), newline="") as handle: - reader = csv.DictReader(handle, delimiter="|") - if reader.fieldnames is None: - raise UniverseBlockedError( - f"{descriptor['source_id']} has no Nasdaq header" - ) - if source_format == "NASDAQ_LISTED": - required = { - "Symbol", - "Security Name", - "ETF", - "Test Issue", - "Financial Status", - } - else: - required = { - "ACT Symbol", - "Security Name", - "Exchange", - "ETF", - "Test Issue", - } - missing = required - set(reader.fieldnames) - if missing: - raise UniverseBlockedError( - f"{descriptor['source_id']} missing Nasdaq headers: {sorted(missing)}" - ) - rows: list[MasterRow] = [] - for row_number, raw in enumerate(reader, start=2): - first_value = next(iter(raw.values()), "") - if str(first_value).startswith("File Creation Time:"): - continue - if ( - source_format == "NASDAQ_OTHER" - and str(raw.get("Exchange", "")).strip().upper() != "N" - ): - continue - symbol_header = ( - "Symbol" if source_format == "NASDAQ_LISTED" else "ACT Symbol" - ) - fields = { - "symbol": str(raw.get(symbol_header, "") or "").strip(), - "name": str(raw.get("Security Name", "") or "").strip(), - "etf": str(raw.get("ETF", "") or "").strip(), - "test_issue": str(raw.get("Test Issue", "") or "").strip(), - } - if source_format == "NASDAQ_LISTED": - fields["status"] = str(raw.get("Financial Status", "") or "").strip() - if not fields["symbol"]: - raise UniverseBlockedError( - f"{descriptor['source_id']} row {row_number} has no symbol" - ) - rows.append(row_from_fields(fields, descriptor, row_number)) - return rows - - -def parse_master_source(descriptor: Mapping[str, Any]) -> list[MasterRow]: - source_format = str(descriptor["format"]) - if source_format in {"KRX_CSV", "KIS_CSV"}: - rows = parse_csv_source(descriptor) - elif source_format == "KIS_FIXED_WIDTH": - rows = parse_fixed_width_source(descriptor) - elif source_format == "KIS_KRX_MASTER": - rows = parse_kis_krx_master(descriptor) - elif source_format == "KIS_OVERSEAS_MASTER": - rows = parse_kis_overseas_master(descriptor) - elif source_format == "KRX_KIND_HTML": - rows = parse_kind_html_source(descriptor) - else: - rows = parse_nasdaq_source(descriptor) - if not rows: - raise UniverseBlockedError(f"{descriptor['source_id']} contains no rows") - by_symbol: dict[str, MasterRow] = {} - for row in rows: - if row.symbol in by_symbol: - if by_symbol[row.symbol] == row: - continue - raise UniverseBlockedError( - f"{descriptor['source_id']} contains conflicting duplicate " - f"symbol {row.symbol}" - ) - by_symbol[row.symbol] = row - return [by_symbol[symbol] for symbol in sorted(by_symbol)] - - -def parse_bool_flag(value: str, label: str) -> bool: - normalized = value.strip().upper() - if normalized in TRUE_VALUES: - return True - if normalized in FALSE_VALUES: - return False - raise UniverseBlockedError( - f"{label} must be one of {sorted(TRUE_VALUES | FALSE_VALUES)}" - ) - - -def metadata_exclusion_reasons( - row: MasterRow, descriptor: Mapping[str, Any] -) -> list[str]: - reasons: list[str] = [] - flag_reasons = { - "etf": "instrument_etf", - "etn": "instrument_etn", - "preferred": "instrument_preferred", - "spac": "instrument_spac", - "unit": "instrument_unit", - "right": "instrument_right", - "warrant": "instrument_warrant", - "test_issue": "test_issue", - "abnormal": "abnormal_status", - "etp": "instrument_etp", - "trading_halt": "trading_halt", - "liquidation": "liquidation", - "administrative": "administrative_issue", - } - for flag, reason in flag_reasons.items(): - if flag in row.flags and parse_bool_flag( - row.flags[flag], f"{row.source_id}:{row.symbol}.{flag}" - ): - reasons.append(reason) - - normal_statuses = set(descriptor["normal_status_values"]) - if normal_statuses and row.status.strip().upper() not in normal_statuses: - reasons.append("abnormal_status") - - raw_type = row.instrument_type.strip().upper() - raw_type_reasons = { - "EF": "instrument_etf", - "FE": "instrument_etf", - "EN": "instrument_etn", - "PF": "instrument_preferred", - "EW": "instrument_warrant", - "SW": "instrument_warrant", - "SR": "instrument_right", - "ETF": "instrument_etf", - "ETN": "instrument_etn", - "WARRANT": "instrument_warrant", - "INDEX": "unsupported_instrument_type", - "MF": "unsupported_instrument_type", - "SC": "unsupported_instrument_type", - "IF": "unsupported_instrument_type", - "BC": "unsupported_instrument_type", - } - if raw_type in raw_type_reasons: - reasons.append(raw_type_reasons[raw_type]) - elif raw_type == "ETP" and not { - "instrument_etf", - "instrument_etn", - }.intersection(reasons): - reasons.append("instrument_etp") - elif raw_type.startswith("UNKNOWN_SECURITY_TYPE_"): - reasons.append("unsupported_instrument_type") - - searchable = f"{row.instrument_type} {row.name}".upper() - token_patterns = ( - (r"\bETF\b|상장지수펀드", "instrument_etf"), - (r"\bETN\b|상장지수증권", "instrument_etn"), - (r"\bPREFERRED\b|우선주", "instrument_preferred"), - (r"\bSPAC\b|\bACQUISITION CORP(?:ORATION)?\b|기업인수목적", "instrument_spac"), - (r"\bUNITS?\b", "instrument_unit"), - (r"\bRIGHTS?\b", "instrument_right"), - (r"\bWARRANTS?\b", "instrument_warrant"), - (r"\bTEST (?:ISSUE|STOCK)\b", "test_issue"), - ) - for pattern, reason in token_patterns: - if re.search(pattern, searchable): - reasons.append(reason) - return sorted(set(reasons)) - - -def parse_eod_catalog( - descriptor: Mapping[str, Any], -) -> tuple[dict[tuple[str, str], CatalogRow], Path]: - path = Path(str(descriptor["path"])) - with path.open("r", encoding=str(descriptor["encoding"]), newline="") as handle: - reader = csv.DictReader(handle) - if reader.fieldnames is None: - raise UniverseBlockedError("eod_catalog has no header") - if tuple(reader.fieldnames) != EOD_CATALOG_FIELDS: - raise UniverseBlockedError( - "eod_catalog headers must be exactly, in order: " - + ",".join(EOD_CATALOG_FIELDS) - ) - output: dict[tuple[str, str], CatalogRow] = {} - for row_number, raw in enumerate(reader, start=2): - if not any(str(value or "").strip() for value in raw.values()): - continue - if None in raw: - raise UniverseBlockedError( - f"eod_catalog row {row_number} has extra cells" - ) - cells = { - field: str(raw[field] or "").strip() for field in EOD_CATALOG_FIELDS - } - exchange = cells["exchange"].upper() - if exchange not in EXCHANGES: - raise UniverseBlockedError( - f"eod_catalog row {row_number}.exchange must be one of {EXCHANGES}" - ) - canonical_symbol = normalized_symbol( - cells["canonical_symbol"], - f"eod_catalog row {row_number}.canonical_symbol", - ) - data_symbol = cells["data_symbol"] - broker_symbol = cells["broker_symbol"].upper() - instrument_type = cells["instrument_type"].upper() - catalog_row = CatalogRow( - exchange=exchange, - canonical_symbol=canonical_symbol, - data_symbol=data_symbol, - broker_symbol=broker_symbol, - instrument_type=instrument_type, - benchmark_id=cells["benchmark_id"].upper(), - ticker_csv=cells["ticker_csv"], - benchmark_csv=cells["benchmark_csv"], - tick_rule_id=cells["tick_rule_id"], - tick_effective_date=cells["tick_effective_date"], - tick_reference_price=cells["tick_reference_price"], - resolved_tick_size=cells["resolved_tick_size"], - source_name=cells["source_name"], - ) - key = (exchange, canonical_symbol) - if key in output: - raise UniverseBlockedError( - f"duplicate eod_catalog row: {exchange}:{canonical_symbol}" - ) - output[key] = catalog_row - if not output: - raise UniverseBlockedError("eod_catalog contains no rows") - return output, path.parent - - -def optional_data_path(raw: str, catalog_directory: Path) -> Optional[Path]: - if not raw: - return None - path = Path(raw) - if not path.is_absolute(): - path = catalog_directory / path - path = path.resolve() - if not path.is_file(): - return None - return path - - -def normalize_tick_contract( - row: CatalogRow, analysis_date: str -) -> tuple[Optional[dict[str, str]], Optional[str]]: - fields = ( - row.tick_rule_id, - row.tick_effective_date, - row.tick_reference_price, - row.resolved_tick_size, - ) - if any(not value for value in fields): - return None, "missing_tick_contract" - try: - effective_date = require_iso_date( - row.tick_effective_date, "tick_effective_date" - ) - if date.fromisoformat(effective_date) > date.fromisoformat(analysis_date): - raise UniverseBlockedError( - "tick_effective_date cannot be later than analysis_date" - ) - reference_price = decimal_string( - row.tick_reference_price, "tick_reference_price", positive=True - ) - resolved_tick_size = decimal_string( - row.resolved_tick_size, "resolved_tick_size", positive=True - ) - except UniverseBlockedError: - return None, "invalid_tick_contract" - return ( - { - "schema": TICK_CONTRACT_SCHEMA, - "kind": TICK_CONTRACT_KIND, - "rule_id": row.tick_rule_id, - "effective_date": effective_date, - "reference_price": reference_price, - "resolved_tick_size": resolved_tick_size, - }, - None, - ) - - -def catalog_exclusion_reasons( - row: CatalogRow, - catalog_directory: Path, - analysis_date: str, -) -> tuple[ - list[str], - Optional[Path], - Optional[Path], - Optional[dict[str, str]], -]: - reasons: list[str] = [] - if row.instrument_type in EXCLUDED_CATALOG_TYPES: - reasons.append(EXCLUDED_CATALOG_TYPES[row.instrument_type]) - elif row.instrument_type not in INCLUDED_INSTRUMENT_TYPES: - reasons.append("unsupported_instrument_type") - if not row.data_symbol: - reasons.append("missing_data_symbol") - if not row.broker_symbol: - reasons.append("missing_broker_symbol") - if row.benchmark_id != EXCHANGE_CONTRACTS[row.exchange]["benchmark_id"]: - reasons.append("invalid_benchmark_id") - if not row.source_name: - reasons.append("missing_eod_source") - - ticker_path = optional_data_path(row.ticker_csv, catalog_directory) - if ticker_path is None: - reasons.append("missing_ticker_csv") - benchmark_path = optional_data_path(row.benchmark_csv, catalog_directory) - if benchmark_path is None: - reasons.append("missing_benchmark_csv") - tick_contract, tick_reason = normalize_tick_contract(row, analysis_date) - if tick_reason is not None: - reasons.append(tick_reason) - return ( - sorted(set(reasons)), - ticker_path, - benchmark_path, - tick_contract, - ) - - -def source_hash_entry( - descriptor: Mapping[str, Any], role: str, exchange: str -) -> dict[str, str]: - path = Path(str(descriptor["path"])) - entry = { - "source_id": str(descriptor["source_id"]), - "role": role, - "provider": str(descriptor["provider"]), - "exchange": exchange, - "as_of": str(descriptor["as_of"]), - "path": str(path.resolve()), - "sha256": sha256_file(path), - } - if not HEX64.fullmatch(entry["sha256"]): - raise AssertionError("sha256_file returned an invalid digest") - return entry - - -def exclusion_record( - exchange: str, - canonical_symbol: str, - broker_symbol: Optional[str], - name: str, - reasons: Sequence[str], - official_source_id: str, - broker_source_id: Optional[str], -) -> dict[str, Any]: - normalized_reasons = sorted(set(reasons)) - if not normalized_reasons: - raise AssertionError("an exclusion must have at least one reason") - return { - "exchange": exchange, - "canonical_symbol": canonical_symbol, - "broker_symbol": broker_symbol, - "name": name, - "reasons": normalized_reasons, - "official_source_id": official_source_id, - "broker_source_id": broker_source_id, - } - - -def build_manifest(raw_spec: dict[str, Any], spec_directory: Path) -> dict[str, Any]: - spec = normalize_build_spec(raw_spec, spec_directory) - exchanges = spec["exchanges"] - official_descriptors = {item["exchange"]: item for item in spec["official_sources"]} - broker_descriptors = {item["exchange"]: item for item in spec["broker_sources"]} - official_rows = { - exchange: { - row.symbol: row - for row in parse_master_source(official_descriptors[exchange]) - } - for exchange in exchanges - } - broker_rows = { - exchange: { - row.symbol: row for row in parse_master_source(broker_descriptors[exchange]) - } - for exchange in broker_descriptors - } - catalog, catalog_directory = parse_eod_catalog(spec["eod_catalog"]) - - source_hashes = [ - source_hash_entry(item, "OFFICIAL_MASTER", item["exchange"]) - for item in spec["official_sources"] - ] - source_hashes.extend( - source_hash_entry(item, "BROKER_MASTER", item["exchange"]) - for item in spec["broker_sources"] - ) - source_hashes.append( - source_hash_entry(spec["eod_catalog"], "EOD_CATALOG", "GLOBAL") - ) - source_hashes.sort(key=lambda item: item["source_id"]) - - instruments: list[dict[str, Any]] = [] - exclusions: list[dict[str, Any]] = [] - path_hash_cache: dict[Path, str] = {} - - for exchange in exchanges: - official_descriptor = official_descriptors[exchange] - broker_descriptor = broker_descriptors[exchange] - broker_by_symbol = broker_rows[exchange] - for canonical_symbol in sorted(official_rows[exchange]): - official_row = official_rows[exchange][canonical_symbol] - catalog_row = catalog.get((exchange, canonical_symbol)) - reasons = metadata_exclusion_reasons(official_row, official_descriptor) - broker_row: Optional[MasterRow] = None - broker_symbol: Optional[str] = None - broker_source_id: Optional[str] = str(broker_descriptor["source_id"]) - ticker_path: Optional[Path] = None - benchmark_path: Optional[Path] = None - tick_contract: Optional[dict[str, str]] = None - - if catalog_row is None: - reasons.append("missing_eod_mapping") - else: - broker_symbol = catalog_row.broker_symbol or None - ( - catalog_reasons, - ticker_path, - benchmark_path, - tick_contract, - ) = catalog_exclusion_reasons( - catalog_row, - catalog_directory, - spec["analysis_date"], - ) - reasons.extend(catalog_reasons) - if catalog_row.broker_symbol: - broker_row = broker_by_symbol.get(catalog_row.broker_symbol.upper()) - if broker_row is None: - reasons.append("not_kis_tradable") - else: - reasons.extend( - metadata_exclusion_reasons(broker_row, broker_descriptor) - ) - - reasons = sorted(set(reasons)) - if reasons: - exclusions.append( - exclusion_record( - exchange=exchange, - canonical_symbol=canonical_symbol, - broker_symbol=broker_symbol, - name=official_row.name, - reasons=reasons, - official_source_id=official_row.source_id, - broker_source_id=broker_source_id, - ) - ) - continue - - if catalog_row is None: - raise AssertionError("catalog row unexpectedly missing") - if catalog_row.instrument_type not in INCLUDED_INSTRUMENT_TYPES: - raise AssertionError("excluded catalog type reached inclusion path") - if ticker_path is None or benchmark_path is None or tick_contract is None: - raise AssertionError("validated EOD inputs unexpectedly missing") - for data_path in (ticker_path, benchmark_path): - if data_path not in path_hash_cache: - path_hash_cache[data_path] = sha256_file(data_path) - - contract = EXCHANGE_CONTRACTS[exchange] - instruments.append( - { - "market": contract["market"], - "exchange": exchange, - "canonical_symbol": canonical_symbol, - "data_symbol": catalog_row.data_symbol, - "broker_symbol": catalog_row.broker_symbol, - "instrument_type": catalog_row.instrument_type, - "benchmark_id": catalog_row.benchmark_id, - "currency": contract["currency"], - "venue": contract["venue"], - "ticker_csv": str(ticker_path), - "benchmark_csv": str(benchmark_path), - "tick_contract": tick_contract, - "source_name": catalog_row.source_name, - "ticker_csv_sha256": path_hash_cache[ticker_path], - "benchmark_csv_sha256": path_hash_cache[benchmark_path], - "broker_tradability_verified": True, - "official_source_id": official_row.source_id, - "broker_source_id": broker_source_id, - } - ) - - official_symbols = set(official_rows[exchange]) - for catalog_key, catalog_row in sorted(catalog.items()): - catalog_exchange, canonical_symbol = catalog_key - if catalog_exchange != exchange or canonical_symbol in official_symbols: - continue - exclusions.append( - exclusion_record( - exchange=exchange, - canonical_symbol=canonical_symbol, - broker_symbol=catalog_row.broker_symbol or None, - name="", - reasons=["not_official_exchange_member"], - official_source_id=str(official_descriptor["source_id"]), - broker_source_id=str(broker_descriptor["source_id"]), - ) - ) - - instruments.sort( - key=lambda item: ( - EXCHANGES.index(item["exchange"]), - item["canonical_symbol"], - item["broker_symbol"], - ) - ) - exclusions.sort( - key=lambda item: ( - EXCHANGES.index(item["exchange"]), - item["canonical_symbol"], - tuple(item["reasons"]), - ) - ) - - by_exchange: dict[str, dict[str, int]] = {} - for exchange in EXCHANGES: - exchange_official_rows = official_rows.get(exchange, {}) - catalog_mapped = sum( - (exchange, symbol) in catalog for symbol in exchange_official_rows - ) - by_exchange[exchange] = { - "official": len(exchange_official_rows), - "broker": len(broker_rows.get(exchange, {})), - "catalog_mapped": catalog_mapped, - "included": sum(item["exchange"] == exchange for item in instruments), - "excluded": sum(item["exchange"] == exchange for item in exclusions), - } - counts = { - "official_rows": sum(item["official"] for item in by_exchange.values()), - "broker_rows": sum(item["broker"] for item in by_exchange.values()), - "catalog_mapped_rows": sum( - item["catalog_mapped"] for item in by_exchange.values() - ), - "included": sum(item["included"] for item in by_exchange.values()), - "excluded": sum(item["excluded"] for item in by_exchange.values()), - "by_exchange": by_exchange, - } - coverage_minimums = spec["catalog_coverage_contract"]["minimum_ratio_by_exchange"] - coverage_ready = all( - meets_minimum_coverage_ratio( - by_exchange[exchange]["catalog_mapped"], - by_exchange[exchange]["official"], - coverage_minimums[exchange], - ) - for exchange in exchanges - ) - screenable_minimums = spec["catalog_coverage_contract"][ - "minimum_screenable_ratio_by_exchange" - ] - screenable_ready = all( - meets_minimum_coverage_ratio( - by_exchange[exchange]["included"], - by_exchange[exchange]["official"], - screenable_minimums[exchange], - ) - for exchange in exchanges - ) - build_status = ( - "READY" - if coverage_ready - and screenable_ready - and all(by_exchange[exchange]["included"] > 0 for exchange in exchanges) - else "BLOCKED" - ) - output: dict[str, Any] = { - "schema": MANIFEST_SCHEMA, - "build_status": build_status, - "as_of": spec["as_of"], - "analysis_date": spec["analysis_date"], - "catalog_coverage_contract": spec["catalog_coverage_contract"], - "source_hashes": source_hashes, - "instruments": instruments, - "exclusions": exclusions, - "counts": counts, - } - output["manifest_hash"] = sha256_json(output) - return output - - -def write_csv( - path: Path, fieldnames: Sequence[str], rows: Sequence[Mapping[str, Any]] -) -> None: - with path.open("w", encoding="utf-8", newline="") as handle: - writer = csv.DictWriter(handle, fieldnames=fieldnames) - writer.writeheader() - writer.writerows(rows) - - -def synthetic_kis_krx_line( - exchange: str, - symbol: str, - name: str, - *, - group_code: str = "ST", - etp_code: str = "0", - spac: str = "N", - preferred: str = "0", - trading_halt: str = "N", - liquidation: str = "N", - administrative: str = "N", -) -> str: - layout = KIS_KRX_LAYOUTS[exchange] - widths = layout["widths"] - values = [""] * len(widths) - values[0] = group_code - values[int(layout["etp_index"])] = etp_code - values[int(layout["spac_index"])] = spac - values[int(layout["halt_index"])] = trading_halt - values[int(layout["liquidation_index"])] = liquidation - values[int(layout["admin_index"])] = administrative - values[int(layout["preferred_index"])] = preferred - tail = "".join( - str(value).ljust(width)[:width] for value, width in zip(values, widths) - ) - tail = tail.ljust(int(layout["tail_length"])) - standard_symbol = f"KR7{symbol}".ljust(12)[:12] - return f"{symbol.ljust(9)}{standard_symbol}{name}{tail}" - - -def synthetic_kis_overseas_line( - exchange: str, - symbol: str, - security_type: str, - dr_flag: str, - classification: str = "", -) -> str: - fields = [""] * 24 - fields[0] = "US" - fields[1] = "22" if exchange == "NASDAQ" else "21" - fields[2] = "NAS" if exchange == "NASDAQ" else "NYS" - fields[3] = exchange - fields[4] = symbol - fields[5] = f"{fields[2]}{symbol}" - fields[6] = symbol - fields[7] = f"{symbol} SYNTHETIC" - fields[8] = security_type - fields[9] = "USD" - fields[17] = dr_flag - fields[22] = classification - return "\t".join(fields) - - -def self_test() -> None: - with tempfile.TemporaryDirectory(prefix="qta-universe-v2-") as directory: - root = Path(directory) - master_paths: dict[str, Path] = {} - broker_paths: dict[str, Path] = {} - kind_path = root / "kind-corpList.xls" - kind_path.write_text( - "" - "" - "" - "" - "" - "" - "" - "" - "
회사명시장구분종목코드
KOSPI Common유가000001
KOSPI ETF유가000002
KOSDAQ Common코스닥100001
KOSDAQ Missing코스닥100002
KOSDAQ Alpha코스닥0156T0
KONEX Ignored코넥스900001
", - encoding="cp949", - ) - for exchange in ("KOSPI", "KOSDAQ"): - master_paths[exchange] = kind_path - broker_path = root / f"{exchange.lower()}_code.mst" - broker_path.write_text( - synthetic_kis_krx_line( - exchange, - "000001" if exchange == "KOSPI" else "100001", - f"{exchange} Common", - ) - + "\n", - encoding="cp949", - ) - broker_paths[exchange] = broker_path - - nasdaq_path = root / "nasdaqlisted.txt" - nasdaq_path.write_text( - "Symbol|Security Name|Market Category|Test Issue|Financial Status|" - "Round Lot Size|ETF|NextShares\n" - "NDAQ1|Nasdaq Common|Q|N|N|100|N|N\n" - "NDAQW|Nasdaq Warrants|Q|N|N|100|N|N\n" - "File Creation Time: 0726202600|||||||\n", - encoding="utf-8", - ) - nyse_path = root / "otherlisted.txt" - nyse_path.write_text( - "ACT Symbol|Security Name|Exchange|CQS Symbol|ETF|Round Lot Size|" - "Test Issue|NASDAQ Symbol\n" - "NYSE1|NYSE ADR|N|NYSE1|N|100|N|NYSE1\n" - "NYSEX|Other Market|A|NYSEX|N|100|N|NYSEX\n" - "File Creation Time: 0726202600|||||||\n", - encoding="utf-8", - ) - master_paths["NASDAQ"] = nasdaq_path - master_paths["NYSE"] = nyse_path - - for exchange, symbol in (("NASDAQ", "NDAQ1"), ("NYSE", "NYSE1")): - broker_path = root / ( - "nasmst.cod" if exchange == "NASDAQ" else "nysmst.cod" - ) - broker_path.write_text( - synthetic_kis_overseas_line( - exchange, - symbol, - "2", - "Y" if exchange == "NYSE" else "N", - ) - + "\n", - encoding="cp949", - ) - broker_paths[exchange] = broker_path - - ticker_paths: dict[str, Path] = {} - benchmark_paths: dict[str, Path] = {} - included_symbols = { - "KOSPI": "000001", - "KOSDAQ": "100001", - "NYSE": "NYSE1", - "NASDAQ": "NDAQ1", - } - for exchange, symbol in included_symbols.items(): - ticker_path = root / f"{exchange.lower()}-{symbol}.csv" - ticker_path.write_text( - "date,adjusted_close\n2026-07-24,100\n", encoding="utf-8" - ) - ticker_paths[exchange] = ticker_path - benchmark_path = root / f"{exchange.lower()}-benchmark.csv" - benchmark_path.write_text( - "date,adjusted_close\n2026-07-24,100\n", encoding="utf-8" - ) - benchmark_paths[exchange] = benchmark_path - - catalog_path = root / "eod-catalog.csv" - catalog_rows: list[dict[str, str]] = [] - for exchange, symbol in included_symbols.items(): - catalog_rows.append( - { - "exchange": exchange, - "canonical_symbol": symbol, - "data_symbol": ( - f"{symbol}.KS" - if exchange == "KOSPI" - else f"{symbol}.KQ" - if exchange == "KOSDAQ" - else symbol - ), - "broker_symbol": symbol, - "instrument_type": "ADR" if exchange == "NYSE" else "COMMON", - "benchmark_id": EXCHANGE_CONTRACTS[exchange]["benchmark_id"], - "ticker_csv": str(ticker_paths[exchange]), - "benchmark_csv": str(benchmark_paths[exchange]), - "tick_rule_id": f"SELFTEST-{exchange}", - "tick_effective_date": "2026-01-01", - "tick_reference_price": "100", - "resolved_tick_size": ( - "1" if exchange in {"KOSPI", "KOSDAQ"} else "0.01" - ), - "source_name": "synthetic-adjusted-eod", - } - ) - catalog_rows.extend( - [ - { - "exchange": "KOSPI", - "canonical_symbol": "000002", - "data_symbol": "", - "broker_symbol": "", - "instrument_type": "ETF", - "benchmark_id": "KOSPI_COMPOSITE", - "ticker_csv": "", - "benchmark_csv": "", - "tick_rule_id": "SELFTEST-KOSPI", - "tick_effective_date": "2026-01-01", - "tick_reference_price": "100", - "resolved_tick_size": "1", - "source_name": "", - }, - { - "exchange": "NASDAQ", - "canonical_symbol": "NDAQW", - "data_symbol": "", - "broker_symbol": "", - "instrument_type": "WARRANT", - "benchmark_id": "NASDAQ_COMPOSITE", - "ticker_csv": "", - "benchmark_csv": "", - "tick_rule_id": "SELFTEST-NASDAQ", - "tick_effective_date": "2026-01-01", - "tick_reference_price": "1", - "resolved_tick_size": "0.01", - "source_name": "", - }, - { - "exchange": "KOSDAQ", - "canonical_symbol": "100002", - "data_symbol": "100002.KQ", - "broker_symbol": "", - "instrument_type": "COMMON", - "benchmark_id": "KOSDAQ_COMPOSITE", - "ticker_csv": "", - "benchmark_csv": "", - "tick_rule_id": "", - "tick_effective_date": "", - "tick_reference_price": "", - "resolved_tick_size": "", - "source_name": "synthetic-adjusted-eod", - }, - { - "exchange": "KOSDAQ", - "canonical_symbol": "0156T0", - "data_symbol": "0156T0.KQ", - "broker_symbol": "0156T0", - "instrument_type": "COMMON", - "benchmark_id": "KOSDAQ_COMPOSITE", - "ticker_csv": str(ticker_paths["KOSDAQ"]), - "benchmark_csv": str(benchmark_paths["KOSDAQ"]), - "tick_rule_id": "SELFTEST-KOSDAQ", - "tick_effective_date": "2026-01-01", - "tick_reference_price": "100", - "resolved_tick_size": "0", - "source_name": "synthetic-adjusted-eod", - }, - ] - ) - write_csv(catalog_path, EOD_CATALOG_FIELDS, catalog_rows) - - official_sources: list[dict[str, Any]] = [] - broker_sources: list[dict[str, Any]] = [] - for exchange in EXCHANGES: - if exchange in {"KOSPI", "KOSDAQ"}: - official_format = "KRX_KIND_HTML" - official_columns = {} - official_delimiter = "" - official_statuses = [] - elif exchange == "NASDAQ": - official_format = "NASDAQ_LISTED" - official_columns = {} - official_delimiter = "|" - official_statuses = ["N"] - else: - official_format = "NASDAQ_OTHER" - official_columns = {} - official_delimiter = "|" - official_statuses = [] - official_sources.append( - { - "source_id": f"official-{exchange.lower()}", - "provider": ( - "KRX" if exchange in {"KOSPI", "KOSDAQ"} else "NASDAQ_TRADER" - ), - "exchange": exchange, - "as_of": "2026-07-26", - "path": str(master_paths[exchange]), - "format": official_format, - "encoding": ( - "cp949" if official_format == "KRX_KIND_HTML" else "utf-8" - ), - "delimiter": official_delimiter, - "skip_rows": 0, - "columns": official_columns, - "normal_status_values": official_statuses, - } - ) - broker_sources.append( - { - "source_id": f"kis-{exchange.lower()}", - "provider": "KIS", - "exchange": exchange, - "as_of": "2026-07-26", - "path": str(broker_paths[exchange]), - "format": ( - "KIS_KRX_MASTER" - if exchange in {"KOSPI", "KOSDAQ"} - else "KIS_OVERSEAS_MASTER" - ), - "encoding": "cp949", - "delimiter": ("" if exchange in {"KOSPI", "KOSDAQ"} else "\t"), - "skip_rows": 0, - "columns": {}, - "normal_status_values": [], - } - ) - - spec = { - "schema": BUILD_SPEC_SCHEMA, - "as_of": "2026-07-26", - "analysis_date": "2026-07-24", - "catalog_coverage_contract": { - "schema": CATALOG_COVERAGE_SCHEMA, - "minimum_ratio_by_exchange": {exchange: "1" for exchange in EXCHANGES}, - "minimum_screenable_ratio_by_exchange": { - exchange: "0.3" for exchange in EXCHANGES - }, - }, - "official_sources": official_sources, - "broker_sources": broker_sources, - "eod_catalog": { - "source_id": "eod-catalog", - "provider": "synthetic-eod", - "as_of": "2026-07-26", - "path": str(catalog_path), - "encoding": "utf-8", - }, - } - first = build_manifest(spec, root) - second = build_manifest( - { - **spec, - "official_sources": list(reversed(official_sources)), - "broker_sources": list(reversed(broker_sources)), - }, - root, - ) - assert first == second - assert first["build_status"] == "READY" - assert first["counts"]["included"] == 4 - assert first["counts"]["by_exchange"]["KOSPI"]["excluded"] == 1 - assert first["counts"]["by_exchange"]["KOSDAQ"]["excluded"] == 2 - assert first["counts"]["by_exchange"]["NASDAQ"]["excluded"] == 1 - assert ( - first["counts"]["catalog_mapped_rows"] == first["counts"]["official_rows"] - ) - assert all( - first["counts"]["by_exchange"][exchange]["catalog_mapped"] - == first["counts"]["by_exchange"][exchange]["official"] - for exchange in EXCHANGES - ) - assert any( - item["canonical_symbol"] == "000002" and "instrument_etf" in item["reasons"] - for item in first["exclusions"] - ) - assert any( - item["canonical_symbol"] == "NDAQW" - and "instrument_warrant" in item["reasons"] - for item in first["exclusions"] - ) - assert any( - item["canonical_symbol"] == "0156T0" - and "invalid_tick_contract" in item["reasons"] - for item in first["exclusions"] - ) - assert any( - item["canonical_symbol"] == "100002" - and { - "missing_tick_contract", - "missing_ticker_csv", - "missing_benchmark_csv", - }.issubset(set(item["reasons"])) - for item in first["exclusions"] - ) - assert all(item["broker_tradability_verified"] for item in first["instruments"]) - assert all( - set(item["tick_contract"]) - == { - "schema", - "kind", - "rule_id", - "effective_date", - "reference_price", - "resolved_tick_size", - } - for item in first["instruments"] - ) - expected_hash = sha256_json( - {key: value for key, value in first.items() if key != "manifest_hash"} - ) - assert first["manifest_hash"] == expected_hash - kr_only = build_manifest( - { - **spec, - "official_sources": [ - item - for item in official_sources - if item["exchange"] in {"KOSPI", "KOSDAQ"} - ], - "broker_sources": [ - item - for item in broker_sources - if item["exchange"] in {"KOSPI", "KOSDAQ"} - ], - }, - root, - ) - assert kr_only["build_status"] == "READY" - assert {item["market"] for item in kr_only["instruments"]} == {"KR"} - assert all( - value == 0 - for exchange in ("NYSE", "NASDAQ") - for value in kr_only["counts"]["by_exchange"][exchange].values() - ) - relaxed_manifest = build_manifest( - { - **spec, - "catalog_coverage_contract": { - "schema": CATALOG_COVERAGE_SCHEMA, - "minimum_ratio_by_exchange": { - exchange: "0.5" for exchange in EXCHANGES - }, - "minimum_screenable_ratio_by_exchange": { - exchange: "0.3" for exchange in EXCHANGES - }, - }, - }, - root, - ) - assert relaxed_manifest["build_status"] == "READY" - assert relaxed_manifest["manifest_hash"] != first["manifest_hash"] - - strict_screenable_manifest = build_manifest( - { - **spec, - "catalog_coverage_contract": { - "schema": CATALOG_COVERAGE_SCHEMA, - "minimum_ratio_by_exchange": { - exchange: "1" for exchange in EXCHANGES - }, - "minimum_screenable_ratio_by_exchange": { - exchange: "0.6" for exchange in EXCHANGES - }, - }, - }, - root, - ) - assert ( - strict_screenable_manifest["counts"]["catalog_mapped_rows"] - == strict_screenable_manifest["counts"]["official_rows"] - ) - assert strict_screenable_manifest["build_status"] == "BLOCKED" - assert strict_screenable_manifest["manifest_hash"] != first["manifest_hash"] - - minimal_catalog_path = root / "minimal-eod-catalog.csv" - write_csv(minimal_catalog_path, EOD_CATALOG_FIELDS, catalog_rows[:4]) - minimal_manifest = build_manifest( - { - **spec, - "eod_catalog": { - **spec["eod_catalog"], - "path": str(minimal_catalog_path), - }, - }, - root, - ) - assert minimal_manifest["counts"]["included"] == 4 - assert minimal_manifest["counts"]["catalog_mapped_rows"] == 4 - assert minimal_manifest["build_status"] == "BLOCKED" - assert minimal_manifest["manifest_hash"] != first["manifest_hash"] - print( - canonical_json( - { - "self_test": "PASS", - "schema": MANIFEST_SCHEMA, - "manifest_hash": first["manifest_hash"], - "counts": first["counts"], - } - ) - ) - - -def emit(value: Mapping[str, Any], output: Optional[str]) -> None: - rendered = json.dumps( - value, - ensure_ascii=False, - sort_keys=True, - indent=2, - allow_nan=False, - ) - if output: - output_path = Path(output).resolve() - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(rendered + "\n", encoding="utf-8") - print( - canonical_json( - { - "schema": value.get("schema"), - "build_status": value.get("build_status"), - "output": str(output_path), - "manifest_hash": value.get("manifest_hash"), - "counts": value.get("counts"), - } - ) - ) - else: - print(rendered) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--build-spec") - parser.add_argument("--output") - parser.add_argument("--self-test", action="store_true") - return parser.parse_args() - - -def main() -> int: - args = parse_args() - if args.self_test: - self_test() - return 0 - if not args.build_spec: - print( - canonical_json( - { - "schema": "qta-universe-build-error/v1", - "build_status": "BLOCKED", - "reason": "--build-spec is required", - } - ) - ) - return 2 - spec_path = Path(args.build_spec).resolve() - try: - manifest = build_manifest( - load_json_object(spec_path), - spec_path.parent, - ) - except ( - UniverseBlockedError, - OSError, - UnicodeError, - json.JSONDecodeError, - ) as exc: - error = { - "schema": "qta-universe-build-error/v1", - "build_status": "BLOCKED", - "reason": str(exc), - } - emit(error, None) - return 2 - emit(manifest, args.output) - return 0 if manifest["build_status"] == "READY" else 2 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/skills/quant-stock-technical/scripts/collect_open1h_research.py b/skills/quant-stock-technical/scripts/collect_open1h_research.py deleted file mode 100644 index 88fc3c1..0000000 --- a/skills/quant-stock-technical/scripts/collect_open1h_research.py +++ /dev/null @@ -1,920 +0,0 @@ -#!/usr/bin/env python3 -"""Collect deterministic first-60-minute KIS bars for QTA research.""" - -from __future__ import annotations - -import argparse -import json -import sys -import tempfile -from datetime import date -from decimal import Decimal, InvalidOperation, ROUND_HALF_EVEN -from pathlib import Path -from typing import Any, Mapping - -import fetch_kis_kr_eod as shared -import screen_universe as screen - - -JOB_SCHEMA = "qta-open1h-research-job/v1" -BARS_SCHEMA = "qta-open1h-bars/v1" -SNAPSHOT_SCHEMA = "qta-open1h-snapshot/v1" -JOB_FIELDS = { - "schema", - "manifest_path", - "market", - "session_date", - "output_directory", - "environment", - "request_interval_ms", -} -MARKET_CONTRACTS = { - "KR": { - "exchanges": ("KOSPI", "KOSDAQ"), - "interval_minutes": 1, - "window_start": "090000", - "window_end": "100000", - "expected_bars": 61, - "path": ( - "/uapi/domestic-stock/v1/quotations/" - "inquire-time-dailychartprice" - ), - "tr_id": "FHKST03010230", - }, - "US": { - "exchanges": ("NYSE", "NASDAQ"), - "interval_minutes": 5, - "window_start": "093000", - "window_end": "103000", - "expected_bars": 13, - "path": ( - "/uapi/overseas-price/v1/quotations/" - "inquire-time-itemchartprice" - ), - "tr_id": "HHDFS76950200", - }, -} -US_EXCHANGE_CODES = {"NYSE": "NYS", "NASDAQ": "NAS"} -BAR_FIELDS = { - "time", - "open", - "high", - "low", - "close", - "volume", - "turnover", -} -BARS_FIELDS = { - "schema", - "source", - "market", - "exchange", - "canonical_symbol", - "broker_symbol", - "session_date", - "interval_minutes", - "window_start", - "window_end", - "bars", - "bars_hash", - "summary", -} -SUMMARY_FIELDS = { - "bar_count", - "start_open", - "end_close", - "window_high", - "window_low", - "window_volume", - "window_turnover", - "return_60m_bps", - "maximum_excursion_bps", - "minimum_excursion_bps", -} - - -class ResearchBlockedError(ValueError): - """Raised when first-hour research data cannot be used without guessing.""" - - -def exact_fields(value: Mapping[str, Any], expected: set[str], label: str) -> None: - if set(value) != expected: - raise ResearchBlockedError( - f"{label} fields mismatch; missing={sorted(expected - set(value))}, " - f"extra={sorted(set(value) - expected)}" - ) - - -def require_date(value: Any, field: str) -> str: - if not isinstance(value, str): - raise ResearchBlockedError(f"{field} must be YYYY-MM-DD") - try: - parsed = date.fromisoformat(value) - except ValueError as exc: - raise ResearchBlockedError(f"{field} must be YYYY-MM-DD") from exc - if parsed.isoformat() != value: - raise ResearchBlockedError(f"{field} must be YYYY-MM-DD") - return value - - -def require_absolute_path(value: Any, field: str) -> Path: - path = Path(str(value)).expanduser() - if not path.is_absolute(): - raise ResearchBlockedError(f"{field} must be absolute") - return path - - -def decimal_value( - value: Any, - field: str, - *, - allow_zero: bool = False, -) -> Decimal: - if isinstance(value, bool): - raise ResearchBlockedError(f"{field} must be decimal") - try: - parsed = Decimal(str(value).replace(",", "").strip()) - except (InvalidOperation, ValueError) as exc: - raise ResearchBlockedError(f"{field} must be decimal") from exc - if not parsed.is_finite() or ( - parsed < 0 if allow_zero else parsed <= 0 - ): - qualifier = "nonnegative" if allow_zero else "positive" - raise ResearchBlockedError(f"{field} must be {qualifier}") - return parsed - - -def decimal_text(value: Decimal) -> str: - rendered = format(value, "f") - if "." in rendered: - rendered = rendered.rstrip("0").rstrip(".") - return rendered or "0" - - -def bps(numerator: Decimal, denominator: Decimal) -> str: - value = ( - (numerator / denominator - Decimal(1)) * Decimal(10000) - ).quantize(Decimal("0.01"), rounding=ROUND_HALF_EVEN) - return format(value, "f") - - -def load_job(path: Path) -> dict[str, Any]: - try: - raw = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as exc: - raise ResearchBlockedError(f"job must be readable JSON: {path}") from exc - if not isinstance(raw, dict): - raise ResearchBlockedError("job must contain one JSON object") - exact_fields(raw, JOB_FIELDS, "job") - if raw["schema"] != JOB_SCHEMA: - raise ResearchBlockedError(f"job.schema must be {JOB_SCHEMA}") - market = str(raw["market"]).upper() - if market not in MARKET_CONTRACTS: - raise ResearchBlockedError("job.market must be KR or US") - session_date = require_date(raw["session_date"], "job.session_date") - manifest_path = require_absolute_path(raw["manifest_path"], "job.manifest_path") - if not manifest_path.is_file() or manifest_path.is_symlink(): - raise ResearchBlockedError( - "job.manifest_path must be a regular non-symlink file" - ) - output_directory = require_absolute_path( - raw["output_directory"], "job.output_directory" - ) - if output_directory.exists() and ( - not output_directory.is_dir() or output_directory.is_symlink() - ): - raise ResearchBlockedError( - "job.output_directory must be a non-symlink directory" - ) - environment = str(raw["environment"]).lower() - if environment not in shared.BASE_URLS: - raise ResearchBlockedError("job.environment must be live or paper") - interval = raw["request_interval_ms"] - if ( - isinstance(interval, bool) - or not isinstance(interval, int) - or interval < (1000 if environment == "paper" else 100) - ): - floor = 1000 if environment == "paper" else 100 - raise ResearchBlockedError( - f"job.request_interval_ms must be an integer >= {floor}" - ) - try: - manifest_raw = json.loads(manifest_path.read_text(encoding="utf-8")) - manifest = screen.normalize_manifest_v2(manifest_raw) - except ( - OSError, - json.JSONDecodeError, - screen.ScreenBlockedError, - ) as exc: - raise ResearchBlockedError(f"manifest validation failed: {exc}") from exc - if manifest["as_of"] != session_date: - raise ResearchBlockedError( - "manifest.as_of must equal the research session_date" - ) - if manifest["analysis_date"] >= session_date: - raise ResearchBlockedError( - "manifest.analysis_date must precede the research session_date" - ) - instruments = [ - instrument - for instrument in manifest["instruments"] - if instrument["market"] == market - ] - if not instruments: - raise ResearchBlockedError("manifest has no instruments for job.market") - return { - "schema": JOB_SCHEMA, - "manifest_path": str(manifest_path), - "manifest": manifest, - "market": market, - "session_date": session_date, - "output_directory": output_directory, - "environment": environment, - "request_interval_ms": interval, - "instruments": sorted( - instruments, - key=lambda item: ( - MARKET_CONTRACTS[market]["exchanges"].index(item["exchange"]), - item["canonical_symbol"], - ), - ), - } - - -def normalize_bar( - raw: Mapping[str, Any], - *, - market: str, - session_date: str, -) -> dict[str, str] | None: - compact_date = session_date.replace("-", "") - if market == "KR": - raw_date = str(raw.get("stck_bsop_date", "")).strip() - raw_time = str(raw.get("stck_cntg_hour", "")).strip() - if raw_date != compact_date: - return None - fields = { - "open": raw.get("stck_oprc"), - "high": raw.get("stck_hgpr"), - "low": raw.get("stck_lwpr"), - "close": raw.get("stck_prpr"), - "volume": raw.get("cntg_vol", "0") or "0", - "turnover": raw.get("acml_tr_pbmn", "0") or "0", - } - else: - raw_date = str(raw.get("xymd", "")).strip() - raw_time = str(raw.get("xhms", "")).strip() - if raw_date != compact_date: - return None - fields = { - "open": raw.get("open"), - "high": raw.get("high"), - "low": raw.get("low"), - "close": raw.get("last"), - "volume": raw.get("evol", "0") or "0", - "turnover": raw.get("eamt", "0") or "0", - } - if len(raw_time) != 6 or not raw_time.isdigit(): - raise ResearchBlockedError("KIS minute row has invalid local time") - output = { - "time": raw_time, - "open": decimal_text(decimal_value(fields["open"], "minute open")), - "high": decimal_text(decimal_value(fields["high"], "minute high")), - "low": decimal_text(decimal_value(fields["low"], "minute low")), - "close": decimal_text(decimal_value(fields["close"], "minute close")), - "volume": decimal_text( - decimal_value(fields["volume"], "minute volume", allow_zero=True) - ), - "turnover": decimal_text( - decimal_value(fields["turnover"], "minute turnover", allow_zero=True) - ), - } - high = Decimal(output["high"]) - low = Decimal(output["low"]) - if high < max( - Decimal(output["open"]), - low, - Decimal(output["close"]), - ): - raise ResearchBlockedError("KIS minute row high is inconsistent") - if low > min( - Decimal(output["open"]), - high, - Decimal(output["close"]), - ): - raise ResearchBlockedError("KIS minute row low is inconsistent") - return output - - -def normalized_window_bars( - raw_rows: Any, - *, - market: str, - session_date: str, -) -> list[dict[str, str]]: - if not isinstance(raw_rows, list) or any( - not isinstance(item, dict) for item in raw_rows - ): - raise ResearchBlockedError("KIS minute output must be an object array") - contract = MARKET_CONTRACTS[market] - by_time: dict[str, dict[str, str]] = {} - for raw in raw_rows: - bar = normalize_bar( - raw, - market=market, - session_date=session_date, - ) - if bar is None: - continue - if not ( - contract["window_start"] - <= bar["time"] - <= contract["window_end"] - ): - continue - existing = by_time.get(bar["time"]) - if existing is not None and existing != bar: - raise ResearchBlockedError( - f"conflicting duplicate minute bar at {bar['time']}" - ) - by_time[bar["time"]] = bar - bars = [by_time[key] for key in sorted(by_time)] - if not bars: - raise ResearchBlockedError("KIS response has no first-hour bars") - return bars - - -def minute_of_day(value: str) -> int: - if len(value) != 6 or not value.isdigit(): - raise ResearchBlockedError("bar time must be HHMMSS") - hour = int(value[:2]) - minute = int(value[2:4]) - second = int(value[4:]) - if hour > 23 or minute > 59 or second != 0: - raise ResearchBlockedError("bar time must be a whole valid minute") - return hour * 60 + minute - - -def coverage_summary( - bars: list[dict[str, str]], - *, - market: str, -) -> dict[str, Any]: - contract = MARKET_CONTRACTS[market] - first_time = bars[0]["time"] - last_time = bars[-1]["time"] - first_minute = minute_of_day(first_time) - last_minute = minute_of_day(last_time) - start_minute = minute_of_day(contract["window_start"]) - end_minute = minute_of_day(contract["window_end"]) - expected = int(contract["expected_bars"]) - interval = int(contract["interval_minutes"]) - if ( - first_time == contract["window_start"] - and last_time == contract["window_end"] - and len(bars) == expected - ): - status = "COMPLETE_GRID" - elif ( - first_time == contract["window_start"] - and last_time == contract["window_end"] - ): - status = "EXACT_ENDPOINTS_SPARSE" - elif ( - first_minute <= start_minute + interval - and last_minute >= end_minute - interval - ): - status = "NEAR_ENDPOINTS" - else: - status = "PARTIAL_WINDOW" - return { - "coverage_status": status, - "first_bar_time": first_time, - "last_bar_time": last_time, - "expected_bar_count": expected, - "missing_bar_count": max(0, expected - len(bars)), - "coverage_fraction": format( - Decimal(len(bars)) / Decimal(expected), "f" - ), - "observed_span_minutes": last_minute - first_minute, - } - - -def summarize_bars( - bars: list[dict[str, str]], - *, - market: str, -) -> dict[str, Any]: - start_open = Decimal(bars[0]["open"]) - end_close = Decimal(bars[-1]["close"]) - window_high = max(Decimal(bar["high"]) for bar in bars) - window_low = min(Decimal(bar["low"]) for bar in bars) - window_volume = sum( - (Decimal(bar["volume"]) for bar in bars), Decimal(0) - ) - if market == "KR": - # The domestic endpoint exposes cumulative traded value on each bar. - window_turnover = Decimal(bars[-1]["turnover"]) - else: - window_turnover = sum( - (Decimal(bar["turnover"]) for bar in bars), Decimal(0) - ) - return { - "bar_count": len(bars), - "start_open": decimal_text(start_open), - "end_close": decimal_text(end_close), - "window_high": decimal_text(window_high), - "window_low": decimal_text(window_low), - "window_volume": decimal_text(window_volume), - "window_turnover": decimal_text(window_turnover), - "return_60m_bps": bps(end_close, start_open), - "maximum_excursion_bps": bps(window_high, start_open), - "minimum_excursion_bps": bps(window_low, start_open), - } - - -def fetch_raw_rows( - client: shared.KisReadClient, - *, - market: str, - exchange: str, - broker_symbol: str, - session_date: str, -) -> list[dict[str, Any]]: - contract = MARKET_CONTRACTS[market] - if market == "KR": - body = client.get( - contract["path"], - contract["tr_id"], - { - "FID_COND_MRKT_DIV_CODE": "J", - "FID_INPUT_ISCD": broker_symbol, - "FID_INPUT_HOUR_1": contract["window_end"], - "FID_INPUT_DATE_1": session_date.replace("-", ""), - "FID_PW_DATA_INCU_YN": "N", - "FID_FAKE_TICK_INCU_YN": "", - }, - ) - else: - body = client.get( - contract["path"], - contract["tr_id"], - { - "AUTH": "", - "EXCD": US_EXCHANGE_CODES[exchange], - "SYMB": broker_symbol, - "NMIN": str(contract["interval_minutes"]), - "PINC": "1", - "NEXT": "", - "NREC": "120", - "FILL": "", - "KEYB": "", - }, - ) - rows = body.get("output2") - if not isinstance(rows, list): - raise ResearchBlockedError("KIS minute response output2 must be an array") - return rows - - -def bars_artifact( - *, - instrument: dict[str, Any], - market: str, - session_date: str, - raw_rows: list[dict[str, Any]], -) -> dict[str, Any]: - contract = MARKET_CONTRACTS[market] - bars = normalized_window_bars( - raw_rows, - market=market, - session_date=session_date, - ) - summary = summarize_bars(bars, market=market) - output = { - "schema": BARS_SCHEMA, - "source": { - "provider": "KIS_OPEN_API", - "endpoint": contract["path"], - "tr_id": contract["tr_id"], - "read_only": True, - }, - "market": market, - "exchange": instrument["exchange"], - "canonical_symbol": instrument["canonical_symbol"], - "broker_symbol": instrument["broker_symbol"], - "session_date": session_date, - "interval_minutes": contract["interval_minutes"], - "window_start": contract["window_start"], - "window_end": contract["window_end"], - "bars": bars, - "bars_hash": shared.sha256_bytes( - shared.canonical_json(bars).encode("utf-8") - ), - "summary": summary, - } - return normalize_bars_artifact(output) - - -def normalize_bars_artifact(value: Any) -> dict[str, Any]: - if not isinstance(value, dict): - raise ResearchBlockedError("bars artifact must be an object") - exact_fields(value, BARS_FIELDS, "bars artifact") - if value["schema"] != BARS_SCHEMA: - raise ResearchBlockedError(f"bars artifact schema must be {BARS_SCHEMA}") - market = str(value["market"]).upper() - if market not in MARKET_CONTRACTS: - raise ResearchBlockedError("bars artifact market must be KR or US") - exchange = str(value["exchange"]).upper() - if exchange not in MARKET_CONTRACTS[market]["exchanges"]: - raise ResearchBlockedError("bars artifact exchange does not match market") - session_date = require_date(value["session_date"], "bars session_date") - source = value["source"] - if not isinstance(source, dict) or source != { - "provider": "KIS_OPEN_API", - "endpoint": MARKET_CONTRACTS[market]["path"], - "tr_id": MARKET_CONTRACTS[market]["tr_id"], - "read_only": True, - }: - raise ResearchBlockedError("bars source contract is invalid") - if value["interval_minutes"] != MARKET_CONTRACTS[market]["interval_minutes"]: - raise ResearchBlockedError("bars interval does not match market") - if value["window_start"] != MARKET_CONTRACTS[market]["window_start"]: - raise ResearchBlockedError("bars window_start does not match market") - if value["window_end"] != MARKET_CONTRACTS[market]["window_end"]: - raise ResearchBlockedError("bars window_end does not match market") - bars = value["bars"] - if not isinstance(bars, list): - raise ResearchBlockedError("bars must be an array") - for index, bar in enumerate(bars): - if not isinstance(bar, dict): - raise ResearchBlockedError(f"bars[{index}] must be an object") - exact_fields(bar, BAR_FIELDS, f"bars[{index}]") - normalized_bars = normalized_window_bars( - [ - ( - { - "stck_bsop_date": session_date.replace("-", ""), - "stck_cntg_hour": bar["time"], - "stck_oprc": bar["open"], - "stck_hgpr": bar["high"], - "stck_lwpr": bar["low"], - "stck_prpr": bar["close"], - "cntg_vol": bar["volume"], - "acml_tr_pbmn": bar["turnover"], - } - if market == "KR" - else { - "xymd": session_date.replace("-", ""), - "xhms": bar["time"], - "open": bar["open"], - "high": bar["high"], - "low": bar["low"], - "last": bar["close"], - "evol": bar["volume"], - "eamt": bar["turnover"], - } - ) - for bar in bars - ], - market=market, - session_date=session_date, - ) - expected_hash = shared.sha256_bytes( - shared.canonical_json(normalized_bars).encode("utf-8") - ) - if value["bars_hash"] != expected_hash: - raise ResearchBlockedError("bars_hash does not match bars") - summary = value["summary"] - if not isinstance(summary, dict): - raise ResearchBlockedError("bars summary must be an object") - exact_fields(summary, SUMMARY_FIELDS, "bars summary") - expected_summary = summarize_bars(normalized_bars, market=market) - if summary != expected_summary: - raise ResearchBlockedError("bars summary does not match bars") - canonical_symbol = str(value["canonical_symbol"]).strip() - broker_symbol = str(value["broker_symbol"]).strip() - if not canonical_symbol or not broker_symbol: - raise ResearchBlockedError("bars symbols must be non-empty") - return { - **value, - "market": market, - "exchange": exchange, - "canonical_symbol": canonical_symbol, - "broker_symbol": broker_symbol, - "session_date": session_date, - "bars": normalized_bars, - "bars_hash": expected_hash, - "summary": expected_summary, - } - - -def record_from_artifact(path: Path, artifact: dict[str, Any]) -> dict[str, Any]: - return { - "market": artifact["market"], - "exchange": artifact["exchange"], - "canonical_symbol": artifact["canonical_symbol"], - "broker_symbol": artifact["broker_symbol"], - "session_date": artifact["session_date"], - "interval_minutes": artifact["interval_minutes"], - "bars_path": str(path.resolve()), - "bars_file_sha256": shared.sha256_file(path), - "bars_hash": artifact["bars_hash"], - **coverage_summary(artifact["bars"], market=artifact["market"]), - **artifact["summary"], - } - - -def cache_path( - output_directory: Path, - instrument: dict[str, Any], -) -> Path: - return ( - output_directory - / "bars" - / instrument["exchange"] - / f"{instrument['canonical_symbol']}.json" - ) - - -def collect( - job: dict[str, Any], - *, - client: shared.KisReadClient | None = None, -) -> dict[str, Any]: - output_directory = job["output_directory"] - output_directory.mkdir(parents=True, exist_ok=True) - if client is None: - key, secret, token = shared.load_api_credentials(job["environment"]) - client = shared.KisReadClient( - environment=job["environment"], - app_key=key, - app_secret=secret, - access_token=token, - interval_ms=job["request_interval_ms"], - ) - records: list[dict[str, Any]] = [] - failures: list[dict[str, str]] = [] - cache_hits = 0 - instruments = job["instruments"] - for index, instrument in enumerate(instruments, start=1): - path = cache_path(output_directory, instrument) - try: - if path.exists(): - artifact = normalize_bars_artifact( - json.loads(path.read_text(encoding="utf-8")) - ) - if ( - artifact["market"] != job["market"] - or artifact["exchange"] != instrument["exchange"] - or artifact["canonical_symbol"] - != instrument["canonical_symbol"] - or artifact["broker_symbol"] != instrument["broker_symbol"] - or artifact["session_date"] != job["session_date"] - ): - raise ResearchBlockedError( - "cached bars identity does not match instrument" - ) - cache_hits += 1 - else: - raw_rows = fetch_raw_rows( - client, - market=job["market"], - exchange=instrument["exchange"], - broker_symbol=instrument["broker_symbol"], - session_date=job["session_date"], - ) - artifact = bars_artifact( - instrument=instrument, - market=job["market"], - session_date=job["session_date"], - raw_rows=raw_rows, - ) - shared.atomic_write_json(path, artifact) - records.append(record_from_artifact(path, artifact)) - except ( - OSError, - json.JSONDecodeError, - shared.EodBlockedError, - ResearchBlockedError, - ) as exc: - failures.append( - { - "market": job["market"], - "exchange": instrument["exchange"], - "canonical_symbol": instrument["canonical_symbol"], - "broker_symbol": instrument["broker_symbol"], - "reason": str(exc), - } - ) - if index % 100 == 0 or index == len(instruments): - print( - shared.canonical_json( - { - "progress": index, - "total": len(instruments), - "ready": len(records), - "blocked": len(failures), - "requests": client.request_count, - } - ), - file=sys.stderr, - flush=True, - ) - records.sort(key=lambda item: (item["exchange"], item["canonical_symbol"])) - failures.sort( - key=lambda item: (item["exchange"], item["canonical_symbol"]) - ) - coverage_counts = { - status: sum( - record["coverage_status"] == status for record in records - ) - for status in ( - "COMPLETE_GRID", - "EXACT_ENDPOINTS_SPARSE", - "NEAR_ENDPOINTS", - "PARTIAL_WINDOW", - ) - } - without_hash = { - "schema": SNAPSHOT_SCHEMA, - "source_skill": "quant-stock-technical", - "source_contract": { - "provider": "KIS_OPEN_API", - "read_only": True, - "market": job["market"], - "tr_id": MARKET_CONTRACTS[job["market"]]["tr_id"], - "endpoint": MARKET_CONTRACTS[job["market"]]["path"], - }, - "manifest_path": job["manifest_path"], - "manifest_hash": job["manifest"]["manifest_hash"], - "analysis_date": job["manifest"]["analysis_date"], - "session_date": job["session_date"], - "market": job["market"], - "window_start": MARKET_CONTRACTS[job["market"]]["window_start"], - "window_end": MARKET_CONTRACTS[job["market"]]["window_end"], - "interval_minutes": MARKET_CONTRACTS[job["market"]][ - "interval_minutes" - ], - "instrument_count": len(instruments), - "ready_count": len(records), - "blocked_count": len(failures), - "cache_hit_count": cache_hits, - "request_count": client.request_count, - "retry_count": client.retry_count, - "coverage_counts": coverage_counts, - "records": records, - "failures": failures, - "api_mutation_count": 0, - "live_enabled": False, - } - snapshot = { - **without_hash, - "snapshot_hash": shared.sha256_bytes( - shared.canonical_json(without_hash).encode("utf-8") - ), - } - shared.atomic_write_json(output_directory / "open1h-snapshot.json", snapshot) - return snapshot - - -def synthetic_rows( - *, - market: str, - session_date: str, -) -> list[dict[str, str]]: - contract = MARKET_CONTRACTS[market] - start_hour = 9 - start_minute = 0 if market == "KR" else 30 - rows: list[dict[str, str]] = [] - for index in range(contract["expected_bars"]): - total_minutes = start_hour * 60 + start_minute + ( - index * contract["interval_minutes"] - ) - stamp = f"{total_minutes // 60:02d}{total_minutes % 60:02d}00" - opening = Decimal("100") + Decimal(index) / Decimal("10") - row = { - "open": decimal_text(opening), - "high": decimal_text(opening + Decimal("1")), - "low": decimal_text(opening - Decimal("1")), - "close": decimal_text(opening + Decimal("0.5")), - "volume": "10", - "turnover": str(1000 + index), - } - if market == "KR": - rows.append( - { - "stck_bsop_date": session_date.replace("-", ""), - "stck_cntg_hour": stamp, - "stck_oprc": row["open"], - "stck_hgpr": row["high"], - "stck_lwpr": row["low"], - "stck_prpr": row["close"], - "cntg_vol": row["volume"], - "acml_tr_pbmn": row["turnover"], - } - ) - else: - rows.append( - { - "xymd": session_date.replace("-", ""), - "xhms": stamp, - "open": row["open"], - "high": row["high"], - "low": row["low"], - "last": row["close"], - "evol": row["volume"], - "eamt": row["turnover"], - } - ) - return list(reversed(rows)) - - -def self_test() -> None: - for market in ("KR", "US"): - exchange = MARKET_CONTRACTS[market]["exchanges"][0] - instrument = { - "exchange": exchange, - "canonical_symbol": "005930" if market == "KR" else "TEST", - "broker_symbol": "005930" if market == "KR" else "TEST", - } - artifact = bars_artifact( - instrument=instrument, - market=market, - session_date="2026-07-28", - raw_rows=synthetic_rows( - market=market, - session_date="2026-07-28", - ), - ) - assert artifact["summary"]["bar_count"] == ( - MARKET_CONTRACTS[market]["expected_bars"] - ) - assert Decimal(artifact["summary"]["return_60m_bps"]) > 0 - with tempfile.TemporaryDirectory( - prefix="qta-open1h-research-" - ) as temporary: - path = Path(temporary) / "bars.json" - shared.atomic_write_json(path, artifact) - loaded = normalize_bars_artifact( - json.loads(path.read_text(encoding="utf-8")) - ) - assert loaded == artifact - print( - shared.canonical_json( - { - "self_test": "PASS", - "schema": SNAPSHOT_SCHEMA, - "markets": ["KR", "US"], - "api_mutation_count": 0, - "live_enabled": False, - } - ) - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--self-test", action="store_true") - subparsers = parser.add_subparsers(dest="command") - collect_parser = subparsers.add_parser("collect") - collect_parser.add_argument("--job", required=True) - return parser.parse_args() - - -def main() -> int: - args = parse_args() - try: - if args.self_test: - self_test() - return 0 - if args.command != "collect": - raise ResearchBlockedError("use collect --job or --self-test") - job_path = require_absolute_path(args.job, "job path") - result = collect(load_job(job_path)) - except ( - ResearchBlockedError, - shared.EodBlockedError, - screen.ScreenBlockedError, - OSError, - json.JSONDecodeError, - ) as exc: - print( - shared.canonical_json( - { - "status": "BLOCKED", - "reason": str(exc), - "api_mutation_count": 0, - "live_enabled": False, - } - ), - file=sys.stderr, - ) - return 2 - print(shared.canonical_json(result)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/skills/quant-stock-technical/scripts/evaluate_open1h_research.py b/skills/quant-stock-technical/scripts/evaluate_open1h_research.py deleted file mode 100644 index 88bf812..0000000 --- a/skills/quant-stock-technical/scripts/evaluate_open1h_research.py +++ /dev/null @@ -1,721 +0,0 @@ -#!/usr/bin/env python3 -"""Evaluate frozen QTA scores against completed first-60-minute outcomes.""" - -from __future__ import annotations - -import argparse -import csv -import json -import math -import statistics -import sys -import tempfile -from decimal import Decimal, InvalidOperation, ROUND_CEILING, ROUND_HALF_EVEN -from pathlib import Path -from typing import Any, Iterable - -import collect_open1h_research as intraday -import fetch_kis_kr_eod as shared -import screen_universe as screen - - -JOB_SCHEMA = "qta-open1h-evaluation-job/v1" -DATASET_SCHEMA = "qta-open1h-evaluation/v1" -JOB_FIELDS = { - "schema", - "screen_path", - "snapshot_path", - "output_path", - "top_fraction", -} -SNAPSHOT_FIELDS = { - "schema", - "source_skill", - "source_contract", - "manifest_path", - "manifest_hash", - "analysis_date", - "session_date", - "market", - "window_start", - "window_end", - "interval_minutes", - "instrument_count", - "ready_count", - "blocked_count", - "cache_hit_count", - "request_count", - "retry_count", - "coverage_counts", - "records", - "failures", - "api_mutation_count", - "live_enabled", - "snapshot_hash", -} - - -class EvaluationBlockedError(ValueError): - """Raised when a QTA/opening-hour comparison would be incomplete.""" - - -def exact_fields(value: dict[str, Any], expected: set[str], label: str) -> None: - if set(value) != expected: - raise EvaluationBlockedError( - f"{label} fields mismatch; missing={sorted(expected - set(value))}, " - f"extra={sorted(set(value) - expected)}" - ) - - -def decimal_value(value: Any, field: str) -> Decimal: - if isinstance(value, bool): - raise EvaluationBlockedError(f"{field} must be decimal") - try: - parsed = Decimal(str(value)) - except (InvalidOperation, ValueError) as exc: - raise EvaluationBlockedError(f"{field} must be decimal") from exc - if not parsed.is_finite(): - raise EvaluationBlockedError(f"{field} must be finite") - return parsed - - -def positive_fraction(value: Any, field: str) -> Decimal: - parsed = decimal_value(value, field) - if parsed <= 0 or parsed >= 1: - raise EvaluationBlockedError(f"{field} must be greater than 0 and below 1") - return parsed - - -def absolute_file(value: Any, field: str, *, must_exist: bool) -> Path: - path = Path(str(value)).expanduser() - if not path.is_absolute(): - raise EvaluationBlockedError(f"{field} must be absolute") - if must_exist and (not path.is_file() or path.is_symlink()): - raise EvaluationBlockedError( - f"{field} must be a regular non-symlink file" - ) - return path - - -def load_object(path: Path, label: str) -> dict[str, Any]: - try: - value = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as exc: - raise EvaluationBlockedError(f"{label} must be readable JSON") from exc - if not isinstance(value, dict): - raise EvaluationBlockedError(f"{label} must be one JSON object") - return value - - -def load_job(path: Path) -> dict[str, Any]: - raw = load_object(path, "evaluation job") - exact_fields(raw, JOB_FIELDS, "evaluation job") - if raw["schema"] != JOB_SCHEMA: - raise EvaluationBlockedError(f"job.schema must be {JOB_SCHEMA}") - screen_path = absolute_file(raw["screen_path"], "screen_path", must_exist=True) - snapshot_path = absolute_file( - raw["snapshot_path"], "snapshot_path", must_exist=True - ) - output_path = absolute_file(raw["output_path"], "output_path", must_exist=False) - return { - "schema": JOB_SCHEMA, - "screen_path": screen_path, - "snapshot_path": snapshot_path, - "output_path": output_path, - "top_fraction": positive_fraction(raw["top_fraction"], "top_fraction"), - } - - -def normalized_screen(path: Path) -> dict[str, Any]: - value = load_object(path, "screen") - supplied_hash = value.get("screen_hash") - if not isinstance(supplied_hash, str): - raise EvaluationBlockedError("screen_hash is required") - without_hash = {key: item for key, item in value.items() if key != "screen_hash"} - if screen.sha256_json(without_hash) != supplied_hash: - raise EvaluationBlockedError("screen_hash does not match screen") - if ( - value.get("schema") != screen.SCREEN_SCHEMA_V2 - or value.get("source_skill") != "quant-stock-technical" - or value.get("method_version") not in {"qta-1.0.0", "qta-2.0.0"} - ): - raise EvaluationBlockedError( - "evaluation requires a qta-screen/v2 QTA 1.0.0 or 2.0.0 artifact" - ) - decisions = value.get("decisions") - if not isinstance(decisions, list) or not decisions: - raise EvaluationBlockedError("screen decisions must be a non-empty array") - return value - - -def normalized_snapshot(path: Path) -> dict[str, Any]: - value = load_object(path, "opening-hour snapshot") - exact_fields(value, SNAPSHOT_FIELDS, "opening-hour snapshot") - if value["schema"] != intraday.SNAPSHOT_SCHEMA: - raise EvaluationBlockedError( - f"snapshot.schema must be {intraday.SNAPSHOT_SCHEMA}" - ) - if value["api_mutation_count"] != 0 or value["live_enabled"] is not False: - raise EvaluationBlockedError("research snapshot must be read-only") - supplied_hash = value["snapshot_hash"] - without_hash = { - key: item for key, item in value.items() if key != "snapshot_hash" - } - expected_hash = shared.sha256_bytes( - shared.canonical_json(without_hash).encode("utf-8") - ) - if supplied_hash != expected_hash: - raise EvaluationBlockedError("snapshot_hash does not match snapshot") - if ( - value["instrument_count"] - != value["ready_count"] + value["blocked_count"] - ): - raise EvaluationBlockedError("snapshot counts do not reconcile") - if value["ready_count"] != len(value["records"]): - raise EvaluationBlockedError("snapshot ready_count does not match records") - if value["blocked_count"] != len(value["failures"]): - raise EvaluationBlockedError("snapshot blocked_count does not match failures") - return value - - -def eod_features( - ticker_path: Path, - *, - analysis_date: str, -) -> dict[str, str]: - try: - with ticker_path.open("r", encoding="utf-8-sig", newline="") as handle: - rows = list(csv.DictReader(handle)) - except OSError as exc: - raise EvaluationBlockedError(f"cannot read ticker CSV: {ticker_path}") from exc - selected = [row for row in rows if str(row.get("date")) <= analysis_date] - if not selected or str(selected[-1].get("date")) != analysis_date: - raise EvaluationBlockedError( - f"ticker CSV has no completed {analysis_date} cutoff" - ) - if len(selected) < 253: - raise EvaluationBlockedError("ticker CSV needs at least 253 cutoff rows") - closes = [ - decimal_value(row.get("close"), "ticker close") for row in selected - ] - volumes = [ - decimal_value(row.get("volume"), "ticker volume") for row in selected - ] - dollar_turnover = [ - closes[index] * volumes[index] - for index in range(max(0, len(selected) - 20), len(selected)) - ] - return { - "previous_close": format(closes[-1], "f"), - "return_5_bps": intraday.bps(closes[-1], closes[-6]), - "return_20_bps": intraday.bps(closes[-1], closes[-21]), - "return_63_bps": intraday.bps(closes[-1], closes[-64]), - "median_20d_dollar_turnover": format( - statistics.median(dollar_turnover), "f" - ), - } - - -def first_bar_close_cross( - bars: list[dict[str, str]], - entry: Decimal, -) -> str | None: - previous: Decimal | None = None - for bar in bars: - current = Decimal(bar["close"]) - if previous is not None and previous < entry <= current: - return bar["time"] - previous = current - return None - - -def combine_rows( - *, - screen_value: dict[str, Any], - snapshot: dict[str, Any], -) -> tuple[list[dict[str, Any]], list[dict[str, str]]]: - if screen_value["manifest_hash"] != snapshot["manifest_hash"]: - raise EvaluationBlockedError("screen and snapshot manifest hashes differ") - if screen_value["analysis_date"] != snapshot["analysis_date"]: - raise EvaluationBlockedError("screen and snapshot analysis dates differ") - decision_by_key: dict[tuple[str, str], dict[str, Any]] = {} - for decision in screen_value["decisions"]: - if not isinstance(decision, dict): - raise EvaluationBlockedError("screen decision must be an object") - key = (str(decision.get("exchange")), str(decision.get("canonical_symbol"))) - if key in decision_by_key: - raise EvaluationBlockedError(f"duplicate screen decision: {key}") - decision_by_key[key] = decision - rows: list[dict[str, Any]] = [] - blocked: list[dict[str, str]] = [] - for record in snapshot["records"]: - if not isinstance(record, dict): - raise EvaluationBlockedError("snapshot record must be an object") - key = (str(record.get("exchange")), str(record.get("canonical_symbol"))) - decision = decision_by_key.get(key) - if decision is None: - raise EvaluationBlockedError(f"snapshot record has no screen decision: {key}") - qta = decision.get("qta") - if ( - not isinstance(qta, dict) - or qta.get("calculation_status") != "READY" - or qta.get("method_version") != screen_value["method_version"] - ): - blocked.append( - { - "exchange": key[0], - "canonical_symbol": key[1], - "reason": str( - qta.get("reason", "QTA calculation is not READY") - if isinstance(qta, dict) - else "QTA payload is missing" - ), - } - ) - continue - bars_path = absolute_file( - record.get("bars_path"), "record.bars_path", must_exist=True - ) - if shared.sha256_file(bars_path) != record.get("bars_file_sha256"): - raise EvaluationBlockedError(f"bars file hash mismatch: {bars_path}") - artifact = intraday.normalize_bars_artifact( - load_object(bars_path, "bars artifact") - ) - if artifact["bars_hash"] != record.get("bars_hash"): - raise EvaluationBlockedError(f"bars hash mismatch: {bars_path}") - instrument = decision.get("instrument") - if not isinstance(instrument, dict): - raise EvaluationBlockedError("screen decision instrument is missing") - ticker_path = absolute_file( - instrument.get("ticker_csv"), "ticker_csv", must_exist=True - ) - if shared.sha256_file(ticker_path) != instrument.get("ticker_csv_sha256"): - raise EvaluationBlockedError(f"ticker CSV hash mismatch: {ticker_path}") - eod = eod_features( - ticker_path, - analysis_date=screen_value["analysis_date"], - ) - entry = decimal_value(qta["entry_price"], "QTA entry price") - start_open = Decimal(record["start_open"]) - end_close = Decimal(record["end_close"]) - cross_time = first_bar_close_cross(artifact["bars"], entry) - rows.append( - { - "market": decision["market"], - "exchange": decision["exchange"], - "canonical_symbol": decision["canonical_symbol"], - "broker_symbol": instrument["broker_symbol"], - "instrument_type": instrument["instrument_type"], - "analysis_date": screen_value["analysis_date"], - "session_date": snapshot["session_date"], - "qta_method_version": qta["method_version"], - "qta_total_score": format( - decimal_value(qta["total_score"], "QTA total score"), "f" - ), - "qta_short_score": format( - decimal_value(qta["short"]["score"], "QTA short score"), "f" - ), - "qta_medium_score": format( - decimal_value(qta["medium"]["score"], "QTA medium score"), "f" - ), - "qta_long_score": format( - decimal_value(qta["long"]["score"], "QTA long score"), "f" - ), - "qta_risk_score": format( - decimal_value(qta["risk"]["score"], "QTA risk score"), "f" - ), - "qta_entry_price": format(entry, "f"), - "qta_stop_price": format( - decimal_value(qta["stop_price"], "QTA stop price"), "f" - ), - "qta_take_profit_price": format( - decimal_value( - qta["take_profit_price"], "QTA take-profit price" - ), - "f", - ), - **eod, - "opening_gap_bps": intraday.bps( - start_open, Decimal(eod["previous_close"]) - ), - "coverage_status": record["coverage_status"], - "first_bar_time": record["first_bar_time"], - "last_bar_time": record["last_bar_time"], - "bar_count": record["bar_count"], - "expected_bar_count": record["expected_bar_count"], - "missing_bar_count": record["missing_bar_count"], - "coverage_fraction": record["coverage_fraction"], - "observed_span_minutes": record["observed_span_minutes"], - "return_60m_bps": record["return_60m_bps"], - "maximum_excursion_bps": record["maximum_excursion_bps"], - "minimum_excursion_bps": record["minimum_excursion_bps"], - "window_volume": record["window_volume"], - "window_turnover": record["window_turnover"], - "entry_distance_from_previous_close_bps": intraday.bps( - entry, Decimal(eod["previous_close"]) - ), - "entry_touched_by_minute_high": ( - Decimal(record["window_high"]) >= entry - ), - "entry_crossed_by_bar_close": cross_time is not None, - "entry_cross_bar_time": cross_time, - "end_return_from_entry_bps": intraday.bps(end_close, entry), - } - ) - rows.sort(key=lambda item: (item["exchange"], item["canonical_symbol"])) - blocked.sort(key=lambda item: (item["exchange"], item["canonical_symbol"])) - return rows, blocked - - -def average_ranks(values: list[Decimal]) -> list[float]: - order = sorted(range(len(values)), key=lambda index: (values[index], index)) - ranks = [0.0] * len(values) - cursor = 0 - while cursor < len(order): - end = cursor + 1 - while end < len(order) and values[order[end]] == values[order[cursor]]: - end += 1 - average = (cursor + 1 + end) / 2.0 - for position in range(cursor, end): - ranks[order[position]] = average - cursor = end - return ranks - - -def pearson(left: Iterable[float], right: Iterable[float]) -> float | None: - x = list(left) - y = list(right) - if len(x) != len(y) or len(x) < 2: - return None - x_mean = sum(x) / len(x) - y_mean = sum(y) / len(y) - numerator = sum((a - x_mean) * (b - y_mean) for a, b in zip(x, y)) - x_scale = math.sqrt(sum((a - x_mean) ** 2 for a in x)) - y_scale = math.sqrt(sum((b - y_mean) ** 2 for b in y)) - if x_scale == 0 or y_scale == 0: - return None - return numerator / (x_scale * y_scale) - - -def rounded_metric(value: float | None) -> str | None: - if value is None: - return None - return format( - Decimal(str(value)).quantize( - Decimal("0.000001"), rounding=ROUND_HALF_EVEN - ), - "f", - ) - - -def group_summary(rows: list[dict[str, Any]]) -> dict[str, Any]: - returns = [Decimal(row["return_60m_bps"]) for row in rows] - touched = sum(bool(row["entry_touched_by_minute_high"]) for row in rows) - crossed = sum(bool(row["entry_crossed_by_bar_close"]) for row in rows) - positive = sum(value > 0 for value in returns) - return { - "count": len(rows), - "mean_return_60m_bps": ( - format(sum(returns, Decimal(0)) / len(returns), "f") - if returns - else None - ), - "median_return_60m_bps": ( - format(statistics.median(returns), "f") if returns else None - ), - "positive_fraction": ( - format(Decimal(positive) / len(rows), "f") if rows else None - ), - "entry_touch_fraction": ( - format(Decimal(touched) / len(rows), "f") if rows else None - ), - "bar_close_cross_fraction": ( - format(Decimal(crossed) / len(rows), "f") if rows else None - ), - } - - -def exchange_diagnostics( - rows: list[dict[str, Any]], - *, - top_fraction: Decimal, -) -> dict[str, Any]: - if not rows: - raise EvaluationBlockedError("exchange diagnostics require rows") - qta_scores = [Decimal(row["qta_total_score"]) for row in rows] - returns = [Decimal(row["return_60m_bps"]) for row in rows] - pearson_value = pearson( - [float(value) for value in qta_scores], - [float(value) for value in returns], - ) - spearman_value = pearson( - average_ranks(qta_scores), - average_ranks(returns), - ) - top_count = max( - 1, - int( - (Decimal(len(rows)) * top_fraction).to_integral_value( - rounding=ROUND_CEILING - ) - ), - ) - by_outcome = sorted( - rows, - key=lambda item: ( - -Decimal(item["return_60m_bps"]), - item["canonical_symbol"], - ), - ) - by_score = sorted( - rows, - key=lambda item: ( - -Decimal(item["qta_total_score"]), - item["canonical_symbol"], - ), - ) - winner_keys = { - row["canonical_symbol"] for row in by_outcome[:top_count] - } - score_keys = {row["canonical_symbol"] for row in by_score[:top_count]} - overlap = len(winner_keys & score_keys) - baseline = Decimal(top_count) / Decimal(len(rows)) - precision = Decimal(overlap) / Decimal(top_count) - lift = precision / baseline - deciles: list[dict[str, Any]] = [] - for index in range(10): - start = len(by_score) * index // 10 - end = len(by_score) * (index + 1) // 10 - bucket = by_score[start:end] - deciles.append( - { - "qta_score_decile": index + 1, - **group_summary(bucket), - } - ) - - def compact(row: dict[str, Any]) -> dict[str, Any]: - return { - key: row[key] - for key in ( - "canonical_symbol", - "qta_total_score", - "qta_short_score", - "qta_medium_score", - "qta_long_score", - "qta_risk_score", - "return_5_bps", - "return_20_bps", - "median_20d_dollar_turnover", - "opening_gap_bps", - "coverage_status", - "first_bar_time", - "last_bar_time", - "bar_count", - "return_60m_bps", - "maximum_excursion_bps", - "minimum_excursion_bps", - "entry_touched_by_minute_high", - "entry_crossed_by_bar_close", - "end_return_from_entry_bps", - ) - } - - return { - "count": len(rows), - "top_fraction": format(top_fraction, "f"), - "top_count": top_count, - "pearson_qta_total_vs_return_60m": rounded_metric(pearson_value), - "spearman_qta_total_vs_return_60m": rounded_metric(spearman_value), - "top_outcome_overlap_count": overlap, - "top_outcome_precision": format(precision, "f"), - "top_outcome_recall": format(precision, "f"), - "top_outcome_lift": format(lift, "f"), - "overall": group_summary(rows), - "qta_score_deciles": deciles, - "top_20_outcomes": [compact(row) for row in by_outcome[:20]], - "bottom_20_outcomes": [compact(row) for row in by_outcome[-20:]], - "top_20_qta_scores": [compact(row) for row in by_score[:20]], - } - - -def evaluate(job: dict[str, Any]) -> dict[str, Any]: - screen_value = normalized_screen(job["screen_path"]) - snapshot = normalized_snapshot(job["snapshot_path"]) - rows, qta_blocked = combine_rows( - screen_value=screen_value, - snapshot=snapshot, - ) - if not rows: - raise EvaluationBlockedError("no READY QTA/opening-hour joined rows") - analysis_rows = [ - row - for row in rows - if row["coverage_status"] - in { - "COMPLETE_GRID", - "EXACT_ENDPOINTS_SPARSE", - "NEAR_ENDPOINTS", - } - ] - if not analysis_rows: - raise EvaluationBlockedError( - "no joined rows have complete or near-complete opening-hour endpoints" - ) - diagnostics = { - exchange: exchange_diagnostics( - [row for row in analysis_rows if row["exchange"] == exchange], - top_fraction=job["top_fraction"], - ) - for exchange in intraday.MARKET_CONTRACTS[snapshot["market"]]["exchanges"] - if any(row["exchange"] == exchange for row in analysis_rows) - } - without_hash = { - "schema": DATASET_SCHEMA, - "source_skill": "quant-stock-technical", - "baseline_method_version": screen_value["method_version"], - "target_definition": ( - "last observed close divided by first observed open; no interpolation; " - "COMPLETE_GRID, EXACT_ENDPOINTS_SPARSE, and NEAR_ENDPOINTS only" - ), - "market": snapshot["market"], - "analysis_date": snapshot["analysis_date"], - "session_date": snapshot["session_date"], - "screen_path": str(job["screen_path"]), - "screen_hash": screen_value["screen_hash"], - "snapshot_path": str(job["snapshot_path"]), - "snapshot_hash": snapshot["snapshot_hash"], - "top_fraction": format(job["top_fraction"], "f"), - "joined_count": len(rows), - "analysis_eligible_count": len(analysis_rows), - "coverage_counts": snapshot["coverage_counts"], - "intraday_blocked_count": snapshot["blocked_count"], - "qta_blocked_after_join_count": len(qta_blocked), - "diagnostics_by_exchange": diagnostics, - "rows": rows, - "qta_blocked_after_join": qta_blocked, - "api_mutation_count": 0, - "live_enabled": False, - } - output = { - **without_hash, - "evaluation_hash": shared.sha256_bytes( - shared.canonical_json(without_hash).encode("utf-8") - ), - } - shared.atomic_write_json(job["output_path"], output) - return output - - -def self_test() -> None: - sample = [ - { - "canonical_symbol": f"T{index}", - "qta_total_score": str(index), - "qta_short_score": str(index), - "qta_medium_score": str(index), - "qta_long_score": str(index), - "qta_risk_score": str(100 - index), - "return_5_bps": str(index), - "return_20_bps": str(index), - "median_20d_dollar_turnover": str(index * 1000000), - "opening_gap_bps": "0", - "coverage_status": "COMPLETE_GRID", - "first_bar_time": "093000", - "last_bar_time": "103000", - "bar_count": 13, - "return_60m_bps": str(index * 2), - "maximum_excursion_bps": str(index * 3), - "minimum_excursion_bps": str(-index), - "entry_touched_by_minute_high": index % 2 == 0, - "entry_crossed_by_bar_close": index % 3 == 0, - "end_return_from_entry_bps": str(index), - } - for index in range(1, 21) - ] - diagnostics = exchange_diagnostics(sample, top_fraction=Decimal("0.1")) - assert diagnostics["spearman_qta_total_vs_return_60m"] == "1.000000" - assert diagnostics["top_outcome_overlap_count"] == 2 - bars = intraday.synthetic_rows(market="US", session_date="2026-07-28") - artifact = intraday.bars_artifact( - instrument={ - "exchange": "NASDAQ", - "canonical_symbol": "TEST", - "broker_symbol": "TEST", - }, - market="US", - session_date="2026-07-28", - raw_rows=bars, - ) - assert first_bar_close_cross(artifact["bars"], Decimal("101.2")) is not None - with tempfile.TemporaryDirectory(prefix="qta-open1h-evaluation-") as temporary: - output = Path(temporary) / "result.json" - shared.atomic_write_json(output, {"self_test": "PASS"}) - assert load_object(output, "self-test")["self_test"] == "PASS" - print( - shared.canonical_json( - { - "self_test": "PASS", - "schema": DATASET_SCHEMA, - "api_mutation_count": 0, - "live_enabled": False, - } - ) - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--self-test", action="store_true") - subparsers = parser.add_subparsers(dest="command") - evaluate_parser = subparsers.add_parser("evaluate") - evaluate_parser.add_argument("--job", required=True) - return parser.parse_args() - - -def main() -> int: - args = parse_args() - try: - if args.self_test: - self_test() - return 0 - if args.command != "evaluate": - raise EvaluationBlockedError("use evaluate --job or --self-test") - job_path = absolute_file(args.job, "job path", must_exist=True) - result = evaluate(load_job(job_path)) - except ( - EvaluationBlockedError, - intraday.ResearchBlockedError, - screen.ScreenBlockedError, - OSError, - json.JSONDecodeError, - ) as exc: - print( - shared.canonical_json( - { - "status": "BLOCKED", - "reason": str(exc), - "api_mutation_count": 0, - "live_enabled": False, - } - ), - file=sys.stderr, - ) - return 2 - print( - shared.canonical_json( - { - "status": "READY", - "market": result["market"], - "joined_count": result["joined_count"], - "evaluation_hash": result["evaluation_hash"], - "api_mutation_count": 0, - "live_enabled": False, - } - ) - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/skills/quant-stock-technical/scripts/fetch_kis_kr_eod.py b/skills/quant-stock-technical/scripts/fetch_kis_kr_eod.py deleted file mode 100644 index 12181da..0000000 --- a/skills/quant-stock-technical/scripts/fetch_kis_kr_eod.py +++ /dev/null @@ -1,1253 +0,0 @@ -#!/usr/bin/env python3 -"""Build a resumable, source-hashed Korean adjusted-EOD bundle from KIS.""" - -from __future__ import annotations - -import argparse -import csv -import hashlib -import json -import os -import re -import stat -import sys -import tempfile -import time -import zipfile -from datetime import date, datetime, timedelta, timezone -from decimal import Decimal, InvalidOperation -from pathlib import Path -from typing import Any, Callable, Mapping, Optional -from urllib.error import HTTPError, URLError -from urllib.parse import urlencode -from urllib.request import Request, urlopen - -import build_universe_manifest as universe - - -JOB_SCHEMA = "qta-kis-kr-eod-job/v1" -RECEIPT_SCHEMA = "qta-kis-kr-eod-receipt/v1" -SNAPSHOT_SCHEMA = "qta-kr-source-snapshot/v1" -CATALOG_FIELDS = universe.EOD_CATALOG_FIELDS -KR_EXCHANGES = ("KOSPI", "KOSDAQ") -JOB_FIELDS = { - "schema", - "as_of", - "analysis_date", - "environment", - "output_directory", - "history_start_date", - "minimum_sessions", - "request_interval_ms", - "official_sources", - "broker_sources", - "catalog_coverage_contract", - "base_eod_catalog", -} -KIND_URL = ( - "https://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13" -) -KIS_MASTER_URLS = { - "KOSPI": ("https://new.real.download.dws.co.kr/common/master/kospi_code.mst.zip"), - "KOSDAQ": ("https://new.real.download.dws.co.kr/common/master/kosdaq_code.mst.zip"), -} -KIS_MASTER_NAMES = { - "KOSPI": "kospi_code.mst", - "KOSDAQ": "kosdaq_code.mst", -} -BASE_URLS = { - "live": "https://openapi.koreainvestment.com:9443", - "paper": "https://openapivts.koreainvestment.com:29443", -} -TOKEN_PATH = "/oauth2/tokenP" -STOCK_PATH = "/uapi/domestic-stock/v1/quotations/inquire-daily-itemchartprice" -INDEX_PATH = "/uapi/domestic-stock/v1/quotations/inquire-daily-indexchartprice" -STOCK_TR_ID = "FHKST03010100" -INDEX_TR_ID = "FHKUP03500100" -INDEX_CODES = {"KOSPI": "0001", "KOSDAQ": "1001"} -BENCHMARK_IDS = { - "KOSPI": "KOSPI_COMPOSITE", - "KOSDAQ": "KOSDAQ_COMPOSITE", -} -DEFAULT_SECRETS_PATH = Path("~/.config/mrcha-skills/secrets.env").expanduser() -DOTENV_KEY = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") - - -class EodBlockedError(ValueError): - """Raised when an EOD bundle cannot be completed without guessing.""" - - -def canonical_json(value: Any) -> str: - return json.dumps( - value, - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - allow_nan=False, - ) - - -def sha256_bytes(value: bytes) -> str: - return hashlib.sha256(value).hexdigest() - - -def sha256_file(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def atomic_write(path: Path, payload: bytes) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - descriptor, temporary = tempfile.mkstemp( - prefix=f".{path.name}.", suffix=".tmp", dir=str(path.parent) - ) - temporary_path = Path(temporary) - try: - with os.fdopen(descriptor, "wb") as handle: - handle.write(payload) - handle.flush() - os.fsync(handle.fileno()) - os.replace(temporary_path, path) - finally: - if temporary_path.exists(): - temporary_path.unlink() - - -def atomic_write_json(path: Path, value: Any) -> None: - atomic_write( - path, - (json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode( - "utf-8" - ), - ) - - -def require_exact_fields( - value: Mapping[str, Any], expected: set[str], label: str -) -> None: - if set(value) != expected: - raise EodBlockedError( - f"{label} fields mismatch; " - f"missing={sorted(expected - set(value))}, " - f"extra={sorted(set(value) - expected)}" - ) - - -def require_iso_date(value: Any, label: str) -> str: - if not isinstance(value, str): - raise EodBlockedError(f"{label} must be YYYY-MM-DD") - try: - date.fromisoformat(value) - except ValueError as exc: - raise EodBlockedError(f"{label} must be YYYY-MM-DD") from exc - return value - - -def require_positive_integer(value: Any, label: str) -> int: - if isinstance(value, bool) or not isinstance(value, int) or value <= 0: - raise EodBlockedError(f"{label} must be a positive integer") - return value - - -def normalize_job(raw: Any, job_directory: Path) -> dict[str, Any]: - if not isinstance(raw, dict): - raise EodBlockedError("job must be one JSON object") - require_exact_fields(raw, JOB_FIELDS, "job") - if raw["schema"] != JOB_SCHEMA: - raise EodBlockedError(f"job.schema must be {JOB_SCHEMA}") - as_of = require_iso_date(raw["as_of"], "job.as_of") - analysis_date = require_iso_date(raw["analysis_date"], "job.analysis_date") - history_start = require_iso_date( - raw["history_start_date"], "job.history_start_date" - ) - if history_start >= analysis_date: - raise EodBlockedError("history_start_date must be before analysis_date") - if as_of < analysis_date: - raise EodBlockedError("as_of cannot be before analysis_date") - environment = str(raw["environment"]).lower() - if environment not in BASE_URLS: - raise EodBlockedError("environment must be live or paper") - output_directory = Path(str(raw["output_directory"])).expanduser() - if not output_directory.is_absolute(): - output_directory = (job_directory / output_directory).resolve() - minimum_sessions = require_positive_integer( - raw["minimum_sessions"], "job.minimum_sessions" - ) - if minimum_sessions < 756: - raise EodBlockedError("minimum_sessions must be at least 756") - request_interval_ms = require_positive_integer( - raw["request_interval_ms"], "job.request_interval_ms" - ) - if environment == "paper" and request_interval_ms < 1000: - raise EodBlockedError("paper request_interval_ms must be at least 1000") - if environment == "live" and request_interval_ms < 100: - raise EodBlockedError("live request_interval_ms must be at least 100") - - def normalize_sources(role: str) -> list[dict[str, Any]]: - sources = raw[role] - if not isinstance(sources, list): - raise EodBlockedError(f"job.{role} must be an array") - normalized: list[dict[str, Any]] = [] - seen: set[str] = set() - for index, source in enumerate(sources): - if not isinstance(source, dict): - raise EodBlockedError(f"job.{role}[{index}] must be an object") - item = universe.normalize_source_descriptor( - source, - ("OFFICIAL_MASTER" if role == "official_sources" else "BROKER_MASTER"), - as_of, - job_directory, - index, - ) - if item["exchange"] in seen: - raise EodBlockedError(f"duplicate {role} exchange {item['exchange']}") - seen.add(item["exchange"]) - normalized.append(item) - missing = set(KR_EXCHANGES) - seen - if missing: - raise EodBlockedError(f"job.{role} missing KR exchanges {sorted(missing)}") - return sorted(normalized, key=lambda item: item["exchange"]) - - base_catalog = str(raw["base_eod_catalog"] or "").strip() - if base_catalog: - base_path = Path(base_catalog).expanduser() - if not base_path.is_absolute(): - base_path = (job_directory / base_path).resolve() - if not base_path.is_file(): - raise EodBlockedError(f"base_eod_catalog is not a file: {base_path}") - base_catalog = str(base_path) - - coverage = universe.normalize_catalog_coverage_contract( - raw["catalog_coverage_contract"] - ) - return { - "schema": JOB_SCHEMA, - "as_of": as_of, - "analysis_date": analysis_date, - "environment": environment, - "output_directory": str(output_directory), - "history_start_date": history_start, - "minimum_sessions": minimum_sessions, - "request_interval_ms": request_interval_ms, - "official_sources": normalize_sources("official_sources"), - "broker_sources": normalize_sources("broker_sources"), - "catalog_coverage_contract": coverage, - "base_eod_catalog": base_catalog, - } - - -def build_spec_sources( - sources: list[dict[str, Any]], -) -> list[dict[str, Any]]: - """Project normalized records back to the exact build-spec input schema.""" - return [ - { - field: source[field] - for field in sorted(universe.SOURCE_FIELDS) - } - for source in sources - ] - - -def inspect_secrets_file(path: Path) -> bool: - try: - metadata = path.lstat() - except FileNotFoundError: - return False - if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode): - raise EodBlockedError("KIS secrets path must be a regular non-symlink file") - if metadata.st_uid != os.getuid() or stat.S_IMODE(metadata.st_mode) != 0o600: - raise EodBlockedError("KIS secrets file must be user-owned mode 0600") - return True - - -def parse_dotenv(path: Path) -> dict[str, str]: - if not inspect_secrets_file(path): - return {} - output: dict[str, str] = {} - for line_number, raw_line in enumerate( - path.read_text(encoding="utf-8").splitlines(), start=1 - ): - stripped = raw_line.strip() - if not stripped or stripped.startswith("#"): - continue - if stripped.startswith("export "): - stripped = stripped[7:].lstrip() - if "=" not in stripped: - raise EodBlockedError(f"invalid dotenv line {line_number}") - key, value = stripped.split("=", 1) - key = key.strip() - if not DOTENV_KEY.fullmatch(key) or key in output: - raise EodBlockedError( - f"invalid or duplicate dotenv key on line {line_number}" - ) - value = value.strip() - if value.startswith('"'): - try: - decoded = json.loads(value) - except json.JSONDecodeError as exc: - raise EodBlockedError( - f"invalid quoted dotenv value on line {line_number}" - ) from exc - if not isinstance(decoded, str): - raise EodBlockedError(f"dotenv line {line_number} is not a string") - value = decoded - elif value.startswith("'"): - if len(value) < 2 or not value.endswith("'"): - raise EodBlockedError( - f"invalid quoted dotenv value on line {line_number}" - ) - value = value[1:-1] - output[key] = value - return output - - -def load_api_credentials(environment: str) -> tuple[str, str, Optional[str]]: - scope = "LIVE" if environment == "live" else "PAPER" - path = Path( - os.environ.get("QTA_SECRETS_FILE", str(DEFAULT_SECRETS_PATH)) - ).expanduser() - dotenv = parse_dotenv(path) - - def value(generic: str, scoped: str) -> Optional[str]: - return ( - os.environ.get(generic) - or os.environ.get(scoped) - or dotenv.get(scoped) - or dotenv.get(generic) - ) - - app_key = value("QTA_KIS_APP_KEY", f"QTA_KIS_{scope}_APP_KEY") - app_secret = value("QTA_KIS_APP_SECRET", f"QTA_KIS_{scope}_APP_SECRET") - access_token = value("QTA_KIS_ACCESS_TOKEN", f"QTA_KIS_{scope}_ACCESS_TOKEN") - missing = [ - name - for name, item in ( - ("QTA_KIS_APP_KEY", app_key), - ("QTA_KIS_APP_SECRET", app_secret), - ) - if not item - ] - if missing: - raise EodBlockedError("missing KIS credentials: " + ", ".join(missing)) - return str(app_key), str(app_secret), access_token - - -class KisReadClient: - """Read-only KIS HTTP client with deterministic pacing and bounded retries.""" - - def __init__( - self, - *, - environment: str, - app_key: str, - app_secret: str, - interval_ms: int, - access_token: Optional[str] = None, - transport: Optional[ - Callable[[str, str, Mapping[str, str], Optional[dict[str, Any]]], Any] - ] = None, - ) -> None: - self.base_url = BASE_URLS[environment] - self.app_key = app_key - self.app_secret = app_secret - self.interval_seconds = interval_ms / 1000.0 - self.access_token = access_token - self.last_request_at = 0.0 - self.request_count = 0 - self.retry_count = 0 - self.transport = transport or self._urllib_transport - - @staticmethod - def _urllib_transport( - method: str, - url: str, - headers: Mapping[str, str], - body: Optional[dict[str, Any]], - ) -> Any: - payload = None - if body is not None: - payload = canonical_json(body).encode("utf-8") - request = Request(url, data=payload, headers=dict(headers), method=method) - try: - with urlopen(request, timeout=15) as response: - raw = response.read() - status_code = int(response.status) - except HTTPError as exc: - raw = exc.read() - status_code = int(exc.code) - except (URLError, TimeoutError, OSError) as exc: - raise EodBlockedError(f"KIS network failure: {type(exc).__name__}") from exc - try: - decoded = json.loads(raw.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise EodBlockedError(f"KIS returned non-JSON HTTP {status_code}") from exc - return status_code, decoded - - def _pace(self) -> None: - now = time.monotonic() - remaining = self.interval_seconds - (now - self.last_request_at) - if remaining > 0: - time.sleep(remaining) - self.last_request_at = time.monotonic() - - def token(self) -> str: - if self.access_token: - return self.access_token - for attempt in range(2): - status, body = self.transport( - "POST", - f"{self.base_url}{TOKEN_PATH}", - {"Content-Type": "application/json"}, - { - "grant_type": "client_credentials", - "appkey": self.app_key, - "appsecret": self.app_secret, - }, - ) - self.request_count += 1 - if status == 200 and isinstance(body, dict) and body.get("access_token"): - self.access_token = str(body["access_token"]) - return self.access_token - code = ( - str(body.get("error_code", body.get("msg_cd", "unknown"))) - if isinstance(body, dict) - else "invalid_body" - ) - if code == "EGW00133" and attempt == 0: - self.retry_count += 1 - time.sleep(61.0) - continue - raise EodBlockedError(f"KIS token request failed: HTTP {status} {code}") - raise AssertionError("bounded KIS token loop fell through") - - def get(self, path: str, tr_id: str, params: Mapping[str, str]) -> dict[str, Any]: - query = urlencode(sorted(params.items())) - headers = { - "authorization": f"Bearer {self.token()}", - "appkey": self.app_key, - "appsecret": self.app_secret, - "tr_id": tr_id, - "custtype": "P", - "Content-Type": "application/json", - } - for attempt in range(4): - self._pace() - status, body = self.transport( - "GET", f"{self.base_url}{path}?{query}", headers, None - ) - self.request_count += 1 - code = ( - str(body.get("msg_cd", body.get("error_code", ""))) - if isinstance(body, dict) - else "" - ) - if status == 200 and isinstance(body, dict) and body.get("rt_cd") == "0": - return body - if status == 429 or code == "EGW00201": - if attempt == 3: - break - self.retry_count += 1 - time.sleep(float(2**attempt)) - continue - message = ( - str(body.get("msg1", body.get("error_description", code))) - if isinstance(body, dict) - else "invalid_body" - ) - raise EodBlockedError( - f"KIS read failed: HTTP {status} {code or 'unknown'} {message}" - ) - raise EodBlockedError("KIS read rate limit persisted after bounded retries") - - -def decimal_text(value: Any, label: str, *, allow_zero: bool = False) -> str: - try: - parsed = Decimal(str(value).replace(",", "").strip()) - except (InvalidOperation, ValueError) as exc: - raise EodBlockedError(f"{label} is not decimal") from exc - if not parsed.is_finite() or (parsed < 0 if allow_zero else parsed <= 0): - raise EodBlockedError( - f"{label} must be {'non-negative' if allow_zero else 'positive'}" - ) - rendered = format(parsed, "f") - return rendered.rstrip("0").rstrip(".") if "." in rendered else rendered - - -def normalize_stock_row(raw: Mapping[str, Any]) -> dict[str, str]: - try: - day = datetime.strptime(str(raw["stck_bsop_date"]), "%Y%m%d").date() - except (KeyError, ValueError) as exc: - raise EodBlockedError("KIS stock row has invalid stck_bsop_date") from exc - output = { - "date": day.isoformat(), - "open": decimal_text(raw.get("stck_oprc"), "stck_oprc"), - "high": decimal_text(raw.get("stck_hgpr"), "stck_hgpr"), - "low": decimal_text(raw.get("stck_lwpr"), "stck_lwpr"), - "close": decimal_text(raw.get("stck_clpr"), "stck_clpr"), - "volume": decimal_text(raw.get("acml_vol", "0"), "acml_vol", allow_zero=True), - } - output["adjusted_close"] = output["close"] - if Decimal(output["high"]) < max( - Decimal(output["open"]), Decimal(output["low"]), Decimal(output["close"]) - ): - raise EodBlockedError("KIS stock row high is inconsistent") - if Decimal(output["low"]) > min( - Decimal(output["open"]), Decimal(output["high"]), Decimal(output["close"]) - ): - raise EodBlockedError("KIS stock row low is inconsistent") - return output - - -def normalize_index_row(raw: Mapping[str, Any]) -> dict[str, str]: - day_value = raw.get("stck_bsop_date", raw.get("bstp_nmix_prdy_vrss_sign")) - try: - day = datetime.strptime(str(day_value), "%Y%m%d").date() - except ValueError as exc: - raise EodBlockedError("KIS index row has invalid stck_bsop_date") from exc - output = { - "date": day.isoformat(), - "open": decimal_text(raw.get("bstp_nmix_oprc"), "bstp_nmix_oprc"), - "high": decimal_text(raw.get("bstp_nmix_hgpr"), "bstp_nmix_hgpr"), - "low": decimal_text(raw.get("bstp_nmix_lwpr"), "bstp_nmix_lwpr"), - "close": decimal_text(raw.get("bstp_nmix_prpr"), "bstp_nmix_prpr"), - "volume": decimal_text( - raw.get("acml_vol", "0") or "0", "acml_vol", allow_zero=True - ), - } - output["adjusted_close"] = output["close"] - if Decimal(output["high"]) < max( - Decimal(output["open"]), Decimal(output["low"]), Decimal(output["close"]) - ): - raise EodBlockedError("KIS index row high is inconsistent") - if Decimal(output["low"]) > min( - Decimal(output["open"]), Decimal(output["high"]), Decimal(output["close"]) - ): - raise EodBlockedError("KIS index row low is inconsistent") - return output - - -def fetch_history( - client: KisReadClient, - *, - kind: str, - symbol: str, - start: date, - end: date, -) -> list[dict[str, str]]: - if kind == "stock": - path, tr_id, market_code = STOCK_PATH, STOCK_TR_ID, "J" - parser = normalize_stock_row - elif kind == "index": - path, tr_id, market_code = INDEX_PATH, INDEX_TR_ID, "U" - parser = normalize_index_row - else: - raise EodBlockedError("history kind must be stock or index") - cursor = end - output: dict[str, dict[str, str]] = {} - for _ in range(32): - params = { - "FID_COND_MRKT_DIV_CODE": market_code, - "FID_INPUT_ISCD": symbol, - "FID_INPUT_DATE_1": start.strftime("%Y%m%d"), - "FID_INPUT_DATE_2": cursor.strftime("%Y%m%d"), - "FID_PERIOD_DIV_CODE": "D", - } - if kind == "stock": - params["FID_ORG_ADJ_PRC"] = "0" - body = client.get(path, tr_id, params) - rows = body.get("output2") - if not isinstance(rows, list): - raise EodBlockedError("KIS history output2 must be an array") - if not rows: - break - normalized = [parser(row) for row in rows if isinstance(row, dict)] - if not normalized: - raise EodBlockedError("KIS history page has no valid rows") - for row in normalized: - day = date.fromisoformat(row["date"]) - if start <= day <= end: - existing = output.get(row["date"]) - if existing is not None and existing != row: - raise EodBlockedError( - f"KIS returned conflicting duplicate date {row['date']}" - ) - output[row["date"]] = row - oldest = min(date.fromisoformat(row["date"]) for row in normalized) - if oldest <= start: - break - next_cursor = oldest - timedelta(days=1) - if next_cursor >= cursor: - raise EodBlockedError("KIS history pagination did not move backward") - cursor = next_cursor - else: - raise EodBlockedError("KIS history exceeded 32 pages") - return [output[key] for key in sorted(output)] - - -def read_csv_rows(path: Path) -> list[dict[str, str]]: - if not path.is_file(): - return [] - with path.open("r", encoding="utf-8", newline="") as handle: - reader = csv.DictReader(handle) - if tuple(reader.fieldnames or ()) != ( - "date", - "open", - "high", - "low", - "close", - "adjusted_close", - "volume", - ): - raise EodBlockedError(f"cached EOD header mismatch: {path}") - rows = [dict(row) for row in reader] - normalized: list[dict[str, str]] = [] - for row in rows: - canonical = { - "date": require_iso_date(row["date"], f"{path}.date"), - "open": decimal_text(row["open"], f"{path}.open"), - "high": decimal_text(row["high"], f"{path}.high"), - "low": decimal_text(row["low"], f"{path}.low"), - "close": decimal_text(row["close"], f"{path}.close"), - "adjusted_close": decimal_text( - row["adjusted_close"], f"{path}.adjusted_close" - ), - "volume": decimal_text(row["volume"], f"{path}.volume", allow_zero=True), - } - normalized.append(canonical) - if len({row["date"] for row in normalized}) != len(normalized): - raise EodBlockedError(f"cached EOD contains duplicate dates: {path}") - return sorted(normalized, key=lambda row: row["date"]) - - -def write_csv_rows(path: Path, rows: list[dict[str, str]]) -> None: - fields = ( - "date", - "open", - "high", - "low", - "close", - "adjusted_close", - "volume", - ) - with tempfile.SpooledTemporaryFile( - mode="w+", encoding="utf-8", newline="" - ) as handle: - writer = csv.DictWriter(handle, fieldnames=fields, lineterminator="\n") - writer.writeheader() - writer.writerows(rows) - handle.seek(0) - atomic_write(path, handle.read().encode("utf-8")) - - -def update_history_file( - client: KisReadClient, - *, - kind: str, - symbol: str, - path: Path, - start: date, - end: date, - minimum_sessions: int, -) -> tuple[list[dict[str, str]], int]: - cached = read_csv_rows(path) - cached_map = { - row["date"]: row - for row in cached - if start <= date.fromisoformat(row["date"]) <= end - } - before_requests = client.request_count - if end.isoformat() not in cached_map or len(cached_map) < minimum_sessions: - fetched = fetch_history(client, kind=kind, symbol=symbol, start=start, end=end) - for row in fetched: - existing = cached_map.get(row["date"]) - if existing is not None and existing != row: - cached_map[row["date"]] = row - else: - cached_map[row["date"]] = row - rows = [cached_map[key] for key in sorted(cached_map)] - if end.isoformat() not in cached_map: - raise EodBlockedError(f"{symbol} has no completed row on {end.isoformat()}") - if len(rows) < minimum_sessions: - raise EodBlockedError( - f"{symbol} needs {minimum_sessions} sessions; found {len(rows)}" - ) - if rows != cached: - write_csv_rows(path, rows) - return rows, client.request_count - before_requests - - -def tick_size(exchange: str, price: Decimal) -> Decimal: - if exchange == "KOSPI": - bands = ( - (Decimal("1000"), Decimal("1")), - (Decimal("5000"), Decimal("5")), - (Decimal("10000"), Decimal("10")), - (Decimal("50000"), Decimal("50")), - (Decimal("100000"), Decimal("100")), - (Decimal("500000"), Decimal("500")), - ) - fallback = Decimal("1000") - elif exchange == "KOSDAQ": - bands = ( - (Decimal("1000"), Decimal("1")), - (Decimal("5000"), Decimal("5")), - (Decimal("10000"), Decimal("10")), - (Decimal("50000"), Decimal("50")), - ) - fallback = Decimal("100") - else: - raise EodBlockedError("tick exchange must be KOSPI or KOSDAQ") - for ceiling, tick in bands: - if price < ceiling: - return tick - return fallback - - -def catalog_type(reasons: list[str]) -> str: - mappings = ( - ("instrument_etf", "ETF"), - ("instrument_etn", "ETN"), - ("instrument_preferred", "PREFERRED"), - ("instrument_spac", "SPAC"), - ("instrument_unit", "UNIT"), - ("instrument_right", "RIGHT"), - ("instrument_warrant", "WARRANT"), - ("test_issue", "TEST"), - ("abnormal_status", "ABNORMAL"), - ) - for reason, rendered in mappings: - if reason in reasons: - return rendered - return "COMMON" - - -def load_base_catalog(path: str) -> dict[tuple[str, str], dict[str, str]]: - if not path: - return {} - output: dict[tuple[str, str], dict[str, str]] = {} - with Path(path).open("r", encoding="utf-8", newline="") as handle: - reader = csv.DictReader(handle) - if tuple(reader.fieldnames or ()) != CATALOG_FIELDS: - raise EodBlockedError("base_eod_catalog header mismatch") - for row in reader: - exchange = str(row["exchange"]).upper() - symbol = str(row["canonical_symbol"]).upper() - if exchange not in universe.EXCHANGES or not symbol: - raise EodBlockedError( - "base_eod_catalog has invalid exchange or canonical_symbol" - ) - key = (exchange, symbol) - if key in output: - raise EodBlockedError(f"duplicate base catalog row {key}") - output[key] = {field: str(row[field]) for field in CATALOG_FIELDS} - return output - - -def write_catalog(path: Path, rows: list[dict[str, str]]) -> None: - with tempfile.SpooledTemporaryFile( - mode="w+", encoding="utf-8", newline="" - ) as handle: - writer = csv.DictWriter(handle, fieldnames=CATALOG_FIELDS, lineterminator="\n") - writer.writeheader() - writer.writerows(rows) - handle.seek(0) - atomic_write(path, handle.read().encode("utf-8")) - - -def build_bundle( - job: dict[str, Any], - *, - client: Optional[KisReadClient] = None, -) -> dict[str, Any]: - output_root = Path(job["output_directory"]) - output_root.mkdir(parents=True, exist_ok=True) - if client is None: - app_key, app_secret, access_token = load_api_credentials(job["environment"]) - client = KisReadClient( - environment=job["environment"], - app_key=app_key, - app_secret=app_secret, - access_token=access_token, - interval_ms=job["request_interval_ms"], - ) - start = date.fromisoformat(job["history_start_date"]) - end = date.fromisoformat(job["analysis_date"]) - minimum_sessions = int(job["minimum_sessions"]) - official_descriptors = { - item["exchange"]: item - for item in job["official_sources"] - if item["exchange"] in KR_EXCHANGES - } - broker_descriptors = { - item["exchange"]: item - for item in job["broker_sources"] - if item["exchange"] in KR_EXCHANGES - } - official_rows = { - exchange: { - row.symbol: row - for row in universe.parse_master_source(official_descriptors[exchange]) - } - for exchange in KR_EXCHANGES - } - broker_rows = { - exchange: { - row.symbol: row - for row in universe.parse_master_source(broker_descriptors[exchange]) - } - for exchange in KR_EXCHANGES - } - - benchmark_paths: dict[str, Path] = {} - benchmark_results: dict[str, dict[str, Any]] = {} - for exchange in KR_EXCHANGES: - benchmark_path = output_root / "benchmarks" / f"{BENCHMARK_IDS[exchange]}.csv" - rows, requests = update_history_file( - client, - kind="index", - symbol=INDEX_CODES[exchange], - path=benchmark_path, - start=start, - end=end, - minimum_sessions=minimum_sessions, - ) - benchmark_paths[exchange] = benchmark_path.resolve() - benchmark_results[exchange] = { - "symbol": INDEX_CODES[exchange], - "path": str(benchmark_path.resolve()), - "sha256": sha256_file(benchmark_path), - "sessions": len(rows), - "requests": requests, - } - - catalog = load_base_catalog(job["base_eod_catalog"]) - failures: list[dict[str, str]] = [] - eligible_count = 0 - ready_count = 0 - cache_hits = 0 - coverage_by_exchange = { - exchange: { - "official": len(official_rows[exchange]), - "eligible": 0, - "ready": 0, - "failed": 0, - } - for exchange in KR_EXCHANGES - } - for exchange in KR_EXCHANGES: - official_descriptor = official_descriptors[exchange] - broker_descriptor = broker_descriptors[exchange] - for symbol in sorted(official_rows[exchange]): - official = official_rows[exchange][symbol] - broker = broker_rows[exchange].get(symbol) - reasons = universe.metadata_exclusion_reasons(official, official_descriptor) - if broker is None: - reasons.append("not_kis_tradable") - else: - reasons.extend( - universe.metadata_exclusion_reasons(broker, broker_descriptor) - ) - reasons = sorted(set(reasons)) - instrument_type = catalog_type(reasons) - ticker_path = output_root / "stocks" / exchange / f"{symbol}.csv" - latest_close = "" - resolved_tick = "" - source_name = "KIS_OPEN_API_ADJUSTED_DAILY:FHKST03010100" - data_path = "" - if not reasons: - eligible_count += 1 - coverage_by_exchange[exchange]["eligible"] += 1 - before = client.request_count - try: - rows, _ = update_history_file( - client, - kind="stock", - symbol=symbol, - path=ticker_path, - start=start, - end=end, - minimum_sessions=minimum_sessions, - ) - if client.request_count == before: - cache_hits += 1 - latest_close = rows[-1]["close"] - resolved_tick = format( - tick_size(exchange, Decimal(latest_close)), "f" - ) - data_path = str(ticker_path.resolve()) - ready_count += 1 - coverage_by_exchange[exchange]["ready"] += 1 - except EodBlockedError as exc: - coverage_by_exchange[exchange]["failed"] += 1 - failures.append( - { - "exchange": exchange, - "symbol": symbol, - "reason": str(exc), - } - ) - catalog[(exchange, symbol)] = { - "exchange": exchange, - "canonical_symbol": symbol, - "data_symbol": symbol, - "broker_symbol": symbol if broker is not None else "", - "instrument_type": instrument_type, - "benchmark_id": BENCHMARK_IDS[exchange], - "ticker_csv": data_path, - "benchmark_csv": str(benchmark_paths[exchange]), - "tick_rule_id": ( - f"KRX_GUIDE_OBSERVED_{job['as_of']}_{exchange}" - if resolved_tick - else "" - ), - "tick_effective_date": job["analysis_date"] if resolved_tick else "", - "tick_reference_price": latest_close, - "resolved_tick_size": resolved_tick, - "source_name": source_name, - } - - catalog_rows = [ - catalog[key] - for key in sorted( - catalog, - key=lambda item: ( - universe.EXCHANGES.index(item[0]), - item[1], - ), - ) - ] - catalog_path = output_root / "eod-catalog.csv" - write_catalog(catalog_path, catalog_rows) - - build_spec = { - "schema": universe.BUILD_SPEC_SCHEMA, - "as_of": job["as_of"], - "analysis_date": job["analysis_date"], - "official_sources": build_spec_sources(job["official_sources"]), - "broker_sources": build_spec_sources(job["broker_sources"]), - "eod_catalog": { - "source_id": f"kis-eod-catalog-{job['as_of']}", - "provider": "KIS", - "as_of": job["as_of"], - "path": str(catalog_path.resolve()), - "encoding": "utf-8", - }, - "catalog_coverage_contract": job["catalog_coverage_contract"], - } - build_spec_path = output_root / "universe-build-spec.json" - atomic_write_json(build_spec_path, build_spec) - - source_hashes = [ - { - "source_id": item["source_id"], - "exchange": item["exchange"], - "role": role, - "path": item["path"], - "sha256": sha256_file(Path(item["path"])), - } - for role, sources in ( - ("OFFICIAL_MASTER", job["official_sources"]), - ("BROKER_MASTER", job["broker_sources"]), - ) - for item in sources - ] - source_hashes.sort(key=lambda item: item["source_id"]) - receipt_without_hash = { - "schema": RECEIPT_SCHEMA, - "status": "READY", - "as_of": job["as_of"], - "analysis_date": job["analysis_date"], - "environment": job["environment"], - "api_mutation_count": 0, - "adjusted_price_parameter": "FID_ORG_ADJ_PRC=0", - "minimum_sessions": minimum_sessions, - "eligible_symbols": eligible_count, - "ready_symbols": ready_count, - "cache_hits": cache_hits, - "coverage_by_exchange": coverage_by_exchange, - "failed_symbols": failures, - "request_count": client.request_count, - "retry_count": client.retry_count, - "benchmarks": benchmark_results, - "source_hashes": source_hashes, - "catalog": { - "path": str(catalog_path.resolve()), - "sha256": sha256_file(catalog_path), - "rows": len(catalog_rows), - }, - "build_spec": { - "path": str(build_spec_path.resolve()), - "sha256": sha256_file(build_spec_path), - }, - "completed_at": datetime.now(timezone.utc).isoformat(), - } - receipt = { - **receipt_without_hash, - "receipt_hash": sha256_bytes( - canonical_json(receipt_without_hash).encode("utf-8") - ), - } - atomic_write_json(output_root / "eod-bundle-receipt.json", receipt) - return receipt - - -def download_bytes(url: str) -> bytes: - request = Request( - url, - headers={ - "User-Agent": "mrcha-skills/quant-stock-technical", - "Accept": "*/*", - }, - ) - try: - with urlopen(request, timeout=30) as response: - return response.read() - except (HTTPError, URLError, TimeoutError, OSError) as exc: - raise EodBlockedError( - f"source snapshot download failed: {url} ({type(exc).__name__})" - ) from exc - - -def snapshot_sources(as_of: str, output_directory: Path) -> dict[str, Any]: - require_iso_date(as_of, "as_of") - output_directory.mkdir(parents=True, exist_ok=True) - kind_path = output_directory / "krx-kind-corporate-list.xls" - atomic_write(kind_path, download_bytes(KIND_URL)) - try: - text = kind_path.read_text(encoding="cp949") - except UnicodeDecodeError as exc: - raise EodBlockedError("KRX KIND snapshot is not CP949 HTML") from exc - if "종목코드" not in text or "시장구분" not in text: - raise EodBlockedError("KRX KIND snapshot is missing expected headers") - - master_paths: dict[str, Path] = {} - for exchange in KR_EXCHANGES: - payload = download_bytes(KIS_MASTER_URLS[exchange]) - with tempfile.TemporaryDirectory() as temporary: - archive_path = Path(temporary) / "master.zip" - archive_path.write_bytes(payload) - try: - with zipfile.ZipFile(archive_path) as archive: - names = archive.namelist() - expected = KIS_MASTER_NAMES[exchange] - matches = [ - name - for name in names - if Path(name).name.lower() == expected.lower() - ] - if len(matches) != 1: - raise EodBlockedError( - f"{exchange} master archive must contain {expected}" - ) - master_payload = archive.read(matches[0]) - except zipfile.BadZipFile as exc: - raise EodBlockedError( - f"{exchange} master download is not a ZIP" - ) from exc - master_path = output_directory / KIS_MASTER_NAMES[exchange] - atomic_write(master_path, master_payload) - master_paths[exchange] = master_path - - official_sources = [ - { - "source_id": f"krx-kind-{exchange.lower()}-{as_of}", - "provider": "KRX", - "exchange": exchange, - "as_of": as_of, - "path": str(kind_path.resolve()), - "format": "KRX_KIND_HTML", - "encoding": "cp949", - "delimiter": "", - "skip_rows": 0, - "columns": {}, - "normal_status_values": [], - } - for exchange in KR_EXCHANGES - ] - broker_sources = [ - { - "source_id": f"kis-master-{exchange.lower()}-{as_of}", - "provider": "KIS", - "exchange": exchange, - "as_of": as_of, - "path": str(master_paths[exchange].resolve()), - "format": "KIS_KRX_MASTER", - "encoding": "cp949", - "delimiter": "", - "skip_rows": 0, - "columns": {}, - "normal_status_values": [], - } - for exchange in KR_EXCHANGES - ] - receipt_without_hash = { - "schema": SNAPSHOT_SCHEMA, - "as_of": as_of, - "official_sources": official_sources, - "broker_sources": broker_sources, - "files": [ - { - "path": str(path.resolve()), - "sha256": sha256_file(path), - } - for path in [kind_path, *master_paths.values()] - ], - "source_urls": [KIND_URL, *KIS_MASTER_URLS.values()], - "captured_at": datetime.now(timezone.utc).isoformat(), - } - receipt = { - **receipt_without_hash, - "receipt_hash": sha256_bytes( - canonical_json(receipt_without_hash).encode("utf-8") - ), - } - atomic_write_json(output_directory / "source-snapshot.json", receipt) - return receipt - - -def _synthetic_rows(end: date, count: int, *, index: bool) -> list[dict[str, str]]: - rows: list[dict[str, str]] = [] - cursor = end - for offset in range(count): - while cursor.weekday() >= 5: - cursor -= timedelta(days=1) - close = Decimal("1000") + Decimal(offset) - if index: - rows.append( - { - "stck_bsop_date": cursor.strftime("%Y%m%d"), - "bstp_nmix_oprc": str(close), - "bstp_nmix_hgpr": str(close + 10), - "bstp_nmix_lwpr": str(close - 10), - "bstp_nmix_prpr": str(close), - "acml_vol": "0", - } - ) - else: - rows.append( - { - "stck_bsop_date": cursor.strftime("%Y%m%d"), - "stck_oprc": str(close), - "stck_hgpr": str(close + 10), - "stck_lwpr": str(close - 10), - "stck_clpr": str(close), - "acml_vol": str(100000 + offset), - } - ) - cursor -= timedelta(days=1) - return rows - - -def self_test() -> None: - end = date(2026, 7, 24) - stock_rows = _synthetic_rows(end, 800, index=False) - index_rows = _synthetic_rows(end, 800, index=True) - - def transport( - method: str, - url: str, - headers: Mapping[str, str], - body: Optional[dict[str, Any]], - ) -> Any: - del headers, body - if method == "POST": - return 200, {"access_token": "fixture-token"} - query = dict(item.split("=", 1) for item in url.split("?", 1)[1].split("&")) - cursor = datetime.strptime(query["FID_INPUT_DATE_2"], "%Y%m%d").date() - source = index_rows if INDEX_PATH in url else stock_rows - page_size = 50 if index_rows is source else 100 - selected = [ - row - for row in source - if datetime.strptime(row["stck_bsop_date"], "%Y%m%d").date() <= cursor - ][:page_size] - return 200, {"rt_cd": "0", "output2": selected} - - client = KisReadClient( - environment="live", - app_key="fixture", - app_secret="fixture", - interval_ms=100, - access_token="fixture-token", - transport=transport, - ) - client.interval_seconds = 0 - start = date(2023, 1, 1) - stocks = fetch_history(client, kind="stock", symbol="005930", start=start, end=end) - indexes = fetch_history(client, kind="index", symbol="0001", start=start, end=end) - assert len(stocks) >= 756 - assert len(indexes) >= 756 - assert stocks[-1]["date"] == end.isoformat() - assert stocks[-1]["adjusted_close"] == stocks[-1]["close"] - assert tick_size("KOSPI", Decimal("999")) == Decimal("1") - assert tick_size("KOSPI", Decimal("100000")) == Decimal("500") - assert tick_size("KOSDAQ", Decimal("500000")) == Decimal("100") - with tempfile.TemporaryDirectory() as temporary: - path = Path(temporary) / "history.csv" - write_csv_rows(path, stocks) - assert read_csv_rows(path) == stocks - print( - json.dumps( - { - "self_test": "PASS", - "stock_sessions": len(stocks), - "index_sessions": len(indexes), - "requests": client.request_count, - }, - sort_keys=True, - ) - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - parser.add_argument("--self-test", action="store_true") - subparsers = parser.add_subparsers(dest="command") - collect = subparsers.add_parser("collect") - collect.add_argument("--job", required=True) - snapshot = subparsers.add_parser("snapshot-sources") - snapshot.add_argument("--as-of", required=True) - snapshot.add_argument("--output-directory", required=True) - return parser.parse_args() - - -def main() -> int: - args = parse_args() - try: - if args.self_test: - self_test() - return 0 - if args.command == "snapshot-sources": - receipt = snapshot_sources( - args.as_of, Path(args.output_directory).expanduser().resolve() - ) - elif args.command == "collect": - job_path = Path(args.job).expanduser().resolve() - raw = json.loads(job_path.read_text(encoding="utf-8")) - receipt = build_bundle(normalize_job(raw, job_path.parent)) - else: - raise EodBlockedError("use collect --job, snapshot-sources, or --self-test") - except ( - EodBlockedError, - universe.UniverseBlockedError, - json.JSONDecodeError, - ) as exc: - print( - json.dumps( - {"status": "BLOCKED", "reason": str(exc)}, - ensure_ascii=False, - sort_keys=True, - ), - file=sys.stderr, - ) - return 2 - print(json.dumps(receipt, ensure_ascii=False, sort_keys=True)) - return 0 if receipt.get("status", "READY") == "READY" else 2 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/skills/quant-stock-technical/scripts/fetch_kis_us_eod.py b/skills/quant-stock-technical/scripts/fetch_kis_us_eod.py deleted file mode 100644 index 2c1db51..0000000 --- a/skills/quant-stock-technical/scripts/fetch_kis_us_eod.py +++ /dev/null @@ -1,997 +0,0 @@ -#!/usr/bin/env python3 -"""Build a resumable, source-hashed U.S. adjusted-EOD bundle from KIS.""" - -from __future__ import annotations - -import argparse -import json -import sys -import tempfile -import zipfile -from collections.abc import Callable, Mapping -from datetime import date, datetime, timedelta, timezone -from decimal import Decimal -from pathlib import Path -from typing import Any, Optional -from urllib.error import HTTPError, URLError -from urllib.parse import urlencode -from urllib.request import Request, urlopen - -import build_universe_manifest as universe -import fetch_kis_kr_eod as shared - -JOB_SCHEMA = "qta-kis-us-eod-job/v1" -RECEIPT_SCHEMA = "qta-kis-us-eod-receipt/v1" -SNAPSHOT_SCHEMA = "qta-us-source-snapshot/v1" -US_EXCHANGES = ("NYSE", "NASDAQ") -JOB_FIELDS = shared.JOB_FIELDS -STOCK_PATH = "/uapi/overseas-price/v1/quotations/dailyprice" -STOCK_TR_ID = "HHDFS76240000" -EXCHANGE_CODES = {"NYSE": "NYS", "NASDAQ": "NAS"} -BENCHMARK_IDS = {"NYSE": "NYSE_COMPOSITE", "NASDAQ": "NASDAQ_COMPOSITE"} -BENCHMARK_SYMBOLS = {"NYSE": "^NYA", "NASDAQ": "^IXIC"} -NASDAQ_URLS = { - "NASDAQ": "https://www.nasdaqtrader.com/dynamic/SymDir/nasdaqlisted.txt", - "NYSE": "https://www.nasdaqtrader.com/dynamic/SymDir/otherlisted.txt", -} -KIS_MASTER_URLS = { - "NASDAQ": "https://new.real.download.dws.co.kr/common/master/nasmst.cod.zip", - "NYSE": "https://new.real.download.dws.co.kr/common/master/nysmst.cod.zip", -} -KIS_MASTER_NAMES = {"NASDAQ": "nasmst.cod", "NYSE": "nysmst.cod"} -YAHOO_CHART_URL = "https://query1.finance.yahoo.com/v8/finance/chart" - - -class UsEodBlockedError(shared.EodBlockedError): - """Raised when a U.S. EOD bundle cannot be completed without guessing.""" - - -class InvalidOhlcGeometry(UsEodBlockedError): - """One parseable provider row has OHLC values that cannot be admitted.""" - - def __init__(self, reason: str, row: dict[str, str]): - super().__init__(reason) - self.reason = reason - self.row = row - - -def validate_ohlc_geometry(row: dict[str, str]) -> None: - if Decimal(row["high"]) < max( - Decimal(row["open"]), Decimal(row["low"]), Decimal(row["close"]) - ): - raise InvalidOhlcGeometry( - "KIS overseas row high is inconsistent", - row, - ) - if Decimal(row["low"]) > min( - Decimal(row["open"]), Decimal(row["high"]), Decimal(row["close"]) - ): - raise InvalidOhlcGeometry( - "KIS overseas row low is inconsistent", - row, - ) - - -def normalize_job(raw: Any, job_directory: Path) -> dict[str, Any]: - if not isinstance(raw, dict): - raise UsEodBlockedError("job must be one JSON object") - shared.require_exact_fields(raw, JOB_FIELDS, "job") - if raw["schema"] != JOB_SCHEMA: - raise UsEodBlockedError(f"job.schema must be {JOB_SCHEMA}") - as_of = shared.require_iso_date(raw["as_of"], "job.as_of") - analysis_date = shared.require_iso_date(raw["analysis_date"], "job.analysis_date") - history_start = shared.require_iso_date( - raw["history_start_date"], "job.history_start_date" - ) - if history_start >= analysis_date: - raise UsEodBlockedError("history_start_date must be before analysis_date") - if as_of < analysis_date: - raise UsEodBlockedError("as_of cannot be before analysis_date") - environment = str(raw["environment"]).lower() - if environment not in shared.BASE_URLS: - raise UsEodBlockedError("environment must be live or paper") - output_directory = Path(str(raw["output_directory"])).expanduser() - if not output_directory.is_absolute(): - output_directory = (job_directory / output_directory).resolve() - minimum_sessions = shared.require_positive_integer( - raw["minimum_sessions"], "job.minimum_sessions" - ) - if minimum_sessions < 756: - raise UsEodBlockedError("minimum_sessions must be at least 756") - interval = shared.require_positive_integer( - raw["request_interval_ms"], "job.request_interval_ms" - ) - if environment == "paper" and interval < 1000: - raise UsEodBlockedError("paper request_interval_ms must be at least 1000") - if environment == "live" and interval < 100: - raise UsEodBlockedError("live request_interval_ms must be at least 100") - - def sources_for(role: str) -> list[dict[str, Any]]: - raw_sources = raw[role] - if not isinstance(raw_sources, list): - raise UsEodBlockedError(f"job.{role} must be an array") - normalized: list[dict[str, Any]] = [] - seen: set[str] = set() - expected_role = ( - "OFFICIAL_MASTER" if role == "official_sources" else "BROKER_MASTER" - ) - for index, source in enumerate(raw_sources): - if not isinstance(source, dict): - raise UsEodBlockedError(f"job.{role}[{index}] must be an object") - item = universe.normalize_source_descriptor( - source, expected_role, as_of, job_directory, index - ) - if item["exchange"] in seen: - raise UsEodBlockedError(f"duplicate {role} exchange {item['exchange']}") - seen.add(item["exchange"]) - normalized.append(item) - missing = set(US_EXCHANGES) - seen - if missing: - raise UsEodBlockedError( - f"job.{role} missing U.S. exchanges {sorted(missing)}" - ) - return sorted( - normalized, key=lambda item: universe.EXCHANGES.index(item["exchange"]) - ) - - base_catalog = str(raw["base_eod_catalog"] or "").strip() - if base_catalog: - base_path = Path(base_catalog).expanduser() - if not base_path.is_absolute(): - base_path = (job_directory / base_path).resolve() - if not base_path.is_file(): - raise UsEodBlockedError(f"base_eod_catalog is not a file: {base_path}") - base_catalog = str(base_path) - coverage = universe.normalize_catalog_coverage_contract( - raw["catalog_coverage_contract"] - ) - return { - "schema": JOB_SCHEMA, - "as_of": as_of, - "analysis_date": analysis_date, - "environment": environment, - "output_directory": str(output_directory), - "history_start_date": history_start, - "minimum_sessions": minimum_sessions, - "request_interval_ms": interval, - "official_sources": sources_for("official_sources"), - "broker_sources": sources_for("broker_sources"), - "catalog_coverage_contract": coverage, - "base_eod_catalog": base_catalog, - } - - -def normalize_stock_row(raw: Mapping[str, Any]) -> dict[str, str]: - try: - day = datetime.strptime(str(raw["xymd"]), "%Y%m%d").date() - except (KeyError, ValueError) as exc: - raise UsEodBlockedError("KIS overseas row has invalid xymd") from exc - output = { - "date": day.isoformat(), - "open": shared.decimal_text(raw.get("open"), "open"), - "high": shared.decimal_text(raw.get("high"), "high"), - "low": shared.decimal_text(raw.get("low"), "low"), - "close": shared.decimal_text(raw.get("clos"), "clos"), - "volume": shared.decimal_text( - raw.get("tvol", "0") or "0", "tvol", allow_zero=True - ), - } - output["adjusted_close"] = output["close"] - validate_ohlc_geometry(output) - return output - - -def fetch_stock_history( - client: shared.KisReadClient, - *, - exchange: str, - symbol: str, - start: date, - end: date, - invalid_rows: Optional[list[dict[str, Any]]] = None, -) -> list[dict[str, str]]: - cursor = end - output: dict[str, dict[str, str]] = {} - for _ in range(32): - body = client.get( - STOCK_PATH, - STOCK_TR_ID, - { - "AUTH": "", - "EXCD": EXCHANGE_CODES[exchange], - "SYMB": symbol, - "GUBN": "0", - "BYMD": cursor.strftime("%Y%m%d"), - "MODP": "1", - }, - ) - rows = body.get("output2") - if not isinstance(rows, list): - raise UsEodBlockedError("KIS overseas output2 must be an array") - if not rows: - break - page_dates: list[date] = [] - normalized: list[dict[str, str]] = [] - for raw_row in rows: - if not isinstance(raw_row, dict): - continue - try: - page_dates.append( - datetime.strptime(str(raw_row["xymd"]), "%Y%m%d").date() - ) - except (KeyError, ValueError) as exc: - raise UsEodBlockedError( - "KIS overseas row has invalid xymd" - ) from exc - try: - normalized.append(normalize_stock_row(raw_row)) - except InvalidOhlcGeometry as exc: - invalid_day = date.fromisoformat(exc.row["date"]) - if invalid_day == end: - raise UsEodBlockedError( - f"{symbol} completed cutoff row has invalid OHLC geometry: " - f"{exc.reason}" - ) from exc - if invalid_rows is not None and start <= invalid_day <= end: - invalid_rows.append( - { - "date": exc.row["date"], - "reason": exc.reason, - "row": exc.row, - "row_sha256": shared.sha256_bytes( - shared.canonical_json(exc.row).encode("utf-8") - ), - } - ) - if not normalized: - if page_dates: - oldest = min(page_dates) - if oldest <= start: - break - next_cursor = oldest - timedelta(days=1) - if next_cursor >= cursor: - raise UsEodBlockedError( - "KIS overseas pagination did not move backward" - ) - cursor = next_cursor - continue - raise UsEodBlockedError("KIS overseas page has no valid rows") - for row in normalized: - day = date.fromisoformat(row["date"]) - if start <= day <= end: - existing = output.get(row["date"]) - if existing is not None and existing != row: - raise UsEodBlockedError( - f"KIS returned conflicting duplicate date {row['date']}" - ) - output[row["date"]] = row - oldest = min(page_dates) - if oldest <= start: - break - next_cursor = oldest - timedelta(days=1) - if next_cursor >= cursor: - raise UsEodBlockedError("KIS overseas pagination did not move backward") - cursor = next_cursor - else: - raise UsEodBlockedError("KIS overseas history exceeded 32 pages") - return [output[key] for key in sorted(output)] - - -def update_stock_file( - client: shared.KisReadClient, - *, - exchange: str, - symbol: str, - path: Path, - start: date, - end: date, - minimum_sessions: int, -) -> tuple[list[dict[str, str]], int, list[dict[str, Any]]]: - invalid_rows: list[dict[str, Any]] = [] - cached: list[dict[str, str]] = [] - cached_rows = shared.read_csv_rows(path) - for row in cached_rows: - try: - validate_ohlc_geometry(row) - except InvalidOhlcGeometry as exc: - invalid_day = date.fromisoformat(exc.row["date"]) - if invalid_day == end: - raise UsEodBlockedError( - f"{symbol} cached cutoff row has invalid OHLC geometry: " - f"{exc.reason}" - ) from exc - if start <= invalid_day <= end: - invalid_rows.append( - { - "date": exc.row["date"], - "reason": exc.reason, - "row": exc.row, - "row_sha256": shared.sha256_bytes( - shared.canonical_json(exc.row).encode("utf-8") - ), - } - ) - continue - cached.append(row) - merged = { - row["date"]: row - for row in cached - if start <= date.fromisoformat(row["date"]) <= end - } - before = client.request_count - if end.isoformat() not in merged or len(merged) < minimum_sessions: - for row in fetch_stock_history( - client, - exchange=exchange, - symbol=symbol, - start=start, - end=end, - invalid_rows=invalid_rows, - ): - merged[row["date"]] = row - rows = [merged[key] for key in sorted(merged)] - if end.isoformat() not in merged: - raise UsEodBlockedError(f"{symbol} has no completed row on {end.isoformat()}") - if len(rows) < minimum_sessions: - raise UsEodBlockedError( - f"{symbol} needs {minimum_sessions} sessions; found {len(rows)}" - ) - if rows != cached_rows: - shared.write_csv_rows(path, rows) - return rows, client.request_count - before, invalid_rows - - -def load_invalid_row_audit(path: Path) -> list[dict[str, Any]]: - if not path.exists(): - return [] - try: - value = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as exc: - raise UsEodBlockedError("invalid-row audit is not readable JSON") from exc - if ( - not isinstance(value, dict) - or value.get("schema") != "qta-kis-us-invalid-eod-rows/v1" - or not isinstance(value.get("records"), list) - ): - raise UsEodBlockedError("invalid-row audit has an unsupported schema") - return [item for item in value["records"] if isinstance(item, dict)] - - -def fetch_yahoo_benchmark(symbol: str, start: date, end: date) -> list[dict[str, str]]: - period1 = int( - datetime.combine(start, datetime.min.time(), timezone.utc).timestamp() - ) - period2 = int( - datetime.combine( - end + timedelta(days=1), datetime.min.time(), timezone.utc - ).timestamp() - ) - query = urlencode( - { - "period1": str(period1), - "period2": str(period2), - "interval": "1d", - "events": "div,splits", - } - ) - request = Request( - f"{YAHOO_CHART_URL}/{symbol}?{query}", - headers={"User-Agent": "mrcha-skills/quant-stock-technical"}, - ) - try: - with urlopen(request, timeout=30) as response: - payload = json.loads(response.read().decode("utf-8")) - except ( - HTTPError, - URLError, - TimeoutError, - OSError, - UnicodeDecodeError, - json.JSONDecodeError, - ) as exc: - raise UsEodBlockedError( - f"benchmark download failed for {symbol}: {type(exc).__name__}" - ) from exc - try: - result = payload["chart"]["result"][0] - timestamps = result["timestamp"] - quote = result["indicators"]["quote"][0] - adjusted = result["indicators"]["adjclose"][0]["adjclose"] - except (KeyError, IndexError, TypeError) as exc: - raise UsEodBlockedError(f"benchmark response malformed for {symbol}") from exc - rows: list[dict[str, str]] = [] - for index, stamp in enumerate(timestamps): - try: - day = datetime.fromtimestamp(int(stamp), timezone.utc).date() - close = Decimal(str(quote["close"][index])) - adjusted_close = Decimal(str(adjusted[index])) - ratio = adjusted_close / close - row = { - "date": day.isoformat(), - "open": shared.decimal_text( - Decimal(str(quote["open"][index])) * ratio, "benchmark.open" - ), - "high": shared.decimal_text( - Decimal(str(quote["high"][index])) * ratio, "benchmark.high" - ), - "low": shared.decimal_text( - Decimal(str(quote["low"][index])) * ratio, "benchmark.low" - ), - "close": shared.decimal_text(adjusted_close, "benchmark.close"), - "adjusted_close": shared.decimal_text( - adjusted_close, "benchmark.adjusted_close" - ), - "volume": shared.decimal_text( - quote["volume"][index] or "0", - "benchmark.volume", - allow_zero=True, - ), - } - except (IndexError, TypeError, ArithmeticError): - continue - if start <= day <= end: - rows.append(row) - if len({row["date"] for row in rows}) != len(rows): - raise UsEodBlockedError(f"benchmark contains duplicate dates for {symbol}") - return sorted(rows, key=lambda row: row["date"]) - - -def update_benchmark_file( - *, - symbol: str, - path: Path, - start: date, - end: date, - minimum_sessions: int, - loader: Callable[[str, date, date], list[dict[str, str]]], -) -> list[dict[str, str]]: - cached = shared.read_csv_rows(path) - merged = { - row["date"]: row - for row in cached - if start <= date.fromisoformat(row["date"]) <= end - } - if end.isoformat() not in merged or len(merged) < minimum_sessions: - for row in loader(symbol, start, end): - merged[row["date"]] = row - rows = [merged[key] for key in sorted(merged)] - if end.isoformat() not in merged: - raise UsEodBlockedError( - f"benchmark {symbol} has no completed row on {end.isoformat()}" - ) - if len(rows) < minimum_sessions: - raise UsEodBlockedError( - f"benchmark {symbol} needs {minimum_sessions} sessions; found {len(rows)}" - ) - if rows != cached: - shared.write_csv_rows(path, rows) - return rows - - -def tick_size(price: Decimal) -> Decimal: - if price <= 0: - raise UsEodBlockedError("reference price must be positive") - return Decimal("0.0001") if price < Decimal("1") else Decimal("0.01") - - -def build_bundle( - job: dict[str, Any], - *, - client: Optional[shared.KisReadClient] = None, - benchmark_loader: Callable[ - [str, date, date], list[dict[str, str]] - ] = fetch_yahoo_benchmark, -) -> dict[str, Any]: - output_root = Path(job["output_directory"]) - output_root.mkdir(parents=True, exist_ok=True) - if client is None: - key, secret, token = shared.load_api_credentials(job["environment"]) - client = shared.KisReadClient( - environment=job["environment"], - app_key=key, - app_secret=secret, - access_token=token, - interval_ms=job["request_interval_ms"], - ) - start = date.fromisoformat(job["history_start_date"]) - end = date.fromisoformat(job["analysis_date"]) - minimum = int(job["minimum_sessions"]) - official_descriptors = { - item["exchange"]: item - for item in job["official_sources"] - if item["exchange"] in US_EXCHANGES - } - broker_descriptors = { - item["exchange"]: item - for item in job["broker_sources"] - if item["exchange"] in US_EXCHANGES - } - official_rows = { - exchange: { - row.symbol: row - for row in universe.parse_master_source(official_descriptors[exchange]) - } - for exchange in US_EXCHANGES - } - broker_rows = { - exchange: { - row.symbol: row - for row in universe.parse_master_source(broker_descriptors[exchange]) - } - for exchange in US_EXCHANGES - } - - benchmark_paths: dict[str, Path] = {} - benchmark_results: dict[str, dict[str, Any]] = {} - for exchange in US_EXCHANGES: - path = output_root / "benchmarks" / f"{BENCHMARK_IDS[exchange]}.csv" - rows = update_benchmark_file( - symbol=BENCHMARK_SYMBOLS[exchange], - path=path, - start=start, - end=end, - minimum_sessions=minimum, - loader=benchmark_loader, - ) - benchmark_paths[exchange] = path.resolve() - benchmark_results[exchange] = { - "symbol": BENCHMARK_SYMBOLS[exchange], - "provider": "YAHOO_CHART", - "path": str(path.resolve()), - "sha256": shared.sha256_file(path), - "sessions": len(rows), - } - - catalog = shared.load_base_catalog(job["base_eod_catalog"]) - failures: list[dict[str, str]] = [] - invalid_audit_path = output_root / "invalid-eod-rows.json" - invalid_records = load_invalid_row_audit(invalid_audit_path) - ready_count = 0 - eligible_count = 0 - cache_hits = 0 - coverage = { - exchange: { - "official": len(official_rows[exchange]), - "eligible": 0, - "ready": 0, - "failed": 0, - } - for exchange in US_EXCHANGES - } - for exchange in US_EXCHANGES: - official_descriptor = official_descriptors[exchange] - broker_descriptor = broker_descriptors[exchange] - for symbol in sorted(official_rows[exchange]): - official = official_rows[exchange][symbol] - broker = broker_rows[exchange].get(symbol) - reasons = universe.metadata_exclusion_reasons(official, official_descriptor) - if broker is None: - reasons.append("not_kis_tradable") - else: - reasons.extend( - universe.metadata_exclusion_reasons(broker, broker_descriptor) - ) - reasons = sorted(set(reasons)) - instrument_type = shared.catalog_type(reasons) - ticker_path = output_root / "stocks" / exchange / f"{symbol}.csv" - data_path = "" - latest_close = "" - resolved_tick = "" - if not reasons: - eligible_count += 1 - coverage[exchange]["eligible"] += 1 - before = client.request_count - try: - rows, _, observed_invalid = update_stock_file( - client, - exchange=exchange, - symbol=symbol, - path=ticker_path, - start=start, - end=end, - minimum_sessions=minimum, - ) - if client.request_count != before or observed_invalid: - invalid_records = [ - item - for item in invalid_records - if not ( - item.get("exchange") == exchange - and item.get("symbol") == symbol - ) - ] - invalid_records.extend( - { - "exchange": exchange, - "symbol": symbol, - **item, - } - for item in observed_invalid - ) - if client.request_count == before: - cache_hits += 1 - latest_close = rows[-1]["close"] - resolved_tick = format(tick_size(Decimal(latest_close)), "f") - data_path = str(ticker_path.resolve()) - ready_count += 1 - coverage[exchange]["ready"] += 1 - except shared.EodBlockedError as exc: - coverage[exchange]["failed"] += 1 - failures.append( - { - "exchange": exchange, - "symbol": symbol, - "reason": str(exc), - } - ) - catalog[(exchange, symbol)] = { - "exchange": exchange, - "canonical_symbol": symbol, - "data_symbol": symbol, - "broker_symbol": symbol if broker is not None else "", - "instrument_type": instrument_type, - "benchmark_id": BENCHMARK_IDS[exchange], - "ticker_csv": data_path, - "benchmark_csv": str(benchmark_paths[exchange]), - "tick_rule_id": ( - f"US_NMS_CONSERVATIVE_{job['as_of']}" if resolved_tick else "" - ), - "tick_effective_date": job["analysis_date"] if resolved_tick else "", - "tick_reference_price": latest_close, - "resolved_tick_size": resolved_tick, - "source_name": "KIS_OPEN_API_ADJUSTED_DAILY:HHDFS76240000", - } - - rows = [ - catalog[key] - for key in sorted( - catalog, - key=lambda item: (universe.EXCHANGES.index(item[0]), item[1]), - ) - ] - catalog_path = output_root / "eod-catalog.csv" - shared.write_catalog(catalog_path, rows) - invalid_records.sort( - key=lambda item: ( - str(item.get("exchange", "")), - str(item.get("symbol", "")), - str(item.get("date", "")), - ) - ) - invalid_records = [ - item - for _, item in { - ( - str(item.get("exchange", "")), - str(item.get("symbol", "")), - str(item.get("date", "")), - ): item - for item in invalid_records - }.items() - ] - invalid_records.sort( - key=lambda item: ( - str(item.get("exchange", "")), - str(item.get("symbol", "")), - str(item.get("date", "")), - ) - ) - invalid_audit = { - "schema": "qta-kis-us-invalid-eod-rows/v1", - "analysis_date": job["analysis_date"], - "records": invalid_records, - } - shared.atomic_write_json(invalid_audit_path, invalid_audit) - build_spec = { - "schema": universe.BUILD_SPEC_SCHEMA, - "as_of": job["as_of"], - "analysis_date": job["analysis_date"], - "official_sources": shared.build_spec_sources(job["official_sources"]), - "broker_sources": shared.build_spec_sources(job["broker_sources"]), - "eod_catalog": { - "source_id": f"kis-cross-market-eod-catalog-{job['as_of']}", - "provider": "KIS", - "as_of": job["as_of"], - "path": str(catalog_path.resolve()), - "encoding": "utf-8", - }, - "catalog_coverage_contract": job["catalog_coverage_contract"], - } - build_spec_path = output_root / "universe-build-spec.json" - shared.atomic_write_json(build_spec_path, build_spec) - source_hashes = [ - { - "source_id": item["source_id"], - "exchange": item["exchange"], - "role": role, - "path": item["path"], - "sha256": shared.sha256_file(Path(item["path"])), - } - for role, sources in ( - ("OFFICIAL_MASTER", job["official_sources"]), - ("BROKER_MASTER", job["broker_sources"]), - ) - for item in sources - ] - source_hashes.sort(key=lambda item: item["source_id"]) - without_hash = { - "schema": RECEIPT_SCHEMA, - "status": "READY", - "as_of": job["as_of"], - "analysis_date": job["analysis_date"], - "environment": job["environment"], - "api_mutation_count": 0, - "adjusted_price_parameter": "MODP=1", - "minimum_sessions": minimum, - "eligible_symbols": eligible_count, - "ready_symbols": ready_count, - "cache_hits": cache_hits, - "coverage_by_exchange": coverage, - "failed_symbols": failures, - "invalid_eod_rows": { - "path": str(invalid_audit_path.resolve()), - "sha256": shared.sha256_file(invalid_audit_path), - "count": len(invalid_records), - "policy": "EXCLUDED_WITHOUT_INTERPOLATION", - }, - "request_count": client.request_count, - "retry_count": client.retry_count, - "benchmarks": benchmark_results, - "source_hashes": source_hashes, - "catalog": { - "path": str(catalog_path.resolve()), - "sha256": shared.sha256_file(catalog_path), - "rows": len(rows), - }, - "build_spec": { - "path": str(build_spec_path.resolve()), - "sha256": shared.sha256_file(build_spec_path), - }, - "completed_at": datetime.now(timezone.utc).isoformat(), - } - receipt = { - **without_hash, - "receipt_hash": shared.sha256_bytes( - shared.canonical_json(without_hash).encode("utf-8") - ), - } - shared.atomic_write_json(output_root / "eod-bundle-receipt.json", receipt) - return receipt - - -def download_bytes(url: str) -> bytes: - request = Request( - url, - headers={ - "User-Agent": "mrcha-skills/quant-stock-technical", - "Accept": "*/*", - }, - ) - try: - with urlopen(request, timeout=30) as response: - return response.read() - except (HTTPError, URLError, TimeoutError, OSError) as exc: - raise UsEodBlockedError( - f"source snapshot download failed: {url} ({type(exc).__name__})" - ) from exc - - -def snapshot_sources(as_of: str, output_directory: Path) -> dict[str, Any]: - shared.require_iso_date(as_of, "as_of") - output_directory.mkdir(parents=True, exist_ok=True) - official_paths: dict[str, Path] = {} - broker_paths: dict[str, Path] = {} - for exchange in US_EXCHANGES: - official_path = output_directory / ( - "otherlisted.txt" if exchange == "NYSE" else "nasdaqlisted.txt" - ) - shared.atomic_write(official_path, download_bytes(NASDAQ_URLS[exchange])) - official_paths[exchange] = official_path - payload = download_bytes(KIS_MASTER_URLS[exchange]) - with tempfile.TemporaryDirectory() as temporary: - archive_path = Path(temporary) / "master.zip" - archive_path.write_bytes(payload) - try: - with zipfile.ZipFile(archive_path) as archive: - expected = KIS_MASTER_NAMES[exchange] - matches = [ - name - for name in archive.namelist() - if Path(name).name.lower() == expected.lower() - ] - if len(matches) != 1: - raise UsEodBlockedError( - f"{exchange} master archive must contain {expected}" - ) - master_payload = archive.read(matches[0]) - except zipfile.BadZipFile as exc: - raise UsEodBlockedError( - f"{exchange} master download is not a ZIP" - ) from exc - broker_path = output_directory / KIS_MASTER_NAMES[exchange] - shared.atomic_write(broker_path, master_payload) - broker_paths[exchange] = broker_path - - official_sources = [ - { - "source_id": f"nasdaq-trader-{exchange.lower()}-{as_of}", - "provider": "NASDAQ_TRADER", - "exchange": exchange, - "as_of": as_of, - "path": str(official_paths[exchange].resolve()), - "format": ("NASDAQ_LISTED" if exchange == "NASDAQ" else "NASDAQ_OTHER"), - "encoding": "utf-8", - "delimiter": "|", - "skip_rows": 0, - "columns": {}, - "normal_status_values": ["N"] if exchange == "NASDAQ" else [], - } - for exchange in US_EXCHANGES - ] - broker_sources = [ - { - "source_id": f"kis-{exchange.lower()}-master-{as_of}", - "provider": "KIS", - "exchange": exchange, - "as_of": as_of, - "path": str(broker_paths[exchange].resolve()), - "format": "KIS_OVERSEAS_MASTER", - "encoding": "cp949", - "delimiter": "\t", - "skip_rows": 0, - "columns": {}, - "normal_status_values": [], - } - for exchange in US_EXCHANGES - ] - for index, descriptor in enumerate(official_sources): - normalized = universe.normalize_source_descriptor( - descriptor, "OFFICIAL_MASTER", as_of, output_directory, index - ) - universe.parse_master_source(normalized) - for index, descriptor in enumerate(broker_sources): - normalized = universe.normalize_source_descriptor( - descriptor, "BROKER_MASTER", as_of, output_directory, index - ) - universe.parse_master_source(normalized) - receipt = { - "schema": SNAPSHOT_SCHEMA, - "as_of": as_of, - "official_sources": official_sources, - "broker_sources": broker_sources, - "files": sorted( - [ - { - "path": str(path.resolve()), - "sha256": shared.sha256_file(path), - "bytes": path.stat().st_size, - } - for path in (*official_paths.values(), *broker_paths.values()) - ], - key=lambda item: item["path"], - ), - "api_mutation_count": 0, - "completed_at": datetime.now(timezone.utc).isoformat(), - } - shared.atomic_write_json(output_directory / "source-snapshot.json", receipt) - return receipt - - -def self_test() -> None: - end = date(2026, 7, 24) - raw = shared._synthetic_rows(end, 800, index=False) - overseas = [ - { - "xymd": item["stck_bsop_date"], - "open": item["stck_oprc"], - "high": item["stck_hgpr"], - "low": item["stck_lwpr"], - "clos": item["stck_clpr"], - "tvol": item["acml_vol"], - } - for item in raw - ] - - def transport( - method: str, - url: str, - headers: Mapping[str, str], - body: Optional[dict[str, Any]], - ) -> tuple[int, dict[str, Any]]: - del headers, body - if method != "GET": - raise AssertionError("self-test must be read-only") - query = dict(item.split("=", 1) for item in url.split("?", 1)[1].split("&")) - cursor = datetime.strptime(query["BYMD"], "%Y%m%d").date() - selected = [ - row - for row in overseas - if datetime.strptime(row["xymd"], "%Y%m%d").date() <= cursor - ][:100] - return 200, {"rt_cd": "0", "output2": selected} - - client = shared.KisReadClient( - environment="live", - app_key="fixture", - app_secret="fixture", - interval_ms=100, - access_token="fixture-token", - transport=transport, - ) - client.interval_seconds = 0 - rows = fetch_stock_history( - client, - exchange="NASDAQ", - symbol="AAPL", - start=end - timedelta(days=1200), - end=end, - ) - if len(rows) != 800 or rows[-1]["date"] != end.isoformat(): - raise AssertionError("U.S. pagination self-test failed") - if tick_size(Decimal("0.5")) != Decimal("0.0001"): - raise AssertionError("sub-dollar tick self-test failed") - if tick_size(Decimal("10")) != Decimal("0.01"): - raise AssertionError("dollar tick self-test failed") - print( - json.dumps( - { - "self_test": "PASS", - "stock_sessions": len(rows), - "requests": client.request_count, - "api_mutation_count": 0, - }, - sort_keys=True, - ) - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - parser.add_argument("--self-test", action="store_true") - subparsers = parser.add_subparsers(dest="command") - collect = subparsers.add_parser("collect") - collect.add_argument("--job", required=True) - snapshot = subparsers.add_parser("snapshot-sources") - snapshot.add_argument("--as-of", required=True) - snapshot.add_argument("--output-directory", required=True) - return parser.parse_args() - - -def main() -> int: - args = parse_args() - try: - if args.self_test: - self_test() - return 0 - if args.command == "snapshot-sources": - receipt = snapshot_sources( - args.as_of, Path(args.output_directory).expanduser().resolve() - ) - elif args.command == "collect": - job_path = Path(args.job).expanduser().resolve() - raw = json.loads(job_path.read_text(encoding="utf-8")) - receipt = build_bundle(normalize_job(raw, job_path.parent)) - else: - raise UsEodBlockedError( - "use collect --job, snapshot-sources, or --self-test" - ) - except ( - shared.EodBlockedError, - universe.UniverseBlockedError, - json.JSONDecodeError, - ) as exc: - print( - json.dumps( - {"status": "BLOCKED", "reason": str(exc)}, - ensure_ascii=False, - sort_keys=True, - ), - file=sys.stderr, - ) - return 2 - print(json.dumps(receipt, ensure_ascii=False, sort_keys=True)) - return 0 if receipt.get("status", "READY") == "READY" else 2 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/skills/quant-stock-technical/scripts/screen_universe.py b/skills/quant-stock-technical/scripts/screen_universe.py deleted file mode 100755 index c4f97f8..0000000 --- a/skills/quant-stock-technical/scripts/screen_universe.py +++ /dev/null @@ -1,1951 +0,0 @@ -#!/usr/bin/env python3 -"""Run an approved QTA method across a frozen v1/v2 universe.""" - -from __future__ import annotations - -import argparse -import concurrent.futures -import hashlib -import json -import os -import re -import sys -from datetime import date -from decimal import Decimal, InvalidOperation -from pathlib import Path -from typing import Any - -import analyze_stock as qta -import analyze_stock_v2 as qta2 - -MANIFEST_SCHEMA = "qta-universe-manifest/v1" -SCREEN_SCHEMA = "qta-screen/v1" -SELECTOR_VERSION = "qta-screen-1.0.0" -SUPPORTED_MARKETS = {"KR", "US"} -MANIFEST_SCHEMA_V2 = "qta-universe-manifest/v2" -SCREEN_SCHEMA_V2 = "qta-screen/v2" -SELECTOR_VERSION_V2 = "qta-screen-1.1.0" -CATALOG_COVERAGE_SCHEMA = "qta-catalog-coverage-contract/v1" -EXCHANGE_ORDER = ("KOSPI", "KOSDAQ", "NYSE", "NASDAQ") -SUPPORTED_EXCHANGES = {"KOSPI", "KOSDAQ", "NYSE", "NASDAQ"} -SUPPORTED_EXCHANGE_SCOPES = { - frozenset(("KOSPI", "KOSDAQ")), - frozenset(("NYSE", "NASDAQ")), - frozenset(SUPPORTED_EXCHANGES), -} -SUPPORTED_SETUP_STATUSES = {"READY", "CONDITIONAL"} -EXCHANGE_CONTRACTS = { - "KOSPI": { - "market": "KR", - "currency": "KRW", - "venue": "KRX", - "benchmark_id": "KOSPI_COMPOSITE", - }, - "KOSDAQ": { - "market": "KR", - "currency": "KRW", - "venue": "KRX", - "benchmark_id": "KOSDAQ_COMPOSITE", - }, - "NYSE": { - "market": "US", - "currency": "USD", - "venue": "NYSE", - "benchmark_id": "NYSE_COMPOSITE", - }, - "NASDAQ": { - "market": "US", - "currency": "USD", - "venue": "NASD", - "benchmark_id": "NASDAQ_COMPOSITE", - }, -} -SUPPORTED_INSTRUMENT_TYPES = {"COMMON", "ADR", "REIT"} -SOURCE_HASH_FIELDS = { - "source_id", - "role", - "provider", - "exchange", - "as_of", - "path", - "sha256", -} -EXCLUSION_FIELDS = { - "exchange", - "canonical_symbol", - "broker_symbol", - "name", - "reasons", - "official_source_id", - "broker_source_id", -} -COUNT_FIELDS = { - "official_rows", - "broker_rows", - "catalog_mapped_rows", - "included", - "excluded", - "by_exchange", -} -EXCHANGE_COUNT_FIELDS = { - "official", - "broker", - "catalog_mapped", - "included", - "excluded", -} -CATALOG_COVERAGE_FIELDS = { - "schema", - "minimum_ratio_by_exchange", - "minimum_screenable_ratio_by_exchange", -} -MANIFEST_V2_FIELDS = { - "schema", - "build_status", - "as_of", - "analysis_date", - "catalog_coverage_contract", - "source_hashes", - "instruments", - "exclusions", - "counts", - "manifest_hash", -} -INSTRUMENT_V2_FIELDS = { - "market", - "exchange", - "canonical_symbol", - "data_symbol", - "broker_symbol", - "instrument_type", - "benchmark_id", - "currency", - "venue", - "ticker_csv", - "benchmark_csv", - "tick_contract", - "source_name", - "ticker_csv_sha256", - "benchmark_csv_sha256", - "broker_tradability_verified", - "official_source_id", - "broker_source_id", -} -TICK_CONTRACT_FIELDS = { - "schema", - "kind", - "rule_id", - "effective_date", - "reference_price", - "resolved_tick_size", -} -SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") - - -class ScreenBlockedError(ValueError): - """Raised when a frozen screen contract is invalid.""" - - -def canonical_json(value: Any) -> str: - return json.dumps( - value, - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - allow_nan=False, - ) - - -def sha256_json(value: Any) -> str: - return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest() - - -def sha256_file(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def load_object(path: Path) -> dict[str, Any]: - value = json.loads(path.read_text(encoding="utf-8")) - if not isinstance(value, dict): - raise ScreenBlockedError(f"{path} must contain a JSON object") - return value - - -def as_decimal(value: Any, field: str) -> Decimal: - if isinstance(value, bool): - raise ScreenBlockedError(f"{field} must be a decimal string or number") - try: - parsed = Decimal(str(value)) - except (InvalidOperation, ValueError) as exc: - raise ScreenBlockedError(f"{field} must be a decimal string or number") from exc - if not parsed.is_finite(): - raise ScreenBlockedError(f"{field} must be finite") - return parsed - - -def as_nonempty_string(value: Any, field: str) -> str: - if not isinstance(value, str) or not value.strip(): - raise ScreenBlockedError(f"{field} must be a non-empty string") - return value.strip() - - -def as_string(value: Any, field: str) -> str: - if not isinstance(value, str): - raise ScreenBlockedError(f"{field} must be a string") - return value - - -def as_iso_date(value: Any, field: str) -> str: - rendered = as_nonempty_string(value, field) - try: - date.fromisoformat(rendered) - except ValueError as exc: - raise ScreenBlockedError(f"{field} must be YYYY-MM-DD") from exc - return rendered - - -def positive_decimal_string(value: Any, field: str) -> str: - parsed = as_decimal(value, field) - if parsed <= 0: - raise ScreenBlockedError(f"{field} must be positive") - return format(parsed, "f") - - -def canonical_coverage_ratio(value: Any, field: str) -> str: - parsed = as_decimal(value, field) - if parsed <= 0 or parsed > 1: - raise ScreenBlockedError(f"{field} must be greater than 0 and at most 1") - rendered = format(parsed.normalize(), "f") - return rendered.rstrip("0").rstrip(".") if "." in rendered else rendered - - -def meets_minimum_coverage_ratio( - numerator: int, - official_rows: int, - minimum: str, -) -> bool: - if official_rows <= 0: - return False - return Decimal(numerator) / Decimal(official_rows) >= Decimal(minimum) - - -def normalize_catalog_coverage_contract(raw: Any) -> dict[str, Any]: - field = "catalog_coverage_contract" - if not isinstance(raw, dict) or set(raw) != CATALOG_COVERAGE_FIELDS: - raise ScreenBlockedError( - f"{field} fields must be exactly {sorted(CATALOG_COVERAGE_FIELDS)}" - ) - if raw["schema"] != CATALOG_COVERAGE_SCHEMA: - raise ScreenBlockedError(f"{field}.schema must be {CATALOG_COVERAGE_SCHEMA}") - normalized_ratios: dict[str, dict[str, str]] = {} - for ratio_field in ( - "minimum_ratio_by_exchange", - "minimum_screenable_ratio_by_exchange", - ): - ratios = raw[ratio_field] - if not isinstance(ratios, dict) or set(ratios) != SUPPORTED_EXCHANGES: - raise ScreenBlockedError( - f"{field}.{ratio_field} must contain exactly " - "KOSPI, KOSDAQ, NYSE, NASDAQ" - ) - normalized_ratios[ratio_field] = { - exchange: canonical_coverage_ratio( - ratios[exchange], - f"{field}.{ratio_field}.{exchange}", - ) - for exchange in EXCHANGE_ORDER - } - return { - "schema": CATALOG_COVERAGE_SCHEMA, - **normalized_ratios, - } - - -def normalize_manifest_v1(manifest: dict[str, Any]) -> dict[str, Any]: - if set(manifest) != {"schema", "analysis_date", "instruments"}: - raise ScreenBlockedError( - "manifest fields must be exactly schema, analysis_date, instruments" - ) - if manifest["schema"] != MANIFEST_SCHEMA: - raise ScreenBlockedError(f"unsupported manifest schema: {manifest['schema']!r}") - try: - date.fromisoformat(str(manifest["analysis_date"])) - except ValueError as exc: - raise ScreenBlockedError("analysis_date must be YYYY-MM-DD") from exc - instruments = manifest["instruments"] - if not isinstance(instruments, list) or not instruments: - raise ScreenBlockedError("instruments must be a non-empty array") - - required = { - "market", - "ticker", - "ticker_csv", - "benchmark_csv", - "tick_size", - "source_name", - } - normalized: list[dict[str, str]] = [] - seen: set[tuple[str, str]] = set() - for index, raw in enumerate(instruments): - if not isinstance(raw, dict) or set(raw) != required: - raise ScreenBlockedError( - f"instruments[{index}] fields must be exactly {sorted(required)}" - ) - market = str(raw["market"]).upper() - ticker = str(raw["ticker"]).upper() - if market not in SUPPORTED_MARKETS: - raise ScreenBlockedError(f"instruments[{index}].market must be KR or US") - if not ticker: - raise ScreenBlockedError(f"instruments[{index}].ticker is empty") - key = (market, ticker) - if key in seen: - raise ScreenBlockedError(f"duplicate instrument: {market}:{ticker}") - seen.add(key) - tick_size = as_decimal(raw["tick_size"], f"instruments[{index}].tick_size") - if tick_size <= 0: - raise ScreenBlockedError(f"instruments[{index}].tick_size must be positive") - normalized.append( - { - "market": market, - "ticker": ticker, - "ticker_csv": str(raw["ticker_csv"]), - "benchmark_csv": str(raw["benchmark_csv"]), - "tick_size": format(tick_size, "f"), - "source_name": str(raw["source_name"]), - } - ) - normalized.sort(key=lambda item: (item["market"], item["ticker"])) - return { - "schema": MANIFEST_SCHEMA, - "analysis_date": str(manifest["analysis_date"]), - "instruments": normalized, - } - - -def normalize_tick_contract( - raw: Any, - field: str, - analysis_date: str, -) -> dict[str, str]: - if not isinstance(raw, dict) or set(raw) != TICK_CONTRACT_FIELDS: - raise ScreenBlockedError( - f"{field} fields must be exactly {sorted(TICK_CONTRACT_FIELDS)}" - ) - if raw["schema"] != "qta-tick-contract/v1": - raise ScreenBlockedError(f"{field}.schema must be qta-tick-contract/v1") - if raw["kind"] != "RESOLVED_PRICE_LADDER": - raise ScreenBlockedError(f"{field}.kind must be RESOLVED_PRICE_LADDER") - effective_date = as_iso_date(raw["effective_date"], f"{field}.effective_date") - if effective_date > analysis_date: - raise ScreenBlockedError( - f"{field}.effective_date must not be after analysis_date" - ) - return { - "schema": "qta-tick-contract/v1", - "kind": "RESOLVED_PRICE_LADDER", - "rule_id": as_nonempty_string(raw["rule_id"], f"{field}.rule_id"), - "effective_date": effective_date, - "reference_price": positive_decimal_string( - raw["reference_price"], f"{field}.reference_price" - ), - "resolved_tick_size": positive_decimal_string( - raw["resolved_tick_size"], f"{field}.resolved_tick_size" - ), - } - - -def exact_sha256(value: Any, field: str) -> str: - rendered = as_nonempty_string(value, field) - if SHA256_PATTERN.fullmatch(rendered) is None: - raise ScreenBlockedError(f"{field} must be 64 lowercase hexadecimal characters") - return rendered - - -def exact_upper_string(value: Any, field: str) -> str: - rendered = as_nonempty_string(value, field) - if rendered != rendered.upper(): - raise ScreenBlockedError(f"{field} must be uppercase") - return rendered - - -def nonnegative_integer(value: Any, field: str) -> int: - if isinstance(value, bool) or not isinstance(value, int) or value < 0: - raise ScreenBlockedError(f"{field} must be an integer >= 0") - return value - - -def normalize_eligible_setup_statuses(raw: Any) -> list[str]: - if ( - not isinstance(raw, list) - or not raw - or any(not isinstance(item, str) for item in raw) - or not set(raw).issubset(SUPPORTED_SETUP_STATUSES) - ): - raise ScreenBlockedError( - "eligible_setup_statuses must be a non-empty subset of " - "READY and CONDITIONAL" - ) - return sorted(set(raw)) - - -def normalize_source_hashes(raw: Any, manifest_as_of: str) -> list[dict[str, str]]: - if not isinstance(raw, list) or not raw: - raise ScreenBlockedError("source_hashes must be a non-empty array") - normalized: list[dict[str, str]] = [] - seen: set[str] = set() - for index, source in enumerate(raw): - field = f"source_hashes[{index}]" - if not isinstance(source, dict) or set(source) != SOURCE_HASH_FIELDS: - raise ScreenBlockedError( - f"{field} fields must be exactly {sorted(SOURCE_HASH_FIELDS)}" - ) - source_id = as_nonempty_string(source["source_id"], f"{field}.source_id") - if source_id in seen: - raise ScreenBlockedError(f"duplicate source_id: {source_id}") - seen.add(source_id) - exchange = exact_upper_string(source["exchange"], f"{field}.exchange") - if exchange not in SUPPORTED_EXCHANGES | {"GLOBAL"}: - raise ScreenBlockedError( - f"{field}.exchange must be GLOBAL or one of " - f"{sorted(SUPPORTED_EXCHANGES)}" - ) - source_date = as_iso_date(source["as_of"], f"{field}.as_of") - if source_date != manifest_as_of: - raise ScreenBlockedError(f"{field}.as_of must equal manifest as_of") - source_path = Path(as_nonempty_string(source["path"], f"{field}.path")) - if not source_path.is_absolute(): - raise ScreenBlockedError(f"{field}.path must be absolute") - normalized.append( - { - "source_id": source_id, - "role": as_nonempty_string(source["role"], f"{field}.role"), - "provider": as_nonempty_string(source["provider"], f"{field}.provider"), - "exchange": exchange, - "as_of": source_date, - "path": str(source_path), - "sha256": exact_sha256(source["sha256"], f"{field}.sha256"), - } - ) - if normalized != sorted(normalized, key=lambda item: item["source_id"]): - raise ScreenBlockedError("source_hashes must be sorted by source_id") - return normalized - - -def normalize_exclusions(raw: Any) -> list[dict[str, Any]]: - if not isinstance(raw, list): - raise ScreenBlockedError("exclusions must be an array") - normalized: list[dict[str, Any]] = [] - for index, exclusion in enumerate(raw): - field = f"exclusions[{index}]" - if not isinstance(exclusion, dict) or set(exclusion) != EXCLUSION_FIELDS: - raise ScreenBlockedError( - f"{field} fields must be exactly {sorted(EXCLUSION_FIELDS)}" - ) - exchange = exact_upper_string(exclusion["exchange"], f"{field}.exchange") - if exchange not in SUPPORTED_EXCHANGES: - raise ScreenBlockedError( - f"{field}.exchange must be one of {sorted(SUPPORTED_EXCHANGES)}" - ) - reasons = exclusion["reasons"] - if ( - not isinstance(reasons, list) - or not reasons - or any(not isinstance(reason, str) or not reason for reason in reasons) - or reasons != sorted(set(reasons)) - ): - raise ScreenBlockedError( - f"{field}.reasons must be a non-empty sorted unique string array" - ) - broker_symbol = exclusion["broker_symbol"] - if broker_symbol is not None: - broker_symbol = as_nonempty_string(broker_symbol, f"{field}.broker_symbol") - broker_source_id = exclusion["broker_source_id"] - if broker_source_id is not None: - broker_source_id = as_nonempty_string( - broker_source_id, f"{field}.broker_source_id" - ) - normalized.append( - { - "exchange": exchange, - "canonical_symbol": as_nonempty_string( - exclusion["canonical_symbol"], f"{field}.canonical_symbol" - ), - "broker_symbol": broker_symbol, - "name": as_string(exclusion["name"], f"{field}.name"), - "reasons": reasons, - "official_source_id": as_nonempty_string( - exclusion["official_source_id"], - f"{field}.official_source_id", - ), - "broker_source_id": broker_source_id, - } - ) - if normalized != sorted( - normalized, - key=lambda item: ( - EXCHANGE_ORDER.index(item["exchange"]), - item["canonical_symbol"], - item["reasons"], - ), - ): - raise ScreenBlockedError( - "exclusions must follow KOSPI, KOSDAQ, NYSE, NASDAQ order and then " - "canonical_symbol, reasons" - ) - return normalized - - -def normalize_counts(raw: Any) -> dict[str, Any]: - if not isinstance(raw, dict) or set(raw) != COUNT_FIELDS: - raise ScreenBlockedError( - f"counts fields must be exactly {sorted(COUNT_FIELDS)}" - ) - by_exchange = raw["by_exchange"] - if not isinstance(by_exchange, dict) or set(by_exchange) != SUPPORTED_EXCHANGES: - raise ScreenBlockedError( - "counts.by_exchange must contain exactly KOSPI, KOSDAQ, NYSE, NASDAQ" - ) - normalized_by_exchange: dict[str, dict[str, int]] = {} - for exchange in sorted(SUPPORTED_EXCHANGES): - counts = by_exchange[exchange] - field = f"counts.by_exchange.{exchange}" - if not isinstance(counts, dict) or set(counts) != EXCHANGE_COUNT_FIELDS: - raise ScreenBlockedError( - f"{field} fields must be exactly {sorted(EXCHANGE_COUNT_FIELDS)}" - ) - normalized_by_exchange[exchange] = { - key: nonnegative_integer(counts[key], f"{field}.{key}") - for key in sorted(EXCHANGE_COUNT_FIELDS) - } - normalized: dict[str, Any] = { - field: nonnegative_integer(raw[field], f"counts.{field}") - for field in ( - "official_rows", - "broker_rows", - "catalog_mapped_rows", - "included", - "excluded", - ) - } - normalized["by_exchange"] = normalized_by_exchange - total_mappings = { - "official_rows": "official", - "broker_rows": "broker", - "catalog_mapped_rows": "catalog_mapped", - "included": "included", - "excluded": "excluded", - } - for total_field, exchange_field in total_mappings.items(): - expected = sum( - counts[exchange_field] for counts in normalized_by_exchange.values() - ) - if normalized[total_field] != expected: - raise ScreenBlockedError( - f"counts.{total_field} must equal the by_exchange sum" - ) - return normalized - - -def normalize_instruments_v2( - raw: Any, - analysis_date: str, - source_ids: set[str], -) -> list[dict[str, Any]]: - if not isinstance(raw, list) or not raw: - raise ScreenBlockedError("instruments must be a non-empty array") - normalized: list[dict[str, Any]] = [] - seen: set[tuple[str, str]] = set() - for index, instrument in enumerate(raw): - field = f"instruments[{index}]" - if not isinstance(instrument, dict) or set(instrument) != INSTRUMENT_V2_FIELDS: - raise ScreenBlockedError( - f"{field} fields must be exactly {sorted(INSTRUMENT_V2_FIELDS)}" - ) - exchange = exact_upper_string(instrument["exchange"], f"{field}.exchange") - if exchange not in SUPPORTED_EXCHANGES: - raise ScreenBlockedError( - f"{field}.exchange must be one of {sorted(SUPPORTED_EXCHANGES)}" - ) - exchange_contract = EXCHANGE_CONTRACTS[exchange] - market = exact_upper_string(instrument["market"], f"{field}.market") - if market != exchange_contract["market"]: - raise ScreenBlockedError( - f"{field}.market must be {exchange_contract['market']} for {exchange}" - ) - canonical_symbol = as_nonempty_string( - instrument["canonical_symbol"], f"{field}.canonical_symbol" - ) - key = (market, canonical_symbol) - if key in seen: - raise ScreenBlockedError( - f"duplicate canonical instrument: {market}:{canonical_symbol}" - ) - seen.add(key) - instrument_type = exact_upper_string( - instrument["instrument_type"], f"{field}.instrument_type" - ) - if instrument_type not in SUPPORTED_INSTRUMENT_TYPES: - raise ScreenBlockedError( - f"{field}.instrument_type must be one of " - f"{sorted(SUPPORTED_INSTRUMENT_TYPES)}" - ) - if market == "KR" and instrument_type == "ADR": - raise ScreenBlockedError(f"{field}.instrument_type ADR requires market US") - for contract_field in ("benchmark_id", "currency", "venue"): - actual = exact_upper_string( - instrument[contract_field], f"{field}.{contract_field}" - ) - if actual != exchange_contract[contract_field]: - raise ScreenBlockedError( - f"{field}.{contract_field} must be " - f"{exchange_contract[contract_field]} for {exchange}" - ) - ticker_path = Path( - as_nonempty_string(instrument["ticker_csv"], f"{field}.ticker_csv") - ) - benchmark_path = Path( - as_nonempty_string(instrument["benchmark_csv"], f"{field}.benchmark_csv") - ) - if not ticker_path.is_absolute() or not benchmark_path.is_absolute(): - raise ScreenBlockedError( - f"{field}.ticker_csv and benchmark_csv must be absolute" - ) - if instrument["broker_tradability_verified"] is not True: - raise ScreenBlockedError( - f"{field}.broker_tradability_verified must be true" - ) - official_source_id = as_nonempty_string( - instrument["official_source_id"], f"{field}.official_source_id" - ) - broker_source_id = as_nonempty_string( - instrument["broker_source_id"], f"{field}.broker_source_id" - ) - if official_source_id not in source_ids or broker_source_id not in source_ids: - raise ScreenBlockedError( - f"{field} references an unknown official_source_id or broker_source_id" - ) - normalized.append( - { - "market": market, - "exchange": exchange, - "canonical_symbol": canonical_symbol, - "data_symbol": as_nonempty_string( - instrument["data_symbol"], f"{field}.data_symbol" - ), - "broker_symbol": as_nonempty_string( - instrument["broker_symbol"], f"{field}.broker_symbol" - ), - "instrument_type": instrument_type, - "benchmark_id": exchange_contract["benchmark_id"], - "currency": exchange_contract["currency"], - "venue": exchange_contract["venue"], - "ticker_csv": str(ticker_path), - "benchmark_csv": str(benchmark_path), - "tick_contract": normalize_tick_contract( - instrument["tick_contract"], - f"{field}.tick_contract", - analysis_date, - ), - "source_name": as_nonempty_string( - instrument["source_name"], f"{field}.source_name" - ), - "ticker_csv_sha256": exact_sha256( - instrument["ticker_csv_sha256"], - f"{field}.ticker_csv_sha256", - ), - "benchmark_csv_sha256": exact_sha256( - instrument["benchmark_csv_sha256"], - f"{field}.benchmark_csv_sha256", - ), - "broker_tradability_verified": True, - "official_source_id": official_source_id, - "broker_source_id": broker_source_id, - } - ) - return normalized - - -def normalize_manifest_v2(manifest: dict[str, Any]) -> dict[str, Any]: - if set(manifest) != MANIFEST_V2_FIELDS: - raise ScreenBlockedError( - f"manifest fields must be exactly {sorted(MANIFEST_V2_FIELDS)}" - ) - if manifest["schema"] != MANIFEST_SCHEMA_V2: - raise ScreenBlockedError(f"unsupported manifest schema: {manifest['schema']!r}") - supplied_hash = exact_sha256(manifest["manifest_hash"], "manifest_hash") - hash_input = { - key: value for key, value in manifest.items() if key != "manifest_hash" - } - if sha256_json(hash_input) != supplied_hash: - raise ScreenBlockedError( - "manifest_hash does not match canonical manifest content" - ) - as_of = as_iso_date(manifest["as_of"], "as_of") - analysis_date = as_iso_date(manifest["analysis_date"], "analysis_date") - if analysis_date > as_of: - raise ScreenBlockedError("analysis_date must not be after as_of") - if manifest["build_status"] not in {"READY", "BLOCKED"}: - raise ScreenBlockedError("build_status must be READY or BLOCKED") - if manifest["build_status"] == "BLOCKED": - raise ScreenBlockedError("manifest build_status is BLOCKED") - coverage_contract = normalize_catalog_coverage_contract( - manifest["catalog_coverage_contract"] - ) - source_hashes = normalize_source_hashes(manifest["source_hashes"], as_of) - source_ids = {source["source_id"] for source in source_hashes} - instruments = normalize_instruments_v2( - manifest["instruments"], - analysis_date, - source_ids, - ) - exclusions = normalize_exclusions(manifest["exclusions"]) - for exclusion in exclusions: - if exclusion["official_source_id"] not in source_ids: - raise ScreenBlockedError( - "exclusion references an unknown official_source_id" - ) - if ( - exclusion["broker_source_id"] is not None - and exclusion["broker_source_id"] not in source_ids - ): - raise ScreenBlockedError("exclusion references an unknown broker_source_id") - counts = normalize_counts(manifest["counts"]) - if counts["included"] != len(instruments): - raise ScreenBlockedError("counts.included must equal instruments length") - if counts["excluded"] != len(exclusions): - raise ScreenBlockedError("counts.excluded must equal exclusions length") - for exchange in sorted(SUPPORTED_EXCHANGES): - included = sum(instrument["exchange"] == exchange for instrument in instruments) - excluded = sum(exclusion["exchange"] == exchange for exclusion in exclusions) - missing_catalog = sum( - exclusion["exchange"] == exchange - and "missing_eod_mapping" in exclusion["reasons"] - for exclusion in exclusions - ) - official = counts["by_exchange"][exchange]["official"] - catalog_mapped = counts["by_exchange"][exchange]["catalog_mapped"] - if catalog_mapped > official: - raise ScreenBlockedError( - f"counts.by_exchange.{exchange}.catalog_mapped cannot exceed official" - ) - if catalog_mapped != official - missing_catalog: - raise ScreenBlockedError( - f"counts.by_exchange.{exchange}.catalog_mapped does not match " - "official rows without missing_eod_mapping" - ) - if counts["by_exchange"][exchange]["included"] > official: - raise ScreenBlockedError( - f"counts.by_exchange.{exchange}.included cannot exceed official" - ) - if counts["by_exchange"][exchange]["included"] != included: - raise ScreenBlockedError( - f"counts.by_exchange.{exchange}.included does not match instruments" - ) - if counts["by_exchange"][exchange]["excluded"] != excluded: - raise ScreenBlockedError( - f"counts.by_exchange.{exchange}.excluded does not match exclusions" - ) - exchange_scope = frozenset( - exchange - for exchange in SUPPORTED_EXCHANGES - if counts["by_exchange"][exchange]["official"] > 0 - ) - if exchange_scope not in SUPPORTED_EXCHANGE_SCOPES: - raise ScreenBlockedError( - "manifest must cover exactly KOSPI/KOSDAQ, NYSE/NASDAQ, " - "or all four exchanges" - ) - inactive_exchanges = SUPPORTED_EXCHANGES - exchange_scope - for exchange in inactive_exchanges: - if any(counts["by_exchange"][exchange].values()): - raise ScreenBlockedError( - f"inactive exchange {exchange} must have zero counts" - ) - expected_status = ( - "READY" - if all( - counts["by_exchange"][exchange]["included"] > 0 - and meets_minimum_coverage_ratio( - counts["by_exchange"][exchange]["catalog_mapped"], - counts["by_exchange"][exchange]["official"], - coverage_contract["minimum_ratio_by_exchange"][exchange], - ) - and meets_minimum_coverage_ratio( - counts["by_exchange"][exchange]["included"], - counts["by_exchange"][exchange]["official"], - coverage_contract["minimum_screenable_ratio_by_exchange"][exchange], - ) - for exchange in exchange_scope - ) - else "BLOCKED" - ) - if manifest["build_status"] != expected_status: - raise ScreenBlockedError( - "build_status must be READY only when every active exchange has " - "included rows and meets its catalog mapping and screenable " - "coverage contracts" - ) - normalized = { - "schema": MANIFEST_SCHEMA_V2, - "build_status": "READY", - "as_of": as_of, - "analysis_date": analysis_date, - "catalog_coverage_contract": coverage_contract, - "source_hashes": source_hashes, - "instruments": instruments, - "exclusions": exclusions, - "counts": counts, - } - normalized["manifest_hash"] = sha256_json(normalized) - if normalized["manifest_hash"] != supplied_hash: - raise ScreenBlockedError("manifest content is not in normalized canonical form") - return normalized - - -def normalize_manifest(manifest: dict[str, Any]) -> dict[str, Any]: - schema = manifest.get("schema") - if schema == MANIFEST_SCHEMA: - return normalize_manifest_v1(manifest) - if schema == MANIFEST_SCHEMA_V2: - return normalize_manifest_v2(manifest) - raise ScreenBlockedError(f"unsupported manifest schema: {schema!r}") - - -def normalize_selector_v1(selector: dict[str, Any]) -> dict[str, Any]: - required = { - "selector_version", - "min_total_score", - "eligible_setup_statuses", - "top_k_by_market", - "max_blocked_fraction", - } - if set(selector) != required: - raise ScreenBlockedError(f"selector fields must be exactly {sorted(required)}") - if selector["selector_version"] != SELECTOR_VERSION: - raise ScreenBlockedError( - f"unsupported selector version: {selector['selector_version']!r}" - ) - minimum = as_decimal(selector["min_total_score"], "min_total_score") - if minimum < 0 or minimum > 100: - raise ScreenBlockedError("min_total_score must be between 0 and 100") - maximum_blocked = as_decimal( - selector["max_blocked_fraction"], "max_blocked_fraction" - ) - if maximum_blocked < 0 or maximum_blocked > 1: - raise ScreenBlockedError("max_blocked_fraction must be between 0 and 1") - - normalized_statuses = normalize_eligible_setup_statuses( - selector["eligible_setup_statuses"] - ) - - top_k = selector["top_k_by_market"] - if not isinstance(top_k, dict) or set(top_k) != SUPPORTED_MARKETS: - raise ScreenBlockedError("top_k_by_market must contain exactly KR and US") - normalized_top_k: dict[str, int] = {} - for market in sorted(SUPPORTED_MARKETS): - value = top_k[market] - if isinstance(value, bool) or not isinstance(value, int) or value < 0: - raise ScreenBlockedError( - f"top_k_by_market.{market} must be an integer >= 0" - ) - normalized_top_k[market] = value - - return { - "selector_version": SELECTOR_VERSION, - "min_total_score": format(minimum, "f"), - "eligible_setup_statuses": normalized_statuses, - "top_k_by_market": normalized_top_k, - "max_blocked_fraction": format(maximum_blocked, "f"), - } - - -def normalize_selector_v2(selector: dict[str, Any]) -> dict[str, Any]: - required = { - "selector_version", - "min_total_score", - "eligible_setup_statuses", - "top_k_by_exchange", - "min_selected_by_exchange", - "max_blocked_fraction", - } - if set(selector) != required: - raise ScreenBlockedError(f"selector fields must be exactly {sorted(required)}") - if selector["selector_version"] != SELECTOR_VERSION_V2: - raise ScreenBlockedError( - f"unsupported selector version: {selector['selector_version']!r}" - ) - minimum = as_decimal(selector["min_total_score"], "min_total_score") - if minimum < 0 or minimum > 100: - raise ScreenBlockedError("min_total_score must be between 0 and 100") - maximum_blocked = as_decimal( - selector["max_blocked_fraction"], "max_blocked_fraction" - ) - if maximum_blocked < 0 or maximum_blocked > 1: - raise ScreenBlockedError("max_blocked_fraction must be between 0 and 1") - normalized_statuses = normalize_eligible_setup_statuses( - selector["eligible_setup_statuses"] - ) - top_k = selector["top_k_by_exchange"] - if not isinstance(top_k, dict) or set(top_k) != SUPPORTED_EXCHANGES: - raise ScreenBlockedError( - "top_k_by_exchange must contain exactly KOSPI, KOSDAQ, NYSE, NASDAQ" - ) - normalized_top_k: dict[str, int] = {} - for exchange in sorted(SUPPORTED_EXCHANGES): - normalized_top_k[exchange] = nonnegative_integer( - top_k[exchange], - f"top_k_by_exchange.{exchange}", - ) - minimum_selected = selector["min_selected_by_exchange"] - if ( - not isinstance(minimum_selected, dict) - or set(minimum_selected) != SUPPORTED_EXCHANGES - ): - raise ScreenBlockedError( - "min_selected_by_exchange must contain exactly KOSPI, KOSDAQ, NYSE, NASDAQ" - ) - normalized_minimum_selected: dict[str, int] = {} - for exchange in sorted(SUPPORTED_EXCHANGES): - value = nonnegative_integer( - minimum_selected[exchange], - f"min_selected_by_exchange.{exchange}", - ) - if value > normalized_top_k[exchange]: - raise ScreenBlockedError( - f"min_selected_by_exchange.{exchange} must be <= " - f"top_k_by_exchange.{exchange}" - ) - normalized_minimum_selected[exchange] = value - return { - "selector_version": SELECTOR_VERSION_V2, - "min_total_score": format(minimum, "f"), - "eligible_setup_statuses": normalized_statuses, - "top_k_by_exchange": normalized_top_k, - "min_selected_by_exchange": normalized_minimum_selected, - "max_blocked_fraction": format(maximum_blocked, "f"), - } - - -def normalize_selector(selector: dict[str, Any]) -> dict[str, Any]: - version = selector.get("selector_version") - if version == SELECTOR_VERSION: - return normalize_selector_v1(selector) - if version == SELECTOR_VERSION_V2: - return normalize_selector_v2(selector) - raise ScreenBlockedError(f"unsupported selector version: {version!r}") - - -def instrument_market(instrument: dict[str, Any]) -> str: - return str(instrument.get("market")) - - -def instrument_ticker(instrument: dict[str, Any]) -> str: - if "ticker" in instrument: - return str(instrument["ticker"]) - return str(instrument["canonical_symbol"]) - - -def instrument_tick_size(instrument: dict[str, Any]) -> str: - if "tick_size" in instrument: - return str(instrument["tick_size"]) - return str(instrument["tick_contract"]["resolved_tick_size"]) - - -def blocked_payload( - instrument: dict[str, Any], - reason: str, - *, - method_version: str, -) -> dict[str, Any]: - return { - "source_skill": "quant-stock-technical", - "result_schema": "quant-stock-technical/v1", - "calculation_status": "BLOCKED", - "reason": reason, - "method_version": method_version, - "market": instrument_market(instrument), - "ticker": instrument_ticker(instrument), - } - - -def analyze_instrument( - instrument: dict[str, Any], - analysis_date: date, - manifest_directory: Path, - *, - calculator: Any = qta, -) -> dict[str, Any]: - ticker_path = Path(instrument["ticker_csv"]) - benchmark_path = Path(instrument["benchmark_csv"]) - if not ticker_path.is_absolute(): - ticker_path = manifest_directory / ticker_path - if not benchmark_path.is_absolute(): - benchmark_path = manifest_directory / benchmark_path - try: - if "ticker_csv_sha256" in instrument: - if sha256_file(ticker_path) != instrument["ticker_csv_sha256"]: - raise ScreenBlockedError("ticker_csv_sha256 does not match ticker_csv") - if sha256_file(benchmark_path) != instrument["benchmark_csv_sha256"]: - raise ScreenBlockedError( - "benchmark_csv_sha256 does not match benchmark_csv" - ) - ticker_rows, benchmark_rows = qta.align( - qta.read_csv(str(ticker_path)), - qta.read_csv(str(benchmark_path)), - analysis_date, - ) - return calculator.calculate( - ticker_rows, - benchmark_rows, - instrument_market(instrument), - instrument_ticker(instrument), - float(Decimal(instrument_tick_size(instrument))), - instrument["source_name"], - ) - except (qta.BlockedError, OSError, ValueError) as exc: - return blocked_payload( - instrument, - str(exc), - method_version=calculator.METHOD_VERSION, - ) - - -def analyze_instrument_task( - task: tuple[dict[str, Any], str, str, str], -) -> dict[str, Any]: - instrument, analysis_date_text, manifest_directory_text, method_version = task - calculators = { - qta.METHOD_VERSION: qta, - qta2.METHOD_VERSION: qta2, - } - calculator = calculators.get(method_version) - if calculator is None: - raise ScreenBlockedError( - f"unsupported QTA method version: {method_version!r}" - ) - return analyze_instrument( - instrument, - date.fromisoformat(analysis_date_text), - Path(manifest_directory_text), - calculator=calculator, - ) - - -def score(payload: dict[str, Any], path: str) -> Decimal: - value: Any = payload - for part in path.split("."): - if not isinstance(value, dict) or part not in value: - raise ScreenBlockedError( - f"{payload.get('market')}:{payload.get('ticker')} missing {path}" - ) - value = value[part] - return as_decimal(value, path) - - -def ranking_key(payload: dict[str, Any]) -> tuple[Any, ...]: - if payload.get("method_version") == qta2.METHOD_VERSION: - return ( - -score(payload, "total_score"), - -score(payload, "short.score"), - -score(payload, "medium.score"), - score(payload, "risk.score"), - -score(payload, "long.score"), - str(payload["market"]), - str(payload["ticker"]), - ) - return ( - -score(payload, "total_score"), - -score(payload, "medium.score"), - -score(payload, "long.score"), - -score(payload, "short.score"), - score(payload, "risk.score"), - str(payload["market"]), - str(payload["ticker"]), - ) - - -def finalize_screen( - manifest: dict[str, Any], - selector: dict[str, Any], - results: list[dict[str, Any]], -) -> dict[str, Any]: - instrument_by_key = { - (item["market"], item["ticker"]): { - "market": item["market"], - "ticker": item["ticker"], - "tick_size": item["tick_size"], - } - for item in manifest["instruments"] - } - result_by_key: dict[tuple[str, str], dict[str, Any]] = {} - for result in results: - key = (str(result.get("market")), str(result.get("ticker"))) - if key in result_by_key: - raise ScreenBlockedError(f"duplicate calculation result: {key}") - result_by_key[key] = result - - ordered_results: list[dict[str, Any]] = [] - for instrument in manifest["instruments"]: - key = (instrument["market"], instrument["ticker"]) - if key not in result_by_key: - raise ScreenBlockedError(f"missing calculation result: {key}") - ordered_results.append(result_by_key[key]) - - blocked_count = sum( - result.get("calculation_status") != "READY" for result in ordered_results - ) - blocked_fraction = Decimal(blocked_count) / Decimal(len(ordered_results)) - maximum_blocked = Decimal(selector["max_blocked_fraction"]) - screen_status = "READY" if blocked_fraction <= maximum_blocked else "BLOCKED" - - decisions: list[dict[str, Any]] = [] - eligible_by_market: dict[str, list[dict[str, Any]]] = { - market: [] for market in SUPPORTED_MARKETS - } - statuses = set(selector["eligible_setup_statuses"]) - minimum = Decimal(selector["min_total_score"]) - for payload in ordered_results: - reasons: list[str] = [] - if payload.get("calculation_status") != "READY": - reasons.append("calculation_not_ready") - else: - if payload.get("setup_status") not in statuses: - reasons.append("setup_status_ineligible") - if score(payload, "total_score") < minimum: - reasons.append("score_below_minimum") - eligible = not reasons - if eligible: - market = str(payload["market"]) - if market not in eligible_by_market: - raise ScreenBlockedError(f"result market is unsupported: {market}") - eligible_by_market[market].append(payload) - decisions.append( - { - "market": payload.get("market"), - "ticker": payload.get("ticker"), - "instrument": instrument_by_key[ - (str(payload.get("market")), str(payload.get("ticker"))) - ], - "eligible": eligible, - "reasons": reasons, - "qta": payload, - } - ) - - selected: dict[str, list[dict[str, Any]]] = {} - selected_keys: set[tuple[str, str]] = set() - for market in sorted(SUPPORTED_MARKETS): - ranked = sorted(eligible_by_market[market], key=ranking_key) - chosen = ranked[: selector["top_k_by_market"][market]] - selected[market] = [ - { - "rank": index, - "instrument": instrument_by_key[(market, str(payload["ticker"]))], - "qta": payload, - } - for index, payload in enumerate(chosen, start=1) - ] - selected_keys.update((market, str(payload["ticker"])) for payload in chosen) - - for decision in decisions: - decision["selected"] = ( - decision["market"], - decision["ticker"], - ) in selected_keys - - method_versions = { - str(result.get("method_version")) for result in ordered_results - } - if len(method_versions) != 1: - raise ScreenBlockedError("screen results contain multiple QTA methods") - output = { - "source_skill": "quant-stock-technical", - "schema": SCREEN_SCHEMA, - "screen_status": screen_status, - "method_version": method_versions.pop(), - "selector_version": SELECTOR_VERSION, - "analysis_date": manifest["analysis_date"], - "manifest_hash": sha256_json(manifest), - "selector_hash": sha256_json(selector), - "blocked_count": blocked_count, - "blocked_fraction": format(blocked_fraction, "f"), - "instrument_count": len(ordered_results), - "selector": selector, - "selected": selected if screen_status == "READY" else {"KR": [], "US": []}, - "decisions": decisions, - } - output["screen_hash"] = sha256_json(output) - return output - - -def finalize_screen_v2( - manifest: dict[str, Any], - selector: dict[str, Any], - results: list[dict[str, Any]], -) -> dict[str, Any]: - instrument_by_key = { - (instrument["market"], instrument["canonical_symbol"]): instrument - for instrument in manifest["instruments"] - } - result_by_key: dict[tuple[str, str], dict[str, Any]] = {} - for result in results: - key = (str(result.get("market")), str(result.get("ticker"))) - if key in result_by_key: - raise ScreenBlockedError(f"duplicate calculation result: {key}") - result_by_key[key] = result - - ordered_results: list[dict[str, Any]] = [] - for instrument in manifest["instruments"]: - key = (instrument["market"], instrument["canonical_symbol"]) - if key not in result_by_key: - raise ScreenBlockedError(f"missing calculation result: {key}") - ordered_results.append(result_by_key[key]) - if len(result_by_key) != len(ordered_results): - raise ScreenBlockedError("calculation results include unknown instruments") - - blocked_count = sum( - result.get("calculation_status") != "READY" for result in ordered_results - ) - blocked_fraction = Decimal(blocked_count) / Decimal(len(ordered_results)) - maximum_blocked = Decimal(selector["max_blocked_fraction"]) - - statuses = set(selector["eligible_setup_statuses"]) - minimum = Decimal(selector["min_total_score"]) - eligible_by_exchange: dict[str, list[dict[str, Any]]] = { - exchange: [] for exchange in SUPPORTED_EXCHANGES - } - decision_by_key: dict[tuple[str, str], dict[str, Any]] = {} - decisions: list[dict[str, Any]] = [] - for payload in ordered_results: - key = (str(payload.get("market")), str(payload.get("ticker"))) - instrument = instrument_by_key[key] - reasons: list[str] = [] - if payload.get("calculation_status") != "READY": - reasons.append("calculation_not_ready") - else: - if payload.get("setup_status") not in statuses: - reasons.append("setup_status_ineligible") - if score(payload, "total_score") < minimum: - reasons.append("score_below_minimum") - eligible = not reasons - if eligible: - eligible_by_exchange[instrument["exchange"]].append(payload) - decision = { - "market": instrument["market"], - "exchange": instrument["exchange"], - "canonical_symbol": instrument["canonical_symbol"], - "instrument": instrument, - "eligible": eligible, - "reasons": reasons, - "exchange_rank": None, - "selected": False, - "qta": payload, - } - decisions.append(decision) - decision_by_key[key] = decision - - selected: dict[str, list[dict[str, Any]]] = { - exchange: [] for exchange in sorted(SUPPORTED_EXCHANGES) - } - ranked_by_exchange: dict[str, list[dict[str, Any]]] = {} - for exchange in sorted(SUPPORTED_EXCHANGES): - ranked = sorted(eligible_by_exchange[exchange], key=ranking_key) - ranked_by_exchange[exchange] = ranked - for exchange_rank, payload in enumerate(ranked, start=1): - key = (str(payload["market"]), str(payload["ticker"])) - decision_by_key[key]["exchange_rank"] = exchange_rank - minimum_selection_met = all( - len(ranked_by_exchange[exchange]) - >= selector["min_selected_by_exchange"][exchange] - for exchange in SUPPORTED_EXCHANGES - ) - screen_status = ( - "READY" - if blocked_fraction <= maximum_blocked and minimum_selection_met - else "BLOCKED" - ) - - selected_keys: set[tuple[str, str]] = set() - if screen_status == "READY": - for exchange in sorted(SUPPORTED_EXCHANGES): - ranked = ranked_by_exchange[exchange] - chosen = ranked[: selector["top_k_by_exchange"][exchange]] - selected[exchange] = [ - { - "exchange_rank": exchange_rank, - "instrument": instrument_by_key[ - (str(payload["market"]), str(payload["ticker"])) - ], - "qta": payload, - } - for exchange_rank, payload in enumerate(chosen, start=1) - ] - selected_keys.update( - (str(payload["market"]), str(payload["ticker"])) for payload in chosen - ) - for key in selected_keys: - decision_by_key[key]["selected"] = True - - method_versions = { - str(result.get("method_version")) for result in ordered_results - } - if len(method_versions) != 1: - raise ScreenBlockedError("screen results contain multiple QTA methods") - output = { - "source_skill": "quant-stock-technical", - "schema": SCREEN_SCHEMA_V2, - "screen_status": screen_status, - "method_version": method_versions.pop(), - "selector_version": SELECTOR_VERSION_V2, - "analysis_date": manifest["analysis_date"], - "manifest_hash": manifest["manifest_hash"], - "selector_hash": sha256_json(selector), - "blocked_count": blocked_count, - "blocked_fraction": format(blocked_fraction, "f"), - "instrument_count": len(ordered_results), - "selector": selector, - "selected": selected, - "decisions": decisions, - } - output["screen_hash"] = sha256_json(output) - return output - - -def build_screen( - manifest: dict[str, Any], - selector: dict[str, Any], - manifest_directory: Path, - *, - method_version: str = qta.METHOD_VERSION, - workers: int = 1, -) -> dict[str, Any]: - normalized_manifest = normalize_manifest(manifest) - normalized_selector = normalize_selector(selector) - if ( - normalized_manifest["schema"] == MANIFEST_SCHEMA - and normalized_selector["selector_version"] != SELECTOR_VERSION - ): - raise ScreenBlockedError("qta-universe-manifest/v1 requires qta-screen-1.0.0") - if ( - normalized_manifest["schema"] == MANIFEST_SCHEMA_V2 - and normalized_selector["selector_version"] != SELECTOR_VERSION_V2 - ): - raise ScreenBlockedError("qta-universe-manifest/v2 requires qta-screen-1.1.0") - analysis_date = date.fromisoformat(normalized_manifest["analysis_date"]) - calculators = { - qta.METHOD_VERSION: qta, - qta2.METHOD_VERSION: qta2, - } - calculator = calculators.get(method_version) - if calculator is None: - raise ScreenBlockedError( - f"unsupported QTA method version: {method_version!r}" - ) - if isinstance(workers, bool) or workers <= 0 or workers > 32: - raise ScreenBlockedError("workers must be an integer between 1 and 32") - tasks = [ - ( - instrument, - analysis_date.isoformat(), - str(manifest_directory), - calculator.METHOD_VERSION, - ) - for instrument in normalized_manifest["instruments"] - ] - if workers == 1: - results = [analyze_instrument_task(task) for task in tasks] - else: - with concurrent.futures.ProcessPoolExecutor( - max_workers=workers - ) as executor: - results = list( - executor.map( - analyze_instrument_task, - tasks, - chunksize=8, - ) - ) - if normalized_manifest["schema"] == MANIFEST_SCHEMA: - return finalize_screen(normalized_manifest, normalized_selector, results) - return finalize_screen_v2(normalized_manifest, normalized_selector, results) - - -def synthetic_result( - market: str, - ticker: str, - total: str, - medium: str, - long: str, - short: str, - risk: str, -) -> dict[str, Any]: - return { - "source_skill": "quant-stock-technical", - "result_schema": "quant-stock-technical/v1", - "calculation_status": "READY", - "setup_status": "READY", - "method_version": qta.METHOD_VERSION, - "analysis_date": "2026-07-25", - "market": market, - "ticker": ticker, - "source_name": "screen-self-test", - "shared_sessions": 1000, - "score_basis": "ticker-relative historical percentile; not probability of profit", - "short": {"opinion": "긍정", "score": float(short)}, - "medium": {"opinion": "긍정", "score": float(medium)}, - "long": {"opinion": "긍정", "score": float(long)}, - "risk": {"score": float(risk), "counterpoint": "self-test"}, - "entry_price": 100.0, - "stop_price": 90.0, - "take_profit_price": 120.0, - "total_score": float(total), - } - - -def self_test() -> None: - raw_instruments = [ - { - "market": "US", - "ticker": "BBB", - "ticker_csv": "bbb.csv", - "benchmark_csv": "bench.csv", - "tick_size": "0.01", - "source_name": "self-test", - }, - { - "market": "US", - "ticker": "AAA", - "ticker_csv": "aaa.csv", - "benchmark_csv": "bench.csv", - "tick_size": "0.01", - "source_name": "self-test", - }, - ] - selector = normalize_selector( - { - "selector_version": SELECTOR_VERSION, - "min_total_score": "60", - "eligible_setup_statuses": ["READY"], - "top_k_by_market": {"KR": 0, "US": 2}, - "max_blocked_fraction": "0", - } - ) - manifest_a = normalize_manifest( - { - "schema": MANIFEST_SCHEMA, - "analysis_date": "2026-07-25", - "instruments": raw_instruments, - } - ) - manifest_b = normalize_manifest( - { - "schema": MANIFEST_SCHEMA, - "analysis_date": "2026-07-25", - "instruments": list(reversed(raw_instruments)), - } - ) - results = [ - synthetic_result("US", "BBB", "70", "70", "70", "70", "30"), - synthetic_result("US", "AAA", "70", "70", "70", "70", "30"), - ] - first = finalize_screen(manifest_a, selector, results) - second = finalize_screen(manifest_b, selector, list(reversed(results))) - assert first == second - assert [item["qta"]["ticker"] for item in first["selected"]["US"]] == [ - "AAA", - "BBB", - ] - assert first["screen_status"] == "READY" - assert len(first["screen_hash"]) == 64 - assert normalize_eligible_setup_statuses(["READY", "CONDITIONAL", "READY"]) == [ - "CONDITIONAL", - "READY", - ] - try: - normalize_selector({**selector, "eligible_setup_statuses": ["BLOCKED"]}) - except ScreenBlockedError as exc: - assert "subset of READY and CONDITIONAL" in str(exc) - else: - raise AssertionError("qta-screen-1.0.0 accepted an unknown setup status") - - exchanges = ["KOSDAQ", "KOSPI", "NASDAQ", "NYSE"] - source_hashes = [] - for source_id in sorted( - f"{role}-{exchange}" - for exchange in exchanges - for role in ("broker", "official") - ): - role, exchange = source_id.split("-", maxsplit=1) - source_hashes.append( - { - "source_id": source_id, - "role": role, - "provider": "self-test", - "exchange": exchange, - "as_of": "2026-07-26", - "path": f"/tmp/{source_id}.txt", - "sha256": "0" * 64, - } - ) - instrument_specs = [ - ("KOSDAQ", "KR", "100002", "1", "KOSDAQ_COMPOSITE", "KRW", "KRX"), - ("KOSPI", "KR", "100001", "1", "KOSPI_COMPOSITE", "KRW", "KRX"), - ("NASDAQ", "US", "AAA", "0.01", "NASDAQ_COMPOSITE", "USD", "NASD"), - ("NASDAQ", "US", "CCC", "0.01", "NASDAQ_COMPOSITE", "USD", "NASD"), - ("NYSE", "US", "BBB", "0.01", "NYSE_COMPOSITE", "USD", "NYSE"), - ] - instruments_v2 = [] - for ( - exchange, - market, - symbol, - tick_size, - benchmark_id, - currency, - venue, - ) in instrument_specs: - instruments_v2.append( - { - "market": market, - "exchange": exchange, - "canonical_symbol": symbol, - "data_symbol": symbol, - "broker_symbol": symbol, - "instrument_type": "COMMON", - "benchmark_id": benchmark_id, - "currency": currency, - "venue": venue, - "ticker_csv": f"/tmp/{exchange}-{symbol}.csv", - "benchmark_csv": f"/tmp/{exchange}-benchmark.csv", - "tick_contract": { - "schema": "qta-tick-contract/v1", - "kind": "RESOLVED_PRICE_LADDER", - "rule_id": f"{exchange}_SELF_TEST", - "effective_date": "2026-07-25", - "reference_price": "100", - "resolved_tick_size": tick_size, - }, - "source_name": "screen-self-test", - "ticker_csv_sha256": "1" * 64, - "benchmark_csv_sha256": "2" * 64, - "broker_tradability_verified": True, - "official_source_id": f"official-{exchange}", - "broker_source_id": f"broker-{exchange}", - } - ) - by_exchange = {} - for exchange in exchanges: - included = sum(spec[0] == exchange for spec in instrument_specs) - by_exchange[exchange] = { - "official": included, - "broker": included, - "catalog_mapped": included, - "included": included, - "excluded": 0, - } - manifest_v2_without_hash = { - "schema": MANIFEST_SCHEMA_V2, - "build_status": "READY", - "as_of": "2026-07-26", - "analysis_date": "2026-07-25", - "catalog_coverage_contract": { - "schema": CATALOG_COVERAGE_SCHEMA, - "minimum_ratio_by_exchange": {exchange: "1" for exchange in EXCHANGE_ORDER}, - "minimum_screenable_ratio_by_exchange": { - exchange: "1" for exchange in EXCHANGE_ORDER - }, - }, - "source_hashes": source_hashes, - "instruments": instruments_v2, - "exclusions": [], - "counts": { - "official_rows": len(instrument_specs), - "broker_rows": len(instrument_specs), - "catalog_mapped_rows": len(instrument_specs), - "included": len(instrument_specs), - "excluded": 0, - "by_exchange": by_exchange, - }, - } - normalized_manifest_v2 = normalize_manifest( - { - **manifest_v2_without_hash, - "manifest_hash": sha256_json(manifest_v2_without_hash), - } - ) - selector_v2 = normalize_selector( - { - "selector_version": SELECTOR_VERSION_V2, - "min_total_score": "60", - "eligible_setup_statuses": ["READY"], - "top_k_by_exchange": { - "KOSDAQ": 1, - "KOSPI": 1, - "NASDAQ": 1, - "NYSE": 1, - }, - "min_selected_by_exchange": { - "KOSDAQ": 1, - "KOSPI": 1, - "NASDAQ": 1, - "NYSE": 1, - }, - "max_blocked_fraction": "0", - } - ) - try: - normalize_selector( - {**selector_v2, "eligible_setup_statuses": ["READY", "UNKNOWN"]} - ) - except ScreenBlockedError as exc: - assert "subset of READY and CONDITIONAL" in str(exc) - else: - raise AssertionError("qta-screen-1.1.0 accepted an unknown setup status") - results_v2 = [ - synthetic_result(market, symbol, "70", "70", "70", "70", "30") - for _, market, symbol, _, _, _, _ in reversed(instrument_specs) - ] - screen_v2_a = finalize_screen_v2( - normalized_manifest_v2, - selector_v2, - results_v2, - ) - screen_v2_b = finalize_screen_v2( - normalized_manifest_v2, - selector_v2, - list(reversed(results_v2)), - ) - assert screen_v2_a == screen_v2_b - assert screen_v2_a["schema"] == SCREEN_SCHEMA_V2 - assert set(screen_v2_a["selected"]) == SUPPORTED_EXCHANGES - assert all( - selected[0]["exchange_rank"] == 1 - for selected in screen_v2_a["selected"].values() - ) - assert all( - set(selected[0]["instrument"]) == INSTRUMENT_V2_FIELDS - for selected in screen_v2_a["selected"].values() - ) - decision_v2_by_symbol = { - decision["canonical_symbol"]: decision for decision in screen_v2_a["decisions"] - } - assert decision_v2_by_symbol["AAA"]["exchange_rank"] == 1 - assert decision_v2_by_symbol["AAA"]["selected"] - assert decision_v2_by_symbol["CCC"]["exchange_rank"] == 2 - assert not decision_v2_by_symbol["CCC"]["selected"] - kr_exchanges = {"KOSPI", "KOSDAQ"} - kr_instruments = [ - item for item in instruments_v2 if item["exchange"] in kr_exchanges - ] - kr_by_exchange = { - exchange: ( - dict(by_exchange[exchange]) - if exchange in kr_exchanges - else { - "official": 0, - "broker": 0, - "catalog_mapped": 0, - "included": 0, - "excluded": 0, - } - ) - for exchange in EXCHANGE_ORDER - } - kr_manifest_without_hash = { - **manifest_v2_without_hash, - "source_hashes": [ - item - for item in source_hashes - if item["exchange"] in kr_exchanges | {"GLOBAL"} - ], - "instruments": kr_instruments, - "counts": { - "official_rows": len(kr_instruments), - "broker_rows": len(kr_instruments), - "catalog_mapped_rows": len(kr_instruments), - "included": len(kr_instruments), - "excluded": 0, - "by_exchange": kr_by_exchange, - }, - } - kr_manifest = normalize_manifest( - { - **kr_manifest_without_hash, - "manifest_hash": sha256_json(kr_manifest_without_hash), - } - ) - kr_selector = normalize_selector( - { - **selector_v2, - "min_selected_by_exchange": { - "KOSPI": 1, - "KOSDAQ": 1, - "NYSE": 0, - "NASDAQ": 0, - }, - } - ) - kr_results = [ - item for item in results_v2 if item["market"] == "KR" - ] - kr_screen = finalize_screen_v2(kr_manifest, kr_selector, kr_results) - assert kr_screen["screen_status"] == "READY" - assert not kr_screen["selected"]["NYSE"] - assert not kr_screen["selected"]["NASDAQ"] - coverage_selector_v2 = normalize_selector( - { - "selector_version": SELECTOR_VERSION_V2, - "min_total_score": "60", - "eligible_setup_statuses": ["READY"], - "top_k_by_exchange": { - "KOSDAQ": 1, - "KOSPI": 1, - "NASDAQ": 1, - "NYSE": 2, - }, - "min_selected_by_exchange": { - "KOSDAQ": 1, - "KOSPI": 1, - "NASDAQ": 1, - "NYSE": 2, - }, - "max_blocked_fraction": "1", - } - ) - coverage_blocked_v2 = finalize_screen_v2( - normalized_manifest_v2, - coverage_selector_v2, - results_v2, - ) - assert coverage_blocked_v2["blocked_fraction"] == "0" - assert coverage_blocked_v2["screen_status"] == "BLOCKED" - assert all(not selected for selected in coverage_blocked_v2["selected"].values()) - tampered_manifest_v2 = { - **manifest_v2_without_hash, - "analysis_date": "2026-07-24", - "manifest_hash": sha256_json(manifest_v2_without_hash), - } - try: - normalize_manifest(tampered_manifest_v2) - except ScreenBlockedError: - pass - else: - raise AssertionError("tampered qta-universe-manifest/v2 was accepted") - - sparse_by_exchange = { - exchange: dict(values) for exchange, values in by_exchange.items() - } - sparse_by_exchange["KOSPI"]["official"] += 1 - sparse_by_exchange["KOSPI"]["excluded"] += 1 - sparse_without_hash = { - **manifest_v2_without_hash, - "catalog_coverage_contract": { - "schema": CATALOG_COVERAGE_SCHEMA, - "minimum_ratio_by_exchange": {exchange: "1" for exchange in EXCHANGE_ORDER}, - "minimum_screenable_ratio_by_exchange": { - exchange: "0.5" for exchange in EXCHANGE_ORDER - }, - }, - "exclusions": [ - { - "exchange": "KOSPI", - "canonical_symbol": "999999", - "broker_symbol": None, - "name": "Missing Catalog", - "reasons": ["missing_eod_mapping"], - "official_source_id": "official-KOSPI", - "broker_source_id": "broker-KOSPI", - } - ], - "counts": { - **manifest_v2_without_hash["counts"], - "official_rows": len(instrument_specs) + 1, - "excluded": 1, - "by_exchange": sparse_by_exchange, - }, - } - try: - normalize_manifest( - { - **sparse_without_hash, - "manifest_hash": sha256_json(sparse_without_hash), - } - ) - except ScreenBlockedError as exc: - assert "catalog mapping and screenable coverage contracts" in str(exc) - else: - raise AssertionError("sparse full-coverage manifest was accepted as READY") - - unscreenable_by_exchange = { - exchange: dict(values) for exchange, values in by_exchange.items() - } - unscreenable_by_exchange["KOSPI"]["official"] += 1 - unscreenable_by_exchange["KOSPI"]["catalog_mapped"] += 1 - unscreenable_by_exchange["KOSPI"]["excluded"] += 1 - unscreenable_without_hash = { - **manifest_v2_without_hash, - "exclusions": [ - { - "exchange": "KOSPI", - "canonical_symbol": "999998", - "broker_symbol": None, - "name": "Not Tradable", - "reasons": ["not_broker_tradable"], - "official_source_id": "official-KOSPI", - "broker_source_id": "broker-KOSPI", - } - ], - "counts": { - **manifest_v2_without_hash["counts"], - "official_rows": len(instrument_specs) + 1, - "catalog_mapped_rows": len(instrument_specs) + 1, - "excluded": 1, - "by_exchange": unscreenable_by_exchange, - }, - } - try: - normalize_manifest( - { - **unscreenable_without_hash, - "manifest_hash": sha256_json(unscreenable_without_hash), - } - ) - except ScreenBlockedError as exc: - assert "catalog mapping and screenable coverage contracts" in str(exc) - else: - raise AssertionError("under-screenable manifest was accepted as READY") - assert ( - normalize_exclusions( - [ - { - "exchange": "KOSPI", - "canonical_symbol": "999999", - "broker_symbol": None, - "name": "", - "reasons": ["not_broker_tradable"], - "official_source_id": "official-KOSPI", - "broker_source_id": None, - } - ] - )[0]["name"] - == "" - ) - print( - canonical_json( - { - "self_test": "PASS", - "selector_version": SELECTOR_VERSION, - "screen_hash": first["screen_hash"], - } - ) - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--manifest") - parser.add_argument("--selector") - parser.add_argument("--output") - parser.add_argument( - "--method-version", - choices=(qta.METHOD_VERSION, qta2.METHOD_VERSION), - default=qta.METHOD_VERSION, - ) - parser.add_argument( - "--workers", - type=int, - default=0, - help="worker processes; 0 selects up to 8 local CPUs", - ) - parser.add_argument("--self-test", action="store_true") - return parser.parse_args() - - -def emit(value: dict[str, Any], output: str | None) -> None: - rendered = json.dumps( - value, ensure_ascii=False, sort_keys=True, indent=2, allow_nan=False - ) - if output: - output_path = Path(output) - output_path.write_text(rendered + "\n", encoding="utf-8") - summary = { - "source_skill": value.get( - "source_skill", - "quant-stock-technical", - ), - "schema": value.get("schema"), - "screen_status": value.get("screen_status"), - "method_version": value.get("method_version"), - "analysis_date": value.get("analysis_date"), - "blocked_count": value.get("blocked_count"), - "screen_hash": value.get("screen_hash"), - "reason": value.get("reason"), - "output": str(output_path.resolve()), - } - print( - json.dumps( - summary, - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - allow_nan=False, - ) - ) - else: - print(rendered) - - -def main() -> int: - args = parse_args() - if args.self_test: - self_test() - return 0 - if not args.manifest or not args.selector: - emit( - { - "source_skill": "quant-stock-technical", - "schema": SCREEN_SCHEMA, - "screen_status": "BLOCKED", - "reason": "--manifest and --selector are required", - }, - args.output, - ) - return 2 - manifest_path = Path(args.manifest).resolve() - failure_schema = SCREEN_SCHEMA - try: - manifest = load_object(manifest_path) - if manifest.get("schema") == MANIFEST_SCHEMA_V2: - failure_schema = SCREEN_SCHEMA_V2 - output = build_screen( - manifest, - load_object(Path(args.selector).resolve()), - manifest_path.parent, - method_version=args.method_version, - workers=( - min(8, max(1, os.cpu_count() or 1)) - if args.workers == 0 - else args.workers - ), - ) - except (ScreenBlockedError, OSError, ValueError, json.JSONDecodeError) as exc: - emit( - { - "source_skill": "quant-stock-technical", - "schema": failure_schema, - "screen_status": "BLOCKED", - "reason": str(exc), - }, - args.output, - ) - return 2 - emit(output, args.output) - return 0 if output["screen_status"] == "READY" else 2 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/skills/quant-stock-technical/scripts/selected_open1h_outcomes.py b/skills/quant-stock-technical/scripts/selected_open1h_outcomes.py deleted file mode 100644 index 49317fb..0000000 --- a/skills/quant-stock-technical/scripts/selected_open1h_outcomes.py +++ /dev/null @@ -1,535 +0,0 @@ -#!/usr/bin/env python3 -"""Collect and compare selected candidates from frozen point-in-time screens.""" - -from __future__ import annotations - -import argparse -import json -import sys -from datetime import date -from decimal import Decimal -from pathlib import Path -from typing import Any - -import analyze_stock as qta1 -import collect_open1h_research as intraday -import evaluate_open1h_research as evaluation -import fetch_kis_kr_eod as shared -import screen_universe as screen -import walk_forward_open1h as walk_forward - - -SCHEMA = "qta-selected-open1h-outcomes/v1" - - -class SelectedOutcomesBlockedError(ValueError): - """Raised when selected point-in-time outcomes cannot be verified.""" - - -def absolute_file(value: str, field: str) -> Path: - path = Path(value).expanduser() - if not path.is_absolute() or not path.is_file() or path.is_symlink(): - raise SelectedOutcomesBlockedError( - f"{field} must be an absolute regular non-symlink file" - ) - return path - - -def absolute_directory(value: str, field: str) -> Path: - path = Path(value).expanduser() - if not path.is_absolute(): - raise SelectedOutcomesBlockedError(f"{field} must be absolute") - if path.exists() and (not path.is_dir() or path.is_symlink()): - raise SelectedOutcomesBlockedError( - f"{field} must be a non-symlink directory" - ) - return path - - -def load_manifest(path: Path) -> dict[str, Any]: - try: - raw = json.loads(path.read_text(encoding="utf-8")) - manifest = screen.normalize_manifest_v2(raw) - except (OSError, json.JSONDecodeError, screen.ScreenBlockedError) as exc: - raise SelectedOutcomesBlockedError( - f"manifest validation failed: {exc}" - ) from exc - supplied_hash = raw.get("manifest_hash") - if not isinstance(supplied_hash, str): - raise SelectedOutcomesBlockedError("manifest_hash is required") - return manifest - - -def selected_rows( - screen_value: dict[str, Any], - *, - market: str, -) -> list[dict[str, Any]]: - selected = screen_value.get("selected") - if not isinstance(selected, dict): - raise SelectedOutcomesBlockedError( - "screen.selected must be an exchange object" - ) - exchanges = intraday.MARKET_CONTRACTS[market]["exchanges"] - rows: list[dict[str, Any]] = [] - for exchange in exchanges: - exchange_rows = selected.get(exchange, []) - if not isinstance(exchange_rows, list): - raise SelectedOutcomesBlockedError( - f"screen.selected.{exchange} must be an array" - ) - for row in exchange_rows: - if ( - not isinstance(row, dict) - or not isinstance(row.get("instrument"), dict) - or not isinstance(row.get("qta"), dict) - ): - raise SelectedOutcomesBlockedError( - "selected row is missing instrument or QTA" - ) - if row["instrument"]["market"] != market: - continue - rows.append(row) - rows.sort( - key=lambda item: ( - exchanges.index(item["instrument"]["exchange"]), - int(item["exchange_rank"]), - item["instrument"]["canonical_symbol"], - ) - ) - return rows - - -def cache_path( - output_directory: Path, - *, - session_date: str, - instrument: dict[str, Any], -) -> Path: - return ( - output_directory - / "bars" - / session_date - / instrument["exchange"] - / f"{instrument['canonical_symbol']}.json" - ) - - -def load_or_fetch( - client: shared.KisReadClient, - output_directory: Path, - *, - market: str, - session_date: str, - instrument: dict[str, Any], -) -> tuple[dict[str, Any], bool]: - path = cache_path( - output_directory, - session_date=session_date, - instrument=instrument, - ) - if path.exists(): - try: - artifact = intraday.normalize_bars_artifact( - json.loads(path.read_text(encoding="utf-8")) - ) - except (OSError, json.JSONDecodeError, intraday.ResearchBlockedError) as exc: - raise SelectedOutcomesBlockedError( - f"cached bars are invalid: {path}: {exc}" - ) from exc - if ( - artifact["market"] != market - or artifact["session_date"] != session_date - or artifact["exchange"] != instrument["exchange"] - or artifact["canonical_symbol"] != instrument["canonical_symbol"] - or artifact["broker_symbol"] != instrument["broker_symbol"] - ): - raise SelectedOutcomesBlockedError( - f"cached bars identity mismatch: {path}" - ) - return artifact, True - raw_rows = intraday.fetch_raw_rows( - client, - market=market, - exchange=instrument["exchange"], - broker_symbol=instrument["broker_symbol"], - session_date=session_date, - ) - artifact = intraday.bars_artifact( - instrument=instrument, - market=market, - session_date=session_date, - raw_rows=raw_rows, - ) - shared.atomic_write_json(path, artifact) - return artifact, False - - -def benchmark_open_regime_bps( - instrument: dict[str, Any], - *, - analysis_date: str, - session_date: str, -) -> Decimal | None: - rows = { - row.day: row - for row in qta1.read_csv(instrument["benchmark_csv"]) - } - previous = rows.get(date.fromisoformat(analysis_date)) - current = rows.get(date.fromisoformat(session_date)) - if previous is None or current is None: - return None - return ( - Decimal(str(current.open)) / Decimal(str(previous.close)) - Decimal(1) - ) * Decimal(10000) - - -def compact( - *, - method: str, - analysis_date: str, - session_date: str, - selected: dict[str, Any], - artifact: dict[str, Any] | None, - bars_reason: str | None, -) -> dict[str, Any]: - instrument = selected["instrument"] - qta = selected["qta"] - regime = benchmark_open_regime_bps( - instrument, - analysis_date=analysis_date, - session_date=session_date, - ) - base = { - "method_version": method, - "analysis_date": analysis_date, - "session_date": session_date, - "exchange": instrument["exchange"], - "canonical_symbol": instrument["canonical_symbol"], - "broker_symbol": instrument["broker_symbol"], - "exchange_rank": selected["exchange_rank"], - "qta_total_score": walk_forward.decimal_text( - walk_forward.decimal_value(qta["total_score"], "total_score") - ), - "entry_price": format( - walk_forward.decimal_value(qta["entry_price"], "entry_price"), - "f", - ), - "stop_price": format( - walk_forward.decimal_value(qta["stop_price"], "stop_price"), - "f", - ), - "take_profit_price": format( - walk_forward.decimal_value( - qta["take_profit_price"], "take_profit_price" - ), - "f", - ), - "market_open_regime_bps": ( - walk_forward.decimal_text(regime) if regime is not None else None - ), - "market_open_regime_pass": ( - regime >= 0 if regime is not None else None - ), - "bars_status": "READY" if artifact is not None else "BLOCKED", - "bars_reason": bars_reason, - } - if artifact is None: - return { - **base, - "coverage_status": None, - "bar_count": 0, - "return_60m_bps": None, - "entry_touched_by_minute_high": None, - "entry_crossed_by_bar_close": None, - "entry_cross_bar_time": None, - "end_return_from_entry_bps": None, - "bars_hash": None, - } - bars = artifact["bars"] - summary = artifact["summary"] - entry = walk_forward.decimal_value(qta["entry_price"], "entry_price") - cross_time = evaluation.first_bar_close_cross(bars, entry) - return { - **base, - "coverage_status": intraday.coverage_summary( - bars, market=artifact["market"] - )["coverage_status"], - "bar_count": len(bars), - "return_60m_bps": summary["return_60m_bps"], - "entry_touched_by_minute_high": ( - Decimal(summary["window_high"]) >= entry - ), - "entry_crossed_by_bar_close": cross_time is not None, - "entry_cross_bar_time": cross_time, - "end_return_from_entry_bps": ( - intraday.bps(Decimal(summary["end_close"]), entry) - if cross_time is not None - else None - ), - "bars_hash": artifact["bars_hash"], - } - - -def run(args: argparse.Namespace) -> dict[str, Any]: - manifest_path = absolute_file(args.manifest, "manifest") - output_directory = absolute_directory( - args.output_directory, "output_directory" - ) - output_directory.mkdir(parents=True, exist_ok=True) - manifest = load_manifest(manifest_path) - if manifest["as_of"] != args.session_date: - raise SelectedOutcomesBlockedError( - "manifest.as_of must equal session_date" - ) - screens: dict[str, dict[str, Any]] = {} - selections: dict[str, list[dict[str, Any]]] = {} - for path_text in args.screen: - path = absolute_file(path_text, "screen") - value = evaluation.normalized_screen(path) - method = value["method_version"] - if method in screens: - raise SelectedOutcomesBlockedError( - f"duplicate method screen: {method}" - ) - if value["manifest_hash"] != manifest["manifest_hash"]: - raise SelectedOutcomesBlockedError( - f"{method} screen manifest hash mismatch" - ) - if value["analysis_date"] != manifest["analysis_date"]: - raise SelectedOutcomesBlockedError( - f"{method} screen analysis date mismatch" - ) - screens[method] = value - selections[method] = selected_rows(value, market=args.market) - if not screens: - raise SelectedOutcomesBlockedError("at least one screen is required") - selected_by_key: dict[tuple[str, str], dict[str, Any]] = {} - for method_rows in selections.values(): - for row in method_rows: - instrument = row["instrument"] - selected_by_key[ - (instrument["exchange"], instrument["canonical_symbol"]) - ] = instrument - key, secret, token = shared.load_api_credentials(args.environment) - client = shared.KisReadClient( - environment=args.environment, - app_key=key, - app_secret=secret, - access_token=token, - interval_ms=args.request_interval_ms, - ) - artifacts: dict[tuple[str, str], dict[str, Any] | None] = {} - errors: dict[tuple[str, str], str | None] = {} - cache_hits = 0 - for key_value, instrument in sorted(selected_by_key.items()): - try: - artifact, cache_hit = load_or_fetch( - client, - output_directory, - market=args.market, - session_date=args.session_date, - instrument=instrument, - ) - artifacts[key_value] = artifact - errors[key_value] = None - cache_hits += int(cache_hit) - except ( - shared.EodBlockedError, - intraday.ResearchBlockedError, - SelectedOutcomesBlockedError, - ) as exc: - artifacts[key_value] = None - errors[key_value] = str(exc) - rows: list[dict[str, Any]] = [] - for method, method_rows in selections.items(): - for selected in method_rows: - instrument = selected["instrument"] - key_value = ( - instrument["exchange"], - instrument["canonical_symbol"], - ) - rows.append( - compact( - method=method, - analysis_date=manifest["analysis_date"], - session_date=args.session_date, - selected=selected, - artifact=artifacts[key_value], - bars_reason=errors[key_value], - ) - ) - methods = sorted({row["method_version"] for row in rows}) - exchanges = intraday.MARKET_CONTRACTS[args.market]["exchanges"] - rows.sort( - key=lambda item: ( - methods.index(item["method_version"]), - exchanges.index(item["exchange"]), - int(item["exchange_rank"]), - item["canonical_symbol"], - ) - ) - summary = { - method: { - exchange: walk_forward.summarize( - [ - row - for row in rows - if row["method_version"] == method - and row["exchange"] == exchange - ] - ) - for exchange in exchanges - } - for method in methods - } - without_hash = { - "schema": SCHEMA, - "source_skill": "quant-stock-technical", - "manifest_path": str(manifest_path), - "manifest_hash": manifest["manifest_hash"], - "analysis_date": manifest["analysis_date"], - "session_date": args.session_date, - "market": args.market, - "methods": methods, - "screen_hashes": { - method: screens[method]["screen_hash"] for method in methods - }, - "selected_union_count": len(selected_by_key), - "summary_by_method_and_exchange": summary, - "candidate_rows": rows, - "request_count": client.request_count, - "retry_count": client.retry_count, - "cache_hit_count": cache_hits, - "api_mutation_count": 0, - "live_enabled": False, - "validation_status": "RESEARCH_ONLY", - } - output = { - **without_hash, - "outcomes_hash": shared.sha256_bytes( - shared.canonical_json(without_hash).encode("utf-8") - ), - } - shared.atomic_write_json(output_directory / "selected-outcomes.json", output) - return output - - -def self_test() -> None: - rows = [ - { - "method_version": "qta-2.0.0", - "session_date": "2026-07-29", - "exchange": "NASDAQ", - "bars_status": "READY", - "return_60m_bps": "50", - "entry_crossed_by_bar_close": True, - "end_return_from_entry_bps": "10", - "market_open_regime_pass": True, - } - ] - summary = walk_forward.summarize(rows) - assert summary["mean_return_60m_bps"] == "50.00" - print( - json.dumps( - { - "schema": SCHEMA, - "self_test": "PASS", - "api_mutation_count": 0, - "live_enabled": False, - }, - sort_keys=True, - ) - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--self-test", action="store_true") - parser.add_argument("--manifest") - parser.add_argument("--screen", action="append", default=[]) - parser.add_argument("--market", choices=("KR", "US")) - parser.add_argument("--session-date") - parser.add_argument("--output-directory") - parser.add_argument("--environment", choices=("live", "paper"), default="live") - parser.add_argument("--request-interval-ms", type=int, default=120) - args = parser.parse_args() - if not args.self_test: - missing = [ - name - for name in ( - "manifest", - "market", - "session_date", - "output_directory", - ) - if getattr(args, name) is None - ] - if missing: - parser.error(f"missing required arguments: {', '.join(missing)}") - if not args.screen: - parser.error("--screen is required at least once") - try: - date.fromisoformat(args.session_date) - except ValueError: - parser.error("--session-date must be YYYY-MM-DD") - floor = 1000 if args.environment == "paper" else 100 - if args.request_interval_ms < floor: - parser.error( - f"--request-interval-ms must be at least {floor}" - ) - return args - - -def main() -> int: - args = parse_args() - if args.self_test: - self_test() - return 0 - try: - output = run(args) - except ( - SelectedOutcomesBlockedError, - walk_forward.WalkForwardBlockedError, - shared.EodBlockedError, - intraday.ResearchBlockedError, - OSError, - ValueError, - ) as exc: - print( - json.dumps( - { - "schema": SCHEMA, - "status": "BLOCKED", - "reason": str(exc), - "api_mutation_count": 0, - "live_enabled": False, - }, - sort_keys=True, - ), - file=sys.stderr, - ) - return 2 - print( - json.dumps( - { - "schema": SCHEMA, - "status": "READY", - "outcomes_hash": output["outcomes_hash"], - "selected_union_count": output["selected_union_count"], - "request_count": output["request_count"], - "retry_count": output["retry_count"], - "api_mutation_count": 0, - "live_enabled": False, - "output": str( - Path(args.output_directory) / "selected-outcomes.json" - ), - }, - sort_keys=True, - ) - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/skills/quant-stock-technical/scripts/walk_forward_open1h.py b/skills/quant-stock-technical/scripts/walk_forward_open1h.py deleted file mode 100644 index b311ecc..0000000 --- a/skills/quant-stock-technical/scripts/walk_forward_open1h.py +++ /dev/null @@ -1,854 +0,0 @@ -#!/usr/bin/env python3 -"""Run a deterministic selected-candidate opening-hour walk-forward study.""" - -from __future__ import annotations - -import argparse -import json -import statistics -import sys -from concurrent.futures import ProcessPoolExecutor -from datetime import date -from decimal import Decimal, InvalidOperation, ROUND_HALF_EVEN -from pathlib import Path -from typing import Any, Iterable - -import analyze_stock as qta1 -import analyze_stock_v2 as qta2 -import collect_open1h_research as intraday -import evaluate_open1h_research as evaluation -import fetch_kis_kr_eod as shared -import screen_universe as screen - - -SCHEMA = "qta-selected-open1h-walk-forward/v1" -UNIVERSE_MODE = "STATIC_SOURCE_MANIFEST_SURVIVOR_BIASED" -METHODS = (qta1.METHOD_VERSION, qta2.METHOD_VERSION) -EXCHANGES = ("KOSPI", "KOSDAQ") - - -class WalkForwardBlockedError(ValueError): - """Raised when a walk-forward study cannot be completed causally.""" - - -def decimal_value(value: Any, field: str) -> Decimal: - if isinstance(value, bool): - raise WalkForwardBlockedError(f"{field} must be decimal") - try: - parsed = Decimal(str(value)) - except (InvalidOperation, ValueError) as exc: - raise WalkForwardBlockedError(f"{field} must be decimal") from exc - if not parsed.is_finite(): - raise WalkForwardBlockedError(f"{field} must be finite") - return parsed - - -def decimal_text(value: Decimal, places: str = "0.01") -> str: - return format( - value.quantize(Decimal(places), rounding=ROUND_HALF_EVEN), - "f", - ) - - -def absolute_file(value: str, field: str) -> Path: - path = Path(value).expanduser() - if not path.is_absolute() or not path.is_file() or path.is_symlink(): - raise WalkForwardBlockedError( - f"{field} must be an absolute regular non-symlink file" - ) - return path - - -def absolute_directory(value: str, field: str) -> Path: - path = Path(value).expanduser() - if not path.is_absolute(): - raise WalkForwardBlockedError(f"{field} must be absolute") - if path.exists() and (not path.is_dir() or path.is_symlink()): - raise WalkForwardBlockedError( - f"{field} must be a non-symlink directory" - ) - return path - - -def load_manifest(path: Path) -> tuple[dict[str, Any], list[dict[str, Any]]]: - try: - raw = json.loads(path.read_text(encoding="utf-8")) - manifest = screen.normalize_manifest_v2(raw) - except (OSError, json.JSONDecodeError, screen.ScreenBlockedError) as exc: - raise WalkForwardBlockedError( - f"source manifest validation failed: {exc}" - ) from exc - instruments = [ - instrument - for instrument in manifest["instruments"] - if instrument["market"] == "KR" - and instrument["exchange"] in EXCHANGES - ] - if not instruments: - raise WalkForwardBlockedError( - "source manifest has no KOSPI/KOSDAQ instruments" - ) - instruments.sort( - key=lambda item: ( - EXCHANGES.index(item["exchange"]), - item["canonical_symbol"], - ) - ) - for instrument in instruments: - ticker_path = absolute_file( - instrument["ticker_csv"], "instrument.ticker_csv" - ) - benchmark_path = absolute_file( - instrument["benchmark_csv"], "instrument.benchmark_csv" - ) - if shared.sha256_file(ticker_path) != instrument["ticker_csv_sha256"]: - raise WalkForwardBlockedError( - f"ticker hash mismatch: {instrument['canonical_symbol']}" - ) - if ( - shared.sha256_file(benchmark_path) - != instrument["benchmark_csv_sha256"] - ): - raise WalkForwardBlockedError( - f"benchmark hash mismatch: {instrument['benchmark_id']}" - ) - return manifest, instruments - - -def shared_session_pairs( - instruments: list[dict[str, Any]], - *, - end_session: date, - sessions: int, -) -> list[tuple[date, date]]: - benchmark_paths = { - exchange: next( - Path(item["benchmark_csv"]) - for item in instruments - if item["exchange"] == exchange - ) - for exchange in EXCHANGES - } - date_sets = [ - {row.day for row in qta1.read_csv(str(path))} - for path in benchmark_paths.values() - ] - common = sorted( - day for day in set.intersection(*date_sets) if day <= end_session - ) - if len(common) < sessions + 1: - raise WalkForwardBlockedError( - f"benchmarks expose only {max(0, len(common) - 1)} usable sessions" - ) - targets = common[-sessions:] - positions = {day: index for index, day in enumerate(common)} - return [(common[positions[target] - 1], target) for target in targets] - - -def blocked_payload( - instrument: dict[str, Any], - reason: str, - method_version: str, -) -> dict[str, Any]: - return { - "source_skill": "quant-stock-technical", - "result_schema": "quant-stock-technical/v1", - "calculation_status": "BLOCKED", - "reason": reason, - "method_version": method_version, - "market": "KR", - "ticker": instrument["canonical_symbol"], - } - - -def analyze_pair_task( - task: tuple[dict[str, Any], str], -) -> dict[str, Any]: - instrument, analysis_date_text = task - analysis_date = date.fromisoformat(analysis_date_text) - try: - ticker_rows, benchmark_rows = qta1.align( - qta1.read_csv(instrument["ticker_csv"]), - qta1.read_csv(instrument["benchmark_csv"]), - analysis_date, - ) - historical_tick = shared.tick_size( - instrument["exchange"], - Decimal(str(ticker_rows[-1].close)), - ) - common = { - "ticker_bars": ticker_rows, - "benchmark_bars": benchmark_rows, - "market": "KR", - "ticker": instrument["canonical_symbol"], - "tick_size": float(historical_tick), - "source_name": instrument["source_name"], - } - payloads = { - qta1.METHOD_VERSION: qta1.calculate(**common), - qta2.METHOD_VERSION: qta2.calculate(**common), - } - previous_close = Decimal(str(ticker_rows[-1].close)) - except (qta1.BlockedError, OSError, ValueError) as exc: - payloads = { - method: blocked_payload(instrument, str(exc), method) - for method in METHODS - } - previous_close = None - return { - "instrument": instrument, - "previous_close": ( - format(previous_close, "f") if previous_close is not None else None - ), - "payloads": payloads, - } - - -def eligible(payload: dict[str, Any]) -> bool: - return ( - payload.get("calculation_status") == "READY" - and payload.get("setup_status") == "READY" - and decimal_value(payload.get("total_score"), "total_score") - >= Decimal("60") - ) - - -def select_candidates( - analyses: list[dict[str, Any]], - *, - method: str, - top_k: int, -) -> list[dict[str, Any]]: - selected: list[dict[str, Any]] = [] - for exchange in EXCHANGES: - ready = [ - item - for item in analyses - if item["instrument"]["exchange"] == exchange - and eligible(item["payloads"][method]) - ] - ready.sort(key=lambda item: screen.ranking_key(item["payloads"][method])) - for rank, item in enumerate(ready[:top_k], start=1): - selected.append( - { - **item, - "exchange_rank": rank, - } - ) - return selected - - -def bars_cache_path( - output_directory: Path, - *, - session_date: str, - instrument: dict[str, Any], -) -> Path: - return ( - output_directory - / "bars" - / session_date - / instrument["exchange"] - / f"{instrument['canonical_symbol']}.json" - ) - - -def load_or_fetch_bars( - client: shared.KisReadClient, - output_directory: Path, - *, - session_date: str, - instrument: dict[str, Any], -) -> tuple[dict[str, Any], bool]: - path = bars_cache_path( - output_directory, - session_date=session_date, - instrument=instrument, - ) - if path.exists(): - try: - artifact = intraday.normalize_bars_artifact( - json.loads(path.read_text(encoding="utf-8")) - ) - except (OSError, json.JSONDecodeError, intraday.ResearchBlockedError) as exc: - raise WalkForwardBlockedError( - f"invalid cached bars: {path}: {exc}" - ) from exc - if ( - artifact["session_date"] != session_date - or artifact["exchange"] != instrument["exchange"] - or artifact["canonical_symbol"] != instrument["canonical_symbol"] - or artifact["broker_symbol"] != instrument["broker_symbol"] - ): - raise WalkForwardBlockedError( - f"cached bars identity mismatch: {path}" - ) - return artifact, True - raw_rows = intraday.fetch_raw_rows( - client, - market="KR", - exchange=instrument["exchange"], - broker_symbol=instrument["broker_symbol"], - session_date=session_date, - ) - artifact = intraday.bars_artifact( - instrument=instrument, - market="KR", - session_date=session_date, - raw_rows=raw_rows, - ) - shared.atomic_write_json(path, artifact) - return artifact, False - - -def benchmark_rows( - instruments: list[dict[str, Any]], -) -> dict[str, dict[date, qta1.RawBar]]: - output: dict[str, dict[date, qta1.RawBar]] = {} - for exchange in EXCHANGES: - path = next( - item["benchmark_csv"] - for item in instruments - if item["exchange"] == exchange - ) - output[exchange] = {row.day: row for row in qta1.read_csv(path)} - return output - - -def open_regime_bps( - rows: dict[date, qta1.RawBar], - *, - analysis_date: date, - session_date: date, -) -> Decimal: - previous = rows.get(analysis_date) - current = rows.get(session_date) - if previous is None or current is None: - raise WalkForwardBlockedError( - "benchmark is missing an analysis/session pair" - ) - return ( - Decimal(str(current.open)) / Decimal(str(previous.close)) - Decimal(1) - ) * Decimal(10000) - - -def compact_candidate( - *, - method: str, - analysis_date: date, - session_date: date, - selected: dict[str, Any], - artifact: dict[str, Any] | None, - bars_error: str | None, - regime_bps: Decimal, -) -> dict[str, Any]: - instrument = selected["instrument"] - qta = selected["payloads"][method] - base = { - "method_version": method, - "analysis_date": analysis_date.isoformat(), - "session_date": session_date.isoformat(), - "exchange": instrument["exchange"], - "canonical_symbol": instrument["canonical_symbol"], - "broker_symbol": instrument["broker_symbol"], - "exchange_rank": selected["exchange_rank"], - "qta_total_score": decimal_text( - decimal_value(qta["total_score"], "qta_total_score") - ), - "qta_short_score": decimal_text( - decimal_value(qta["short"]["score"], "qta_short_score") - ), - "qta_medium_score": decimal_text( - decimal_value(qta["medium"]["score"], "qta_medium_score") - ), - "qta_long_score": decimal_text( - decimal_value(qta["long"]["score"], "qta_long_score") - ), - "qta_risk_score": decimal_text( - decimal_value(qta["risk"]["score"], "qta_risk_score") - ), - "entry_price": format(decimal_value(qta["entry_price"], "entry"), "f"), - "stop_price": format(decimal_value(qta["stop_price"], "stop"), "f"), - "take_profit_price": format( - decimal_value(qta["take_profit_price"], "take_profit"), "f" - ), - "previous_close": selected["previous_close"], - "market_open_regime_bps": decimal_text(regime_bps), - "market_open_regime_pass": regime_bps >= 0, - "bars_status": "READY" if artifact is not None else "BLOCKED", - "bars_reason": bars_error, - } - if artifact is None: - return { - **base, - "coverage_status": None, - "bar_count": 0, - "return_60m_bps": None, - "maximum_excursion_bps": None, - "minimum_excursion_bps": None, - "entry_touched_by_minute_high": None, - "entry_crossed_by_bar_close": None, - "entry_cross_bar_time": None, - "end_return_from_entry_bps": None, - "bars_hash": None, - } - bars = artifact["bars"] - summary = artifact["summary"] - entry = decimal_value(qta["entry_price"], "entry") - cross_time = evaluation.first_bar_close_cross(bars, entry) - end_close = Decimal(summary["end_close"]) - return { - **base, - "coverage_status": intraday.coverage_summary( - bars, market="KR" - )["coverage_status"], - "bar_count": len(bars), - "return_60m_bps": summary["return_60m_bps"], - "maximum_excursion_bps": summary["maximum_excursion_bps"], - "minimum_excursion_bps": summary["minimum_excursion_bps"], - "entry_touched_by_minute_high": ( - Decimal(summary["window_high"]) >= entry - ), - "entry_crossed_by_bar_close": cross_time is not None, - "entry_cross_bar_time": cross_time, - "end_return_from_entry_bps": ( - intraday.bps(end_close, entry) if cross_time is not None else None - ), - "bars_hash": artifact["bars_hash"], - } - - -def mean(values: list[Decimal]) -> str | None: - if not values: - return None - return decimal_text(sum(values, Decimal(0)) / Decimal(len(values))) - - -def median(values: list[Decimal]) -> str | None: - if not values: - return None - return decimal_text(statistics.median(values)) - - -def fraction(numerator: int, denominator: int) -> str | None: - if denominator == 0: - return None - return decimal_text( - Decimal(numerator) / Decimal(denominator), - "0.000001", - ) - - -def summarize(rows: list[dict[str, Any]]) -> dict[str, Any]: - ready = [row for row in rows if row["bars_status"] == "READY"] - returns = [ - decimal_value(row["return_60m_bps"], "return_60m_bps") - for row in ready - ] - crossed = [ - row for row in ready if row["entry_crossed_by_bar_close"] is True - ] - entry_returns = [ - decimal_value(row["end_return_from_entry_bps"], "entry return") - for row in crossed - ] - regime_evaluated = [ - row - for row in ready - if isinstance(row["market_open_regime_pass"], bool) - ] - regime_ready = [ - row for row in ready if row["market_open_regime_pass"] is True - ] - regime_returns = [ - decimal_value(row["return_60m_bps"], "regime return") - for row in regime_ready - ] - regime_crossed = [ - row - for row in crossed - if row["market_open_regime_pass"] is True - ] - regime_entry_returns = [ - decimal_value(row["end_return_from_entry_bps"], "regime entry return") - for row in regime_crossed - ] - session_means: list[Decimal] = [] - for session_date in sorted({row["session_date"] for row in ready}): - values = [ - decimal_value(row["return_60m_bps"], "session return") - for row in ready - if row["session_date"] == session_date - ] - if values: - session_means.append( - sum(values, Decimal(0)) / Decimal(len(values)) - ) - return { - "selected_count": len(rows), - "bars_ready_count": len(ready), - "bars_blocked_count": len(rows) - len(ready), - "mean_return_60m_bps": mean(returns), - "median_return_60m_bps": median(returns), - "positive_fraction": fraction( - sum(value > 0 for value in returns), len(returns) - ), - "entry_cross_count": len(crossed), - "entry_cross_fraction": fraction(len(crossed), len(ready)), - "mean_end_return_from_entry_bps": mean(entry_returns), - "median_end_return_from_entry_bps": median(entry_returns), - "positive_end_return_from_entry_fraction": fraction( - sum(value > 0 for value in entry_returns), - len(entry_returns), - ), - "market_open_regime_evaluated_count": len(regime_evaluated), - "market_open_regime_pass_count": len(regime_ready), - "market_open_regime_pass_fraction": fraction( - len(regime_ready), len(regime_evaluated) - ), - "market_open_regime_pass_mean_return_60m_bps": mean(regime_returns), - "market_open_regime_entry_cross_count": len(regime_crossed), - "market_open_regime_cross_mean_end_return_from_entry_bps": mean( - regime_entry_returns - ), - "mean_equal_weight_session_return_60m_bps": mean(session_means), - } - - -def grouped_summary(rows: list[dict[str, Any]]) -> dict[str, Any]: - return { - method: { - exchange: summarize( - [ - row - for row in rows - if row["method_version"] == method - and row["exchange"] == exchange - ] - ) - for exchange in EXCHANGES - } - for method in METHODS - } - - -def run(args: argparse.Namespace) -> dict[str, Any]: - manifest_path = absolute_file(args.manifest, "manifest") - output_directory = absolute_directory( - args.output_directory, "output_directory" - ) - output_directory.mkdir(parents=True, exist_ok=True) - manifest, instruments = load_manifest(manifest_path) - pairs = shared_session_pairs( - instruments, - end_session=date.fromisoformat(args.end_session), - sessions=args.sessions, - ) - benchmark = benchmark_rows(instruments) - key, secret, token = shared.load_api_credentials(args.environment) - client = shared.KisReadClient( - environment=args.environment, - app_key=key, - app_secret=secret, - access_token=token, - interval_ms=args.request_interval_ms, - ) - workers = min(8, max(1, args.workers)) - candidate_rows: list[dict[str, Any]] = [] - session_receipts: list[dict[str, Any]] = [] - cache_hits = 0 - for analysis_date, session_date in pairs: - tasks = [ - (instrument, analysis_date.isoformat()) - for instrument in instruments - ] - with ProcessPoolExecutor(max_workers=workers) as executor: - analyses = list( - executor.map(analyze_pair_task, tasks, chunksize=16) - ) - selections = { - method: select_candidates( - analyses, - method=method, - top_k=args.top_k, - ) - for method in METHODS - } - selected_by_key: dict[ - tuple[str, str], dict[str, Any] - ] = {} - for method_rows in selections.values(): - for item in method_rows: - instrument = item["instrument"] - selected_by_key[ - (instrument["exchange"], instrument["canonical_symbol"]) - ] = item - artifacts: dict[tuple[str, str], dict[str, Any] | None] = {} - errors: dict[tuple[str, str], str | None] = {} - for key_value, item in sorted(selected_by_key.items()): - try: - artifact, cache_hit = load_or_fetch_bars( - client, - output_directory, - session_date=session_date.isoformat(), - instrument=item["instrument"], - ) - artifacts[key_value] = artifact - errors[key_value] = None - cache_hits += int(cache_hit) - except ( - shared.EodBlockedError, - intraday.ResearchBlockedError, - WalkForwardBlockedError, - ) as exc: - artifacts[key_value] = None - errors[key_value] = str(exc) - for method, method_rows in selections.items(): - for item in method_rows: - instrument = item["instrument"] - key_value = ( - instrument["exchange"], - instrument["canonical_symbol"], - ) - regime = open_regime_bps( - benchmark[instrument["exchange"]], - analysis_date=analysis_date, - session_date=session_date, - ) - candidate_rows.append( - compact_candidate( - method=method, - analysis_date=analysis_date, - session_date=session_date, - selected=item, - artifact=artifacts[key_value], - bars_error=errors[key_value], - regime_bps=regime, - ) - ) - session_receipts.append( - { - "analysis_date": analysis_date.isoformat(), - "session_date": session_date.isoformat(), - "analyzed_instrument_count": len(analyses), - "qta1_blocked_count": sum( - item["payloads"][qta1.METHOD_VERSION].get( - "calculation_status" - ) - != "READY" - for item in analyses - ), - "qta2_blocked_count": sum( - item["payloads"][qta2.METHOD_VERSION].get( - "calculation_status" - ) - != "READY" - for item in analyses - ), - "qta1_selected_count": len( - selections[qta1.METHOD_VERSION] - ), - "qta2_selected_count": len( - selections[qta2.METHOD_VERSION] - ), - "unique_bar_request_candidates": len(selected_by_key), - "bars_blocked_count": sum( - artifact is None for artifact in artifacts.values() - ), - } - ) - print( - json.dumps(session_receipts[-1], sort_keys=True), - file=sys.stderr, - flush=True, - ) - candidate_rows.sort( - key=lambda item: ( - item["session_date"], - METHODS.index(item["method_version"]), - EXCHANGES.index(item["exchange"]), - item["exchange_rank"], - item["canonical_symbol"], - ) - ) - without_hash = { - "schema": SCHEMA, - "source_skill": "quant-stock-technical", - "source_manifest_path": str(manifest_path), - "source_manifest_hash": manifest["manifest_hash"], - "source_manifest_as_of": manifest["as_of"], - "universe_mode": UNIVERSE_MODE, - "universe_limitations": [ - ( - "the source manifest membership is held static across earlier " - "sessions, creating survivorship and membership bias" - ), - ( - "historical bid-ask spreads and exact quote-cycle market-regime " - "states are unavailable" - ), - ( - "market_open_regime_bps is the exact benchmark opening gap, " - "not the runner's later intraday regime observation" - ), - ], - "market": "KR", - "methods": list(METHODS), - "top_k_per_exchange": args.top_k, - "minimum_total_score": "60", - "session_count": len(pairs), - "first_session_date": pairs[0][1].isoformat(), - "last_session_date": pairs[-1][1].isoformat(), - "session_receipts": session_receipts, - "summary_by_method_and_exchange": grouped_summary(candidate_rows), - "candidate_rows": candidate_rows, - "request_count": client.request_count, - "retry_count": client.retry_count, - "cache_hit_count": cache_hits, - "api_mutation_count": 0, - "live_enabled": False, - "validation_status": "RESEARCH_ONLY", - } - output = { - **without_hash, - "walk_forward_hash": shared.sha256_bytes( - shared.canonical_json(without_hash).encode("utf-8") - ), - } - output_path = output_directory / "walk-forward.json" - shared.atomic_write_json(output_path, output) - return output - - -def self_test() -> None: - rows = [ - { - "method_version": qta1.METHOD_VERSION, - "session_date": "2026-01-02", - "exchange": "KOSPI", - "bars_status": "READY", - "return_60m_bps": "100", - "entry_crossed_by_bar_close": True, - "end_return_from_entry_bps": "25", - "market_open_regime_pass": True, - }, - { - "method_version": qta1.METHOD_VERSION, - "session_date": "2026-01-03", - "exchange": "KOSPI", - "bars_status": "READY", - "return_60m_bps": "-50", - "entry_crossed_by_bar_close": False, - "end_return_from_entry_bps": None, - "market_open_regime_pass": False, - }, - ] - summary = summarize(rows) - assert summary["mean_return_60m_bps"] == "25.00" - assert summary["positive_fraction"] == "0.500000" - assert summary["entry_cross_fraction"] == "0.500000" - assert summary["positive_end_return_from_entry_fraction"] == "1.000000" - assert summary["market_open_regime_pass_mean_return_60m_bps"] == "100.00" - assert summary["market_open_regime_entry_cross_count"] == 1 - print( - json.dumps( - { - "schema": SCHEMA, - "self_test": "PASS", - "api_mutation_count": 0, - "live_enabled": False, - }, - sort_keys=True, - ) - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--self-test", action="store_true") - parser.add_argument("--manifest") - parser.add_argument("--output-directory") - parser.add_argument("--end-session") - parser.add_argument("--sessions", type=int, default=20) - parser.add_argument("--top-k", type=int, default=2) - parser.add_argument("--environment", choices=("live", "paper"), default="live") - parser.add_argument("--request-interval-ms", type=int, default=120) - parser.add_argument("--workers", type=int, default=8) - args = parser.parse_args() - if not args.self_test: - missing = [ - name - for name in ("manifest", "output_directory", "end_session") - if getattr(args, name) is None - ] - if missing: - parser.error(f"missing required arguments: {', '.join(missing)}") - try: - date.fromisoformat(args.end_session) - except ValueError as exc: - parser.error("--end-session must be YYYY-MM-DD") - if args.sessions < 2 or args.sessions > 252: - parser.error("--sessions must be between 2 and 252") - if args.top_k < 1 or args.top_k > 20: - parser.error("--top-k must be between 1 and 20") - floor = 1000 if args.environment == "paper" else 100 - if args.request_interval_ms < floor: - parser.error( - f"--request-interval-ms must be at least {floor}" - ) - if args.workers < 1 or args.workers > 8: - parser.error("--workers must be between 1 and 8") - return args - - -def main() -> int: - args = parse_args() - if args.self_test: - self_test() - return 0 - try: - output = run(args) - except ( - WalkForwardBlockedError, - shared.EodBlockedError, - intraday.ResearchBlockedError, - OSError, - ValueError, - ) as exc: - print( - json.dumps( - { - "schema": SCHEMA, - "status": "BLOCKED", - "reason": str(exc), - "api_mutation_count": 0, - "live_enabled": False, - }, - sort_keys=True, - ), - file=sys.stderr, - ) - return 2 - print( - json.dumps( - { - "schema": SCHEMA, - "status": "READY", - "session_count": output["session_count"], - "walk_forward_hash": output["walk_forward_hash"], - "output": str( - Path(args.output_directory) / "walk-forward.json" - ), - "request_count": output["request_count"], - "retry_count": output["retry_count"], - "api_mutation_count": 0, - "live_enabled": False, - }, - sort_keys=True, - ) - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/skills/stock-scenario-story/SKILL.md b/skills/stock-scenario-story/SKILL.md index 86d57e0..08a335c 100644 --- a/skills/stock-scenario-story/SKILL.md +++ b/skills/stock-scenario-story/SKILL.md @@ -1,6 +1,6 @@ --- name: stock-scenario-story -description: Research and perform a vivid, numbers-free, evidence-grounded stock story only after a complete quant-stock-technical JSON result has passed the bundled handoff validator. Use when the user wants a flamboyant showman-style qualitative tale about a company's products, characters, competition, customers, catalysts, culture, policy pressures, dreams, and dangers as pure entertainment. Never run standalone, reveal or restate upstream numbers, reconstruct missing quant inputs, or present the performance as a forecast or personalized investment advice. +description: Research and perform a vivid, numbers-free, evidence-grounded stock story only after a complete external QTA JSON result matching quant-stock-technical/v1 has passed the bundled handoff validator. Use when the user wants a flamboyant showman-style qualitative tale about a company's products, characters, competition, customers, catalysts, culture, policy pressures, dreams, and dangers as pure entertainment. Never run standalone, reveal or restate upstream numbers, reconstruct missing quant inputs, or present the performance as a forecast or personalized investment advice. --- # Stock Scenario Story @@ -9,7 +9,10 @@ Turn a validated deterministic technical result into an entertaining but auditab ## Mandatory invocation gate -Require the complete JSON output from `$quant-stock-technical`. A summary, screenshot, Markdown table, manually typed score, or output from another calculator is insufficient. +Require the complete JSON output from the standalone QTA system. It must match +the legacy `quant-stock-technical/v1` contract; the producer is not a required +marketplace skill. A summary, screenshot, Markdown table, manually typed score, +or output from another calculator is insufficient. Run: @@ -17,7 +20,9 @@ Run: python3 scripts/validate_quant_handoff.py /absolute/path/to/quant-result.json --output /absolute/path/to/quant-receipt.json ``` -Continue only when the receipt has `gate_status: PASSED`. If it returns `BLOCKED`, stop and request a fresh JSON run from `$quant-stock-technical`. Never repair, infer, or hand-create the missing upstream fields. +Continue only when the receipt has `gate_status: PASSED`. If it returns +`BLOCKED`, stop and request a fresh JSON export from the standalone QTA system. +Never repair, infer, or hand-create the missing upstream fields. Treat the receipt as an invisible backstage pass. Use it only to prove that the deterministic analysis ran first. Never quote, paraphrase, compare, hint at, or display its numeric contents in the qualitative output. diff --git a/skills/stock-scenario-story/agents/openai.yaml b/skills/stock-scenario-story/agents/openai.yaml index bacba26..f448aad 100644 --- a/skills/stock-scenario-story/agents/openai.yaml +++ b/skills/stock-scenario-story/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Stock Scenario Story" short_description: "Research-driven stories from validated quant results" - default_prompt: "Use $stock-scenario-story after $quant-stock-technical to research and narrate evidence-backed market scenarios." + default_prompt: "Use $stock-scenario-story with a complete external QTA JSON result matching quant-stock-technical/v1 to research and narrate evidence-backed market scenarios." diff --git a/tests/test_daily_pipeline.py b/tests/test_daily_pipeline.py deleted file mode 100644 index b034fa8..0000000 --- a/tests/test_daily_pipeline.py +++ /dev/null @@ -1,401 +0,0 @@ -#!/usr/bin/env python3 -"""Offline regression tests for the recurring daily shadow pipeline.""" - -from __future__ import annotations - -import json -import importlib.util -import os -import subprocess -import sys -import tempfile -import unittest -from datetime import datetime -from pathlib import Path -from unittest.mock import patch -from zoneinfo import ZoneInfo - -ROOT = Path(__file__).resolve().parents[1] -SCRIPT = ( - ROOT - / "skills" - / "quant-stock-polling-trader" - / "scripts" - / "daily_pipeline.py" -) -sys.path.insert(0, str(SCRIPT.parent)) -SPEC = importlib.util.spec_from_file_location("daily_pipeline_contract", SCRIPT) -if SPEC is None or SPEC.loader is None: - raise RuntimeError(f"cannot load {SCRIPT}") -MODULE = importlib.util.module_from_spec(SPEC) -sys.modules[SPEC.name] = MODULE -SPEC.loader.exec_module(MODULE) - - -class DailyPipelineTests(unittest.TestCase): - def test_embedded_self_test(self) -> None: - result = subprocess.run( - [sys.executable, "-B", "-s", str(SCRIPT), "self-test"], - check=False, - capture_output=True, - text=True, - ) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(json.loads(result.stdout)["self_test"], "PASS") - - def test_v2_config_needs_no_approval_or_external_broker_files(self) -> None: - with tempfile.TemporaryDirectory(prefix="qta-daily-config-v2-") as temporary: - root = Path(temporary) - risk_path = ( - ROOT - / "skills" - / "quant-stock-polling-trader" - / "references" - / "fixtures" - / "aggressive-2m-risk-policy.json" - ) - raw = { - "schema": MODULE.CONFIG_SCHEMA, - "runtime_root": str(root / "runtime"), - "broker": "kis-live", - "mode": "shadow", - "history_start_date": "2021-01-01", - "minimum_sessions": 756, - "request_interval_ms": 120, - "catalog_coverage_contract": {}, - "selector": {"selector_version": "qta-screen-1.1.0"}, - "risk_policy_path": str(risk_path), - "execution_policy": { - "snapshot_max_age_seconds": 1800, - "poll_interval_seconds": 2, - "quote_max_age_seconds": 5, - "max_spread_bps": "25", - "max_gap_bps": "20", - "trigger_mode": "CROSS_FROM_BELOW", - "order_ttl_seconds": 30, - "order_type": "LIMIT", - "time_in_force": "DAY", - "allow_partial_fill": True, - "cancel_remainder_at_window_end": True, - }, - "account_aliases": { - "KR": "kis-live-kr", - "US": "kis-live-us", - }, - "prepare_complete_seconds_before_open": 600, - } - normalized = MODULE.normalize_config(raw, root / "daily.json") - self.assertEqual(normalized["exposure_component_paths"], []) - self.assertNotIn("approved_trader_tree_sha256", normalized) - self.assertNotIn("required_manual_exposure_brokers", normalized) - - def test_blocked_stage_is_persisted_as_current_terminal_state(self) -> None: - with tempfile.TemporaryDirectory(prefix="qta-daily-blocked-") as temporary: - root = Path(temporary) - runtime = root / "runtime" - config = root / "invalid-config.json" - config.write_text( - json.dumps({"runtime_root": str(runtime)}), - encoding="utf-8", - ) - environment = dict(os.environ) - environment["PYTHONDONTWRITEBYTECODE"] = "1" - environment.pop("PYTHONPATH", None) - environment.pop("PYTHONHOME", None) - command = [ - sys.executable, - "-B", - "-s", - str(SCRIPT), - "prepare", - "--config", - str(config), - "--market", - "KR", - "--technical-skill-root", - str(root / "technical"), - "--trader-skill-root", - str(root / "trader"), - ] - first = subprocess.run( - command, - check=False, - capture_output=True, - text=True, - env=environment, - ) - self.assertEqual(first.returncode, 2) - session = datetime.now(ZoneInfo("Asia/Seoul")).date().isoformat() - descriptor_path = runtime / "current" / "kr.json" - descriptor = json.loads(descriptor_path.read_text(encoding="utf-8")) - self.assertEqual(descriptor["status"], "BLOCKED") - self.assertEqual(descriptor["blocked_stage"], "prepare") - self.assertEqual(descriptor["session_date"], session) - self.assertFalse(descriptor["live_enabled"]) - self.assertEqual(descriptor["api_mutation_count"], 0) - receipt_path = ( - runtime - / "workflows" - / "kr" - / session - / "prepare-status.json" - ) - self.assertEqual( - json.loads(receipt_path.read_text(encoding="utf-8"))["status"], - "BLOCKED", - ) - - def test_same_analysis_date_ready_cache_is_seeded(self) -> None: - with tempfile.TemporaryDirectory(prefix="qta-daily-cache-") as temporary: - runtime = Path(temporary) - ready_root = runtime / "eod" / "2026-07-27" / "approved-a" / "kr" - blocked_root = runtime / "eod" / "2026-07-27" / "approved-z" / "kr" - target_root = runtime / "eod" / "2026-07-27" / "current" / "kr" - for candidate_root, status in ( - (ready_root, "READY"), - (blocked_root, "BLOCKED"), - ): - (candidate_root / "stocks" / "KOSPI").mkdir(parents=True) - (candidate_root / "benchmarks").mkdir() - (candidate_root / "stocks" / "KOSPI" / "005930.csv").write_text( - f"{status}\n", - encoding="utf-8", - ) - (candidate_root / "benchmarks" / "KOSPI_COMPOSITE.csv").write_text( - f"{status}\n", - encoding="utf-8", - ) - (candidate_root / "eod-catalog.csv").write_text( - "catalog\n", - encoding="utf-8", - ) - (candidate_root / "eod-bundle-receipt.json").write_text( - json.dumps( - { - "status": status, - "analysis_date": "2026-07-27", - "api_mutation_count": 0, - } - ), - encoding="utf-8", - ) - - MODULE.seed_eod_cache( - runtime, - target_root, - "2026-07-27", - "kr", - ) - - self.assertEqual( - (target_root / "stocks" / "KOSPI" / "005930.csv").read_text( - encoding="utf-8" - ), - "READY\n", - ) - self.assertEqual( - ( - target_root / "benchmarks" / "KOSPI_COMPOSITE.csv" - ).read_text(encoding="utf-8"), - "READY\n", - ) - - def test_cached_eod_build_spec_rebinds_current_coverage_policy(self) -> None: - with tempfile.TemporaryDirectory(prefix="qta-daily-build-spec-") as temporary: - root = Path(temporary) - us_output = root / "eod" / "us" - workflow = root / "workflow" - source_path = us_output / "universe-build-spec.json" - original_contract = { - "schema": "qta-catalog-coverage-contract/v1", - "minimum_ratio_by_exchange": { - exchange: "1" - for exchange in ("KOSPI", "KOSDAQ", "NYSE", "NASDAQ") - }, - "minimum_screenable_ratio_by_exchange": { - exchange: "0.5" - for exchange in ("KOSPI", "KOSDAQ", "NYSE", "NASDAQ") - }, - } - current_contract = json.loads(json.dumps(original_contract)) - current_contract["minimum_screenable_ratio_by_exchange"][ - "NASDAQ" - ] = "0.35" - source = { - "schema": "qta-universe-build-spec/v1", - "as_of": "2026-07-28", - "analysis_date": "2026-07-27", - "official_sources": [ - {"exchange": "NYSE"}, - {"exchange": "NASDAQ"}, - ], - "broker_sources": [ - {"exchange": "NYSE"}, - {"exchange": "NASDAQ"}, - ], - "eod_catalog": {}, - "catalog_coverage_contract": original_contract, - } - MODULE.atomic_write_json(source_path, source) - MODULE.atomic_write_json( - us_output / "eod-bundle-receipt.json", - { - "build_spec": { - "path": str(source_path.resolve()), - "sha256": MODULE.sha256_file(source_path), - } - }, - ) - - derived_path = MODULE.derive_universe_build_spec( - config={"catalog_coverage_contract": current_contract}, - eod_output=us_output, - workflow=workflow, - market="US", - ) - - self.assertEqual( - json.loads(derived_path.read_text(encoding="utf-8"))[ - "catalog_coverage_contract" - ], - current_contract, - ) - self.assertEqual( - json.loads(source_path.read_text(encoding="utf-8"))[ - "catalog_coverage_contract" - ], - original_contract, - ) - source["official_sources"].append({"exchange": "KOSPI"}) - MODULE.atomic_write_json(source_path, source) - MODULE.atomic_write_json( - us_output / "eod-bundle-receipt.json", - { - "build_spec": { - "path": str(source_path.resolve()), - "sha256": MODULE.sha256_file(source_path), - } - }, - ) - with self.assertRaisesRegex( - MODULE.PipelineBlockedError, - "official_sources must contain exactly", - ): - MODULE.derive_universe_build_spec( - config={"catalog_coverage_contract": current_contract}, - eod_output=us_output, - workflow=workflow, - market="US", - ) - - def test_eod_job_uses_only_the_selected_market(self) -> None: - with tempfile.TemporaryDirectory(prefix="qta-daily-market-eod-") as temporary: - root = Path(temporary) - technical = root / "technical" - scripts = technical / "scripts" - scripts.mkdir(parents=True) - for name in ("fetch_kis_kr_eod.py", "fetch_kis_us_eod.py"): - (scripts / name).write_text("# fixture\n", encoding="utf-8") - config = { - "runtime_root": root / "runtime", - "history_start_date": "2021-01-01", - "minimum_sessions": 756, - "request_interval_ms": 120, - "catalog_coverage_contract": {"schema": "fixture"}, - } - - for market, exchanges in ( - ("KR", {"KOSPI", "KOSDAQ"}), - ("US", {"NYSE", "NASDAQ"}), - ): - sources = { - "official_sources": [ - {"exchange": exchange} for exchange in sorted(exchanges) - ], - "broker_sources": [ - {"exchange": exchange} for exchange in sorted(exchanges) - ], - } - - def complete_job(command: list[str], label: str) -> None: - self.assertEqual(label, f"KIS {market} adjusted EOD") - job = json.loads(Path(command[-1]).read_text(encoding="utf-8")) - self.assertEqual( - {item["exchange"] for item in job["official_sources"]}, - exchanges, - ) - self.assertEqual( - {item["exchange"] for item in job["broker_sources"]}, - exchanges, - ) - self.assertEqual(job["base_eod_catalog"], "") - output = Path(job["output_directory"]) - output.mkdir(parents=True, exist_ok=True) - MODULE.atomic_write_json( - output / "eod-bundle-receipt.json", - { - "status": "READY", - "analysis_date": "2026-07-28", - "api_mutation_count": 0, - }, - ) - - with patch.object(MODULE, "run_command", side_effect=complete_job): - output = MODULE.ensure_eod( - config=config, - technical_root=technical, - workflow=root / "workflow" / market.lower(), - as_of="2026-07-29", - analysis_date="2026-07-28", - market=market, - sources=sources, - cache_namespace=f"test-{market.lower()}", - ) - self.assertEqual(output.name, market.lower()) - - def test_legacy_harness_descriptor_does_not_block_v2_prepare(self) -> None: - with tempfile.TemporaryDirectory(prefix="qta-daily-v2-") as temporary: - runtime = Path(temporary) - session = "2026-07-28" - workflow = runtime / "workflows" / "us" / session - current = runtime / "current" / "us.json" - workflow.mkdir(parents=True) - current.parent.mkdir(parents=True) - MODULE.atomic_write_json( - current, - { - "schema": "qta-daily-shadow-descriptor/v1", - "market": "US", - "session_date": session, - "status": "PREPARED", - }, - ) - - with ( - patch.object(MODULE, "current_market_date", return_value=session), - patch.object( - MODULE.market_calendar, - "snapshot", - return_value={ - "status": "MARKET_CLOSED", - "session_date": session, - }, - ) as calendar_snapshot, - ): - receipt = MODULE.prepare( - {"runtime_root": runtime}, - "US", - runtime / "technical", - runtime / "trader", - ) - - self.assertEqual(receipt["status"], "MARKET_CLOSED") - calendar_snapshot.assert_called_once() - self.assertEqual( - json.loads(current.read_text(encoding="utf-8"))["schema"], - MODULE.DESCRIPTOR_SCHEMA, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_handoff_integration.py b/tests/test_handoff_integration.py index bc3eb86..cb4105d 100644 --- a/tests/test_handoff_integration.py +++ b/tests/test_handoff_integration.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""End-to-end contract test between the paired stock-analysis skills.""" +"""Validate the story skill against the external QTA handoff contract.""" from __future__ import annotations @@ -9,7 +9,6 @@ ROOT = Path(__file__).resolve().parents[1] -QUANT_PATH = ROOT / "skills/quant-stock-technical/scripts/analyze_stock.py" STORY_GATE_PATH = ROOT / "skills/stock-scenario-story/scripts/validate_quant_handoff.py" @@ -24,11 +23,9 @@ def load_module(name: str, path: Path): def main() -> None: - quant = load_module("quant_calculator", QUANT_PATH) story_gate = load_module("story_gate", STORY_GATE_PATH) - ticker_bars = quant.synthetic_bars(1100, 0.0006, 0.0) - benchmark_bars = quant.synthetic_bars(1100, 0.0003, 1.0) - result = quant.calculate(ticker_bars, benchmark_bars, "TEST", "SYNTH", 0.01, "integration-test") + result = story_gate.valid_fixture() + result["source_name"] = "external-qta-contract-fixture" receipt = story_gate.validate(result) assert receipt["gate_status"] == "PASSED" assert receipt["source_skill"] == "quant-stock-technical" diff --git a/tests/test_kis_kr_eod.py b/tests/test_kis_kr_eod.py deleted file mode 100644 index c2ba3c5..0000000 --- a/tests/test_kis_kr_eod.py +++ /dev/null @@ -1,283 +0,0 @@ -#!/usr/bin/env python3 -"""Offline contract tests for the read-only KIS Korean EOD collector.""" - -from __future__ import annotations - -import importlib.util -import json -import sys -import tempfile -import unittest -from datetime import date, datetime -from decimal import Decimal -from pathlib import Path -from unittest.mock import patch - -ROOT = Path(__file__).resolve().parents[1] -SCRIPTS = ROOT / "skills" / "quant-stock-technical" / "scripts" -MODULE_PATH = SCRIPTS / "fetch_kis_kr_eod.py" -sys.path.insert(0, str(SCRIPTS)) -SPEC = importlib.util.spec_from_file_location("fetch_kis_kr_eod_contract", MODULE_PATH) -if SPEC is None or SPEC.loader is None: - raise RuntimeError(f"cannot load {MODULE_PATH}") -MODULE = importlib.util.module_from_spec(SPEC) -sys.modules[SPEC.name] = MODULE -SPEC.loader.exec_module(MODULE) - - -class KisKrEodTests(unittest.TestCase): - def test_embedded_pagination_and_csv_round_trip(self) -> None: - MODULE.self_test() - - def test_adjusted_price_flag_is_fixed(self) -> None: - self.assertEqual(MODULE.STOCK_TR_ID, "FHKST03010100") - self.assertEqual( - MODULE.STOCK_PATH, - "/uapi/domestic-stock/v1/quotations/inquire-daily-itemchartprice", - ) - row = MODULE.normalize_stock_row( - { - "stck_bsop_date": "20260724", - "stck_oprc": "1000", - "stck_hgpr": "1100", - "stck_lwpr": "900", - "stck_clpr": "1050", - "acml_vol": "123456", - } - ) - self.assertEqual(row["adjusted_close"], row["close"]) - - def test_kospi_and_kosdaq_tick_ladders_remain_distinct(self) -> None: - self.assertEqual(MODULE.tick_size("KOSPI", Decimal("100000")), Decimal("500")) - self.assertEqual(MODULE.tick_size("KOSDAQ", Decimal("100000")), Decimal("100")) - - def test_history_minimum_cannot_be_weakened(self) -> None: - with self.assertRaisesRegex(MODULE.EodBlockedError, "at least 756"): - MODULE.normalize_job( - { - "schema": MODULE.JOB_SCHEMA, - "as_of": "2026-07-27", - "analysis_date": "2026-07-24", - "environment": "live", - "output_directory": "output", - "history_start_date": "2021-01-01", - "minimum_sessions": 755, - "request_interval_ms": 120, - "official_sources": [], - "broker_sources": [], - "catalog_coverage_contract": { - "schema": MODULE.universe.CATALOG_COVERAGE_SCHEMA, - "minimum_ratio_by_exchange": { - exchange: "1" for exchange in MODULE.universe.EXCHANGES - }, - "minimum_screenable_ratio_by_exchange": { - exchange: "0.5" for exchange in MODULE.universe.EXCHANGES - }, - }, - "base_eod_catalog": "", - }, - ROOT, - ) - - def test_token_throttle_retries_once_without_exposing_secrets(self) -> None: - responses = iter( - ( - (403, {"error_code": "EGW00133"}), - (200, {"access_token": "fixture-token"}), - ) - ) - - def transport(method, url, headers, body): - del method, url, headers, body - return next(responses) - - client = MODULE.KisReadClient( - environment="live", - app_key="fixture-key", - app_secret="fixture-secret", - interval_ms=100, - transport=transport, - ) - with patch.object(MODULE.time, "sleep") as sleep: - self.assertEqual(client.token(), "fixture-token") - sleep.assert_called_once_with(61.0) - self.assertEqual(client.request_count, 2) - self.assertEqual(client.retry_count, 1) - - def test_complete_cache_is_not_rewritten(self) -> None: - end = date(2026, 7, 24) - rows = [ - { - "date": (end - MODULE.timedelta(days=offset)).isoformat(), - "open": "100", - "high": "110", - "low": "90", - "close": "105", - "adjusted_close": "105", - "volume": "1000", - } - for offset in range(3) - ] - rows.sort(key=lambda row: row["date"]) - with tempfile.TemporaryDirectory() as temporary: - path = Path(temporary) / "cached.csv" - MODULE.write_csv_rows(path, rows) - client = MODULE.KisReadClient( - environment="live", - app_key="fixture", - app_secret="fixture", - interval_ms=100, - access_token="fixture-token", - transport=lambda *args: self.fail("cache should satisfy the job"), - ) - with patch.object(MODULE, "write_csv_rows") as writer: - result, requests = MODULE.update_history_file( - client, - kind="stock", - symbol="005930", - path=path, - start=end - MODULE.timedelta(days=3), - end=end, - minimum_sessions=2, - ) - self.assertEqual(result, rows) - self.assertEqual(requests, 0) - writer.assert_not_called() - - def test_offline_bundle_emits_catalog_spec_and_zero_mutations(self) -> None: - end = date(2026, 7, 24) - stock_rows = MODULE._synthetic_rows(end, 800, index=False) - index_rows = MODULE._synthetic_rows(end, 800, index=True) - - def transport(method, url, headers, body): - del headers, body - self.assertEqual(method, "GET") - query = dict(item.split("=", 1) for item in url.split("?", 1)[1].split("&")) - cursor = datetime.strptime(query["FID_INPUT_DATE_2"], "%Y%m%d").date() - source = index_rows if MODULE.INDEX_PATH in url else stock_rows - page_size = 50 if source is index_rows else 100 - selected = [ - row - for row in source - if datetime.strptime(row["stck_bsop_date"], "%Y%m%d").date() <= cursor - ][:page_size] - return 200, {"rt_cd": "0", "output2": selected} - - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - sources = root / "sources" - sources.mkdir() - official_paths = {} - broker_paths = {} - for exchange, symbol in (("KOSPI", "005930"), ("KOSDAQ", "035720")): - official_path = sources / f"official-{exchange}.csv" - broker_path = sources / f"broker-{exchange}.csv" - payload = ( - "symbol,name,instrument_type,status\n" - f"{symbol},fixture,COMMON,NORMAL\n" - ) - official_path.write_text(payload, encoding="utf-8") - broker_path.write_text(payload, encoding="utf-8") - official_paths[exchange] = official_path - broker_paths[exchange] = broker_path - - def descriptor(exchange, path, provider, source_id): - return { - "source_id": source_id, - "provider": provider, - "exchange": exchange, - "as_of": "2026-07-27", - "path": str(path), - "format": "KRX_CSV" if provider == "KRX" else "KIS_CSV", - "encoding": "utf-8", - "delimiter": ",", - "skip_rows": 0, - "columns": { - "symbol": "symbol", - "name": "name", - "instrument_type": "instrument_type", - "status": "status", - }, - "normal_status_values": ["NORMAL"], - } - - official_sources = [ - descriptor( - exchange, - official_paths[exchange], - "KRX", - f"official-{exchange}", - ) - for exchange in MODULE.KR_EXCHANGES - ] - broker_sources = [ - descriptor( - exchange, - broker_paths[exchange], - "KIS", - f"broker-{exchange}", - ) - for exchange in MODULE.KR_EXCHANGES - ] - job_path = root / "job.json" - raw_job = { - "schema": MODULE.JOB_SCHEMA, - "as_of": "2026-07-27", - "analysis_date": "2026-07-24", - "environment": "live", - "output_directory": str(root / "output"), - "history_start_date": "2023-01-01", - "minimum_sessions": 756, - "request_interval_ms": 100, - "official_sources": official_sources, - "broker_sources": broker_sources, - "catalog_coverage_contract": { - "schema": MODULE.universe.CATALOG_COVERAGE_SCHEMA, - "minimum_ratio_by_exchange": { - exchange: "1" for exchange in MODULE.universe.EXCHANGES - }, - "minimum_screenable_ratio_by_exchange": { - exchange: "0.5" for exchange in MODULE.universe.EXCHANGES - }, - }, - "base_eod_catalog": "", - } - job_path.write_text(json.dumps(raw_job), encoding="utf-8") - job = MODULE.normalize_job(raw_job, root) - client = MODULE.KisReadClient( - environment="live", - app_key="fixture", - app_secret="fixture", - interval_ms=100, - access_token="fixture-token", - transport=transport, - ) - client.interval_seconds = 0 - receipt = MODULE.build_bundle(job, client=client) - - self.assertEqual(receipt["status"], "READY") - self.assertEqual(receipt["ready_symbols"], 2) - self.assertEqual(receipt["api_mutation_count"], 0) - self.assertEqual(receipt["failed_symbols"], []) - self.assertTrue(Path(receipt["catalog"]["path"]).is_file()) - self.assertTrue(Path(receipt["build_spec"]["path"]).is_file()) - self.assertEqual(receipt["coverage_by_exchange"]["KOSPI"]["ready"], 1) - build_spec_path = Path(receipt["build_spec"]["path"]) - build_spec = json.loads(build_spec_path.read_text(encoding="utf-8")) - for role, sources in ( - ("OFFICIAL_MASTER", build_spec["official_sources"]), - ("BROKER_MASTER", build_spec["broker_sources"]), - ): - for index, source in enumerate(sources): - self.assertEqual(set(source), MODULE.universe.SOURCE_FIELDS) - MODULE.universe.normalize_source_descriptor( - source, - role, - build_spec["as_of"], - build_spec_path.parent, - index, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_kis_us_eod.py b/tests/test_kis_us_eod.py deleted file mode 100644 index 9325c02..0000000 --- a/tests/test_kis_us_eod.py +++ /dev/null @@ -1,372 +0,0 @@ -#!/usr/bin/env python3 -"""Offline contract tests for the KIS U.S. EOD collector.""" - -from __future__ import annotations - -import importlib.util -import json -import sys -import tempfile -import unittest -from datetime import date, datetime, timedelta -from decimal import Decimal -from pathlib import Path -from unittest.mock import patch - -ROOT = Path(__file__).resolve().parents[1] -SCRIPTS = ROOT / "skills" / "quant-stock-technical" / "scripts" -sys.path.insert(0, str(SCRIPTS)) -SPEC = importlib.util.spec_from_file_location( - "fetch_kis_us_eod_contract", SCRIPTS / "fetch_kis_us_eod.py" -) -if SPEC is None or SPEC.loader is None: - raise RuntimeError("cannot load fetch_kis_us_eod.py") -MODULE = importlib.util.module_from_spec(SPEC) -sys.modules[SPEC.name] = MODULE -SPEC.loader.exec_module(MODULE) - - -class KisUsEodTests(unittest.TestCase): - def test_embedded_pagination_is_read_only(self) -> None: - MODULE.self_test() - - def test_adjusted_price_contract_is_fixed(self) -> None: - self.assertEqual(MODULE.STOCK_TR_ID, "HHDFS76240000") - self.assertEqual( - MODULE.STOCK_PATH, - "/uapi/overseas-price/v1/quotations/dailyprice", - ) - - def test_exchange_codes_are_not_collapsed(self) -> None: - self.assertEqual(MODULE.EXCHANGE_CODES["NASDAQ"], "NAS") - self.assertEqual(MODULE.EXCHANGE_CODES["NYSE"], "NYS") - - def test_conservative_us_tick_ladder(self) -> None: - self.assertEqual(MODULE.tick_size(Decimal("0.25")), Decimal("0.0001")) - self.assertEqual(MODULE.tick_size(Decimal("1")), Decimal("0.01")) - self.assertEqual(MODULE.tick_size(Decimal("250")), Decimal("0.01")) - - def test_benchmark_identity_is_exchange_specific(self) -> None: - self.assertEqual(MODULE.BENCHMARK_SYMBOLS["NYSE"], "^NYA") - self.assertEqual(MODULE.BENCHMARK_SYMBOLS["NASDAQ"], "^IXIC") - - def test_interior_invalid_geometry_is_audited_without_interpolation(self) -> None: - end = date(2026, 7, 24) - rows = MODULE.shared._synthetic_rows(end, 800, index=False) - overseas_rows = [ - { - "xymd": item["stck_bsop_date"], - "open": item["stck_oprc"], - "high": item["stck_hgpr"], - "low": item["stck_lwpr"], - "clos": item["stck_clpr"], - "tvol": item["acml_vol"], - } - for item in rows - ] - invalid = overseas_rows[400] - invalid["low"] = str(Decimal(invalid["open"]) + Decimal("1")) - - def transport(method, url, headers, body): - del headers, body - self.assertEqual(method, "GET") - query = dict(item.split("=", 1) for item in url.split("?", 1)[1].split("&")) - cursor = datetime.strptime(query["BYMD"], "%Y%m%d").date() - selected = [ - row - for row in overseas_rows - if datetime.strptime(row["xymd"], "%Y%m%d").date() <= cursor - ][:100] - return 200, {"rt_cd": "0", "output2": selected} - - client = MODULE.shared.KisReadClient( - environment="live", - app_key="fixture", - app_secret="fixture", - interval_ms=100, - access_token="fixture-token", - transport=transport, - ) - client.interval_seconds = 0 - audit = [] - result = MODULE.fetch_stock_history( - client, - exchange="NYSE", - symbol="A", - start=end - timedelta(days=1200), - end=end, - invalid_rows=audit, - ) - self.assertEqual(len(result), 799) - self.assertEqual(len(audit), 1) - self.assertEqual(audit[0]["date"], datetime.strptime(invalid["xymd"], "%Y%m%d").date().isoformat()) - self.assertNotIn(audit[0]["date"], {row["date"] for row in result}) - - def test_invalid_completed_cutoff_remains_blocked(self) -> None: - end = date(2026, 7, 24) - row = { - "xymd": end.strftime("%Y%m%d"), - "open": "10", - "high": "11", - "low": "10.5", - "clos": "10.25", - "tvol": "100", - } - - def transport(method, url, headers, body): - del method, url, headers, body - return 200, {"rt_cd": "0", "output2": [row]} - - client = MODULE.shared.KisReadClient( - environment="live", - app_key="fixture", - app_secret="fixture", - interval_ms=100, - access_token="fixture-token", - transport=transport, - ) - client.interval_seconds = 0 - with self.assertRaisesRegex( - MODULE.UsEodBlockedError, - "completed cutoff row has invalid OHLC geometry", - ): - MODULE.fetch_stock_history( - client, - exchange="NYSE", - symbol="A", - start=end - timedelta(days=10), - end=end, - invalid_rows=[], - ) - - def test_cached_invalid_geometry_is_excluded_and_audited(self) -> None: - end = date(2026, 7, 24) - rows = [ - { - "date": (end - timedelta(days=offset)).isoformat(), - "open": "10", - "high": "11", - "low": "9", - "close": "10.5", - "adjusted_close": "10.5", - "volume": "100", - } - for offset in range(3) - ] - rows[1]["low"] = "10.25" - with tempfile.TemporaryDirectory() as temporary: - path = Path(temporary) / "cached.csv" - MODULE.shared.write_csv_rows(path, sorted(rows, key=lambda row: row["date"])) - client = MODULE.shared.KisReadClient( - environment="live", - app_key="fixture", - app_secret="fixture", - interval_ms=100, - access_token="fixture-token", - transport=lambda *args: self.fail("cache should satisfy the job"), - ) - result, requests, audit = MODULE.update_stock_file( - client, - exchange="NYSE", - symbol="A", - path=path, - start=end - timedelta(days=3), - end=end, - minimum_sessions=2, - ) - self.assertEqual(requests, 0) - self.assertEqual(len(result), 2) - self.assertEqual(len(audit), 1) - self.assertNotIn(audit[0]["date"], {row["date"] for row in result}) - - def test_complete_cache_is_not_rewritten(self) -> None: - end = date(2026, 7, 24) - rows = [ - { - "date": (end - timedelta(days=offset)).isoformat(), - "open": "10", - "high": "11", - "low": "9", - "close": "10.5", - "adjusted_close": "10.5", - "volume": "100", - } - for offset in range(3) - ] - rows.sort(key=lambda row: row["date"]) - with tempfile.TemporaryDirectory() as temporary: - path = Path(temporary) / "cached.csv" - MODULE.shared.write_csv_rows(path, rows) - client = MODULE.shared.KisReadClient( - environment="live", - app_key="fixture", - app_secret="fixture", - interval_ms=100, - access_token="fixture-token", - transport=lambda *args: self.fail("cache should satisfy the job"), - ) - with patch.object(MODULE.shared, "write_csv_rows") as writer: - result, requests, audit = MODULE.update_stock_file( - client, - exchange="NASDAQ", - symbol="AAPL", - path=path, - start=end - timedelta(days=3), - end=end, - minimum_sessions=2, - ) - self.assertEqual(result, rows) - self.assertEqual(requests, 0) - self.assertEqual(audit, []) - writer.assert_not_called() - - def test_offline_bundle_emits_cross_market_inputs(self) -> None: - end = date(2026, 7, 24) - raw_rows = MODULE.shared._synthetic_rows(end, 800, index=False) - overseas_rows = [ - { - "xymd": item["stck_bsop_date"], - "open": item["stck_oprc"], - "high": item["stck_hgpr"], - "low": item["stck_lwpr"], - "clos": item["stck_clpr"], - "tvol": item["acml_vol"], - } - for item in raw_rows - ] - benchmark_rows = [MODULE.shared.normalize_stock_row(item) for item in raw_rows] - - def transport(method, url, headers, body): - del headers, body - self.assertEqual(method, "GET") - query = dict(item.split("=", 1) for item in url.split("?", 1)[1].split("&")) - cursor = datetime.strptime(query["BYMD"], "%Y%m%d").date() - selected = [ - row - for row in overseas_rows - if datetime.strptime(row["xymd"], "%Y%m%d").date() <= cursor - ][:100] - return 200, {"rt_cd": "0", "output2": selected} - - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - nasdaq = root / "nasdaqlisted.txt" - nyse = root / "otherlisted.txt" - nasdaq.write_text( - "Symbol|Security Name|Market Category|Test Issue|Financial Status|" - "Round Lot Size|ETF|NextShares\n" - "AAPL|Apple Inc|Q|N|N|100|N|N\n", - encoding="utf-8", - ) - nyse.write_text( - "ACT Symbol|Security Name|Exchange|CQS Symbol|ETF|Round Lot Size|" - "Test Issue|NASDAQ Symbol\n" - "BA|Boeing|N|BA|N|100|N|BA\n", - encoding="utf-8", - ) - masters = {} - for exchange, symbol in (("NASDAQ", "AAPL"), ("NYSE", "BA")): - path = root / ("nasmst.cod" if exchange == "NASDAQ" else "nysmst.cod") - path.write_text( - MODULE.universe.synthetic_kis_overseas_line( - exchange, symbol, "2", "N" - ) - + "\n", - encoding="cp949", - ) - masters[exchange] = path - official_sources = [ - { - "source_id": f"official-{exchange}", - "provider": "NASDAQ_TRADER", - "exchange": exchange, - "as_of": "2026-07-27", - "path": str(nasdaq if exchange == "NASDAQ" else nyse), - "format": ( - "NASDAQ_LISTED" if exchange == "NASDAQ" else "NASDAQ_OTHER" - ), - "encoding": "utf-8", - "delimiter": "|", - "skip_rows": 0, - "columns": {}, - "normal_status_values": ["N"] if exchange == "NASDAQ" else [], - } - for exchange in MODULE.US_EXCHANGES - ] - broker_sources = [ - { - "source_id": f"broker-{exchange}", - "provider": "KIS", - "exchange": exchange, - "as_of": "2026-07-27", - "path": str(masters[exchange]), - "format": "KIS_OVERSEAS_MASTER", - "encoding": "cp949", - "delimiter": "\t", - "skip_rows": 0, - "columns": {}, - "normal_status_values": [], - } - for exchange in MODULE.US_EXCHANGES - ] - raw_job = { - "schema": MODULE.JOB_SCHEMA, - "as_of": "2026-07-27", - "analysis_date": "2026-07-24", - "environment": "live", - "output_directory": str(root / "output"), - "history_start_date": "2023-01-01", - "minimum_sessions": 756, - "request_interval_ms": 100, - "official_sources": official_sources, - "broker_sources": broker_sources, - "catalog_coverage_contract": { - "schema": MODULE.universe.CATALOG_COVERAGE_SCHEMA, - "minimum_ratio_by_exchange": { - exchange: "1" for exchange in MODULE.universe.EXCHANGES - }, - "minimum_screenable_ratio_by_exchange": { - exchange: "0.5" for exchange in MODULE.universe.EXCHANGES - }, - }, - "base_eod_catalog": "", - } - job_path = root / "job.json" - job_path.write_text(json.dumps(raw_job), encoding="utf-8") - client = MODULE.shared.KisReadClient( - environment="live", - app_key="fixture", - app_secret="fixture", - interval_ms=100, - access_token="fixture-token", - transport=transport, - ) - client.interval_seconds = 0 - receipt = MODULE.build_bundle( - MODULE.normalize_job(raw_job, root), - client=client, - benchmark_loader=lambda symbol, start, end: benchmark_rows, - ) - self.assertEqual(receipt["status"], "READY") - self.assertEqual(receipt["ready_symbols"], 2) - self.assertEqual(receipt["api_mutation_count"], 0) - self.assertTrue(Path(receipt["catalog"]["path"]).is_file()) - build_spec_path = Path(receipt["build_spec"]["path"]) - build_spec = json.loads(build_spec_path.read_text(encoding="utf-8")) - for role, sources in ( - ("OFFICIAL_MASTER", build_spec["official_sources"]), - ("BROKER_MASTER", build_spec["broker_sources"]), - ): - for index, source in enumerate(sources): - self.assertEqual(set(source), MODULE.universe.SOURCE_FIELDS) - MODULE.universe.normalize_source_descriptor( - source, - role, - build_spec["as_of"], - build_spec_path.parent, - index, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_market_calendar.py b/tests/test_market_calendar.py deleted file mode 100644 index e02bc42..0000000 --- a/tests/test_market_calendar.py +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env python3 -"""Offline tests for official market-calendar parsing and freezing.""" - -from __future__ import annotations - -import importlib.util -import json -import sys -import tempfile -import unittest -from datetime import datetime -from pathlib import Path -from zoneinfo import ZoneInfo - -ROOT = Path(__file__).resolve().parents[1] -SCRIPTS = ROOT / "skills" / "quant-stock-polling-trader" / "scripts" -sys.path.insert(0, str(SCRIPTS)) -SPEC = importlib.util.spec_from_file_location( - "qta_market_calendar_contract", - SCRIPTS / "market_calendar.py", -) -if SPEC is None or SPEC.loader is None: - raise RuntimeError("cannot load market_calendar.py") -MODULE = importlib.util.module_from_spec(SPEC) -sys.modules[SPEC.name] = MODULE -SPEC.loader.exec_module(MODULE) - - -NASDAQ = b""" - - - - - - -
2025HolidayStatus
January 1, 2025New YearClosed
2026HolidayStatus
January 1, 2026New YearClosed
November 27, 2026Early Close1:00 p.m.
-""" -NYSE = b""" -

Core Trading Session: 9:30 a.m. to 4:00 p.m. ET

- - - -
Holiday20252026
New YearWednesday, January 1Thursday, January 1
-

Each market will close early at 1:00 p.m. on Friday, -November 27, 2026.

-""" - - -class MarketCalendarTests(unittest.TestCase): - def test_embedded_self_test(self) -> None: - MODULE.self_test() - - def test_us_sources_must_agree(self) -> None: - self.assertEqual( - MODULE.parse_nasdaq_calendar(NASDAQ, 2026), - MODULE.parse_nyse_calendar(NYSE, 2026), - ) - mismatched = NASDAQ.replace( - b"January 1, 2026New YearClosed", - b"January 1, 2026New Year1:00 p.m.", - 1, - ) - with self.assertRaisesRegex( - MODULE.CalendarBlockedError, "no closed dates" - ): - MODULE.parse_nasdaq_calendar(mismatched, 2026) - - def test_krx_holiday_page_contract_is_required(self) -> None: - valid = ( - f"".encode() - ) - MODULE.validate_krx_holiday_page(valid) - with self.assertRaisesRegex( - MODULE.CalendarBlockedError, "request contract" - ): - MODULE.validate_krx_holiday_page(b"") - - def test_mocked_us_snapshot_is_source_hashed(self) -> None: - responses = { - MODULE.NYSE_CALENDAR_URL: NYSE, - MODULE.NASDAQ_CALENDAR_URL: NASDAQ, - } - - def fetcher(url: str, data: bytes | None = None) -> bytes: - self.assertIsNone(data) - return responses[url] - - with tempfile.TemporaryDirectory(prefix="qta-market-calendar-") as temporary: - receipt = MODULE.snapshot( - market="US", - session_date="2026-07-28", - output_directory=Path(temporary), - now=datetime( - 2026, - 7, - 28, - 8, - 0, - tzinfo=ZoneInfo("America/New_York"), - ), - fetcher=fetcher, - ) - self.assertEqual(receipt["status"], "READY") - self.assertEqual(receipt["previous_session_date"], "2026-07-27") - self.assertEqual(receipt["api_mutation_count"], 0) - bound = json.loads( - Path(receipt["market_session_path"]).read_text(encoding="utf-8") - ) - self.assertEqual(bound["regular_open"], "2026-07-28T09:30:00-04:00") - self.assertEqual(bound["regular_close"], "2026-07-28T16:00:00-04:00") - - def test_closed_session_does_not_emit_bound_market_session(self) -> None: - responses = { - MODULE.NYSE_CALENDAR_URL: NYSE, - MODULE.NASDAQ_CALENDAR_URL: NASDAQ, - } - - def fetcher(url: str, data: bytes | None = None) -> bytes: - self.assertIsNone(data) - return responses[url] - - with tempfile.TemporaryDirectory(prefix="qta-market-closed-") as temporary: - receipt = MODULE.snapshot( - market="US", - session_date="2026-01-01", - output_directory=Path(temporary), - now=datetime( - 2026, - 1, - 1, - 8, - 0, - tzinfo=ZoneInfo("America/New_York"), - ), - fetcher=fetcher, - ) - self.assertEqual(receipt["status"], "MARKET_CLOSED") - self.assertEqual(receipt["market_session_path"], "") - self.assertFalse((Path(temporary) / "market-session.json").exists()) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_marketplace_packaging.py b/tests/test_marketplace_packaging.py index c7a1629..a99048c 100644 --- a/tests/test_marketplace_packaging.py +++ b/tests/test_marketplace_packaging.py @@ -15,8 +15,6 @@ "advisor-review", "katok-reply-reuse", "learnus-course-copilot", - "quant-stock-polling-trader", - "quant-stock-technical", "stock-scenario-story", "yonsei-central-student-governance-counsel", } @@ -60,7 +58,7 @@ def main() -> None: claude_entries = {entry["name"]: entry for entry in claude_market["plugins"]} assert set(codex_entries) == EXPECTED_PLUGINS assert set(claude_entries) == EXPECTED_PLUGINS - assert len(codex_market["plugins"]) == len(claude_market["plugins"]) == 8 + assert len(codex_market["plugins"]) == len(claude_market["plugins"]) == 6 assert not (PLUGINS / "mrcha-skills").exists(), ( "aggregate plugin must not remain installable" ) @@ -104,7 +102,7 @@ def main() -> None: packaged / relative ).read_bytes(), f"{plugin_name}/{relative} content drift" - print("catalog-backed eight-plugin marketplace packaging: PASS") + print("catalog-backed six-plugin marketplace packaging: PASS") if __name__ == "__main__": diff --git a/tests/test_position_lifecycle.py b/tests/test_position_lifecycle.py deleted file mode 100644 index 2df4f6a..0000000 --- a/tests/test_position_lifecycle.py +++ /dev/null @@ -1,565 +0,0 @@ -#!/usr/bin/env python3 -"""Regression tests for deterministic strategy-position management.""" - -from __future__ import annotations - -import importlib.util -import sys -import tempfile -import unittest -from datetime import datetime, timedelta, timezone -from decimal import Decimal -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -SCRIPTS = ROOT / "skills" / "quant-stock-polling-trader" / "scripts" -sys.path.insert(0, str(SCRIPTS)) -SPEC = importlib.util.spec_from_file_location( - "qta_position_lifecycle", - SCRIPTS / "position_lifecycle.py", -) -if SPEC is None or SPEC.loader is None: - raise RuntimeError("cannot load position_lifecycle.py") -MODULE = importlib.util.module_from_spec(SPEC) -sys.modules[SPEC.name] = MODULE -SPEC.loader.exec_module(MODULE) - -import broker_adapters as BROKERS -import run_session as RUNNER - - -def policy() -> dict[str, object]: - return { - "schema": MODULE.POSITION_POLICY_SCHEMA, - "close_liquidation_seconds_before_close": 600, - "exit_order_ttl_seconds": 15, - "max_exit_replacements": 3, - "quote_max_age_seconds": 5, - "partial_entry_action": "CANCEL_REMAINDER_MANAGE_FILLED", - "gap_down_action": "MARKETABLE_LIMIT", - "daily_loss_action": "ENTRY_FREEZE_LIQUIDATE", - "overnight_residual_action": "EXIT_ONLY_NEXT_SESSION", - } - - -def intent() -> dict[str, str]: - return { - "intent_id": "b" * 32, - "market": "KR", - "exchange": "KOSPI", - "symbol": "005930", - "venue": "KRX", - "currency": "KRW", - "quantity": "10", - "stop_price": "65000", - "take_profit_price": "80000", - "resolved_tick_size": "100", - } - - -def open_position(quantity: str = "10") -> dict[str, object]: - value = intent() - value["quantity"] = quantity - position = MODULE.position_from_intent( - plan_hash="a" * 64, - intent=value, - session_date="2026-07-29", - fx_to_krw="1", - ) - return MODULE.record_entry_fill( - position, - cumulative_filled_quantity=quantity, - average_fill_price="70000", - session_date="2026-07-29", - ) - - -def quote( - now: datetime, - *, - last: str, - bid: str, - ask: str, -) -> dict[str, str]: - return { - "market": "KR", - "symbol": "005930", - "last_price": last, - "best_bid": bid, - "best_ask": ask, - "trade_timestamp": now.isoformat(), - "book_timestamp": now.isoformat(), - "received_at": now.isoformat(), - } - - -def account_snapshot( - *, - strategy_quantity: str, - open_orders: list[dict[str, str]] | None = None, -) -> dict[str, object]: - return { - "schema": "qta-account-snapshot/v2", - "broker": "kis", - "environment": "shadow", - "account_alias": "live-main", - "market": "KR", - "currency": "KRW", - "as_of": "2026-07-30T08:50:00+09:00", - "settled_cash": "150000", - "borrowed_buying_power": "0", - "fx_to_krw": "1", - "positions": [ - { - "market": "KR", - "exchange": "KOSPI", - "symbol": "005930", - "quantity": strategy_quantity, - "market_value_krw": "700000", - }, - { - "market": "KR", - "exchange": "KOSDAQ", - "symbol": "035900", - "quantity": "1", - "market_value_krw": "100000", - }, - ], - "open_orders": open_orders or [], - } - - -class PositionLifecycleTests(unittest.TestCase): - def setUp(self) -> None: - self.zone = timezone(timedelta(hours=9)) - self.open = datetime(2026, 7, 29, 9, 0, tzinfo=self.zone) - self.close = datetime(2026, 7, 29, 15, 30, tzinfo=self.zone) - - def decide( - self, - position: dict[str, object], - observed: datetime, - *, - last: str, - bid: str, - ask: str, - first: bool = False, - daily_loss: bool = False, - ) -> dict[str, object]: - return MODULE.exit_decision( - position, - quote(observed, last=last, bid=bid, ask=ask), - policy(), - observed_at=observed, - regular_close=self.close, - first_valid_quote_of_session=first, - daily_loss_breached=daily_loss, - ) - - def test_gap_below_stop_exits_at_current_bid(self) -> None: - decision = self.decide( - open_position(), - self.open, - last="64000", - bid="63900", - ask="64000", - first=True, - ) - self.assertEqual(decision["action"], "SUBMIT_EXIT") - self.assertEqual(decision["reason"], "STOP_GAP") - self.assertEqual(decision["limit_price"], "63900") - - def test_stop_take_profit_and_hold_are_distinct(self) -> None: - position = open_position() - stop = self.decide( - position, - self.open + timedelta(minutes=1), - last="65000", - bid="65000", - ask="65100", - ) - target = self.decide( - position, - self.open + timedelta(minutes=2), - last="80100", - bid="80000", - ask="80100", - ) - hold = self.decide( - position, - self.open + timedelta(minutes=3), - last="72000", - bid="71900", - ask="72000", - ) - self.assertEqual(stop["reason"], "STOP") - self.assertEqual(target["reason"], "TAKE_PROFIT") - self.assertEqual(hold["action"], "HOLD") - - def test_close_liquidation_begins_ten_minutes_before_close(self) -> None: - before = self.decide( - open_position(), - self.close - timedelta(seconds=601), - last="72000", - bid="71900", - ask="72000", - ) - at_cutoff = self.decide( - open_position(), - self.close - timedelta(seconds=600), - last="72000", - bid="71900", - ask="72000", - ) - self.assertEqual(before["action"], "HOLD") - self.assertEqual(at_cutoff["reason"], "MARKET_CLOSE") - - def test_residual_after_close_is_manual_block(self) -> None: - decision = self.decide( - open_position(), - self.close, - last="72000", - bid="71900", - ask="72000", - ) - self.assertEqual(decision["action"], "MANUAL_BLOCK") - self.assertEqual(decision["reason"], "MARKET_CLOSED_WITH_RESIDUAL") - self.assertIsNone(decision["quote"]) - - def test_daily_loss_has_priority(self) -> None: - decision = self.decide( - open_position(), - self.open + timedelta(minutes=1), - last="80000", - bid="79900", - ask="80000", - daily_loss=True, - ) - self.assertEqual(decision["reason"], "DAILY_LOSS_LIMIT") - - def test_partial_entry_cancels_remainder_and_manages_fill(self) -> None: - position = MODULE.position_from_intent( - plan_hash="a" * 64, - intent=intent(), - session_date="2026-07-29", - fx_to_krw="1", - ) - position = MODULE.record_entry_fill( - position, - cumulative_filled_quantity="3", - average_fill_price="70000", - session_date="2026-07-29", - ) - self.assertTrue( - MODULE.partial_entry_requires_cancel( - position, broker_remaining_quantity="7" - ) - ) - self.assertEqual(MODULE.open_quantity(position), Decimal(3)) - - def test_partial_exit_preserves_only_remaining_quantity(self) -> None: - position = MODULE.record_exit_fill( - open_position(), - cumulative_exited_quantity="4", - average_exit_price="68000", - ) - self.assertEqual(position["status"], "EXIT_PARTIAL") - self.assertEqual(MODULE.open_quantity(position), Decimal(6)) - - def test_carry_rolls_to_exit_only_and_does_not_become_new_entry(self) -> None: - partially_exited = MODULE.record_exit_fill( - open_position(), - cumulative_exited_quantity="2", - average_exit_price="68000", - ) - carried = MODULE.roll_position_session( - partially_exited, session_date="2026-07-30" - ) - self.assertEqual(carried["status"], "CARRY_EXIT_ONLY") - self.assertIsNone(carried["daily_baseline_price"]) - based = MODULE.set_daily_baseline( - carried, - session_date="2026-07-30", - price="69000", - ) - with self.assertRaisesRegex(MODULE.BlockedError, "immutable"): - MODULE.set_daily_baseline( - based, - session_date="2026-07-30", - price="69100", - ) - finished = MODULE.record_exit_fill( - based, - cumulative_exited_quantity="10", - average_exit_price="69600", - ) - self.assertEqual(finished["status"], "CLOSED") - self.assertEqual(finished["daily_realized_pnl_native"], "8000") - - def test_loss_threshold_is_exact_and_current_150k_cannot_reach_2m(self) -> None: - self.assertFalse( - MODULE.daily_loss_limit_breached("150000", "2000000") - ) - self.assertTrue( - MODULE.daily_loss_limit_breached("2000000", "2000000") - ) - - def test_stale_or_mismatched_quote_never_triggers_exit(self) -> None: - observed = self.open + timedelta(seconds=10) - stale = quote( - self.open, - last="64000", - bid="63900", - ask="64000", - ) - with self.assertRaisesRegex(MODULE.BlockedError, "stale"): - MODULE.exit_decision( - open_position(), - stale, - policy(), - observed_at=observed, - regular_close=self.close, - first_valid_quote_of_session=True, - daily_loss_breached=False, - ) - - def test_position_registry_survives_session_directories(self) -> None: - carried = MODULE.roll_position_session( - open_position(), session_date="2026-07-30" - ) - with tempfile.TemporaryDirectory(prefix="qta-position-test-") as temporary: - path = Path(temporary) / "account" / "KR" / "positions.sqlite3" - ledger = MODULE.PositionLedger(path) - try: - ledger.put( - carried, - event_type="POSITION_CARRIED", - event_payload={"from": "2026-07-29", "to": "2026-07-30"}, - ) - finally: - ledger.close() - reopened = MODULE.PositionLedger(path) - try: - stored = reopened.open_positions() - self.assertEqual(len(stored), 1) - self.assertEqual(stored[0].status, "CARRY_EXIT_ONLY") - finally: - reopened.close() - - def test_entry_fill_snapshot_creates_and_updates_persistent_position( - self, - ) -> None: - plan = { - "plan_hash": "a" * 64, - "frozen_inputs": {"account": {"fx_to_krw": "1"}}, - } - with tempfile.TemporaryDirectory(prefix="qta-fill-ledger-") as temporary: - path = Path(temporary) / "account" / "KR" / "positions.sqlite3" - ledger = MODULE.PositionLedger(path) - try: - self.assertIsNone( - RUNNER.persist_entry_fill( - ledger, - plan=plan, - intent=intent(), - snapshot={ - "filled_quantity": "0", - "average_fill_price": None, - }, - session_date="2026-07-29", - ) - ) - self.assertEqual(ledger.open_positions(), []) - first = RUNNER.persist_entry_fill( - ledger, - plan=plan, - intent=intent(), - snapshot={ - "broker_order_id": "123", - "normalized_status": "PARTIALLY_FILLED", - "ordered_quantity": "10", - "filled_quantity": "3", - "average_fill_price": "70000", - "remaining_quantity": "7", - }, - session_date="2026-07-29", - ) - self.assertIsNotNone(first) - self.assertEqual(first["acquired_quantity"], "3") - position_id = first["position_id"] - self.assertEqual(len(ledger.events(position_id)), 1) - repeated = RUNNER.persist_entry_fill( - ledger, - plan=plan, - intent=intent(), - snapshot={ - "broker_order_id": "123", - "normalized_status": "PARTIALLY_FILLED", - "ordered_quantity": "10", - "filled_quantity": "3", - "average_fill_price": "70000", - "remaining_quantity": "7", - }, - session_date="2026-07-29", - ) - self.assertEqual(repeated, first) - self.assertEqual(len(ledger.events(position_id)), 1) - updated = RUNNER.persist_entry_fill( - ledger, - plan=plan, - intent=intent(), - snapshot={ - "broker_order_id": "123", - "normalized_status": "PARTIALLY_FILLED", - "ordered_quantity": "10", - "filled_quantity": "5", - "average_fill_price": "70100", - "remaining_quantity": "5", - }, - session_date="2026-07-29", - ) - self.assertEqual(updated["acquired_quantity"], "5") - self.assertEqual(updated["average_entry_price"], "70100") - self.assertEqual(len(ledger.events(position_id)), 2) - finally: - ledger.close() - - def test_entry_fill_without_average_price_is_blocked(self) -> None: - plan = { - "plan_hash": "a" * 64, - "frozen_inputs": {"account": {"fx_to_krw": "1"}}, - } - with tempfile.TemporaryDirectory(prefix="qta-fill-ledger-") as temporary: - ledger = MODULE.PositionLedger(Path(temporary) / "positions.sqlite3") - try: - with self.assertRaisesRegex( - MODULE.BlockedError, "average_fill_price" - ): - RUNNER.persist_entry_fill( - ledger, - plan=plan, - intent=intent(), - snapshot={ - "filled_quantity": "1", - "average_fill_price": None, - }, - session_date="2026-07-29", - ) - self.assertEqual(ledger.open_positions(), []) - finally: - ledger.close() - - def test_kis_order_snapshots_expose_authoritative_average_fill_price( - self, - ) -> None: - broker = BROKERS.KisBroker( - app_key="key", - app_secret="secret", - account_prefix="12345678", - account_product="01", - environment="live", - access_token="token", - ) - cases = ( - ( - "KR", - { - "odno": "123", - "ord_qty": "10", - "tot_ccld_qty": "3", - "rmn_qty": "7", - "avg_prvs": "70000", - }, - "70000", - ), - ( - "US", - { - "odno": "123", - "ft_ord_qty": "10", - "ft_ccld_qty": "3", - "nccs_qty": "7", - "ft_ccld_unpr3": "201.25", - }, - "201.25", - ), - ) - for market, record, expected in cases: - broker.order_history = lambda **_kwargs: [record] - snapshot = broker.get_order( - market=market, - trading_date="2026-07-29", - symbol="005930" if market == "KR" else "AAPL", - venue="KRX" if market == "KR" else "NASD", - broker_order_id="123", - ) - self.assertEqual(snapshot["average_fill_price"], expected) - self.assertEqual(snapshot["normalized_status"], "PARTIALLY_FILLED") - - def test_account_reconciliation_rolls_exact_carry_and_lists_manual_holdings( - self, - ) -> None: - with tempfile.TemporaryDirectory(prefix="qta-reconcile-") as temporary: - ledger = MODULE.PositionLedger(Path(temporary) / "positions.sqlite3") - try: - position = open_position() - ledger.put( - position, - event_type="ENTRY_FILL_CREATED", - event_payload={"filled_quantity": "10"}, - ) - receipt = MODULE.reconcile_account_positions( - ledger, - account_snapshot=account_snapshot(strategy_quantity="10"), - session_date="2026-07-30", - ) - self.assertEqual(receipt["status"], "READY") - self.assertEqual( - receipt["managed_positions"][0]["state"], - "CARRY_EXIT_ONLY", - ) - self.assertEqual( - receipt["unmanaged_broker_positions"], - [ - { - "exchange": "KOSDAQ", - "symbol": "035900", - "quantity": "1", - } - ], - ) - self.assertEqual(receipt["api_mutation_count"], 0) - self.assertFalse(receipt["live_enabled"]) - finally: - ledger.close() - - def test_account_quantity_mismatch_becomes_manual_block(self) -> None: - with tempfile.TemporaryDirectory(prefix="qta-reconcile-") as temporary: - ledger = MODULE.PositionLedger(Path(temporary) / "positions.sqlite3") - try: - position = open_position() - ledger.put( - position, - event_type="ENTRY_FILL_CREATED", - event_payload={"filled_quantity": "10"}, - ) - receipt = MODULE.reconcile_account_positions( - ledger, - account_snapshot=account_snapshot(strategy_quantity="9"), - session_date="2026-07-30", - ) - self.assertEqual(receipt["status"], "MANUAL_BLOCK") - self.assertIn( - "broker_quantity_differs_from_strategy_ledger", - receipt["managed_positions"][0]["reasons"], - ) - self.assertEqual( - ledger.open_positions()[0].status, - "MANUAL_BLOCK", - ) - finally: - ledger.close() - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_qta2_execution.py b/tests/test_qta2_execution.py deleted file mode 100644 index b6d4f81..0000000 --- a/tests/test_qta2_execution.py +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env python3 -"""Regression tests for the QTA 2.0 shadow execution contract.""" - -from __future__ import annotations - -import sys -import unittest -from copy import deepcopy -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -SCRIPTS = ROOT / "skills" / "quant-stock-polling-trader" / "scripts" -sys.path.insert(0, str(ROOT)) -sys.path.insert(0, str(SCRIPTS)) - -import execution_core -import run_session -from tests.test_quant_execution import ( - complete_v2_screen, - rehash_screen, - v2_selected_item, -) - - -def qta2_screen() -> dict[str, object]: - selected = { - "KOSPI": [ - v2_selected_item( - "KOSPI", - 1, - "005930", - "005930", - "005930", - ) - ], - "KOSDAQ": [], - "NYSE": [], - "NASDAQ": [], - } - screen = complete_v2_screen(execution_core, selected) - screen["method_version"] = execution_core.QTA_METHOD_V2 - payloads = [ - selected["KOSPI"][0]["qta"], - screen["decisions"][0]["qta"], - ] - for payload in payloads: - payload.update( - { - "method_version": execution_core.QTA_METHOD_V2, - "validation_status": "RESEARCH_ONLY", - "score_basis": execution_core.QTA2_SCORE_BASIS, - "liquidity": { - "median_20_session_turnover": 2_000_000_000.0, - "minimum_turnover": 1_000_000_000.0, - "currency": "KRW", - "status": "READY", - }, - "market_regime": { - "metric": execution_core.QTA2_REGIME_METRIC, - "minimum_change_bps": 0.0, - "max_age_seconds": ( - execution_core.QTA2_REGIME_MAX_AGE_SECONDS - ), - "status": "REQUIRED", - }, - "assumptions": list(execution_core.QTA2_ASSUMPTIONS), - } - ) - screen["selected"]["KOSPI"][0]["qta"] = deepcopy(payloads[0]) - rehash_screen(execution_core, screen) - return screen - - -class Qta2ExecutionTests(unittest.TestCase): - def test_qta2_screen_is_valid_and_shadow_only(self) -> None: - screen = qta2_screen() - execution_core.validate_screen(screen) - plan = { - "context": {"broker": "kis", "environment": "shadow"}, - "frozen_inputs": {"screen": screen}, - } - run_session.validate_mode(plan, "kis-live", "shadow") - with self.assertRaisesRegex( - execution_core.BlockedError, "RESEARCH_ONLY" - ): - paper_plan = { - "context": {"broker": "kis", "environment": "paper"}, - "frozen_inputs": {"screen": screen}, - } - run_session.validate_mode(paper_plan, "kis-paper", "paper") - - def test_qta2_formula_tamper_is_rejected(self) -> None: - screen = qta2_screen() - screen["selected"]["KOSPI"][0]["qta"]["total_score"] = 71.0 - screen["decisions"][0]["qta"]["total_score"] = 71.0 - rehash_screen(execution_core, screen) - with self.assertRaisesRegex( - execution_core.BlockedError, - "qta-2.0.0 score formula", - ): - execution_core.validate_screen(screen) - - def test_qta2_liquidity_status_tamper_is_rejected(self) -> None: - screen = qta2_screen() - for payload in ( - screen["selected"]["KOSPI"][0]["qta"], - screen["decisions"][0]["qta"], - ): - payload["liquidity"]["status"] = "BLOCKED" - rehash_screen(execution_core, screen) - with self.assertRaisesRegex( - execution_core.BlockedError, - "liquidity.status", - ): - execution_core.validate_screen(screen) - - def test_qta2_market_regime_tamper_is_rejected(self) -> None: - screen = qta2_screen() - for payload in ( - screen["selected"]["KOSPI"][0]["qta"], - screen["decisions"][0]["qta"], - ): - payload["market_regime"]["minimum_change_bps"] = -50.0 - rehash_screen(execution_core, screen) - with self.assertRaisesRegex( - execution_core.BlockedError, - "minimum_change_bps", - ): - execution_core.validate_screen(screen) - - def test_screen_and_nested_qta_methods_must_match(self) -> None: - screen = qta2_screen() - screen["method_version"] = execution_core.QTA_METHOD_V1 - rehash_screen(execution_core, screen) - with self.assertRaisesRegex( - execution_core.BlockedError, - "method differs", - ): - execution_core.validate_screen(screen) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_qta2_market_regime.py b/tests/test_qta2_market_regime.py deleted file mode 100644 index 20cc733..0000000 --- a/tests/test_qta2_market_regime.py +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/env python3 -"""Regression tests for QTA2 same-session market-regime admission.""" - -from __future__ import annotations - -import sys -import unittest -from datetime import datetime, timedelta, timezone -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -SCRIPTS = ROOT / "skills" / "quant-stock-polling-trader" / "scripts" -sys.path.insert(0, str(SCRIPTS)) - -from broker_adapters import HttpResponse, KisBroker, QueueTransport -from execution_core import BlockedError -from run_session import evaluate_market_regime - - -def kis_broker(responses: list[HttpResponse], *, environment: str = "live") -> KisBroker: - return KisBroker( - app_key="test-key", - app_secret="test-secret", - account_prefix="12345678", - account_product="01", - environment=environment, - transport=QueueTransport(responses), - access_token="test-token", - ) - - -class Qta2MarketRegimeTests(unittest.TestCase): - def test_kr_benchmark_quote_and_admission(self) -> None: - broker = kis_broker( - [ - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "msg_cd": "0", - "msg1": "OK", - "output": [ - { - "stck_cntg_hour": "090005", - "bstp_nmix_prpr": "3200", - "bstp_nmix_prdy_vrss": "16", - "prdy_vrss_sign": "2", - } - ], - }, - ) - ] - ) - quote = broker.benchmark_quote( - exchange="KOSPI", - session_date="2026-07-29", - ) - observed = datetime.fromisoformat(quote["source_timestamp"]) + timedelta( - seconds=1 - ) - quote["received_at"] = observed.astimezone(timezone.utc).isoformat() - decision = evaluate_market_regime( - exchange="KOSPI", - quote=quote, - observed_at=observed, - session_date="2026-07-29", - ) - self.assertTrue(decision["quality_valid"]) - self.assertTrue(decision["admitted"]) - self.assertEqual(quote["previous_close"], "3184") - - def test_negative_benchmark_blocks_entry(self) -> None: - now = datetime(2026, 7, 29, 9, 0, 5, tzinfo=timezone(timedelta(hours=9))) - quote = { - "schema": "qta-market-regime-quote/v1", - "broker": "kis", - "exchange": "KOSDAQ", - "benchmark_id": "KOSDAQ_COMPOSITE", - "regime_proxy_id": "KOSDAQ_COMPOSITE", - "provider_symbol": "1001", - "current": "990", - "previous_close": "1000", - "change_bps": "-100.00", - "source_timestamp": now.isoformat(), - "received_at": now.isoformat(), - "raw_status": "OK", - } - decision = evaluate_market_regime( - exchange="KOSDAQ", - quote=quote, - observed_at=now, - session_date="2026-07-29", - ) - self.assertTrue(decision["quality_valid"]) - self.assertFalse(decision["admitted"]) - self.assertIn("benchmark_below_minimum", decision["reasons"]) - - def test_stale_benchmark_blocks_entry(self) -> None: - now = datetime(2026, 7, 29, 9, 2, 0, tzinfo=timezone(timedelta(hours=9))) - stale = now - timedelta(seconds=91) - quote = { - "schema": "qta-market-regime-quote/v1", - "broker": "kis", - "exchange": "KOSPI", - "benchmark_id": "KOSPI_COMPOSITE", - "regime_proxy_id": "KOSPI_COMPOSITE", - "provider_symbol": "0001", - "current": "1001", - "previous_close": "1000", - "change_bps": "10.00", - "source_timestamp": stale.isoformat(), - "received_at": now.isoformat(), - "raw_status": "OK", - } - decision = evaluate_market_regime( - exchange="KOSPI", - quote=quote, - observed_at=now, - session_date="2026-07-29", - ) - self.assertFalse(decision["quality_valid"]) - self.assertFalse(decision["admitted"]) - self.assertIn("source_timestamp_stale", decision["reasons"]) - - def test_us_benchmark_uses_timestamped_intraday_price(self) -> None: - broker = kis_broker( - [ - HttpResponse( - 200, - {}, - { - "rt_cd": "0", - "msg_cd": "0", - "msg1": "OK", - "output1": { - "ovrs_nmix_prdy_clpr": "25000", - "ovrs_nmix_prpr": "25100", - }, - "output2": [ - { - "stck_bsop_date": "20260729", - "stck_cntg_hour": "093000", - "optn_prpr": "25025", - "optn_oprc": "25020", - "optn_hgpr": "25030", - "optn_lwpr": "25010", - "cntg_vol": "0", - } - ], - }, - ) - ] - ) - quote = broker.benchmark_quote( - exchange="NASDAQ", - session_date="2026-07-29", - ) - self.assertEqual(quote["current"], "25025") - self.assertEqual(quote["previous_close"], "25000") - self.assertEqual(quote["change_bps"], "10.00") - self.assertEqual(quote["regime_proxy_id"], "NASDAQ_COMPOSITE") - - def test_us_paper_benchmark_is_rejected(self) -> None: - broker = kis_broker([], environment="paper") - with self.assertRaisesRegex(BlockedError, "paper benchmark"): - broker.benchmark_quote( - exchange="NASDAQ", - session_date="2026-07-29", - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_quant_execution.py b/tests/test_quant_execution.py deleted file mode 100755 index c9cabc5..0000000 --- a/tests/test_quant_execution.py +++ /dev/null @@ -1,1576 +0,0 @@ -#!/usr/bin/env python3 -"""Regression suite for deterministic screening and broker execution helpers.""" - -from __future__ import annotations - -import os -import subprocess -import sys -import tempfile -from copy import deepcopy -from datetime import datetime, timedelta, timezone -from pathlib import Path -from unittest.mock import patch - -ROOT = Path(__file__).resolve().parents[1] -EXECUTION_SCRIPTS = ROOT / "skills" / "quant-stock-polling-trader" / "scripts" -TECHNICAL_SCRIPTS = ROOT / "skills" / "quant-stock-technical" / "scripts" -QTA_REFERENCE_FIELDS = { - "return_5", - "return_20", - "close_over_sma_20", - "sma_5_over_sma_20", - "signed_log_volume_ratio_20", - "rsi_14", - "return_63", - "close_over_sma_50", - "sma_20_over_sma_50", - "relative_return_63", - "adx_direction_14", - "return_126", - "return_252", - "close_over_sma_200", - "sma_50_over_sma_200", - "relative_return_252", - "ATR14/close", - "20d annualized volatility", - "252d maximum drawdown", - "60d gap p95", -} - - -def run(*arguments: str) -> None: - result = subprocess.run( - [sys.executable, "-B", *arguments], - cwd=ROOT, - text=True, - capture_output=True, - check=False, - ) - assert result.returncode == 0, ( - f"{' '.join(arguments)} failed\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}" - ) - assert '"self_test"' in result.stdout or '"self_test": "PASS"' in result.stdout - - -def assert_credential_guidance_contract() -> None: - skill_text = ( - ROOT / "skills" / "quant-stock-polling-trader" / "SKILL.md" - ).read_text(encoding="utf-8") - status_command = "python3 scripts/broker_credentials.py status --environment live" - configure_command = ( - "python3 scripts/broker_credentials.py configure --environment live" - ) - auth_command = ( - "python3 scripts/broker_credentials.py auth-check --environment live" - ) - first_status = skill_text.index(status_command) - configure = skill_text.index(configure_command) - second_status = skill_text.index(status_command, configure + 1) - auth_check = skill_text.index(auth_command) - assert first_status < configure < second_status < auth_check - for required_guidance in ( - "Never ask the user to paste a credential into chat.", - "Wait for the user to confirm", - "Authentication is proven only when the result says `AUTHENTICATED`.", - "does not enable live trading", - ): - assert required_guidance in skill_text - - openai_text = ( - ROOT / "skills" / "quant-stock-polling-trader" / "agents" / "openai.yaml" - ).read_text(encoding="utf-8") - assert "$quant-stock-polling-trader" in openai_text - assert "direct KIS shadow workflow" in openai_text - - plugin_text = ( - ROOT - / "plugins" - / "quant-stock-polling-trader" - / ".codex-plugin" - / "plugin.json" - ).read_text(encoding="utf-8") - assert "Securely guide KIS credential setup and verify authentication." in ( - plugin_text - ) - - -def v2_instrument( - exchange: str, - canonical_symbol: str, - data_symbol: str, - broker_symbol: str, -) -> dict[str, object]: - contracts = { - "KOSPI": ("KR", "KOSPI_COMPOSITE", "KRW", "KRX"), - "KOSDAQ": ("KR", "KOSDAQ_COMPOSITE", "KRW", "KRX"), - "NYSE": ("US", "NYSE_COMPOSITE", "USD", "NYSE"), - "NASDAQ": ("US", "NASDAQ_COMPOSITE", "USD", "NASD"), - } - market, benchmark_id, currency, venue = contracts[exchange] - return { - "market": market, - "exchange": exchange, - "canonical_symbol": canonical_symbol, - "data_symbol": data_symbol, - "broker_symbol": broker_symbol, - "instrument_type": "COMMON", - "benchmark_id": benchmark_id, - "currency": currency, - "venue": venue, - "ticker_csv": f"/fixture/data/{data_symbol}.csv", - "benchmark_csv": f"/fixture/data/{benchmark_id}.csv", - "tick_contract": { - "schema": "qta-tick-contract/v1", - "kind": "RESOLVED_PRICE_LADDER", - "rule_id": f"{exchange.lower()}-tick", - "effective_date": "2026-07-24", - "reference_price": "100", - "resolved_tick_size": "0.01" if market == "US" else "100", - }, - "source_name": "regression-fixture", - "ticker_csv_sha256": "a" * 64, - "benchmark_csv_sha256": "b" * 64, - "broker_tradability_verified": True, - "official_source_id": f"official:{exchange}", - "broker_source_id": f"kis:{exchange}", - } - - -def opinion(score: float) -> str: - if score >= 80: - return "강한 긍정" - if score >= 60: - return "긍정" - if score >= 40: - return "중립" - if score >= 20: - return "부정" - return "강한 부정" - - -def v2_qta( - market: str, - canonical_symbol: str, - *, - total_score: float = 70.0, - short_score: float = 70.0, - medium_score: float = 70.0, - long_score: float = 70.0, - risk_score: float = 30.0, -) -> dict[str, object]: - entry_price, stop_price, take_profit_price = ( - (70000.0, 65000.0, 80000.0) if market == "KR" else (100.0, 90.0, 120.0) - ) - return { - "source_skill": "quant-stock-technical", - "result_schema": "quant-stock-technical/v1", - "calculation_status": "READY", - "setup_status": "READY" if total_score >= 60 else "CONDITIONAL", - "method_version": "qta-1.0.0", - "analysis_date": "2026-07-24", - "market": market, - "ticker": canonical_symbol, - "source_name": "regression-fixture", - "shared_sessions": 1000, - "score_basis": ( - "ticker-relative historical percentile; not probability of profit" - ), - "short": {"opinion": opinion(short_score), "score": short_score}, - "medium": {"opinion": opinion(medium_score), "score": medium_score}, - "long": {"opinion": opinion(long_score), "score": long_score}, - "risk": {"score": risk_score, "counterpoint": "regression fixture"}, - "entry_price": entry_price, - "stop_price": stop_price, - "take_profit_price": take_profit_price, - "total_score": total_score, - "reference_observations": { - feature: 756 for feature in sorted(QTA_REFERENCE_FIELDS) - }, - "assumptions": [ - "input rows are finalized completed daily sessions", - ( - "prices are corporate-action adjusted and aligned to " - "analysis-date raw price scale" - ), - "fees, tax, FX, slippage, position size, and execution are excluded", - ], - } - - -def v2_selected_item( - exchange: str, - exchange_rank: int, - canonical_symbol: str, - data_symbol: str, - broker_symbol: str, -) -> dict[str, object]: - market = "KR" if exchange in {"KOSPI", "KOSDAQ"} else "US" - return { - "exchange_rank": exchange_rank, - "instrument": v2_instrument( - exchange, - canonical_symbol, - data_symbol, - broker_symbol, - ), - "qta": v2_qta(market, canonical_symbol), - } - - -def rehash_screen(execution_core: object, screen: dict[str, object]) -> None: - screen.pop("screen_hash", None) - screen["screen_hash"] = execution_core.canonical_screen_hash(screen) - - -def rehash_plan(execution_core: object, plan: dict[str, object]) -> None: - plan.pop("plan_hash", None) - plan["plan_hash"] = execution_core.sha256_json(plan) - - -def complete_v2_screen( - execution_core: object, - selected: dict[str, list[dict[str, object]]], -) -> dict[str, object]: - exchanges = ("KOSPI", "KOSDAQ", "NYSE", "NASDAQ") - selector = { - "selector_version": execution_core.V2_SELECTOR_VERSION, - "min_total_score": "0", - "eligible_setup_statuses": ["READY"], - "top_k_by_exchange": { - exchange: len(selected[exchange]) for exchange in exchanges - }, - "min_selected_by_exchange": {exchange: 0 for exchange in exchanges}, - "max_blocked_fraction": "0", - } - decisions = [ - { - "market": item["instrument"]["market"], - "exchange": exchange, - "canonical_symbol": item["instrument"]["canonical_symbol"], - "instrument": deepcopy(item["instrument"]), - "eligible": True, - "reasons": [], - "exchange_rank": item["exchange_rank"], - "selected": True, - "qta": deepcopy(item["qta"]), - } - for exchange in exchanges - for item in selected[exchange] - ] - screen = { - "source_skill": "quant-stock-technical", - "schema": execution_core.SCREEN_SCHEMA_V2, - "screen_status": "READY", - "method_version": "qta-1.0.0", - "selector_version": execution_core.V2_SELECTOR_VERSION, - "analysis_date": "2026-07-24", - "manifest_hash": "c" * 64, - "selector_hash": execution_core.sha256_json(selector), - "blocked_count": 0, - "blocked_fraction": "0", - "instrument_count": len(decisions), - "selector": selector, - "selected": selected, - "decisions": decisions, - } - rehash_screen(execution_core, screen) - return screen - - -def main() -> None: - assert_credential_guidance_contract() - run("skills/quant-stock-technical/scripts/analyze_stock.py", "--self-test") - run( - "skills/quant-stock-technical/scripts/fetch_kis_kr_eod.py", - "--self-test", - ) - run( - "skills/quant-stock-technical/scripts/fetch_kis_us_eod.py", - "--self-test", - ) - run( - "skills/quant-stock-technical/scripts/selected_open1h_outcomes.py", - "--self-test", - ) - run( - "skills/quant-stock-technical/scripts/walk_forward_open1h.py", - "--self-test", - ) - run("skills/quant-stock-technical/scripts/screen_universe.py", "--self-test") - run( - "skills/quant-stock-polling-trader/scripts/execution_core.py", - "--self-test", - ) - run( - "skills/quant-stock-polling-trader/scripts/freeze_market_session.py", - "--self-test", - ) - run( - "skills/quant-stock-polling-trader/scripts/broker_adapters.py", - "--self-test", - ) - run( - "skills/quant-stock-polling-trader/scripts/broker_credentials.py", - "self-test", - ) - run( - "skills/quant-stock-polling-trader/scripts/account_snapshot.py", - "self-test", - ) - run("skills/quant-stock-polling-trader/scripts/plan_orders.py", "--self-test") - run("skills/quant-stock-polling-trader/scripts/run_session.py", "self-test") - run( - "skills/quant-stock-polling-trader/scripts/reconcile.py", - "--self-test", - ) - run( - "skills/quant-stock-polling-trader/scripts/systemd_units.py", - "self-test", - ) - sys.path.insert(0, str(TECHNICAL_SCRIPTS)) - import screen_universe - - sys.path.insert(0, str(EXECUTION_SCRIPTS)) - import execution_core - import plan_orders - import run_session - - def assert_blocked(callable_object: object, expected: str) -> None: - try: - callable_object() - except execution_core.BlockedError as exc: - assert expected in str(exc), str(exc) - else: - raise AssertionError(f"expected BlockedError containing {expected!r}") - - assert execution_core.normalize_symbol("KR", "5930") == "005930" - assert execution_core.normalize_symbol("KR", "0156t0") == "0156T0" - assert "PARTIALLY_FILLED" not in execution_core.TRANSITIONS["CANCEL_PENDING"] - redacted_identity = execution_core.redact( - { - "account_seq": "7", - "accountSeq": "7", - "account_prefix": "12345678", - "account_product": "01", - "ACNT_PRDT_CD": "01", - } - ) - assert set(redacted_identity.values()) == {"[REDACTED]"} - assert_blocked( - lambda: execution_core.normalize_symbol("KR", "156T0"), - "invalid KR symbol", - ) - legacy_inputs = list(plan_orders.fixture_inputs()) - rehash_screen(execution_core, legacy_inputs[0]) - plan = execution_core.plan_orders(*legacy_inputs) - assert "exchange" not in plan["intents"][0] - assert "broker_account_identity_hash" not in legacy_inputs[1] - assert "broker_account_identity_hash" not in plan["context"] - runtime_plan = deepcopy(plan) - runtime_now = datetime.now(timezone.utc) - runtime_plan["entry_window"]["start"] = ( - runtime_now - timedelta(milliseconds=100) - ).isoformat() - runtime_plan["entry_window"]["end"] = ( - runtime_now + timedelta(seconds=8) - ).isoformat() - runtime_plan["entry_window"]["poll_interval_seconds"] = 4 - assert_blocked( - lambda: run_session.preflight_venues(runtime_plan, {}), - "venue map missing KR:005930", - ) - - class ShadowQuoteBroker: - def token(self) -> str: - return "not-logged" - - def preview_submit( - self, - intent: dict[str, object], - *, - venue: str, - ) -> dict[str, object]: - return { - "request_hash": "c" * 64, - "venue": venue, - "intent_id": intent["intent_id"], - } - - def quote( - self, - market: str, - symbol: str, - *, - venue: str, - ) -> dict[str, object]: - observed = datetime.now(timezone.utc).isoformat() - return { - "market": market, - "symbol": symbol, - "last_price": "69000", - "best_ask": "69100", - "best_bid": "68900", - "received_at": observed, - "trade_timestamp": observed, - "book_timestamp": observed, - "raw_status": "OK", - } - - with ( - tempfile.TemporaryDirectory(prefix="qta-run-smoke-") as state_directory, - patch.object(run_session, "validate_plan"), - patch.object(run_session, "validate_mode"), - patch.object(run_session, "validate_runtime_capabilities"), - patch.object(run_session, "create_broker", return_value=ShadowQuoteBroker()), - patch.object( - run_session, - "warm_broker_before_open", - return_value={ - "scheduled_at": runtime_now.isoformat(), - "started_at": runtime_now.isoformat(), - "completed_at": runtime_now.isoformat(), - "latency_ms": 0, - "token_material_logged": False, - }, - ), - ): - runtime_receipt = run_session.run_session( - runtime_plan, - "kis-paper", - "paper", - {"KR:005930": "KRX"}, - Path(state_directory), - max_cycles=1, - ) - assert runtime_receipt["freeze_reason"] == "max_cycles_reached" - assert runtime_receipt["polling_metrics"]["cycles_started"] == 1 - assert runtime_receipt["polling_metrics"]["cycles_completed"] == 1 - assert runtime_receipt["polling_metrics"]["quotes_evaluated"] == 1 - assert runtime_receipt["polling_metrics"]["expected_http_requests_started"] == 2 - assert "arm_hash" not in runtime_receipt - with tempfile.TemporaryDirectory(prefix="qta-cross-plan-") as state_directory: - foreign_ledger = execution_core.Ledger(Path(state_directory) / "ledger.sqlite3") - foreign_ledger.create_intent("b" * 64, {"intent_id": "old-intent"}) - foreign_ledger.transition("old-intent", "WAIT_TRIGGER", {}) - foreign_ledger.transition("old-intent", "RESERVED", {}) - foreign_ledger.transition( - "old-intent", - "SUBMITTING", - {"request_hash": "e" * 64}, - request_hash="e" * 64, - ) - foreign_ledger.transition("old-intent", "UNKNOWN", {}) - foreign_ledger.transition("old-intent", "RECONCILING", {}) - foreign_ledger.transition("old-intent", "MANUAL_BLOCK", {}) - foreign_ledger.close() - - class NeverQuoteBroker: - def token(self) -> str: - raise AssertionError("foreign-plan block must precede warm-up") - - def preview_submit( - self, - intent: dict[str, object], - *, - venue: str, - ) -> dict[str, object]: - return { - "request_hash": "d" * 64, - "venue": venue, - "intent_id": intent["intent_id"], - } - - def quote(self, *_: object, **__: object) -> dict[str, object]: - raise AssertionError("foreign-plan block must precede quote") - - with ( - patch.object(run_session, "validate_plan"), - patch.object(run_session, "validate_mode"), - patch.object(run_session, "validate_runtime_capabilities"), - patch.object( - run_session, - "create_broker", - return_value=NeverQuoteBroker(), - ), - ): - assert_blocked( - lambda: run_session.run_session( - runtime_plan, - "kis-paper", - "paper", - {"KR:005930": "KRX"}, - Path(state_directory), - max_cycles=1, - ), - "nonterminal intents outside the current plan", - ) - with tempfile.TemporaryDirectory(prefix="qta-manual-block-") as state_directory: - blocked_ledger = execution_core.Ledger(Path(state_directory) / "ledger.sqlite3") - blocked_intent = runtime_plan["intents"][0] - blocked_ledger.create_intent(runtime_plan["plan_hash"], blocked_intent) - blocked_ledger.transition(blocked_intent["intent_id"], "WAIT_TRIGGER", {}) - blocked_ledger.transition(blocked_intent["intent_id"], "RESERVED", {}) - blocked_ledger.transition( - blocked_intent["intent_id"], - "SUBMITTING", - {"request_hash": "f" * 64}, - request_hash="f" * 64, - ) - blocked_ledger.transition(blocked_intent["intent_id"], "UNKNOWN", {}) - blocked_ledger.transition(blocked_intent["intent_id"], "RECONCILING", {}) - blocked_ledger.transition(blocked_intent["intent_id"], "MANUAL_BLOCK", {}) - assert_blocked( - lambda: run_session.validate_ledger_scope( - blocked_ledger, - runtime_plan, - ), - "unresolved mutation states", - ) - blocked_ledger.close() - toss_inputs = deepcopy(legacy_inputs) - toss_inputs[1].update( - { - "broker": "toss", - "environment": "shadow", - "account_alias": "toss-shadow-kr", - } - ) - toss_plan = execution_core.plan_orders(*toss_inputs) - assert toss_plan["context"]["account_alias"] == "toss-shadow-kr" - stale_legacy_inputs = deepcopy(legacy_inputs) - stale_legacy_inputs[1]["as_of"] = "2026-07-27T08:00:00+09:00" - stale_legacy_inputs[2]["as_of"] = "2026-07-27T08:00:00+09:00" - assert_blocked( - lambda: execution_core.plan_orders(*stale_legacy_inputs), - "account snapshots exceed snapshot_max_age_seconds", - ) - tampered_plan_quantity = deepcopy(plan) - tampered_plan_quantity["intents"][0]["quantity"] = "999999" - rehash_plan(execution_core, tampered_plan_quantity) - assert_blocked( - lambda: run_session.validate_plan(tampered_plan_quantity), - "intent_hash does not match intent contents", - ) - tampered_plan_limit = deepcopy(plan) - tampered_plan_limit["intents"][0]["limit_price"] = "1" - rehash_plan(execution_core, tampered_plan_limit) - assert_blocked( - lambda: run_session.validate_plan(tampered_plan_limit), - "intent_hash does not match intent contents", - ) - tampered_plan_window = deepcopy(plan) - tampered_plan_window["entry_window"]["start"] = "2026-07-27T08:00:00+09:00" - rehash_plan(execution_core, tampered_plan_window) - assert_blocked( - lambda: run_session.validate_plan(tampered_plan_window), - "must be the first hour of the regular session", - ) - tampered_plan_broker = deepcopy(plan) - tampered_plan_broker["context"]["broker"] = "toss" - rehash_plan(execution_core, tampered_plan_broker) - assert_blocked( - lambda: run_session.validate_plan(tampered_plan_broker), - "intent_id does not match deterministic seed", - ) - tampered_order_policy = deepcopy(plan) - tampered_order_policy["order_policy"]["cancel_remainder_at_window_end"] = False - rehash_plan(execution_core, tampered_order_policy) - assert_blocked( - lambda: run_session.validate_plan(tampered_order_policy), - "does not match the embedded execution policy", - ) - tampered_quote_policy = deepcopy(plan) - tampered_quote_policy["quote_policy"]["max_spread_bps"] = "9999" - rehash_plan(execution_core, tampered_quote_policy) - assert_blocked( - lambda: run_session.validate_plan(tampered_quote_policy), - "does not match the embedded execution policy", - ) - tampered_plan_status = deepcopy(plan) - tampered_plan_status["plan_status"] = "NO_ORDERS" - rehash_plan(execution_core, tampered_plan_status) - assert_blocked( - lambda: run_session.validate_plan(tampered_plan_status), - "plan_status does not match intents", - ) - fully_rehashed_quantity = deepcopy(plan) - fully_rehashed_intent = fully_rehashed_quantity["intents"][0] - fully_rehashed_intent["quantity"] = "2" - fully_rehashed_intent["reserved_cash"] = format( - execution_core.decimal_value( - plan["intents"][0]["reserved_cash"], - "reserved_cash", - ) - * 2, - "f", - ) - fully_rehashed_quantity["settled_cash_unreserved"] = format( - execution_core.decimal_value( - fully_rehashed_quantity["settled_cash_start"], - "settled_cash_start", - ) - - execution_core.decimal_value( - fully_rehashed_intent["reserved_cash"], - "reserved_cash", - ), - "f", - ) - fully_rehashed_seed = { - "plan_seed": execution_core.sha256_json(fully_rehashed_quantity["context"]), - "rank": fully_rehashed_intent["rank"], - "market": fully_rehashed_intent["market"], - "symbol": fully_rehashed_intent["symbol"], - "side": fully_rehashed_intent["side"], - "order_type": fully_rehashed_intent["order_type"], - "time_in_force": fully_rehashed_intent["time_in_force"], - "quantity": fully_rehashed_intent["quantity"], - "limit_price": fully_rehashed_intent["limit_price"], - } - fully_rehashed_intent["intent_id"] = execution_core.sha256_json( - fully_rehashed_seed - )[:32] - fully_rehashed_intent["client_order_id"] = ( - f"qta-{fully_rehashed_intent['intent_id'][:28]}" - ) - fully_rehashed_intent.pop("intent_hash") - fully_rehashed_intent["intent_hash"] = execution_core.sha256_json( - fully_rehashed_intent - ) - rehash_plan(execution_core, fully_rehashed_quantity) - assert_blocked( - lambda: run_session.validate_plan(fully_rehashed_quantity), - "does not match deterministic rebuild from frozen_inputs", - ) - tampered_legacy_qta = deepcopy(legacy_inputs[0]) - tampered_legacy_qta["selected"]["KR"][0]["qta"]["entry_price"] = 71000 - assert_blocked( - lambda: execution_core.plan_orders( - tampered_legacy_qta, - *legacy_inputs[1:], - ), - "screen_hash does not match canonical screen payload", - ) - preview = run_session.preview_requests(plan, "kis-paper", {"KR:005930": "KRX"}) - assert preview["status"] == "READY" - assert preview["mutation_sent"] is False - assert preview["previews"][0]["request"]["tr_id"] == "VTTC0012U" - toss_preview = run_session.preview_requests(plan, "toss", {}) - assert toss_preview["status"] == "READY" - assert toss_preview["account_bound"] is False - assert len(toss_preview["previews"][0]["request"]["request_hash"]) == 64 - toss_environment = { - "QTA_TOSS_CLIENT_ID": "test-client", - "QTA_TOSS_CLIENT_SECRET": "test-secret", - "QTA_TOSS_ACCOUNT_SEQ": "1", - "QTA_TOSS_ACCESS_TOKEN": "test-token", - } - with patch.dict(os.environ, toss_environment, clear=True): - try: - run_session.create_broker("toss") - except execution_core.BlockedError as exc: - assert "expiration timestamp" in str(exc) - else: - raise AssertionError("Toss external token without expiry must be blocked") - toss_environment["QTA_TOSS_ACCESS_TOKEN_EXPIRES_AT"] = "2099-01-01T00:00:00+00:00" - with patch.dict(os.environ, toss_environment, clear=True): - toss_broker = run_session.create_broker("toss") - assert toss_broker.account_seq == 1 - try: - run_session.validate_mode(plan, "kis-paper", "live") - except execution_core.BlockedError: - pass - else: - raise AssertionError("live promotion must remain blocked") - - _, account, _, risk, execution = plan_orders.fixture_inputs() - account.update( - { - "account_alias": "paper-us-v2", - "market": "US", - "currency": "USD", - "settled_cash": "5000", - "fx_to_krw": "1", - "as_of": "2026-07-27T09:00:00-04:00", - } - ) - risk.update( - { - "per_trade_risk_krw": "100", - "max_symbol_notional_krw": "1000", - "max_concurrent_positions": 3, - } - ) - execution.update( - { - "market": "US", - "timezone": "America/New_York", - "entry_window_start": "2026-07-27T09:30:00-04:00", - "entry_window_end": "2026-07-27T10:30:00-04:00", - "market_session": execution_core.market_session_from_source( - ROOT - / "skills" - / "quant-stock-polling-trader" - / "references" - / "fixtures" - / "us-market-session-2026-07-27.json" - ), - } - ) - nyse_rank_two = v2_selected_item("NYSE", 2, "ZZZ", "ZZZ", "ZZZ") - nyse_rank_one = v2_selected_item("NYSE", 1, "BRK.B", "BRK-B", "BRK-B") - nasdaq_rank_one = v2_selected_item( - "NASDAQ", - 1, - "BRK.B", - "BRK-B", - "BRK-B", - ) - screen_v2 = complete_v2_screen( - execution_core, - { - "KOSPI": [], - "KOSDAQ": [], - "NYSE": [nyse_rank_one, nyse_rank_two], - "NASDAQ": [nasdaq_rank_one], - }, - ) - exposure_v2 = { - "schema": execution_core.EXPOSURE_SCHEMA_V2, - "as_of": "2026-07-27T09:00:00-04:00", - "positions": [ - { - "broker": "toss", - "market": "US", - "exchange": "NYSE", - "symbol": "BRK-B", - "quantity": "0.5", - "market_value_krw": "70000", - } - ], - } - partial_self_hashed_screen = { - "schema": execution_core.SCREEN_SCHEMA_V2, - "screen_status": "READY", - "method_version": "qta-1.0.0", - "selector_version": execution_core.V2_SELECTOR_VERSION, - "selected": screen_v2["selected"], - } - rehash_screen(execution_core, partial_self_hashed_screen) - assert_blocked( - lambda: execution_core.plan_orders( - partial_self_hashed_screen, - account, - exposure_v2, - risk, - execution, - ), - "qta-screen/v2 fields mismatch", - ) - minimal_qta_screen = deepcopy(screen_v2) - minimal_qta = { - "source_skill": "quant-stock-technical", - "result_schema": "quant-stock-technical/v1", - "calculation_status": "READY", - "setup_status": "READY", - "method_version": "qta-1.0.0", - "market": "US", - "ticker": "BRK.B", - "entry_price": 100.0, - "stop_price": 90.0, - "take_profit_price": 120.0, - } - minimal_qta_screen["selected"]["NASDAQ"][0]["qta"] = deepcopy(minimal_qta) - for decision in minimal_qta_screen["decisions"]: - if decision["exchange"] == "NASDAQ": - decision["qta"] = deepcopy(minimal_qta) - rehash_screen(execution_core, minimal_qta_screen) - assert_blocked( - lambda: execution_core.plan_orders( - minimal_qta_screen, - account, - exposure_v2, - risk, - execution, - ), - "qta fields mismatch", - ) - mismatched_qta_date = deepcopy(screen_v2) - mismatched_qta_date["selected"]["NASDAQ"][0]["qta"]["analysis_date"] = "2026-07-23" - for decision in mismatched_qta_date["decisions"]: - if decision["exchange"] == "NASDAQ": - decision["qta"]["analysis_date"] = "2026-07-23" - rehash_screen(execution_core, mismatched_qta_date) - assert_blocked( - lambda: execution_core.plan_orders( - mismatched_qta_date, - account, - exposure_v2, - risk, - execution, - ), - "analysis_date must equal screen.analysis_date", - ) - hacked_status_selector = deepcopy(screen_v2) - hacked_status_selector["selector"]["eligible_setup_statuses"] = ["HACKED"] - hacked_status_selector["selector_hash"] = execution_core.sha256_json( - hacked_status_selector["selector"] - ) - rehash_screen(execution_core, hacked_status_selector) - assert_blocked( - lambda: execution_core.plan_orders( - hacked_status_selector, - account, - exposure_v2, - risk, - execution, - ), - "subset of READY and CONDITIONAL", - ) - wrong_nested_score = deepcopy(screen_v2) - wrong_nested_score["selected"]["NASDAQ"][0]["qta"]["risk"]["score"] = 101.0 - for decision in wrong_nested_score["decisions"]: - if decision["exchange"] == "NASDAQ": - decision["qta"]["risk"]["score"] = 101.0 - rehash_screen(execution_core, wrong_nested_score) - assert_blocked( - lambda: execution_core.plan_orders( - wrong_nested_score, - account, - exposure_v2, - risk, - execution, - ), - "risk.score must be >= 0 and <= 100", - ) - toss_v2_account = deepcopy(account) - toss_v2_account["broker"] = "toss" - assert_blocked( - lambda: execution_core.plan_orders( - screen_v2, - toss_v2_account, - exposure_v2, - risk, - execution, - ), - "requires account broker kis", - ) - v2_plan = execution_core.plan_orders( - screen_v2, - account, - exposure_v2, - risk, - execution, - ) - blocked_decision_screen = deepcopy(screen_v2) - blocked_decision_screen["selector"]["max_blocked_fraction"] = "1" - blocked_decision_screen["selector_hash"] = execution_core.sha256_json( - blocked_decision_screen["selector"] - ) - blocked_decision_screen["selected"]["NASDAQ"] = [] - for decision in blocked_decision_screen["decisions"]: - if decision["exchange"] == "NASDAQ": - decision["qta"] = { - "source_skill": "quant-stock-technical", - "result_schema": "quant-stock-technical/v1", - "calculation_status": "BLOCKED", - "reason": "fixture data unavailable", - "method_version": "qta-1.0.0", - "market": "US", - "ticker": "BRK.B", - } - decision["eligible"] = False - decision["reasons"] = ["calculation_not_ready"] - decision["exchange_rank"] = None - decision["selected"] = False - blocked_decision_screen["blocked_count"] = 1 - blocked_decision_screen["blocked_fraction"] = "0.3333333333333333333333333333" - rehash_screen(execution_core, blocked_decision_screen) - blocked_decision_plan = execution_core.plan_orders( - blocked_decision_screen, - account, - exposure_v2, - risk, - execution, - ) - assert [item["symbol"] for item in blocked_decision_plan["intents"]] == ["ZZZ"] - malformed_blocked_decision = deepcopy(blocked_decision_screen) - for decision in malformed_blocked_decision["decisions"]: - if decision["exchange"] == "NASDAQ": - decision["qta"]["analysis_date"] = "2026-07-24" - rehash_screen(execution_core, malformed_blocked_decision) - assert_blocked( - lambda: execution_core.plan_orders( - malformed_blocked_decision, - account, - exposure_v2, - risk, - execution, - ), - "qta fields mismatch", - ) - conditional_screen_v2 = deepcopy(screen_v2) - conditional_screen_v2["selector"]["eligible_setup_statuses"] = [ - "CONDITIONAL", - "READY", - ] - conditional_screen_v2["selected"]["NASDAQ"][0]["qta"]["setup_status"] = ( - "CONDITIONAL" - ) - conditional_screen_v2["selected"]["NASDAQ"][0]["qta"]["total_score"] = 50.0 - for horizon in ("short", "medium", "long"): - conditional_screen_v2["selected"]["NASDAQ"][0]["qta"][horizon] = { - "opinion": "중립", - "score": 50.0, - } - for decision in conditional_screen_v2["decisions"]: - if decision["exchange"] == "NASDAQ": - decision["qta"]["setup_status"] = "CONDITIONAL" - decision["qta"]["total_score"] = 50.0 - for horizon in ("short", "medium", "long"): - decision["qta"][horizon] = { - "opinion": "중립", - "score": 50.0, - } - conditional_screen_v2["selector_hash"] = execution_core.sha256_json( - conditional_screen_v2["selector"] - ) - rehash_screen(execution_core, conditional_screen_v2) - execution_core.plan_orders( - conditional_screen_v2, - account, - exposure_v2, - risk, - execution, - ) - rounded_boundary_screen = deepcopy(screen_v2) - rounded_boundary_screen["selector"]["eligible_setup_statuses"] = [ - "CONDITIONAL", - "READY", - ] - rounded_boundary_payloads = [ - rounded_boundary_screen["selected"]["NASDAQ"][0]["qta"], - *[ - decision["qta"] - for decision in rounded_boundary_screen["decisions"] - if decision["exchange"] == "NASDAQ" - ], - ] - for payload in rounded_boundary_payloads: - payload["setup_status"] = "CONDITIONAL" - payload["total_score"] = 60.0 - for horizon in ("short", "medium", "long"): - payload[horizon] = {"opinion": "중립", "score": 60.0} - rounded_boundary_screen["selector_hash"] = execution_core.sha256_json( - rounded_boundary_screen["selector"] - ) - rehash_screen(execution_core, rounded_boundary_screen) - execution_core.plan_orders( - rounded_boundary_screen, - account, - exposure_v2, - risk, - execution, - ) - assert_blocked( - lambda: run_session.validate_mode(v2_plan, "toss", "shadow"), - "plan broker does not match selected adapter", - ) - reversed_screen_v2 = deepcopy(screen_v2) - reversed_screen_v2["selected"]["NYSE"].reverse() - assert_blocked( - lambda: execution_core.plan_orders( - reversed_screen_v2, - account, - exposure_v2, - risk, - execution, - ), - "screen_hash does not match canonical screen payload", - ) - rehash_screen(execution_core, reversed_screen_v2) - assert_blocked( - lambda: execution_core.plan_orders( - reversed_screen_v2, - account, - exposure_v2, - risk, - execution, - ), - "ordered and contiguous", - ) - wrong_rank_screen = deepcopy(screen_v2) - wrong_rank_screen["selected"]["NYSE"].reverse() - for exchange_rank, item in enumerate( - wrong_rank_screen["selected"]["NYSE"], - start=1, - ): - item["exchange_rank"] = exchange_rank - for decision in wrong_rank_screen["decisions"]: - if ( - decision["exchange"] == "NYSE" - and decision["canonical_symbol"] - == item["instrument"]["canonical_symbol"] - ): - decision["exchange_rank"] = exchange_rank - rehash_screen(execution_core, wrong_rank_screen) - assert_blocked( - lambda: execution_core.plan_orders( - wrong_rank_screen, - account, - exposure_v2, - risk, - execution, - ), - "exchange_rank does not match qta ranking_key", - ) - assert v2_plan["context"]["exchange_order"] == ["NYSE", "NASDAQ"] - assert v2_plan["context"]["candidate_order_contract"].startswith( - "exchange_contract_order" - ) - assert [ - (item["exchange"], item["exchange_rank"], item["rank"], item["symbol"]) - for item in v2_plan["intents"] - ] == [ - ("NYSE", 2, 2, "ZZZ"), - ("NASDAQ", 1, 3, "BRK-B"), - ] - assert v2_plan["intents"][1]["canonical_symbol"] == "BRK.B" - assert v2_plan["intents"][1]["data_symbol"] == "BRK-B" - assert v2_plan["intents"][1]["broker_symbol"] == "BRK-B" - assert v2_plan["intents"][1]["venue"] == "NASD" - assert v2_plan["skipped"] == [ - { - "market": "US", - "symbol": "BRK-B", - "reason": "existing_exposure", - "exchange": "NYSE", - "exchange_rank": 1, - "canonical_symbol": "BRK.B", - } - ] - assert v2_plan["context"]["screen_hash"] == screen_v2["screen_hash"] - assert ( - v2_plan["context"]["market_session_hash"] - == execution["market_session"]["session_hash"] - ) - rehashed_session_tamper = deepcopy(v2_plan) - rehashed_session_tamper["context"]["market_session_hash"] = "0" * 64 - rehashed_session_tamper.pop("plan_hash") - rehashed_session_tamper["plan_hash"] = execution_core.sha256_json( - rehashed_session_tamper - ) - assert_blocked( - lambda: run_session.validate_plan(rehashed_session_tamper), - "market_session_hash does not match market_session", - ) - - tampered_qta = deepcopy(screen_v2) - tampered_qta["selected"]["NASDAQ"][0]["qta"]["entry_price"] = 101 - assert_blocked( - lambda: execution_core.plan_orders( - tampered_qta, - account, - exposure_v2, - risk, - execution, - ), - "screen_hash does not match canonical screen payload", - ) - forged_total = deepcopy(screen_v2) - forged_total_payloads = [ - forged_total["selected"]["NASDAQ"][0]["qta"], - *[ - decision["qta"] - for decision in forged_total["decisions"] - if decision["exchange"] == "NASDAQ" - ], - ] - for payload in forged_total_payloads: - payload["total_score"] = 99.0 - rehash_screen(execution_core, forged_total) - assert_blocked( - lambda: execution_core.plan_orders( - forged_total, - account, - exposure_v2, - risk, - execution, - ), - "total_score does not match the qta-1.0.0 score formula", - ) - forged_opinion = deepcopy(screen_v2) - forged_opinion_payloads = [ - forged_opinion["selected"]["NASDAQ"][0]["qta"], - *[ - decision["qta"] - for decision in forged_opinion["decisions"] - if decision["exchange"] == "NASDAQ" - ], - ] - for payload in forged_opinion_payloads: - payload["short"]["opinion"] = "강한 부정" - rehash_screen(execution_core, forged_opinion) - assert_blocked( - lambda: execution_core.plan_orders( - forged_opinion, - account, - exposure_v2, - risk, - execution, - ), - "opinion does not match its score", - ) - off_tick_target = deepcopy(screen_v2) - off_tick_payloads = [ - off_tick_target["selected"]["NASDAQ"][0]["qta"], - *[ - decision["qta"] - for decision in off_tick_target["decisions"] - if decision["exchange"] == "NASDAQ" - ], - ] - for payload in off_tick_payloads: - payload["take_profit_price"] = 120.005 - rehash_screen(execution_core, off_tick_target) - assert_blocked( - lambda: execution_core.plan_orders( - off_tick_target, - account, - exposure_v2, - risk, - execution, - ), - "must align to the resolved tick", - ) - forged_target = deepcopy(screen_v2) - forged_target_payloads = [ - forged_target["selected"]["NASDAQ"][0]["qta"], - *[ - decision["qta"] - for decision in forged_target["decisions"] - if decision["exchange"] == "NASDAQ" - ], - ] - for payload in forged_target_payloads: - payload["take_profit_price"] = 121.0 - rehash_screen(execution_core, forged_target) - assert_blocked( - lambda: execution_core.plan_orders( - forged_target, - account, - exposure_v2, - risk, - execution, - ), - "must equal the qta-1.0.0 2R target", - ) - tampered_metadata = deepcopy(screen_v2) - tampered_metadata["selected"]["NASDAQ"][0]["instrument"]["source_name"] = ( - "tampered-but-structurally-valid" - ) - assert_blocked( - lambda: execution_core.plan_orders( - tampered_metadata, - account, - exposure_v2, - risk, - execution, - ), - "screen_hash does not match canonical screen payload", - ) - uppercase_hash = deepcopy(screen_v2) - uppercase_hash["screen_hash"] = str(uppercase_hash["screen_hash"]).upper() - assert_blocked( - lambda: execution_core.plan_orders( - uppercase_hash, - account, - exposure_v2, - risk, - execution, - ), - "lowercase SHA-256", - ) - wrong_us_offset = deepcopy(execution) - wrong_us_offset.update( - { - "entry_window_start": "2026-07-27T09:30:00-05:00", - "entry_window_end": "2026-07-27T10:30:00-05:00", - } - ) - assert_blocked( - lambda: execution_core.normalized_execution_policy(wrong_us_offset), - "local clock and UTC offset must match America/New_York", - ) - wrong_us_window = deepcopy(execution) - wrong_us_window.update( - { - "entry_window_start": "2026-07-27T10:00:00-04:00", - "entry_window_end": "2026-07-27T11:00:00-04:00", - } - ) - assert_blocked( - lambda: execution_core.normalized_execution_policy(wrong_us_window), - "US entry window must be 09:30-10:30 America/New_York", - ) - winter_us_window = deepcopy(execution) - winter_us_window.update( - { - "entry_window_start": "2026-12-07T09:30:00-05:00", - "entry_window_end": "2026-12-07T10:30:00-05:00", - "market_session": execution_core.market_session_from_source( - ROOT - / "skills" - / "quant-stock-polling-trader" - / "references" - / "fixtures" - / "us-market-session-2026-12-07.json" - ), - } - ) - normalized_winter = execution_core.normalized_execution_policy(winter_us_window) - assert normalized_winter["entry_window_start"].endswith("-05:00") - sunday_us_window = deepcopy(execution) - sunday_us_window.update( - { - "entry_window_start": "2026-07-26T09:30:00-04:00", - "entry_window_end": "2026-07-26T10:30:00-04:00", - "market_session": execution_core.market_session_from_source( - ROOT - / "skills" - / "quant-stock-polling-trader" - / "references" - / "fixtures" - / "us-market-session-2026-07-26-invalid.json" - ), - } - ) - assert_blocked( - lambda: execution_core.normalized_execution_policy(sunday_us_window), - "session_date must be a weekday", - ) - oversized_snapshot_age = deepcopy(execution) - oversized_snapshot_age["snapshot_max_age_seconds"] = 3601 - assert_blocked( - lambda: execution_core.normalized_execution_policy(oversized_snapshot_age), - "snapshot_max_age_seconds must be <= 3600", - ) - tampered_market_session = deepcopy(execution) - tampered_market_session["market_session"]["source_id"] = "forged-source" - tampered_market_session["market_session"]["session_hash"] = ( - execution_core.canonical_market_session_hash( - tampered_market_session["market_session"] - ) - ) - assert_blocked( - lambda: execution_core.normalized_execution_policy(tampered_market_session), - "do not match the hashed source snapshot", - ) - _, _, _, _, wrong_kr_offset = plan_orders.fixture_inputs() - wrong_kr_offset.update( - { - "entry_window_start": "2026-07-27T09:00:00+08:00", - "entry_window_end": "2026-07-27T10:00:00+08:00", - } - ) - assert_blocked( - lambda: execution_core.normalized_execution_policy(wrong_kr_offset), - "local clock and UTC offset must match Asia/Seoul", - ) - - mismatched_snapshot = deepcopy(exposure_v2) - mismatched_snapshot["as_of"] = "2026-07-27T09:00:01-04:00" - assert_blocked( - lambda: execution_core.plan_orders( - screen_v2, - account, - mismatched_snapshot, - risk, - execution, - ), - "must share one frozen as_of instant", - ) - stale_snapshot_account = deepcopy(account) - stale_snapshot = deepcopy(exposure_v2) - stale_snapshot_account["as_of"] = "2026-07-26T09:00:00-04:00" - stale_snapshot["as_of"] = stale_snapshot_account["as_of"] - assert_blocked( - lambda: execution_core.plan_orders( - screen_v2, - stale_snapshot_account, - stale_snapshot, - risk, - execution, - ), - "must be from the entry session local date", - ) - late_snapshot_account = deepcopy(account) - late_snapshot = deepcopy(exposure_v2) - late_snapshot_account["as_of"] = "2026-07-27T09:31:00-04:00" - late_snapshot["as_of"] = late_snapshot_account["as_of"] - assert_blocked( - lambda: execution_core.plan_orders( - screen_v2, - late_snapshot_account, - late_snapshot, - risk, - execution, - ), - "must be frozen no later than entry_window_start", - ) - too_old_snapshot_account = deepcopy(account) - too_old_snapshot = deepcopy(exposure_v2) - too_old_snapshot_account["as_of"] = "2026-07-27T08:00:00-04:00" - too_old_snapshot["as_of"] = too_old_snapshot_account["as_of"] - assert_blocked( - lambda: execution_core.plan_orders( - screen_v2, - too_old_snapshot_account, - too_old_snapshot, - risk, - execution, - ), - "exceed snapshot_max_age_seconds", - ) - same_day_analysis = deepcopy(screen_v2) - same_day_analysis["analysis_date"] = "2026-07-27" - for exchange_items in same_day_analysis["selected"].values(): - for selected_item in exchange_items: - selected_item["qta"]["analysis_date"] = "2026-07-27" - for decision in same_day_analysis["decisions"]: - if decision["qta"]["calculation_status"] == "READY": - decision["qta"]["analysis_date"] = "2026-07-27" - rehash_screen(execution_core, same_day_analysis) - assert_blocked( - lambda: execution_core.plan_orders( - same_day_analysis, - account, - exposure_v2, - risk, - execution, - ), - "analysis_date must equal market_session.previous_session_date", - ) - changed_selector = deepcopy(screen_v2) - changed_selector["selector"]["min_total_score"] = "1" - rehash_screen(execution_core, changed_selector) - assert_blocked( - lambda: execution_core.plan_orders( - changed_selector, - account, - exposure_v2, - risk, - execution, - ), - "selector_hash does not match screen.selector", - ) - - account_with_open_buy = deepcopy(account) - account_with_open_buy["open_orders"] = [ - { - "side": "buy", - "market": "us", - "exchange": "nasdaq", - "symbol": "brk-b", - } - ] - open_buy_plan = execution_core.plan_orders( - screen_v2, - account_with_open_buy, - exposure_v2, - risk, - execution, - ) - assert [item["symbol"] for item in open_buy_plan["intents"]] == ["ZZZ"] - assert { - (item["exchange"], item["symbol"], item["reason"]) - for item in open_buy_plan["skipped"] - } == { - ("NYSE", "BRK-B", "existing_exposure"), - ("NASDAQ", "BRK-B", "existing_open_buy_order"), - } - - occupied_risk = deepcopy(risk) - occupied_risk["max_concurrent_positions"] = 2 - occupied_plan = execution_core.plan_orders( - screen_v2, - account_with_open_buy, - exposure_v2, - occupied_risk, - execution, - ) - assert occupied_plan["intents"] == [] - assert {item["reason"] for item in occupied_plan["skipped"]} == { - "existing_exposure", - "existing_open_buy_order", - "max_concurrent_positions", - } - - duplicate_account_position = deepcopy(account) - duplicate_account_position["positions"] = [ - { - "market": "us", - "exchange": "nyse", - "symbol": "brk-b", - } - ] - unique_slot_plan = execution_core.plan_orders( - screen_v2, - duplicate_account_position, - exposure_v2, - occupied_risk, - execution, - ) - assert [item["symbol"] for item in unique_slot_plan["intents"]] == ["ZZZ"] - assert unique_slot_plan["skipped"][-1]["reason"] == "max_concurrent_positions" - - missing_open_order_market = deepcopy(account_with_open_buy) - del missing_open_order_market["open_orders"][0]["market"] - assert_blocked( - lambda: execution_core.plan_orders( - screen_v2, - missing_open_order_market, - exposure_v2, - risk, - execution, - ), - "missing fields needed for exposure key", - ) - missing_open_order_exchange = deepcopy(account_with_open_buy) - del missing_open_order_exchange["open_orders"][0]["exchange"] - assert_blocked( - lambda: execution_core.plan_orders( - screen_v2, - missing_open_order_exchange, - exposure_v2, - risk, - execution, - ), - "requires exchange for qta-screen/v2", - ) - - _, kr_account, _, kr_risk, kr_execution = plan_orders.fixture_inputs() - kr_instrument = v2_instrument( - "KOSDAQ", - "0156T0", - "0156T0.KQ", - "0156T0", - ) - kr_selector = screen_universe.normalize_selector_v2( - { - "selector_version": "qta-screen-1.1.0", - "min_total_score": "0", - "eligible_setup_statuses": ["READY"], - "top_k_by_exchange": { - "KOSPI": 0, - "KOSDAQ": 1, - "NYSE": 0, - "NASDAQ": 0, - }, - "min_selected_by_exchange": { - "KOSPI": 0, - "KOSDAQ": 1, - "NYSE": 0, - "NASDAQ": 0, - }, - "max_blocked_fraction": "0", - } - ) - kr_qta = v2_qta("KR", "0156T0") - kr_screen_v2 = screen_universe.finalize_screen_v2( - { - "analysis_date": "2026-07-24", - "manifest_hash": "d" * 64, - "instruments": [kr_instrument], - }, - kr_selector, - [kr_qta], - ) - kr_exposure_v2 = { - "schema": execution_core.EXPOSURE_SCHEMA_V2, - "as_of": "2026-07-27T08:50:00+09:00", - "positions": [], - } - kr_v2_plan = execution_core.plan_orders( - kr_screen_v2, - kr_account, - kr_exposure_v2, - kr_risk, - kr_execution, - ) - assert kr_v2_plan["intents"][0]["symbol"] == "0156T0" - assert kr_v2_plan["intents"][0]["exchange"] == "KOSDAQ" - assert kr_v2_plan["intents"][0]["venue"] == "KRX" - - legacy_exposure = { - "schema": execution_core.EXPOSURE_SCHEMA, - "as_of": exposure_v2["as_of"], - "positions": [], - } - assert_blocked( - lambda: execution_core.plan_orders( - screen_v2, - account, - legacy_exposure, - risk, - execution, - ), - "requires exposure schema qta-exposure-snapshot/v2", - ) - invalid_keys = deepcopy(screen_v2) - del invalid_keys["selected"]["KOSDAQ"] - rehash_screen(execution_core, invalid_keys) - assert_blocked( - lambda: execution_core.plan_orders( - invalid_keys, - account, - exposure_v2, - risk, - execution, - ), - "selected must contain exactly", - ) - invalid_venue = deepcopy(screen_v2) - invalid_venue["selected"]["NASDAQ"][0]["instrument"]["venue"] = "NASDAQ" - rehash_screen(execution_core, invalid_venue) - assert_blocked( - lambda: execution_core.plan_orders( - invalid_venue, - account, - exposure_v2, - risk, - execution, - ), - ".venue must be NASD", - ) - invalid_metadata = deepcopy(screen_v2) - invalid_metadata["selected"]["NASDAQ"][0]["instrument"]["unexpected"] = True - rehash_screen(execution_core, invalid_metadata) - assert_blocked( - lambda: execution_core.plan_orders( - invalid_metadata, - account, - exposure_v2, - risk, - execution, - ), - "fields mismatch", - ) - invalid_qta_symbol = deepcopy(screen_v2) - invalid_qta_symbol["selected"]["NASDAQ"][0]["qta"]["ticker"] = "OTHER" - rehash_screen(execution_core, invalid_qta_symbol) - assert_blocked( - lambda: execution_core.plan_orders( - invalid_qta_symbol, - account, - exposure_v2, - risk, - execution, - ), - "qta.ticker must equal instrument.canonical_symbol", - ) - print("quant screening and execution regression: PASS") - - -if __name__ == "__main__": - main() diff --git a/tests/test_screen_universe.py b/tests/test_screen_universe.py deleted file mode 100644 index a31b4b7..0000000 --- a/tests/test_screen_universe.py +++ /dev/null @@ -1,106 +0,0 @@ -"""Focused contract tests for deterministic universe screening.""" - -from __future__ import annotations - -import importlib.util -import sys -import unittest -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -SCRIPTS = ROOT / "skills" / "quant-stock-technical" / "scripts" -MODULE_PATH = SCRIPTS / "screen_universe.py" -sys.path.insert(0, str(SCRIPTS)) -SPEC = importlib.util.spec_from_file_location("screen_universe_contract", MODULE_PATH) -if SPEC is None or SPEC.loader is None: - raise RuntimeError(f"cannot load {MODULE_PATH}") -MODULE = importlib.util.module_from_spec(SPEC) -sys.modules[SPEC.name] = MODULE -SPEC.loader.exec_module(MODULE) - - -class ScreenUniverseContractTests(unittest.TestCase): - def test_catalog_coverage_contract_requires_both_ratio_maps(self) -> None: - self.assertTrue(MODULE.meets_minimum_coverage_ratio(1, 2, "0.5")) - self.assertFalse(MODULE.meets_minimum_coverage_ratio(1, 2, "0.5001")) - self.assertFalse(MODULE.meets_minimum_coverage_ratio(0, 0, "0.1")) - contract = MODULE.normalize_catalog_coverage_contract( - { - "schema": MODULE.CATALOG_COVERAGE_SCHEMA, - "minimum_ratio_by_exchange": { - exchange: "1.00" for exchange in MODULE.EXCHANGE_ORDER - }, - "minimum_screenable_ratio_by_exchange": { - exchange: "0.500" for exchange in MODULE.EXCHANGE_ORDER - }, - } - ) - self.assertEqual( - contract["minimum_ratio_by_exchange"], - {exchange: "1" for exchange in MODULE.EXCHANGE_ORDER}, - ) - self.assertEqual( - contract["minimum_screenable_ratio_by_exchange"], - {exchange: "0.5" for exchange in MODULE.EXCHANGE_ORDER}, - ) - with self.assertRaisesRegex( - MODULE.ScreenBlockedError, - "fields must be exactly", - ): - MODULE.normalize_catalog_coverage_contract( - { - "schema": MODULE.CATALOG_COVERAGE_SCHEMA, - "minimum_ratio_by_exchange": { - exchange: "1" for exchange in MODULE.EXCHANGE_ORDER - }, - } - ) - - def test_v1_selector_allows_only_known_setup_statuses(self) -> None: - selector = { - "selector_version": MODULE.SELECTOR_VERSION, - "min_total_score": "60", - "eligible_setup_statuses": ["READY", "CONDITIONAL", "READY"], - "top_k_by_market": {"KR": 1, "US": 1}, - "max_blocked_fraction": "0", - } - normalized = MODULE.normalize_selector_v1(selector) - self.assertEqual( - normalized["eligible_setup_statuses"], - ["CONDITIONAL", "READY"], - ) - with self.assertRaisesRegex( - MODULE.ScreenBlockedError, - "subset of READY and CONDITIONAL", - ): - MODULE.normalize_selector_v1( - {**selector, "eligible_setup_statuses": ["READY", "BLOCKED"]} - ) - - def test_v2_selector_allows_only_known_setup_statuses(self) -> None: - selector = { - "selector_version": MODULE.SELECTOR_VERSION_V2, - "min_total_score": "60", - "eligible_setup_statuses": ["CONDITIONAL"], - "top_k_by_exchange": {exchange: 1 for exchange in MODULE.EXCHANGE_ORDER}, - "min_selected_by_exchange": { - exchange: 0 for exchange in MODULE.EXCHANGE_ORDER - }, - "max_blocked_fraction": "0", - } - normalized = MODULE.normalize_selector_v2(selector) - self.assertEqual( - normalized["eligible_setup_statuses"], - ["CONDITIONAL"], - ) - with self.assertRaisesRegex( - MODULE.ScreenBlockedError, - "subset of READY and CONDITIONAL", - ): - MODULE.normalize_selector_v2( - {**selector, "eligible_setup_statuses": ["conditional"]} - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_systemd_units.py b/tests/test_systemd_units.py deleted file mode 100644 index 4818175..0000000 --- a/tests/test_systemd_units.py +++ /dev/null @@ -1,438 +0,0 @@ -#!/usr/bin/env python3 -"""Offline contract tests for deterministic QTA systemd generation.""" - -from __future__ import annotations - -import importlib.util -import json -import os -import shutil -import subprocess -import sys -import tempfile -import unittest -from unittest import mock -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -MODULE_PATH = ( - ROOT / "skills" / "quant-stock-polling-trader" / "scripts" / "systemd_units.py" -) -SPEC = importlib.util.spec_from_file_location("qta_systemd_contract", MODULE_PATH) -if SPEC is None or SPEC.loader is None: - raise RuntimeError("cannot load systemd_units.py") -MODULE = importlib.util.module_from_spec(SPEC) -sys.modules[SPEC.name] = MODULE -SPEC.loader.exec_module(MODULE) - - -class SystemdUnitTests(unittest.TestCase): - def test_embedded_generator_contract(self) -> None: - MODULE.self_test() - - def test_entry_timers_are_not_persistent(self) -> None: - self.assertEqual( - MODULE.SCHEDULES[("entry", "KR")], - "Mon..Fri *-*-* 08:59:00 Asia/Seoul", - ) - self.assertEqual( - MODULE.SCHEDULES[("entry", "US")], - "Mon..Fri *-*-* 09:29:00 America/New_York", - ) - self.assertEqual( - MODULE.SCHEDULES[("snapshot", "KR")], - "Mon..Fri *-*-* 08:50:00 Asia/Seoul", - ) - self.assertEqual( - MODULE.SCHEDULES[("snapshot", "US")], - "Mon..Fri *-*-* 09:20:00 America/New_York", - ) - self.assertEqual( - MODULE.SCHEDULES[("daily-prepare", "KR")], - "Mon..Fri *-*-* 01:00:00 Asia/Seoul", - ) - self.assertEqual( - MODULE.SCHEDULES[("daily-prepare", "US")], - "Mon..Fri *-*-* 01:00:00 America/New_York", - ) - self.assertEqual( - MODULE.SCHEDULES[("daily-entry", "US")], - "Mon..Fri *-*-* 09:29:00 America/New_York", - ) - - def test_systemd_escaping_disables_specifiers(self) -> None: - self.assertEqual(MODULE.systemd_quote("/tmp/100%"), '"/tmp/100%%"') - - def test_directive_path_escaping_is_unquoted(self) -> None: - self.assertEqual( - MODULE.systemd_path("/tmp/qta 100%"), - r"/tmp/qta\x20100%%", - ) - - def test_ntfy_is_optional_and_secrets_are_not_reported(self) -> None: - job = {"market": "US", "kind": "daily-entry"} - self.assertEqual( - MODULE.publish_ntfy( - environment={}, - job=job, - outcome="READY", - duration_seconds=60, - details=[], - ), - {"channel": "ntfy", "status": "DISABLED"}, - ) - - response = mock.MagicMock() - response.__enter__.return_value.status = 200 - with mock.patch.object(MODULE.urllib.request, "urlopen", return_value=response): - receipt = MODULE.publish_ntfy( - environment={ - "QTA_NTFY_TOPIC_URL": "https://ntfy.example/qta-secret-topic", - "QTA_NTFY_TOKEN": "secret-token", - }, - job=job, - outcome="READY", - duration_seconds=60, - details=["session=2026-07-29", "filled=0"], - ) - self.assertEqual(receipt, {"channel": "ntfy", "status": "SENT"}) - self.assertNotIn("secret", json.dumps(receipt)) - - invalid = MODULE.publish_ntfy( - environment={ - "QTA_NTFY_TOPIC_URL": "https://user:password@ntfy.example/topic" - }, - job=job, - outcome="BLOCKED", - duration_seconds=0, - details=[], - ) - self.assertEqual(invalid["reason"], "INVALID_TOPIC_URL") - - def test_generated_bundle_round_trips_through_execute_reader(self) -> None: - with tempfile.TemporaryDirectory(prefix="qta-systemd-roundtrip-") as temporary: - root = Path(temporary) - technical = root / "technical" - trader = root / "trader" - runtime = root / "runtime" - output = root / "generated" - for path in (technical / "scripts", trader / "scripts", runtime): - path.mkdir(parents=True) - for path in ( - technical / "scripts" / "fetch_kis_kr_eod.py", - technical / "scripts" / "fetch_kis_us_eod.py", - trader / "scripts" / "account_snapshot.py", - trader / "scripts" / "daily_pipeline.py", - trader / "scripts" / "market_calendar.py", - trader / "scripts" / "run_session.py", - trader / "scripts" / "systemd_units.py", - runtime / "us-eod.json", - ): - path.write_text("{}\n", encoding="utf-8") - marker = runtime / "worker-result.json" - (technical / "scripts" / "fetch_kis_us_eod.py").write_text( - "import json, os, sys\n" - "from pathlib import Path\n" - f"Path({str(marker)!r}).write_text(json.dumps({{" - "'argv': sys.argv[1:], " - "'pythonpath': os.environ.get('PYTHONPATH'), " - "'pythonhome': os.environ.get('PYTHONHOME')" - "}), encoding='utf-8')\n", - encoding="utf-8", - ) - environment = root / "secrets.env" - environment.write_text("QTA_TEST=1\n", encoding="utf-8") - environment.chmod(0o600) - raw = { - "schema": MODULE.BUNDLE_SCHEMA, - "unit_prefix": "qta", - "python_executable": str(Path(sys.executable).resolve()), - "technical_skill_root": str(technical), - "trader_skill_root": str(trader), - "environment_file": str(environment), - "runtime_directory": str(runtime), - "jobs": [ - { - "name": "eod-us", - "kind": "eod", - "market": "US", - "input_path": str(runtime / "us-eod.json"), - "plan_path": "", - "state_directory": "", - "output_path": "", - "broker": "", - "mode": "", - "venue_map": "", - "max_cycles": 0, - "timeout_start_seconds": 7200, - } - ], - } - normalized = MODULE.normalize_bundle(raw) - MODULE.generate(normalized, output) - - serialized = json.loads( - (output / "systemd-bundle.json").read_text(encoding="utf-8") - ) - self.assertEqual(set(serialized["jobs"][0]), MODULE.JOB_FIELDS) - self.assertNotIn("schedule", serialized["jobs"][0]) - self.assertNotIn("persistent", serialized["jobs"][0]) - self.assertEqual( - MODULE.read_bundle(output / "systemd-bundle.json"), - normalized, - ) - service = (output / "qta-eod-us.service").read_text(encoding="utf-8") - self.assertIn( - "UnsetEnvironment=PYTHONPATH PYTHONHOME\n", - service, - ) - environment_for_execute = dict(os.environ) - environment_for_execute["PYTHONPATH"] = "/tmp/qta-test-pythonpath" - environment_for_execute["PYTHONHOME"] = sys.prefix - result = subprocess.run( - [ - sys.executable, - "-B", - "-s", - str(MODULE_PATH), - "execute", - "--bundle", - str(output / "systemd-bundle.json"), - "--name", - "eod-us", - ], - check=False, - capture_output=True, - text=True, - env=environment_for_execute, - ) - self.assertEqual(result.returncode, 0, result.stderr) - worker_result = json.loads(marker.read_text(encoding="utf-8")) - self.assertEqual( - worker_result["argv"], - ["collect", "--job", str(runtime / "us-eod.json")], - ) - self.assertIsNone(worker_result["pythonpath"]) - self.assertIsNone(worker_result["pythonhome"]) - - def test_nonempty_output_is_rejected(self) -> None: - with tempfile.TemporaryDirectory(prefix="qta-systemd-stale-") as temporary: - output = Path(temporary) - (output / "stale.timer").write_text("stale\n", encoding="utf-8") - with self.assertRaisesRegex( - MODULE.UnitBlockedError, - "must be empty", - ): - MODULE.generate({}, output) - - def test_daily_pipeline_bundle_uses_dynamic_config_and_dependencies(self) -> None: - with tempfile.TemporaryDirectory(prefix="qta-systemd-daily-") as temporary: - root = Path(temporary) - technical = root / "technical" - trader = root / "trader" - runtime = root / "runtime" - output = root / "generated" - for path in (technical / "scripts", trader / "scripts", runtime): - path.mkdir(parents=True) - for path in ( - technical / "scripts" / "fetch_kis_kr_eod.py", - technical / "scripts" / "fetch_kis_us_eod.py", - trader / "scripts" / "account_snapshot.py", - trader / "scripts" / "daily_pipeline.py", - trader / "scripts" / "market_calendar.py", - trader / "scripts" / "run_session.py", - trader / "scripts" / "systemd_units.py", - ): - path.write_text("{}\n", encoding="utf-8") - config = runtime / "daily-config.json" - config.write_text("{}\n", encoding="utf-8") - environment = root / "secrets.env" - environment.write_text("QTA_TEST=1\n", encoding="utf-8") - environment.chmod(0o600) - - def job(name: str, kind: str, max_cycles: int = 0) -> dict[str, object]: - return { - "name": name, - "kind": kind, - "market": "US", - "input_path": str(config), - "plan_path": "", - "state_directory": "", - "output_path": "", - "broker": "kis-live", - "mode": "shadow", - "venue_map": "", - "max_cycles": max_cycles, - "timeout_start_seconds": 7200, - } - - bundle = MODULE.normalize_bundle( - { - "schema": MODULE.BUNDLE_SCHEMA, - "unit_prefix": "qta", - "python_executable": str(Path(sys.executable).resolve()), - "technical_skill_root": str(technical), - "trader_skill_root": str(trader), - "environment_file": str(environment), - "runtime_directory": str(runtime), - "jobs": [ - job("daily-prepare-us", "daily-prepare"), - job("daily-snapshot-us", "daily-snapshot"), - job("daily-entry-us", "daily-entry", 3), - ], - } - ) - MODULE.generate(bundle, output) - entry = next( - item for item in bundle["jobs"] if item["kind"] == "daily-entry" - ) - command = MODULE.command_for(bundle, entry) - self.assertIn("daily_pipeline.py", command[3]) - self.assertEqual(command[4:8], ["entry", "--config", str(config), "--market"]) - self.assertEqual(command[-2:], ["--max-cycles", "3"]) - service = (output / "qta-daily-entry-us.service").read_text( - encoding="utf-8" - ) - self.assertIn( - "After=network-online.target qta-daily-snapshot-us.service", - service, - ) - prepare_timer = ( - output / "qta-daily-prepare-us.timer" - ).read_text(encoding="utf-8") - entry_timer = (output / "qta-daily-entry-us.timer").read_text( - encoding="utf-8" - ) - self.assertIn("Persistent=yes", prepare_timer) - self.assertIn("Persistent=no", entry_timer) - - def test_symlink_lock_directory_is_rejected(self) -> None: - with tempfile.TemporaryDirectory(prefix="qta-systemd-lock-") as temporary: - root = Path(temporary) - technical = root / "technical" - trader = root / "trader" - runtime = root / "runtime" - outside = root / "outside-locks" - for path in ( - technical / "scripts", - trader / "scripts", - runtime, - outside, - ): - path.mkdir(parents=True) - for path in ( - technical / "scripts" / "fetch_kis_kr_eod.py", - technical / "scripts" / "fetch_kis_us_eod.py", - trader / "scripts" / "account_snapshot.py", - trader / "scripts" / "daily_pipeline.py", - trader / "scripts" / "market_calendar.py", - trader / "scripts" / "run_session.py", - trader / "scripts" / "systemd_units.py", - runtime / "kr-eod.json", - ): - path.write_text("{}\n", encoding="utf-8") - environment = root / "secrets.env" - environment.write_text("QTA_TEST=1\n", encoding="utf-8") - environment.chmod(0o600) - (runtime / "locks").symlink_to(outside, target_is_directory=True) - bundle = MODULE.normalize_bundle( - { - "schema": MODULE.BUNDLE_SCHEMA, - "unit_prefix": "qta", - "python_executable": str(Path(sys.executable).resolve()), - "technical_skill_root": str(technical), - "trader_skill_root": str(trader), - "environment_file": str(environment), - "runtime_directory": str(runtime), - "jobs": [ - { - "name": "eod-kr", - "kind": "eod", - "market": "KR", - "input_path": str(runtime / "kr-eod.json"), - "plan_path": "", - "state_directory": "", - "output_path": "", - "broker": "", - "mode": "", - "venue_map": "", - "max_cycles": 0, - "timeout_start_seconds": 7200, - } - ], - } - ) - with self.assertRaisesRegex( - MODULE.UnitBlockedError, - "lock directory must be a non-symlink directory", - ): - MODULE.execute(bundle, "eod-kr") - - @unittest.skipUnless(shutil.which("systemd-analyze"), "systemd-analyze unavailable") - def test_generated_units_pass_systemd_analyze(self) -> None: - with tempfile.TemporaryDirectory(prefix="qta-systemd-test-") as temporary: - root = Path(temporary) / "qta 100%" - technical = root / "technical" - trader = root / "trader" - runtime = root / "runtime" - output = runtime / "generated" - for path in (technical / "scripts", trader / "scripts", runtime): - path.mkdir(parents=True) - for path in ( - technical / "scripts" / "fetch_kis_kr_eod.py", - technical / "scripts" / "fetch_kis_us_eod.py", - trader / "scripts" / "account_snapshot.py", - trader / "scripts" / "daily_pipeline.py", - trader / "scripts" / "market_calendar.py", - trader / "scripts" / "run_session.py", - trader / "scripts" / "systemd_units.py", - runtime / "kr-eod.json", - ): - path.write_text("{}\n", encoding="utf-8") - environment = root / "secrets.env" - environment.write_text("QTA_TEST=1\n", encoding="utf-8") - environment.chmod(0o600) - bundle = MODULE.normalize_bundle( - { - "schema": MODULE.BUNDLE_SCHEMA, - "unit_prefix": "qta", - "python_executable": str(Path(sys.executable).resolve()), - "technical_skill_root": str(technical), - "trader_skill_root": str(trader), - "environment_file": str(environment), - "runtime_directory": str(runtime), - "jobs": [ - { - "name": "eod-kr", - "kind": "eod", - "market": "KR", - "input_path": str(runtime / "kr-eod.json"), - "plan_path": "", - "state_directory": "", - "output_path": "", - "broker": "", - "mode": "", - "venue_map": "", - "max_cycles": 0, - "timeout_start_seconds": 7200, - } - ], - } - ) - receipt = MODULE.generate(bundle, output) - result = subprocess.run( - [ - shutil.which("systemd-analyze"), - "--user", - "verify", - *(item["path"] for item in receipt["unit_files"]), - ], - check=False, - capture_output=True, - text=True, - ) - self.assertEqual(result.returncode, 0, result.stderr) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_universe_builder.py b/tests/test_universe_builder.py deleted file mode 100755 index 7e8b36c..0000000 --- a/tests/test_universe_builder.py +++ /dev/null @@ -1,299 +0,0 @@ -#!/usr/bin/env python3 -"""Focused regression tests for the market-scoped universe manifest builder.""" - -from __future__ import annotations - -import importlib.util -import sys -import tempfile -import unittest -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -MODULE_PATH = ( - ROOT / "skills" / "quant-stock-technical" / "scripts" / "build_universe_manifest.py" -) -SPEC = importlib.util.spec_from_file_location("build_universe_manifest", MODULE_PATH) -if SPEC is None or SPEC.loader is None: - raise RuntimeError(f"cannot load {MODULE_PATH}") -MODULE = importlib.util.module_from_spec(SPEC) -sys.modules[SPEC.name] = MODULE -SPEC.loader.exec_module(MODULE) - - -class UniverseBuilderTests(unittest.TestCase): - def test_embedded_offline_contract(self) -> None: - MODULE.self_test() - - def test_no_implicit_build_spec_fields(self) -> None: - with self.assertRaisesRegex(MODULE.UniverseBlockedError, "fields mismatch"): - MODULE.normalize_build_spec( - { - "schema": MODULE.BUILD_SPEC_SCHEMA, - "as_of": "2026-07-26", - "analysis_date": "2026-07-24", - "official_sources": [], - "broker_sources": [], - "eod_catalog": {}, - "guessed_tick_size": "0.01", - }, - ROOT, - ) - - def test_tick_contract_requires_positive_decimals(self) -> None: - with self.assertRaisesRegex(MODULE.UniverseBlockedError, "must be positive"): - MODULE.decimal_string("0", "resolved_tick_size", positive=True) - self.assertEqual( - MODULE.decimal_string("0.0100", "resolved_tick_size", positive=True), - "0.0100", - ) - row = MODULE.CatalogRow( - exchange="KOSPI", - canonical_symbol="005930", - data_symbol="005930.KS", - broker_symbol="005930", - instrument_type="COMMON", - benchmark_id="KOSPI_COMPOSITE", - ticker_csv="ticker.csv", - benchmark_csv="benchmark.csv", - tick_rule_id="KRX-2026", - tick_effective_date="2026-07-25", - tick_reference_price="100", - resolved_tick_size="1", - source_name="fixture", - ) - contract, reason = MODULE.normalize_tick_contract(row, "2026-07-24") - self.assertIsNone(contract) - self.assertEqual(reason, "invalid_tick_contract") - - def test_catalog_coverage_contract_is_exact_and_canonical(self) -> None: - self.assertTrue(MODULE.meets_minimum_coverage_ratio(1, 2, "0.5")) - self.assertFalse(MODULE.meets_minimum_coverage_ratio(1, 2, "0.5001")) - self.assertFalse(MODULE.meets_minimum_coverage_ratio(0, 0, "0.1")) - contract = MODULE.normalize_catalog_coverage_contract( - { - "schema": MODULE.CATALOG_COVERAGE_SCHEMA, - "minimum_ratio_by_exchange": { - "KOSPI": "1.00", - "KOSDAQ": "0.9500", - "NYSE": 1, - "NASDAQ": "0.5", - }, - "minimum_screenable_ratio_by_exchange": { - "KOSPI": "0.500", - "KOSDAQ": "0.400", - "NYSE": "0.300", - "NASDAQ": "0.200", - }, - } - ) - self.assertEqual( - contract, - { - "schema": MODULE.CATALOG_COVERAGE_SCHEMA, - "minimum_ratio_by_exchange": { - "KOSPI": "1", - "KOSDAQ": "0.95", - "NYSE": "1", - "NASDAQ": "0.5", - }, - "minimum_screenable_ratio_by_exchange": { - "KOSPI": "0.5", - "KOSDAQ": "0.4", - "NYSE": "0.3", - "NASDAQ": "0.2", - }, - }, - ) - with self.assertRaisesRegex( - MODULE.UniverseBlockedError, - "fields mismatch", - ): - MODULE.normalize_catalog_coverage_contract( - { - "schema": MODULE.CATALOG_COVERAGE_SCHEMA, - "minimum_ratio_by_exchange": { - exchange: "1" for exchange in MODULE.EXCHANGES - }, - } - ) - with self.assertRaisesRegex( - MODULE.UniverseBlockedError, - "must contain exactly", - ): - MODULE.normalize_catalog_coverage_contract( - { - "schema": MODULE.CATALOG_COVERAGE_SCHEMA, - "minimum_ratio_by_exchange": { - "KOSPI": "1", - "KOSDAQ": "1", - "NYSE": "1", - }, - "minimum_screenable_ratio_by_exchange": { - "KOSPI": "0.5", - "KOSDAQ": "0.5", - "NYSE": "0.5", - "NASDAQ": "0.5", - }, - } - ) - for invalid in ("0", "1.01", "NaN"): - with ( - self.subTest(invalid=invalid), - self.assertRaisesRegex( - MODULE.UniverseBlockedError, - "greater than 0 and at most 1", - ), - ): - MODULE.canonical_ratio_string(invalid, "coverage") - - def test_exchange_contract_is_complete(self) -> None: - self.assertEqual( - set(MODULE.EXCHANGE_CONTRACTS), - {"KOSPI", "KOSDAQ", "NYSE", "NASDAQ"}, - ) - self.assertEqual( - MODULE.EXCHANGE_CONTRACTS["NASDAQ"]["venue"], - "NASD", - ) - self.assertEqual( - MODULE.EXCHANGE_CONTRACTS["KOSDAQ"]["benchmark_id"], - "KOSDAQ_COMPOSITE", - ) - - def test_kind_cp949_html_preserves_alphanumeric_code(self) -> None: - with tempfile.TemporaryDirectory(prefix="kind-html-test-") as directory: - path = Path(directory) / "corpList.xls" - path.write_text( - "" - "" - "" - "" - "" - "" - "" - "
회사명시장구분종목코드
에이치엘지노믹스코스닥0156T0
에이치엘지노믹스코스닥0156T0
삼성전자유가005930
", - encoding="cp949", - ) - base = { - "source_id": "kind", - "provider": "KRX", - "as_of": "2026-07-26", - "path": str(path), - "format": "KRX_KIND_HTML", - "encoding": "cp949", - "delimiter": "", - "skip_rows": 0, - "columns": {}, - "normal_status_values": [], - "role": "OFFICIAL_MASTER", - } - kosdaq_rows = MODULE.parse_master_source({**base, "exchange": "KOSDAQ"}) - kospi_rows = MODULE.parse_master_source({**base, "exchange": "KOSPI"}) - self.assertEqual([row.symbol for row in kosdaq_rows], ["0156T0"]) - self.assertEqual([row.symbol for row in kospi_rows], ["005930"]) - - path.write_text( - "" - "" - "" - "" - "
회사명시장구분종목코드
회사 A코스닥0156T0
회사 B코스닥0156T0
", - encoding="cp949", - ) - with self.assertRaisesRegex( - MODULE.UniverseBlockedError, "conflicting duplicate" - ): - MODULE.parse_master_source({**base, "exchange": "KOSDAQ"}) - - def test_kis_domestic_raw_master_flags(self) -> None: - self.assertEqual( - sum(MODULE.KIS_KOSPI_WIDTHS), - MODULE.KIS_KRX_LAYOUTS["KOSPI"]["tail_length"], - ) - self.assertEqual( - sum(MODULE.KIS_KOSDAQ_WIDTHS), - MODULE.KIS_KRX_LAYOUTS["KOSDAQ"]["tail_length"], - ) - with tempfile.TemporaryDirectory(prefix="kis-krx-test-") as directory: - path = Path(directory) / "kospi_code.mst" - path.write_text( - MODULE.synthetic_kis_krx_line( - "KOSPI", - "005935", - "삼성전자우", - etp_code="3", - spac="Y", - preferred="1", - trading_halt="Y", - liquidation="Y", - administrative="Y", - ) - + "\n", - encoding="cp949", - ) - descriptor = { - "source_id": "kis-kospi", - "provider": "KIS", - "exchange": "KOSPI", - "as_of": "2026-07-26", - "path": str(path), - "format": "KIS_KRX_MASTER", - "encoding": "cp949", - "delimiter": "", - "skip_rows": 0, - "columns": {}, - "normal_status_values": [], - "role": "BROKER_MASTER", - } - rows = MODULE.parse_kis_krx_master(descriptor) - self.assertEqual(rows[0].symbol, "005935") - reasons = MODULE.metadata_exclusion_reasons(rows[0], descriptor) - self.assertTrue( - { - "instrument_etn", - "instrument_spac", - "instrument_preferred", - "trading_halt", - "liquidation", - "administrative_issue", - }.issubset(set(reasons)) - ) - - def test_kis_overseas_headerless_24_column_master(self) -> None: - with tempfile.TemporaryDirectory(prefix="kis-us-test-") as directory: - path = Path(directory) / "nysmst.cod" - path.write_text( - MODULE.synthetic_kis_overseas_line("NYSE", "ADR1", "2", "Y") - + "\n" - + MODULE.synthetic_kis_overseas_line("NYSE", "ETF1", "3", "N", "001") - + "\n", - encoding="cp949", - ) - descriptor = { - "source_id": "kis-nyse", - "provider": "KIS", - "exchange": "NYSE", - "as_of": "2026-07-26", - "path": str(path), - "format": "KIS_OVERSEAS_MASTER", - "encoding": "cp949", - "delimiter": "\t", - "skip_rows": 0, - "columns": {}, - "normal_status_values": [], - "role": "BROKER_MASTER", - } - rows = MODULE.parse_kis_overseas_master(descriptor) - self.assertEqual( - [(row.symbol, row.instrument_type) for row in rows], - [("ADR1", "DR"), ("ETF1", "ETP")], - ) - self.assertIn( - "instrument_etf", - MODULE.metadata_exclusion_reasons(rows[1], descriptor), - ) - - -if __name__ == "__main__": - unittest.main()