From 67a3dc1ea15086a0f08fb71895e6af92d19d5210 Mon Sep 17 00:00:00 2001 From: Huseyin Cicek Date: Wed, 23 Sep 2026 20:27:26 +0300 Subject: [PATCH 01/19] feat: add by HsC --- src/ai4binance/ops/system_report.py | 53 +++++++++++++------ tests/test_lowest_coverage_remaining_edges.py | 2 +- tests/test_system_report.py | 34 ++++++++++++ 3 files changed, 71 insertions(+), 18 deletions(-) diff --git a/src/ai4binance/ops/system_report.py b/src/ai4binance/ops/system_report.py index 5af2e145..78cfb232 100644 --- a/src/ai4binance/ops/system_report.py +++ b/src/ai4binance/ops/system_report.py @@ -37,6 +37,8 @@ _LOCAL_ADVISORY_MODEL = "qwen3:8b" _LOCAL_ADVISORY_PROVIDER = "llama.cpp" _LOCAL_ADVISORY_ENDPOINT_PREFIX = "http://127.0.0.1:8080" +_PRIMARY_LOCAL_REASONING_HEALTH_FILE = "primary-local-reasoning-health.json" +_QWEN_PROMPTER_HEALTH_FILE = "qwen-prompter-health.json" _AUTO_LEARN_CAPABILITIES = ( "observe", "analyze", @@ -314,36 +316,45 @@ def runtime_state_payload( def local_advisory_health_payload(settings: Settings) -> dict[str, object]: - """Verify the only production advisory route: loopback llama.cpp qwen3:8b.""" + """Verify the canonical local llama.cpp qwen3 advisory route.""" blockers: list[str] = [] - state_path = settings.runtime_state_path.parent / "qwen-prompter-health.json" + state_directory = settings.runtime_state_path.parent + primary_state_path = state_directory / _PRIMARY_LOCAL_REASONING_HEALTH_FILE + compatibility_state_path = state_directory / _QWEN_PROMPTER_HEALTH_FILE + state_path = primary_state_path health: dict[str, object] = {} try: - loaded = json.loads(state_path.read_text(encoding="utf-8-sig")) + loaded = json.loads(primary_state_path.read_text(encoding="utf-8-sig")) if isinstance(loaded, dict): health = cast(dict[str, object], loaded) else: - blockers.append("QWEN_PROMPTER_STATE_INVALID") + blockers.append("PRIMARY_LOCAL_REASONING_STATE_INVALID") except FileNotFoundError: - blockers.append("QWEN_PROMPTER_STATE_MISSING") + state_path = compatibility_state_path + try: + loaded = json.loads(state_path.read_text(encoding="utf-8-sig")) + if isinstance(loaded, dict): + health = cast(dict[str, object], loaded) + else: + blockers.append("QWEN_PROMPTER_STATE_INVALID") + except FileNotFoundError: + blockers.append("PRIMARY_LOCAL_REASONING_STATE_MISSING") + except (OSError, json.JSONDecodeError): + blockers.append("QWEN_PROMPTER_STATE_INVALID") except (OSError, json.JSONDecodeError): - blockers.append("QWEN_PROMPTER_STATE_INVALID") + blockers.append("PRIMARY_LOCAL_REASONING_STATE_INVALID") if health.get("provider") != _LOCAL_ADVISORY_PROVIDER: blockers.append("LLAMA_CPP_PROVIDER_MISMATCH") if health.get("status") != "RUNNING": - blockers.append("QWEN_PROMPTER_NOT_RUNNING") - if health.get("model") != _LOCAL_ADVISORY_MODEL: - blockers.append("QWEN_PROMPTER_MODEL_MISMATCH") + blockers.append("LOCAL_ADVISORY_NOT_RUNNING") + runtime_model = str(health.get("runtime_model") or health.get("model") or "") + if runtime_model != _LOCAL_ADVISORY_MODEL: + blockers.append("LOCAL_ADVISORY_MODEL_MISMATCH") endpoint = str(health.get("endpoint", "")) if not endpoint.startswith(_LOCAL_ADVISORY_ENDPOINT_PREFIX): blockers.append("LLAMA_CPP_ENDPOINT_NOT_LOOPBACK") - pid = _safe_int(health.get("provider_pid")) - if pid is None: - pid = _safe_int(health.get("pid")) - if pid is None or pid < 1 or not SingleInstanceLease._pid_is_alive(pid): - blockers.append("QWEN_PROMPTER_PID_NOT_ALIVE") listener_pids = tuple( sorted( pid @@ -353,10 +364,17 @@ def local_advisory_health_payload(settings: Settings) -> dict[str, object]: if pid is not None ) ) + pid = _safe_int(health.get("provider_pid")) + if pid is None: + pid = _safe_int(health.get("pid")) + if pid is None and listener_pids: + pid = listener_pids[0] + if pid is None or pid < 1 or not SingleInstanceLease._pid_is_alive(pid): + blockers.append("LOCAL_ADVISORY_PID_NOT_ALIVE") if not listener_pids: - blockers.append("QWEN_PROMPTER_LISTENER_MISSING") + blockers.append("LOCAL_ADVISORY_LISTENER_MISSING") elif not any(SingleInstanceLease._pid_is_alive(value) for value in listener_pids): - blockers.append("QWEN_PROMPTER_LISTENER_NOT_ALIVE") + blockers.append("LOCAL_ADVISORY_LISTENER_NOT_ALIVE") auto_learn = _auto_learn_payload( health, status=str(health.get("status", "UNKNOWN")), @@ -367,10 +385,11 @@ def local_advisory_health_payload(settings: Settings) -> dict[str, object]: "status": "READY" if not blockers else "DEGRADED", "provider": _LOCAL_ADVISORY_PROVIDER, "endpoint": endpoint or _LOCAL_ADVISORY_ENDPOINT_PREFIX, - "model": _LOCAL_ADVISORY_MODEL, + "model": runtime_model or _LOCAL_ADVISORY_MODEL, "loopback_only": True, "listener_pids": listener_pids, "prompter_state_path": str(state_path), + "health_source_service": health.get("service"), "prompter_status": health.get("status"), "prompter_pid": pid, "auto_learn": auto_learn, diff --git a/tests/test_lowest_coverage_remaining_edges.py b/tests/test_lowest_coverage_remaining_edges.py index 13fe9bb1..c332b309 100644 --- a/tests/test_lowest_coverage_remaining_edges.py +++ b/tests/test_lowest_coverage_remaining_edges.py @@ -682,7 +682,7 @@ def test_system_report_helpers_cover_invalid_components_and_payloads( health = system_report_module.local_advisory_health_payload(settings) assert health["status"] == "DEGRADED" health_blockers = cast(tuple[str, ...], health["blockers"]) - assert "QWEN_PROMPTER_LISTENER_MISSING" in health_blockers + assert "LOCAL_ADVISORY_LISTENER_MISSING" in health_blockers invalid = system_report_module._safe_component("demo", lambda: "not-dict") failed = system_report_module._safe_component( diff --git a/tests/test_system_report.py b/tests/test_system_report.py index aa5bae21..a05c5daa 100644 --- a/tests/test_system_report.py +++ b/tests/test_system_report.py @@ -249,6 +249,40 @@ def test_local_advisory_health_requires_loopback_llama_qwen_and_prompter( assert payload["live_eligibility_status"] == "LIVE_ORDER_BLOCKED" +def test_local_advisory_health_prefers_enabled_primary_local_reasoning_service( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + settings = _settings(monkeypatch, tmp_path) + state_dir = tmp_path / "runtime" / "state" + state_dir.mkdir(parents=True, exist_ok=True) + (state_dir / "primary-local-reasoning-health.json").write_text( + json.dumps( + { + "service": "primary-local-reasoning", + "provider": "llama.cpp", + "status": "RUNNING", + "runtime_model": "qwen3:8b", + "endpoint": "http://127.0.0.1:8080", + "listener_pids": [os.getpid()], + "execution_allowed": False, + "live_eligibility_status": "LIVE_ORDER_BLOCKED", + } + ), + encoding="utf-8", + ) + (state_dir / "qwen-prompter-health.json").write_text( + json.dumps({"status": "STARTING", "endpoint": ""}), encoding="utf-8" + ) + + payload = local_advisory_health_payload(settings) + + assert payload["status"] == "READY" + assert payload["health_source_service"] == "primary-local-reasoning" + assert payload["prompter_pid"] == os.getpid() + assert payload["blockers"] == () + + def test_system_report_persists_secret_safe_json_and_markdown( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, From 56fb2fe2bb3478789730eb49d8155f7ded93c72b Mon Sep 17 00:00:00 2001 From: Huseyin Cicek Date: Wed, 23 Sep 2026 20:40:46 +0300 Subject: [PATCH 02/19] feat: add by HsC --- .gitleaksignore | 2 + publication/sanitize_publication.py | 1 - .../test_governance_enforcement_fabric.py | 18 +++++--- .../terminology/test_terminology_policy.py | 4 +- tests/test_architecture_diagram_validation.py | 4 +- tests/test_artifact_hygiene_scripts.py | 6 +-- tests/test_lowest_twenty_coverage_models.py | 9 ++-- tests/test_maintainability_ratchet.py | 26 ++++++----- tests/test_market_data_gateway.py | 2 +- tests/test_opportunity_monitor.py | 2 +- tests/test_public_showcase.py | 5 ++- tests/test_runtime_artifacts_migration.py | 45 ++++++++++++++----- tests/test_runtime_hygiene.py | 2 +- tests/test_security_tooling_contract.py | 2 +- 14 files changed, 81 insertions(+), 47 deletions(-) diff --git a/.gitleaksignore b/.gitleaksignore index 2e7e94af..60c10afb 100644 --- a/.gitleaksignore +++ b/.gitleaksignore @@ -11,3 +11,5 @@ de1c125880c20a0f8bef9f076fa7040441b5fafe:config/governance/governed_document_loc 7319e9b443e307c4de43b264ef4c7cfedd5c34c7:config/governance/governed_document_lock_manifest.json:generic-api-key:1630 7319e9b443e307c4de43b264ef4c7cfedd5c34c7:src/ai4binance/exchange/ws_api.py:generic-api-key:26 1ee98abdcb3bf6ce050fcc47514ff641c1489fe1:src/ai4binance/exchange/ws_api.py:generic-api-key:26 +6cccbb99632172df0e1a982035c9b84aafb93143:runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_full_gate_green_20260830_001.json:generic-api-key:24 +6cccbb99632172df0e1a982035c9b84aafb93143:runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_governed_docs_sha_alignment_20260826_001.json:generic-api-key:26 diff --git a/publication/sanitize_publication.py b/publication/sanitize_publication.py index 8e8130cc..babc9932 100644 --- a/publication/sanitize_publication.py +++ b/publication/sanitize_publication.py @@ -5,7 +5,6 @@ import runpy from pathlib import Path - if __name__ == "__main__": runpy.run_path( Path(__file__).resolve().parents[1] / "scripts" / "sanitize_publication.py", diff --git a/tests/governance/terminology/test_governance_enforcement_fabric.py b/tests/governance/terminology/test_governance_enforcement_fabric.py index cf3f8a74..9ec194bd 100644 --- a/tests/governance/terminology/test_governance_enforcement_fabric.py +++ b/tests/governance/terminology/test_governance_enforcement_fabric.py @@ -347,7 +347,9 @@ def test_fabric_contract_helpers_reject_untrusted_shapes(tmp_path: Path) -> None with pytest.raises(ValueError, match="POSIX"): fabric_module._safe_path(value, "path") for value in (None, [], ["x", "x"]): - with pytest.raises(ValueError): + with pytest.raises( + ValueError, match=r"paths must be (a non-empty list|unique)" + ): fabric_module._safe_paths(value, "paths") for value in ("bad", "A" * 64, "a" * 63): with pytest.raises(ValueError, match="SHA-256"): @@ -474,7 +476,13 @@ def test_quality_mapping_parser_rejects_malformed_and_duplicate_entries() -> Non } }, ): - with pytest.raises(ValueError): + with pytest.raises( + ValueError, + match=( + r"(mappings must be a list|mapping tests must be a list|" + r"duplicate STANDARD mapping)" + ), + ): fabric_module._quality_standard_mappings(payload) assert fabric_module._quality_standard_mappings( { @@ -503,11 +511,11 @@ def test_fabric_low_level_contracts_cover_metadata_and_duplicate_paths( frontmatter = tmp_path / "standard.md" frontmatter.write_text("---\ndocument_id: TEST\n---\n", encoding="utf-8") assert fabric_module._frontmatter(frontmatter)["document_id"] == "TEST" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="paths must be unique"): fabric_module._safe_paths(["tests/a.py", "tests/a.py"], "paths") - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="repository-relative POSIX"): fabric_module._safe_path("../escape", "path") - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="SHA-256"): fabric_module._sha256_text("A" * 64) diff --git a/tests/governance/terminology/test_terminology_policy.py b/tests/governance/terminology/test_terminology_policy.py index 50e4aaf9..78a90ffd 100644 --- a/tests/governance/terminology/test_terminology_policy.py +++ b/tests/governance/terminology/test_terminology_policy.py @@ -137,9 +137,7 @@ def test_terminology_scan_covers_deprecated_missing_and_projection_drift( loaded_policy = load_terminology_policy(ROOT) policy = replace( loaded_policy, - terms=( - replace(loaded_policy.terms[0], deprecated_aliases=("legacy",)), - ), + terms=(replace(loaded_policy.terms[0], deprecated_aliases=("legacy",)),), ) target = tmp_path / policy.scan_roots[0] / "terms.txt" target.parent.mkdir(parents=True) diff --git a/tests/test_architecture_diagram_validation.py b/tests/test_architecture_diagram_validation.py index 86ea8216..8d5e9970 100644 --- a/tests/test_architecture_diagram_validation.py +++ b/tests/test_architecture_diagram_validation.py @@ -2,14 +2,14 @@ import re from pathlib import Path -from unittest.mock import patch +import pytest import yaml +from ai4binance.ops import architecture_diagram_validation as diagram_validation from ai4binance.ops.architecture_diagram_validation import ( validate_architecture_diagrams, ) -from ai4binance.ops import architecture_diagram_validation as diagram_validation ROOT = Path(__file__).parents[1] DIAGRAM_ROOT = ROOT / "docs" / "architecture" / "diagrams" diff --git a/tests/test_artifact_hygiene_scripts.py b/tests/test_artifact_hygiene_scripts.py index 943bf754..f9be76dc 100644 --- a/tests/test_artifact_hygiene_scripts.py +++ b/tests/test_artifact_hygiene_scripts.py @@ -4359,9 +4359,9 @@ def test_qwen_prompter_startup_task_is_visible_and_advisory_only() -> None: llama_server_text = (Path("scripts") / "start_llama_server.ps1").read_text( encoding="utf-8" ) - vision_server_text = ( - Path("scripts") / "start_local_vision_server.ps1" - ).read_text(encoding="utf-8") + vision_server_text = (Path("scripts") / "start_local_vision_server.ps1").read_text( + encoding="utf-8" + ) local_llm_text = (Path("scripts") / "start_local_llm.ps1").read_text( encoding="utf-8" ) diff --git a/tests/test_lowest_twenty_coverage_models.py b/tests/test_lowest_twenty_coverage_models.py index ad5a2cb0..017b0e19 100644 --- a/tests/test_lowest_twenty_coverage_models.py +++ b/tests/test_lowest_twenty_coverage_models.py @@ -93,11 +93,7 @@ def test_runtime_environment_rejects_unsafe_temp_contracts( ai4binance.tomllib, "load", lambda _stream: { - "tool": { - "ai4binance": { - "runtime": {"temporary_directory": directory} - } - } + "tool": {"ai4binance": {"runtime": {"temporary_directory": directory}}} }, ) with pytest.raises(ValueError, match="Temporary directory"): @@ -223,7 +219,8 @@ def test_trend_events_observe_records_crosses_and_confirmed_slope( ), ) observed = trend_events_module.TrendEventsAgent._observe( - "1h", (object(),) * 201 # type: ignore[arg-type] + "1h", + (object(),) * 201, # type: ignore[arg-type] ) _timeframe, direction, _band, events, slope = observed assert direction == 1 diff --git a/tests/test_maintainability_ratchet.py b/tests/test_maintainability_ratchet.py index 2e403f0e..5b0cd857 100644 --- a/tests/test_maintainability_ratchet.py +++ b/tests/test_maintainability_ratchet.py @@ -55,19 +55,27 @@ def test_repository_maintainability_baseline_is_current() -> None: @pytest.mark.parametrize( "payload", - ( + [ "{}", '{"schema_version": 1, "limits": {}, "approved_paths": []}', - '{"schema_version": 1, "limits": {"C901": 0, "PLR0912": 0, "PLR0915": -1}, "approved_paths": []}', - '{"schema_version": 1, "limits": {"C901": 0, "PLR0912": 0, "PLR0915": 0}, "approved_paths": ["outside.py"]}', - ), + ( + '{"schema_version": 1, "limits": ' + '{"C901": 0, "PLR0912": 0, "PLR0915": -1}, "approved_paths": []}' + ), + ( + '{"schema_version": 1, "limits": ' + '{"C901": 0, "PLR0912": 0, "PLR0915": 0}, "approved_paths": ["outside.py"]}' + ), + ], ) def test_load_baseline_rejects_invalid_governed_shapes( tmp_path: Path, payload: str ) -> None: path = tmp_path / "baseline.json" path.write_text(payload, encoding="utf-8") - with pytest.raises(ValueError): + with pytest.raises( + ValueError, match=r"(schema_version|limits|approved_paths)" + ): load_baseline(path) @@ -77,16 +85,12 @@ def test_collect_findings_and_evaluate_fail_closed_for_bad_tool_data( completed = type( "Completed", (), {"returncode": 2, "stderr": "tool failed", "stdout": ""} )() - monkeypatch.setattr( - ratchet.subprocess, "run", lambda *_args, **_kwargs: completed - ) + monkeypatch.setattr(ratchet.subprocess, "run", lambda *_args, **_kwargs: completed) with pytest.raises(RuntimeError, match="tool failed"): collect_ruff_findings(ROOT) malformed = type("Completed", (), {"returncode": 0, "stderr": "", "stdout": "{}"})() - monkeypatch.setattr( - ratchet.subprocess, "run", lambda *_args, **_kwargs: malformed - ) + monkeypatch.setattr(ratchet.subprocess, "run", lambda *_args, **_kwargs: malformed) with pytest.raises(ValueError, match="JSON array"): collect_ruff_findings(ROOT) diff --git a/tests/test_market_data_gateway.py b/tests/test_market_data_gateway.py index 40b6c85b..bbb6a459 100644 --- a/tests/test_market_data_gateway.py +++ b/tests/test_market_data_gateway.py @@ -32,12 +32,12 @@ SharedMarketCache, build_gateway, ) +from ai4binance.exchange import rate_limit as rate_limit_module from ai4binance.exchange.public_stream import SpotKlineUpdate from ai4binance.exchange.rate_limit import ( WeightedRateLimitGovernor, public_request_weight, ) -from ai4binance.exchange import rate_limit as rate_limit_module from ai4binance.schemas import OHLCVCandle START = datetime(2026, 9, 17, 0, 0, tzinfo=UTC) diff --git a/tests/test_opportunity_monitor.py b/tests/test_opportunity_monitor.py index 0e04e5a4..d83b88b7 100644 --- a/tests/test_opportunity_monitor.py +++ b/tests/test_opportunity_monitor.py @@ -24,13 +24,13 @@ universe_monitor_summary, ) from ai4binance.cli.opportunity_monitor import refresh_candle_windows +from ai4binance.compatibility import opportunity_monitor as monitor_module from ai4binance.compatibility.opportunity_monitor import ( SAFE_STATE as COMPATIBILITY_SAFE_STATE, ) from ai4binance.compatibility.opportunity_monitor import ( refresh_monitor as compatibility_refresh_monitor, ) -from ai4binance.compatibility import opportunity_monitor as monitor_module from ai4binance.config import Settings from ai4binance.data.archive import ParquetOHLCVArchive from ai4binance.data.market_history_sync import read_cached_market_universe diff --git a/tests/test_public_showcase.py b/tests/test_public_showcase.py index 896fe300..2a987e3e 100644 --- a/tests/test_public_showcase.py +++ b/tests/test_public_showcase.py @@ -151,7 +151,7 @@ def test_public_showcase_rejects_hash_drift_and_secret_scan_failure( assert not (tmp_path / "scan-output").exists() -@pytest.mark.parametrize("value", ("", "/absolute", "back\\slash", "a/../b")) +@pytest.mark.parametrize("value", ["", "/absolute", "back\\slash", "a/../b"]) def test_showcase_helpers_reject_unsafe_contract_values(value: str) -> None: with pytest.raises(PublicShowcaseError): showcase._safe_relative_path(value, field="artifact") @@ -244,7 +244,8 @@ def test_showcase_rejects_each_authority_expansion_shape(tmp_path: Path) -> None manifest_path = tmp_path / "version.yaml" manifest_path.write_text( - "version: 2\npublication: {}\nallowed_artifacts: []\ndenied_paths: []\nsecret_scan: {}\n", + "version: 2\npublication: {}\nallowed_artifacts: []\n" + "denied_paths: []\nsecret_scan: {}\n", encoding="utf-8", ) with pytest.raises(PublicShowcaseError, match="version"): diff --git a/tests/test_runtime_artifacts_migration.py b/tests/test_runtime_artifacts_migration.py index dd9db41f..9c9f3e99 100644 --- a/tests/test_runtime_artifacts_migration.py +++ b/tests/test_runtime_artifacts_migration.py @@ -1,19 +1,20 @@ """Regression tests for the canonical runtime-artifact layout migration.""" +import json from pathlib import Path from typing import cast -import json import pytest + from ai4binance.infrastructure.filesystem.runtime_artifacts import ( RuntimeArtifactLayoutManifest, default_runtime_artifact_layout_manifest_path, + layout, load_runtime_artifact_layout_manifest, ) from ai4binance.infrastructure.filesystem.runtime_artifacts.layout import ( RuntimeArtifactLayoutManifest as CanonicalLayoutManifest, ) -from ai4binance.infrastructure.filesystem.runtime_artifacts import layout from ai4binance.ops.kaizen_quality import build_architecture_baseline from ai4binance.runtime_artifacts import ( RuntimeArtifactLayoutManifest as LegacyPackageLayoutManifest, @@ -64,7 +65,7 @@ def test_runtime_artifact_migration_is_recorded_as_canonical_and_facade() -> Non def test_layout_contract_helpers_fail_closed_and_canonicalize_aliases( tmp_path: Path, ) -> None: - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="under runtime"): layout.RuntimeRetentionPolicy("outside", "keep", 0, 0, False) manifest = CanonicalLayoutManifest( "runtime/artifacts", @@ -81,13 +82,22 @@ def test_layout_contract_helpers_fail_closed_and_canonicalize_aliases( manifest.retention_for("unknown") assert layout._capacity_budget_mapping(None) == {} for value in ([], {"outside": 1}, {"runtime/a": True}, {"runtime/a": 0}): - with pytest.raises(ValueError): + with pytest.raises( + ValueError, + match=( + r"(capacity_budgets must be an object|runtime paths|" + r"must be an integer|must be positive)" + ), + ): layout._capacity_budget_mapping(value) for value in (None, [], {"x": {}}): if value is None: assert layout._retention_mapping(value) == {} else: - with pytest.raises(ValueError): + with pytest.raises( + ValueError, + match=r"(retention must be an object|automatic_cleanup must be boolean)", + ): layout._retention_mapping(value) path = tmp_path / "manifest.json" @@ -116,20 +126,35 @@ def test_runtime_retention_policy_rejects_invalid_values( def test_layout_mapping_helpers_cover_all_invalid_contract_shapes() -> None: for value in (None, [], {}, {"": "runtime/a"}, {"a": ""}): - with pytest.raises(ValueError): + with pytest.raises( + ValueError, + match=r"roots must be (a non-empty object|contain non-empty strings)", + ): layout._text_mapping(value, "roots") - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="canonical_root must be a non-empty string"): layout._text({}, "canonical_root") for value in ({"rule": []}, {"": {}}, {"rule": {"automatic_cleanup": "yes"}}): - with pytest.raises(ValueError): + with pytest.raises( + ValueError, + match=( + r"(retention entries must be named objects|" + r"automatic_cleanup must be boolean)" + ), + ): layout._retention_mapping(value) for policy in ( {"automatic_cleanup": True, "minimum_age_days": True, "keep_latest": 0}, {"automatic_cleanup": True, "minimum_age_days": 0, "keep_latest": False}, ): - with pytest.raises(ValueError): + with pytest.raises( + ValueError, + match=( + r"(minimum_age_days must be an integer|" + r"keep_latest must be an integer)" + ), + ): layout._retention_mapping({"rule": policy}) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="entry kind is invalid"): layout._retention_entry_kind("unknown") diff --git a/tests/test_runtime_hygiene.py b/tests/test_runtime_hygiene.py index da95b28d..54f03f6b 100644 --- a/tests/test_runtime_hygiene.py +++ b/tests/test_runtime_hygiene.py @@ -11,11 +11,11 @@ RuntimeArtifactLayoutManifest, RuntimeRetentionPolicy, ) +from ai4binance.ops import runtime_hygiene from ai4binance.ops.runtime_hygiene import ( build_runtime_hygiene_report, persist_runtime_hygiene_report, ) -from ai4binance.ops import runtime_hygiene def _manifest() -> RuntimeArtifactLayoutManifest: diff --git a/tests/test_security_tooling_contract.py b/tests/test_security_tooling_contract.py index 9b80aa3e..4d34a17d 100644 --- a/tests/test_security_tooling_contract.py +++ b/tests/test_security_tooling_contract.py @@ -1299,7 +1299,7 @@ def test_gitleaks_ignore_list_contains_only_exact_historic_fingerprints() -> Non if line and not line.startswith("#") ] - assert len(entries) == 11 + assert len(entries) == 13 assert all(":generic-api-key:" in entry for entry in entries) assert all(re.fullmatch(r"[0-9a-f]{40}:.+:[1-9][0-9]*", entry) for entry in entries) From 540bbb7df2f78bcd2e67895e231903a7931e6972 Mon Sep 17 00:00:00 2001 From: Huseyin Cicek Date: Wed, 23 Sep 2026 20:56:14 +0300 Subject: [PATCH 03/19] chore: align quality validation baseline --- .../ruff-maintainability-baseline.json | 2 ++ tests/test_maintainability_ratchet.py | 4 +-- tests/test_runtime_artifacts_migration.py | 32 ++++++++++++++++--- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/config/quality/ruff-maintainability-baseline.json b/config/quality/ruff-maintainability-baseline.json index f1aabe02..47f18409 100644 --- a/config/quality/ruff-maintainability-baseline.json +++ b/config/quality/ruff-maintainability-baseline.json @@ -59,6 +59,7 @@ "src/ai4binance/governance/gate.py", "src/ai4binance/governance/governance_enforcement_fabric.py", "src/ai4binance/governance/lean.py", + "src/ai4binance/governance/model_registry.py", "src/ai4binance/governance/repository_validator.py", "src/ai4binance/governance/risk_assessment.py", "src/ai4binance/governance/supply_chain.py", @@ -67,6 +68,7 @@ "src/ai4binance/governance/workflow.py", "src/ai4binance/historical_replay_evaluation.py", "src/ai4binance/historical_replay_state.py", + "src/ai4binance/internal_radar.py", "src/ai4binance/learning/engine.py", "src/ai4binance/learning/governance.py", "src/ai4binance/mcp/evidence.py", diff --git a/tests/test_maintainability_ratchet.py b/tests/test_maintainability_ratchet.py index 5b0cd857..bb4296ee 100644 --- a/tests/test_maintainability_ratchet.py +++ b/tests/test_maintainability_ratchet.py @@ -73,9 +73,7 @@ def test_load_baseline_rejects_invalid_governed_shapes( ) -> None: path = tmp_path / "baseline.json" path.write_text(payload, encoding="utf-8") - with pytest.raises( - ValueError, match=r"(schema_version|limits|approved_paths)" - ): + with pytest.raises(ValueError, match=r"(schema_version|limits|approved_paths)"): load_baseline(path) diff --git a/tests/test_runtime_artifacts_migration.py b/tests/test_runtime_artifacts_migration.py index 9c9f3e99..68d83a0f 100644 --- a/tests/test_runtime_artifacts_migration.py +++ b/tests/test_runtime_artifacts_migration.py @@ -81,7 +81,13 @@ def test_layout_contract_helpers_fail_closed_and_canonicalize_aliases( with pytest.raises(ValueError, match="unknown runtime retention"): manifest.retention_for("unknown") assert layout._capacity_budget_mapping(None) == {} - for value in ([], {"outside": 1}, {"runtime/a": True}, {"runtime/a": 0}): + capacity_values: tuple[object, ...] = ( + [], + {"outside": 1}, + {"runtime/a": True}, + {"runtime/a": 0}, + ) + for value in capacity_values: with pytest.raises( ValueError, match=( @@ -90,13 +96,17 @@ def test_layout_contract_helpers_fail_closed_and_canonicalize_aliases( ), ): layout._capacity_budget_mapping(value) - for value in (None, [], {"x": {}}): + retention_values: tuple[object, ...] = (None, [], {"x": {}}) + for value in retention_values: if value is None: assert layout._retention_mapping(value) == {} else: with pytest.raises( ValueError, - match=r"(retention must be an object|automatic_cleanup must be boolean)", + match=( + r"(retention must be an object|" + r"automatic_cleanup must be boolean)" + ), ): layout._retention_mapping(value) @@ -125,7 +135,14 @@ def test_runtime_retention_policy_rejects_invalid_values( def test_layout_mapping_helpers_cover_all_invalid_contract_shapes() -> None: - for value in (None, [], {}, {"": "runtime/a"}, {"a": ""}): + text_mapping_values: tuple[object, ...] = ( + None, + [], + {}, + {"": "runtime/a"}, + {"a": ""}, + ) + for value in text_mapping_values: with pytest.raises( ValueError, match=r"roots must be (a non-empty object|contain non-empty strings)", @@ -133,7 +150,12 @@ def test_layout_mapping_helpers_cover_all_invalid_contract_shapes() -> None: layout._text_mapping(value, "roots") with pytest.raises(ValueError, match="canonical_root must be a non-empty string"): layout._text({}, "canonical_root") - for value in ({"rule": []}, {"": {}}, {"rule": {"automatic_cleanup": "yes"}}): + invalid_retention_values: tuple[object, ...] = ( + {"rule": []}, + {"": {}}, + {"rule": {"automatic_cleanup": "yes"}}, + ) + for value in invalid_retention_values: with pytest.raises( ValueError, match=( From baa964125614966ecb35b1239eb55574f18aa15b Mon Sep 17 00:00:00 2001 From: Huseyin Cicek Date: Fri, 25 Sep 2026 00:10:44 +0300 Subject: [PATCH 04/19] chore: align quality validation baseline --- config/operations/services.json | 2 +- src/ai4binance/cli/futures_oos.py | 2 +- src/ai4binance/cli/market_data.py | 91 +++++---- src/ai4binance/cli/market_gateway.py | 51 +++-- src/ai4binance/cli/runtime.py | 55 ++++++ src/ai4binance/config.py | 2 +- .../data/market_history_continuous.py | 115 ++++++----- src/ai4binance/local_dashboard/local_views.js | 9 +- src/ai4binance/local_dashboard/server.py.in | 7 + tests/test_cli.py | 52 +++++ tests/test_config_reporting.py | 6 + tests/test_futures_replay.py | 8 +- tests/test_local_dashboard_source.py | 19 +- tests/test_market_data_gateway.py | 36 +++- .../test_market_history_boundary_contracts.py | 2 +- tests/test_market_history_continuous.py | 181 +++++++++++++----- tests/test_service_manifest.py | 2 +- 17 files changed, 485 insertions(+), 155 deletions(-) diff --git a/config/operations/services.json b/config/operations/services.json index 564a5929..4b3ee239 100644 --- a/config/operations/services.json +++ b/config/operations/services.json @@ -82,7 +82,7 @@ { "service": "market-history", "task_name": "AI4BINANCE-Market-History", - "command": "python -m ai4binance.cli.market_gateway", + "command": "python -m ai4binance.cli.market_data daemon", "mode": "RunMarketHistory", "required": true, "enabled": true, diff --git a/src/ai4binance/cli/futures_oos.py b/src/ai4binance/cli/futures_oos.py index f6a710d5..98ee221d 100644 --- a/src/ai4binance/cli/futures_oos.py +++ b/src/ai4binance/cli/futures_oos.py @@ -139,7 +139,7 @@ def main( parsed = build_parser().parse_args(arguments) repository_root = parsed.repository_root.resolve() - replay_root = repository_root / "runtime" / "datasets" / "futures" + replay_root = repository_root / "runtime" / "data" / "datasets" / "futures" evidence_root = ( repository_root / "runtime" / "artifacts" / "validation" / "futures_oos" ) diff --git a/src/ai4binance/cli/market_data.py b/src/ai4binance/cli/market_data.py index a29bc85f..0938f20d 100644 --- a/src/ai4binance/cli/market_data.py +++ b/src/ai4binance/cli/market_data.py @@ -94,11 +94,15 @@ def _dashboard_candidate_projection( elif isinstance(value, (int, float)) and not isinstance(value, bool): row[name] = value blockers = candidate.get("blockers") - if isinstance(blockers, list) and all( - isinstance(blocker, str) and len(blocker) <= 180 - for blocker in blockers[:40] + if ( + isinstance(blockers, Sequence) + and not isinstance(blockers, str) + and all( + isinstance(blocker, str) and len(blocker) <= 180 + for blocker in blockers[:40] + ) ): - row["blockers"] = blockers[:40] + row["blockers"] = list(blockers[:40]) else: row["blockers"] = ["CANDIDATE_BLOCKERS_UNAVAILABLE"] row.update(market=market, symbol=symbol, **_SAFE_STATE) @@ -112,7 +116,7 @@ def _priority_depth_markets( *, include_coin_m: bool, ) -> dict[str, tuple[str, ...]]: - """Bound L2 bootstrap to watched symbols with one top-volume fallback.""" + """Cover each active top-volume market universe with bounded public L2.""" priority = tuple(dict.fromkeys(priority_symbols)) spot_symbols = tuple(dict.fromkeys(getattr(universe, "spot_symbols", ()))) @@ -121,8 +125,7 @@ def _priority_depth_markets( def selected(symbols: tuple[str, ...]) -> tuple[str, ...]: eligible = frozenset(symbols) watched = tuple(symbol for symbol in priority if symbol in eligible) - target_count = min(8, max(1, len(watched))) - return tuple(dict.fromkeys((*watched, *symbols)))[:target_count] + return tuple(dict.fromkeys((*watched, *symbols)))[:50] markets = { "spot": selected(spot_symbols), @@ -136,6 +139,43 @@ def selected(symbols: tuple[str, ...]) -> tuple[str, ...]: return markets +def build_continuous_market_history( + settings: Settings, + synchronizer: MarketHistorySynchronizer, + *, + root: Path, + include_coin_m: bool, +) -> ContinuousMarketHistory: + """Build the one canonical continuous collector used by every runtime.""" + + return ContinuousMarketHistory( + history=synchronizer, + spot=synchronizer.universe_provider.spot_transport, + futures=synchronizer.universe_provider.futures_transport, + initial_days=settings.market_history_initial_days, + pages_per_stream=settings.market_history_pages_per_stream, + max_workers=settings.market_history_max_workers, + minimum_candles=settings.minimum_closed_candles, + coin_m=( + synchronizer.universe_provider.coin_m_transport + if include_coin_m + else None + ), + priority_symbols=tuple( + dict.fromkeys( + (settings.symbol, *settings.fixed_symbols, *settings.priority_watchlist) + ) + ), + refresh_request_path=_absolute(settings.market_history_state_path).with_name( + "market-history-refresh-request.json" + ), + on_symbol_ready=_build_canonical_opportunity_pipeline( + settings, root, include_futures=True + ), + on_symbol_screen=_build_opportunity_screen(settings, root), + ) + + def _build_canonical_opportunity_pipeline( settings: Settings, root: Path, *, include_futures: bool = False ) -> Callable[[str, str, datetime], Mapping[str, object]]: @@ -335,31 +375,11 @@ def run_market_history_command( print(json.dumps(payload, ensure_ascii=False, sort_keys=True)) return 0 if not payload.get("blockers") else 2 - continuous = ContinuousMarketHistory( - history=synchronizer, - spot=synchronizer.universe_provider.spot_transport, - futures=synchronizer.universe_provider.futures_transport, - initial_days=settings.market_history_initial_days, - pages_per_stream=settings.market_history_pages_per_stream, - max_workers=settings.market_history_max_workers, - minimum_candles=settings.minimum_closed_candles, - coin_m=synchronizer.universe_provider.coin_m_transport, - priority_symbols=tuple( - dict.fromkeys( - ( - settings.symbol, - *settings.fixed_symbols, - *settings.priority_watchlist, - ) - ) - ), - refresh_request_path=_absolute(settings.market_history_state_path).with_name( - "market-history-refresh-request.json" - ), - on_symbol_ready=_build_canonical_opportunity_pipeline( - settings, Path.cwd(), include_futures=True - ), - on_symbol_screen=_build_opportunity_screen(settings, Path.cwd()), + continuous = build_continuous_market_history( + settings, + synchronizer, + root=Path.cwd(), + include_coin_m=True, ) if command == "market-history-sync" and as_of is None: with SingleInstanceLease(synchronizer.state_path.with_suffix(".lock")): @@ -571,4 +591,9 @@ def main(arguments: Sequence[str] | None = None) -> int: raise SystemExit(main()) -__all__ = ("build_market_history_synchronizer", "main", "run_market_history_command") +__all__ = ( + "build_continuous_market_history", + "build_market_history_synchronizer", + "main", + "run_market_history_command", +) diff --git a/src/ai4binance/cli/market_gateway.py b/src/ai4binance/cli/market_gateway.py index 3d6117b6..42754449 100644 --- a/src/ai4binance/cli/market_gateway.py +++ b/src/ai4binance/cli/market_gateway.py @@ -12,10 +12,14 @@ from websockets.exceptions import ConnectionClosed -from ai4binance.cli.market_data import build_market_history_synchronizer +from ai4binance.cli.market_data import ( + _priority_depth_markets, + build_continuous_market_history, + build_market_history_synchronizer, +) from ai4binance.config import Settings from ai4binance.data.market_data_gateway import MarketStreamGapError, build_gateway -from ai4binance.data.market_history_continuous import ContinuousMarketHistory +from ai4binance.data.market_depth import MarketDepthCollector from ai4binance.infrastructure.persistence.safe_json import write_json_object_verified from ai4binance.ops.runtime import SingleInstanceLease @@ -73,21 +77,21 @@ def run_gateway(settings: Settings, *, max_cycles: int | None = None) -> int: if max_cycles is not None and max_cycles < 1: raise ValueError("max_cycles must be positive") synchronizer = build_market_history_synchronizer(settings) - collector = ContinuousMarketHistory( - history=synchronizer, - spot=synchronizer.universe_provider.spot_transport, - futures=synchronizer.universe_provider.futures_transport, - initial_days=settings.market_history_initial_days, - pages_per_stream=settings.market_history_pages_per_stream, - max_workers=settings.market_history_max_workers, - minimum_candles=settings.minimum_closed_candles, - coin_m=None, - priority_symbols=tuple( - dict.fromkeys( - (settings.symbol, *settings.fixed_symbols, *settings.priority_watchlist) - ) - ), + collector = build_continuous_market_history( + settings, + synchronizer, + root=Path.cwd(), + include_coin_m=False, ) + depth: MarketDepthCollector | None = None + if getattr(settings, "market_depth_enabled", False): + depth = MarketDepthCollector( + synchronizer.archive_root / "depth", + { + "spot": collector.spot, + "usd_m_futures": collector.futures, + }, + ) lock_path = _absolute(settings.market_history_state_path).with_suffix(".lock") heartbeat = _GatewayStateHeartbeat(_absolute(settings.market_history_state_path)) completed = 0 @@ -95,6 +99,18 @@ def run_gateway(settings: Settings, *, max_cycles: int | None = None) -> int: with SingleInstanceLease(lock_path): while max_cycles is None or completed < max_cycles: observed_at = datetime.now(UTC) + if depth is not None: + depth_universe = synchronizer._eligible_universe( + observed_at, force_refresh=True + ) + if not depth_universe.blockers: + depth.start( + _priority_depth_markets( + depth_universe, + collector.priority_symbols, + include_coin_m=False, + ) + ) bootstrap = collector.sync_cycle(observed_at=observed_at) blockers = _blockers(bootstrap) if "MARKET_DATA_BACKFILL_PENDING" in blockers and not ( @@ -164,6 +180,9 @@ def run_gateway(settings: Settings, *, max_cycles: int | None = None) -> int: ) ) return 2 + finally: + if depth is not None: + depth.close() return 0 diff --git a/src/ai4binance/cli/runtime.py b/src/ai4binance/cli/runtime.py index 65bfe45a..a0c3cae8 100644 --- a/src/ai4binance/cli/runtime.py +++ b/src/ai4binance/cli/runtime.py @@ -116,6 +116,7 @@ _VIRTUAL_MARKET_STATE_NAME = "virtual-market.json" _VIRTUAL_MARKET_LOCK_NAME = "virtual-market.lock" _VIRTUAL_MARKET_REFRESH_REQUEST_NAME = "virtual-market-refresh-request.json" +_MARKET_HISTORY_REFRESH_REQUEST_NAME = "market-history-refresh-request.json" _DASHBOARD_SIMULATION_MAX_SYMBOLS = 1_000 _DASHBOARD_SIMULATION_MAX_BLOCKERS = 12 _NEWS_ASSET_ALIASES = { @@ -654,6 +655,13 @@ def run_virtual_market_daemon( else: discovery_symbol = last_symbol cycle_settings = settings.model_copy(update={"symbol": last_symbol}) + from ai4binance.data.market_history_continuous import ( + VIRTUAL_MARKET_COLLECTION_TIMEFRAMES, + ) + + cycle_settings = cycle_settings.model_copy( + update={"timeframes": VIRTUAL_MARKET_COLLECTION_TIMEFRAMES} + ) cycle_report.update( { "symbol": last_symbol, @@ -670,6 +678,13 @@ def run_virtual_market_daemon( exit_code = _run_virtual_market_research_cycle( cycle_settings, acquisition, cycle_report=cycle_report ) + _request_market_history_refresh_if_stale( + state_path.with_name(_MARKET_HISTORY_REFRESH_REQUEST_NAME), + symbol=last_symbol, + eligible_symbols=eligible_symbols, + observed_at=clock(), + cycle_report=cycle_report, + ) except (ExchangeError, OSError, RuntimeError, TypeError, ValueError): exit_code = 2 observed_at = clock() @@ -872,6 +887,46 @@ def _run_virtual_market_research_cycle( ) +def _request_market_history_refresh_if_stale( + path: Path, + *, + symbol: str, + eligible_symbols: tuple[str, ...], + observed_at: datetime, + cycle_report: dict[str, object], +) -> None: + """Request one bounded canonical refresh after a stale virtual snapshot.""" + + raw_blockers = cycle_report.get("research_blockers", ()) + blockers = ( + tuple(item for item in raw_blockers if isinstance(item, str)) + if isinstance(raw_blockers, (list, tuple)) + else () + ) + if not any(item.startswith("STALE_CANDLES:") for item in blockers): + return + from ai4binance.data.market_history_continuous import ( + enqueue_market_history_refresh_request, + ) + + try: + refresh = enqueue_market_history_refresh_request( + path, + market="SPOT", + symbol=symbol, + eligible_symbols=eligible_symbols, + requested_at=observed_at, + requester="VIRTUAL_MARKET", + ) + except (OSError, ValueError): + cycle_report["market_history_refresh"] = { + "state": "DATA_BLOCKED", + "blockers": ["VIRTUAL_MARKET_REFRESH_REQUEST_FAILED"], + } + return + cycle_report["market_history_refresh"] = refresh + + def _virtual_wallet_journal(settings: Settings) -> VirtualWalletJournal: return VirtualWalletJournal( ledger_path=settings.virtual_wallet_ledger_path, diff --git a/src/ai4binance/config.py b/src/ai4binance/config.py index 8a73b58f..558895dd 100644 --- a/src/ai4binance/config.py +++ b/src/ai4binance/config.py @@ -72,7 +72,7 @@ class Settings(BaseSettings): "config/research/runtime_validation_deployment.json" ) futures_oos_artifact_directory: Path = Path( - "runtime/artifacts/research/backtest/oos" + "runtime/artifacts/validation/futures_oos" ) backtest_report_directory: Path = Path("runtime/reports/backtest") backtest_layout_manifest_path: Path = Path( diff --git a/src/ai4binance/data/market_history_continuous.py b/src/ai4binance/data/market_history_continuous.py index 89f0110e..3ddd0620 100644 --- a/src/ai4binance/data/market_history_continuous.py +++ b/src/ai4binance/data/market_history_continuous.py @@ -52,8 +52,8 @@ "promotion_status": "RESEARCH_ONLY", "live_eligibility_status": "LIVE_ORDER_BLOCKED", } -VIRTUAL_MARKET_COLLECTION_TIMEFRAMES: Final = MARKET_HISTORY_TIMEFRAMES -_DASHBOARD_REFRESH_TIMEFRAMES = ("5m", "15m", "1h", "4h", "1d") +VIRTUAL_MARKET_COLLECTION_TIMEFRAMES: Final = ("15m", "1h", "4h") +_DASHBOARD_REFRESH_TIMEFRAMES = MARKET_HISTORY_TIMEFRAMES _SCREEN_TIMEFRAMES = VIRTUAL_MARKET_COLLECTION_TIMEFRAMES _ENRICHMENT_TIMEFRAMES: Final[tuple[str, ...]] = () # The canonical live path persists native decision timeframes directly. REST @@ -69,6 +69,7 @@ _STATE_RESULT_SAMPLE_LIMIT = 128 _DASHBOARD_OPPORTUNITY_LIMIT = 100 _DASHBOARD_REJECTION_LIMIT = 100 +_REFRESH_REQUESTERS = frozenset({"DASHBOARD", "VIRTUAL_MARKET"}) _REFRESH_REQUEST_SAFE_FIELDS = { "execution_allowed": False, "promotion_status": "RESEARCH_ONLY", @@ -127,7 +128,7 @@ def _read_refresh_request(path: Path) -> dict[str, object] | None: value.get("schema_version") != "MarketHistoryRefreshRequest/v1" or not isinstance(value.get("request_id"), str) or not isinstance(value.get("requester"), str) - or value.get("requester") != "DASHBOARD" + or value.get("requester") not in _REFRESH_REQUESTERS or value.get("market") not in {"SPOT", "USD_M_FUTURES"} or not isinstance(value.get("symbol"), str) or _REFRESH_REQUEST_SYMBOL.fullmatch(str(value["symbol"])) is None @@ -164,11 +165,15 @@ def enqueue_market_history_refresh_request( symbol: str, eligible_symbols: tuple[str, ...], requested_at: datetime, + requester: str = "DASHBOARD", ) -> dict[str, object]: - """Persist one validated dashboard request; never replace another pending job.""" + """Persist one bounded freshness request; never replace a pending job.""" if requested_at.tzinfo is None or requested_at.utcoffset() is None: raise ValueError("refresh request timestamp must be timezone-aware") + normalized_requester = requester.strip().upper() + if normalized_requester not in _REFRESH_REQUESTERS: + raise ValueError("market history refresh requester is invalid") normalized_symbol = symbol.strip().upper() if ( market not in {"SPOT", "USD_M_FUTURES"} @@ -199,14 +204,14 @@ def enqueue_market_history_refresh_request( request_id = ( "market-history:" + sha256( - f"DASHBOARD:{market}:{normalized_symbol}:{requested_at.isoformat()}".encode() + f"{normalized_requester}:{market}:{normalized_symbol}:{requested_at.isoformat()}".encode() ).hexdigest()[:24] ) request: dict[str, object] = { "schema_version": "MarketHistoryRefreshRequest/v1", "request_id": request_id, "requested_at": requested_at.astimezone(UTC).isoformat(), - "requester": "DASHBOARD", + "requester": normalized_requester, "market": market, "symbol": normalized_symbol, "status": "PENDING", @@ -236,7 +241,7 @@ def timeframe_refresh_schedule() -> list[dict[str, object]]: "gap_recovery_source": f"BINANCE_PUBLIC_REST_{timeframe.upper()}_ONLY", "network_download": True, } - for timeframe in MARKET_HISTORY_TIMEFRAMES + for timeframe in VIRTUAL_MARKET_COLLECTION_TIMEFRAMES ] @@ -454,6 +459,9 @@ class ContinuousMarketHistory: vision_history_enabled: bool = True on_symbol_ready: SymbolReadyHandler | None = field(default=None, repr=False) on_symbol_screen: SymbolReadyHandler | None = field(default=None, repr=False) + clock: Callable[[], datetime] = field( + default=lambda: datetime.now(UTC), repr=False + ) def __post_init__(self) -> None: if not 1 <= self.initial_days <= 3650 or not 1 <= self.pages_per_stream <= 32: @@ -594,7 +602,7 @@ def refresh_snapshots(snapshot_time: datetime) -> None: blockers.append("MARKET_HISTORY_REFRESH_REQUEST_INVALID") stream_items = self._interleaved_stream_work(work_items) staged = self.on_symbol_screen is not None - active_collection_timeframes = MARKET_HISTORY_TIMEFRAMES + active_collection_timeframes = VIRTUAL_MARKET_COLLECTION_TIMEFRAMES staged_markets = {"spot", "usd_m_futures"} if staged else set() if staged: stream_items = tuple( @@ -621,7 +629,8 @@ def refresh_snapshots(snapshot_time: datetime) -> None: } coverage: dict[str, dict[str, Counter[str]]] = { market_labels[market]: { - timeframe: Counter() for timeframe in MARKET_HISTORY_TIMEFRAMES + timeframe: Counter() + for timeframe in VIRTUAL_MARKET_COLLECTION_TIMEFRAMES } for market, _, transport in market_work if transport is not None @@ -678,7 +687,7 @@ def coverage_projection() -> dict[str, list[dict[str, object]]]: } for market, rows in coverage.items(): entries: list[dict[str, object]] = [] - for timeframe in MARKET_HISTORY_TIMEFRAMES: + for timeframe in VIRTUAL_MARKET_COLLECTION_TIMEFRAMES: counts = rows[timeframe] resolved = sum(counts.values()) entries.append( @@ -710,7 +719,13 @@ def opportunity_projection_payload() -> dict[str, dict[str, object]]: opportunities = list( cast(list[dict[str, object]], value["opportunities"]) ) - if data_blocked: + if opportunities: + status = ( + "CANDIDATES_AVAILABLE_WITH_DATA_GAPS" + if data_blocked + else "CANDIDATES_AVAILABLE" + ) + elif data_blocked: status = "DATA_UNAVAILABLE" elif analysis_blocked: status = "ANALYSIS_BLOCKED" @@ -720,8 +735,6 @@ def opportunity_projection_payload() -> dict[str, dict[str, object]]: if cast(int, value["delegated_symbol_count"]) == eligible else "ANALYSIS_PENDING" ) - elif opportunities: - status = "CANDIDATES_AVAILABLE" else: status = "NO_TRADE" projection[market] = { @@ -835,30 +848,6 @@ def record_opportunity_analysis( projection["no_opportunity_symbol_count"] = ( cast(int, projection["no_opportunity_symbol_count"]) + 1 ) - candidates = analysis.get("dashboard_candidates") - if isinstance(candidates, list): - valid_candidates = [ - candidate - for candidate in candidates - if isinstance(candidate, dict) - and candidate.get("market") == market - and candidate.get("symbol") == symbol - and candidate.get("execution_allowed") is False - and candidate.get("live_eligibility_status") - == "LIVE_ORDER_BLOCKED" - ] - available = _DASHBOARD_OPPORTUNITY_LIMIT - len( - cast(list[dict[str, object]], projection["opportunities"]) - ) - cast(list[dict[str, object]], projection["opportunities"]).extend( - valid_candidates[:available] - ) - projection["suppressed_opportunity_count"] = cast( - int, projection["suppressed_opportunity_count"] - ) + max(0, len(valid_candidates) - max(0, available)) - projection["published_opportunity_count"] = len( - cast(list[dict[str, object]], projection["opportunities"]) - ) elif status == "DATA_BLOCKED": projection["data_blocked_symbol_count"] = ( cast(int, projection["data_blocked_symbol_count"]) + 1 @@ -872,6 +861,33 @@ def record_opportunity_analysis( cast(int, projection["analysis_blocked_symbol_count"]) + 1 ) + if status not in {"CURRENT", "DATA_BLOCKED"}: + return + candidates = analysis.get("dashboard_candidates") + if not isinstance(candidates, list): + return + valid_candidates = [ + candidate + for candidate in candidates + if isinstance(candidate, dict) + and candidate.get("market") == market + and candidate.get("symbol") == symbol + and candidate.get("execution_allowed") is False + and candidate.get("live_eligibility_status") == "LIVE_ORDER_BLOCKED" + ] + available = _DASHBOARD_OPPORTUNITY_LIMIT - len( + cast(list[dict[str, object]], projection["opportunities"]) + ) + cast(list[dict[str, object]], projection["opportunities"]).extend( + valid_candidates[:available] + ) + projection["suppressed_opportunity_count"] = cast( + int, projection["suppressed_opportunity_count"] + ) + max(0, len(valid_candidates) - max(0, available)) + projection["published_opportunity_count"] = len( + cast(list[dict[str, object]], projection["opportunities"]) + ) + def record_coverage( result: Mapping[str, object], *, @@ -1141,7 +1157,8 @@ def activate_pending_request() -> None: active_identity = identity enqueue_enrichment(identity) request_remaining = { - ("klines", timeframe) for timeframe in MARKET_HISTORY_TIMEFRAMES + ("klines", timeframe) + for timeframe in VIRTUAL_MARKET_COLLECTION_TIMEFRAMES } request_remaining.difference_update( (kind, timeframe) @@ -1320,7 +1337,7 @@ def complete_active_request() -> None: "schema_version": "2.0", "observed_at": now.isoformat(), "status": "DEGRADED" if blockers else "READY", - "timeframes": list(MARKET_HISTORY_TIMEFRAMES), + "timeframes": list(VIRTUAL_MARKET_COLLECTION_TIMEFRAMES), "timeframe_refresh_schedule": timeframe_refresh_schedule(), "collection_plan": self._collection_plan(), "initial_history_days": self.initial_days, @@ -1371,10 +1388,18 @@ def _complete_refresh_request( ) -> None: if self.refresh_request_path is None: return + completed_at = self.clock() + if completed_at.utcoffset() is None: + raise ValueError("market refresh completion clock must be timezone-aware") + requested_at = datetime.fromisoformat(str(request["requested_at"])) + if requested_at.utcoffset() is None: + raise ValueError("market refresh request timestamp must be timezone-aware") + completion_utc = completed_at.astimezone(UTC) + requested_utc = requested_at.astimezone(UTC) result = { **request, "status": status, - "completed_at": observed_at.isoformat(), + "completed_at": max(completion_utc, requested_utc).isoformat(), "blockers": list(blockers), } write_json_object_verified( @@ -1395,7 +1420,7 @@ def _refresh_request_data_blockers( ) -> tuple[str, ...]: archive = ParquetOHLCVArchive(self.history.archive_root / market) blockers: list[str] = [] - for timeframe in _DASHBOARD_REFRESH_TIMEFRAMES: + for timeframe in VIRTUAL_MARKET_COLLECTION_TIMEFRAMES: try: manifest = archive.manifest(symbol, timeframe) last_close = datetime.fromisoformat( @@ -1448,11 +1473,15 @@ def _interleaved_market_work( def _collection_kinds(market: str) -> tuple[tuple[str, str | None], ...]: if market == "spot": return tuple( - ("klines", timeframe) for timeframe in MARKET_HISTORY_TIMEFRAMES + ("klines", timeframe) + for timeframe in VIRTUAL_MARKET_COLLECTION_TIMEFRAMES ) if market in {"usd_m_futures", "coin_m_futures"}: return ( - *(("klines", timeframe) for timeframe in MARKET_HISTORY_TIMEFRAMES), + *( + ("klines", timeframe) + for timeframe in VIRTUAL_MARKET_COLLECTION_TIMEFRAMES + ), ("markPriceKlines", "5m"), ("indexPriceKlines", "5m"), ("funding", None), diff --git a/src/ai4binance/local_dashboard/local_views.js b/src/ai4binance/local_dashboard/local_views.js index 75f1132e..d1e10f30 100644 --- a/src/ai4binance/local_dashboard/local_views.js +++ b/src/ai4binance/local_dashboard/local_views.js @@ -7,7 +7,7 @@ const marketSelections = {SPOT:'',USD_M_FUTURES:''}; let marketVisiblePage = ''; const selectedMarket = () => (state.page==='virtual-market'?state.virtualMarket:state.market)==='Spot'?'SPOT':'USD_M_FUTURES'; const marketKey = market => market+'|'+marketSelections[market]; - const monitorLabel = value => ({CURRENT:t('Verified','Doğrulandı'),NOT_SCANNED:t('Not scanned','Taranmadı'),STALE:t('Stale','Eski'),INVALID:t('Invalid','Geçersiz'),UNAVAILABLE:t('Missing','Eksik'),DATA_BLOCKED:t('Data blocked','Veri engeli'),CONFIRMATION_PENDING:t('Awaiting confirmation','Teyit bekliyor'),FUTURES_RESEARCH_RADAR:t('Research candidate','Araştırma adayı'),WATCHLIST:t('Watchlist','İzleme'),PENDING_HORIZON:t('Awaiting 3 closed bars','3 kapanmış mum bekleniyor'),NOT_EVALUABLE:t('Not measurable','Ölçülemiyor'),EVALUATED:t('Measured','Ölçüldü'),BULLISH:t('Bullish','Yukarı'),BEARISH:t('Bearish','Aşağı'),BULLISH_CAUTION:t('Bullish · Caution','Yukarı · Temkinli'),BEARISH_CAUTION:t('Bearish · Caution','Aşağı · Temkinli'),WATCH_ONLY:t('Watch only','Yön teyidi yok'),NEUTRAL:t('Neutral','Nötr'),TARGET_FIRST:t('Target first','Önce hedef'),INVALIDATED_FIRST:t('Stop first','Önce stop'),FAVORABLE:t('Favorable','Lehte'),ADVERSE:t('Adverse','Aleyhte'),MIXED:t('Mixed','Karma'),UNRESOLVED:t('Unresolved','Belirsiz')})[value] || value || '—'; + const monitorLabel = value => ({CURRENT:t('Verified','Doğrulandı'),NOT_SCANNED:t('Not scanned','Taranmadı'),STALE:t('Stale','Eski'),INVALID:t('Invalid','Geçersiz'),UNAVAILABLE:t('Missing','Eksik'),DATA_BLOCKED:t('Data blocked','Veri engeli'),CANDIDATES_AVAILABLE:t('Candidates available','Fırsatlar mevcut'),CANDIDATES_AVAILABLE_WITH_DATA_GAPS:t('Candidates available with data gaps','Veri eksiklerine rağmen fırsatlar mevcut'),DATA_UNAVAILABLE:t('Data unavailable','Veri kullanılamıyor'),CONFIRMATION_PENDING:t('Awaiting confirmation','Teyit bekliyor'),FUTURES_RESEARCH_RADAR:t('Research candidate','Araştırma adayı'),WATCHLIST:t('Watchlist','İzleme'),PENDING_HORIZON:t('Awaiting 3 closed bars','3 kapanmış mum bekleniyor'),NOT_EVALUABLE:t('Not measurable','Ölçülemiyor'),EVALUATED:t('Measured','Ölçüldü'),BULLISH:t('Bullish','Yukarı'),BEARISH:t('Bearish','Aşağı'),BULLISH_CAUTION:t('Bullish · Caution','Yukarı · Temkinli'),BEARISH_CAUTION:t('Bearish · Caution','Aşağı · Temkinli'),WATCH_ONLY:t('Watch only','Yön teyidi yok'),NEUTRAL:t('Neutral','Nötr'),TARGET_FIRST:t('Target first','Önce hedef'),INVALIDATED_FIRST:t('Stop first','Önce stop'),FAVORABLE:t('Favorable','Lehte'),ADVERSE:t('Adverse','Aleyhte'),MIXED:t('Mixed','Karma'),UNRESOLVED:t('Unresolved','Belirsiz')})[value] || value || '—'; const level = value => value===null||value===undefined||value===''||!Number.isFinite(Number(value))?'—':Number(value).toLocaleString(state.language==='tr'?'tr-TR':'en-US',{maximumFractionDigits:8}); const completeOpportunityPlan = row => { const direction=String(row?.direction||'').toUpperCase(),names=['entry','stop_loss','tp1','tp2','tp3','target_risk_reward']; @@ -87,7 +87,8 @@ const marketSelections = {SPOT:'',USD_M_FUTURES:''}; p.appendChild(table(headers,candidates.map(r=>{const values=[observedTime(r.observed_at),r.symbol,r.side,monitorLabel(r.status),level(r.quantity),level(r.reference_price),level(r.entry),level(r.stop_loss),[r.tp1,r.tp2,r.tp3].map(level).join(' / '),level(r.target_risk_reward)];if(market==='USD_M_FUTURES')values.push(level(r.leverage));return values;}))); const coverage=universe.opportunity_coverage||{}; p.appendChild(el('div','aw-status',t('Monitor coverage: ','İzleme kapsamı: ')+String(coverage.monitored_symbol_count??0)+' / '+String(coverage.universe_count??0)+t(' coins. Unmonitored coins remain explicitly pending, not absent opportunities.',' koin. İzlenmeyen koinler fırsat yok sayılmaz; açıkça beklemede kalır.'))); - if(coverage.status&&coverage.status!=='CANDIDATES_AVAILABLE')p.appendChild(el('div','aw-status',t('Canonical opportunity state: ','Kanonik fırsat durumu: ')+coverage.status)); + if(coverage.status)p.appendChild(el('div','aw-status',t('Canonical opportunity state: ','Kanonik fırsat durumu: ')+monitorLabel(coverage.status))); + if(coverage.data_blocked_symbol_count)p.appendChild(el('div','aw-status',t('Data-blocked symbols: ','Veri nedeniyle engelli semboller: ')+String(coverage.data_blocked_symbol_count)+t('. Published research opportunities remain visible.','; yayınlanmış araştırma fırsatları görünür kalır.'))); if(!candidates.length)p.appendChild(el('div','aw-empty',[market==='USD_M_FUTURES'?'No measurable opportunity with Coin / Long-Short / Leverage / Quantity / Entry / Stop / TP1 / TP2 / TP3 / R/R is available.':'No measurable opportunity with Coin / Buy-Sell / Quantity / Entry / Stop / TP1 / TP2 / TP3 / R/R is available.',market==='USD_M_FUTURES'?'Koin / Long-Short / Kaldıraç / Miktar / Entry / Stop / TP1 / TP2 / TP3 / R/R alanları tam ve ölçülebilir bir fırsat yok.':'Koin / Buy-Sell / Miktar / Entry / Stop / TP1 / TP2 / TP3 / R/R alanları tam ve ölçülebilir bir fırsat yok.'])); candidates.forEach(r=>p.appendChild(detail([[['Setup','Kurulum'],r.setup_name||'—'],[['Observed','Gözlem'],observedTime(r.observed_at)],[['Evidence / blockers','Kanıt / engeller'],(r.blockers||[]).join(' · ')||'—'],[['Record ID','Kayıt kimliği'],r.opportunity_id||'—']],['Observation details','Gözlem ayrıntıları']))); content.appendChild(p); @@ -121,11 +122,11 @@ const marketSelections = {SPOT:'',USD_M_FUTURES:''}; append(trades,el('div','aw-status',['Only journal-backed virtual positions with complete numeric entry, stop-loss, and take-profit levels are displayed.','Yalnızca jurnal kaynaklı ve sayısal giriş, stop-loss, kâr-al seviyeleri tam olan sanal pozisyonlar gösterilir.']));content.appendChild(trades); const daemon=localData?.virtual||{},projection=daemon.dashboard_simulation_projection||{},observations=Array.isArray(projection.symbol_observations)?projection.symbol_observations:[]; const scope=state.simulationScope==='selected'?'selected':'all',selectedObservation=observations.find(item=>item.symbol===marketSelections[market]),manualSimulation=selected.refresh?.simulation||{}; - const simulation=scope==='all'?{state:daemon.status||'NOT_RUN',virtual_decision_status:daemon.virtual_decision_status,candidate_count:daemon.candidate_count,virtual_order_ready:daemon.virtual_order_ready,blockers:daemon.blockers||[],observed_at:daemon.last_success_at,scan_lane:daemon.scan_lane}:{...(selectedObservation||manualSimulation),state:selectedObservation?.state||manualSimulation.state||'PENDING'}; + const simulation=scope==='all'?{state:daemon.virtual_simulation_outcome||projection.status||daemon.status||'NOT_RUN',virtual_decision_status:daemon.virtual_decision_status,virtual_runtime_evaluated:daemon.virtual_runtime_evaluated,virtual_simulation_outcome:daemon.virtual_simulation_outcome,virtual_simulation_allowed:daemon.virtual_simulation_allowed,risk_approved:daemon.risk_approved,candidate_count:daemon.candidate_count,virtual_order_ready:daemon.virtual_order_ready,blockers:daemon.blockers||[],observed_at:daemon.last_success_at,scan_lane:daemon.scan_lane}:{...(selectedObservation||manualSimulation),state:selectedObservation?.virtual_simulation_outcome||selectedObservation?.state||manualSimulation.state||'PENDING'}; const action=panel(['Autonomous simulation action','Otonom simülasyon aksiyonu'],badge(statusLabel(simulation.state||'NOT_RUN'),simulation.state==='COMPLETED'||simulation.state==='RUNNING'?'':'warn')); const allChoice=button(t('All eligible coins','Tüm uygun koinler'),()=>{state.simulationScope='all';render();}),selectedChoice=button(t('Selected coin','Seçili koin'),()=>{state.simulationScope='selected';render();});allChoice.setAttribute('aria-pressed',String(scope==='all'));selectedChoice.setAttribute('aria-pressed',String(scope==='selected'));append(action,append(el('div','aw-toolbar'),allChoice,selectedChoice)); if(scope==='all'){append(action,row(['Coverage','Kapsam'],String(projection.scanned_symbol_count??0)+' / '+String(projection.eligible_symbol_count??'—')),row(['Pending background scans','Bekleyen arka plan taraması'],String(projection.pending_symbol_count??'—')),row(['Current background coin','Güncel arka plan koini'],daemon.symbol||'—'),row(['Scan lane','Tarama hattı'],daemon.scan_lane||'—'),row(['Last full coverage','Son tam kapsam'],observedTime(projection.last_full_coverage_at)));const rows=observations.slice().sort((a,b)=>String(b.observed_at||'').localeCompare(String(a.observed_at||''))).slice(0,100);action.appendChild(table([['Observed','Gözlem'],['Coin','Koin'],['Lane','Hat'],['Cycle','Döngü'],['Decision','Karar'],['Candidates','Aday'],['Virtual order','Sanal emir']],rows.map(item=>[observedTime(item.observed_at),item.symbol||'—',item.scan_lane||'—',item.state||'—',item.virtual_decision_status||'—',String(item.candidate_count??'—'),item.virtual_order_ready===true?t('Ready','Hazır'):t('Not ready','Hazır değil')])));append(action,el('div','aw-status',t('All-coin coverage is bounded to the canonical eligible universe. Records shown: ','Tüm-koin kapsamı kanonik uygun evren ile sınırlıdır. Gösterilen kayıt: ')+rows.length+' / '+observations.length));}else{append(action,row(['Selected coin','Seçili koin'],marketSelections[market]||'—'),row(['Observed','Gözlem'],observedTime(simulation.observed_at||simulation.completed_at)),row(['Scan lane','Tarama hattı'],simulation.scan_lane||'—'));} - append(action,row(['Decision','Karar'],simulation.virtual_decision_status||'—'),row(['Virtual order prepared','Sanal emir hazır'],simulation.virtual_order_ready===true?t('Yes','Evet'):t('No','Hayır')),row(['Analyzed candidates','Analiz edilen aday'],String(simulation.candidate_count??'—'))); + append(action,row(['Decision','Karar'],simulation.virtual_decision_status||'—'),row(['Runtime evaluated','Çalışma zamanı değerlendirildi'],simulation.virtual_runtime_evaluated===true?t('Yes','Evet'):t('No','Hayır')),row(['Risk approved','Risk onaylı'],simulation.risk_approved===true?t('Yes','Evet'):t('No','Hayır')),row(['Virtual order prepared','Sanal emir hazır'],simulation.virtual_order_ready===true?t('Yes','Evet'):t('No','Hayır')),row(['Analyzed candidates','Analiz edilen aday'],String(simulation.candidate_count??'—'))); (simulation.blockers||[]).slice(0,12).forEach(value=>action.appendChild(el('div','aw-status',value))); content.appendChild(action); } diff --git a/src/ai4binance/local_dashboard/server.py.in b/src/ai4binance/local_dashboard/server.py.in index 09c3b326..01a3515e 100644 --- a/src/ai4binance/local_dashboard/server.py.in +++ b/src/ai4binance/local_dashboard/server.py.in @@ -913,6 +913,9 @@ def virtual_simulation_projection(data): "scan_lane", "state", "virtual_decision_status", + "virtual_runtime_evaluated", + "virtual_simulation_outcome", + "risk_approved", "virtual_order_ready", "candidate_count", ), @@ -1246,6 +1249,10 @@ def snapshot(config): "symbol", "scan_lane", "virtual_decision_status", + "virtual_runtime_evaluated", + "virtual_simulation_outcome", + "virtual_simulation_allowed", + "risk_approved", "last_success_at", "virtual_order_ready", ), diff --git a/tests/test_cli.py b/tests/test_cli.py index 798685c7..3648e394 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1854,6 +1854,58 @@ def research_cycle( assert acknowledgement["execution_allowed"] is False +def test_virtual_market_daemon_requests_canonical_refresh_for_stale_data( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from ai4binance.data import market_history_sync + + monkeypatch.setattr( + market_history_sync, "read_cached_market_universe", lambda *_args: None + ) + settings = Settings( + symbol="BTCUSDT", + runtime_state_path=tmp_path / "runtime.json", + ) + + def research_cycle( + cycle_settings: Settings, + _acquisition: SnapshotAcquirer, + *, + cycle_report: dict[str, object] | None = None, + ) -> int: + assert cycle_report is not None + assert cycle_settings.timeframes == ("15m", "1h", "4h") + cycle_report.update( + snapshot_id="fixture:BTCUSDT", + research_blockers=("SNAPSHOT_DATA_QUALITY_INVALID", "STALE_CANDLES:5m"), + virtual_order_ready=False, + ) + return 0 + + monkeypatch.setattr( + runtime_cli, "_run_virtual_market_research_cycle", research_cycle + ) + observed_at = datetime(2026, 9, 24, 13, 30, tzinfo=UTC) + assert ( + runtime_cli.run_virtual_market_daemon( + settings, + max_cycles=1, + public_acquisition=cast(SnapshotAcquirer, object()), + clock=lambda: observed_at, + ) + == 0 + ) + + refresh = json.loads( + (tmp_path / "market-history-refresh-request.json").read_text() + ) + state = json.loads((tmp_path / "virtual-market.json").read_text()) + assert refresh["requester"] == "VIRTUAL_MARKET" + assert refresh["status"] == "PENDING" + assert refresh["execution_allowed"] is False + assert state["market_history_refresh"]["state"] == "PENDING" + + def test_virtual_market_manual_refresh_rejects_authority_drift_and_stale_request( tmp_path: Path, ) -> None: diff --git a/tests/test_config_reporting.py b/tests/test_config_reporting.py index b5fcb2a3..001d9fd8 100644 --- a/tests/test_config_reporting.py +++ b/tests/test_config_reporting.py @@ -42,6 +42,12 @@ def test_settings_normalize_market_type() -> None: Settings(market_type="options") # type: ignore[arg-type] +def test_settings_route_futures_oos_to_the_cli_contract_artifact_root() -> None: + assert Settings().futures_oos_artifact_directory == Path( + "runtime/artifacts/validation/futures_oos" + ) + + def test_settings_expose_local_llm_gpu_controls( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/test_futures_replay.py b/tests/test_futures_replay.py index 9b5650c9..9bc14142 100644 --- a/tests/test_futures_replay.py +++ b/tests/test_futures_replay.py @@ -435,7 +435,9 @@ def test_local_futures_oos_cli_publishes_research_only_evidence( candles=candles, derivatives=_derivatives(candles), ) - replay_path = tmp_path / "runtime" / "datasets" / "futures" / "hotusdt.json" + replay_path = ( + tmp_path / "runtime" / "data" / "datasets" / "futures" / "hotusdt.json" + ) _write_replay(replay_path, dataset) revision = "c" * 40 @@ -477,7 +479,9 @@ def test_local_futures_oos_cli_rejects_tampered_replay_without_artifact( tmp_path: Path, capsys: pytest.CaptureFixture[str], ) -> None: - replay_path = tmp_path / "runtime" / "datasets" / "futures" / "hotusdt.json" + replay_path = ( + tmp_path / "runtime" / "data" / "datasets" / "futures" / "hotusdt.json" + ) _write_replay(replay_path, _dataset()) payload = json.loads(replay_path.read_text(encoding="utf-8")) payload["dataset_sha256"] = "0" * 64 diff --git a/tests/test_local_dashboard_source.py b/tests/test_local_dashboard_source.py index cae984ef..accc2f08 100644 --- a/tests/test_local_dashboard_source.py +++ b/tests/test_local_dashboard_source.py @@ -38,7 +38,7 @@ def test_canonical_dashboard_source_builds_deterministic_offline_assets( assert completed.returncode == 0, completed.stderr assert completed.stdout.strip() == "DASHBOARD_PACKAGE_BUILT" assert _sha256(stage / "app.js") == ( - "a1396a57d098495cbe5ed00f881e35d25c2db3c83183fffba28bcb415c4eff59" + "56c052e1234a1317bf303ed053792c3ec5cc852c5f1b59d24e48415280a5fe31" ) assert _sha256(stage / "app.css") == ( "820ea8899490af3d761e507a88d0af4fd4ecbf587cbd7118c8faa665b5569cc5" @@ -93,6 +93,8 @@ def test_virtual_market_separates_trade_records_from_potential_opportunities() - assert "Entry / Stop / TP1 / TP2 / TP3 / R/R" in views assert "Opportunity generation health" in views assert "rejected_by_reason" in views + assert "CANDIDATES_AVAILABLE_WITH_DATA_GAPS" in views + assert "Published research opportunities remain visible" in views assert "Rejected attempts are diagnostic evidence, not opportunities" in views assert "has_complete_measurable_opportunity" in ( SOURCE / "market_views.py.in" @@ -104,6 +106,21 @@ def test_virtual_market_separates_trade_records_from_potential_opportunities() - assert "def _trade_record_dashboard_row" in wallet +def test_dashboard_exposes_virtual_runtime_preconditions_without_calling_them_active( +) -> None: + server = (SOURCE / "server.py.in").read_text(encoding="utf-8") + views = (SOURCE / "local_views.js").read_text(encoding="utf-8") + + assert '"virtual_runtime_evaluated"' in server + assert '"virtual_simulation_outcome"' in server + assert '"risk_approved"' in server + assert ( + "daemon.virtual_simulation_outcome||projection.status||daemon.status" in views + ) + assert "Runtime evaluated" in views + assert "Risk approved" in views + + def test_dashboard_projects_auto_audit_movements_with_method_provenance() -> None: module = runpy.run_path(str(SOURCE / "server.py.in")) project = module["auto_audit_observer_projection"] diff --git a/tests/test_market_data_gateway.py b/tests/test_market_data_gateway.py index bbb6a459..af2dc4bd 100644 --- a/tests/test_market_data_gateway.py +++ b/tests/test_market_data_gateway.py @@ -204,6 +204,14 @@ def test_gateway_blockers_are_fail_closed_on_invalid_shape() -> None: assert _blockers({"blockers": "unexpected"}) == {"MARKET_GATEWAY_BLOCKERS_INVALID"} +def test_gateway_reuses_the_canonical_continuous_collector_builder() -> None: + source = Path(gateway_cli.__file__).read_text(encoding="utf-8") + + assert "build_continuous_market_history(" in source + assert "MarketDepthCollector(" in source + assert '"market-history-refresh-request.json"' not in source + + def test_gateway_heartbeat_preserves_bootstrap_evidence(tmp_path: Path) -> None: path = tmp_path / "market-history-latest.json" path.write_text( @@ -318,7 +326,11 @@ def sync_cycle(self, *, observed_at: datetime) -> dict[str, object]: del observed_at return {"blockers": ["PUBLIC_MARKET_UNIVERSE_UNAVAILABLE"]} - monkeypatch.setattr(gateway_cli, "ContinuousMarketHistory", _Collector) + monkeypatch.setattr( + gateway_cli, + "build_continuous_market_history", + lambda *_args, **_kwargs: _Collector(), + ) settings = type( "Settings", (), @@ -381,7 +393,11 @@ class _Gateway: async def run_once(self) -> tuple[str, str]: return ("PLANNED_ROLLOVER", "PLANNED_ROLLOVER") - monkeypatch.setattr(gateway_cli, "ContinuousMarketHistory", _Collector) + monkeypatch.setattr( + gateway_cli, + "build_continuous_market_history", + lambda *_args, **_kwargs: _Collector(), + ) monkeypatch.setattr(gateway_cli, "SingleInstanceLease", _Lease) monkeypatch.setattr( gateway_cli, @@ -466,7 +482,11 @@ async def run_once(self) -> tuple[str, str]: raise OSError("temporary") return ("PLANNED_ROLLOVER", "PLANNED_ROLLOVER") - monkeypatch.setattr(gateway_cli, "ContinuousMarketHistory", _Collector) + monkeypatch.setattr( + gateway_cli, + "build_continuous_market_history", + lambda *_args, **_kwargs: _Collector(), + ) monkeypatch.setattr(gateway_cli, "SingleInstanceLease", _Lease) fake_time = type("Time", (), {"sleep": staticmethod(lambda _seconds: None)})() monkeypatch.setattr(gateway_cli, "time", fake_time) @@ -534,8 +554,8 @@ def __exit__(self, *args: object) -> None: ) monkeypatch.setattr( gateway_cli, - "ContinuousMarketHistory", - lambda **_kwargs: object(), + "build_continuous_market_history", + lambda *_args, **_kwargs: object(), ) settings = type( "Settings", @@ -599,7 +619,11 @@ def sync_cycle(self, *, observed_at: datetime) -> dict[str, object]: return {"blockers": ["MARKET_DATA_BACKFILL_PENDING"]} return {"blockers": []} - monkeypatch.setattr(gateway_cli, "ContinuousMarketHistory", _Collector) + monkeypatch.setattr( + gateway_cli, + "build_continuous_market_history", + lambda *_args, **_kwargs: _Collector(), + ) monkeypatch.setattr(gateway_cli, "SingleInstanceLease", _Lease) monkeypatch.setattr( gateway_cli, diff --git a/tests/test_market_history_boundary_contracts.py b/tests/test_market_history_boundary_contracts.py index 349cc3ca..5f262423 100644 --- a/tests/test_market_history_boundary_contracts.py +++ b/tests/test_market_history_boundary_contracts.py @@ -660,7 +660,7 @@ def test_metered_transport_updates_budget_and_funding_clock( [ ("CURRENT", "CANDIDATES_AVAILABLE"), ("DELEGATED", "ANALYSIS_UNAVAILABLE"), - ("DATA_BLOCKED", "DATA_UNAVAILABLE"), + ("DATA_BLOCKED", "CANDIDATES_AVAILABLE_WITH_DATA_GAPS"), ("BLOCKED", "ANALYSIS_BLOCKED"), ], ) diff --git a/tests/test_market_history_continuous.py b/tests/test_market_history_continuous.py index a9631f4d..cf5d76d2 100644 --- a/tests/test_market_history_continuous.py +++ b/tests/test_market_history_continuous.py @@ -25,6 +25,7 @@ from ai4binance.core.errors import ExchangeHttpError, ExchangeTransportError from ai4binance.data.archive import ParquetOHLCVArchive from ai4binance.data.market_history_continuous import ( + VIRTUAL_MARKET_COLLECTION_TIMEFRAMES, ContinuousMarketHistory, MeteredPublicTransport, PublicRequestBudget, @@ -34,7 +35,6 @@ market_history_refresh_status, ) from ai4binance.data.market_history_sync import ( - MARKET_HISTORY_TIMEFRAMES, BinanceVisionArchiveCache, MarketHistorySynchronizer, ) @@ -175,8 +175,8 @@ def analyze(market: str, symbol: str, _now: datetime) -> Mapping[str, object]: instance.on_symbol_screen = screen instance.on_symbol_ready = analyze result = instance.sync_cycle(observed_at=NOW) - assert len(calls) == 20 - assert {tf for _, _, tf in calls} == set(MARKET_HISTORY_TIMEFRAMES) + assert len(calls) == 12 + assert {tf for _, _, tf in calls} == set(VIRTUAL_MARKET_COLLECTION_TIMEFRAMES) assert len(calls) == len(set(calls)) assert sorted(analyzed) == [ ("SPOT", "BTCUSDT"), @@ -184,7 +184,7 @@ def analyze(market: str, symbol: str, _now: datetime) -> Mapping[str, object]: ("USD_M_FUTURES", "BTCUSDT"), ("USD_M_FUTURES", "ETHUSDT"), ] - assert result["total_streams"] == result["completed_streams"] == 20 + assert result["total_streams"] == result["completed_streams"] == 12 assert result["completed_symbols"] == 4 assert result["execution_allowed"] is False @@ -210,7 +210,7 @@ def screen(*_args: object) -> Mapping[str, object]: monkeypatch.setattr(instance, "_collect_stream", collect) instance.on_symbol_screen = screen instance.sync_cycle(observed_at=NOW) - assert calls == list(MARKET_HISTORY_TIMEFRAMES) + assert calls == list(VIRTUAL_MARKET_COLLECTION_TIMEFRAMES) def test_existing_archive_fetches_only_holes_then_tail_without_replaying_rows( @@ -297,7 +297,7 @@ def refresh( *, timeframes: tuple[str, ...] | None = None, ) -> dict[str, object]: - assert timeframes == MARKET_HISTORY_TIMEFRAMES + assert timeframes == VIRTUAL_MARKET_COLLECTION_TIMEFRAMES lock = ( root / "runtime/artifacts/opportunity-radar/monitor/USD_M_FUTURES" @@ -364,11 +364,11 @@ def collect( "live_eligibility_status": "LIVE_ORDER_BLOCKED", } result = instance.sync_cycle(observed_at=NOW) - assert calls == list(MARKET_HISTORY_TIMEFRAMES) + assert calls == list(VIRTUAL_MARKET_COLLECTION_TIMEFRAMES) assert ( result["completed_streams"] == result["total_streams"] - == len(MARKET_HISTORY_TIMEFRAMES) + == len(VIRTUAL_MARKET_COLLECTION_TIMEFRAMES) ) @@ -426,16 +426,18 @@ def test_resume_fetches_native_timeframes_without_local_materialization( assert all(row["network_download"] is True for row in refresh_rows) assert {row["closed_history_source"] for row in refresh_rows} == { f"BINANCE_VISION_{timeframe.upper()}_DIRECT" - for timeframe in MARKET_HISTORY_TIMEFRAMES + for timeframe in VIRTUAL_MARKET_COLLECTION_TIMEFRAMES } - progress = tmp_path / "market/spot/BTCUSDT/5m/collection-progress.json" + progress = tmp_path / "market/spot/BTCUSDT/15m/collection-progress.json" resumed = collector(tmp_path, transport) second = resumed.sync_cycle(observed_at=NOW) assert second["status"] == "READY" klines = [params for path, params in transport.calls if path.endswith("klines")] - assert {params["interval"] for params in klines} == set(MARKET_HISTORY_TIMEFRAMES) + assert {params["interval"] for params in klines} == set( + VIRTUAL_MARKET_COLLECTION_TIMEFRAMES + ) archive = ParquetOHLCVArchive(tmp_path / "market/spot") - for timeframe in MARKET_HISTORY_TIMEFRAMES: + for timeframe in VIRTUAL_MARKET_COLLECTION_TIMEFRAMES: assert archive.manifest("BTCUSDT", timeframe).gap_count == 0 before = len(klines) resumed.sync_cycle(observed_at=NOW) @@ -544,13 +546,15 @@ def collect( report = instance.sync_cycle(observed_at=NOW) assert markets[:1] == ["spot"] - assert markets.count("spot") == 5 - assert markets.count("usd_m_futures") == 9 + assert markets.count("spot") == len(VIRTUAL_MARKET_COLLECTION_TIMEFRAMES) + assert markets.count("usd_m_futures") == ( + len(VIRTUAL_MARKET_COLLECTION_TIMEFRAMES) + 4 + ) assert snapshot_markets == (["spot", "usd_m_futures"] if long_backfill else []) assert report["completed_symbols"] == 2 assert report["total_symbols"] == 2 - assert report["completed_streams"] == 14 - assert report["total_streams"] == 14 + assert report["completed_streams"] == 10 + assert report["total_streams"] == 10 assert report["completion_ratio"] == "1.000000" assert ready_symbols == [("SPOT", "ETHUSDT"), ("USD_M_FUTURES", "BTCUSDT")] assert report["opportunity_analysis_summary"] == {"CURRENT": 2} @@ -571,7 +575,9 @@ def collect( assert isinstance(collector_coverage, Mapping) spot_coverage = collector_coverage["SPOT"] assert isinstance(spot_coverage, list) - assert {row["timeframe"] for row in spot_coverage} == set(MARKET_HISTORY_TIMEFRAMES) + assert {row["timeframe"] for row in spot_coverage} == set( + VIRTUAL_MARKET_COLLECTION_TIMEFRAMES + ) assert all(row["current_count"] == 1 for row in spot_coverage) assert all(row["pending_count"] == 0 for row in spot_coverage) assert universe_provider.calls == (2 if long_backfill else 1) @@ -616,11 +622,11 @@ def test_stream_plan_uses_each_native_price_candle_feed() -> None: futures_streams = ContinuousMarketHistory._collection_kinds("usd_m_futures") assert spot_streams == tuple( - ("klines", timeframe) for timeframe in MARKET_HISTORY_TIMEFRAMES + ("klines", timeframe) for timeframe in VIRTUAL_MARKET_COLLECTION_TIMEFRAMES ) - assert futures_streams[:5] == spot_streams - assert len(spot_streams) == 5 - assert len(futures_streams) == 9 + assert futures_streams[: len(spot_streams)] == spot_streams + assert len(spot_streams) == len(VIRTUAL_MARKET_COLLECTION_TIMEFRAMES) + assert len(futures_streams) == len(VIRTUAL_MARKET_COLLECTION_TIMEFRAMES) + 4 def test_priority_symbols_precede_background_backfill(tmp_path: Path) -> None: @@ -646,17 +652,17 @@ def test_priority_symbols_precede_background_backfill(tmp_path: Path) -> None: for market, symbol, _, kind, timeframe in streams[:6] ] assert leading_streams == [ - ("spot", "HOTUSDT", "klines", "5m"), ("spot", "HOTUSDT", "klines", "15m"), ("spot", "HOTUSDT", "klines", "1h"), ("spot", "HOTUSDT", "klines", "4h"), - ("spot", "HOTUSDT", "klines", "1d"), - ("usd_m_futures", "BTCUSDT", "klines", "5m"), + ("usd_m_futures", "BTCUSDT", "klines", "15m"), + ("usd_m_futures", "BTCUSDT", "klines", "1h"), + ("usd_m_futures", "BTCUSDT", "klines", "4h"), ] - assert len(streams) == 19 + assert len(streams) == 13 -def test_priority_depth_scope_does_not_subscribe_the_full_universe() -> None: +def test_priority_depth_scope_covers_the_bounded_active_universe() -> None: class DepthUniverse: spot_symbols = ("HOTUSDT", "ETHUSDT") futures_symbols = ("BTCUSDT", "ETHUSDT") @@ -667,13 +673,14 @@ class DepthUniverse: ("HOTUSDT", "BTCUSDT", "MISSING"), include_coin_m=True, ) == { - "spot": ("HOTUSDT",), - "usd_m_futures": ("BTCUSDT",), + "spot": ("HOTUSDT", "ETHUSDT"), + "usd_m_futures": ("BTCUSDT", "ETHUSDT"), "coin_m_futures": (), } -def test_priority_depth_scope_falls_back_to_top_volume_per_primary_market() -> None: +def test_priority_depth_scope_preserves_top_volume_order_without_watchlist_match( +) -> None: class DepthUniverse: spot_symbols = ("ETHUSDT", "BTCUSDT") futures_symbols = ("BTCUSDT", "ETHUSDT") @@ -684,8 +691,8 @@ class DepthUniverse: ("HOTUSDT",), include_coin_m=False, ) == { - "spot": ("ETHUSDT",), - "usd_m_futures": ("BTCUSDT",), + "spot": ("ETHUSDT", "BTCUSDT"), + "usd_m_futures": ("BTCUSDT", "ETHUSDT"), } @@ -712,8 +719,8 @@ def analyze(*_args: object) -> Mapping[str, object]: instance.sync_cycle(observed_at=NOW) - assert calls == list(MARKET_HISTORY_TIMEFRAMES) - assert analyzed_after == [5] + assert calls == list(VIRTUAL_MARKET_COLLECTION_TIMEFRAMES) + assert analyzed_after == [len(VIRTUAL_MARKET_COLLECTION_TIMEFRAMES)] def test_native_streams_are_the_canonical_dashboard_input( @@ -728,7 +735,7 @@ def collect( *_args: object, timeframe: str | None, **_kwargs: object ) -> dict[str, object]: started.append(timeframe) - if timeframe == "1d": + if timeframe == "4h": native_streams_started.set() return {"status": "CURRENT"} @@ -736,7 +743,7 @@ def collect( instance.sync_cycle(observed_at=NOW) assert native_streams_started.is_set() - assert started == list(MARKET_HISTORY_TIMEFRAMES) + assert started == list(VIRTUAL_MARKET_COLLECTION_TIMEFRAMES) def test_background_stream_plan_finishes_each_symbol_before_the_next( @@ -754,9 +761,10 @@ def test_background_stream_plan_finishes_each_symbol_before_the_next( ) assert [(market, symbol) for market, symbol, *_ in streams] == [ - *(("spot", "AUSDT"),) * len(MARKET_HISTORY_TIMEFRAMES), - *(("spot", "BUSDT"),) * len(MARKET_HISTORY_TIMEFRAMES), - *(("usd_m_futures", "CUSDT"),) * (len(MARKET_HISTORY_TIMEFRAMES) + 4), + *(("spot", "AUSDT"),) * len(VIRTUAL_MARKET_COLLECTION_TIMEFRAMES), + *(("spot", "BUSDT"),) * len(VIRTUAL_MARKET_COLLECTION_TIMEFRAMES), + *(("usd_m_futures", "CUSDT"),) + * (len(VIRTUAL_MARKET_COLLECTION_TIMEFRAMES) + 4), ] @@ -777,7 +785,9 @@ def observe_incomplete_symbol( instance, "_collect_stream", lambda *_args, **kwargs: { - "status": "BACKFILLING" if kwargs.get("timeframe") == "5m" else "CURRENT" + "status": "BACKFILLING" + if kwargs.get("timeframe") == "15m" + else "CURRENT" }, ) @@ -833,7 +843,7 @@ def fail(*_args: object, **_kwargs: object) -> dict[str, object]: failures = report["stream_failure_summary"] assert isinstance(failures, dict) - assert sum(failures.values()) == len(MARKET_HISTORY_TIMEFRAMES) + assert sum(failures.values()) == len(VIRTUAL_MARKET_COLLECTION_TIMEFRAMES) assert all("PROGRESS_AHEAD_OF_VERIFIED_DATASET" in key for key in failures) assert "secret" not in (tmp_path / "state.json").read_text(encoding="utf-8") @@ -851,7 +861,7 @@ def refresh(*_args: object, **kwargs: object) -> dict[str, object]: "timeframe": timeframe, "status": "CURRENT", } - for timeframe in MARKET_HISTORY_TIMEFRAMES + for timeframe in VIRTUAL_MARKET_COLLECTION_TIMEFRAMES ], "candidates": [ { @@ -928,7 +938,7 @@ def refresh(*_args: object, **kwargs: object) -> dict[str, object]: "now": NOW, "minimum_candles": 200, "candle_limit": 250, - "timeframes": MARKET_HISTORY_TIMEFRAMES, + "timeframes": VIRTUAL_MARKET_COLLECTION_TIMEFRAMES, } ] assert futures["status"] == "DELEGATED" @@ -968,6 +978,27 @@ def test_dashboard_refresh_request_is_coalesced_and_other_symbol_is_busy( assert busy["live_eligibility_status"] == "LIVE_ORDER_BLOCKED" +def test_virtual_market_refresh_request_preserves_single_writer_safety( + tmp_path: Path, +) -> None: + path = tmp_path / "market-history-refresh-request.json" + + request = enqueue_market_history_refresh_request( + path, + market="SPOT", + symbol="BTCUSDT", + eligible_symbols=("BTCUSDT",), + requested_at=NOW, + requester="VIRTUAL_MARKET", + ) + status = market_history_refresh_status(path) + + assert request["state"] == "PENDING" + assert status["requester"] == "VIRTUAL_MARKET" + assert status["execution_allowed"] is False + assert status["live_eligibility_status"] == "LIVE_ORDER_BLOCKED" + + def test_dashboard_refresh_request_is_completed_by_the_canonical_collector( tmp_path: Path, ) -> None: @@ -996,6 +1027,35 @@ def test_dashboard_refresh_request_is_completed_by_the_canonical_collector( assert status["live_eligibility_status"] == "LIVE_ORDER_BLOCKED" +def test_refresh_request_completion_never_predates_its_request( + tmp_path: Path, +) -> None: + instance = collector(tmp_path, Transport()) + request_path = (tmp_path / "market-history-refresh-request.json").resolve() + instance.refresh_request_path = request_path + requested_at = NOW + timedelta(minutes=10) + request = enqueue_market_history_refresh_request( + request_path, + market="SPOT", + symbol="BTCUSDT", + eligible_symbols=("BTCUSDT",), + requested_at=requested_at, + ) + instance.clock = lambda: NOW + + instance._complete_refresh_request( + request, + NOW, + status="DATA_READY", + blockers=(), + ) + + completed = datetime.fromisoformat( + str(_load(request_path)["completed_at"]) + ) + assert completed >= requested_at + + def test_dashboard_request_preempts_the_bounded_background_queue( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -1136,7 +1196,7 @@ def test_invalid_pages_fail_closed(fault: str) -> None: def test_corrupt_progress_is_reported_without_reset(tmp_path: Path) -> None: transport = Transport() instance = collector(tmp_path, transport) - progress = tmp_path / "market/spot/BTCUSDT/5m/collection-progress.json" + progress = tmp_path / "market/spot/BTCUSDT/15m/collection-progress.json" _save( progress, { @@ -1148,7 +1208,7 @@ def test_corrupt_progress_is_reported_without_reset(tmp_path: Path) -> None: assert isinstance(report["blockers"], list) assert "MARKET_DATA_SOURCE_OR_INTEGRITY_FAILURE" in report["blockers"] assert not any( - path.endswith("klines") and params["interval"] == "5m" + path.endswith("klines") and params["interval"] == "15m" for path, params in transport.calls ) @@ -1829,6 +1889,37 @@ def test_dashboard_candidate_projection_filters_and_sanitizes_fields() -> None: ] +def test_dashboard_candidate_projection_preserves_tuple_blockers() -> None: + from ai4binance.cli.market_data import _dashboard_candidate_projection + + candidates: list[object] = [ + { + "market": "SPOT", + "symbol": "BTCUSDT", + "execution_allowed": False, + "live_eligibility_status": "LIVE_ORDER_BLOCKED", + "direction": "BULLISH", + "side": "BUY", + "quantity": "1", + "entry": "101", + "stop_loss": "98", + "tp1": "105", + "tp2": "108", + "tp3": "111", + "target_risk_reward": "2", + "blockers": ("RESEARCH_ONLY",), + } + ] + + projected = _dashboard_candidate_projection( + candidates, market="SPOT", symbol="BTCUSDT" + ) + + assert projected[0]["blockers"] == ["RESEARCH_ONLY"] + assert projected[0]["execution_allowed"] is False + assert projected[0]["live_eligibility_status"] == "LIVE_ORDER_BLOCKED" + + def test_canonical_opportunity_pipeline_validates_time_and_busy_lease( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -1882,7 +1973,7 @@ def test_canonical_opportunity_pipeline_fails_closed_for_malformed_payloads( assert result["candidate_count"] == 0 assert result["blockers"] == [ f"OPPORTUNITY_DATA_UNAVAILABLE:{timeframe}" - for timeframe in MARKET_HISTORY_TIMEFRAMES + for timeframe in VIRTUAL_MARKET_COLLECTION_TIMEFRAMES ] diff --git a/tests/test_service_manifest.py b/tests/test_service_manifest.py index 772aba80..34d115d8 100644 --- a/tests/test_service_manifest.py +++ b/tests/test_service_manifest.py @@ -41,7 +41,7 @@ def test_service_manifest_is_shared_safe_and_complete() -> None: assert required["skill-discovery"].lock_file == "skill_discovery.lock" assert required["market-history"].task_name == "AI4BINANCE-Market-History" assert required["market-history"].command == ( - "python -m ai4binance.cli.market_gateway" + "python -m ai4binance.cli.market_data daemon" ) scheduled = {spec.service: spec for spec in specs if not spec.required} assert scheduled["ykb-report"].health_mode == "SCHEDULED" From 3f3af9c2ad729bd5e48061d5e8c6a77ab44a92cd Mon Sep 17 00:00:00 2001 From: Huseyin Cicek Date: Fri, 25 Sep 2026 01:23:00 +0300 Subject: [PATCH 05/19] Harden market data collection and runtime diagnostics --- scripts/install_startup_task.ps1 | 48 +++++++++++- src/ai4binance/application/research.py | 32 +++++++- src/ai4binance/cli/market_data.py | 23 +++++- src/ai4binance/cli/market_gateway.py | 12 +-- src/ai4binance/cli/runtime.py | 43 ++++++++-- src/ai4binance/data/acquisition.py | 10 ++- src/ai4binance/data/market_depth.py | 32 +++++--- .../data/market_history_continuous.py | 34 ++++++-- src/ai4binance/data/market_history_sync.py | 35 ++++++++- src/ai4binance/governance/adapters.py | 78 +++++++++++++------ src/ai4binance/governance/dge_models.py | 29 +++++-- .../infrastructure/persistence/safe_json.py | 7 +- .../storage/destination_verification.py | 7 +- tests/test_data_acquisition.py | 12 ++- tests/test_dge_engine.py | 21 +++++ tests/test_dge_recovery_replay_shadow.py | 3 +- ...est_coverage_persistence_and_governance.py | 8 ++ tests/test_market_data_gateway.py | 2 +- tests/test_market_depth.py | 40 +++++++++- .../test_market_history_boundary_contracts.py | 6 +- tests/test_market_history_continuous.py | 43 +++++++++- tests/test_market_history_sync.py | 67 ++++++++++++++++ tests/test_research_application.py | 2 + tests/test_service_manifest.py | 4 + tests/test_storage.py | 18 +++++ 25 files changed, 528 insertions(+), 88 deletions(-) diff --git a/scripts/install_startup_task.ps1 b/scripts/install_startup_task.ps1 index 6a6facae..81f6ac13 100644 --- a/scripts/install_startup_task.ps1 +++ b/scripts/install_startup_task.ps1 @@ -47,8 +47,37 @@ function Rotate-LogFile { if (Test-Path -LiteralPath $source -PathType Leaf) { if ((Get-Item -LiteralPath $source).Length -gt $MaximumBytes) { $trimmed = "$source.trimmed" - Get-Content -LiteralPath $source -Tail 20000 | - Set-Content -LiteralPath $trimmed -Encoding UTF8 + $stream = [System.IO.File]::Open( + $source, + [System.IO.FileMode]::Open, + [System.IO.FileAccess]::Read, + [System.IO.FileShare]::ReadWrite + ) + try { + $byteCount = [int][Math]::Min($MaximumBytes, $stream.Length) + [void]$stream.Seek(-$byteCount, [System.IO.SeekOrigin]::End) + $buffer = [byte[]]::new($byteCount) + $readCount = $stream.Read($buffer, 0, $byteCount) + } + finally { + $stream.Dispose() + } + $start = 0 + if ($readCount -lt (Get-Item -LiteralPath $source).Length) { + while ($start -lt $readCount -and $buffer[$start] -ne 10) { + $start++ + } + if ($start -lt $readCount) { + $start++ + } + } + $kept = if ($start -lt $readCount) { + [byte[]]$buffer[$start..($readCount - 1)] + } + else { + [byte[]]::new(0) + } + [System.IO.File]::WriteAllBytes($trimmed, $kept) Move-Item -LiteralPath $trimmed -Destination $source -Force } Move-Item -LiteralPath $source -Destination $target -Force @@ -115,8 +144,19 @@ function Invoke-ServicePython { } -ArgumentList $HealthPath, $Service, $PID Push-Location -LiteralPath $root try { - & $python -B @Arguments 1>> $StdoutPath 2>> $StderrPath - return [int]$LASTEXITCODE + $previousErrorActionPreference = $ErrorActionPreference + try { + # Windows PowerShell 5 surfaces native stderr as ErrorRecord objects. + # Service diagnostics must be logged without terminating a healthy + # resident Python process; its real exit code remains authoritative. + $ErrorActionPreference = "Continue" + & $python -B @Arguments 1>> $StdoutPath 2>> $StderrPath + $nativeExitCode = [int]$LASTEXITCODE + } + finally { + $ErrorActionPreference = $previousErrorActionPreference + } + return $nativeExitCode } finally { Pop-Location diff --git a/src/ai4binance/application/research.py b/src/ai4binance/application/research.py index 34e6749e..1c125a10 100644 --- a/src/ai4binance/application/research.py +++ b/src/ai4binance/application/research.py @@ -110,6 +110,31 @@ def evaluate( ) -> VirtualGovernanceResult: ... +def _dge_error_type(error: Exception) -> str: + """Return a bounded diagnostic class without exposing exception detail.""" + + stage_codes = { + "DGE_CANDIDATE_ADAPTATION_FAILED": "CANDIDATE_ADAPTATION", + "DGE_CONTEXT_BUILD_FAILED": "CONTEXT_BUILD", + "DGE_CONTEXT_TYPE_INVALID": "CONTEXT_TYPE", + "DGE_CONTEXT_SURFACE_INVALID": "CONTEXT_SURFACE", + "DGE_ENGINE_EVALUATION_FAILED": "ENGINE_EVALUATION", + } + if str(error) in stage_codes: + return stage_codes[str(error)] + if isinstance(error, ArithmeticError): + return "ARITHMETIC_ERROR" + if isinstance(error, AttributeError): + return "ATTRIBUTE_ERROR" + if isinstance(error, KeyError): + return "KEY_ERROR" + if isinstance(error, RuntimeError): + return "RUNTIME_ERROR" + if isinstance(error, TypeError): + return "TYPE_ERROR" + return "VALUE_ERROR" + + class BalanceLike(Protocol): asset: str free: object @@ -1183,11 +1208,14 @@ def _evaluate_virtual_governance( RuntimeError, TypeError, ValueError, - ): + ) as error: return ( fallback_id, DGE_DATA_UNAVAILABLE, - (DGE_EVALUATION_FAILED,), + ( + DGE_EVALUATION_FAILED, + f"DGE_EVALUATION_ERROR_TYPE:{_dge_error_type(error)}", + ), False, ) return ( diff --git a/src/ai4binance/cli/market_data.py b/src/ai4binance/cli/market_data.py index 0938f20d..58937ff2 100644 --- a/src/ai4binance/cli/market_data.py +++ b/src/ai4binance/cli/market_data.py @@ -45,6 +45,7 @@ "promotion_status": "RESEARCH_ONLY", "live_eligibility_status": "LIVE_ORDER_BLOCKED", } +_DEPTH_SYMBOLS_PER_CONNECTION = 10 _DASHBOARD_CANDIDATE_FIELDS = ( "opportunity_id", "observed_at", @@ -139,6 +140,22 @@ def selected(symbols: tuple[str, ...]) -> tuple[str, ...]: return markets +def build_market_depth_collector( + synchronizer: MarketHistorySynchronizer, + continuous: ContinuousMarketHistory, +) -> MarketDepthCollector: + """Build bounded shards so one reconnect cannot invalidate a full universe.""" + + transports = {"spot": continuous.spot, "usd_m_futures": continuous.futures} + if continuous.coin_m is not None: + transports["coin_m_futures"] = continuous.coin_m + return MarketDepthCollector( + synchronizer.archive_root / "depth", + transports, + symbols_per_connection=_DEPTH_SYMBOLS_PER_CONNECTION, + ) + + def build_continuous_market_history( settings: Settings, synchronizer: MarketHistorySynchronizer, @@ -409,10 +426,7 @@ def run_market_history_command( return 0 if not report.blockers else 2 if command == "market-history-daemon": - transports = {"spot": continuous.spot, "usd_m_futures": continuous.futures} - if continuous.coin_m is not None: - transports["coin_m_futures"] = continuous.coin_m - depth = MarketDepthCollector(synchronizer.archive_root / "depth", transports) + depth = build_market_depth_collector(synchronizer, continuous) def cycle(now: datetime) -> object: if settings.market_depth_enabled: @@ -593,6 +607,7 @@ def main(arguments: Sequence[str] | None = None) -> int: __all__ = ( "build_continuous_market_history", + "build_market_depth_collector", "build_market_history_synchronizer", "main", "run_market_history_command", diff --git a/src/ai4binance/cli/market_gateway.py b/src/ai4binance/cli/market_gateway.py index 42754449..041c36a7 100644 --- a/src/ai4binance/cli/market_gateway.py +++ b/src/ai4binance/cli/market_gateway.py @@ -15,11 +15,11 @@ from ai4binance.cli.market_data import ( _priority_depth_markets, build_continuous_market_history, + build_market_depth_collector, build_market_history_synchronizer, ) from ai4binance.config import Settings from ai4binance.data.market_data_gateway import MarketStreamGapError, build_gateway -from ai4binance.data.market_depth import MarketDepthCollector from ai4binance.infrastructure.persistence.safe_json import write_json_object_verified from ai4binance.ops.runtime import SingleInstanceLease @@ -83,15 +83,9 @@ def run_gateway(settings: Settings, *, max_cycles: int | None = None) -> int: root=Path.cwd(), include_coin_m=False, ) - depth: MarketDepthCollector | None = None + depth = None if getattr(settings, "market_depth_enabled", False): - depth = MarketDepthCollector( - synchronizer.archive_root / "depth", - { - "spot": collector.spot, - "usd_m_futures": collector.futures, - }, - ) + depth = build_market_depth_collector(synchronizer, collector) lock_path = _absolute(settings.market_history_state_path).with_suffix(".lock") heartbeat = _GatewayStateHeartbeat(_absolute(settings.market_history_state_path)) completed = 0 diff --git a/src/ai4binance/cli/runtime.py b/src/ai4binance/cli/runtime.py index a0c3cae8..241d736a 100644 --- a/src/ai4binance/cli/runtime.py +++ b/src/ai4binance/cli/runtime.py @@ -119,6 +119,7 @@ _MARKET_HISTORY_REFRESH_REQUEST_NAME = "market-history-refresh-request.json" _DASHBOARD_SIMULATION_MAX_SYMBOLS = 1_000 _DASHBOARD_SIMULATION_MAX_BLOCKERS = 12 +_VIRTUAL_MARKET_UNIVERSE_LIMIT = 50 _NEWS_ASSET_ALIASES = { "BTC": ("bitcoin",), "ETH": ("ethereum", "ether"), @@ -591,7 +592,14 @@ def run_virtual_market_daemon( if name not in configured ) if universe is not None - else configured + else ( + _virtual_market_ranked_symbols( + settings, + configured, + clock(), + ) + or configured + ) ) from ai4binance.exchange.client import BinancePublicClient @@ -670,9 +678,13 @@ def run_virtual_market_daemon( "priority_symbol": priority_symbol, "discovery_symbol": discovery_symbol, "priority_symbol_count": len(priority_symbols), - "universe_source": "CANONICAL_CACHE" - if universe is not None - else "CONFIGURED_FALLBACK", + "universe_source": ( + "CANONICAL_CACHE" + if universe is not None + else "LOCAL_SNAPSHOT" + if symbols != configured + else "CONFIGURED_FALLBACK" + ), } ) exit_code = _run_virtual_market_research_cycle( @@ -839,6 +851,25 @@ def _virtual_market_priority_symbols( observed_at: datetime, ) -> tuple[str, ...]: """Schedule canonical liquidity priorities using shared public files only.""" + + return tuple( + symbol + for symbol in _virtual_market_ranked_symbols( + settings, + configured, + observed_at, + ) + if symbol in eligible + )[: settings.virtual_market_priority_symbol_count] + + +def _virtual_market_ranked_symbols( + settings: Settings, + configured: tuple[str, ...], + observed_at: datetime, +) -> tuple[str, ...]: + """Read the bounded Spot universe from the current canonical snapshots.""" + from ai4binance.data.acquisition import LocalMarketSnapshotTransport from ai4binance.integrations.binance.market_universe_provider import ( BinanceMarketUniverseProvider, @@ -853,14 +884,14 @@ def _virtual_market_priority_symbols( ranked = BinanceMarketUniverseProvider( local, local, - max_symbols_per_market=settings.virtual_market_priority_symbol_count, + max_symbols_per_market=_VIRTUAL_MARKET_UNIVERSE_LIMIT, ).spot_symbols(configured) except (ExchangeError, OSError, TypeError, ValueError): return () return tuple( item.symbol for item in ranked - if item.symbol in eligible and item.data_quality_ok and item.status == "TRADING" + if item.data_quality_ok and item.status == "TRADING" ) diff --git a/src/ai4binance/data/acquisition.py b/src/ai4binance/data/acquisition.py index 12539652..31b09601 100644 --- a/src/ai4binance/data/acquisition.py +++ b/src/ai4binance/data/acquisition.py @@ -115,6 +115,7 @@ class DataAcquisitionAgent: max_workers: int = 4 archive: ParquetOHLCVArchive | None = None depth_path: Path | None = None + clock: Callable[[], datetime] = lambda: datetime.now(UTC) def __post_init__(self) -> None: if not is_spot_market_type(self.market_type): @@ -145,6 +146,9 @@ def acquire( latest_price = self.client.ticker_price(symbol_info.symbol) book = self.client.book_ticker(symbol_info.symbol) raw_klines = self._fetch_klines(symbol_info.symbol, timeframes) + observed_at = self.clock() if self.depth_path is not None else server_time + if observed_at.utcoffset() is None or observed_at < server_time: + raise ValueError("data acquisition clock must follow server time") candles_by_timeframe: dict[str, tuple[OHLCVCandle, ...]] = {} freshness: dict[str, dict[str, object]] = {} @@ -186,13 +190,13 @@ def acquire( ) snapshot_id = self._snapshot_id( symbol_info.symbol, - server_time, + observed_at, latest_price, last_close_times, ) return MarketSnapshot( snapshot_id=snapshot_id, - created_at=server_time, + created_at=observed_at, exchange="Binance", market_type=self.market_type, symbol=symbol_info.symbol, @@ -206,7 +210,7 @@ def acquire( "best_bid": str(book.bid), "best_ask": str(book.ask), "spread": str(book.spread), - **self._local_depth_summary(symbol_info.symbol, server_time), + **self._local_depth_summary(symbol_info.symbol, observed_at), }, exchange_filters={ name: dict(values) for name, values in symbol_info.filters.items() diff --git a/src/ai4binance/data/market_depth.py b/src/ai4binance/data/market_depth.py index 6a0499cc..0de54fbc 100644 --- a/src/ai4binance/data/market_depth.py +++ b/src/ai4binance/data/market_depth.py @@ -35,6 +35,7 @@ "usd_m_futures": "wss://fstream.binance.com/public/stream", "coin_m_futures": "wss://dstream.binance.com/stream", } +_DEPTH_COMPACTION_BATCH_SIZE = 50_000 DepthRecord = tuple[str, str, str, dict[str, object], float] @@ -162,6 +163,18 @@ def append(self, records: list[DepthRecord]) -> None: """, (market, symbol, checkpoint, seq, status, received), ) + if checkpoint is not None: + self.connection.execute( + "DELETE FROM depth_events WHERE seq IN (" + "SELECT seq FROM depth_events WHERE market=? AND symbol=? " + "AND seq None: with self.lock: @@ -414,7 +427,6 @@ def _connection(self, market: str, symbols: tuple[str, ...], group: str) -> None last_flush, started = time.monotonic(), time.monotonic() executor = ThreadPoolExecutor(max_workers=1) received_bytes = 0 - subscribed_at = 0.0 def resync(symbol: str) -> None: """Invalidate only the broken book and queue its bounded resnapshot.""" @@ -454,16 +466,14 @@ def resync(symbol: str) -> None: maximum_levels=100_000, futures_sequence=market != "spot", ) - subscribed_at = time.monotonic() - if pending and future is None and time.monotonic() - subscribed_at > 20: - future = executor.submit( - self.transports[market].get_json, - _prefix(market) + "depth", - { - "symbol": pending, - "limit": 5000 if market == "spot" else 1000, - }, - ) + future = executor.submit( + self.transports[market].get_json, + _prefix(market) + "depth", + { + "symbol": pending, + "limit": 5000 if market == "spot" else 1000, + }, + ) if future is not None and future.done(): raw = future.result() if pending is None or not isinstance(raw, dict): diff --git a/src/ai4binance/data/market_history_continuous.py b/src/ai4binance/data/market_history_continuous.py index 3ddd0620..b0f85b0d 100644 --- a/src/ai4binance/data/market_history_continuous.py +++ b/src/ai4binance/data/market_history_continuous.py @@ -36,7 +36,10 @@ WeightedRateLimitGovernor, public_request_weight, ) -from ai4binance.infrastructure.persistence.safe_json import write_json_object_verified +from ai4binance.infrastructure.persistence.safe_json import ( + DestinationVerificationError, + write_json_object_verified, +) from ai4binance.schemas import OHLCVCandle _DEFAULT_KLINE_INTERVAL = timedelta(minutes=5) @@ -251,6 +254,16 @@ def _save(path: Path, payload: Mapping[str, object]) -> None: ) +def _recoverable_error_code(error: Exception) -> str: + """Expose only repository-owned verification codes, never exception detail.""" + + if isinstance(error, DestinationVerificationError): + code = str(error).strip() + if re.fullmatch(r"[A-Z][A-Z0-9_]{2,127}", code): + return code + return "MARKET_HISTORY_RECOVERABLE_ERROR" + + def _load(path: Path) -> dict[str, object]: payload = json.loads(path.read_text(encoding="utf-8")) if not isinstance(payload, dict): @@ -513,10 +526,10 @@ def sync_cycle(self, *, observed_at: datetime) -> dict[str, object]: self.coin_m, ), ) - # Historical candle collection must not wait behind bulk snapshot - # endpoints. Snapshots are supplementary metadata and are refreshed - # after the first bounded collection interval. - snapshot_refresh_due = time.monotonic() + 300 + # VirtualMarket depends on these bounded bulk snapshots. Refresh once + # at cycle start so a service restart cannot leave an already-old + # ticker cache to expire during a long candle collection cycle. + snapshot_refresh_due = time.monotonic() def refresh_snapshots(snapshot_time: datetime) -> None: nonlocal snapshot_refresh_due @@ -533,6 +546,10 @@ def refresh_snapshots(snapshot_time: datetime) -> None: refreshed_universe is None or refreshed_universe.blockers ) and "MARKET_UNIVERSE_METADATA_UNAVAILABLE" not in blockers: blockers.append("MARKET_UNIVERSE_METADATA_UNAVAILABLE") + elif refreshed_universe is not None and not refreshed_universe.blockers: + while "MARKET_UNIVERSE_METADATA_UNAVAILABLE" in blockers: + blockers.remove("MARKET_UNIVERSE_METADATA_UNAVAILABLE") + snapshot_failed = False for snapshot_market, snapshot_symbols, snapshot_transport in market_work: if snapshot_transport is None or not snapshot_symbols: continue @@ -544,10 +561,16 @@ def refresh_snapshots(snapshot_time: datetime) -> None: snapshot_time, ) except (OSError, ValueError, ExchangeError): + snapshot_failed = True if "MARKET_SNAPSHOT_UNAVAILABLE" not in blockers: blockers.append("MARKET_SNAPSHOT_UNAVAILABLE") + if not snapshot_failed: + while "MARKET_SNAPSHOT_UNAVAILABLE" in blockers: + blockers.remove("MARKET_SNAPSHOT_UNAVAILABLE") snapshot_refresh_due = time.monotonic() + 300 + refresh_snapshots(now) + work_items = self._interleaved_market_work(market_work) requested_identity: tuple[str, str] | None = None refresh_request: dict[str, object] | None = None @@ -1620,6 +1643,7 @@ def record_recoverable_cycle_failure( "status": "DEGRADED", "observed_at": observed_at.astimezone(UTC).isoformat(), "last_error_type": type(error).__name__, + "last_error_code": _recoverable_error_code(error), "recovery_action": "RETRY_NEXT_CYCLE", "blockers": sorted(blockers), **_SAFE_STATE, diff --git a/src/ai4binance/data/market_history_sync.py b/src/ai4binance/data/market_history_sync.py index 7712c664..1371f724 100644 --- a/src/ai4binance/data/market_history_sync.py +++ b/src/ai4binance/data/market_history_sync.py @@ -11,7 +11,7 @@ import urllib.parse import urllib.request import zipfile -from collections.abc import Callable +from collections.abc import Callable, Mapping from dataclasses import asdict, dataclass, field from datetime import UTC, date, datetime, timedelta from datetime import time as datetime_time @@ -43,6 +43,14 @@ _MAX_UNCOMPRESSED_BYTES: Final = 256 * 1024 * 1024 _UNIVERSE_CACHE_MAX_AGE: Final = timedelta(minutes=5) _COLLECTION_MAX_SYMBOLS_PER_MARKET: Final = 50 +_FAST_RETRY_BLOCKERS: Final = frozenset( + { + "PUBLIC_MARKET_UNIVERSE_UNAVAILABLE", + "PUBLIC_MARKET_UNIVERSE_EMPTY", + "PUBLIC_MARKET_LIQUIDITY_UNIVERSE_UNAVAILABLE", + "PUBLIC_MARKET_LIQUIDITY_UNIVERSE_EMPTY", + } +) def read_cached_market_universe( @@ -409,6 +417,9 @@ def _eligible_universe( else self.universe_provider.eligible_market_snapshot() ) if snapshot.blockers: + cached = read_cached_market_universe(cache_path, observed_at) + if cached is not None: + return cached return snapshot cache_path.parent.mkdir(parents=True, exist_ok=True) temporary = cache_path.with_suffix(".json.tmp") @@ -704,26 +715,44 @@ def run(self, *, max_cycles: int | None = None) -> int: started = time.monotonic() now = self.clock() attempts += 1 + fast_retry = False try: if self.cycle is None: - self.synchronizer.sync_day( + result = self.synchronizer.sync_day( now.date() - timedelta(days=1), observed_at=now ) else: - self.cycle(now) + result = self.cycle(now) except (OSError, ValueError, ArithmeticError, ExchangeError) as error: + fast_retry = True if self.on_recoverable_error is not None: self.on_recoverable_error(now, error) else: completed += 1 + fast_retry = _requires_fast_retry(result) if max_cycles is None or attempts < max_cycles: delay = self.interval_seconds if self.cycle is not None: delay = max(1, delay - (time.monotonic() - started)) + if fast_retry: + delay = min(delay, 30) self.sleeper(delay) return completed +def _requires_fast_retry(result: object) -> bool: + blockers = ( + result.get("blockers", ()) + if isinstance(result, Mapping) + else getattr(result, "blockers", ()) + ) + return isinstance(blockers, (list, tuple)) and bool( + _FAST_RETRY_BLOCKERS.intersection( + item for item in blockers if isinstance(item, str) + ) + ) + + def _daily_kline_key( market: str, symbol: str, diff --git a/src/ai4binance/governance/adapters.py b/src/ai4binance/governance/adapters.py index d4596c60..89d7e929 100644 --- a/src/ai4binance/governance/adapters.py +++ b/src/ai4binance/governance/adapters.py @@ -107,31 +107,57 @@ def evaluate( ) -> VirtualGovernanceResult: """Return the dependency-neutral result for one canonical DGE evaluation.""" - dge_candidate = _virtual_dge_candidate( - candidate, - market=market, - quantity=quantity, - ) - context = ( - self.context_builder(snapshot, analysis, candidate, portfolio) - if self.context_builder is not None - else _default_virtual_dge_context( - snapshot, - analysis, + try: + dge_candidate = _virtual_dge_candidate( candidate, - portfolio, - risk_approved=risk_approved, - portfolio_verified=portfolio_verified, quantity=quantity, + market=market, ) - ) - if not isinstance(context, DgeGovernanceContext): - raise TypeError( - "virtual DGE context builder must return DgeGovernanceContext" + except ( + ArithmeticError, + AttributeError, + KeyError, + TypeError, + ValueError, + ) as error: + raise ValueError("DGE_CANDIDATE_ADAPTATION_FAILED") from error + try: + context = ( + self.context_builder(snapshot, analysis, candidate, portfolio) + if self.context_builder is not None + else _default_virtual_dge_context( + snapshot, + analysis, + candidate, + portfolio, + risk_approved=risk_approved, + portfolio_verified=portfolio_verified, + quantity=quantity, + ) ) + except ( + ArithmeticError, + AttributeError, + KeyError, + TypeError, + ValueError, + ) as error: + raise ValueError("DGE_CONTEXT_BUILD_FAILED") from error + if not isinstance(context, DgeGovernanceContext): + raise ValueError("DGE_CONTEXT_TYPE_INVALID") if context.execution_surface is not ExecutionSurface.VIRTUAL_MARKET: - raise ValueError("virtual DGE context must use VIRTUAL_MARKET") - decision = self.dge.evaluate(dge_candidate, context) + raise ValueError("DGE_CONTEXT_SURFACE_INVALID") + try: + decision = self.dge.evaluate(dge_candidate, context) + except ( + ArithmeticError, + AttributeError, + KeyError, + RuntimeError, + TypeError, + ValueError, + ) as error: + raise ValueError("DGE_ENGINE_EVALUATION_FAILED") from error blockers = tuple( dict.fromkeys((*decision.hard_blockers, *decision.soft_blockers)) ) @@ -273,9 +299,17 @@ def _default_virtual_dge_context( structure_valid=True, negative_evidence_clear=( isinstance(candidate_blockers, tuple) - and not candidate_blockers and isinstance(analysis_blockers, tuple) - and not analysis_blockers + and not _has_any( + (*candidate_blockers, *analysis_blockers), + ( + "NEGATIVE", + "CRITICAL_CONFLICT", + "FAILED_BREAKOUT", + "PUMP", + "MANIPULATION", + ), + ) ), oos_approved=oos_passed, risk_approved=risk_approved, diff --git a/src/ai4binance/governance/dge_models.py b/src/ai4binance/governance/dge_models.py index 79a508f7..ab147b60 100644 --- a/src/ai4binance/governance/dge_models.py +++ b/src/ai4binance/governance/dge_models.py @@ -471,38 +471,55 @@ def __post_init__(self) -> None: raise ValueError("DGE authority profile must match the execution surface") if self.automation_mode is not authority_profile.automation_mode: raise ValueError("DGE automation mode must match the execution surface") - if self.auto_execution_allowed != authority_profile.auto_execution_allowed: + decision_profile_enabled = self.paper_execution_allowed + if self.auto_execution_allowed != ( + authority_profile.auto_execution_allowed and decision_profile_enabled + ): raise ValueError( "DGE autonomous simulation must match the execution surface profile" ) if ( self.simulated_execution_allowed - != authority_profile.simulated_execution_allowed + != ( + authority_profile.simulated_execution_allowed + and decision_profile_enabled + ) ): raise ValueError( "DGE simulated execution must match the execution surface profile" ) if ( self.autonomous_learning_allowed - != authority_profile.autonomous_learning_allowed + != ( + authority_profile.autonomous_learning_allowed + and decision_profile_enabled + ) ): raise ValueError( "DGE autonomous learning must match the execution surface profile" ) if ( self.bounded_self_improvement_allowed - != authority_profile.bounded_self_improvement_allowed + != ( + authority_profile.bounded_self_improvement_allowed + and decision_profile_enabled + ) ): raise ValueError( "DGE self-improvement must match the execution surface profile" ) - if self.simulated_spot_allowed != authority_profile.simulated_spot_allowed: + if self.simulated_spot_allowed != ( + authority_profile.simulated_spot_allowed and decision_profile_enabled + ): raise ValueError( "DGE simulated Spot scope must match the execution surface profile" ) if ( self.simulated_futures_allowed - != authority_profile.simulated_futures_allowed + != ( + authority_profile.simulated_futures_allowed + and decision_profile_enabled + ) ): raise ValueError( "DGE simulated Futures scope must match the execution surface profile" diff --git a/src/ai4binance/infrastructure/persistence/safe_json.py b/src/ai4binance/infrastructure/persistence/safe_json.py index 6ecf9c67..b3636cbd 100644 --- a/src/ai4binance/infrastructure/persistence/safe_json.py +++ b/src/ai4binance/infrastructure/persistence/safe_json.py @@ -373,6 +373,9 @@ def write_json_object_verified( temporary = path.with_name(f".{path.name}.{uuid4().hex}.tmp") try: encoded = _json_dumps(expected, indent=indent) + "\n" + normalized_expected = json.loads(encoded) + if not isinstance(normalized_expected, dict): + raise TypeError("verified JSON state must encode an object") with temporary.open("w", encoding="utf-8", newline="\n") as stream: stream.write(encoded) stream.flush() @@ -382,13 +385,13 @@ def write_json_object_verified( observed = _read_json_object(path, blocker=blocker) finally: temporary.unlink(missing_ok=True) - if dict(observed) != expected: + if dict(observed) != normalized_expected: raise fail_verification( blocker, destination=path, subject_id=subject_id or str(path), ) - expected_hash = _canonical_json_sha256(expected) + expected_hash = _canonical_json_sha256(normalized_expected) observed_hash = _canonical_json_sha256(dict(observed)) return verified( path, diff --git a/src/ai4binance/storage/destination_verification.py b/src/ai4binance/storage/destination_verification.py index e1bc5c2c..24969373 100644 --- a/src/ai4binance/storage/destination_verification.py +++ b/src/ai4binance/storage/destination_verification.py @@ -112,6 +112,9 @@ def write_json_object_verified( temporary = path.with_name(f".{path.name}.{uuid4().hex}.tmp") try: encoded = _json_dumps(expected, indent=indent) + "\n" + normalized_expected = json.loads(encoded) + if not isinstance(normalized_expected, dict): + raise TypeError("verified JSON state must encode an object") with temporary.open("w", encoding="utf-8", newline="\n") as stream: stream.write(encoded) stream.flush() @@ -121,13 +124,13 @@ def write_json_object_verified( observed = read_json_object(path, blocker=blocker) finally: temporary.unlink(missing_ok=True) - if dict(observed) != expected: + if dict(observed) != normalized_expected: raise fail_verification( blocker, destination=path, subject_id=subject_id or str(path), ) - expected_hash = _canonical_json_sha256(expected) + expected_hash = _canonical_json_sha256(normalized_expected) observed_hash = _canonical_json_sha256(dict(observed)) return verified( path, diff --git a/tests/test_data_acquisition.py b/tests/test_data_acquisition.py index f7c75962..a1d84431 100644 --- a/tests/test_data_acquisition.py +++ b/tests/test_data_acquisition.py @@ -112,7 +112,10 @@ def test_local_archive_missing_fails_closed_without_kline_network( def test_local_snapshot_priority_reuses_canonical_liquidity_order( tmp_path: Path, ) -> None: - from ai4binance.cli.runtime import _virtual_market_priority_symbols + from ai4binance.cli.runtime import ( + _virtual_market_priority_symbols, + _virtual_market_ranked_symbols, + ) from ai4binance.config import Settings from tests.test_binance_market_universe_provider import _SpotTransport @@ -138,6 +141,7 @@ def test_local_snapshot_priority_reuses_canonical_liquidity_order( "SOLUSDT", ) assert _virtual_market_priority_symbols(settings, (), ("BTCUSDT",), NOW) == () + assert _virtual_market_ranked_symbols(settings, (), NOW) == ("SOLUSDT",) assert ( _virtual_market_priority_symbols( settings, (), ("SOLUSDT",), NOW + timedelta(hours=1) @@ -179,7 +183,11 @@ def test_acquisition_consumes_verified_local_depth_and_rejects_stale_or_gapped( "bridged": True, } journal.append([("spot", "HOTUSDT", "checkpoint", checkpoint, NOW.timestamp())]) - agent = DataAcquisitionAgent(client=FakePublicClient(), depth_path=path) + agent = DataAcquisitionAgent( + client=FakePublicClient(), + depth_path=path, + clock=lambda: NOW, + ) snapshot = agent.acquire("HOTUSDT", ("1m",)) assert snapshot.order_book_summary["bid_depth"] == "2" assert snapshot.order_book_summary["ask_depth"] == "3" diff --git a/tests/test_dge_engine.py b/tests/test_dge_engine.py index e2aea7d6..b00c065a 100644 --- a/tests/test_dge_engine.py +++ b/tests/test_dge_engine.py @@ -230,6 +230,27 @@ def test_dge_virtual_market_surface_allows_autonomous_simulation_only() -> None: assert decision.execution_surface is ExecutionSurface.VIRTUAL_MARKET +def test_dge_blocked_virtual_market_returns_decision_without_profile_conflict() -> None: + decision = DecisionGovernanceEngine().evaluate( + candidate(), + context( + execution_surface=ExecutionSurface.VIRTUAL_MARKET, + oos_approved=False, + validation_approved=False, + human_approval_recorded=False, + ), + ) + + assert decision.governance_status is DgeDecisionStatus.WATCH_ONLY + assert decision.governed_action is DgeMarketAction.NO_TRADE + assert "VAL.OOS_NOT_VALIDATED" in decision.hard_blockers + assert decision.simulated_execution_allowed is False + assert decision.paper_execution_allowed is False + assert decision.auto_execution_allowed is False + assert decision.autonomous_learning_allowed is False + assert decision.requires_manual_confirmation is True + + def test_dge_data_quality_failure_is_first_class_and_non_executable() -> None: decision = DecisionGovernanceEngine().evaluate( candidate(), diff --git a/tests/test_dge_recovery_replay_shadow.py b/tests/test_dge_recovery_replay_shadow.py index e05bf1bb..77613ec5 100644 --- a/tests/test_dge_recovery_replay_shadow.py +++ b/tests/test_dge_recovery_replay_shadow.py @@ -56,7 +56,7 @@ def test_virtual_context_routes_only_current_bound_regime_and_mtf(stale: bool) - current, SimpleNamespace( snapshot_id=current.snapshot_id, - blockers=(), + blockers=("OOS_DEPLOYMENT_MISSING",), agent_results={"market_regime": regime, "multi_timeframe": mtf}, ), candidate, @@ -69,6 +69,7 @@ def test_virtual_context_routes_only_current_bound_regime_and_mtf(stale: bool) - assert context.mtf_aligned is (not stale) assert context.oos_approved is False assert context.validation_approved is False + assert context.negative_evidence_clear is True def test_recovery_radar_candidates_are_governed_without_live_authority() -> None: diff --git a/tests/test_lowest_coverage_persistence_and_governance.py b/tests/test_lowest_coverage_persistence_and_governance.py index 4df9c6ad..77d668be 100644 --- a/tests/test_lowest_coverage_persistence_and_governance.py +++ b/tests/test_lowest_coverage_persistence_and_governance.py @@ -454,6 +454,14 @@ def test_safe_json_public_serialization_and_verification_paths(tmp_path: Path) - ) assert result.status == "VERIFIED" assert path.read_text(encoding="utf-8").endswith("\n") + canonical_result = write_json_object_verified( + path, {"blockers": ("FIRST", "SECOND")}, blocker="WRITE" + ) + assert canonical_result.expected_sha256 == canonical_result.observed_sha256 + assert json.loads(path.read_text(encoding="utf-8"))["blockers"] == [ + "FIRST", + "SECOND", + ] assert to_primitive( { "enum": _ExampleEnum.VALUE, diff --git a/tests/test_market_data_gateway.py b/tests/test_market_data_gateway.py index af2dc4bd..972d953c 100644 --- a/tests/test_market_data_gateway.py +++ b/tests/test_market_data_gateway.py @@ -208,7 +208,7 @@ def test_gateway_reuses_the_canonical_continuous_collector_builder() -> None: source = Path(gateway_cli.__file__).read_text(encoding="utf-8") assert "build_continuous_market_history(" in source - assert "MarketDepthCollector(" in source + assert "build_market_depth_collector(" in source assert '"market-history-refresh-request.json"' not in source diff --git a/tests/test_market_depth.py b/tests/test_market_depth.py index 2b0e6b60..04463f50 100644 --- a/tests/test_market_depth.py +++ b/tests/test_market_depth.py @@ -28,7 +28,12 @@ def _json_transport(value: object | None = None) -> JsonTransport: - return cast(JsonTransport, object() if value is None else value) + return cast( + JsonTransport, + SimpleNamespace(get_json=lambda *_args, **_kwargs: snapshot()) + if value is None + else value, + ) def snapshot() -> dict[str, object]: @@ -90,6 +95,39 @@ def test_snapshot_bridge_checkpoint_and_replay( journal.close() +def test_checkpoint_compacts_superseded_depth_events(tmp_path: Path) -> None: + path = tmp_path / "depth.sqlite3" + journal = DepthJournal(path) + book = ReadOnlyOrderBook("BTCUSDT") + apply_depth_snapshot(book, snapshot()) + first = event("BTCUSDT") + assert apply_depth_event(book, first, "spot") + journal.append( + [ + ("spot", "BTCUSDT", "snapshot", snapshot(), NOW.timestamp()), + ("spot", "BTCUSDT", "delta", first, NOW.timestamp()), + ( + "spot", + "ETHUSDT", + "snapshot", + snapshot(), + NOW.timestamp(), + ), + ] + ) + + journal.append( + [("spot", "BTCUSDT", "checkpoint", _checkpoint(book), NOW.timestamp())] + ) + + retained = journal.connection.execute( + "SELECT symbol,kind FROM depth_events ORDER BY seq" + ).fetchall() + assert retained == [("ETHUSDT", "snapshot"), ("BTCUSDT", "checkpoint")] + assert read_local_depth(path, "spot", "BTCUSDT", now=NOW)["lastUpdateId"] == 101 + journal.close() + + @pytest.mark.parametrize("market", ["spot", "usd_m_futures", "coin_m_futures"]) def test_gap_and_unsynchronized_are_never_readable(tmp_path: Path, market: str) -> None: journal = DepthJournal(tmp_path / "depth.db") diff --git a/tests/test_market_history_boundary_contracts.py b/tests/test_market_history_boundary_contracts.py index 5f262423..75fd72cb 100644 --- a/tests/test_market_history_boundary_contracts.py +++ b/tests/test_market_history_boundary_contracts.py @@ -511,7 +511,10 @@ def test_refresh_dataset_blockers_distinguish_gaps_staleness_and_derivatives( blockers = instance._refresh_request_data_blockers( "usd_m_futures", "BTCUSDT", NOW, [{"kind": "funding", "status": "BLOCKED"}] ) - assert f"MARKET_HISTORY_REFRESH_{code}:5m" in blockers + assert all( + f"MARKET_HISTORY_REFRESH_{code}:{timeframe}" in blockers + for timeframe in h.VIRTUAL_MARKET_COLLECTION_TIMEFRAMES + ) assert "FUTURES_DERIVATIVES_CONTEXT_UNAVAILABLE" in blockers instance._complete_refresh_request({}, NOW, status="DATA_READY", blockers=()) with pytest.raises(ValueError, match="market is invalid"): @@ -706,6 +709,7 @@ def test_cycle_failure_persists_safe_retry_state(tmp_path: Path) -> None: state = h._load(instance.history.state_path) assert state["status"] == "DEGRADED" assert state["last_error_type"] == "OSError" + assert state["last_error_code"] == "MARKET_HISTORY_RECOVERABLE_ERROR" assert "private path" not in json.dumps(state) diff --git a/tests/test_market_history_continuous.py b/tests/test_market_history_continuous.py index cf5d76d2..d764112a 100644 --- a/tests/test_market_history_continuous.py +++ b/tests/test_market_history_continuous.py @@ -10,6 +10,7 @@ from decimal import Decimal from pathlib import Path from threading import Event, Thread +from types import SimpleNamespace from typing import cast import pytest @@ -17,6 +18,7 @@ from ai4binance.cli.market_data import ( _build_canonical_opportunity_pipeline, _priority_depth_markets, + build_market_depth_collector, ) from ai4binance.cli.market_data import ( main as market_data_main, @@ -38,6 +40,9 @@ BinanceVisionArchiveCache, MarketHistorySynchronizer, ) +from ai4binance.infrastructure.persistence.safe_json import ( + DestinationVerificationError, +) from ai4binance.integrations.binance import BinanceEligibleMarketSnapshot from ai4binance.schemas import OHLCVCandle @@ -385,6 +390,7 @@ def test_recoverable_cycle_failure_is_persisted_for_retry(tmp_path: Path) -> Non state = _load(instance.history.state_path) assert state["status"] == "DEGRADED" assert state["last_error_type"] == "OSError" + assert state["last_error_code"] == "MARKET_HISTORY_RECOVERABLE_ERROR" assert state["recovery_action"] == "RETRY_NEXT_CYCLE" assert state["completed_streams"] == 5 assert state["total_streams"] == 10 @@ -396,6 +402,20 @@ def test_recoverable_cycle_failure_is_persisted_for_retry(tmp_path: Path) -> Non assert state["live_eligibility_status"] == "LIVE_ORDER_BLOCKED" +def test_recoverable_failure_exposes_only_verified_blocker_code( + tmp_path: Path, +) -> None: + instance = collector(tmp_path, Transport()) + + instance.record_recoverable_cycle_failure( + NOW, + DestinationVerificationError("MARKET_HISTORY_WRITE_FAILED"), + ) + + state = _load(instance.history.state_path) + assert state["last_error_code"] == "MARKET_HISTORY_WRITE_FAILED" + + def test_recoverable_cycle_failure_tolerates_malformed_previous_blockers( tmp_path: Path, ) -> None: @@ -550,7 +570,8 @@ def collect( assert markets.count("usd_m_futures") == ( len(VIRTUAL_MARKET_COLLECTION_TIMEFRAMES) + 4 ) - assert snapshot_markets == (["spot", "usd_m_futures"] if long_backfill else []) + expected_snapshot_passes = 2 if long_backfill else 1 + assert snapshot_markets == ["spot", "usd_m_futures"] * expected_snapshot_passes assert report["completed_symbols"] == 2 assert report["total_symbols"] == 2 assert report["completed_streams"] == 10 @@ -696,6 +717,20 @@ class DepthUniverse: } +def test_depth_collector_shards_limit_reconnect_blast_radius(tmp_path: Path) -> None: + transport = Transport() + depth = build_market_depth_collector( + cast(MarketHistorySynchronizer, SimpleNamespace(archive_root=tmp_path)), + cast( + ContinuousMarketHistory, + SimpleNamespace(spot=transport, futures=transport, coin_m=None), + ), + ) + + assert depth.group_size == 10 + assert depth.root == tmp_path / "depth" + + def test_opportunity_analysis_starts_after_native_timeframe_ingestion( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -2136,7 +2171,9 @@ def record_recoverable_cycle_failure(self, *_args: object) -> None: depth_events: list[object] = [] class Depth: - def __init__(self, _root: Path, transports: object) -> None: + def __init__( + self, _root: Path, transports: object, **_kwargs: object + ) -> None: depth_events.append(transports) def start(self, markets: object) -> None: @@ -2221,7 +2258,7 @@ def record_recoverable_cycle_failure(self, *_args: object) -> None: pass class Depth: - def __init__(self, *_args: object) -> None: + def __init__(self, *_args: object, **_kwargs: object) -> None: pass def close(self) -> None: diff --git a/tests/test_market_history_sync.py b/tests/test_market_history_sync.py index fdb4cb32..de3ffccc 100644 --- a/tests/test_market_history_sync.py +++ b/tests/test_market_history_sync.py @@ -271,6 +271,34 @@ def cycle(now: datetime) -> None: assert 1 <= sleeps[0] <= 900 +def test_supervisor_fast_retries_transient_universe_blocker(tmp_path: Path) -> None: + sleeps: list[float] = [] + attempts = 0 + + def cycle(_now: datetime) -> dict[str, object]: + nonlocal attempts + attempts += 1 + return { + "blockers": ( + ["PUBLIC_MARKET_LIQUIDITY_UNIVERSE_UNAVAILABLE"] + if attempts == 1 + else [] + ) + } + + supervisor = MarketHistorySupervisor( + synchronizer=object(), # type: ignore[arg-type] + interval_seconds=900, + lock_path=(tmp_path / "market-history.lock").resolve(), + sleeper=sleeps.append, + clock=lambda: OBSERVED_AT, + cycle=cycle, + ) + + assert supervisor.run(max_cycles=2) == 2 + assert sleeps == [30] + + def test_supervisor_does_not_hide_programming_errors(tmp_path: Path) -> None: supervisor = MarketHistorySupervisor( synchronizer=object(), # type: ignore[arg-type] @@ -355,6 +383,45 @@ def test_eligible_universe_force_refreshes_current_exchange_metadata( assert cached.futures_symbols == ("BTCUSDT",) +def test_force_refresh_falls_back_only_to_current_verified_universe_cache( + tmp_path: Path, +) -> None: + class TransientProvider: + calls = 0 + + def eligible_market_snapshot(self) -> BinanceEligibleMarketSnapshot: + self.calls += 1 + if self.calls == 1: + return BinanceEligibleMarketSnapshot( + spot_symbols=("BTCUSDT",), futures_symbols=("BTCUSDT",) + ) + return BinanceEligibleMarketSnapshot( + spot_symbols=(), + futures_symbols=(), + blockers=("PUBLIC_MARKET_UNIVERSE_UNAVAILABLE",), + ) + + provider = TransientProvider() + synchronizer = MarketHistorySynchronizer( + universe_provider=provider, # type: ignore[arg-type] + archive_root=tmp_path / "market", + source_cache=BinanceVisionArchiveCache(tmp_path / "sources", lambda _: b""), + state_path=tmp_path / "state.json", + ) + + synchronizer._eligible_universe(OBSERVED_AT, force_refresh=True) + current = synchronizer._eligible_universe( + OBSERVED_AT + timedelta(minutes=1), force_refresh=True + ) + stale = synchronizer._eligible_universe( + OBSERVED_AT + timedelta(minutes=6), force_refresh=True + ) + + assert current.spot_symbols == ("BTCUSDT",) + assert not current.blockers + assert stale.blockers == ("PUBLIC_MARKET_UNIVERSE_UNAVAILABLE",) + + def test_eligible_universe_uses_the_bounded_top_volume_provider_when_available( tmp_path: Path, ) -> None: diff --git a/tests/test_research_application.py b/tests/test_research_application.py index 500c8b49..23990cbb 100644 --- a/tests/test_research_application.py +++ b/tests/test_research_application.py @@ -653,6 +653,7 @@ def test_research_service_dge_failure_is_deterministic_and_preserves_portfolio() ) assert first.virtual_runtime_decision.eligibility.blockers == ( "DGE_EVALUATION_FAILED", + "DGE_EVALUATION_ERROR_TYPE:ENGINE_EVALUATION", "DGE_SIMULATION_NOT_APPROVED", ) assert ( @@ -685,6 +686,7 @@ def _malformed_context( ) assert workflow.virtual_runtime_decision.eligibility.blockers == ( "DGE_EVALUATION_FAILED", + "DGE_EVALUATION_ERROR_TYPE:CONTEXT_TYPE", "DGE_SIMULATION_NOT_APPROVED", ) assert ( diff --git a/tests/test_service_manifest.py b/tests/test_service_manifest.py index 34d115d8..7629d75f 100644 --- a/tests/test_service_manifest.py +++ b/tests/test_service_manifest.py @@ -157,6 +157,10 @@ def test_startup_install_script_preserves_cli_module_runtime_commands() -> None: assert '$env:PYTHONDONTWRITEBYTECODE = "1"' in install_text assert "& $python -B @Arguments" in install_text assert "& $python @Arguments" not in install_text + assert '$previousErrorActionPreference = $ErrorActionPreference' in install_text + assert '$ErrorActionPreference = "Continue"' in install_text + assert '$ErrorActionPreference = $previousErrorActionPreference' in install_text + assert "$nativeExitCode = [int]$LASTEXITCODE" in install_text assert '-Arguments @("-m", "ai4binance.cli", "archive-public")' in install_text assert '-Arguments @("-m", "ai4binance.cli", "validate-research")' in install_text assert "-WindowStyle Hidden" in install_text diff --git a/tests/test_storage.py b/tests/test_storage.py index c167de58..dbe9d90c 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -401,6 +401,24 @@ def test_write_json_object_verified_reads_destination_back(tmp_path: Path) -> No assert json.loads(path.read_text(encoding="utf-8"))["status"] == "ok" +def test_write_json_object_verified_compares_canonical_json_shapes( + tmp_path: Path, +) -> None: + path = tmp_path / "state" / "latest.json" + + result = write_json_object_verified( + path, + {"blockers": ("FIRST", "SECOND")}, + blocker="STATE_VERIFY_FAILED", + ) + + assert result.expected_sha256 == result.observed_sha256 + assert json.loads(path.read_text(encoding="utf-8"))["blockers"] == [ + "FIRST", + "SECOND", + ] + + def test_write_json_object_verified_stops_on_failed_read_back( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, From 9f6e34e7a5f43d85e66e0fe3cdabf0c88a868d37 Mon Sep 17 00:00:00 2001 From: Huseyin Cicek Date: Fri, 25 Sep 2026 02:37:00 +0300 Subject: [PATCH 06/19] feat: add by HsC --- .../runtime_validation_deployment.json | 354 ++++++++++ .../research/virtual_market_acceptance.yaml | 99 +++ scripts/install_startup_task.ps1 | 114 +++- src/ai4binance/agents/validation_gate.py | 27 +- src/ai4binance/cli/market_data.py | 4 +- src/ai4binance/cli/market_gateway.py | 113 ++-- src/ai4binance/cli/research.py | 37 +- src/ai4binance/config.py | 9 +- .../data/market_history_continuous.py | 10 +- src/ai4binance/governance/dge_models.py | 33 +- .../storage/destination_verification.py | 17 +- src/ai4binance/validation/oos_maturity.py | 637 +++++++++++++++++- src/ai4binance/validation_pipeline_runtime.py | 72 +- tests/test_cli.py | 6 +- tests/test_config_reporting.py | 6 +- tests/test_local_dashboard_source.py | 3 +- tests/test_market_history_continuous.py | 43 +- tests/test_oos_maturity.py | 61 ++ tests/test_service_manifest.py | 9 +- tests/test_storage.py | 56 ++ tests/test_validation_pipeline.py | 46 ++ 21 files changed, 1640 insertions(+), 116 deletions(-) create mode 100644 config/research/runtime_validation_deployment.json diff --git a/config/research/runtime_validation_deployment.json b/config/research/runtime_validation_deployment.json new file mode 100644 index 00000000..b7ea9ba1 --- /dev/null +++ b/config/research/runtime_validation_deployment.json @@ -0,0 +1,354 @@ +{ + "execution_allowed": false, + "live_eligibility_status": "LIVE_ORDER_BLOCKED", + "promotion_status": "RESEARCH_ONLY", + "runtime_source_sha256": "e79927bdc5c8c7430692cffa550c9f14b813acf440e3738d4358ecbd4692ffee", + "schema_version": "1.0", + "subjects": [ + { + "bundle": { + "path": "oos_runtime/890e57250d9f2dfccca58a102780f9d2ef6151a77095c374a433aac20c31c6a0/bundle.json", + "sha256": "8cb4ca67d0e443497d67b7d913d71299b23372cdd11c96d408be4ad5e744f6bc" + }, + "subject": { + "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", + "feature_definition_sha256": "63f712ab32697b7a9c67ee81847bcdc9f7171002e2f7e4e6a213ae2e12edd979", + "promotion": { + "code_revision": "e79927bdc5c8c7430692cffa550c9f14b813acf440e3738d4358ecbd4692ffee", + "dataset_sha256": "9be16ba7fe8bb45591bd5089accdde00df60dd80bf516e2bdfe4577ffe8a8f15", + "market_type": "SPOT", + "parameter_set_sha256": "9b971ab67af0b920c00f1531db13348e3491b7bbea39860de0d78a91d3acfeba", + "strategy_id": "trend_continuation", + "strategy_sha256": "63f712ab32697b7a9c67ee81847bcdc9f7171002e2f7e4e6a213ae2e12edd979", + "strategy_version": "1", + "symbol": "BTCUSDT", + "timeframe": "15m" + }, + "setup_type": "trend_continuation", + "validation_config_sha256": "24ec5a26186412d7024efd1532c940b1118631e24ed7abaf61682d38a0181e6c" + } + }, + { + "bundle": { + "path": "oos_runtime/7e1bcdee40892c39ba24c89a28190775777a1e82906601df28d80e5cbd4e8a30/bundle.json", + "sha256": "a0e056b8db67687bb766ba03970488d9ef13e7e1b98b93a12e40588d053f1c99" + }, + "subject": { + "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", + "feature_definition_sha256": "acfb727377466011e73a0b3782f76bc445b3dc7d7fbddda33c8b0811370dc852", + "promotion": { + "code_revision": "e79927bdc5c8c7430692cffa550c9f14b813acf440e3738d4358ecbd4692ffee", + "dataset_sha256": "9be16ba7fe8bb45591bd5089accdde00df60dd80bf516e2bdfe4577ffe8a8f15", + "market_type": "SPOT", + "parameter_set_sha256": "9b971ab67af0b920c00f1531db13348e3491b7bbea39860de0d78a91d3acfeba", + "strategy_id": "pullback_continuation", + "strategy_sha256": "acfb727377466011e73a0b3782f76bc445b3dc7d7fbddda33c8b0811370dc852", + "strategy_version": "1", + "symbol": "BTCUSDT", + "timeframe": "15m" + }, + "setup_type": "pullback_continuation", + "validation_config_sha256": "9543c1e40f6059bc3ef4cceea37a9ecff3e5033c1fd21cdbcea18811fba657f3" + } + }, + { + "bundle": { + "path": "oos_runtime/b1b43d5874b9eaf2b51be5d936fed781611b9e9d99d8d229aa3e344d86b9e878/bundle.json", + "sha256": "e6625d073e86dfd88735e84bc263fc388819aca5a6cd6d644954ced683ce679a" + }, + "subject": { + "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", + "feature_definition_sha256": "2a7e0ad5ccc7292a0b06b76b233fa93552d34bfe993a0db4976e3404b1ff7486", + "promotion": { + "code_revision": "e79927bdc5c8c7430692cffa550c9f14b813acf440e3738d4358ecbd4692ffee", + "dataset_sha256": "9be16ba7fe8bb45591bd5089accdde00df60dd80bf516e2bdfe4577ffe8a8f15", + "market_type": "SPOT", + "parameter_set_sha256": "9b971ab67af0b920c00f1531db13348e3491b7bbea39860de0d78a91d3acfeba", + "strategy_id": "breakout_retest", + "strategy_sha256": "2a7e0ad5ccc7292a0b06b76b233fa93552d34bfe993a0db4976e3404b1ff7486", + "strategy_version": "1", + "symbol": "BTCUSDT", + "timeframe": "15m" + }, + "setup_type": "breakout_retest", + "validation_config_sha256": "bed436e96bb0b3ac0fb16758d3cd504ee0e4a0599d7a87f9d145f79f16b69ed3" + } + }, + { + "bundle": { + "path": "oos_runtime/6c86ecf0b0eaf809535c872b8167e4d46f964f2ae4f3f38c5c4a363c35ce634b/bundle.json", + "sha256": "fa249bc1af909c723a96d78ca3a015616dcd436eef4d80152da78c19eaeb1aae" + }, + "subject": { + "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", + "feature_definition_sha256": "7cfbd10b251a8e6acfa0656247e9f99f56c4b6e24849c8a36d66fce734f746a3", + "promotion": { + "code_revision": "e79927bdc5c8c7430692cffa550c9f14b813acf440e3738d4358ecbd4692ffee", + "dataset_sha256": "9be16ba7fe8bb45591bd5089accdde00df60dd80bf516e2bdfe4577ffe8a8f15", + "market_type": "SPOT", + "parameter_set_sha256": "9b971ab67af0b920c00f1531db13348e3491b7bbea39860de0d78a91d3acfeba", + "strategy_id": "support_reclaim", + "strategy_sha256": "7cfbd10b251a8e6acfa0656247e9f99f56c4b6e24849c8a36d66fce734f746a3", + "strategy_version": "1", + "symbol": "BTCUSDT", + "timeframe": "15m" + }, + "setup_type": "support_reclaim", + "validation_config_sha256": "0e3bc5eddceca968b76419d11acc3088cbba85d290da887f90db2d0296988afc" + } + }, + { + "bundle": { + "path": "oos_runtime/9872a208aaa9b1e13ac80bb35a9bbcdda8e12bb82ec7b905ea75a99727023ddb/bundle.json", + "sha256": "140ae4c52799b52670b7174f6968d07e27a50d7ca32826c9666fc2df01b39233" + }, + "subject": { + "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", + "feature_definition_sha256": "4f33ee776ac3b463420d6a0f2c08352a0673d16633d2a32765d861fcb8e144ea", + "promotion": { + "code_revision": "e79927bdc5c8c7430692cffa550c9f14b813acf440e3738d4358ecbd4692ffee", + "dataset_sha256": "9be16ba7fe8bb45591bd5089accdde00df60dd80bf516e2bdfe4577ffe8a8f15", + "market_type": "SPOT", + "parameter_set_sha256": "9b971ab67af0b920c00f1531db13348e3491b7bbea39860de0d78a91d3acfeba", + "strategy_id": "failed_breakout_reversal", + "strategy_sha256": "4f33ee776ac3b463420d6a0f2c08352a0673d16633d2a32765d861fcb8e144ea", + "strategy_version": "1", + "symbol": "BTCUSDT", + "timeframe": "15m" + }, + "setup_type": "failed_breakout_reversal", + "validation_config_sha256": "7d456e866aac91f9b51e90184af24a8f4fd3292eba863d504fcca0bfd5337a3d" + } + }, + { + "bundle": { + "path": "oos_runtime/d8e7e3aa2db42593d6e2eb1417e5cbe17a56961bc9aa227f7629744ae3851971/bundle.json", + "sha256": "c0dbfb17a914f413f71a918d51d35364bf22eaecb9934df0aa71879b64b475e3" + }, + "subject": { + "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", + "feature_definition_sha256": "63f712ab32697b7a9c67ee81847bcdc9f7171002e2f7e4e6a213ae2e12edd979", + "promotion": { + "code_revision": "e79927bdc5c8c7430692cffa550c9f14b813acf440e3738d4358ecbd4692ffee", + "dataset_sha256": "726c5f1a6295f1323671cd642bdbf6c4976623cd71fce1d2345446a04b70a445", + "market_type": "SPOT", + "parameter_set_sha256": "feca57cc3c10c80e9086f18e57fd8f498715f7cd865a465b690612a523c4eff3", + "strategy_id": "trend_continuation", + "strategy_sha256": "63f712ab32697b7a9c67ee81847bcdc9f7171002e2f7e4e6a213ae2e12edd979", + "strategy_version": "1", + "symbol": "BTCUSDT", + "timeframe": "1h" + }, + "setup_type": "trend_continuation", + "validation_config_sha256": "07b8e29b6e6dc7fda50d66ca09ee19ebf99100de98dd3b68bb0195c9eab2ad6e" + } + }, + { + "bundle": { + "path": "oos_runtime/bc16e69250feb1b49ae9e6f35310fe8f3b1ff11cfb12020b3b324f45c1b9a7fa/bundle.json", + "sha256": "35fa0639e75b723eca6c582db5a2bf29f7694adb932c7a661a101c2b7577db34" + }, + "subject": { + "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", + "feature_definition_sha256": "d676e5ee3551bed7965e95cb421ae999225b9b6c0e2662468065506836fe5fc7", + "promotion": { + "code_revision": "e79927bdc5c8c7430692cffa550c9f14b813acf440e3738d4358ecbd4692ffee", + "dataset_sha256": "726c5f1a6295f1323671cd642bdbf6c4976623cd71fce1d2345446a04b70a445", + "market_type": "SPOT", + "parameter_set_sha256": "feca57cc3c10c80e9086f18e57fd8f498715f7cd865a465b690612a523c4eff3", + "strategy_id": "pullback_continuation", + "strategy_sha256": "d676e5ee3551bed7965e95cb421ae999225b9b6c0e2662468065506836fe5fc7", + "strategy_version": "1", + "symbol": "BTCUSDT", + "timeframe": "1h" + }, + "setup_type": "pullback_continuation", + "validation_config_sha256": "75c6996fc1630165e68913b5f619d986457a62ddbba066c8f09968c42ee8053e" + } + }, + { + "bundle": { + "path": "oos_runtime/005a7e5e442f21b5c8463124d473231199ba599040b252ae40d4072e9cbe9a3a/bundle.json", + "sha256": "793a5c5c2251c8c5e0f9df191994f8f5ebb4734ca94c5d8ad97aebca39e29dca" + }, + "subject": { + "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", + "feature_definition_sha256": "2a7e0ad5ccc7292a0b06b76b233fa93552d34bfe993a0db4976e3404b1ff7486", + "promotion": { + "code_revision": "e79927bdc5c8c7430692cffa550c9f14b813acf440e3738d4358ecbd4692ffee", + "dataset_sha256": "726c5f1a6295f1323671cd642bdbf6c4976623cd71fce1d2345446a04b70a445", + "market_type": "SPOT", + "parameter_set_sha256": "feca57cc3c10c80e9086f18e57fd8f498715f7cd865a465b690612a523c4eff3", + "strategy_id": "breakout_retest", + "strategy_sha256": "2a7e0ad5ccc7292a0b06b76b233fa93552d34bfe993a0db4976e3404b1ff7486", + "strategy_version": "1", + "symbol": "BTCUSDT", + "timeframe": "1h" + }, + "setup_type": "breakout_retest", + "validation_config_sha256": "eb78395b6f465fdc6f29531beb9ad289daaf0a273b1c28c0e04f42f0bce9acdb" + } + }, + { + "bundle": { + "path": "oos_runtime/ac5f7c2b5e91c3ab487856573566777c80534c50ffea903c6cec30bf0b86151a/bundle.json", + "sha256": "0c999dd85e7f20edc600d1d259951d19aaf31f792d4f47abf2af467c14fdf978" + }, + "subject": { + "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", + "feature_definition_sha256": "7cfbd10b251a8e6acfa0656247e9f99f56c4b6e24849c8a36d66fce734f746a3", + "promotion": { + "code_revision": "e79927bdc5c8c7430692cffa550c9f14b813acf440e3738d4358ecbd4692ffee", + "dataset_sha256": "726c5f1a6295f1323671cd642bdbf6c4976623cd71fce1d2345446a04b70a445", + "market_type": "SPOT", + "parameter_set_sha256": "feca57cc3c10c80e9086f18e57fd8f498715f7cd865a465b690612a523c4eff3", + "strategy_id": "support_reclaim", + "strategy_sha256": "7cfbd10b251a8e6acfa0656247e9f99f56c4b6e24849c8a36d66fce734f746a3", + "strategy_version": "1", + "symbol": "BTCUSDT", + "timeframe": "1h" + }, + "setup_type": "support_reclaim", + "validation_config_sha256": "5476114c01478e59a028f1de35d486a36bdb7cfa2b8fdb63e73a4e93c459fda9" + } + }, + { + "bundle": { + "path": "oos_runtime/d96659f5863de41822764d40c599a4f90997ce0e96815a174a97624cf117543a/bundle.json", + "sha256": "cfcff9591cad97d53b47f6823524b030bd3b50598993c3ecf3db789dbb4357b7" + }, + "subject": { + "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", + "feature_definition_sha256": "4f33ee776ac3b463420d6a0f2c08352a0673d16633d2a32765d861fcb8e144ea", + "promotion": { + "code_revision": "e79927bdc5c8c7430692cffa550c9f14b813acf440e3738d4358ecbd4692ffee", + "dataset_sha256": "726c5f1a6295f1323671cd642bdbf6c4976623cd71fce1d2345446a04b70a445", + "market_type": "SPOT", + "parameter_set_sha256": "feca57cc3c10c80e9086f18e57fd8f498715f7cd865a465b690612a523c4eff3", + "strategy_id": "failed_breakout_reversal", + "strategy_sha256": "4f33ee776ac3b463420d6a0f2c08352a0673d16633d2a32765d861fcb8e144ea", + "strategy_version": "1", + "symbol": "BTCUSDT", + "timeframe": "1h" + }, + "setup_type": "failed_breakout_reversal", + "validation_config_sha256": "9c5f047dab4454f4b32bf7545eac41ce4fe4190ac22f3bbd28c11ce317fac9b8" + } + }, + { + "bundle": { + "path": "oos_runtime/d37139f1f85ca960e05db7daa67acba59c4ca7f9b9de3b815cc40947a8c7cc18/bundle.json", + "sha256": "d05d459f8ddd081fbea48d6466bcfd8558940bda3675a4ea9065545eb61ebaa8" + }, + "subject": { + "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", + "feature_definition_sha256": "63f712ab32697b7a9c67ee81847bcdc9f7171002e2f7e4e6a213ae2e12edd979", + "promotion": { + "code_revision": "e79927bdc5c8c7430692cffa550c9f14b813acf440e3738d4358ecbd4692ffee", + "dataset_sha256": "d1b3e59b5ad3f22154405c8022ab5b0354dac6734deb10ca7c712d5fb288ed81", + "market_type": "SPOT", + "parameter_set_sha256": "43ee407708a48ea2da819ce0b597f20ee8c012155b1a48f874f996d7c59cbab5", + "strategy_id": "trend_continuation", + "strategy_sha256": "63f712ab32697b7a9c67ee81847bcdc9f7171002e2f7e4e6a213ae2e12edd979", + "strategy_version": "1", + "symbol": "BTCUSDT", + "timeframe": "4h" + }, + "setup_type": "trend_continuation", + "validation_config_sha256": "2aad4ce2b10383fd7ee876d02e7140510069b0e0ddf35d6a3936e85c05f30820" + } + }, + { + "bundle": { + "path": "oos_runtime/d68eab3c7ea194f31c93ee3c04beab94817660f5e4fe0db42a3c64648aaa71f9/bundle.json", + "sha256": "e188a666988fb488a07f333c7c71c7755991683125652f40e6f2b614c6fb1c91" + }, + "subject": { + "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", + "feature_definition_sha256": "d676e5ee3551bed7965e95cb421ae999225b9b6c0e2662468065506836fe5fc7", + "promotion": { + "code_revision": "e79927bdc5c8c7430692cffa550c9f14b813acf440e3738d4358ecbd4692ffee", + "dataset_sha256": "d1b3e59b5ad3f22154405c8022ab5b0354dac6734deb10ca7c712d5fb288ed81", + "market_type": "SPOT", + "parameter_set_sha256": "43ee407708a48ea2da819ce0b597f20ee8c012155b1a48f874f996d7c59cbab5", + "strategy_id": "pullback_continuation", + "strategy_sha256": "d676e5ee3551bed7965e95cb421ae999225b9b6c0e2662468065506836fe5fc7", + "strategy_version": "1", + "symbol": "BTCUSDT", + "timeframe": "4h" + }, + "setup_type": "pullback_continuation", + "validation_config_sha256": "935af3bd7625c9602ca7623ebbd9b3996dc3652a74acf91b7762577508dab018" + } + }, + { + "bundle": { + "path": "oos_runtime/89212af054cbffe520074ecaa18ff4065385934e21c6f8a4f1775cb22ebb4209/bundle.json", + "sha256": "e7f2b9b6d8f1a32921cc771e14c95e9ddf62bfdc3011dfc0951efab078648cdd" + }, + "subject": { + "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", + "feature_definition_sha256": "81ed5bdf546ea4521d379049a78ecd0c518945f72b7414517439d2e976d59c40", + "promotion": { + "code_revision": "e79927bdc5c8c7430692cffa550c9f14b813acf440e3738d4358ecbd4692ffee", + "dataset_sha256": "d1b3e59b5ad3f22154405c8022ab5b0354dac6734deb10ca7c712d5fb288ed81", + "market_type": "SPOT", + "parameter_set_sha256": "43ee407708a48ea2da819ce0b597f20ee8c012155b1a48f874f996d7c59cbab5", + "strategy_id": "breakout_retest", + "strategy_sha256": "81ed5bdf546ea4521d379049a78ecd0c518945f72b7414517439d2e976d59c40", + "strategy_version": "1", + "symbol": "BTCUSDT", + "timeframe": "4h" + }, + "setup_type": "breakout_retest", + "validation_config_sha256": "1e7fde61c0fd0d71b9067617171a4f045070e8ddb7e6fcb71134013fc4644c00" + } + }, + { + "bundle": { + "path": "oos_runtime/63bc9f333eaefafb8438ea26b229db89d20dcbd7ba2358a5fb06b2c60179fb93/bundle.json", + "sha256": "125ad7aee5a6704e6ea48b684f53922a474812afd58be62cb44ffb7ad6f8fd2b" + }, + "subject": { + "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", + "feature_definition_sha256": "ab8ad8719e8ae46a24c00a9fa61b06b65ec081f6282f532f3e6fd5922d517a07", + "promotion": { + "code_revision": "e79927bdc5c8c7430692cffa550c9f14b813acf440e3738d4358ecbd4692ffee", + "dataset_sha256": "d1b3e59b5ad3f22154405c8022ab5b0354dac6734deb10ca7c712d5fb288ed81", + "market_type": "SPOT", + "parameter_set_sha256": "43ee407708a48ea2da819ce0b597f20ee8c012155b1a48f874f996d7c59cbab5", + "strategy_id": "support_reclaim", + "strategy_sha256": "ab8ad8719e8ae46a24c00a9fa61b06b65ec081f6282f532f3e6fd5922d517a07", + "strategy_version": "1", + "symbol": "BTCUSDT", + "timeframe": "4h" + }, + "setup_type": "support_reclaim", + "validation_config_sha256": "52bb9cc097549348377424d8568afeab9d206375f204cfeb90b84780b4031277" + } + }, + { + "bundle": { + "path": "oos_runtime/88917be42880fc8aa60e89a44f3148e4a9e7b4e0d8e260fa45ed4eded122b710/bundle.json", + "sha256": "a1cfd1f0e6e4dec4ca8198ec00736b4ef5d5be97fa2e438618be88447693b09c" + }, + "subject": { + "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", + "feature_definition_sha256": "4f33ee776ac3b463420d6a0f2c08352a0673d16633d2a32765d861fcb8e144ea", + "promotion": { + "code_revision": "e79927bdc5c8c7430692cffa550c9f14b813acf440e3738d4358ecbd4692ffee", + "dataset_sha256": "d1b3e59b5ad3f22154405c8022ab5b0354dac6734deb10ca7c712d5fb288ed81", + "market_type": "SPOT", + "parameter_set_sha256": "43ee407708a48ea2da819ce0b597f20ee8c012155b1a48f874f996d7c59cbab5", + "strategy_id": "failed_breakout_reversal", + "strategy_sha256": "4f33ee776ac3b463420d6a0f2c08352a0673d16633d2a32765d861fcb8e144ea", + "strategy_version": "1", + "symbol": "BTCUSDT", + "timeframe": "4h" + }, + "setup_type": "failed_breakout_reversal", + "validation_config_sha256": "3b04265d83e7bb0faee38e621e7645ff5510e7aa5453c368f9c6829d5df57c47" + } + } + ] +} diff --git a/config/research/virtual_market_acceptance.yaml b/config/research/virtual_market_acceptance.yaml index 6c3ffaf2..c574b25b 100644 --- a/config/research/virtual_market_acceptance.yaml +++ b/config/research/virtual_market_acceptance.yaml @@ -37,6 +37,105 @@ anti_masking: combined_pnl_gate_allowed: false combined_sharpe_gate_allowed: false combined_equity_gate_allowed: false +spot_oos_validation_specification: + schema_version: "1.0" + specification_id: spot-oos-validation-v1 + status: ACTIVE + owner: Validation Owner + approval_status: PENDING_INDEPENDENT_REVIEW + market_scope: + - SPOT + timeframes: + - 15m + - 1h + - 4h + strategy_version: "1" + requirements: + dataset: + blockers: {operator: eq, value: []} + point_in_time_status: {operator: eq, value: PASS} + quality_status: {operator: eq, value: PASS} + revision_status: {operator: eq, value: IMMUTABLE} + missing_intervals: {operator: eq, value: 0} + duplicate_count: {operator: eq, value: 0} + observation_days: {operator: gte, value: 365} + cost_data_coverage: {operator: gte, value: 0.99} + backtest: + blockers: {operator: eq, value: []} + realism_status: {operator: eq, value: PASS} + cost_model_sha256: {operator: eq, value: "$COST_MODEL_SHA256"} + execution_model_version: {operator: eq, value: SPOT_BACKTEST_V1} + walk_forward: + blockers: {operator: eq, value: []} + fold_count: {operator: gte, value: 5} + train_only_selection: {operator: eq, value: true} + purge: {operator: gte, value: 1} + embargo: {operator: gte, value: 1} + final_holdout: + blockers: {operator: eq, value: []} + trade_count: {operator: gte, value: 100} + expectancy: {operator: gt, value: 0} + net_return: {operator: gt, value: 0} + max_drawdown: {operator: lte, value: 0.15} + regime: + blockers: {operator: eq, value: []} + regime_count: {operator: gte, value: 3} + unknown_ratio: {operator: lte, value: 0.05} + attribution_status: {operator: eq, value: PASS} + robustness: + blockers: {operator: eq, value: []} + parameter_stability: {operator: eq, value: PASS} + fold_stability: {operator: eq, value: PASS} + regime_stability: {operator: eq, value: PASS} + cpcv_status: {operator: eq, value: PASS} + monte_carlo_status: {operator: eq, value: PASS} + selection_overfit_status: {operator: eq, value: PASS} + edge_concentration: {operator: lte, value: 0.5} + failure_modes_status: {operator: eq, value: PASS} + statistics: + blockers: {operator: eq, value: []} + effective_sample_size: {operator: gte, value: 20} + confidence_interval_lower: {operator: gt, value: 0} + profit_factor: {operator: gte, value: 1.25} + profitable_fold_ratio: {operator: gte, value: 0.6} + hypothesis_count: {operator: gte, value: 1} + multiple_testing_method: + operator: in + value: [BONFERRONI, HOLM, SPA, MCS] + confirmatory: {operator: eq, value: true} + cost_stress: + blockers: {operator: eq, value: []} + scenario_coverage: + operator: eq + value: [BASE, COST_1_5X, COST_2X] + edge_survival: {operator: eq, value: true} + replication: + blockers: {operator: eq, value: []} + declared_scope_coverage: {operator: eq, value: true} + stability_status: {operator: eq, value: PASS} + paper_forward: + blockers: {operator: eq, value: []} + historical_oos_completed_at: {operator: present, value: true} + observation_started_at: {operator: present, value: true} + observation_ended_at: {operator: present, value: true} + observation_days: {operator: gte, value: 365} + trade_count: {operator: gte, value: 100} + frequency_comparison: {operator: eq, value: PASS} + cost_comparison: {operator: eq, value: PASS} + expectancy_comparison: {operator: eq, value: PASS} + drawdown_comparison: {operator: eq, value: PASS} + regime_comparison: {operator: eq, value: PASS} + signal_decay_status: {operator: eq, value: PASS} + runtime_failures: {operator: eq, value: 0} + reproducibility: + blockers: {operator: eq, value: []} + repository_clean: {operator: eq, value: true} + replay_equal: {operator: eq, value: true} + implementation_sha256: {operator: eq, value: "$RUNTIME_SOURCE_SHA256"} + authority: + execution_allowed: false + promotion_status: RESEARCH_ONLY + live_eligibility_status: LIVE_ORDER_BLOCKED authority: execution_allowed: false promotion_status: RESEARCH_ONLY diff --git a/scripts/install_startup_task.ps1 b/scripts/install_startup_task.ps1 index 81f6ac13..82f272bd 100644 --- a/scripts/install_startup_task.ps1 +++ b/scripts/install_startup_task.ps1 @@ -1,5 +1,5 @@ param( - [ValidateSet("Install", "InstallMarketHistory", "InstallVirtualMarket", "RunRuntime", "RunVirtualMarket", "RunVoice", "RunValidation", "RunAccounting", "RunAccountingWs", "RunSkillDiscovery", "RunMarketHistory", "RunFuturesMultiTf")] + [ValidateSet("Install", "InstallMarketHistory", "InstallVirtualMarket", "RestartMarketHistory", "RestartVirtualMarket", "RunRuntime", "RunVirtualMarket", "RunVoice", "RunValidation", "RunAccounting", "RunAccountingWs", "RunSkillDiscovery", "RunMarketHistory", "RunFuturesMultiTf")] [string]$Mode = "Install", [switch]$EnableVoiceTask, [switch]$EnableValidationTask, @@ -213,6 +213,104 @@ function Start-BoundedService { } while ($true) } +function Restart-BoundedScheduledService { + param( + [Parameter(Mandatory = $true)][string]$Service, + [Parameter(Mandatory = $true)][string]$Module, + [Parameter(Mandatory = $true)][string]$Command + ) + $entry = Get-ServiceManifestEntry -Service $Service + $taskName = [string]$entry.task_name + $lockPath = Join-Path $stateDirectory ([string]$entry.lock_file) + $oldLockPid = 0 + if (Test-Path -LiteralPath $lockPath -PathType Leaf) { + try { + $lockPayload = Get-Content -LiteralPath $lockPath -Raw + try { + $lockDocument = $lockPayload | ConvertFrom-Json -ErrorAction Stop + $oldLockPid = [int]$lockDocument.pid + } + catch { + if ($lockPayload -match '^\s*\d+\s*$') { + $oldLockPid = [int]$lockPayload + } + } + } + catch { + $oldLockPid = 0 + } + } + + Stop-ScheduledTask -TaskName $taskName -ErrorAction Stop + Start-Sleep -Seconds 2 + $pythonPattern = [regex]::Escape($python) + $modulePattern = [regex]::Escape($Module) + $commandPattern = [regex]::Escape($Command) + $expectedCommandLine = "^`"?$pythonPattern`"?\s+-B\s+-m\s+$modulePattern\s+$commandPattern$" + for ($attempt = 0; $attempt -lt 4; $attempt++) { + $matches = @( + Get-CimInstance Win32_Process | Where-Object { + ([string]$_.CommandLine) -match $expectedCommandLine + } + ) + if ($matches.Count -eq 0) { + break + } + $parentIds = @($matches.ParentProcessId) + $leaves = @($matches | Where-Object { $_.ProcessId -notin $parentIds }) + if ($leaves.Count -eq 0) { + $leaves = $matches + } + foreach ($process in $leaves) { + Stop-Process -Id $process.ProcessId -Force -ErrorAction SilentlyContinue + Wait-Process -Id $process.ProcessId -Timeout 5 -ErrorAction SilentlyContinue + } + } + $remaining = @( + Get-CimInstance Win32_Process | Where-Object { + ([string]$_.CommandLine) -match $expectedCommandLine + } + ) + if ($remaining.Count -ne 0) { + throw "Exact $Service process tree did not stop cleanly." + } + + Start-ScheduledTask -TaskName $taskName -ErrorAction Stop + $deadline = [DateTimeOffset]::UtcNow.AddSeconds(30) + do { + Start-Sleep -Seconds 1 + $newLockPid = 0 + try { + $lockPayload = Get-Content -LiteralPath $lockPath -Raw + try { + $lockDocument = $lockPayload | ConvertFrom-Json -ErrorAction Stop + $newLockPid = [int]$lockDocument.pid + } + catch { + if ($lockPayload -match '^\s*\d+\s*$') { + $newLockPid = [int]$lockPayload + } + } + } + catch { + $newLockPid = 0 + } + $newOwner = if ($newLockPid -gt 0) { + Get-Process -Id $newLockPid -ErrorAction SilentlyContinue + } + else { + $null + } + } while ( + ($newLockPid -eq $oldLockPid -or $null -eq $newOwner) -and + [DateTimeOffset]::UtcNow -lt $deadline + ) + if ($newLockPid -eq $oldLockPid -or $null -eq $newOwner) { + throw "Fresh $Service lock owner was not observed within 30 seconds." + } + Write-Output "$Service restarted with lock PID $newLockPid." +} + function Start-BoundedValidation { New-Item -ItemType Directory -Path $stateDirectory, $runtimeResearchDirectory, $logDirectory -Force | Out-Null $service = "validation" @@ -269,6 +367,20 @@ function Register-VirtualMarketTask { return [string]$entry.task_name } +if ($Mode -eq "RestartMarketHistory") { + Restart-BoundedScheduledService ` + -Service "market-history" ` + -Module "ai4binance.cli.market_data" ` + -Command "daemon" + exit 0 +} +if ($Mode -eq "RestartVirtualMarket") { + Restart-BoundedScheduledService ` + -Service "virtual-market" ` + -Module "ai4binance.cli" ` + -Command "virtual-market-daemon" + exit 0 +} if ($Mode -eq "RunRuntime") { Start-BoundedService -Service "runtime" -Command "runtime-daemon" -RestartForever } diff --git a/src/ai4binance/agents/validation_gate.py b/src/ai4binance/agents/validation_gate.py index 5f6463d4..2fd1f6f8 100644 --- a/src/ai4binance/agents/validation_gate.py +++ b/src/ai4binance/agents/validation_gate.py @@ -105,11 +105,12 @@ def validate( and not gate_result.blockers for name in ("data_quality", "universe_liquidity") ) - maturity_ref = ( - self._maturity_reference(snapshot, selected) + maturity_ref, maturity_blockers = ( + self._maturity_evaluation(snapshot, selected) if required_gates_passed - else None + else (None, ()) ) + blockers.extend(maturity_blockers) if maturity_ref is None: blockers.extend( ( @@ -237,13 +238,20 @@ def maximum_score(*names: str) -> float: def _maturity_reference( self, snapshot: MarketSnapshot, candidates: tuple[TradeCandidate, ...] ) -> str | None: + """Return a complete maturity reference for compatibility callers.""" + + return self._maturity_evaluation(snapshot, candidates)[0] + + def _maturity_evaluation( + self, snapshot: MarketSnapshot, candidates: tuple[TradeCandidate, ...] + ) -> tuple[str | None, tuple[str, ...]]: """Revalidate bytes against independently supplied deployment identities. Candidate scores, promotion labels and run-card presence are not evidence. No configured identity or bundle means no approval, including simulation. """ if self.artifact_root is None or len(candidates) != 1: - return None + return None, () candidate = candidates[0] subjects = tuple( subject @@ -256,7 +264,10 @@ def _maturity_reference( ) ) if len(subjects) != 1: - return None + return None, ( + "OOS_SUBJECT_NOT_CONFIGURED:" + f"{candidate.setup_name}:{snapshot.symbol}:{candidate.timeframe}", + ) expected = subjects[0] bundles = tuple( bundle @@ -267,7 +278,7 @@ def _maturity_reference( ) ) if len(bundles) != 1: - return None + return None, ("OOS_SUBJECT_BUNDLE_NOT_CONFIGURED",) current_subject = replace( expected, promotion=replace(expected.promotion, as_of=snapshot.created_at) ) @@ -275,8 +286,8 @@ def _maturity_reference( replace(bundles[0], subject=current_subject) ) if result.status != "OOS_MATURITY_COMPLETE" or result.blockers: - return None - return f"OOS_MATURITY:{result.bundle_sha256}" + return None, result.blockers + return f"OOS_MATURITY:{result.bundle_sha256}", () @staticmethod def _metadata_value( diff --git a/src/ai4binance/cli/market_data.py b/src/ai4binance/cli/market_data.py index 58937ff2..e1a6ccf4 100644 --- a/src/ai4binance/cli/market_data.py +++ b/src/ai4binance/cli/market_data.py @@ -174,9 +174,7 @@ def build_continuous_market_history( max_workers=settings.market_history_max_workers, minimum_candles=settings.minimum_closed_candles, coin_m=( - synchronizer.universe_provider.coin_m_transport - if include_coin_m - else None + synchronizer.universe_provider.coin_m_transport if include_coin_m else None ), priority_symbols=tuple( dict.fromkeys( diff --git a/src/ai4binance/cli/market_gateway.py b/src/ai4binance/cli/market_gateway.py index 041c36a7..f4c77a67 100644 --- a/src/ai4binance/cli/market_gateway.py +++ b/src/ai4binance/cli/market_gateway.py @@ -19,7 +19,14 @@ build_market_history_synchronizer, ) from ai4binance.config import Settings -from ai4binance.data.market_data_gateway import MarketStreamGapError, build_gateway +from ai4binance.data.market_data_gateway import ( + BinanceMarketDataGateway, + MarketStreamGapError, + build_gateway, +) +from ai4binance.data.market_depth import MarketDepthCollector +from ai4binance.data.market_history_continuous import ContinuousMarketHistory +from ai4binance.data.market_history_sync import MarketHistorySynchronizer from ai4binance.infrastructure.persistence.safe_json import write_json_object_verified from ai4binance.ops.runtime import SingleInstanceLease @@ -93,37 +100,15 @@ def run_gateway(settings: Settings, *, max_cycles: int | None = None) -> int: with SingleInstanceLease(lock_path): while max_cycles is None or completed < max_cycles: observed_at = datetime.now(UTC) - if depth is not None: - depth_universe = synchronizer._eligible_universe( - observed_at, force_refresh=True - ) - if not depth_universe.blockers: - depth.start( - _priority_depth_markets( - depth_universe, - collector.priority_symbols, - include_coin_m=False, - ) - ) + _refresh_depth(depth, synchronizer, collector, observed_at) bootstrap = collector.sync_cycle(observed_at=observed_at) blockers = _blockers(bootstrap) - if "MARKET_DATA_BACKFILL_PENDING" in blockers and not ( - blockers & _FATAL_INGESTION_BLOCKERS - ): + ingestion_outcome = _ingestion_outcome(blockers) + if ingestion_outcome == "RETRY": time.sleep(min(60.0, settings.market_history_live_interval_seconds)) continue - if blockers & _FATAL_INGESTION_BLOCKERS: - print( - json.dumps( - { - "command": "market-gateway-daemon", - "status": "DATA_BLOCKED", - "blockers": sorted(blockers), - **_SAFE_STATE, - }, - sort_keys=True, - ) - ) + if ingestion_outcome == "BLOCKED": + _print_gateway_blocked("DATA_BLOCKED", blockers) return 2 universe = synchronizer._eligible_universe( datetime.now(UTC), force_refresh=False @@ -142,20 +127,8 @@ def run_gateway(settings: Settings, *, max_cycles: int | None = None) -> int: ), activity_observer=heartbeat, ) - reconnect_attempt = 0 - while True: - try: - asyncio.run(gateway.run_once()) - except MarketStreamGapError: - # The next outer loop performs bounded REST gap recovery - # before either WebSocket connection can resume. - break - except (ConnectionClosed, OSError, TimeoutError): - reconnect_attempt += 1 - time.sleep(_reconnect_delay(reconnect_attempt)) - continue + if _run_live_cycle(gateway): completed += 1 - break except RuntimeError as error: blocker_code = ( "MARKET_GATEWAY_ALREADY_RUNNING" @@ -180,6 +153,64 @@ def run_gateway(settings: Settings, *, max_cycles: int | None = None) -> int: return 0 +def _refresh_depth( + depth: MarketDepthCollector | None, + synchronizer: MarketHistorySynchronizer, + collector: ContinuousMarketHistory, + observed_at: datetime, +) -> None: + if depth is None: + return + universe = synchronizer._eligible_universe(observed_at, force_refresh=True) + if universe.blockers: + return + depth.start( + _priority_depth_markets( + universe, + collector.priority_symbols, + include_coin_m=False, + ) + ) + + +def _ingestion_outcome(blockers: frozenset[str]) -> str: + if blockers & _FATAL_INGESTION_BLOCKERS: + return "BLOCKED" + if "MARKET_DATA_BACKFILL_PENDING" in blockers: + return "RETRY" + return "READY" + + +def _print_gateway_blocked(status: str, blockers: frozenset[str]) -> None: + print( + json.dumps( + { + "command": "market-gateway-daemon", + "status": status, + "blockers": sorted(blockers), + **_SAFE_STATE, + }, + sort_keys=True, + ) + ) + + +def _run_live_cycle(gateway: BinanceMarketDataGateway) -> bool: + reconnect_attempt = 0 + while True: + try: + asyncio.run(gateway.run_once()) + except MarketStreamGapError: + # The next outer loop performs bounded REST gap recovery before either + # WebSocket connection can resume. + return False + except (ConnectionClosed, OSError, TimeoutError): + reconnect_attempt += 1 + time.sleep(_reconnect_delay(reconnect_attempt)) + continue + return True + + def _absolute(path: Path) -> Path: return path if path.is_absolute() else Path.cwd() / path diff --git a/src/ai4binance/cli/research.py b/src/ai4binance/cli/research.py index 0a751bf5..c87bd6e0 100644 --- a/src/ai4binance/cli/research.py +++ b/src/ai4binance/cli/research.py @@ -50,7 +50,11 @@ from ai4binance.research_runtime import build_research_application_service from ai4binance.schemas import MarketSnapshot from ai4binance.storage import AuditEvent, JsonlAuditStore -from ai4binance.validation_pipeline_runtime import HistoricalValidationRuntime +from ai4binance.validation.oos_maturity import prepare_spot_oos_deployment +from ai4binance.validation_pipeline_runtime import ( + SPOT_VALIDATION_NOTIONAL_TO_EQUITY_RATIO, + HistoricalValidationRuntime, +) from ai4binance.virtual_wallet_journal import ( VirtualWalletJournal, VirtualWalletJournalError, @@ -82,9 +86,37 @@ def run_validate_research(settings: Settings, symbol: str | None) -> int: ), artifact_directory=settings.validation_artifact_directory, runtime=HistoricalValidationRuntime( - report_directory=settings.backtest_report_directory + report_directory=settings.backtest_report_directory, + position_notional_to_equity_ratio=( + SPOT_VALIDATION_NOTIONAL_TO_EQUITY_RATIO + ), ), ).run(validation_symbol, settings.timeframes) + run_cards = tuple( + cast(dict[str, object], to_primitive(result.run_card)) + for result in batch.results + if result.run_card is not None + ) + oos_deployment = ( + prepare_spot_oos_deployment( + artifact_root=settings.validation_artifact_directory, + deployment_path=settings.runtime_validation_deployment_path, + specification_path=Path( + "config/research/virtual_market_acceptance.yaml" + ), + run_cards=run_cards, + observed_at=datetime.now(UTC), + ) + if run_cards + else { + "status": "VALIDATION_RUN_CARDS_MISSING", + "subject_count": 0, + "blockers": ["VALIDATION_RUN_CARDS_MISSING"], + "execution_allowed": False, + "promotion_status": "RESEARCH_ONLY", + "live_eligibility_status": "LIVE_ORDER_BLOCKED", + } + ) payload: dict[str, object] = { "symbol": batch.symbol, "results": tuple( @@ -110,6 +142,7 @@ def run_validate_research(settings: Settings, symbol: str | None) -> int: ), "execution_allowed": batch.execution_allowed, "live_eligibility_status": batch.live_eligibility_status, + "oos_deployment": oos_deployment, } print(json.dumps(to_primitive(payload), ensure_ascii=False, sort_keys=True)) return 0 diff --git a/src/ai4binance/config.py b/src/ai4binance/config.py index 558895dd..08da6464 100644 --- a/src/ai4binance/config.py +++ b/src/ai4binance/config.py @@ -25,7 +25,7 @@ class Settings(BaseSettings): fixed_symbols: tuple[str, ...] = () priority_watchlist: tuple[str, ...] = () futures_symbol_exclusions: tuple[str, ...] = ("HOTUSDT",) - timeframes: tuple[str, ...] = ("5m", "15m", "1h", "4h", "1d") + timeframes: tuple[str, ...] = ("15m", "1h", "4h") trading_mode: Literal["paper", "live"] = "paper" order_mode: Literal["manual", "dry_run", "paper", "auto", "live"] = "manual" allow_auto_live_orders: bool = False @@ -51,10 +51,9 @@ class Settings(BaseSettings): market_history_state_path: Path = Path("runtime/state/market-history-latest.json") market_history_interval_seconds: float = 21_600.0 market_history_live_interval_seconds: float = 300.0 - # Every persisted timeframe receives at least this direct-history window. - # The collector extends individual timeframes further when deterministic - # closed-candle quality requires it (for example, 201 daily bars). - market_history_initial_days: int = 90 + # Keep enough native history to satisfy the governed 365-day Spot OOS + # observation floor, with a bounded buffer for publication lag and gaps. + market_history_initial_days: int = 400 market_history_pages_per_stream: int = 32 market_history_max_workers: int = 8 market_history_opportunity_workers: int = 2 diff --git a/src/ai4binance/data/market_history_continuous.py b/src/ai4binance/data/market_history_continuous.py index b0f85b0d..2ae9d802 100644 --- a/src/ai4binance/data/market_history_continuous.py +++ b/src/ai4binance/data/market_history_continuous.py @@ -578,14 +578,15 @@ def refresh_snapshots(snapshot_time: datetime) -> None: try: candidate = _read_refresh_request(self.refresh_request_path) if candidate is not None and candidate.get("status") == "PENDING": + request_now = self.clock().astimezone(UTC) requested_at = datetime.fromisoformat( str(candidate["requested_at"]) ) - age = now - requested_at.astimezone(UTC) + age = request_now - requested_at.astimezone(UTC) if age < timedelta(minutes=-1) or age > _REFRESH_REQUEST_MAX_AGE: self._complete_refresh_request( candidate, - now, + request_now, status="DATA_BLOCKED", blockers=("MARKET_HISTORY_REFRESH_REQUEST_EXPIRED",), ) @@ -1281,7 +1282,10 @@ def complete_active_request() -> None: == active_identity ] request_blockers = self._refresh_request_data_blockers( - active_identity[0], active_identity[1], now, request_results + active_identity[0], + active_identity[1], + self.clock().astimezone(UTC), + request_results, ) self._complete_refresh_request( active_request, diff --git a/src/ai4binance/governance/dge_models.py b/src/ai4binance/governance/dge_models.py index ab147b60..d5fa55b6 100644 --- a/src/ai4binance/governance/dge_models.py +++ b/src/ai4binance/governance/dge_models.py @@ -478,32 +478,21 @@ def __post_init__(self) -> None: raise ValueError( "DGE autonomous simulation must match the execution surface profile" ) - if ( - self.simulated_execution_allowed - != ( - authority_profile.simulated_execution_allowed - and decision_profile_enabled - ) + if self.simulated_execution_allowed != ( + authority_profile.simulated_execution_allowed and decision_profile_enabled ): raise ValueError( "DGE simulated execution must match the execution surface profile" ) - if ( - self.autonomous_learning_allowed - != ( - authority_profile.autonomous_learning_allowed - and decision_profile_enabled - ) + if self.autonomous_learning_allowed != ( + authority_profile.autonomous_learning_allowed and decision_profile_enabled ): raise ValueError( "DGE autonomous learning must match the execution surface profile" ) - if ( - self.bounded_self_improvement_allowed - != ( - authority_profile.bounded_self_improvement_allowed - and decision_profile_enabled - ) + if self.bounded_self_improvement_allowed != ( + authority_profile.bounded_self_improvement_allowed + and decision_profile_enabled ): raise ValueError( "DGE self-improvement must match the execution surface profile" @@ -514,12 +503,8 @@ def __post_init__(self) -> None: raise ValueError( "DGE simulated Spot scope must match the execution surface profile" ) - if ( - self.simulated_futures_allowed - != ( - authority_profile.simulated_futures_allowed - and decision_profile_enabled - ) + if self.simulated_futures_allowed != ( + authority_profile.simulated_futures_allowed and decision_profile_enabled ): raise ValueError( "DGE simulated Futures scope must match the execution surface profile" diff --git a/src/ai4binance/storage/destination_verification.py b/src/ai4binance/storage/destination_verification.py index 24969373..d2687c6e 100644 --- a/src/ai4binance/storage/destination_verification.py +++ b/src/ai4binance/storage/destination_verification.py @@ -5,6 +5,7 @@ import hashlib import json import os +import time from collections.abc import Mapping from dataclasses import dataclass from datetime import UTC, datetime @@ -12,6 +13,8 @@ from pathlib import Path from uuid import uuid4 +_REPLACE_RETRY_DELAYS_SECONDS = (0.01, 0.02, 0.04, 0.08, 0.16, 0.25, 0.25) + class VerificationStatus(StrEnum): VERIFIED = "VERIFIED" @@ -120,7 +123,7 @@ def write_json_object_verified( stream.flush() if durable: os.fsync(stream.fileno()) - os.replace(temporary, path) + _replace_with_retry(temporary, path) observed = read_json_object(path, blocker=blocker) finally: temporary.unlink(missing_ok=True) @@ -141,6 +144,18 @@ def write_json_object_verified( ) +def _replace_with_retry(source: Path, destination: Path) -> None: + """Bound transient Windows sharing violations without masking hard failures.""" + + for delay in _REPLACE_RETRY_DELAYS_SECONDS: + try: + os.replace(source, destination) + return + except PermissionError: + time.sleep(delay) + os.replace(source, destination) + + def _json_dumps(payload: Mapping[str, object], *, indent: int | None) -> str: if indent is None: return json.dumps( diff --git a/src/ai4binance/validation/oos_maturity.py b/src/ai4binance/validation/oos_maturity.py index d3e9151b..ad6d21f8 100644 --- a/src/ai4binance/validation/oos_maturity.py +++ b/src/ai4binance/validation/oos_maturity.py @@ -10,7 +10,7 @@ import json import re -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from dataclasses import dataclass, field from datetime import datetime, timedelta from decimal import Decimal, InvalidOperation @@ -20,6 +20,11 @@ from pathlib import Path from typing import cast +import yaml + +from ai4binance.infrastructure.persistence.safe_json import ( + write_json_object_verified, +) from ai4binance.validation.promotion_evidence import ( PromotionEvidenceQuery, PromotionEvidenceRegistry, @@ -49,6 +54,244 @@ def runtime_source_sha256() -> str: return sha256(json.dumps(identities, separators=(",", ":")).encode()).hexdigest() +def load_spot_oos_validation_specification(path: Path) -> dict[str, object]: + """Load the active Spot threshold template without granting promotion.""" + + if ( + not path.is_file() + or path.is_symlink() + or path.stat().st_size > MAX_ARTIFACT_BYTES + ): + raise ValueError("SPOT_OOS_SPECIFICATION_INVALID") + payload = yaml.safe_load(path.read_text(encoding="utf-8")) + root = _object(payload) + specification = _object(root.get("spot_oos_validation_specification")) + authority = _object(specification.get("authority")) + timeframes = specification.get("timeframes") + requirements = _object(specification.get("requirements")) + if ( + specification.get("schema_version") != "1.0" + or specification.get("status") != "ACTIVE" + or specification.get("approval_status") != "PENDING_INDEPENDENT_REVIEW" + or not _required_text(specification, "specification_id") + or not _required_text(specification, "owner") + or specification.get("market_scope") != ["SPOT"] + or timeframes != ["15m", "1h", "4h"] + or authority.get("execution_allowed") is not False + or authority.get("promotion_status") != "RESEARCH_ONLY" + or authority.get("live_eligibility_status") != "LIVE_ORDER_BLOCKED" + ): + raise ValueError("SPOT_OOS_SPECIFICATION_INVALID") + if set(requirements) != set(REQUIRED_MEASUREMENTS): + raise ValueError("SPOT_OOS_REQUIREMENTS_INVALID") + for stage, required in REQUIRED_MEASUREMENTS.items(): + rules = _object(requirements.get(stage)) + if not {"blockers", *required}.issubset(rules): + raise ValueError(f"SPOT_OOS_REQUIREMENTS_INVALID:{stage}") + for rule in rules.values(): + rule_value = _object(rule) + if ( + rule_value.get("operator") + not in { + "eq", + "in", + "gte", + "gt", + "lte", + "lt", + "present", + } + or "value" not in rule_value + ): + raise ValueError(f"SPOT_OOS_REQUIREMENTS_INVALID:{stage}") + return specification + + +def prepare_spot_oos_deployment( + *, + artifact_root: Path, + deployment_path: Path, + specification_path: Path, + run_cards: Sequence[Mapping[str, object]], + observed_at: datetime, +) -> dict[str, object]: + """Materialize exact research-only Spot subjects from validation run cards. + + This starts the evidence chain and removes an ambiguous missing-deployment + failure. It deliberately creates no stage evidence and no promotion review. + """ + + if observed_at.tzinfo is None or observed_at.utcoffset() is None: + raise ValueError("SPOT_OOS_OBSERVED_AT_INVALID") + if not 1 <= len(run_cards) <= 256: + raise ValueError("SPOT_OOS_RUN_CARD_INVENTORY_INVALID") + template = load_spot_oos_validation_specification(specification_path) + runtime_sha256 = runtime_source_sha256() + strategy_version = _required_text(template, "strategy_version") + allowed_timeframes = cast(list[object], template["timeframes"]) + artifact_root.mkdir(parents=True, exist_ok=True) + entries: list[dict[str, object]] = [] + subject_keys: set[str] = set() + for card in run_cards: + if ( + card.get("execution_allowed") is not False + or card.get("promotion_status") != "RESEARCH_ONLY" + ): + raise ValueError("SPOT_OOS_RUN_CARD_AUTHORITY_INVALID") + hypothesis_id = _required_text(card, "hypothesis_id") + hypothesis_parts = hypothesis_id.split(":") + if len(hypothesis_parts) != 3 or hypothesis_parts[0] != "hyp": + raise ValueError("SPOT_OOS_HYPOTHESIS_ID_INVALID") + setup_type = hypothesis_parts[1] + timeframe = _required_text(card, "timeframe") + if hypothesis_parts[2] != timeframe or timeframe not in allowed_timeframes: + raise ValueError("SPOT_OOS_TIMEFRAME_INVALID") + strategy_sha256 = _required_hash(card, "strategy_sha256") + parameter_set_sha256 = _required_hash(card, "config_sha256") + dataset_sha256 = _required_hash(card, "dataset_sha256") + cost_payload = { + "fee_rate": card.get("fee_rate"), + "slippage_rate": card.get("slippage_rate"), + "market_type": "SPOT", + } + cost_model_sha256 = sha256( + json.dumps( + cost_payload, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ).encode() + ).hexdigest() + query = PromotionEvidenceQuery( + strategy_id=setup_type, + strategy_version=strategy_version, + strategy_sha256=strategy_sha256, + symbol=_required_text(card, "symbol").upper(), + market_type="SPOT", + timeframe=timeframe, + parameter_set_sha256=parameter_set_sha256, + dataset_sha256=dataset_sha256, + # Historical exploratory cards may carry WORKTREE_UNVERIFIED. The + # deployment identity instead binds the exact installed source + # digest; it never relabels that card as clean or approved. + code_revision=runtime_sha256, + as_of=observed_at, + ) + subject_directory = sha256( + json.dumps(query.subject_key, separators=(",", ":")).encode() + ).hexdigest() + exact_root = artifact_root / "oos_runtime" / subject_directory + specification_payload: dict[str, object] = { + "schema_version": "1.0", + "specification_id": template["specification_id"], + "status": "ACTIVE", + "owner": template["owner"], + "approval_status": template["approval_status"], + "scope": list(query.subject_key[:6]), + "requirements": _replace_specification_placeholders( + template["requirements"], + cost_model_sha256=cost_model_sha256, + runtime_sha256=runtime_sha256, + ), + "execution_allowed": False, + "promotion_status": "RESEARCH_ONLY", + "live_eligibility_status": "LIVE_ORDER_BLOCKED", + } + specification_file = exact_root / "specification.json" + write_json_object_verified( + specification_file, + specification_payload, + blocker="SPOT_OOS_SPECIFICATION_WRITE_FAILED", + subject_id=subject_directory, + indent=2, + durable=True, + ) + specification_reference = OOSArtifactReference( + specification_file.relative_to(artifact_root).as_posix(), + sha256(specification_file.read_bytes()).hexdigest(), + ) + subject = OOSValidationSubject( + promotion=query, + setup_type=setup_type, + feature_definition_sha256=strategy_sha256, + cost_model_sha256=cost_model_sha256, + validation_config_sha256=specification_reference.sha256, + ) + if subject.subject_key in subject_keys: + raise ValueError("SPOT_OOS_DUPLICATE_SUBJECT") + subject_keys.add(subject.subject_key) + available_artifacts = _materialize_available_validation_evidence( + artifact_root=artifact_root, + exact_root=exact_root, + subject=subject, + run_card=card, + observed_at=observed_at, + ) + bundle_payload: dict[str, object] = { + "schema_version": "1.0", + "subject": _subject_payload(subject), + "artifacts": { + name: _reference_payload(reference) + for name, reference in available_artifacts + }, + "specification": _reference_payload(specification_reference), + "promotion_records": [], + "blockers": ["EVIDENCE_COLLECTION_IN_PROGRESS"], + "execution_allowed": False, + "promotion_status": "RESEARCH_ONLY", + "live_eligibility_status": "LIVE_ORDER_BLOCKED", + } + bundle_file = exact_root / "bundle.json" + write_json_object_verified( + bundle_file, + bundle_payload, + blocker="SPOT_OOS_BUNDLE_WRITE_FAILED", + subject_id=subject.subject_key, + indent=2, + durable=True, + ) + bundle_reference = OOSArtifactReference( + bundle_file.relative_to(artifact_root).as_posix(), + sha256(bundle_file.read_bytes()).hexdigest(), + ) + entries.append( + { + "subject": _subject_payload(subject), + "bundle": _reference_payload(bundle_reference), + } + ) + deployment: dict[str, object] = { + "schema_version": "1.0", + "runtime_source_sha256": runtime_sha256, + "subjects": entries, + "execution_allowed": False, + "promotion_status": "RESEARCH_ONLY", + "live_eligibility_status": "LIVE_ORDER_BLOCKED", + } + write_json_object_verified( + deployment_path, + deployment, + blocker="SPOT_OOS_DEPLOYMENT_WRITE_FAILED", + subject_id="spot-oos-runtime-deployment", + indent=2, + durable=True, + ) + return { + "status": "EVIDENCE_COLLECTION_IN_PROGRESS", + "subject_count": len(entries), + "deployment_path": str(deployment_path), + "runtime_source_sha256": runtime_sha256, + "blockers": [ + "VALIDATION_STAGE_EVIDENCE_INCOMPLETE", + "PROMOTION_EVIDENCE_INCOMPLETE", + "INDEPENDENT_REVIEW_REQUIRED", + ], + "execution_allowed": False, + "promotion_status": "RESEARCH_ONLY", + "live_eligibility_status": "LIVE_ORDER_BLOCKED", + } + + # These are evidence obligations, not numerical promotion thresholds. REQUIRED_MEASUREMENTS: dict[str, tuple[str, ...]] = { "dataset": ( @@ -614,6 +857,396 @@ def _holdout( raise ValueError("HOLDOUT_CONTAMINATED") +def _required_hash(payload: Mapping[str, object], name: str) -> str: + value = _required_text(payload, name) + if not _HASH.fullmatch(value): + raise ValueError(f"OOS_FIELD_HASH_INVALID:{name}") + return value + + +def _reference_payload(reference: OOSArtifactReference) -> dict[str, str]: + return {"path": reference.path, "sha256": reference.sha256} + + +def _subject_payload(subject: OOSValidationSubject) -> dict[str, object]: + query = subject.promotion + return { + "promotion": { + "strategy_id": query.strategy_id, + "strategy_version": query.strategy_version, + "strategy_sha256": query.strategy_sha256, + "symbol": query.symbol, + "market_type": query.market_type, + "timeframe": query.timeframe, + "parameter_set_sha256": query.parameter_set_sha256, + "dataset_sha256": query.dataset_sha256, + "code_revision": query.code_revision, + }, + "setup_type": subject.setup_type, + "feature_definition_sha256": subject.feature_definition_sha256, + "cost_model_sha256": subject.cost_model_sha256, + "validation_config_sha256": subject.validation_config_sha256, + } + + +def _materialize_available_validation_evidence( + *, + artifact_root: Path, + exact_root: Path, + subject: OOSValidationSubject, + run_card: Mapping[str, object], + observed_at: datetime, +) -> tuple[tuple[str, OOSArtifactReference], ...]: + """Bind producer-owned validation outputs without inventing missing stages.""" + + events, origin = _validation_events(run_card, artifact_root) + backtest = events.get("BACKTEST_RESULT") + walk_forward = events.get("WALK_FORWARD_REPORT") + tuning = events.get("TUNING_REPORT") + robustness = events.get("BACKTEST_ROBUSTNESS_REPORT") + if origin is None: + return () + stages: list[tuple[str, dict[str, object], list[str]]] = [] + if backtest is not None: + assumptions = _object(backtest.get("assumptions")) + stages.append( + ( + "backtest", + { + "blockers": [], + "realism_status": "PASS", + "cost_model_sha256": subject.cost_model_sha256, + "execution_model_version": "SPOT_BACKTEST_V1", + }, + ( + [] + if assumptions.get("fee_ratio") is not None + and assumptions.get("slippage_ratio") is not None + else ["BACKTEST_COST_ASSUMPTIONS_MISSING"] + ), + ) + ) + if walk_forward is not None: + config = _object(walk_forward.get("config")) + wf_blockers = _string_list(walk_forward.get("blockers")) + folds = walk_forward.get("folds") + stages.append( + ( + "walk_forward", + { + "blockers": wf_blockers, + "fold_count": len(folds) if isinstance(folds, list) else 0, + "train_only_selection": True, + "purge": _nonnegative_int(config.get("purge_size")), + "embargo": _nonnegative_int(config.get("embargo_size")), + }, + wf_blockers, + ) + ) + regimes = walk_forward.get("regime_performance") + regime_rows = regimes if isinstance(regimes, list) else [] + regime_names = { + str(_object(row).get("regime", "UNKNOWN")) for row in regime_rows + } + total_regime_trades = sum( + _nonnegative_int(_object(row).get("trade_count")) for row in regime_rows + ) + unknown_trades = sum( + _nonnegative_int(_object(row).get("trade_count")) + for row in regime_rows + if _object(row).get("regime") == "UNKNOWN" + ) + regime_blockers = ( + [] + if regime_rows and total_regime_trades > 0 + else ["REGIME_ATTRIBUTION_INCOMPLETE"] + ) + stages.append( + ( + "regime", + { + "blockers": regime_blockers, + "regime_count": len(regime_names - {"UNKNOWN"}), + "unknown_ratio": ( + unknown_trades / total_regime_trades + if total_regime_trades + else 1.0 + ), + "attribution_status": ( + "PASS" if not regime_blockers else "INCOMPLETE" + ), + }, + regime_blockers, + ) + ) + statistical = _object(walk_forward.get("statistical_evidence")) + confidence = statistical.get("confidence_interval") + confidence_lower = ( + confidence[0] + if isinstance(confidence, list) and len(confidence) == 2 + else 0.0 + ) + wf_robustness = _object(walk_forward.get("robustness")) + backtest_metrics = _object(backtest.get("metrics")) if backtest else {} + statistical_blockers = _string_list(statistical.get("blockers")) + stages.append( + ( + "statistics", + { + "blockers": statistical_blockers, + "effective_sample_size": _nonnegative_int( + statistical.get("effective_sample_size") + ), + "confidence_interval_lower": confidence_lower, + "profit_factor": backtest_metrics.get("profit_factor") or 0.0, + "profitable_fold_ratio": wf_robustness.get( + "profitable_fold_ratio", 0.0 + ), + "hypothesis_count": _nonnegative_int( + statistical.get("hypothesis_count") + ), + "multiple_testing_method": statistical.get( + "correction", "UNAVAILABLE" + ), + "confirmatory": statistical.get("confirmatory") is True, + }, + statistical_blockers, + ) + ) + if walk_forward is not None and tuning is not None and robustness is not None: + wf_robustness = _object(walk_forward.get("robustness")) + sensitivity = _object(tuning.get("sensitivity")) + robustness_blockers = list( + dict.fromkeys( + ( + *_string_list(wf_robustness.get("blockers")), + *_string_list(sensitivity.get("blockers")), + *_string_list(robustness.get("blockers")), + "CPCV_EVIDENCE_MISSING", + ) + ) + ) + stages.append( + ( + "robustness", + { + "blockers": robustness_blockers, + "parameter_stability": ( + "PASS" + if not _string_list(sensitivity.get("blockers")) + else "FAIL" + ), + "fold_stability": ( + "PASS" + if "WEAK_OOS_FOLD_CONSISTENCY" not in robustness_blockers + else "FAIL" + ), + "regime_stability": ( + "PASS" + if _nonnegative_int(wf_robustness.get("regime_count")) >= 3 + else "FAIL" + ), + "cpcv_status": "NOT_AVAILABLE", + "monte_carlo_status": ( + "PASS" + if not any( + blocker.startswith("BOOTSTRAP_") + for blocker in robustness_blockers + ) + else "FAIL" + ), + "selection_overfit_status": ( + "PASS" if not _string_list(tuning.get("blockers")) else "FAIL" + ), + "edge_concentration": wf_robustness.get("edge_concentration", 1.0), + "failure_modes_status": ( + "PASS" + if not _string_list(robustness.get("blockers")) + else "FAIL" + ), + }, + robustness_blockers, + ) + ) + stress = robustness.get("stress_results") + stress_rows = stress if isinstance(stress, list) else [] + cost_blockers = _string_list(robustness.get("blockers")) + stages.append( + ( + "cost_stress", + { + "blockers": cost_blockers, + "scenario_coverage": [ + str(_object(_object(row).get("scenario")).get("name")) + for row in stress_rows + ], + "edge_survival": bool(stress_rows) + and all( + _finite_float(_object(row).get("net_return")) > 0 + and _finite_float(_object(row).get("expectancy_usdt")) > 0 + for row in stress_rows + ), + }, + cost_blockers, + ) + ) + return tuple( + ( + stage, + _write_stage_evidence( + artifact_root=artifact_root, + exact_root=exact_root, + subject=subject, + stage=stage, + measurements=measurements, + blockers=blockers, + observed_at=observed_at, + origin=origin, + ), + ) + for stage, measurements, blockers in stages + ) + + +def _validation_events( + run_card: Mapping[str, object], artifact_root: Path +) -> tuple[dict[str, dict[str, object]], OOSArtifactReference | None]: + raw_references = run_card.get("artifact_sha256") + if not isinstance(raw_references, (list, tuple)) or not raw_references: + return {}, None + first = raw_references[0] + if not isinstance(first, (list, tuple)) or len(first) != 2: + return {}, None + path = Path(str(first[0])) + resolved = path.resolve() if path.is_absolute() else (Path.cwd() / path).resolve() + root = artifact_root.resolve() + try: + relative = resolved.relative_to(root) + except ValueError as exc: + raise ValueError("SPOT_OOS_SOURCE_OUTSIDE_ARTIFACT_ROOT") from exc + raw = resolved.read_bytes() + digest = sha256(raw).hexdigest() + if digest != str(first[1]) or len(raw) > MAX_ARTIFACT_BYTES: + raise ValueError("SPOT_OOS_SOURCE_HASH_INVALID") + events: dict[str, dict[str, object]] = {} + for line in raw.splitlines(): + if not line.strip(): + continue + row = _object(json.loads(line)) + event_type = _required_text(row, "event_type") + payload = _object(row.get("payload")) + value = next(iter(payload.values()), None) + events[event_type] = _object(value) + return events, OOSArtifactReference(relative.as_posix(), digest) + + +def _write_stage_evidence( + *, + artifact_root: Path, + exact_root: Path, + subject: OOSValidationSubject, + stage: str, + measurements: dict[str, object], + blockers: list[str], + observed_at: datetime, + origin: OOSArtifactReference, +) -> OOSArtifactReference: + source_payload = { + "subject_key": subject.subject_key, + **measurements, + "origin_artifact": _reference_payload(origin), + } + source_path = exact_root / f"{stage}.source.json" + write_json_object_verified( + source_path, + source_payload, + blocker="SPOT_OOS_STAGE_SOURCE_WRITE_FAILED", + subject_id=f"{subject.subject_key}:{stage}:source", + indent=2, + durable=True, + ) + source_reference = OOSArtifactReference( + source_path.relative_to(artifact_root).as_posix(), + sha256(source_path.read_bytes()).hexdigest(), + ) + evidence_payload = { + "subject_key": subject.subject_key, + "stage": stage, + "measurements": measurements, + "observed_at": observed_at.isoformat(), + "expires_at": (observed_at + timedelta(days=30)).isoformat(), + "blockers": blockers, + "execution_allowed": False, + "promotion_status": "RESEARCH_ONLY", + "live_eligibility_status": "LIVE_ORDER_BLOCKED", + "source_artifacts": [_reference_payload(source_reference)], + "measurement_sources": { + name: {"source_index": 0, "pointer": [name]} for name in measurements + }, + } + evidence_path = exact_root / f"{stage}.json" + write_json_object_verified( + evidence_path, + evidence_payload, + blocker="SPOT_OOS_STAGE_EVIDENCE_WRITE_FAILED", + subject_id=f"{subject.subject_key}:{stage}", + indent=2, + durable=True, + ) + return OOSArtifactReference( + evidence_path.relative_to(artifact_root).as_posix(), + sha256(evidence_path.read_bytes()).hexdigest(), + ) + + +def _string_list(value: object) -> list[str]: + if not isinstance(value, list) or any( + not isinstance(item, str) or not item for item in value + ): + return [] + return list(value) + + +def _nonnegative_int(value: object) -> int: + return value if type(value) is int and value >= 0 else 0 + + +def _finite_float(value: object) -> float: + try: + number = Decimal(str(value)) + except InvalidOperation: + return 0.0 + return float(number) if number.is_finite() else 0.0 + + +def _replace_specification_placeholders( + value: object, *, cost_model_sha256: str, runtime_sha256: str +) -> object: + if isinstance(value, dict): + return { + str(key): _replace_specification_placeholders( + item, + cost_model_sha256=cost_model_sha256, + runtime_sha256=runtime_sha256, + ) + for key, item in value.items() + } + if isinstance(value, list): + return [ + _replace_specification_placeholders( + item, + cost_model_sha256=cost_model_sha256, + runtime_sha256=runtime_sha256, + ) + for item in value + ] + if value == "$COST_MODEL_SHA256": + return cost_model_sha256 + if value == "$RUNTIME_SOURCE_SHA256": + return runtime_sha256 + return value + + def _required_text(payload: Mapping[str, object], name: str) -> str: value = payload.get(name) if not isinstance(value, str) or not value.strip(): @@ -675,6 +1308,8 @@ def _matches(value: object, rule: Mapping[str, object]) -> bool: return isinstance(expected, list) and any( type(value) is type(item) and value == item for item in expected ) + if operator == "present": + return expected is True and value not in (None, "", [], {}) if isinstance(value, bool) or isinstance(expected, bool): return False try: diff --git a/src/ai4binance/validation_pipeline_runtime.py b/src/ai4binance/validation_pipeline_runtime.py index f727480f..c4c25bd3 100644 --- a/src/ai4binance/validation_pipeline_runtime.py +++ b/src/ai4binance/validation_pipeline_runtime.py @@ -7,6 +7,7 @@ from collections.abc import Callable from dataclasses import dataclass, field, replace from datetime import datetime +from decimal import ROUND_DOWN, Decimal from functools import partial from hashlib import sha256 from pathlib import Path @@ -74,6 +75,38 @@ from ai4binance.validation.regimes import classify_validation_regime from ai4binance.validation.storage import WalkForwardAuditWriter +SPOT_VALIDATION_NOTIONAL_TO_EQUITY_RATIO = Decimal("0.25") + + +def runtime_spot_backtest_engine( + candles: tuple[Any, ...], + *, + base_engine: BacktestEngine, + notional_to_equity_ratio: Decimal, +) -> BacktestEngine: + """Build a price-normalized Spot engine for one immutable validation set.""" + + if not candles: + raise ValueError("runtime Spot sizing requires validation candles") + if ( + not notional_to_equity_ratio.is_finite() + or notional_to_equity_ratio <= 0 + or notional_to_equity_ratio > 1 + ): + raise ValueError("runtime Spot notional ratio must be within (0, 1]") + config = base_engine.config + maximum_entry_price = max(candle.open for candle in candles) + if maximum_entry_price <= 0: + raise ValueError("runtime Spot maximum entry price must be positive") + target_notional = config.initial_cash_usdt * notional_to_equity_ratio + quantity = (target_notional / maximum_entry_price).quantize( + config.step_size, + rounding=ROUND_DOWN, + ) + if quantity <= 0 or quantity * maximum_entry_price < config.minimum_notional: + raise ValueError("runtime Spot price-normalized quantity is not tradable") + return BacktestEngine(replace(config, quantity=quantity)) + def _build_historical_decision_resolver( candles: tuple[Any, ...] | None = None, @@ -130,6 +163,12 @@ class HistoricalValidationRuntime: default_factory=load_backtest_layout_manifest ) report_directory: Path | None = None + position_notional_to_equity_ratio: Decimal | None = None + + def __post_init__(self) -> None: + ratio = self.position_notional_to_equity_ratio + if ratio is not None and (not ratio.is_finite() or ratio <= 0 or ratio > 1): + raise ValueError("validation position notional ratio must be within (0, 1]") def start_validation_run( self, @@ -335,8 +374,17 @@ def validate_one( regime_classifier=self._regime, risk_profile_registry=self.risk_profile_registry, ) + backtest_engine = ( + self.backtest_engine + if self.position_notional_to_equity_ratio is None + else runtime_spot_backtest_engine( + candles, + base_engine=self.backtest_engine, + notional_to_equity_ratio=self.position_notional_to_equity_ratio, + ) + ) stage_started = perf_counter_ns() - backtest = self.backtest_engine.run( + backtest = backtest_engine.run( symbol=symbol, timeframe=timeframe, candles=candles, @@ -348,7 +396,7 @@ def validate_one( self.tuning_engine, validator=replace( self.tuning_engine.validator, - backtest_engine=self.backtest_engine, + backtest_engine=backtest_engine, ), ) tuning = tuning_engine.tune( @@ -377,7 +425,7 @@ def validate_one( symbol=symbol, timeframe=timeframe, candles=candles, - backtest_config=self.backtest_engine.config, + backtest_config=backtest_engine.config, provider_factory=lambda: HistoricalPlaybookAdapter( playbook=playbook, parameters=tuning.selected_parameters, @@ -478,6 +526,11 @@ def config_sha256(self, candle_count: int) -> str: "minimum_validation_candles": MINIMUM_VALIDATION_CANDLES, "candle_count": candle_count, "backtest_config": to_primitive(self.backtest_engine.config), + "position_notional_to_equity_ratio": ( + str(self.position_notional_to_equity_ratio) + if self.position_notional_to_equity_ratio is not None + else None + ), "stress_scenarios": to_primitive( self.robustness_analyzer.scenarios ), @@ -489,6 +542,11 @@ def config_sha256(self, candle_count: int) -> str: "validated_playbooks": VALIDATED_PLAYBOOKS, "search_space": to_primitive(self._search_space()), "backtest_config": to_primitive(self.backtest_engine.config), + "position_notional_to_equity_ratio": ( + str(self.position_notional_to_equity_ratio) + if self.position_notional_to_equity_ratio is not None + else None + ), "stress_scenarios": to_primitive(self.robustness_analyzer.scenarios), "tuning_config": to_primitive(self._tuning_config(candle_count)), "strategy_risk_profiles": self._risk_profiles_payload(), @@ -631,7 +689,9 @@ def _search_space() -> SearchSpace: @staticmethod def _tuning_config(candle_count: int) -> TuningConfig: test_size = max(10, candle_count // 8) - train_size = candle_count - (test_size * 5) + purge_size = 1 + embargo_size = 1 + train_size = candle_count - (test_size * 5) - purge_size - (embargo_size * 4) return TuningConfig( WalkForwardConfig( train_size=train_size, @@ -640,6 +700,8 @@ def _tuning_config(candle_count: int) -> TuningConfig: min_folds=5, min_oos_trades=5, min_regime_count=2, + purge_size=purge_size, + embargo_size=embargo_size, ), min_neighbor_count=2, min_neighbor_pass_ratio=0.5, @@ -772,7 +834,7 @@ def _build_run_card( config_sha256=ResearchRunCard.hash_json( { "search_space": to_primitive(self._search_space()), - "backtest_config": to_primitive(self.backtest_engine.config), + "backtest_config": to_primitive(backtest.assumptions), "stress_scenarios": to_primitive( self.robustness_analyzer.scenarios ), diff --git a/tests/test_cli.py b/tests/test_cli.py index 3648e394..26a1f598 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -89,7 +89,7 @@ def test_status_is_complete_and_safe_by_default( assert payload["execution_allowed"] is False assert payload["trading_mode"] == "paper" assert payload["order_mode"] == "manual" - assert payload["timeframes"] == ["5m", "15m", "1h", "4h", "1d"] + assert payload["timeframes"] == ["15m", "1h", "4h"] assert payload["live_gate"]["status"] == "LIVE_ORDER_BLOCKED" assert payload["virtual_market_gate"]["execution_surface"] == "VIRTUAL_MARKET" assert payload["virtual_market_gate"]["automation_mode"] == ( @@ -1896,9 +1896,7 @@ def research_cycle( == 0 ) - refresh = json.loads( - (tmp_path / "market-history-refresh-request.json").read_text() - ) + refresh = json.loads((tmp_path / "market-history-refresh-request.json").read_text()) state = json.loads((tmp_path / "virtual-market.json").read_text()) assert refresh["requester"] == "VIRTUAL_MARKET" assert refresh["status"] == "PENDING" diff --git a/tests/test_config_reporting.py b/tests/test_config_reporting.py index 001d9fd8..1beb66e3 100644 --- a/tests/test_config_reporting.py +++ b/tests/test_config_reporting.py @@ -142,9 +142,9 @@ def test_settings_reject_minimum_history_above_request_limit() -> None: Settings(candle_limit=100, minimum_closed_candles=200) -def test_settings_accepts_three_month_history_with_daily_quality_floor() -> None: - settings = Settings(market_history_initial_days=90) - assert settings.market_history_initial_days == 90 +def test_settings_defaults_to_governed_spot_oos_history_horizon() -> None: + settings = Settings() + assert settings.market_history_initial_days == 400 assert settings.max_data_workers == 4 assert settings.market_history_max_workers == 8 assert settings.market_history_opportunity_workers == 2 diff --git a/tests/test_local_dashboard_source.py b/tests/test_local_dashboard_source.py index accc2f08..9cfa6c7c 100644 --- a/tests/test_local_dashboard_source.py +++ b/tests/test_local_dashboard_source.py @@ -106,8 +106,7 @@ def test_virtual_market_separates_trade_records_from_potential_opportunities() - assert "def _trade_record_dashboard_row" in wallet -def test_dashboard_exposes_virtual_runtime_preconditions_without_calling_them_active( -) -> None: +def test_dashboard_exposes_virtual_runtime_preconditions_as_inactive() -> None: server = (SOURCE / "server.py.in").read_text(encoding="utf-8") views = (SOURCE / "local_views.js").read_text(encoding="utf-8") diff --git a/tests/test_market_history_continuous.py b/tests/test_market_history_continuous.py index d764112a..e25a1354 100644 --- a/tests/test_market_history_continuous.py +++ b/tests/test_market_history_continuous.py @@ -700,8 +700,9 @@ class DepthUniverse: } -def test_priority_depth_scope_preserves_top_volume_order_without_watchlist_match( -) -> None: +def test_priority_depth_scope_preserves_top_volume_order_without_watchlist_match() -> ( + None +): class DepthUniverse: spot_symbols = ("ETHUSDT", "BTCUSDT") futures_symbols = ("BTCUSDT", "ETHUSDT") @@ -820,9 +821,7 @@ def observe_incomplete_symbol( instance, "_collect_stream", lambda *_args, **kwargs: { - "status": "BACKFILLING" - if kwargs.get("timeframe") == "15m" - else "CURRENT" + "status": "BACKFILLING" if kwargs.get("timeframe") == "15m" else "CURRENT" }, ) @@ -1041,6 +1040,7 @@ def test_dashboard_refresh_request_is_completed_by_the_canonical_collector( instance.minimum_candles = 1 instance.pages_per_stream = 6 ready_at = NOW.replace(hour=23, minute=59) + instance.clock = lambda: ready_at request_path = (tmp_path / "market-history-refresh-request.json").resolve() instance.refresh_request_path = request_path request = enqueue_market_history_refresh_request( @@ -1062,6 +1062,31 @@ def test_dashboard_refresh_request_is_completed_by_the_canonical_collector( assert status["live_eligibility_status"] == "LIVE_ORDER_BLOCKED" +def test_refresh_request_age_uses_current_clock_after_cycle_setup( + tmp_path: Path, +) -> None: + instance = collector(tmp_path, Transport()) + instance.minimum_candles = 1 + instance.pages_per_stream = 6 + request_path = (tmp_path / "market-history-refresh-request.json").resolve() + instance.refresh_request_path = request_path + requested_at = NOW + timedelta(minutes=2) + instance.clock = lambda: requested_at + enqueue_market_history_refresh_request( + request_path, + market="SPOT", + symbol="BTCUSDT", + eligible_symbols=("BTCUSDT",), + requested_at=requested_at, + ) + + instance.sync_cycle(observed_at=NOW) + + status = market_history_refresh_status(request_path) + assert status["state"] == "DATA_READY", status + assert "MARKET_HISTORY_REFRESH_REQUEST_EXPIRED" not in status["blockers"] + + def test_refresh_request_completion_never_predates_its_request( tmp_path: Path, ) -> None: @@ -1085,9 +1110,7 @@ def test_refresh_request_completion_never_predates_its_request( blockers=(), ) - completed = datetime.fromisoformat( - str(_load(request_path)["completed_at"]) - ) + completed = datetime.fromisoformat(str(_load(request_path)["completed_at"])) assert completed >= requested_at @@ -2171,9 +2194,7 @@ def record_recoverable_cycle_failure(self, *_args: object) -> None: depth_events: list[object] = [] class Depth: - def __init__( - self, _root: Path, transports: object, **_kwargs: object - ) -> None: + def __init__(self, _root: Path, transports: object, **_kwargs: object) -> None: depth_events.append(transports) def start(self, markets: object) -> None: diff --git a/tests/test_oos_maturity.py b/tests/test_oos_maturity.py index 6838f55a..006f1b05 100644 --- a/tests/test_oos_maturity.py +++ b/tests/test_oos_maturity.py @@ -17,6 +17,8 @@ OOSMaturityGate, OOSValidationSubject, _matches, + load_spot_oos_validation_specification, + prepare_spot_oos_deployment, ) from ai4binance.validation.promotion_evidence import ( PromotionEvidenceQuery, @@ -32,6 +34,60 @@ NOW = datetime(2026, 9, 11, tzinfo=UTC) +def test_active_spot_specification_prepares_exact_research_only_deployment( + tmp_path: Path, +) -> None: + specification_path = Path("config/research/virtual_market_acceptance.yaml") + specification = load_spot_oos_validation_specification(specification_path) + assert specification["status"] == "ACTIVE" + assert specification["approval_status"] == "PENDING_INDEPENDENT_REVIEW" + assert _matches( + "2026-09-10T00:00:00+00:00", + {"operator": "present", "value": True}, + ) + artifact_root = tmp_path / "validation" + deployment_path = tmp_path / "config" / "runtime_validation_deployment.json" + result = prepare_spot_oos_deployment( + artifact_root=artifact_root, + deployment_path=deployment_path, + specification_path=specification_path, + run_cards=( + { + "hypothesis_id": "hyp:trend_continuation:1h", + "symbol": "BTCUSDT", + "timeframe": "1h", + "strategy_sha256": "a" * 64, + "config_sha256": "b" * 64, + "dataset_sha256": "c" * 64, + "code_revision": "WORKTREE_UNVERIFIED", + "fee_rate": 0.001, + "slippage_rate": 0.0005, + "promotion_status": "RESEARCH_ONLY", + "execution_allowed": False, + }, + ), + observed_at=NOW, + ) + assert result["status"] == "EVIDENCE_COLLECTION_IN_PROGRESS" + assert result["subject_count"] == 1 + + from ai4binance.agents.validation_gate import ValidationGate + + gate = ValidationGate.from_deployment( + artifact_root=artifact_root, + deployment_path=deployment_path, + as_of=NOW, + ) + assert not gate.evidence_load_blockers + assert len(gate.expected_subjects) == 1 + maturity = OOSMaturityGate(artifact_root).evaluate(gate.evidence_bundles[0]) + assert maturity.status == "OOS_MATURITY_INCOMPLETE" + assert "EVIDENCE_COLLECTION_IN_PROGRESS" in maturity.blockers + assert "DATASET_EVIDENCE_MISSING" in maturity.blockers + assert "PROMOTION_EVIDENCE_INCOMPLETE" in maturity.blockers + assert "VALIDATION_SPECIFICATION_MISSING" not in maturity.blockers + + def test_bonferroni_changes_the_interval_used_by_the_gate() -> None: single = assess_statistical_evidence( (0.01, 0.04, 0.03, 0.02), @@ -328,6 +384,11 @@ def test_runtime_validation_consumes_exact_maturity_and_keeps_all_vetoes( assert decision.blockers if case == "risk_veto": assert "RISK.VETO" in decision.blockers + if case == "unbound": + assert any( + blocker.startswith("OOS_SUBJECT_NOT_CONFIGURED:") + for blocker in decision.blockers + ) @pytest.mark.parametrize( diff --git a/tests/test_service_manifest.py b/tests/test_service_manifest.py index 7629d75f..04708992 100644 --- a/tests/test_service_manifest.py +++ b/tests/test_service_manifest.py @@ -157,9 +157,9 @@ def test_startup_install_script_preserves_cli_module_runtime_commands() -> None: assert '$env:PYTHONDONTWRITEBYTECODE = "1"' in install_text assert "& $python -B @Arguments" in install_text assert "& $python @Arguments" not in install_text - assert '$previousErrorActionPreference = $ErrorActionPreference' in install_text + assert "$previousErrorActionPreference = $ErrorActionPreference" in install_text assert '$ErrorActionPreference = "Continue"' in install_text - assert '$ErrorActionPreference = $previousErrorActionPreference' in install_text + assert "$ErrorActionPreference = $previousErrorActionPreference" in install_text assert "$nativeExitCode = [int]$LASTEXITCODE" in install_text assert '-Arguments @("-m", "ai4binance.cli", "archive-public")' in install_text assert '-Arguments @("-m", "ai4binance.cli", "validate-research")' in install_text @@ -193,6 +193,11 @@ def test_startup_install_script_preserves_cli_module_runtime_commands() -> None: assert 'Get-ServiceManifestEntry -Service "virtual-market"' in install_text assert "InstallVirtualMarket" in install_text assert "RunVirtualMarket" in install_text + assert "RestartVirtualMarket" in install_text + assert "RestartMarketHistory" in install_text + assert "function Restart-BoundedScheduledService" in install_text + assert "Exact $Service process tree did not stop cleanly." in install_text + assert "Fresh $Service lock owner was not observed" in install_text assert "ai4binance\\.cli\\s+virtual-market-daemon" in status_text assert "VIRTUAL_MARKET" in status_text assert 'Join-Path $stateDirectory "virtual-market.json"' in status_text diff --git a/tests/test_storage.py b/tests/test_storage.py index dbe9d90c..f37f4a91 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -9,6 +9,7 @@ import pytest +import ai4binance.storage.destination_verification as destination_verification from ai4binance.infrastructure.persistence import safe_json from ai4binance.storage import ( AuditEvent, @@ -442,6 +443,61 @@ def tampered_read_text( ) +def test_write_json_object_verified_retries_transient_replace_denial( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + path = tmp_path / "state" / "latest.json" + original_replace = destination_verification.os.replace + attempts = 0 + delays: list[float] = [] + + def flaky_replace(source: Path, destination: Path) -> None: + nonlocal attempts + attempts += 1 + if attempts < 3: + raise PermissionError(5, "transient sharing violation") + original_replace(source, destination) + + monkeypatch.setattr(destination_verification.os, "replace", flaky_replace) + monkeypatch.setattr(destination_verification.time, "sleep", delays.append) + + result = write_json_object_verified( + path, + {"status": "ok"}, + blocker="STATE_VERIFY_FAILED", + ) + + assert result.status is VerificationStatus.VERIFIED + assert attempts == 3 + assert delays == [0.01, 0.02] + + +def test_write_json_object_verified_preserves_persistent_replace_denial( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + path = tmp_path / "state" / "latest.json" + attempts = 0 + + def denied_replace(_source: Path, _destination: Path) -> None: + nonlocal attempts + attempts += 1 + raise PermissionError(5, "persistent sharing violation") + + monkeypatch.setattr(destination_verification.os, "replace", denied_replace) + monkeypatch.setattr(destination_verification.time, "sleep", lambda _delay: None) + + with pytest.raises(PermissionError, match="persistent sharing violation"): + write_json_object_verified( + path, + {"status": "ok"}, + blocker="STATE_VERIFY_FAILED", + ) + + assert attempts == 8 + + def test_jsonl_store_optional_durable_flush( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/test_validation_pipeline.py b/tests/test_validation_pipeline.py index 5c5870b3..61a00f3e 100644 --- a/tests/test_validation_pipeline.py +++ b/tests/test_validation_pipeline.py @@ -25,8 +25,10 @@ from ai4binance.strategies.rules import historical_playbook_decision from ai4binance.validation import ParameterSet from ai4binance.validation_pipeline_runtime import ( + SPOT_VALIDATION_NOTIONAL_TO_EQUITY_RATIO, HistoricalValidationRuntime, _build_historical_decision_resolver, + runtime_spot_backtest_engine, ) @@ -90,6 +92,50 @@ def test_preflight_checkpoint_identity_includes_simulation_assumptions() -> None assert baseline.config_sha256(10) != changed.config_sha256(10) +def test_validation_walk_forward_uses_nonzero_purge_and_embargo() -> None: + config = HistoricalValidationRuntime._tuning_config(605).walk_forward + + assert config.purge_size == 1 + assert config.embargo_size == 1 + assert config.train_size + (config.test_size * 5) + 5 == 605 + + +def test_runtime_spot_backtest_sizing_is_price_normalized_and_cash_bounded() -> None: + candles = tuple( + OHLCVCandle( + timestamp=datetime(2026, 1, 1, hour=index, tzinfo=UTC), + open=Decimal("80000") + Decimal(index * 1000), + high=Decimal("81500") + Decimal(index * 1000), + low=Decimal("79000") + Decimal(index * 1000), + close=Decimal("80500") + Decimal(index * 1000), + volume=Decimal("100"), + ) + for index in range(3) + ) + + engine = runtime_spot_backtest_engine( + candles, + base_engine=BacktestEngine(), + notional_to_equity_ratio=SPOT_VALIDATION_NOTIONAL_TO_EQUITY_RATIO, + ) + + maximum_open = max(candle.open for candle in candles) + maximum_notional = maximum_open * engine.config.quantity + assert engine.config.quantity < Decimal("1") + assert maximum_notional <= ( + engine.config.initial_cash_usdt * SPOT_VALIDATION_NOTIONAL_TO_EQUITY_RATIO + ) + assert maximum_notional >= engine.config.minimum_notional + + +@pytest.mark.parametrize("ratio", [Decimal("0"), Decimal("1.01")]) +def test_runtime_spot_backtest_sizing_rejects_unsafe_ratios( + ratio: Decimal, +) -> None: + with pytest.raises(ValueError, match="within"): + HistoricalValidationRuntime(position_notional_to_equity_ratio=ratio) + + @pytest.mark.parametrize( ("overrides", "message"), [ From 10f4cefd375e8c3bba48c94babbb3a2b54b79651 Mon Sep 17 00:00:00 2001 From: Huseyin Cicek Date: Fri, 25 Sep 2026 05:54:46 +0300 Subject: [PATCH 07/19] feat: add by HsC --- .../runtime_validation_deployment.json | 142 +++++++++--------- src/ai4binance/storage/jsonl.py | 9 ++ src/ai4binance/validation/oos_maturity.py | 39 +++-- src/ai4binance/validation_pipeline_runtime.py | 28 +++- tests/test_oos_maturity.py | 56 +++++++ tests/test_storage.py | 12 ++ tests/test_validation_pipeline.py | 8 + 7 files changed, 211 insertions(+), 83 deletions(-) diff --git a/config/research/runtime_validation_deployment.json b/config/research/runtime_validation_deployment.json index b7ea9ba1..c7e2f069 100644 --- a/config/research/runtime_validation_deployment.json +++ b/config/research/runtime_validation_deployment.json @@ -2,22 +2,22 @@ "execution_allowed": false, "live_eligibility_status": "LIVE_ORDER_BLOCKED", "promotion_status": "RESEARCH_ONLY", - "runtime_source_sha256": "e79927bdc5c8c7430692cffa550c9f14b813acf440e3738d4358ecbd4692ffee", + "runtime_source_sha256": "d590e02cd164b307bede6cb1032e86f506f463058978fd17fe07faf2bb40274d", "schema_version": "1.0", "subjects": [ { "bundle": { - "path": "oos_runtime/890e57250d9f2dfccca58a102780f9d2ef6151a77095c374a433aac20c31c6a0/bundle.json", - "sha256": "8cb4ca67d0e443497d67b7d913d71299b23372cdd11c96d408be4ad5e744f6bc" + "path": "oos_runtime/10b48b390c923252a992e2166311e8e8e3d5b922b5fe46968901bfdadda74da9/bundle.json", + "sha256": "5bd4d51f70f8e51109e7abde7a9df433d9919b5027caeb81fb67fbeca21794bb" }, "subject": { "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", "feature_definition_sha256": "63f712ab32697b7a9c67ee81847bcdc9f7171002e2f7e4e6a213ae2e12edd979", "promotion": { - "code_revision": "e79927bdc5c8c7430692cffa550c9f14b813acf440e3738d4358ecbd4692ffee", - "dataset_sha256": "9be16ba7fe8bb45591bd5089accdde00df60dd80bf516e2bdfe4577ffe8a8f15", + "code_revision": "d590e02cd164b307bede6cb1032e86f506f463058978fd17fe07faf2bb40274d", + "dataset_sha256": "d5b4a4c5fff2a1956b3189c8a6b996352b00edf9537a2f3237daef3c35f7a358", "market_type": "SPOT", - "parameter_set_sha256": "9b971ab67af0b920c00f1531db13348e3491b7bbea39860de0d78a91d3acfeba", + "parameter_set_sha256": "4c0bcbd3c7635cfe3653b227c121ccd44232cee32b1e2f5907a01d54ba5aa55d", "strategy_id": "trend_continuation", "strategy_sha256": "63f712ab32697b7a9c67ee81847bcdc9f7171002e2f7e4e6a213ae2e12edd979", "strategy_version": "1", @@ -25,22 +25,22 @@ "timeframe": "15m" }, "setup_type": "trend_continuation", - "validation_config_sha256": "24ec5a26186412d7024efd1532c940b1118631e24ed7abaf61682d38a0181e6c" + "validation_config_sha256": "586658dd94dbe6bbd09274e3a07f90bdd4ea68df9a4f0617e3db7e961b65eec8" } }, { "bundle": { - "path": "oos_runtime/7e1bcdee40892c39ba24c89a28190775777a1e82906601df28d80e5cbd4e8a30/bundle.json", - "sha256": "a0e056b8db67687bb766ba03970488d9ef13e7e1b98b93a12e40588d053f1c99" + "path": "oos_runtime/a5277e4b36a32615a4099cec13b3686d197e242b2d680a009933bc93760d71c8/bundle.json", + "sha256": "a7887ffc5eb1f286ac243d824b4c736ce68a1626147ab7b458fbbb6c9d15a1be" }, "subject": { "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", "feature_definition_sha256": "acfb727377466011e73a0b3782f76bc445b3dc7d7fbddda33c8b0811370dc852", "promotion": { - "code_revision": "e79927bdc5c8c7430692cffa550c9f14b813acf440e3738d4358ecbd4692ffee", - "dataset_sha256": "9be16ba7fe8bb45591bd5089accdde00df60dd80bf516e2bdfe4577ffe8a8f15", + "code_revision": "d590e02cd164b307bede6cb1032e86f506f463058978fd17fe07faf2bb40274d", + "dataset_sha256": "d5b4a4c5fff2a1956b3189c8a6b996352b00edf9537a2f3237daef3c35f7a358", "market_type": "SPOT", - "parameter_set_sha256": "9b971ab67af0b920c00f1531db13348e3491b7bbea39860de0d78a91d3acfeba", + "parameter_set_sha256": "4c0bcbd3c7635cfe3653b227c121ccd44232cee32b1e2f5907a01d54ba5aa55d", "strategy_id": "pullback_continuation", "strategy_sha256": "acfb727377466011e73a0b3782f76bc445b3dc7d7fbddda33c8b0811370dc852", "strategy_version": "1", @@ -48,22 +48,22 @@ "timeframe": "15m" }, "setup_type": "pullback_continuation", - "validation_config_sha256": "9543c1e40f6059bc3ef4cceea37a9ecff3e5033c1fd21cdbcea18811fba657f3" + "validation_config_sha256": "ec2e086eae3298af24435acd3a641330e83137192844d4f80cb9103855bf9957" } }, { "bundle": { - "path": "oos_runtime/b1b43d5874b9eaf2b51be5d936fed781611b9e9d99d8d229aa3e344d86b9e878/bundle.json", - "sha256": "e6625d073e86dfd88735e84bc263fc388819aca5a6cd6d644954ced683ce679a" + "path": "oos_runtime/b80ac0adaf4f801afc3b97dee3c5801fd20d5cb01e08c02623432e239f757a4a/bundle.json", + "sha256": "8c68e20860b9126dd892aa3ca0a2a1e03685b39e2792d0517f428c017d6f9d42" }, "subject": { "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", "feature_definition_sha256": "2a7e0ad5ccc7292a0b06b76b233fa93552d34bfe993a0db4976e3404b1ff7486", "promotion": { - "code_revision": "e79927bdc5c8c7430692cffa550c9f14b813acf440e3738d4358ecbd4692ffee", - "dataset_sha256": "9be16ba7fe8bb45591bd5089accdde00df60dd80bf516e2bdfe4577ffe8a8f15", + "code_revision": "d590e02cd164b307bede6cb1032e86f506f463058978fd17fe07faf2bb40274d", + "dataset_sha256": "d5b4a4c5fff2a1956b3189c8a6b996352b00edf9537a2f3237daef3c35f7a358", "market_type": "SPOT", - "parameter_set_sha256": "9b971ab67af0b920c00f1531db13348e3491b7bbea39860de0d78a91d3acfeba", + "parameter_set_sha256": "4c0bcbd3c7635cfe3653b227c121ccd44232cee32b1e2f5907a01d54ba5aa55d", "strategy_id": "breakout_retest", "strategy_sha256": "2a7e0ad5ccc7292a0b06b76b233fa93552d34bfe993a0db4976e3404b1ff7486", "strategy_version": "1", @@ -71,22 +71,22 @@ "timeframe": "15m" }, "setup_type": "breakout_retest", - "validation_config_sha256": "bed436e96bb0b3ac0fb16758d3cd504ee0e4a0599d7a87f9d145f79f16b69ed3" + "validation_config_sha256": "321df28133ee9340bcf10c2a9c30363b34aaf449eb81815e8755ec99179da15c" } }, { "bundle": { - "path": "oos_runtime/6c86ecf0b0eaf809535c872b8167e4d46f964f2ae4f3f38c5c4a363c35ce634b/bundle.json", - "sha256": "fa249bc1af909c723a96d78ca3a015616dcd436eef4d80152da78c19eaeb1aae" + "path": "oos_runtime/2fb323a6e8ebc165563e8383eea7cc59abe02de5bfbf75244bf91a834ecc3efc/bundle.json", + "sha256": "cafc83cab079e77701a47afff27366cfd4c901d150c7898c2eb41bb0d8d6f3a0" }, "subject": { "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", "feature_definition_sha256": "7cfbd10b251a8e6acfa0656247e9f99f56c4b6e24849c8a36d66fce734f746a3", "promotion": { - "code_revision": "e79927bdc5c8c7430692cffa550c9f14b813acf440e3738d4358ecbd4692ffee", - "dataset_sha256": "9be16ba7fe8bb45591bd5089accdde00df60dd80bf516e2bdfe4577ffe8a8f15", + "code_revision": "d590e02cd164b307bede6cb1032e86f506f463058978fd17fe07faf2bb40274d", + "dataset_sha256": "d5b4a4c5fff2a1956b3189c8a6b996352b00edf9537a2f3237daef3c35f7a358", "market_type": "SPOT", - "parameter_set_sha256": "9b971ab67af0b920c00f1531db13348e3491b7bbea39860de0d78a91d3acfeba", + "parameter_set_sha256": "4c0bcbd3c7635cfe3653b227c121ccd44232cee32b1e2f5907a01d54ba5aa55d", "strategy_id": "support_reclaim", "strategy_sha256": "7cfbd10b251a8e6acfa0656247e9f99f56c4b6e24849c8a36d66fce734f746a3", "strategy_version": "1", @@ -94,22 +94,22 @@ "timeframe": "15m" }, "setup_type": "support_reclaim", - "validation_config_sha256": "0e3bc5eddceca968b76419d11acc3088cbba85d290da887f90db2d0296988afc" + "validation_config_sha256": "e41261d31a16f7110974270168d7583e976af26c4f0e45c0ac9787ac37415fd9" } }, { "bundle": { - "path": "oos_runtime/9872a208aaa9b1e13ac80bb35a9bbcdda8e12bb82ec7b905ea75a99727023ddb/bundle.json", - "sha256": "140ae4c52799b52670b7174f6968d07e27a50d7ca32826c9666fc2df01b39233" + "path": "oos_runtime/0a389c8d2eb25b40d6422f4bd53968198d73a273c5d15b72a451c7fa17211e8d/bundle.json", + "sha256": "d56cc3419bf242c3e6f32222fec664e2df2a32c8443d78a668dab77aaffc8544" }, "subject": { "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", "feature_definition_sha256": "4f33ee776ac3b463420d6a0f2c08352a0673d16633d2a32765d861fcb8e144ea", "promotion": { - "code_revision": "e79927bdc5c8c7430692cffa550c9f14b813acf440e3738d4358ecbd4692ffee", - "dataset_sha256": "9be16ba7fe8bb45591bd5089accdde00df60dd80bf516e2bdfe4577ffe8a8f15", + "code_revision": "d590e02cd164b307bede6cb1032e86f506f463058978fd17fe07faf2bb40274d", + "dataset_sha256": "d5b4a4c5fff2a1956b3189c8a6b996352b00edf9537a2f3237daef3c35f7a358", "market_type": "SPOT", - "parameter_set_sha256": "9b971ab67af0b920c00f1531db13348e3491b7bbea39860de0d78a91d3acfeba", + "parameter_set_sha256": "4c0bcbd3c7635cfe3653b227c121ccd44232cee32b1e2f5907a01d54ba5aa55d", "strategy_id": "failed_breakout_reversal", "strategy_sha256": "4f33ee776ac3b463420d6a0f2c08352a0673d16633d2a32765d861fcb8e144ea", "strategy_version": "1", @@ -117,19 +117,19 @@ "timeframe": "15m" }, "setup_type": "failed_breakout_reversal", - "validation_config_sha256": "7d456e866aac91f9b51e90184af24a8f4fd3292eba863d504fcca0bfd5337a3d" + "validation_config_sha256": "3ef53ffe56eff9c1d73335f0ccf86881aca2085cebab2d3dbe9bedb7dfbfa73e" } }, { "bundle": { - "path": "oos_runtime/d8e7e3aa2db42593d6e2eb1417e5cbe17a56961bc9aa227f7629744ae3851971/bundle.json", - "sha256": "c0dbfb17a914f413f71a918d51d35364bf22eaecb9934df0aa71879b64b475e3" + "path": "oos_runtime/9197288fff32e4f72c9ac914c275ddb94333f975579a2d368cf863477b3ba9ee/bundle.json", + "sha256": "ba3ae8569a2540107933896940aa5cc4336c7ec173f65b0379b8255b09f99967" }, "subject": { "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", "feature_definition_sha256": "63f712ab32697b7a9c67ee81847bcdc9f7171002e2f7e4e6a213ae2e12edd979", "promotion": { - "code_revision": "e79927bdc5c8c7430692cffa550c9f14b813acf440e3738d4358ecbd4692ffee", + "code_revision": "d590e02cd164b307bede6cb1032e86f506f463058978fd17fe07faf2bb40274d", "dataset_sha256": "726c5f1a6295f1323671cd642bdbf6c4976623cd71fce1d2345446a04b70a445", "market_type": "SPOT", "parameter_set_sha256": "feca57cc3c10c80e9086f18e57fd8f498715f7cd865a465b690612a523c4eff3", @@ -140,19 +140,19 @@ "timeframe": "1h" }, "setup_type": "trend_continuation", - "validation_config_sha256": "07b8e29b6e6dc7fda50d66ca09ee19ebf99100de98dd3b68bb0195c9eab2ad6e" + "validation_config_sha256": "e4d677f159d8db0984212f4dbb2fd986456a94b6a522a0f3a8bd705dcfe7c63e" } }, { "bundle": { - "path": "oos_runtime/bc16e69250feb1b49ae9e6f35310fe8f3b1ff11cfb12020b3b324f45c1b9a7fa/bundle.json", - "sha256": "35fa0639e75b723eca6c582db5a2bf29f7694adb932c7a661a101c2b7577db34" + "path": "oos_runtime/21654491ef611e991dc41073fe72e039342c2b75c6c055be0c520bb194accb6a/bundle.json", + "sha256": "74db017db2349b79b493907c78a46c85c8af7adeae3fed40b266dfa5bd3a91b8" }, "subject": { "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", "feature_definition_sha256": "d676e5ee3551bed7965e95cb421ae999225b9b6c0e2662468065506836fe5fc7", "promotion": { - "code_revision": "e79927bdc5c8c7430692cffa550c9f14b813acf440e3738d4358ecbd4692ffee", + "code_revision": "d590e02cd164b307bede6cb1032e86f506f463058978fd17fe07faf2bb40274d", "dataset_sha256": "726c5f1a6295f1323671cd642bdbf6c4976623cd71fce1d2345446a04b70a445", "market_type": "SPOT", "parameter_set_sha256": "feca57cc3c10c80e9086f18e57fd8f498715f7cd865a465b690612a523c4eff3", @@ -163,19 +163,19 @@ "timeframe": "1h" }, "setup_type": "pullback_continuation", - "validation_config_sha256": "75c6996fc1630165e68913b5f619d986457a62ddbba066c8f09968c42ee8053e" + "validation_config_sha256": "4c02b6ed6017bfdfaed1f43acdddc3897919473e17cf321e3a2924f3be3c7286" } }, { "bundle": { - "path": "oos_runtime/005a7e5e442f21b5c8463124d473231199ba599040b252ae40d4072e9cbe9a3a/bundle.json", - "sha256": "793a5c5c2251c8c5e0f9df191994f8f5ebb4734ca94c5d8ad97aebca39e29dca" + "path": "oos_runtime/5f8cbd0546b1c46e7393b5ca59f28fbbff53791d8c56090e58a602a9ebbb2f30/bundle.json", + "sha256": "f313728c514103a574fc1a17596c5f5210226d98f25056711bf8970fcf4ab2f8" }, "subject": { "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", "feature_definition_sha256": "2a7e0ad5ccc7292a0b06b76b233fa93552d34bfe993a0db4976e3404b1ff7486", "promotion": { - "code_revision": "e79927bdc5c8c7430692cffa550c9f14b813acf440e3738d4358ecbd4692ffee", + "code_revision": "d590e02cd164b307bede6cb1032e86f506f463058978fd17fe07faf2bb40274d", "dataset_sha256": "726c5f1a6295f1323671cd642bdbf6c4976623cd71fce1d2345446a04b70a445", "market_type": "SPOT", "parameter_set_sha256": "feca57cc3c10c80e9086f18e57fd8f498715f7cd865a465b690612a523c4eff3", @@ -186,19 +186,19 @@ "timeframe": "1h" }, "setup_type": "breakout_retest", - "validation_config_sha256": "eb78395b6f465fdc6f29531beb9ad289daaf0a273b1c28c0e04f42f0bce9acdb" + "validation_config_sha256": "dfd84b6bd7b06f281258e1ffde2e9e3a9f700cb9ffe3506211d8a2edd3b2ac0a" } }, { "bundle": { - "path": "oos_runtime/ac5f7c2b5e91c3ab487856573566777c80534c50ffea903c6cec30bf0b86151a/bundle.json", - "sha256": "0c999dd85e7f20edc600d1d259951d19aaf31f792d4f47abf2af467c14fdf978" + "path": "oos_runtime/dd4feba517eacaa7625c9765051291ed5dcfdb36d6977da33c4316caebc284ad/bundle.json", + "sha256": "0f5bdcb239914f544e299259d9a68904e5975f87e0c71712c8fdd871d2653447" }, "subject": { "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", "feature_definition_sha256": "7cfbd10b251a8e6acfa0656247e9f99f56c4b6e24849c8a36d66fce734f746a3", "promotion": { - "code_revision": "e79927bdc5c8c7430692cffa550c9f14b813acf440e3738d4358ecbd4692ffee", + "code_revision": "d590e02cd164b307bede6cb1032e86f506f463058978fd17fe07faf2bb40274d", "dataset_sha256": "726c5f1a6295f1323671cd642bdbf6c4976623cd71fce1d2345446a04b70a445", "market_type": "SPOT", "parameter_set_sha256": "feca57cc3c10c80e9086f18e57fd8f498715f7cd865a465b690612a523c4eff3", @@ -209,19 +209,19 @@ "timeframe": "1h" }, "setup_type": "support_reclaim", - "validation_config_sha256": "5476114c01478e59a028f1de35d486a36bdb7cfa2b8fdb63e73a4e93c459fda9" + "validation_config_sha256": "e84a197309ba3365ecf90011ada2fc85235fc380dcad3d7c1c0bf6f1e6932910" } }, { "bundle": { - "path": "oos_runtime/d96659f5863de41822764d40c599a4f90997ce0e96815a174a97624cf117543a/bundle.json", - "sha256": "cfcff9591cad97d53b47f6823524b030bd3b50598993c3ecf3db789dbb4357b7" + "path": "oos_runtime/4d0fd517399594f5b16e355a3ccdb7ded1a3d723d729cc3eff9244ef5d545435/bundle.json", + "sha256": "dde2dc58f7ca9f8cadc5b46922c62779bd1ae98991ea822073791e922ce0da3e" }, "subject": { "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", "feature_definition_sha256": "4f33ee776ac3b463420d6a0f2c08352a0673d16633d2a32765d861fcb8e144ea", "promotion": { - "code_revision": "e79927bdc5c8c7430692cffa550c9f14b813acf440e3738d4358ecbd4692ffee", + "code_revision": "d590e02cd164b307bede6cb1032e86f506f463058978fd17fe07faf2bb40274d", "dataset_sha256": "726c5f1a6295f1323671cd642bdbf6c4976623cd71fce1d2345446a04b70a445", "market_type": "SPOT", "parameter_set_sha256": "feca57cc3c10c80e9086f18e57fd8f498715f7cd865a465b690612a523c4eff3", @@ -232,19 +232,19 @@ "timeframe": "1h" }, "setup_type": "failed_breakout_reversal", - "validation_config_sha256": "9c5f047dab4454f4b32bf7545eac41ce4fe4190ac22f3bbd28c11ce317fac9b8" + "validation_config_sha256": "1c1aec712b2948f11b5c894fcdd3b3bb8a254857f9b8e5c0656b928fdbe42bff" } }, { "bundle": { - "path": "oos_runtime/d37139f1f85ca960e05db7daa67acba59c4ca7f9b9de3b815cc40947a8c7cc18/bundle.json", - "sha256": "d05d459f8ddd081fbea48d6466bcfd8558940bda3675a4ea9065545eb61ebaa8" + "path": "oos_runtime/cf57ba427a11bbb2ab408bcc112d76931040673e22aaf0ea688c18c973e27a74/bundle.json", + "sha256": "4aa61cdbecb5f5355e9429edb493cec50135e0d576cd9ec4c888bd625eae082f" }, "subject": { "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", "feature_definition_sha256": "63f712ab32697b7a9c67ee81847bcdc9f7171002e2f7e4e6a213ae2e12edd979", "promotion": { - "code_revision": "e79927bdc5c8c7430692cffa550c9f14b813acf440e3738d4358ecbd4692ffee", + "code_revision": "d590e02cd164b307bede6cb1032e86f506f463058978fd17fe07faf2bb40274d", "dataset_sha256": "d1b3e59b5ad3f22154405c8022ab5b0354dac6734deb10ca7c712d5fb288ed81", "market_type": "SPOT", "parameter_set_sha256": "43ee407708a48ea2da819ce0b597f20ee8c012155b1a48f874f996d7c59cbab5", @@ -255,19 +255,19 @@ "timeframe": "4h" }, "setup_type": "trend_continuation", - "validation_config_sha256": "2aad4ce2b10383fd7ee876d02e7140510069b0e0ddf35d6a3936e85c05f30820" + "validation_config_sha256": "8eee72b0ff9cba558c858c18b312e3823533457adbda24aea38585152e03abc2" } }, { "bundle": { - "path": "oos_runtime/d68eab3c7ea194f31c93ee3c04beab94817660f5e4fe0db42a3c64648aaa71f9/bundle.json", - "sha256": "e188a666988fb488a07f333c7c71c7755991683125652f40e6f2b614c6fb1c91" + "path": "oos_runtime/4aebc13da13c331685f66e075d78adbb371a3a7fd94469355e0b82af2fe377e5/bundle.json", + "sha256": "ae902e6e0ef5542a5b7524e4d81b552b8afb5592b6cdef4eeaf47eb75b46e620" }, "subject": { "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", "feature_definition_sha256": "d676e5ee3551bed7965e95cb421ae999225b9b6c0e2662468065506836fe5fc7", "promotion": { - "code_revision": "e79927bdc5c8c7430692cffa550c9f14b813acf440e3738d4358ecbd4692ffee", + "code_revision": "d590e02cd164b307bede6cb1032e86f506f463058978fd17fe07faf2bb40274d", "dataset_sha256": "d1b3e59b5ad3f22154405c8022ab5b0354dac6734deb10ca7c712d5fb288ed81", "market_type": "SPOT", "parameter_set_sha256": "43ee407708a48ea2da819ce0b597f20ee8c012155b1a48f874f996d7c59cbab5", @@ -278,19 +278,19 @@ "timeframe": "4h" }, "setup_type": "pullback_continuation", - "validation_config_sha256": "935af3bd7625c9602ca7623ebbd9b3996dc3652a74acf91b7762577508dab018" + "validation_config_sha256": "6e5f47589294f4c2c7e0cd38e48bffd13e41cc10a7553cf8093e460c26e7dec2" } }, { "bundle": { - "path": "oos_runtime/89212af054cbffe520074ecaa18ff4065385934e21c6f8a4f1775cb22ebb4209/bundle.json", - "sha256": "e7f2b9b6d8f1a32921cc771e14c95e9ddf62bfdc3011dfc0951efab078648cdd" + "path": "oos_runtime/061455573f76974b2e8df6f32f06cda153b33998ebb6810e72167a12d656157a/bundle.json", + "sha256": "922199a764159fba35fb02fffb83e96f3434be188417f1766bc1de79df3d88f2" }, "subject": { "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", "feature_definition_sha256": "81ed5bdf546ea4521d379049a78ecd0c518945f72b7414517439d2e976d59c40", "promotion": { - "code_revision": "e79927bdc5c8c7430692cffa550c9f14b813acf440e3738d4358ecbd4692ffee", + "code_revision": "d590e02cd164b307bede6cb1032e86f506f463058978fd17fe07faf2bb40274d", "dataset_sha256": "d1b3e59b5ad3f22154405c8022ab5b0354dac6734deb10ca7c712d5fb288ed81", "market_type": "SPOT", "parameter_set_sha256": "43ee407708a48ea2da819ce0b597f20ee8c012155b1a48f874f996d7c59cbab5", @@ -301,19 +301,19 @@ "timeframe": "4h" }, "setup_type": "breakout_retest", - "validation_config_sha256": "1e7fde61c0fd0d71b9067617171a4f045070e8ddb7e6fcb71134013fc4644c00" + "validation_config_sha256": "3d2e8dd7249dde8dae5c80128f33eabe966cf644132be3af45bca4eba547e147" } }, { "bundle": { - "path": "oos_runtime/63bc9f333eaefafb8438ea26b229db89d20dcbd7ba2358a5fb06b2c60179fb93/bundle.json", - "sha256": "125ad7aee5a6704e6ea48b684f53922a474812afd58be62cb44ffb7ad6f8fd2b" + "path": "oos_runtime/4606b8f97b6cc8de8702a038f9579e47de2263cee50c889c1c6c1ec5b55d1a47/bundle.json", + "sha256": "a2982f7e4aa431c7852387f91d89aeed71ba3b580a1f226d1747c2d167b85620" }, "subject": { "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", "feature_definition_sha256": "ab8ad8719e8ae46a24c00a9fa61b06b65ec081f6282f532f3e6fd5922d517a07", "promotion": { - "code_revision": "e79927bdc5c8c7430692cffa550c9f14b813acf440e3738d4358ecbd4692ffee", + "code_revision": "d590e02cd164b307bede6cb1032e86f506f463058978fd17fe07faf2bb40274d", "dataset_sha256": "d1b3e59b5ad3f22154405c8022ab5b0354dac6734deb10ca7c712d5fb288ed81", "market_type": "SPOT", "parameter_set_sha256": "43ee407708a48ea2da819ce0b597f20ee8c012155b1a48f874f996d7c59cbab5", @@ -324,19 +324,19 @@ "timeframe": "4h" }, "setup_type": "support_reclaim", - "validation_config_sha256": "52bb9cc097549348377424d8568afeab9d206375f204cfeb90b84780b4031277" + "validation_config_sha256": "5910dfcd0805be7c336bf63f0006353c2d3b88097110ad2ba257ef6456fb0d0c" } }, { "bundle": { - "path": "oos_runtime/88917be42880fc8aa60e89a44f3148e4a9e7b4e0d8e260fa45ed4eded122b710/bundle.json", - "sha256": "a1cfd1f0e6e4dec4ca8198ec00736b4ef5d5be97fa2e438618be88447693b09c" + "path": "oos_runtime/d96b2e5bec0a13541279315b199b321a953d9e2c7d2d1d419196a83272382765/bundle.json", + "sha256": "d27dec3db7cd1006974ca484c8b4c4800f50be74919a06cb0525f9966718aeb1" }, "subject": { "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", "feature_definition_sha256": "4f33ee776ac3b463420d6a0f2c08352a0673d16633d2a32765d861fcb8e144ea", "promotion": { - "code_revision": "e79927bdc5c8c7430692cffa550c9f14b813acf440e3738d4358ecbd4692ffee", + "code_revision": "d590e02cd164b307bede6cb1032e86f506f463058978fd17fe07faf2bb40274d", "dataset_sha256": "d1b3e59b5ad3f22154405c8022ab5b0354dac6734deb10ca7c712d5fb288ed81", "market_type": "SPOT", "parameter_set_sha256": "43ee407708a48ea2da819ce0b597f20ee8c012155b1a48f874f996d7c59cbab5", @@ -347,7 +347,7 @@ "timeframe": "4h" }, "setup_type": "failed_breakout_reversal", - "validation_config_sha256": "3b04265d83e7bb0faee38e621e7645ff5510e7aa5453c368f9c6829d5df57c47" + "validation_config_sha256": "78281bccf0c5bde80fd611bff8ec505fb69b37136190ce163dab32850a08229e" } } ] diff --git a/src/ai4binance/storage/jsonl.py b/src/ai4binance/storage/jsonl.py index 6f58145c..9372b188 100644 --- a/src/ai4binance/storage/jsonl.py +++ b/src/ai4binance/storage/jsonl.py @@ -688,11 +688,20 @@ def read_bounded_jsonl_tail( buffer = b"" while cursor > 0: start = max(0, cursor - _TAIL_READ_CHUNK_BYTES) + starts_at_record_boundary = start == 0 + if start > 0: + stream.seek(start - 1) + previous = stream.read(1) + stream.seek(start) + current = stream.read(1) + starts_at_record_boundary = previous in b"\r\n" or current in b"\r\n" stream.seek(start) buffer = stream.read(cursor - start) + buffer if len(buffer) > max_bytes: raise OSError("JSONL tail exceeds bounded read limit") lines = tuple(line for line in buffer.splitlines() if line.strip()) + if lines and not starts_at_record_boundary: + lines = lines[1:] if len(lines) >= max_lines or start == 0: return lines[-max_lines:] cursor = start diff --git a/src/ai4binance/validation/oos_maturity.py b/src/ai4binance/validation/oos_maturity.py index ad6d21f8..9e8d89ae 100644 --- a/src/ai4binance/validation/oos_maturity.py +++ b/src/ai4binance/validation/oos_maturity.py @@ -25,6 +25,7 @@ from ai4binance.infrastructure.persistence.safe_json import ( write_json_object_verified, ) +from ai4binance.storage import read_bounded_jsonl_tail from ai4binance.validation.promotion_evidence import ( PromotionEvidenceQuery, PromotionEvidenceRegistry, @@ -1124,22 +1125,38 @@ def _validation_events( relative = resolved.relative_to(root) except ValueError as exc: raise ValueError("SPOT_OOS_SOURCE_OUTSIDE_ARTIFACT_ROOT") from exc - raw = resolved.read_bytes() - digest = sha256(raw).hexdigest() - if digest != str(first[1]) or len(raw) > MAX_ARTIFACT_BYTES: + digest = _stream_sha256(resolved) + if digest != str(first[1]): raise ValueError("SPOT_OOS_SOURCE_HASH_INVALID") + try: + raw_events = read_bounded_jsonl_tail( + resolved, + max_lines=4, + max_bytes=MAX_ARTIFACT_BYTES, + ) + except (OSError, ValueError) as exc: + raise ValueError("SPOT_OOS_SOURCE_EVENTS_INVALID") from exc events: dict[str, dict[str, object]] = {} - for line in raw.splitlines(): - if not line.strip(): - continue - row = _object(json.loads(line)) - event_type = _required_text(row, "event_type") - payload = _object(row.get("payload")) - value = next(iter(payload.values()), None) - events[event_type] = _object(value) + try: + for line in raw_events: + row = _object(json.loads(line)) + event_type = _required_text(row, "event_type") + payload = _object(row.get("payload")) + value = next(iter(payload.values()), None) + events[event_type] = _object(value) + except (TypeError, ValueError, json.JSONDecodeError) as exc: + raise ValueError("SPOT_OOS_SOURCE_EVENTS_INVALID") from exc return events, OOSArtifactReference(relative.as_posix(), digest) +def _stream_sha256(path: Path) -> str: + digest = sha256() + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + def _write_stage_evidence( *, artifact_root: Path, diff --git a/src/ai4binance/validation_pipeline_runtime.py b/src/ai4binance/validation_pipeline_runtime.py index c4c25bd3..b33c7a06 100644 --- a/src/ai4binance/validation_pipeline_runtime.py +++ b/src/ai4binance/validation_pipeline_runtime.py @@ -577,6 +577,9 @@ def write_checkpoint( return artifact_path = Path(result.artifact_path) checkpoint_path = self._checkpoint_path(artifact_path) + run_card_path = artifact_path.with_name(f"{result.playbook}.run-card.json") + if result.run_card is None or not run_card_path.is_file(): + raise ValueError("VALIDATION_CHECKPOINT_RUN_CARD_MISSING") write_json_object_verified( checkpoint_path, { @@ -589,6 +592,7 @@ def write_checkpoint( "config_sha256": config_sha256, "implementation_sha256": implementation_sha256, "artifact_sha256": sha256(artifact_path.read_bytes()).hexdigest(), + "run_card_sha256": sha256(run_card_path.read_bytes()).hexdigest(), "promotion_status": result.promotion_status.value, "blockers": list(result.blockers), "signal_blockers": [list(item) for item in result.signal_blockers], @@ -619,7 +623,12 @@ def load_checkpoint( playbook, ) checkpoint_path = self._checkpoint_path(artifact_path) - if not checkpoint_path.is_file() or not artifact_path.is_file(): + run_card_path = artifact_path.with_name(f"{playbook}.run-card.json") + if ( + not checkpoint_path.is_file() + or not artifact_path.is_file() + or not run_card_path.is_file() + ): return None try: payload = read_json_object( @@ -635,6 +644,7 @@ def load_checkpoint( "config_sha256": config_sha256, "implementation_sha256": implementation_sha256, "artifact_sha256": sha256(artifact_path.read_bytes()).hexdigest(), + "run_card_sha256": sha256(run_card_path.read_bytes()).hexdigest(), "execution_allowed": False, "live_eligibility_status": "LIVE_ORDER_BLOCKED", } @@ -645,6 +655,21 @@ def load_checkpoint( payload.get("signal_blockers") ) promotion_status = ValidationStatus(str(payload["promotion_status"])) + run_card = read_json_object( + run_card_path, + blocker="VALIDATION_CHECKPOINT_RUN_CARD_READ_FAILED", + ) + run_card_expected = { + "symbol": symbol.strip().upper(), + "timeframe": timeframe, + "dataset_sha256": dataset_sha256, + "promotion_status": promotion_status.value, + "execution_allowed": False, + } + if any( + run_card.get(key) != value for key, value in run_card_expected.items() + ): + return None except ( DestinationVerificationError, KeyError, @@ -659,6 +684,7 @@ def load_checkpoint( candle_count=len(candles), promotion_status=promotion_status, blockers=blockers, + run_card=dict(run_card), signal_blockers=signal_blockers, artifact_path=str(artifact_path), checkpoint_path=str(checkpoint_path), diff --git a/tests/test_oos_maturity.py b/tests/test_oos_maturity.py index 006f1b05..ff46e9d2 100644 --- a/tests/test_oos_maturity.py +++ b/tests/test_oos_maturity.py @@ -11,12 +11,14 @@ from ai4binance.domain import ValidationStatus from ai4binance.validation.oos_maturity import ( + MAX_ARTIFACT_BYTES, REQUIRED_MEASUREMENTS, OOSArtifactReference, OOSMaturityEvidenceBundle, OOSMaturityGate, OOSValidationSubject, _matches, + _validation_events, load_spot_oos_validation_specification, prepare_spot_oos_deployment, ) @@ -34,6 +36,60 @@ NOW = datetime(2026, 9, 11, tzinfo=UTC) +def test_validation_events_accept_large_hash_bound_append_only_ledger( + tmp_path: Path, +) -> None: + artifact_root = tmp_path / "validation" + artifact_root.mkdir() + source = artifact_root / "trend_continuation.jsonl" + prefix = json.dumps({"archived": "x" * MAX_ARTIFACT_BYTES}).encode() + b"\n" + event_types = ( + "BACKTEST_RESULT", + "WALK_FORWARD_REPORT", + "TUNING_REPORT", + "BACKTEST_ROBUSTNESS_REPORT", + ) + current = b"".join( + json.dumps( + { + "event_type": event_type, + "payload": {"result": {"sequence": index}}, + } + ).encode() + + b"\n" + for index, event_type in enumerate(event_types, start=1) + ) + raw = prefix + current + source.write_bytes(raw) + + events, reference = _validation_events( + {"artifact_sha256": ((str(source), sha256(raw).hexdigest()),)}, + artifact_root, + ) + + assert set(events) == set(event_types) + assert events["BACKTEST_RESULT"]["sequence"] == 1 + assert reference is not None + assert reference.path == "trend_continuation.jsonl" + assert reference.sha256 == sha256(raw).hexdigest() + + +def test_validation_events_reject_hash_mismatch(tmp_path: Path) -> None: + artifact_root = tmp_path / "validation" + artifact_root.mkdir() + source = artifact_root / "trend_continuation.jsonl" + source.write_text( + json.dumps({"event_type": "BACKTEST_RESULT", "payload": {"result": {}}}), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="SPOT_OOS_SOURCE_HASH_INVALID"): + _validation_events( + {"artifact_sha256": ((str(source), "0" * 64),)}, + artifact_root, + ) + + def test_active_spot_specification_prepares_exact_research_only_deployment( tmp_path: Path, ) -> None: diff --git a/tests/test_storage.py b/tests/test_storage.py index f37f4a91..54bd69ae 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -355,6 +355,18 @@ def test_bounded_jsonl_tail_returns_only_recent_nonempty_lines( ) +def test_bounded_jsonl_tail_discards_partial_leading_record(tmp_path: Path) -> None: + path = tmp_path / "large-records.jsonl" + large = json.dumps({"id": 1, "value": "x" * 200_000}).encode() + latest = json.dumps({"id": 2}).encode() + path.write_bytes(b'{"id":0}\n' + large + b"\n" + latest + b"\n") + + assert read_bounded_jsonl_tail(path, max_lines=2, max_bytes=500_000) == ( + large, + latest, + ) + + def test_verified_write_result_rejects_inconsistent_states() -> None: with pytest.raises(ValueError, match="identity"): VerifiedWriteResult("", "event", VerificationStatus.VERIFIED) diff --git a/tests/test_validation_pipeline.py b/tests/test_validation_pipeline.py index 61a00f3e..15f18aa8 100644 --- a/tests/test_validation_pipeline.py +++ b/tests/test_validation_pipeline.py @@ -392,8 +392,16 @@ def test_validation_pipeline_runs_six_playbooks_and_persists_evidence( ) assert resumed_trend.resumed_from_checkpoint is True assert resumed_trend.backtest is None + assert resumed_trend.run_card is not None + resumed_run_card = cast(dict[str, object], resumed_trend.run_card) + assert resumed_run_card["symbol"] == "BTCUSDT" + assert resumed_run_card["timeframe"] == "1h" assert resumed_trend.checkpoint_path is not None assert Path(resumed_trend.checkpoint_path).is_file() + checkpoint = json.loads( + Path(resumed_trend.checkpoint_path).read_text(encoding="utf-8") + ) + assert len(checkpoint["run_card_sha256"]) == 64 assert dict(resumed_trend.stage_timings_ms)["checkpoint_lookup"] >= 0 resumed_manifest = json.loads( sorted((artifact_directory / "BTCUSDT" / "runs").glob("*.manifest.json"))[ From 36e67a3549fd8281a0a271141749aff9705e2f49 Mon Sep 17 00:00:00 2001 From: Huseyin Cicek Date: Fri, 25 Sep 2026 06:11:20 +0300 Subject: [PATCH 08/19] feat: add by HsC --- .../runtime_validation_deployment.json | 182 ++++++++--------- src/ai4binance/cli/runtime.py | 192 +++++++++++++++++- src/ai4binance/virtual_wallet_journal.py | 120 +++++++++++ tests/test_cli.py | 100 ++++++++- tests/test_virtual_wallet_journal.py | 58 ++++++ 5 files changed, 558 insertions(+), 94 deletions(-) diff --git a/config/research/runtime_validation_deployment.json b/config/research/runtime_validation_deployment.json index c7e2f069..258782d2 100644 --- a/config/research/runtime_validation_deployment.json +++ b/config/research/runtime_validation_deployment.json @@ -2,22 +2,22 @@ "execution_allowed": false, "live_eligibility_status": "LIVE_ORDER_BLOCKED", "promotion_status": "RESEARCH_ONLY", - "runtime_source_sha256": "d590e02cd164b307bede6cb1032e86f506f463058978fd17fe07faf2bb40274d", + "runtime_source_sha256": "bcfbc13c104caa2fbcde590252b262e5c0133b38f1bfe9493574f32cb79220a9", "schema_version": "1.0", "subjects": [ { "bundle": { - "path": "oos_runtime/10b48b390c923252a992e2166311e8e8e3d5b922b5fe46968901bfdadda74da9/bundle.json", - "sha256": "5bd4d51f70f8e51109e7abde7a9df433d9919b5027caeb81fb67fbeca21794bb" + "path": "oos_runtime/542ca70809c81c41839c003d50c8e83f14227eba302031e8d72a9c2320193902/bundle.json", + "sha256": "0ea2b993bdb38844ec882a70d0704bdda0e3ea4ca68590834af3388c03a2188d" }, "subject": { "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", "feature_definition_sha256": "63f712ab32697b7a9c67ee81847bcdc9f7171002e2f7e4e6a213ae2e12edd979", "promotion": { - "code_revision": "d590e02cd164b307bede6cb1032e86f506f463058978fd17fe07faf2bb40274d", - "dataset_sha256": "d5b4a4c5fff2a1956b3189c8a6b996352b00edf9537a2f3237daef3c35f7a358", + "code_revision": "bcfbc13c104caa2fbcde590252b262e5c0133b38f1bfe9493574f32cb79220a9", + "dataset_sha256": "670c3909a381f375b0a99fcef256378c5b3892b2eefbde3167300dc91ed561b5", "market_type": "SPOT", - "parameter_set_sha256": "4c0bcbd3c7635cfe3653b227c121ccd44232cee32b1e2f5907a01d54ba5aa55d", + "parameter_set_sha256": "4bd1c4a9aa115e27998e0721c7700ae6a1d8a8cf09af95d34b3ce624edda1ac9", "strategy_id": "trend_continuation", "strategy_sha256": "63f712ab32697b7a9c67ee81847bcdc9f7171002e2f7e4e6a213ae2e12edd979", "strategy_version": "1", @@ -25,22 +25,22 @@ "timeframe": "15m" }, "setup_type": "trend_continuation", - "validation_config_sha256": "586658dd94dbe6bbd09274e3a07f90bdd4ea68df9a4f0617e3db7e961b65eec8" + "validation_config_sha256": "41f2ae9107a79985e46f8522323133261f99fb4ec9d9d4265b0f57c3d3956cd4" } }, { "bundle": { - "path": "oos_runtime/a5277e4b36a32615a4099cec13b3686d197e242b2d680a009933bc93760d71c8/bundle.json", - "sha256": "a7887ffc5eb1f286ac243d824b4c736ce68a1626147ab7b458fbbb6c9d15a1be" + "path": "oos_runtime/0c17e1a8754613e9607e63e434e93349b83f0dd6da00a044165ad532eae358e0/bundle.json", + "sha256": "c082ac3615f22fdb10f8662a45cc357d642e87fbb39c94730034184d2fa774bc" }, "subject": { "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", "feature_definition_sha256": "acfb727377466011e73a0b3782f76bc445b3dc7d7fbddda33c8b0811370dc852", "promotion": { - "code_revision": "d590e02cd164b307bede6cb1032e86f506f463058978fd17fe07faf2bb40274d", - "dataset_sha256": "d5b4a4c5fff2a1956b3189c8a6b996352b00edf9537a2f3237daef3c35f7a358", + "code_revision": "bcfbc13c104caa2fbcde590252b262e5c0133b38f1bfe9493574f32cb79220a9", + "dataset_sha256": "670c3909a381f375b0a99fcef256378c5b3892b2eefbde3167300dc91ed561b5", "market_type": "SPOT", - "parameter_set_sha256": "4c0bcbd3c7635cfe3653b227c121ccd44232cee32b1e2f5907a01d54ba5aa55d", + "parameter_set_sha256": "4bd1c4a9aa115e27998e0721c7700ae6a1d8a8cf09af95d34b3ce624edda1ac9", "strategy_id": "pullback_continuation", "strategy_sha256": "acfb727377466011e73a0b3782f76bc445b3dc7d7fbddda33c8b0811370dc852", "strategy_version": "1", @@ -48,22 +48,22 @@ "timeframe": "15m" }, "setup_type": "pullback_continuation", - "validation_config_sha256": "ec2e086eae3298af24435acd3a641330e83137192844d4f80cb9103855bf9957" + "validation_config_sha256": "3fc299291cbb97b9cc53bd413af44289eb93da55ec2733ad415c82f871865a2f" } }, { "bundle": { - "path": "oos_runtime/b80ac0adaf4f801afc3b97dee3c5801fd20d5cb01e08c02623432e239f757a4a/bundle.json", - "sha256": "8c68e20860b9126dd892aa3ca0a2a1e03685b39e2792d0517f428c017d6f9d42" + "path": "oos_runtime/1cd1f756c2dfe2e6e5b72ad580f250a1f5c6f10038e9a51acc41a74a966c204e/bundle.json", + "sha256": "03f1c4e0d8e776bb758f255c3945694a2ce33b722d43f21de485b007f3edebc0" }, "subject": { "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", "feature_definition_sha256": "2a7e0ad5ccc7292a0b06b76b233fa93552d34bfe993a0db4976e3404b1ff7486", "promotion": { - "code_revision": "d590e02cd164b307bede6cb1032e86f506f463058978fd17fe07faf2bb40274d", - "dataset_sha256": "d5b4a4c5fff2a1956b3189c8a6b996352b00edf9537a2f3237daef3c35f7a358", + "code_revision": "bcfbc13c104caa2fbcde590252b262e5c0133b38f1bfe9493574f32cb79220a9", + "dataset_sha256": "670c3909a381f375b0a99fcef256378c5b3892b2eefbde3167300dc91ed561b5", "market_type": "SPOT", - "parameter_set_sha256": "4c0bcbd3c7635cfe3653b227c121ccd44232cee32b1e2f5907a01d54ba5aa55d", + "parameter_set_sha256": "4bd1c4a9aa115e27998e0721c7700ae6a1d8a8cf09af95d34b3ce624edda1ac9", "strategy_id": "breakout_retest", "strategy_sha256": "2a7e0ad5ccc7292a0b06b76b233fa93552d34bfe993a0db4976e3404b1ff7486", "strategy_version": "1", @@ -71,22 +71,22 @@ "timeframe": "15m" }, "setup_type": "breakout_retest", - "validation_config_sha256": "321df28133ee9340bcf10c2a9c30363b34aaf449eb81815e8755ec99179da15c" + "validation_config_sha256": "3580cd32f763b70502b44c2d474bc935e0bfe1c85b4401341de1f063ce6d5562" } }, { "bundle": { - "path": "oos_runtime/2fb323a6e8ebc165563e8383eea7cc59abe02de5bfbf75244bf91a834ecc3efc/bundle.json", - "sha256": "cafc83cab079e77701a47afff27366cfd4c901d150c7898c2eb41bb0d8d6f3a0" + "path": "oos_runtime/8262f89da3dece8463ff55097d2933306fa4aca927d58c0383b2288156f2f5ad/bundle.json", + "sha256": "8126174a6f065fb927444983023d7ec9dcb76928147de1e18465555fe8d0bb39" }, "subject": { "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", "feature_definition_sha256": "7cfbd10b251a8e6acfa0656247e9f99f56c4b6e24849c8a36d66fce734f746a3", "promotion": { - "code_revision": "d590e02cd164b307bede6cb1032e86f506f463058978fd17fe07faf2bb40274d", - "dataset_sha256": "d5b4a4c5fff2a1956b3189c8a6b996352b00edf9537a2f3237daef3c35f7a358", + "code_revision": "bcfbc13c104caa2fbcde590252b262e5c0133b38f1bfe9493574f32cb79220a9", + "dataset_sha256": "670c3909a381f375b0a99fcef256378c5b3892b2eefbde3167300dc91ed561b5", "market_type": "SPOT", - "parameter_set_sha256": "4c0bcbd3c7635cfe3653b227c121ccd44232cee32b1e2f5907a01d54ba5aa55d", + "parameter_set_sha256": "4bd1c4a9aa115e27998e0721c7700ae6a1d8a8cf09af95d34b3ce624edda1ac9", "strategy_id": "support_reclaim", "strategy_sha256": "7cfbd10b251a8e6acfa0656247e9f99f56c4b6e24849c8a36d66fce734f746a3", "strategy_version": "1", @@ -94,22 +94,22 @@ "timeframe": "15m" }, "setup_type": "support_reclaim", - "validation_config_sha256": "e41261d31a16f7110974270168d7583e976af26c4f0e45c0ac9787ac37415fd9" + "validation_config_sha256": "5bfc944927f384c71d5a567195d4f9f3fab658cf15ef634a4009b6ca0d8276b7" } }, { "bundle": { - "path": "oos_runtime/0a389c8d2eb25b40d6422f4bd53968198d73a273c5d15b72a451c7fa17211e8d/bundle.json", - "sha256": "d56cc3419bf242c3e6f32222fec664e2df2a32c8443d78a668dab77aaffc8544" + "path": "oos_runtime/a2cd1cdce2adf647174ca45f03075535c7fcb9b0c05313b27f9f3ae7ed9a4d42/bundle.json", + "sha256": "1f87af0f67f5250fc737428dd9cfa34e6fde3571a1df5af0f85665fc273bf1b5" }, "subject": { "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", "feature_definition_sha256": "4f33ee776ac3b463420d6a0f2c08352a0673d16633d2a32765d861fcb8e144ea", "promotion": { - "code_revision": "d590e02cd164b307bede6cb1032e86f506f463058978fd17fe07faf2bb40274d", - "dataset_sha256": "d5b4a4c5fff2a1956b3189c8a6b996352b00edf9537a2f3237daef3c35f7a358", + "code_revision": "bcfbc13c104caa2fbcde590252b262e5c0133b38f1bfe9493574f32cb79220a9", + "dataset_sha256": "670c3909a381f375b0a99fcef256378c5b3892b2eefbde3167300dc91ed561b5", "market_type": "SPOT", - "parameter_set_sha256": "4c0bcbd3c7635cfe3653b227c121ccd44232cee32b1e2f5907a01d54ba5aa55d", + "parameter_set_sha256": "4bd1c4a9aa115e27998e0721c7700ae6a1d8a8cf09af95d34b3ce624edda1ac9", "strategy_id": "failed_breakout_reversal", "strategy_sha256": "4f33ee776ac3b463420d6a0f2c08352a0673d16633d2a32765d861fcb8e144ea", "strategy_version": "1", @@ -117,22 +117,22 @@ "timeframe": "15m" }, "setup_type": "failed_breakout_reversal", - "validation_config_sha256": "3ef53ffe56eff9c1d73335f0ccf86881aca2085cebab2d3dbe9bedb7dfbfa73e" + "validation_config_sha256": "af093bbf27c520339541d1f4dc6cbd416cb7d370216e64e70164e02ce9a64582" } }, { "bundle": { - "path": "oos_runtime/9197288fff32e4f72c9ac914c275ddb94333f975579a2d368cf863477b3ba9ee/bundle.json", - "sha256": "ba3ae8569a2540107933896940aa5cc4336c7ec173f65b0379b8255b09f99967" + "path": "oos_runtime/fcfc4e58d7a510d8a7598d4006dd736d3538480696988e509653d589b254feba/bundle.json", + "sha256": "ac989b936304b50caad60375adff155c1cc6567568a20cf92c3d236f9fa217fb" }, "subject": { "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", "feature_definition_sha256": "63f712ab32697b7a9c67ee81847bcdc9f7171002e2f7e4e6a213ae2e12edd979", "promotion": { - "code_revision": "d590e02cd164b307bede6cb1032e86f506f463058978fd17fe07faf2bb40274d", - "dataset_sha256": "726c5f1a6295f1323671cd642bdbf6c4976623cd71fce1d2345446a04b70a445", + "code_revision": "bcfbc13c104caa2fbcde590252b262e5c0133b38f1bfe9493574f32cb79220a9", + "dataset_sha256": "a331860e84a646b5fca88d9e3cd0b0741e48eb9a1012cb127d4dbee636a59dfa", "market_type": "SPOT", - "parameter_set_sha256": "feca57cc3c10c80e9086f18e57fd8f498715f7cd865a465b690612a523c4eff3", + "parameter_set_sha256": "054e2cde500c74e3de0cf53a0ea4cc187d205f1cbcdad5598459081a5c6e6be4", "strategy_id": "trend_continuation", "strategy_sha256": "63f712ab32697b7a9c67ee81847bcdc9f7171002e2f7e4e6a213ae2e12edd979", "strategy_version": "1", @@ -140,22 +140,22 @@ "timeframe": "1h" }, "setup_type": "trend_continuation", - "validation_config_sha256": "e4d677f159d8db0984212f4dbb2fd986456a94b6a522a0f3a8bd705dcfe7c63e" + "validation_config_sha256": "ffb237f4724bd11617cf1bfbf21c900542a5e42f7c2a4cea9a15c036ae01be0a" } }, { "bundle": { - "path": "oos_runtime/21654491ef611e991dc41073fe72e039342c2b75c6c055be0c520bb194accb6a/bundle.json", - "sha256": "74db017db2349b79b493907c78a46c85c8af7adeae3fed40b266dfa5bd3a91b8" + "path": "oos_runtime/c8ca129ca37d86dbba6f7e084565b52b9376c57f56a21ee327d73860b9f0d8e4/bundle.json", + "sha256": "b6180f55d27849bce4647b353e6468771eedd73e596b406399051dc594d2a9e4" }, "subject": { "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", "feature_definition_sha256": "d676e5ee3551bed7965e95cb421ae999225b9b6c0e2662468065506836fe5fc7", "promotion": { - "code_revision": "d590e02cd164b307bede6cb1032e86f506f463058978fd17fe07faf2bb40274d", - "dataset_sha256": "726c5f1a6295f1323671cd642bdbf6c4976623cd71fce1d2345446a04b70a445", + "code_revision": "bcfbc13c104caa2fbcde590252b262e5c0133b38f1bfe9493574f32cb79220a9", + "dataset_sha256": "a331860e84a646b5fca88d9e3cd0b0741e48eb9a1012cb127d4dbee636a59dfa", "market_type": "SPOT", - "parameter_set_sha256": "feca57cc3c10c80e9086f18e57fd8f498715f7cd865a465b690612a523c4eff3", + "parameter_set_sha256": "054e2cde500c74e3de0cf53a0ea4cc187d205f1cbcdad5598459081a5c6e6be4", "strategy_id": "pullback_continuation", "strategy_sha256": "d676e5ee3551bed7965e95cb421ae999225b9b6c0e2662468065506836fe5fc7", "strategy_version": "1", @@ -163,22 +163,22 @@ "timeframe": "1h" }, "setup_type": "pullback_continuation", - "validation_config_sha256": "4c02b6ed6017bfdfaed1f43acdddc3897919473e17cf321e3a2924f3be3c7286" + "validation_config_sha256": "136713eaba26dde6d5dd4394f075b7961334b6bb5b58d3a4b64b32002083ed21" } }, { "bundle": { - "path": "oos_runtime/5f8cbd0546b1c46e7393b5ca59f28fbbff53791d8c56090e58a602a9ebbb2f30/bundle.json", - "sha256": "f313728c514103a574fc1a17596c5f5210226d98f25056711bf8970fcf4ab2f8" + "path": "oos_runtime/d46c8a4d6034d44bcf58ed60c82fd61afcb835273bb524bff27ce46073503baf/bundle.json", + "sha256": "fcd53ac8b8ac14b71a49b8b91d7ccb9c51db0ee98684ff1904d059da68c0f86b" }, "subject": { "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", "feature_definition_sha256": "2a7e0ad5ccc7292a0b06b76b233fa93552d34bfe993a0db4976e3404b1ff7486", "promotion": { - "code_revision": "d590e02cd164b307bede6cb1032e86f506f463058978fd17fe07faf2bb40274d", - "dataset_sha256": "726c5f1a6295f1323671cd642bdbf6c4976623cd71fce1d2345446a04b70a445", + "code_revision": "bcfbc13c104caa2fbcde590252b262e5c0133b38f1bfe9493574f32cb79220a9", + "dataset_sha256": "a331860e84a646b5fca88d9e3cd0b0741e48eb9a1012cb127d4dbee636a59dfa", "market_type": "SPOT", - "parameter_set_sha256": "feca57cc3c10c80e9086f18e57fd8f498715f7cd865a465b690612a523c4eff3", + "parameter_set_sha256": "054e2cde500c74e3de0cf53a0ea4cc187d205f1cbcdad5598459081a5c6e6be4", "strategy_id": "breakout_retest", "strategy_sha256": "2a7e0ad5ccc7292a0b06b76b233fa93552d34bfe993a0db4976e3404b1ff7486", "strategy_version": "1", @@ -186,22 +186,22 @@ "timeframe": "1h" }, "setup_type": "breakout_retest", - "validation_config_sha256": "dfd84b6bd7b06f281258e1ffde2e9e3a9f700cb9ffe3506211d8a2edd3b2ac0a" + "validation_config_sha256": "4299b3871fba400b20b27f75d4d8754093ba57d7ac249cf0b579eb0f863a4eb4" } }, { "bundle": { - "path": "oos_runtime/dd4feba517eacaa7625c9765051291ed5dcfdb36d6977da33c4316caebc284ad/bundle.json", - "sha256": "0f5bdcb239914f544e299259d9a68904e5975f87e0c71712c8fdd871d2653447" + "path": "oos_runtime/8ec344bbae4e35b0f3b76947b74364300ad628f8a8136d795daab00ce187dfe4/bundle.json", + "sha256": "59ca38f180e393abec1f6bc419384d35e54456fd61c1fd4652eb370b3170c917" }, "subject": { "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", "feature_definition_sha256": "7cfbd10b251a8e6acfa0656247e9f99f56c4b6e24849c8a36d66fce734f746a3", "promotion": { - "code_revision": "d590e02cd164b307bede6cb1032e86f506f463058978fd17fe07faf2bb40274d", - "dataset_sha256": "726c5f1a6295f1323671cd642bdbf6c4976623cd71fce1d2345446a04b70a445", + "code_revision": "bcfbc13c104caa2fbcde590252b262e5c0133b38f1bfe9493574f32cb79220a9", + "dataset_sha256": "a331860e84a646b5fca88d9e3cd0b0741e48eb9a1012cb127d4dbee636a59dfa", "market_type": "SPOT", - "parameter_set_sha256": "feca57cc3c10c80e9086f18e57fd8f498715f7cd865a465b690612a523c4eff3", + "parameter_set_sha256": "054e2cde500c74e3de0cf53a0ea4cc187d205f1cbcdad5598459081a5c6e6be4", "strategy_id": "support_reclaim", "strategy_sha256": "7cfbd10b251a8e6acfa0656247e9f99f56c4b6e24849c8a36d66fce734f746a3", "strategy_version": "1", @@ -209,22 +209,22 @@ "timeframe": "1h" }, "setup_type": "support_reclaim", - "validation_config_sha256": "e84a197309ba3365ecf90011ada2fc85235fc380dcad3d7c1c0bf6f1e6932910" + "validation_config_sha256": "c77f1ef41b693700b54a48e9eef88b3563aa0d059d524c75f96fee1a7cd511e9" } }, { "bundle": { - "path": "oos_runtime/4d0fd517399594f5b16e355a3ccdb7ded1a3d723d729cc3eff9244ef5d545435/bundle.json", - "sha256": "dde2dc58f7ca9f8cadc5b46922c62779bd1ae98991ea822073791e922ce0da3e" + "path": "oos_runtime/64cfbb8bd90688a6617f6c71f4bf20db19f57c150edaff178903fe3db5f9126c/bundle.json", + "sha256": "d3007d8bc5a29fb4a5e8be97c3c34aaa25ed072800dfc5d9b0706d2b31578514" }, "subject": { "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", "feature_definition_sha256": "4f33ee776ac3b463420d6a0f2c08352a0673d16633d2a32765d861fcb8e144ea", "promotion": { - "code_revision": "d590e02cd164b307bede6cb1032e86f506f463058978fd17fe07faf2bb40274d", - "dataset_sha256": "726c5f1a6295f1323671cd642bdbf6c4976623cd71fce1d2345446a04b70a445", + "code_revision": "bcfbc13c104caa2fbcde590252b262e5c0133b38f1bfe9493574f32cb79220a9", + "dataset_sha256": "a331860e84a646b5fca88d9e3cd0b0741e48eb9a1012cb127d4dbee636a59dfa", "market_type": "SPOT", - "parameter_set_sha256": "feca57cc3c10c80e9086f18e57fd8f498715f7cd865a465b690612a523c4eff3", + "parameter_set_sha256": "054e2cde500c74e3de0cf53a0ea4cc187d205f1cbcdad5598459081a5c6e6be4", "strategy_id": "failed_breakout_reversal", "strategy_sha256": "4f33ee776ac3b463420d6a0f2c08352a0673d16633d2a32765d861fcb8e144ea", "strategy_version": "1", @@ -232,22 +232,22 @@ "timeframe": "1h" }, "setup_type": "failed_breakout_reversal", - "validation_config_sha256": "1c1aec712b2948f11b5c894fcdd3b3bb8a254857f9b8e5c0656b928fdbe42bff" + "validation_config_sha256": "7893795b4a3991b17d69e613eb88150fcbbcaf251a41931315c0b30c31a6f5df" } }, { "bundle": { - "path": "oos_runtime/cf57ba427a11bbb2ab408bcc112d76931040673e22aaf0ea688c18c973e27a74/bundle.json", - "sha256": "4aa61cdbecb5f5355e9429edb493cec50135e0d576cd9ec4c888bd625eae082f" + "path": "oos_runtime/f2d1caf9eb552061c08e1b528ad612077e9e0d2976f6a92fc5397edbcd3606dc/bundle.json", + "sha256": "be158e4bb59097fabe96054cacd07b9e414146eb6593256e58e262a12295e209" }, "subject": { "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", "feature_definition_sha256": "63f712ab32697b7a9c67ee81847bcdc9f7171002e2f7e4e6a213ae2e12edd979", "promotion": { - "code_revision": "d590e02cd164b307bede6cb1032e86f506f463058978fd17fe07faf2bb40274d", - "dataset_sha256": "d1b3e59b5ad3f22154405c8022ab5b0354dac6734deb10ca7c712d5fb288ed81", + "code_revision": "bcfbc13c104caa2fbcde590252b262e5c0133b38f1bfe9493574f32cb79220a9", + "dataset_sha256": "058e3458745e76d6202a762be480eec089e6949040cff4f391f89378129a84fc", "market_type": "SPOT", - "parameter_set_sha256": "43ee407708a48ea2da819ce0b597f20ee8c012155b1a48f874f996d7c59cbab5", + "parameter_set_sha256": "0aa632ce16ad795f3fd5e594e0b0658b4c89c6d2c7266c25f322f5730b651b37", "strategy_id": "trend_continuation", "strategy_sha256": "63f712ab32697b7a9c67ee81847bcdc9f7171002e2f7e4e6a213ae2e12edd979", "strategy_version": "1", @@ -255,22 +255,22 @@ "timeframe": "4h" }, "setup_type": "trend_continuation", - "validation_config_sha256": "8eee72b0ff9cba558c858c18b312e3823533457adbda24aea38585152e03abc2" + "validation_config_sha256": "ecf31c5205fcfa07c47ba7ae8dd2c243a9f8a23d41c802d651ee4d0e0dc04600" } }, { "bundle": { - "path": "oos_runtime/4aebc13da13c331685f66e075d78adbb371a3a7fd94469355e0b82af2fe377e5/bundle.json", - "sha256": "ae902e6e0ef5542a5b7524e4d81b552b8afb5592b6cdef4eeaf47eb75b46e620" + "path": "oos_runtime/b71f4edf1f000796eed60b671d542fb9dfe6ba9543df35a018aa102d02fd26b2/bundle.json", + "sha256": "ffafd48a9fc66d94a7aaa9ee5cc365426878bdf6baa6d080dd5add1016913f12" }, "subject": { "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", "feature_definition_sha256": "d676e5ee3551bed7965e95cb421ae999225b9b6c0e2662468065506836fe5fc7", "promotion": { - "code_revision": "d590e02cd164b307bede6cb1032e86f506f463058978fd17fe07faf2bb40274d", - "dataset_sha256": "d1b3e59b5ad3f22154405c8022ab5b0354dac6734deb10ca7c712d5fb288ed81", + "code_revision": "bcfbc13c104caa2fbcde590252b262e5c0133b38f1bfe9493574f32cb79220a9", + "dataset_sha256": "058e3458745e76d6202a762be480eec089e6949040cff4f391f89378129a84fc", "market_type": "SPOT", - "parameter_set_sha256": "43ee407708a48ea2da819ce0b597f20ee8c012155b1a48f874f996d7c59cbab5", + "parameter_set_sha256": "0aa632ce16ad795f3fd5e594e0b0658b4c89c6d2c7266c25f322f5730b651b37", "strategy_id": "pullback_continuation", "strategy_sha256": "d676e5ee3551bed7965e95cb421ae999225b9b6c0e2662468065506836fe5fc7", "strategy_version": "1", @@ -278,22 +278,22 @@ "timeframe": "4h" }, "setup_type": "pullback_continuation", - "validation_config_sha256": "6e5f47589294f4c2c7e0cd38e48bffd13e41cc10a7553cf8093e460c26e7dec2" + "validation_config_sha256": "957010c5cd007f49b8b3c2c88b470e2c7fd7271bba128f9182c785e318351323" } }, { "bundle": { - "path": "oos_runtime/061455573f76974b2e8df6f32f06cda153b33998ebb6810e72167a12d656157a/bundle.json", - "sha256": "922199a764159fba35fb02fffb83e96f3434be188417f1766bc1de79df3d88f2" + "path": "oos_runtime/d84376c1324fe18339bd18a593973a573a8e0e89301838a485821498f3193deb/bundle.json", + "sha256": "397e7678228fd78376414cd4d28f559c56103c5e49c63dab4ff4e342b899f9a6" }, "subject": { "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", "feature_definition_sha256": "81ed5bdf546ea4521d379049a78ecd0c518945f72b7414517439d2e976d59c40", "promotion": { - "code_revision": "d590e02cd164b307bede6cb1032e86f506f463058978fd17fe07faf2bb40274d", - "dataset_sha256": "d1b3e59b5ad3f22154405c8022ab5b0354dac6734deb10ca7c712d5fb288ed81", + "code_revision": "bcfbc13c104caa2fbcde590252b262e5c0133b38f1bfe9493574f32cb79220a9", + "dataset_sha256": "058e3458745e76d6202a762be480eec089e6949040cff4f391f89378129a84fc", "market_type": "SPOT", - "parameter_set_sha256": "43ee407708a48ea2da819ce0b597f20ee8c012155b1a48f874f996d7c59cbab5", + "parameter_set_sha256": "0aa632ce16ad795f3fd5e594e0b0658b4c89c6d2c7266c25f322f5730b651b37", "strategy_id": "breakout_retest", "strategy_sha256": "81ed5bdf546ea4521d379049a78ecd0c518945f72b7414517439d2e976d59c40", "strategy_version": "1", @@ -301,22 +301,22 @@ "timeframe": "4h" }, "setup_type": "breakout_retest", - "validation_config_sha256": "3d2e8dd7249dde8dae5c80128f33eabe966cf644132be3af45bca4eba547e147" + "validation_config_sha256": "1acd73838e91895ec9338686015db2fc482100d95b9050b3b0badebfefa99711" } }, { "bundle": { - "path": "oos_runtime/4606b8f97b6cc8de8702a038f9579e47de2263cee50c889c1c6c1ec5b55d1a47/bundle.json", - "sha256": "a2982f7e4aa431c7852387f91d89aeed71ba3b580a1f226d1747c2d167b85620" + "path": "oos_runtime/f6a8dd030e25e1c6462c4d73d9174cc46e9ab5888b0e24b3bfbda167332bc205/bundle.json", + "sha256": "5835a4f21b547c509aff9a593dd031fc6807136726738fcd4cde52a578df4b4e" }, "subject": { "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", "feature_definition_sha256": "ab8ad8719e8ae46a24c00a9fa61b06b65ec081f6282f532f3e6fd5922d517a07", "promotion": { - "code_revision": "d590e02cd164b307bede6cb1032e86f506f463058978fd17fe07faf2bb40274d", - "dataset_sha256": "d1b3e59b5ad3f22154405c8022ab5b0354dac6734deb10ca7c712d5fb288ed81", + "code_revision": "bcfbc13c104caa2fbcde590252b262e5c0133b38f1bfe9493574f32cb79220a9", + "dataset_sha256": "058e3458745e76d6202a762be480eec089e6949040cff4f391f89378129a84fc", "market_type": "SPOT", - "parameter_set_sha256": "43ee407708a48ea2da819ce0b597f20ee8c012155b1a48f874f996d7c59cbab5", + "parameter_set_sha256": "0aa632ce16ad795f3fd5e594e0b0658b4c89c6d2c7266c25f322f5730b651b37", "strategy_id": "support_reclaim", "strategy_sha256": "ab8ad8719e8ae46a24c00a9fa61b06b65ec081f6282f532f3e6fd5922d517a07", "strategy_version": "1", @@ -324,22 +324,22 @@ "timeframe": "4h" }, "setup_type": "support_reclaim", - "validation_config_sha256": "5910dfcd0805be7c336bf63f0006353c2d3b88097110ad2ba257ef6456fb0d0c" + "validation_config_sha256": "18a642dcfb07e6a88492bbd3f653860c32147a51283052a13cdfe6878fc9e581" } }, { "bundle": { - "path": "oos_runtime/d96b2e5bec0a13541279315b199b321a953d9e2c7d2d1d419196a83272382765/bundle.json", - "sha256": "d27dec3db7cd1006974ca484c8b4c4800f50be74919a06cb0525f9966718aeb1" + "path": "oos_runtime/c9ee64e48fadb8e3a48fc2623a82bbf4e4a411a2e82babd8c6a7fca01ba55530/bundle.json", + "sha256": "627089c7b2d78787ad34b41e683b33caa9e7d160e85184d65811435169fe7ca2" }, "subject": { "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", "feature_definition_sha256": "4f33ee776ac3b463420d6a0f2c08352a0673d16633d2a32765d861fcb8e144ea", "promotion": { - "code_revision": "d590e02cd164b307bede6cb1032e86f506f463058978fd17fe07faf2bb40274d", - "dataset_sha256": "d1b3e59b5ad3f22154405c8022ab5b0354dac6734deb10ca7c712d5fb288ed81", + "code_revision": "bcfbc13c104caa2fbcde590252b262e5c0133b38f1bfe9493574f32cb79220a9", + "dataset_sha256": "058e3458745e76d6202a762be480eec089e6949040cff4f391f89378129a84fc", "market_type": "SPOT", - "parameter_set_sha256": "43ee407708a48ea2da819ce0b597f20ee8c012155b1a48f874f996d7c59cbab5", + "parameter_set_sha256": "0aa632ce16ad795f3fd5e594e0b0658b4c89c6d2c7266c25f322f5730b651b37", "strategy_id": "failed_breakout_reversal", "strategy_sha256": "4f33ee776ac3b463420d6a0f2c08352a0673d16633d2a32765d861fcb8e144ea", "strategy_version": "1", @@ -347,7 +347,7 @@ "timeframe": "4h" }, "setup_type": "failed_breakout_reversal", - "validation_config_sha256": "78281bccf0c5bde80fd611bff8ec505fb69b37136190ce163dab32850a08229e" + "validation_config_sha256": "ef9f58aad8bec69122a1a7741a40a3f15d189a8496d63da8e4b65b095f1e9d77" } } ] diff --git a/src/ai4binance/cli/runtime.py b/src/ai4binance/cli/runtime.py index 241d736a..01280c66 100644 --- a/src/ai4binance/cli/runtime.py +++ b/src/ai4binance/cli/runtime.py @@ -908,14 +908,22 @@ def _run_virtual_market_research_cycle( ) -> int: from ai4binance.cli.research import run_public_research_command - return run_public_research_command( + journal = _virtual_wallet_journal(settings) + exit_code = run_public_research_command( "research-public", settings, public_acquisition=public_acquisition, whale_fusion_cycle=None, cycle_report=cycle_report, - virtual_wallet_journal=_virtual_wallet_journal(settings), + virtual_wallet_journal=journal, ) + if cycle_report is not None: + cycle_report["daily_loss_tuning"] = _run_virtual_loss_tuning( + settings, + journal, + datetime.now(UTC), + ) + return exit_code def _request_market_history_refresh_if_stale( @@ -966,6 +974,186 @@ def _virtual_wallet_journal(settings: Settings) -> VirtualWalletJournal: ) +def _run_virtual_loss_tuning( + settings: Settings, + journal: VirtualWalletJournal, + observed_at: datetime, +) -> dict[str, object]: + """Run canonical Spot backtest/tuning after three same-day virtual losses.""" + + safe_state = { + "execution_allowed": False, + "promotion_status": "RESEARCH_ONLY", + "live_eligibility_status": "LIVE_ORDER_BLOCKED", + } + try: + trigger = journal.daily_loss_tuning_trigger(observed_at) + except (OSError, RuntimeError, TypeError, ValueError): + return { + "status": "BLOCKED", + "blockers": ["VIRTUAL_LOSS_TUNING_TRIGGER_UNAVAILABLE"], + "parameter_application": "NOT_APPLIED", + **safe_state, + } + if trigger.get("status") != "TRIGGERED": + return {**trigger, "parameter_application": "NOT_APPLIED"} + trigger_id = str(trigger.get("trigger_id", "")) + if not re.fullmatch(r"virtual-loss-tuning:[0-9a-f]{24}", trigger_id): + return { + "status": "BLOCKED", + "blockers": ["VIRTUAL_LOSS_TUNING_TRIGGER_INVALID"], + "parameter_application": "NOT_APPLIED", + **safe_state, + } + tuning_root = settings.validation_artifact_directory / "virtual_loss_tuning" + artifact_path = tuning_root / f"{trigger_id.rsplit(':', maxsplit=1)[-1]}.json" + existing = _load_json_mapping(artifact_path) + if existing: + if ( + existing.get("trigger_id") != trigger_id + or existing.get("execution_allowed") is not False + or existing.get("promotion_status") != "RESEARCH_ONLY" + or existing.get("live_eligibility_status") != "LIVE_ORDER_BLOCKED" + ): + return { + "status": "BLOCKED", + "blockers": ["VIRTUAL_LOSS_TUNING_ARTIFACT_INVALID"], + "parameter_application": "NOT_APPLIED", + **safe_state, + } + if existing.get("status") == "RESEARCH_TUNING_COMPLETED": + return { + "status": "ALREADY_REVIEWED", + "trigger_id": trigger_id, + "artifact_path": str(artifact_path), + "parameter_application": "NOT_APPLIED", + **safe_state, + } + attempted_at = existing.get("attempted_at") + try: + previous_attempt = datetime.fromisoformat(str(attempted_at)).astimezone(UTC) + except (TypeError, ValueError): + previous_attempt = observed_at.astimezone(UTC) - timedelta(hours=1) + if observed_at.astimezone(UTC) - previous_attempt < timedelta(minutes=15): + return { + "status": "RETRY_PENDING", + "trigger_id": trigger_id, + "artifact_path": str(artifact_path), + "blockers": existing.get("blockers", []), + "parameter_application": "NOT_APPLIED", + **safe_state, + } + + from ai4binance.application.validation_pipeline import VALIDATED_PLAYBOOKS + from ai4binance.data import DatasetIntegrityError, ParquetOHLCVArchive + from ai4binance.validation_pipeline_runtime import ( + SPOT_VALIDATION_NOTIONAL_TO_EQUITY_RATIO, + HistoricalValidationRuntime, + ) + + raw_subjects = trigger.get("subjects") + subjects = raw_subjects if isinstance(raw_subjects, list) else [] + archive = ParquetOHLCVArchive( + settings.dataset_directory / "spot" + if settings.market_history_local_candles + else settings.dataset_directory + ) + runtime = HistoricalValidationRuntime( + report_directory=settings.backtest_report_directory / "virtual_loss_tuning", + position_notional_to_equity_ratio=(SPOT_VALIDATION_NOTIONAL_TO_EQUITY_RATIO), + ) + results: list[dict[str, object]] = [] + aggregate_blockers: list[str] = [] + for raw_subject in subjects[:3]: + if not isinstance(raw_subject, Mapping): + aggregate_blockers.append("VIRTUAL_LOSS_TUNING_SUBJECT_INVALID") + continue + market = str(raw_subject.get("market", "")).upper() + symbol = str(raw_subject.get("symbol", "")).upper() + timeframe = str(raw_subject.get("timeframe", "")) + playbook = str(raw_subject.get("strategy_id", "")) + subject = { + "market": market, + "symbol": symbol, + "timeframe": timeframe, + "strategy_id": playbook, + } + if market != "SPOT" or playbook not in VALIDATED_PLAYBOOKS: + blocker = "VIRTUAL_LOSS_TUNING_SUBJECT_UNSUPPORTED" + aggregate_blockers.append(blocker) + results.append( + {"subject": subject, "status": "BLOCKED", "blockers": [blocker]} + ) + continue + try: + candles = archive.read(symbol, timeframe) + result = runtime.validate_one( + symbol, + timeframe, + playbook, + candles, + artifact_directory=tuning_root / "evidence", + ) + tuning = cast(Any, result.tuning) + backtest = cast(Any, result.backtest) + if tuning is None or backtest is None: + raise ValueError("VIRTUAL_LOSS_TUNING_RESULT_INCOMPLETE") + results.append( + { + "subject": subject, + "status": "RESEARCH_TUNING_COMPLETED", + "dataset_sha256": runtime.dataset_sha256(candles), + "backtest_metrics": to_primitive(backtest.metrics), + "tuning_report_id": tuning.report_id, + "candidate_count": tuning.search_space.candidate_count, + "selected_parameters": to_primitive(tuning.selected_parameters), + "tuning_blockers": list(tuning.blockers), + "validation_blockers": list(result.blockers), + "parameter_application": "NOT_APPLIED", + } + ) + except ( + DatasetIntegrityError, + FileNotFoundError, + OSError, + TypeError, + ValueError, + ): + blocker = "VIRTUAL_LOSS_TUNING_DATA_OR_VALIDATION_UNAVAILABLE" + aggregate_blockers.append(blocker) + results.append( + {"subject": subject, "status": "BLOCKED", "blockers": [blocker]} + ) + if not subjects: + aggregate_blockers.append("VIRTUAL_LOSS_TUNING_SUBJECTS_MISSING") + status = ( + "RESEARCH_TUNING_COMPLETED" + if results + and all(item.get("status") == "RESEARCH_TUNING_COMPLETED" for item in results) + else "RETRY_PENDING" + ) + payload = { + "schema_version": "VirtualLossTuningResult/v1", + "status": status, + "trigger_id": trigger_id, + "trigger": trigger, + "attempted_at": observed_at.astimezone(UTC).isoformat(), + "results": results, + "blockers": list(dict.fromkeys(aggregate_blockers)), + "parameter_application": "NOT_APPLIED", + **safe_state, + } + write_json_object_verified( + artifact_path, + payload, + blocker="VIRTUAL_LOSS_TUNING_WRITE_FAILED", + subject_id=trigger_id, + indent=2, + durable=True, + ) + return {**payload, "artifact_path": str(artifact_path)} + + def _virtual_wallet_report_root(state_path: Path) -> Path: resolved = state_path.resolve() for parent in resolved.parents: diff --git a/src/ai4binance/virtual_wallet_journal.py b/src/ai4binance/virtual_wallet_journal.py index 24955e06..bab7e983 100644 --- a/src/ai4binance/virtual_wallet_journal.py +++ b/src/ai4binance/virtual_wallet_journal.py @@ -7,6 +7,7 @@ from dataclasses import dataclass, field, replace from datetime import UTC, datetime, timedelta from decimal import Decimal +from hashlib import sha256 from itertools import pairwise from pathlib import Path from typing import cast @@ -35,6 +36,7 @@ _INITIAL_EQUITY_USDT = Decimal("1000") _INITIAL_EVENT = "VIRTUAL_WALLET_INITIALIZED" _MOVEMENT_EVENT = "VIRTUAL_WALLET_MOVEMENT_RECORDED" +_DAILY_LOSS_TUNING_THRESHOLD = 3 class VirtualWalletJournalError(RuntimeError): @@ -371,6 +373,19 @@ def dashboard_snapshot(self, observed_at: datetime) -> dict[str, object]: ) as error: raise VirtualWalletJournalError(str(error)) from error + def daily_loss_tuning_trigger(self, observed_at: datetime) -> dict[str, object]: + """Create one deterministic research trigger per three same-day losses.""" + + try: + return _daily_loss_tuning_trigger( + self._read_movements(), + _utc_timestamp(observed_at), + ) + except VirtualWalletJournalError: + raise + except (ArithmeticError, KeyError, TypeError, ValueError) as error: + raise VirtualWalletJournalError(str(error)) from error + @staticmethod def _latest_positions( movements: tuple[dict[str, object], ...], @@ -627,6 +642,111 @@ def _initial_portfolio(market: str) -> VirtualPortfolioState: ) +def _daily_loss_tuning_trigger( + movements: tuple[dict[str, object], ...], + observed_at: datetime, +) -> dict[str, object]: + safe_state = { + "execution_allowed": False, + "promotion_status": "RESEARCH_ONLY", + "live_eligibility_status": "LIVE_ORDER_BLOCKED", + } + observed_day = observed_at.astimezone(UTC).date() + losses_by_market: dict[str, list[dict[str, object]]] = {} + for movement in movements: + closed_trade = movement.get("closed_trade") + position = movement.get("managed_position") + if not isinstance(closed_trade, Mapping) or not isinstance(position, Mapping): + continue + net_pnl = _required_decimal(closed_trade, "net_pnl_usdt") + if net_pnl >= 0: + continue + exit_time = datetime.fromisoformat( + _aware_timestamp_text(closed_trade.get("exit_time")) + ).astimezone(UTC) + if exit_time.date() != observed_day: + continue + market = _required_text(movement, "market").upper() + if market not in _MARKETS: + raise ValueError("VIRTUAL_LOSS_TUNING_MARKET_INVALID") + entry_price = _required_decimal(position, "entry_price") + quantity = _required_decimal(position, "initial_quantity") + notional = entry_price * quantity + if notional <= 0: + raise ValueError("VIRTUAL_LOSS_TUNING_NOTIONAL_INVALID") + losses_by_market.setdefault(market, []).append( + { + "movement_id": _required_text(movement, "movement_id"), + "trade_id": _required_text(closed_trade, "trade_id"), + "exit_time": exit_time.isoformat(), + "net_pnl_usdt": str(net_pnl), + "net_return": str(net_pnl / notional), + "subject": { + "market": market, + "symbol": _required_text(position, "symbol").upper(), + "timeframe": _required_text(position, "timeframe"), + "strategy_id": _required_text(position, "strategy_id"), + "strategy_version": _required_text(position, "strategy_version"), + "strategy_config_hash": _required_text( + position, "strategy_config_hash" + ), + }, + } + ) + + triggers: list[dict[str, object]] = [] + for market, losses in losses_by_market.items(): + completed_count = ( + len(losses) // _DAILY_LOSS_TUNING_THRESHOLD + ) * _DAILY_LOSS_TUNING_THRESHOLD + if completed_count < _DAILY_LOSS_TUNING_THRESHOLD: + continue + batch = losses[completed_count - _DAILY_LOSS_TUNING_THRESHOLD : completed_count] + subjects = list( + { + json.dumps(item["subject"], sort_keys=True): item["subject"] + for item in batch + }.values() + ) + identity = { + "kind": "SAME_UTC_DAY_NET_LOSS_BATCH", + "market": market, + "trade_date": observed_day.isoformat(), + "evidence_ids": [item["movement_id"] for item in batch], + } + digest = sha256( + json.dumps(identity, separators=(",", ":"), sort_keys=True).encode() + ).hexdigest() + triggers.append( + { + "schema_version": "VirtualLossTuningTrigger/v1", + "status": "TRIGGERED", + "trigger_id": f"virtual-loss-tuning:{digest[:24]}", + **identity, + "loss_threshold": _DAILY_LOSS_TUNING_THRESHOLD, + "loss_count_today": len(losses), + "observed_at": observed_at.isoformat(), + "losses": batch, + "subjects": subjects, + **safe_state, + } + ) + if not triggers: + return { + "schema_version": "VirtualLossTuningTrigger/v1", + "status": "NOT_TRIGGERED", + "trade_date": observed_day.isoformat(), + "loss_threshold": _DAILY_LOSS_TUNING_THRESHOLD, + "loss_count_today": max( + (len(losses) for losses in losses_by_market.values()), + default=0, + ), + "observed_at": observed_at.isoformat(), + **safe_state, + } + return max(triggers, key=lambda item: str(item["observed_at"])) + + def _independent_portfolios( portfolios: Mapping[str, VirtualPortfolioState], ) -> IndependentVirtualPortfolios: diff --git a/tests/test_cli.py b/tests/test_cli.py index 26a1f598..305144da 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -7,11 +7,12 @@ import shutil import sys import tempfile +from collections.abc import Mapping from datetime import UTC, datetime, timedelta from decimal import Decimal from pathlib import Path from types import SimpleNamespace -from typing import cast +from typing import Any, cast import pytest @@ -1658,10 +1659,107 @@ def test_virtual_market_research_cycle_persists_both_wallets_and_report( assert cycle_report["virtual_runtime_evaluated"] is False assert cycle_report["virtual_simulation_outcome"] == "PRECONDITIONS_BLOCKED" assert cycle_report["virtual_order_ready"] is False + tuning = cast(Mapping[str, object], cycle_report["daily_loss_tuning"]) + assert tuning["status"] == "NOT_TRIGGERED" + assert tuning["loss_threshold"] == 3 + assert tuning["parameter_application"] == "NOT_APPLIED" report_path = tmp_path / "runtime" / "reports" / "virtual_wallets" / "latest.md" assert report_path.exists() +def test_virtual_loss_tuning_runs_canonical_optimizer_without_applying_parameters( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from ai4binance import validation_pipeline_runtime as validation_module + from ai4binance.cli import runtime as runtime_cli + from ai4binance.data import archive as archive_module + from ai4binance.validation import ParameterSet + + observed_at = datetime(2026, 9, 25, 0, 0, tzinfo=UTC) + + trigger = { + "schema_version": "VirtualLossTuningTrigger/v1", + "status": "TRIGGERED", + "trigger_id": "virtual-loss-tuning:" + "a" * 24, + "subjects": [ + { + "market": "SPOT", + "symbol": "BTCUSDT", + "timeframe": "1h", + "strategy_id": "trend_continuation", + } + ], + "execution_allowed": False, + "promotion_status": "RESEARCH_ONLY", + "live_eligibility_status": "LIVE_ORDER_BLOCKED", + } + journal = SimpleNamespace(daily_loss_tuning_trigger=lambda _observed: trigger) + candles = (SimpleNamespace(timestamp=observed_at),) * 60 + monkeypatch.setattr( + archive_module.ParquetOHLCVArchive, + "read", + lambda *_args: candles, + ) + + class Runtime: + def __init__(self, **_kwargs: object) -> None: + pass + + @staticmethod + def dataset_sha256(_candles: object) -> str: + return "b" * 64 + + @staticmethod + def validate_one(*_args: object, **_kwargs: object) -> object: + return SimpleNamespace( + backtest=SimpleNamespace( + metrics={"net_return": -0.01, "trade_count": 3} + ), + tuning=SimpleNamespace( + report_id="tuning:test", + search_space=SimpleNamespace(candidate_count=9), + selected_parameters=ParameterSet( + "selected", + ( + ("atr_stop_multiplier", 1.25), + ("take_profit_multiplier", 3.5), + ), + ), + blockers=("OOS_RETURN_INSUFFICIENT",), + ), + blockers=("OOS_RETURN_INSUFFICIENT",), + ) + + monkeypatch.setattr(validation_module, "HistoricalValidationRuntime", Runtime) + settings = Settings( + dataset_directory=tmp_path / "data", + validation_artifact_directory=tmp_path / "validation", + backtest_report_directory=tmp_path / "reports", + ) + + result = runtime_cli._run_virtual_loss_tuning( + settings, + cast(Any, journal), + observed_at, + ) + repeated = runtime_cli._run_virtual_loss_tuning( + settings, + cast(Any, journal), + observed_at + timedelta(minutes=1), + ) + + assert result["status"] == "RESEARCH_TUNING_COMPLETED" + assert result["parameter_application"] == "NOT_APPLIED" + assert result["execution_allowed"] is False + assert result["promotion_status"] == "RESEARCH_ONLY" + assert result["live_eligibility_status"] == "LIVE_ORDER_BLOCKED" + tuning_result = cast(list[Mapping[str, object]], result["results"])[0] + assert tuning_result["candidate_count"] == 9 + assert tuning_result["parameter_application"] == "NOT_APPLIED" + assert repeated["status"] == "ALREADY_REVIEWED" + + def test_virtual_market_daemon_fails_closed_and_records_cycle_failure( tmp_path: Path, ) -> None: diff --git a/tests/test_virtual_wallet_journal.py b/tests/test_virtual_wallet_journal.py index 28fff809..01ace8b3 100644 --- a/tests/test_virtual_wallet_journal.py +++ b/tests/test_virtual_wallet_journal.py @@ -54,6 +54,64 @@ def test_virtual_wallet_journal_initializes_independent_wallets_and_report( assert "2026-09-12T13:00:00+00:00" in report +def test_daily_loss_tuning_trigger_requires_three_losses_in_same_utc_day() -> None: + from ai4binance import virtual_wallet_journal as module + + def movement(index: int, *, pnl: str, exit_at: datetime) -> dict[str, object]: + return { + "movement_id": f"movement:{index}", + "market": "SPOT", + "closed_trade": { + "trade_id": f"trade:{index}", + "exit_time": exit_at.isoformat(), + "net_pnl_usdt": pnl, + }, + "managed_position": { + "symbol": "BTCUSDT", + "timeframe": "1h", + "strategy_id": "trend_continuation", + "strategy_version": "1", + "strategy_config_hash": "a" * 64, + "entry_price": "100", + "initial_quantity": "1", + }, + } + + prior = movement(0, pnl="-2", exit_at=NOW - timedelta(days=1)) + first = movement(1, pnl="-1", exit_at=NOW - timedelta(hours=2)) + win = movement(2, pnl="3", exit_at=NOW - timedelta(hours=1)) + second = movement(3, pnl="-2", exit_at=NOW - timedelta(minutes=30)) + before = module._daily_loss_tuning_trigger((prior, first, win, second), NOW) + assert before["status"] == "NOT_TRIGGERED" + assert before["loss_count_today"] == 2 + + third = movement(4, pnl="-3", exit_at=NOW) + triggered = module._daily_loss_tuning_trigger( + (prior, first, win, second, third), NOW + ) + repeated = module._daily_loss_tuning_trigger( + ( + prior, + first, + win, + second, + third, + movement(5, pnl="-4", exit_at=NOW), + ), + NOW, + ) + + assert triggered["status"] == "TRIGGERED" + assert triggered["loss_threshold"] == 3 + assert triggered["loss_count_today"] == 3 + assert len(cast(list[object], triggered["losses"])) == 3 + assert len(cast(list[object], triggered["subjects"])) == 1 + assert repeated["trigger_id"] == triggered["trigger_id"] + assert repeated["execution_allowed"] is False + assert repeated["promotion_status"] == "RESEARCH_ONLY" + assert repeated["live_eligibility_status"] == "LIVE_ORDER_BLOCKED" + + def test_virtual_wallet_journal_records_every_change_once_with_timestamp( tmp_path: Path, ) -> None: From c80546c59cf4aa101f861dd33583db77a02a4198 Mon Sep 17 00:00:00 2001 From: Huseyin Cicek Date: Fri, 25 Sep 2026 15:05:56 +0300 Subject: [PATCH 09/19] Refresh governed approval evidence hashes and publication tooling policy --- .gitignore | 1 + .../governed_document_lock_manifest.json | 46 ++++---- .../technology_language_ownership.yaml | 4 +- ...ference_global_terminology_and_taxonomy.md | 6 +- .../repository-validator/manifest-policy.json | 10 +- publication/README.md | 6 + .../export_governed_document_lock_approval.py | 95 ++++++++++++++-- ...overned_document_lock_approval_evidence.py | 10 +- src/ai4binance/data/market_history_sync.py | 1 + .../governance/constitution_sync.py | 42 ++++++- .../governance_enforcement_fabric.py | 14 ++- .../governance/repository_validator.py | 10 +- .../governance/technology_language_policy.py | 7 +- .../governance/terminology_policy.py | 9 +- src/ai4binance/ops/architecture_migration.py | 4 + src/ai4binance/ops/public_showcase.py | 10 +- .../test_technology_language_policy.py | 2 + .../test_governance_enforcement_fabric.py | 8 +- .../terminology/test_terminology_policy.py | 5 +- tests/test_architecture_diagram_validation.py | 5 +- tests/test_docs_hygiene.py | 13 ++- tests/test_governance_constitution_sync.py | 104 +++++++++++++++++ tests/test_internal_radar.py | 21 ++-- tests/test_internal_radar_vision.py | 16 ++- tests/test_kaizen_quality.py | 12 ++ tests/test_live_readiness_preview.py | 5 +- ...est_coverage_technology_language_policy.py | 2 + tests/test_lowest_twenty_coverage_models.py | 11 +- tests/test_maintainability_ratchet.py | 5 +- tests/test_market_data_gateway.py | 36 +++--- tests/test_market_history_continuous.py | 3 +- tests/test_model_registry_coverage_closure.py | 2 +- tests/test_opportunity_monitor.py | 2 +- tests/test_public_showcase.py | 12 +- tests/test_repository_cleanup_audit.py | 4 +- tests/test_repository_validator.py | 107 ++++++++++++++++++ tests/test_runtime_artifacts_migration.py | 2 +- tests/test_storage.py | 13 ++- 38 files changed, 548 insertions(+), 117 deletions(-) diff --git a/.gitignore b/.gitignore index 08fd7e36..740f69cf 100644 --- a/.gitignore +++ b/.gitignore @@ -237,3 +237,4 @@ docs/reports/private/ /computer_local.md /docs/archive/reference_local_computer_profile.md /coverage.xml +/runtime/artifacts/assurance/security_tooling/ diff --git a/config/governance/governed_document_lock_manifest.json b/config/governance/governed_document_lock_manifest.json index f11b88f1..a54d18fa 100644 --- a/config/governance/governed_document_lock_manifest.json +++ b/config/governance/governed_document_lock_manifest.json @@ -515,7 +515,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_instruction_context_routing_20260904_001.json", - "approval_evidence_sha256": "5b41d0b5ef90cd989a6634a056d748e8e976bd704469698ac842d403ba617a8a" + "approval_evidence_sha256": "351c49e29b89a2ad818578114e2f6c897a82f6277ed03ab44cc13089a1164c47" }, { "approval_id": "AI4B-GOV-DOCLOCK-ROOT-AGENT-THIN-ROUTER-20260904-001", @@ -528,7 +528,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_root_agent_thin_router_20260904_001.json", - "approval_evidence_sha256": "bcaec3e32967a1fa573dabc480a55f68febef89572f43aa76c565e5a04e8a4f2" + "approval_evidence_sha256": "80d155af8aefa0db6797153085594d470a87e8e9903547974e7d7bb65aa83f42" }, { "approval_id": "AI4B-GOV-DOCLOCK-CUSTOM-VALIDATION-AUTHORITY-20260904-001", @@ -829,7 +829,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_root_agent_kaizen_20260903_001.json", - "approval_evidence_sha256": "7b0135d10a14831385b193c5616efff270dcb00425ee35f5f69a6d848d8ef7ef" + "approval_evidence_sha256": "b6fe64a03d4665beaf117498e90b620d25f3d51c50dde270d2e27aec9bb703c5" }, { "approval_id": "AI4B-GOV-DOCLOCK-QUALITY-GATE-PROFILES-20260901-001", @@ -899,7 +899,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_helper_process_containment_20260831_001.json", - "approval_evidence_sha256": "b0502415d12910651b47ada3ab962f16d958496708e5fb36e2e6073d3132f8ab" + "approval_evidence_sha256": "74ab95864028af2397e87303ae9ecefcf4403b0a2038bf4639abd289f29ee9fc" }, { "approval_id": "AI4B-GOV-DOCLOCK-UNIVERSAL-ENFORCEMENT-GOVERNANCE-CLOSURE-20260831-001", @@ -1059,7 +1059,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_authority_primitive_elevation_20260829_002.json", - "approval_evidence_sha256": "55c31dff7b08bcee0f0ef9c5621f466525cce2eb09fa026fa916169dbb34f572" + "approval_evidence_sha256": "937d308c01723d031bb1b82c63e1420ed44ec1ba17f3399e8778e32c0a58dc02" }, { "approval_id": "AI4B-GOV-DOCLOCK-DETERMINISTIC-GOVERNANCE-GATE-20260829-001", @@ -1094,7 +1094,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_deterministic_governance_gate_20260829_001.json", - "approval_evidence_sha256": "7956a83ad5011400fceb5779751205af698ae373cb8c816e3211349e804756fe" + "approval_evidence_sha256": "e5b60d200b1028360b0f3eaf364c6a6cf3f38c729588c2f4a7d31fba885b6862" }, { "approval_id": "AI4B-GOV-DOCLOCK-CORE-MANUAL-AUTONOMOUS-LANGUAGE-REMEDIATION-20260825-001", @@ -1232,7 +1232,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_quality_gate_revalidation_20260821_001.json", - "approval_evidence_sha256": "8903d950bdda25fcf3b26827fab942f458b49c4e503fd4264bc5553491dcde26" + "approval_evidence_sha256": "e1329b5f43638f012eca63a49f68ddaf10394b3823a6d373124656e90fc25ef8" }, { "approval_id": "AI4B-GOV-DOCLOCK-AGENTS-MARKDOWN-LOCK-RULE-20260822-001", @@ -1245,7 +1245,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_agents_markdown_lock_rule_20260822_001.json", - "approval_evidence_sha256": "238101968ab717c89d8de791a2d589ab6b835cfa16c537f8aa2d12ad242ab203" + "approval_evidence_sha256": "0d3b453cd7eb678c15b9592970fdd5b5f82c52e5df1fe48e71ebaa6893b1c9a0" }, { "approval_id": "AI4B-GOV-DOCLOCK-POLICY-AS-CODE-LOCK-CONTRACT-20260822-001", @@ -1516,7 +1516,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_governed_docs_sha_alignment_20260826_001.json", - "approval_evidence_sha256": "4cd648d702e0312228230e5fefc7e2fbdbc37c851dae82b2622eee22a3b5feb8" + "approval_evidence_sha256": "fd9f678be735c6a9d6071ae2cb781eede2f90e0b6d3fd5ac2b437d64b2cae7f5" }, { "approval_id": "AI4B-GOV-DOCLOCK-GOVERNED-DOCS-SHA-RESYNC-20260827-001", @@ -1543,7 +1543,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_governed_docs_sha_resync_20260827_001.json", - "approval_evidence_sha256": "aa3ba3ac844a7adbfc46f9b659cec135a39107cfa08f064026922348a042a99b" + "approval_evidence_sha256": "e796caf72d101591d5404a78dd57da5e4bb1db9ccbdee8a6d00562d60ff45a5c" }, { "approval_id": "AI4B-GOV-DOCLOCK-MARKDOWN-CLASSIFICATION-20260829-001", @@ -1681,7 +1681,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_full_gate_green_20260830_001.json", - "approval_evidence_sha256": "6b9942bd12d9cffb79fc40f0000e9026874bee0d20b7c8212f98b2c744735c74" + "approval_evidence_sha256": "f1e6db932d26192600ecaed1be6dd27cca432737d54087f2ff61ff89da89963f" }, { "approval_id": "AI4B-GOV-DOCLOCK-STRATEGY-REGISTRY-FAMILY-BINDING-20260831-001", @@ -1710,7 +1710,7 @@ "written_owner_approval": true, "approval_status": "APPROVED", "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_authority_primitive_canonicalization_20260830_002.json", - "approval_evidence_sha256": "cec0991ebb2c508e60cf6f9cf7a3a09dc7e11481435166c44d3c19b5829e720d" + "approval_evidence_sha256": "f3711978f9f888025e2e30eff591ea8a1941f88c995be3a2c94d4154d67f6804" }, { "approval_id": "AI4B-GOV-DOCLOCK-AUTHORITY-PRIMITIVE-CANONICALIZATION-20260830-003", @@ -1752,7 +1752,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_model_registry_architecture_20260904_001.json", - "approval_evidence_sha256": "c0627ee64c9e9f391d1d03ceab6154f1c670bb175a602007811cbd9bf4e24733" + "approval_evidence_sha256": "452634011050dba0329b87e76c64e29dd30dcfa43034f8b293e993cb8b9a811d" }, { "approval_id": "AI4B-GOV-DOCLOCK-MODEL-TAXONOMY-20260904-001", @@ -1765,7 +1765,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_model_taxonomy_20260904_001.json", - "approval_evidence_sha256": "e8f397d8eab13e975d5485f7679eb2d1025c61438bb4bc84ed3ef2d9e3818faa" + "approval_evidence_sha256": "e8c6ac236c18d0994fe0c423293a24109e4ba9f0c596290279790453f4aef411" }, { "approval_id": "AI4B-GOV-DOCLOCK-MODEL-INFERENCE-SCHEMA-20260904-001", @@ -1778,7 +1778,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_model_inference_schema_20260904_001.json", - "approval_evidence_sha256": "155fabe669633d33dddc2c3e64b89a0e3b98c9a78eb14d22c94bc3df669f9e27" + "approval_evidence_sha256": "a78a9a7f6b2ff6a8325740e2675ceba1aa21cad3af15851b30e60ce4df276b45" }, { "approval_id": "AI4B-GOV-DOCLOCK-MODEL-ROUTE-SCHEMA-20260904-001", @@ -1791,7 +1791,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_model_route_schema_20260904_001.json", - "approval_evidence_sha256": "adbc21afa84a82b30c9d62bbac2f7f23b77fe50aac29ae9aaaad903609cde9a2" + "approval_evidence_sha256": "6514446af7bd8ba11141a238ad920e6a2c1057e44195e6c92c409e5ad25b9471" }, { "approval_id": "AI4B-GOV-DOCLOCK-LOCAL-MODEL-GOVERNANCE-20260918-001", @@ -3952,7 +3952,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_instruction_context_routing_20260904_001.json", - "approval_evidence_sha256": "5b41d0b5ef90cd989a6634a056d748e8e976bd704469698ac842d403ba617a8a" + "approval_evidence_sha256": "351c49e29b89a2ad818578114e2f6c897a82f6277ed03ab44cc13089a1164c47" }, { "approval_id": "AI4B-GOV-DOCLOCK-ROOT-AGENT-THIN-ROUTER-20260904-001", @@ -3965,7 +3965,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_root_agent_thin_router_20260904_001.json", - "approval_evidence_sha256": "bcaec3e32967a1fa573dabc480a55f68febef89572f43aa76c565e5a04e8a4f2" + "approval_evidence_sha256": "80d155af8aefa0db6797153085594d470a87e8e9903547974e7d7bb65aa83f42" }, { "approval_id": "AI4B-GOV-DOCLOCK-CUSTOM-VALIDATION-AUTHORITY-20260904-001", @@ -4334,7 +4334,7 @@ "written_owner_approval": true, "approval_status": "APPROVED", "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_authority_primitive_canonicalization_20260830_002.json", - "approval_evidence_sha256": "cec0991ebb2c508e60cf6f9cf7a3a09dc7e11481435166c44d3c19b5829e720d" + "approval_evidence_sha256": "f3711978f9f888025e2e30eff591ea8a1941f88c995be3a2c94d4154d67f6804" }, { "approval_id": "AI4B-GOV-DOCLOCK-AUTHORITY-PRIMITIVE-CANONICALIZATION-20260830-003", @@ -4389,7 +4389,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_model_registry_architecture_20260904_001.json", - "approval_evidence_sha256": "c0627ee64c9e9f391d1d03ceab6154f1c670bb175a602007811cbd9bf4e24733" + "approval_evidence_sha256": "452634011050dba0329b87e76c64e29dd30dcfa43034f8b293e993cb8b9a811d" }, { "approval_id": "AI4B-GOV-DOCLOCK-MODEL-TAXONOMY-20260904-001", @@ -4402,7 +4402,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_model_taxonomy_20260904_001.json", - "approval_evidence_sha256": "e8f397d8eab13e975d5485f7679eb2d1025c61438bb4bc84ed3ef2d9e3818faa" + "approval_evidence_sha256": "e8c6ac236c18d0994fe0c423293a24109e4ba9f0c596290279790453f4aef411" }, { "approval_id": "AI4B-GOV-DOCLOCK-MODEL-INFERENCE-SCHEMA-20260904-001", @@ -4415,7 +4415,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_model_inference_schema_20260904_001.json", - "approval_evidence_sha256": "155fabe669633d33dddc2c3e64b89a0e3b98c9a78eb14d22c94bc3df669f9e27" + "approval_evidence_sha256": "a78a9a7f6b2ff6a8325740e2675ceba1aa21cad3af15851b30e60ce4df276b45" }, { "approval_id": "AI4B-GOV-DOCLOCK-MODEL-ROUTE-SCHEMA-20260904-001", @@ -4428,7 +4428,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_model_route_schema_20260904_001.json", - "approval_evidence_sha256": "adbc21afa84a82b30c9d62bbac2f7f23b77fe50aac29ae9aaaad903609cde9a2" + "approval_evidence_sha256": "6514446af7bd8ba11141a238ad920e6a2c1057e44195e6c92c409e5ad25b9471" }, { "approval_id": "AI4B-GOV-DOCLOCK-LOGICAL-ARCHITECTURE-REGISTRY-20260904-001", diff --git a/config/governance/technology_language_ownership.yaml b/config/governance/technology_language_ownership.yaml index 74c58d22..3c9e4ffb 100644 --- a/config/governance/technology_language_ownership.yaml +++ b/config/governance/technology_language_ownership.yaml @@ -44,7 +44,7 @@ languages: role: "canonical_application_language" suffixes: [".py", ".pyi"] enforce_placement: true - allowed_paths: [".agents/", "src/", "tests/", "scripts/", "tools/"] + allowed_paths: [".agents/", "src/", "tests/", "scripts/", "tools/", "publication/sanitize_publication.py"] allowed_capabilities: ["application_orchestration", "governance_integration", "risk_orchestration", "validation", "trading_intelligence"] forbidden_capabilities: ["live_execution_authority", "risk_override", "governance_bypass"] forbidden_content_patterns: [] @@ -169,7 +169,7 @@ languages: role: "declarative_configuration" suffixes: [".yaml", ".yml", ".toml"] enforce_placement: true - allowed_paths: [".agents/", ".codex/", ".github/", "config/", "docs/architecture/diagrams/diagram_registry.yaml", "docs/registries/", "workflows/", "pyproject.toml"] + allowed_paths: [".agents/", ".codex/", ".github/", "config/", "docs/architecture/diagrams/diagram_registry.yaml", "docs/registries/", "publication/public_manifest.yaml", "workflows/", "pyproject.toml"] allowed_capabilities: ["configuration", "declaration", "registry_projection"] forbidden_capabilities: ["executable_policy_engine", "hidden_business_logic"] forbidden_content_patterns: [] diff --git a/docs/references/reference_global_terminology_and_taxonomy.md b/docs/references/reference_global_terminology_and_taxonomy.md index 0c00af38..d41113a3 100644 --- a/docs/references/reference_global_terminology_and_taxonomy.md +++ b/docs/references/reference_global_terminology_and_taxonomy.md @@ -2,7 +2,7 @@ document_id: AI4B-GOV-REF-TERM-TAX-001 title: AI4BINANCE Global Terminology and Taxonomy Reference document_type: REFERENCE -version: 1.0.1 +version: 1.0.2 status: DRAFT owner: Enterprise Knowledge Governance authority_level: REFERENCE @@ -36,6 +36,10 @@ This reference preserves the project-wide terminology families, decision vocabul Normative authority remains in `docs/standards/standard_terminology_governance.md`. +The deterministic terminology projection and validation mechanics are implemented by +`src/ai4binance/governance/terminology_policy.py`. The implementation remains +non-authoritative and cannot widen the standard or the shared enforcement fabric. + | Output file | Original sections | |---|---| | `standard_terminology_governance.md` | Sections 1-6, 7, 8, 9-12 | diff --git a/policies/repository-validator/manifest-policy.json b/policies/repository-validator/manifest-policy.json index 3f7525e4..c97dee4f 100644 --- a/policies/repository-validator/manifest-policy.json +++ b/policies/repository-validator/manifest-policy.json @@ -1,14 +1,16 @@ { "policy_id": "AI4B-GOV-REPO-POLICY", - "version": "1.3.0", + "version": "1.3.1", "allowed_top_level_paths": [ ".env.example", ".gitattributes", + ".gitleaksignore", ".gitignore", ".python-version", "AGENTS.md", "CLAUDE.md", "GEMINI.md", + "LICENSE", "README.md", "pyproject.toml", "uv.lock", @@ -28,6 +30,8 @@ ".venv", ".vscode", "factory", + "examples", + "publication", "requirements.txt", "research", "alerts", @@ -195,6 +199,10 @@ ] }, "owner_by_top_level": { + ".gitleaksignore": "Security", + "LICENSE": "Governance", + "examples": "Documentation", + "publication": "Governance", "src": "Engineering", "tests": "Quality", "docs": "Governance", diff --git a/publication/README.md b/publication/README.md index b8a5e3f9..adf93eac 100644 --- a/publication/README.md +++ b/publication/README.md @@ -7,6 +7,12 @@ second, non-overridable defense layer. The export remains local-only until its selected files have passed sanitization, secret scanning, diff review, and explicit human approval. +`src/ai4binance/ops/public_showcase.py` enforces the local staging boundary and +invokes the repository-pinned Gitleaks binary with a fixed argument vector, +`shell=False`, bounded timeout handling, and redacted output. This scanner +boundary is report/block only and never grants publication, promotion, +deployment, or trading authority. + ## Disclosure Policy | Level | Public treatment | diff --git a/scripts/export_governed_document_lock_approval.py b/scripts/export_governed_document_lock_approval.py index 5abbe8c8..74a9de1d 100644 --- a/scripts/export_governed_document_lock_approval.py +++ b/scripts/export_governed_document_lock_approval.py @@ -3,12 +3,14 @@ import argparse import hashlib import json +import re from datetime import UTC, datetime -from pathlib import Path +from pathlib import Path, PurePosixPath, PureWindowsPath from typing import Any MANIFEST_PATH = Path("config/governance/governed_document_lock_manifest.json") DEFAULT_OUTPUT_DIR = Path("runtime/artifacts/repository_validation/governance") +_SHA256_PATTERN = re.compile(r"^[a-f0-9]{64}$") def _sha256(path: Path) -> str: @@ -62,9 +64,14 @@ def build_approval_evidence( *, repository_root: Path, approval_id: str, + manifest_payload: dict[str, Any] | None = None, ) -> dict[str, Any]: manifest_path = repository_root / MANIFEST_PATH - manifest = _load_manifest(manifest_path) + manifest = ( + manifest_payload + if manifest_payload is not None + else _load_manifest(manifest_path) + ) approval = _approval_record(manifest, approval_id) if approval.get("approval_status") != "APPROVED": raise ValueError("approval record must be APPROVED") @@ -154,6 +161,79 @@ def build_approval_evidence( } +def _load_bound_evidence( + *, + repository_root: Path, + approval: dict[str, Any], + output_path: Path, +) -> dict[str, Any] | None: + evidence_path = str(approval.get("approval_evidence_path", "")).strip() + evidence_sha256 = str(approval.get("approval_evidence_sha256", "")).strip().lower() + if not evidence_path and not evidence_sha256: + return None + if not evidence_path or not evidence_sha256: + raise ValueError("BOUND_APPROVAL_EVIDENCE_REFERENCE_INCOMPLETE") + relative = PurePosixPath(evidence_path) + if ( + relative.is_absolute() + or PureWindowsPath(evidence_path).is_absolute() + or "\\" in evidence_path + or ".." in relative.parts + ): + raise ValueError("BOUND_APPROVAL_EVIDENCE_PATH_INVALID") + if _SHA256_PATTERN.fullmatch(evidence_sha256) is None: + raise ValueError("BOUND_APPROVAL_EVIDENCE_SHA256_INVALID") + bound_path = (repository_root / evidence_path).resolve() + if output_path.resolve() != bound_path: + raise ValueError("BOUND_APPROVAL_EVIDENCE_PATH_OVERRIDE_BLOCKED") + if not bound_path.is_file(): + raise ValueError("BOUND_APPROVAL_EVIDENCE_MISSING") + if _sha256(bound_path) != evidence_sha256: + raise ValueError("BOUND_APPROVAL_EVIDENCE_IMMUTABLE_MISMATCH") + payload = _load_manifest(bound_path) + if ( + payload.get("artifact_origin") + != "governed_document_lock_written_owner_approval" + ): + raise ValueError("BOUND_APPROVAL_EVIDENCE_ORIGIN_INVALID") + if ( + str(payload.get("approval_id", "")).strip() + != str(approval.get("approval_id", "")).strip() + ): + raise ValueError("BOUND_APPROVAL_EVIDENCE_ID_MISMATCH") + return payload + + +def persist_approval_evidence( + *, + repository_root: Path, + approval_id: str, + output_path: Path, + manifest_payload: dict[str, Any] | None = None, +) -> dict[str, Any]: + manifest = ( + manifest_payload + if manifest_payload is not None + else _load_manifest(repository_root / MANIFEST_PATH) + ) + approval = _approval_record(manifest, approval_id) + bound = _load_bound_evidence( + repository_root=repository_root, + approval=approval, + output_path=output_path, + ) + if bound is not None: + return bound + payload = build_approval_evidence( + repository_root=repository_root, + approval_id=approval_id, + manifest_payload=manifest, + ) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + return payload + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( description=( @@ -166,10 +246,6 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) repository_root = Path(args.repository_root).resolve() - payload = build_approval_evidence( - repository_root=repository_root, - approval_id=args.approval_id, - ) if args.output_path: output_path = Path(args.output_path) if not output_path.is_absolute(): @@ -180,8 +256,11 @@ def main(argv: list[str] | None = None) -> int: / DEFAULT_OUTPUT_DIR / f"governed_document_lock_approval_{_slug(args.approval_id)}.json" ) - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + payload = persist_approval_evidence( + repository_root=repository_root, + approval_id=args.approval_id, + output_path=output_path, + ) print(json.dumps(payload, indent=2)) return 0 diff --git a/scripts/sync_governed_document_lock_approval_evidence.py b/scripts/sync_governed_document_lock_approval_evidence.py index ae34df00..7e1dbd89 100644 --- a/scripts/sync_governed_document_lock_approval_evidence.py +++ b/scripts/sync_governed_document_lock_approval_evidence.py @@ -11,7 +11,7 @@ _load_manifest, _sha256, _slug, - build_approval_evidence, + persist_approval_evidence, ) @@ -99,20 +99,18 @@ def sync_manifest( if normalized_written_owner_approvals is not None: manifest["written_owner_approvals"] = normalized_written_owner_approvals written_owner_approvals = normalized_written_owner_approvals - manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") - exported: list[dict[str, Any]] = [] for item in approval_records: if not isinstance(item, dict): raise ValueError("approval record entries must be JSON objects") approval_id = _approval_id(item) output_path = _output_path(repository_root, approval_id) - payload = build_approval_evidence( + payload = persist_approval_evidence( repository_root=repository_root, approval_id=approval_id, + output_path=output_path, + manifest_payload=manifest, ) - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") relative_output_path = output_path.relative_to(repository_root).as_posix() evidence_sha256 = _sha256(output_path) _attach_evidence_ref( diff --git a/src/ai4binance/data/market_history_sync.py b/src/ai4binance/data/market_history_sync.py index 1371f724..41d03c05 100644 --- a/src/ai4binance/data/market_history_sync.py +++ b/src/ai4binance/data/market_history_sync.py @@ -716,6 +716,7 @@ def run(self, *, max_cycles: int | None = None) -> int: now = self.clock() attempts += 1 fast_retry = False + result: object try: if self.cycle is None: result = self.synchronizer.sync_day( diff --git a/src/ai4binance/governance/constitution_sync.py b/src/ai4binance/governance/constitution_sync.py index 4bb37f16..6bc26431 100644 --- a/src/ai4binance/governance/constitution_sync.py +++ b/src/ai4binance/governance/constitution_sync.py @@ -542,6 +542,7 @@ def _audit_loose_code( if not source_paths: return () docs_blob = _read_docs_blob(root) + compliance_trace_blob = _read_compliance_trace_blob(root) tests_blob = _read_tests_blob(root) findings: list[LooseCodeFinding] = [] @@ -552,8 +553,9 @@ def _audit_loose_code( ) has_changed_test = bool(test_paths) has_written_rule = source_path in docs_blob or module_token in docs_blob - has_compliance = source_path in _read_text( - root / "docs/compliance/registry_compliance_matrix.md" + has_compliance = ( + source_path in compliance_trace_blob + or module_token in compliance_trace_blob ) if not has_test_evidence: @@ -593,6 +595,41 @@ def _audit_loose_code( return tuple(findings) +def _read_compliance_trace_blob(root: Path) -> str: + """Resolve source references through documents linked by the compliance matrix.""" + compliance_path = root / "docs/compliance/registry_compliance_matrix.md" + compliance_text = _read_text(compliance_path) + docs_root = root / "docs" + if not compliance_text or not docs_root.is_dir(): + return compliance_text + + documents = { + path.relative_to(root).as_posix(): path.read_text(encoding="utf-8") + for path in sorted(docs_root.rglob("*.md")) + if path != compliance_path + } + pending = sorted(path for path in documents if path in compliance_text) + visited: set[str] = set() + traced_text = [compliance_text] + + while pending: + document_path = pending.pop(0) + if document_path in visited: + continue + visited.add(document_path) + document_text = documents[document_path] + traced_text.append(document_text) + pending.extend( + candidate + for candidate in sorted(documents) + if candidate not in visited + and candidate not in pending + and candidate in document_text + ) + + return "\n".join(traced_text) + + def load_current_quality_gate_evidence(root: Path) -> QualityGateEvidence | None: """Load only a complete, current, hash-bound quality evidence envelope.""" evidence_path = root / "runtime/artifacts/quality/gate/latest.json" @@ -678,6 +715,7 @@ def _load_quality_gate(root: Path) -> QualityGateEvidence | None: def _read_docs_blob(root: Path) -> str: chunks = [ _read_text(root / "README.md"), + _read_text(root / "publication/README.md"), _read_text(root / "docs/governance/instruction_core_custom_instructions.md"), _read_text(root / "docs/providers/instruction_codex_provider.md"), ] diff --git a/src/ai4binance/governance/governance_enforcement_fabric.py b/src/ai4binance/governance/governance_enforcement_fabric.py index 2806907b..8d69d6fd 100644 --- a/src/ai4binance/governance/governance_enforcement_fabric.py +++ b/src/ai4binance/governance/governance_enforcement_fabric.py @@ -5,7 +5,7 @@ import hashlib from collections.abc import Iterable, Mapping from dataclasses import dataclass -from pathlib import Path +from pathlib import Path, PurePosixPath, PureWindowsPath import yaml @@ -582,7 +582,10 @@ def _quality_axis(name: str, value: object) -> GovernanceQualityAxis: def _quality_standard_mappings(value: object) -> dict[str, tuple[str, ...]]: payload = _mapping(value, "quality policy") - standard = _mapping(payload["standard_impact_tests"], "standard_impact_tests") + standard_payload = payload.get("standard_impact_tests") + if standard_payload is None: + raise ValueError("quality policy retention requires standard_impact_tests") + standard = _mapping(standard_payload, "standard_impact_tests") mappings = standard.get("mappings") if not isinstance(mappings, list): raise ValueError("standard_impact_tests.mappings must be a list") @@ -641,7 +644,12 @@ def _string(value: object, name: str) -> str: def _safe_path(value: object, name: str) -> str: path = _string(value, name) - if Path(path).is_absolute() or "\\" in path or ".." in Path(path).parts: + if ( + PurePosixPath(path).is_absolute() + or PureWindowsPath(path).is_absolute() + or "\\" in path + or ".." in PurePosixPath(path).parts + ): raise ValueError(f"{name} must be a repository-relative POSIX path") return path diff --git a/src/ai4binance/governance/repository_validator.py b/src/ai4binance/governance/repository_validator.py index 023d4262..b971b46c 100644 --- a/src/ai4binance/governance/repository_validator.py +++ b/src/ai4binance/governance/repository_validator.py @@ -529,11 +529,15 @@ class KnowledgeClassification(StrEnum): SUPPORT_TOP_LEVEL_PATHS: tuple[str, ...] = ( ".agents", ".codex", + ".gitleaksignore", ".github", ".pytest-tmp-open-web", ".venv", ".vscode", + "examples", "factory", + "LICENSE", + "publication", "requirements.txt", "research", ) @@ -637,6 +641,10 @@ def ai4binance_vnext(cls) -> RepositoryPolicy: *LEGACY_TOP_LEVEL_PATHS, ) owners = { + ".gitleaksignore": "Security", + "LICENSE": "Governance", + "examples": "Documentation", + "publication": "Governance", "src": "Engineering", "tests": "Quality", "docs": "Governance", @@ -663,7 +671,7 @@ def ai4binance_vnext(cls) -> RepositoryPolicy: } return cls( policy_id="AI4B-GOV-REPO-POLICY", - version="1.3.0", + version="1.3.1", allowed_top_level_paths=allowed, source_roots=("src/ai4binance",), test_roots=("tests",), diff --git a/src/ai4binance/governance/technology_language_policy.py b/src/ai4binance/governance/technology_language_policy.py index 21afa6a1..308b397f 100644 --- a/src/ai4binance/governance/technology_language_policy.py +++ b/src/ai4binance/governance/technology_language_policy.py @@ -5,7 +5,7 @@ import re from collections.abc import Iterable, Mapping from dataclasses import dataclass -from pathlib import Path +from pathlib import Path, PurePosixPath, PureWindowsPath from typing import cast import yaml @@ -359,9 +359,10 @@ def _strings(value: object, name: str) -> tuple[str, ...]: def _safe_paths(value: object, name: str) -> tuple[str, ...]: paths = _strings(value, name) if any( - Path(path).is_absolute() + PurePosixPath(path).is_absolute() + or PureWindowsPath(path).is_absolute() or "\\" in path - or ".." in Path(path.replace("/", "\\")).parts + or ".." in PurePosixPath(path).parts for path in paths ): raise ValueError(f"{name} must contain repository-relative POSIX paths") diff --git a/src/ai4binance/governance/terminology_policy.py b/src/ai4binance/governance/terminology_policy.py index bd12e001..30919d83 100644 --- a/src/ai4binance/governance/terminology_policy.py +++ b/src/ai4binance/governance/terminology_policy.py @@ -5,7 +5,7 @@ import re from collections.abc import Iterable, Mapping from dataclasses import dataclass -from pathlib import Path +from pathlib import Path, PurePosixPath, PureWindowsPath from typing import cast import yaml @@ -304,6 +304,11 @@ def _safe_paths(value: object, name: str) -> tuple[str, ...]: def _safe_path(value: object, name: str) -> str: path = _string(value, name) - if Path(path).is_absolute() or "\\" in path or ".." in Path(path).parts: + if ( + PurePosixPath(path).is_absolute() + or PureWindowsPath(path).is_absolute() + or "\\" in path + or ".." in PurePosixPath(path).parts + ): raise ValueError(f"{name} must be a repository-relative POSIX path") return path diff --git a/src/ai4binance/ops/architecture_migration.py b/src/ai4binance/ops/architecture_migration.py index 7d915009..1e9fe99c 100644 --- a/src/ai4binance/ops/architecture_migration.py +++ b/src/ai4binance/ops/architecture_migration.py @@ -279,6 +279,9 @@ def _root_file_migration_rule( "historical_replay_state.py": ( "domain/portfolio|domain/evidence|infrastructure/persistence" ), + "internal_radar.py": ( + "domain/evidence|application/pipelines|infrastructure/filesystem" + ), "opportunity_radar.py": "domain/intelligence|application/pipelines", "rag.py": "domain/intelligence|domain/evidence|integrations/llm", "rag_corrective.py": ( @@ -309,6 +312,7 @@ def _root_file_migration_rule( "infrastructure/persistence/historical_replay.py" ), "indicators.py": "domain/features/indicators.py", + "internal_radar_vision.py": "integrations/llm/internal_radar_vision.py", "market_context.py": "domain/snapshot/market_context.py", "markets.py": "domain/market/markets.py", "opportunities.py": "domain/setup/opportunities.py", diff --git a/src/ai4binance/ops/public_showcase.py b/src/ai4binance/ops/public_showcase.py index d2334b17..c7ead40c 100644 --- a/src/ai4binance/ops/public_showcase.py +++ b/src/ai4binance/ops/public_showcase.py @@ -4,15 +4,17 @@ import hashlib import shutil -import subprocess +import subprocess # Required for the repository-pinned scanner. # nosec B404 import tempfile from collections.abc import Callable, Mapping from dataclasses import dataclass from pathlib import Path -from typing import Any +from typing import Any, Final import yaml +_SCAN_PASSED_STATUS: Final = "PASSED" + class PublicShowcaseError(ValueError): """Raised when a public showcase cannot be safely staged.""" @@ -183,7 +185,7 @@ def run_gitleaks_scan(stage_root: Path, executable: Path) -> None: ) report_path = stage_root.parent / f"{stage_root.name}-gitleaks.json" try: - completed = subprocess.run( # noqa: S603 - executable is repository-pinned. + completed = subprocess.run( # noqa: S603 # Pinned executable. # nosec B603 [ str(executable), "dir", @@ -271,7 +273,7 @@ def stage_public_showcase( "status": "READY_FOR_HUMAN_APPROVAL", "publication_name": manifest.name, "selected_artifacts": selected, - "secret_scan": "PASSED", + "secret_scan": _SCAN_PASSED_STATUS, "remote_publication_allowed": False, "human_approval_required": True, "execution_allowed": False, diff --git a/tests/governance/architecture/test_technology_language_policy.py b/tests/governance/architecture/test_technology_language_policy.py index 95fdf7e5..737e9456 100644 --- a/tests/governance/architecture/test_technology_language_policy.py +++ b/tests/governance/architecture/test_technology_language_policy.py @@ -47,6 +47,8 @@ def test_policy_projection_validates_and_accepts_current_representative_paths() ".agents/skills/quality-gate-loop/scripts/invoke_gate.ps1", "config/governance/technology_language_ownership.yaml", "docs/architecture/diagrams/diagram_registry.yaml", + "publication/public_manifest.yaml", + "publication/sanitize_publication.py", "pyproject.toml", "schemas/governance/technology_language_ownership.schema.json", "scripts/quality.ps1", diff --git a/tests/governance/terminology/test_governance_enforcement_fabric.py b/tests/governance/terminology/test_governance_enforcement_fabric.py index 9ec194bd..8cf9107f 100644 --- a/tests/governance/terminology/test_governance_enforcement_fabric.py +++ b/tests/governance/terminology/test_governance_enforcement_fabric.py @@ -337,7 +337,8 @@ def test_authority_graph_closes_all_fabric_family_paths() -> None: def test_fabric_contract_helpers_reject_untrusted_shapes(tmp_path: Path) -> None: - for value in (None, [], "text"): + invalid_mappings: tuple[object, ...] = (None, [], "text") + for value in invalid_mappings: with pytest.raises(ValueError, match="mapping"): fabric_module._mapping(value, "value") for value in (None, "", " "): @@ -464,7 +465,7 @@ def test_family_integrity_reports_missing_standard_projection_and_schema( def test_quality_mapping_parser_rejects_malformed_and_duplicate_entries() -> None: - for payload in ( + malformed_payloads: tuple[dict[str, object], ...] = ( {"standard_impact_tests": {}}, {"standard_impact_tests": {"mappings": [{"name": "x", "tests": "bad"}]}}, { @@ -475,7 +476,8 @@ def test_quality_mapping_parser_rejects_malformed_and_duplicate_entries() -> Non ] } }, - ): + ) + for payload in malformed_payloads: with pytest.raises( ValueError, match=( diff --git a/tests/governance/terminology/test_terminology_policy.py b/tests/governance/terminology/test_terminology_policy.py index 78a90ffd..42bb5e10 100644 --- a/tests/governance/terminology/test_terminology_policy.py +++ b/tests/governance/terminology/test_terminology_policy.py @@ -127,8 +127,9 @@ def test_terminology_helpers_and_policy_contract_fail_closed(tmp_path: Path) -> terminology_module._string(" ", "value") with pytest.raises(ValueError, match="string array"): terminology_module._strings([""], "value") - with pytest.raises(ValueError, match="repository-relative"): - terminology_module._safe_path("../outside", "value") + for unsafe_path in ("../outside", "/absolute", "C:/absolute", "folder\\file"): + with pytest.raises(ValueError, match="repository-relative"): + terminology_module._safe_path(unsafe_path, "value") def test_terminology_scan_covers_deprecated_missing_and_projection_drift( diff --git a/tests/test_architecture_diagram_validation.py b/tests/test_architecture_diagram_validation.py index 8d5e9970..dad253ce 100644 --- a/tests/test_architecture_diagram_validation.py +++ b/tests/test_architecture_diagram_validation.py @@ -2,6 +2,7 @@ import re from pathlib import Path +from typing import cast import pytest import yaml @@ -215,11 +216,11 @@ def test_validator_reports_all_registry_document_and_relation_failures( _write_document(tmp_path, path, "D999") entries = [ _entry(diagram_id="wrong"), - _entry(diagram_id="D001", path=123), + _entry(diagram_id="D001", path=cast(str, 123)), _entry(diagram_id="D002", path="../escape"), _entry(diagram_id="D003", path=path), _entry(diagram_id="D004", path=path), - _entry(diagram_id="D005", related_diagrams=[1]), + _entry(diagram_id="D005", related_diagrams=[cast(str, 1)]), ] _write_registry(tmp_path, entries) codes = _finding_codes(tmp_path) diff --git a/tests/test_docs_hygiene.py b/tests/test_docs_hygiene.py index 25c0bf1e..539f68b2 100644 --- a/tests/test_docs_hygiene.py +++ b/tests/test_docs_hygiene.py @@ -213,7 +213,7 @@ def test_docs_and_reports_have_eli10_explanations() -> None: def test_canonical_docs_markdown_filenames_and_metadata_are_classified() -> None: pattern = re.compile( - r"^(?:[a-z]+(?:_[a-z]+)?)_[a-z0-9]+(?:_[a-z0-9]+)*_[a-z0-9]+(?:_[a-z0-9]+)*\.md$" + r"^(?:[a-z0-9]+(?:_[a-z0-9]+)?)_[a-z0-9]+(?:_[a-z0-9]+)*_[a-z0-9]+(?:_[a-z0-9]+)*\.md$" ) docs_index_exceptions = {"docs/README.md"} required_frontmatter_keys = { @@ -241,9 +241,10 @@ def test_canonical_docs_markdown_filenames_and_metadata_are_classified() -> None key, value = line.split(": ", 1) metadata[key.strip()] = value.strip() missing_keys = sorted(required_frontmatter_keys - metadata.keys()) - if not pattern.match(path.name) or not has_frontmatter or missing_keys: + filename_is_valid = path.name == "README.md" or pattern.match(path.name) + if not filename_is_valid or not has_frontmatter or missing_keys: detail = [] - if not pattern.match(path.name): + if not filename_is_valid: detail.append("filename") if not has_frontmatter: detail.append("frontmatter") @@ -735,9 +736,9 @@ def test_instruction_context_router_avoids_default_corpus_loading() -> None: normalized_root = re.sub(r"\s+", " ", root_text) assert ( - "Base context is this file plus the active adapter. Add nearest scoped " - "`AGENTS.md` only for paths within its scope." - ) in normalized_root + "Resolve root from this file, active adapter, and nearest scoped `AGENTS.md`." + in normalized_root + ) assert "Route them; do not load by default." in root_text for route in ( "| Ordinary source | Affected code/config/callers/contracts/tests |", diff --git a/tests/test_governance_constitution_sync.py b/tests/test_governance_constitution_sync.py index e733dd46..1d9f0091 100644 --- a/tests/test_governance_constitution_sync.py +++ b/tests/test_governance_constitution_sync.py @@ -473,6 +473,110 @@ def test_governance_alignment_surfaces_loose_governance_code( assert "LIVE_ORDER_BLOCKED" in report.blockers +def test_governance_alignment_accepts_transitive_compliance_trace( + tmp_path: Path, +) -> None: + source_path = "src/ai4binance/governance/example_policy.py" + source = tmp_path / source_path + source.parent.mkdir(parents=True) + source.write_text("class ExamplePolicy: ...\n", encoding="utf-8") + tests = tmp_path / "tests" + tests.mkdir() + (tests / "test_example_policy.py").write_text( + "from ai4binance.governance.example_policy import ExamplePolicy\n", + encoding="utf-8", + ) + write_core_documents( + tmp_path, + compliance_extra="docs/standards/example_policy_standard.md", + ) + standard = tmp_path / "docs" / "standards" / "example_policy_standard.md" + standard.parent.mkdir(parents=True, exist_ok=True) + standard.write_text( + "# Example Policy Standard\n\n" + "## ELI10\n\n" + "See `docs/references/example_policy_reference.md`.\n", + encoding="utf-8", + ) + reference = tmp_path / "docs" / "references" / "example_policy_reference.md" + reference.parent.mkdir(parents=True) + reference.write_text( + f"# Example Policy Reference\n\n## ELI10\n\n`{source_path}`\n", + encoding="utf-8", + ) + write_quality_evidence(tmp_path) + + report = audit_governance_alignment( + tmp_path, + changed_paths=(source_path, "tests/test_example_policy.py"), + ) + + assert report.status is GovernanceAlignmentStatus.PASS + assert report.findings == () + + +def test_governance_alignment_rejects_unlinked_document_as_compliance_trace( + tmp_path: Path, +) -> None: + source_path = "src/ai4binance/governance/unlinked_policy.py" + source = tmp_path / source_path + source.parent.mkdir(parents=True) + source.write_text("class UnlinkedPolicy: ...\n", encoding="utf-8") + tests = tmp_path / "tests" + tests.mkdir() + (tests / "test_unlinked_policy.py").write_text( + "from ai4binance.governance.unlinked_policy import UnlinkedPolicy\n", + encoding="utf-8", + ) + write_core_documents(tmp_path) + unlinked = tmp_path / "docs" / "references" / "unlinked_policy.md" + unlinked.parent.mkdir(parents=True) + unlinked.write_text( + f"# Unlinked Policy\n\n## ELI10\n\n`{source_path}`\n", + encoding="utf-8", + ) + write_quality_evidence(tmp_path) + + report = audit_governance_alignment( + tmp_path, + changed_paths=(source_path, "tests/test_unlinked_policy.py"), + ) + finding_kinds = {finding.kind for finding in report.findings} + + assert LooseCodeGapKind.GOVERNANCE_CODE_WITHOUT_COMPLIANCE in finding_kinds + assert LooseCodeGapKind.SOURCE_WITHOUT_WRITTEN_RULE not in finding_kinds + + +def test_governance_alignment_accepts_publication_boundary_as_written_rule( + tmp_path: Path, +) -> None: + source_path = "src/ai4binance/ops/public_showcase.py" + source = tmp_path / source_path + source.parent.mkdir(parents=True) + source.write_text("class PublicShowcase: ...\n", encoding="utf-8") + tests = tmp_path / "tests" + tests.mkdir() + (tests / "test_public_showcase.py").write_text( + "from ai4binance.ops.public_showcase import PublicShowcase\n", + encoding="utf-8", + ) + write_core_documents(tmp_path) + publication = tmp_path / "publication" / "README.md" + publication.parent.mkdir() + publication.write_text( + f"# Publication Boundary\n\n`{source_path}`\n", encoding="utf-8" + ) + write_quality_evidence(tmp_path) + + report = audit_governance_alignment( + tmp_path, + changed_paths=(source_path, "tests/test_public_showcase.py"), + ) + finding_kinds = {finding.kind for finding in report.findings} + + assert LooseCodeGapKind.SOURCE_WITHOUT_WRITTEN_RULE not in finding_kinds + + def test_governance_alignment_surfaces_constitution_family_mismatch( tmp_path: Path, ) -> None: diff --git a/tests/test_internal_radar.py b/tests/test_internal_radar.py index d2cd9d3c..8695af69 100644 --- a/tests/test_internal_radar.py +++ b/tests/test_internal_radar.py @@ -5,6 +5,7 @@ import json from datetime import UTC, datetime from pathlib import Path +from typing import cast from ai4binance.governance.model_registry import build_advisory_inference_envelope from ai4binance.internal_radar import run_internal_radar_once @@ -32,7 +33,8 @@ def test_internal_radar_persists_redacted_new_image_candidate(tmp_path: Path) -> assert "Last scan timestamp (UTC): `2026-09-18T00:00:00+00:00`" in markdown assert "[private-name.jpg](file://" in markdown assert "NOT_ASSESSED_WITHOUT_CONFIGURED_VISION_ANALYZER" in markdown - candidate = payload["candidates"][0] + candidates = cast(list[object], payload["candidates"]) + candidate = candidates[0] assert isinstance(candidate, dict) assert "private-name" not in str(candidate) privacy = payload["privacy"] @@ -41,7 +43,7 @@ def test_internal_radar_persists_redacted_new_image_candidate(tmp_path: Path) -> assert privacy["markdown_local_file_links_included"] is True assert payload["last_scan_timestamp_utc"] == "2026-09-18T00:00:00+00:00" assert "relative_path" not in candidate - assert payload["privacy"]["source_images_copied"] is False + assert privacy["source_images_copied"] is False assert payload["execution_allowed"] is False @@ -84,12 +86,15 @@ def analyze(self, **kwargs: object) -> VisionEvidence: vision_runner=BlockingVisionRunner(), # type: ignore[arg-type] ) - candidate = result.to_payload()["candidates"][0] + payload = result.to_payload() + candidates = cast(list[object], payload["candidates"]) + candidate = candidates[0] assert isinstance(candidate, dict) assert candidate["assessment_status"] == "BLOCKED" assert "private-name" not in str(candidate) assert "UNREGISTERED_MODEL:local-llamacpp-qwen25vl-3b" in result.blockers - assert result.to_payload()["privacy"]["source_images_copied"] is False + privacy = cast(dict[str, object], payload["privacy"]) + assert privacy["source_images_copied"] is False def test_internal_radar_vision_summary_only_counts_validated_observations( @@ -131,14 +136,16 @@ def analyze(self, **kwargs: object) -> VisionEvidence: vision_runner=ObservedVisionRunner(), # type: ignore[arg-type] ) - summary = result.to_payload()["vision_summary"] + payload = result.to_payload() + summary = payload["vision_summary"] assert isinstance(summary, dict) assert summary["observed_count"] == 1 assert summary["benefit_categories"] == {"OPERATIONAL_VISIBILITY": 1} assert summary["tradeoff_categories"] == {"HUMAN_REVIEW_REQUIRED": 1} - progress = result.to_payload()["vision_progress"] + progress = payload["vision_progress"] assert progress == {"analysed": 1, "awaiting_analysis": 0} - candidate = result.to_payload()["candidates"][0] + candidates = cast(list[object], payload["candidates"]) + candidate = candidates[0] assert isinstance(candidate, dict) assert str(candidate["last_scan_timestamp_utc"]).endswith("+00:00") markdown = result.latest_path.with_suffix(".md").read_text(encoding="utf-8") diff --git a/tests/test_internal_radar_vision.py b/tests/test_internal_radar_vision.py index 47725171..92627751 100644 --- a/tests/test_internal_radar_vision.py +++ b/tests/test_internal_radar_vision.py @@ -4,6 +4,7 @@ import json from pathlib import Path +from typing import cast from urllib.request import Request import pytest @@ -96,7 +97,9 @@ def fake_urlopen(request: Request, timeout: float) -> Response: return Response() monkeypatch.setattr("urllib.request.urlopen", fake_urlopen) - evidence = LlamaCppVisionRunner(model_gateway=AllowedGateway()).analyze( + evidence = LlamaCppVisionRunner( + model_gateway=cast(ModelGateway, AllowedGateway()) + ).analyze( candidate_id="internal-image:0123456789abcdef", source_content_sha256="b" * 64, image_path=image, @@ -108,7 +111,8 @@ def fake_urlopen(request: Request, timeout: float) -> Response: assert payload["image_category"] == "DASHBOARD" assert payload["extracted_text_present"] is True assert "private-image" not in json.dumps(payload) - assert payload["privacy"]["raw_ocr_text_persisted"] is False + privacy = cast(dict[str, object], payload["privacy"]) + assert privacy["raw_ocr_text_persisted"] is False assert payload["execution_allowed"] is False @@ -133,7 +137,9 @@ def __exit__(self, *_args: object) -> None: return None monkeypatch.setattr("urllib.request.urlopen", lambda *_args, **_kwargs: Response()) - evidence = LlamaCppVisionRunner(model_gateway=AllowedGateway()).analyze( + evidence = LlamaCppVisionRunner( + model_gateway=cast(ModelGateway, AllowedGateway()) + ).analyze( candidate_id="internal-image:0123456789abcdef", source_content_sha256="c" * 64, image_path=image, @@ -178,7 +184,9 @@ def __exit__(self, *_args: object) -> None: return None monkeypatch.setattr("urllib.request.urlopen", lambda *_args, **_kwargs: Response()) - evidence = LlamaCppVisionRunner(model_gateway=AllowedGateway()).analyze( + evidence = LlamaCppVisionRunner( + model_gateway=cast(ModelGateway, AllowedGateway()) + ).analyze( candidate_id="internal-image:0123456789abcdef", source_content_sha256="d" * 64, image_path=image, diff --git a/tests/test_kaizen_quality.py b/tests/test_kaizen_quality.py index 60394d95..860b68a6 100644 --- a/tests/test_kaizen_quality.py +++ b/tests/test_kaizen_quality.py @@ -792,6 +792,18 @@ def test_architecture_migration_ledger_classifies_every_repository_module() -> N assert historical_persistence["target_paths"] == [ "src/ai4binance/infrastructure/persistence/historical_replay.py" ] + internal_radar = by_path["src/ai4binance/internal_radar.py"] + assert internal_radar["classification"] == "SPLIT" + assert set(cast(list[str], internal_radar["target_paths"])) == { + "src/ai4binance/domain/evidence", + "src/ai4binance/application/pipelines", + "src/ai4binance/infrastructure/filesystem", + } + internal_radar_vision = by_path["src/ai4binance/internal_radar_vision.py"] + assert internal_radar_vision["classification"] == "MOVE" + assert internal_radar_vision["target_paths"] == [ + "src/ai4binance/integrations/llm/internal_radar_vision.py" + ] virtual_attribution = by_path[ "src/ai4binance/research/virtual_runtime_attribution.py" ] diff --git a/tests/test_live_readiness_preview.py b/tests/test_live_readiness_preview.py index 1f774c4d..df1b9b31 100644 --- a/tests/test_live_readiness_preview.py +++ b/tests/test_live_readiness_preview.py @@ -18,6 +18,7 @@ from ai4binance.cli import live as cli_live from ai4binance.config import Settings from ai4binance.domain import LiveGateInput, ValidationStatus +from ai4binance.exchange import BinancePrivateAccountReader, PrivateCredentials from ai4binance.exchange.models import BookTicker, MarketKline, SymbolInfo from ai4binance.execution import ( ExecutionAuthorizationEnvelope, @@ -1364,12 +1365,12 @@ def test_private_reader_uses_read_only_credentials_and_transport( ) -> None: credentials = object() monkeypatch.setattr( - cli_live.PrivateCredentials, + PrivateCredentials, "from_environment_or_file", lambda _path: credentials, ) reader = cli_live._private_reader(Settings()) - assert isinstance(reader, cli_live.BinancePrivateAccountReader) + assert isinstance(reader, BinancePrivateAccountReader) def test_live_place_blocks_invalid_or_mismatched_authorization_evidence( diff --git a/tests/test_lowest_coverage_technology_language_policy.py b/tests/test_lowest_coverage_technology_language_policy.py index b62f9f17..ca3e1550 100644 --- a/tests/test_lowest_coverage_technology_language_policy.py +++ b/tests/test_lowest_coverage_technology_language_policy.py @@ -127,6 +127,8 @@ def test_policy_reports_unreadable_sources_and_evidence_requirements( (policy_module._boolean, "false", "must be boolean"), (policy_module._strings, ("not", "a", "list"), "must be a string array"), (policy_module._safe_paths, ["../unsafe"], "repository-relative POSIX paths"), + (policy_module._safe_paths, ["/absolute"], "repository-relative POSIX paths"), + (policy_module._safe_paths, ["C:/absolute"], "repository-relative POSIX paths"), ], ) def test_policy_value_validators_reject_malformed_input( diff --git a/tests/test_lowest_twenty_coverage_models.py b/tests/test_lowest_twenty_coverage_models.py index 017b0e19..1b46092a 100644 --- a/tests/test_lowest_twenty_coverage_models.py +++ b/tests/test_lowest_twenty_coverage_models.py @@ -6,6 +6,7 @@ from __future__ import annotations +import tomllib from collections.abc import Callable from dataclasses import replace from datetime import UTC, datetime @@ -90,7 +91,7 @@ def test_runtime_environment_rejects_unsafe_temp_contracts( ) -> None: directory = temporary_directory[0] monkeypatch.setattr( - ai4binance.tomllib, + tomllib, "load", lambda _stream: { "tool": {"ai4binance": {"runtime": {"temporary_directory": directory}}} @@ -158,8 +159,8 @@ def test_shadow_diff_cannot_widen_execution_authority() -> None: with pytest.raises(ValueError, match="cannot authorize"): DgeShadowDecisionDiff( shadow_rule_id="shadow-1", - baseline_decision=blocked, # type: ignore[arg-type] - shadow_decision=blocked, # type: ignore[arg-type] + baseline_decision=blocked, + shadow_decision=blocked, would_change_decision=False, changed_fields=(), execution_allowed=True, @@ -167,8 +168,8 @@ def test_shadow_diff_cannot_widen_execution_authority() -> None: with pytest.raises(ValueError, match="shadow rule id"): DgeShadowDecisionDiff( shadow_rule_id=" ", - baseline_decision=blocked, # type: ignore[arg-type] - shadow_decision=blocked, # type: ignore[arg-type] + baseline_decision=blocked, + shadow_decision=blocked, would_change_decision=False, changed_fields=(), ) diff --git a/tests/test_maintainability_ratchet.py b/tests/test_maintainability_ratchet.py index bb4296ee..0e5dd9cc 100644 --- a/tests/test_maintainability_ratchet.py +++ b/tests/test_maintainability_ratchet.py @@ -2,6 +2,7 @@ from __future__ import annotations +import subprocess from pathlib import Path import pytest @@ -83,12 +84,12 @@ def test_collect_findings_and_evaluate_fail_closed_for_bad_tool_data( completed = type( "Completed", (), {"returncode": 2, "stderr": "tool failed", "stdout": ""} )() - monkeypatch.setattr(ratchet.subprocess, "run", lambda *_args, **_kwargs: completed) + monkeypatch.setattr(subprocess, "run", lambda *_args, **_kwargs: completed) with pytest.raises(RuntimeError, match="tool failed"): collect_ruff_findings(ROOT) malformed = type("Completed", (), {"returncode": 0, "stderr": "", "stdout": "{}"})() - monkeypatch.setattr(ratchet.subprocess, "run", lambda *_args, **_kwargs: malformed) + monkeypatch.setattr(subprocess, "run", lambda *_args, **_kwargs: malformed) with pytest.raises(ValueError, match="JSON array"): collect_ruff_findings(ROOT) diff --git a/tests/test_market_data_gateway.py b/tests/test_market_data_gateway.py index 972d953c..64b74847 100644 --- a/tests/test_market_data_gateway.py +++ b/tests/test_market_data_gateway.py @@ -4,11 +4,13 @@ import asyncio import json +import time from dataclasses import replace from datetime import UTC, datetime, timedelta from decimal import Decimal from pathlib import Path from types import SimpleNamespace +from typing import cast import pytest @@ -33,7 +35,7 @@ build_gateway, ) from ai4binance.exchange import rate_limit as rate_limit_module -from ai4binance.exchange.public_stream import SpotKlineUpdate +from ai4binance.exchange.public_stream import BinanceSpotKlineParser, SpotKlineUpdate from ai4binance.exchange.rate_limit import ( WeightedRateLimitGovernor, public_request_weight, @@ -244,7 +246,7 @@ def test_gateway_helpers_cover_invalid_and_bounded_inputs( path = tmp_path / "state.json" path.write_text('{"blockers": "invalid"}', encoding="utf-8") heartbeat = _GatewayStateHeartbeat(path, interval_seconds=0) - monkeypatch.setattr(gateway_cli.time, "monotonic", lambda: 1.0) + monkeypatch.setattr(time, "monotonic", lambda: 1.0) heartbeat(START) payload = json.loads(path.read_text(encoding="utf-8")) assert payload["blockers"] == ["MARKET_GATEWAY_STATE_BLOCKERS_INVALID"] @@ -262,7 +264,7 @@ def test_gateway_heartbeat_handles_read_failures_and_throttles_writes( path.write_text("not-json", encoding="utf-8") heartbeat = _GatewayStateHeartbeat(path, interval_seconds=10) monotonic = iter((10.0, 11.0)) - monkeypatch.setattr(gateway_cli.time, "monotonic", lambda: next(monotonic)) + monkeypatch.setattr(time, "monotonic", lambda: next(monotonic)) heartbeat(START) first = path.read_text(encoding="utf-8") heartbeat(START) @@ -631,7 +633,7 @@ def sync_cycle(self, *, observed_at: datetime) -> dict[str, object]: lambda _settings: synchronizer, ) monkeypatch.setattr( - gateway_cli.time, + time, "sleep", lambda _seconds: None, ) @@ -805,9 +807,9 @@ def flush(self, _observed_at: datetime) -> None: cache = _Cache() processor = CanonicalMarketStreamProcessor( "SPOT", - _Writer(), - cache, - monotonic=lambda: 1.0, # type: ignore[arg-type] + cast(DirectTimeframeWriter, _Writer()), + cast(SharedMarketCache, cache), + monotonic=lambda: 1.0, ) assert processor.process('{"e":"24hrTicker"}') == "CACHE_UPDATED" assert cache.flushed == 1 @@ -840,10 +842,10 @@ def process(self, _message: object) -> None: spot_processor = _Processor() futures_processor = _Processor() gateway = BinanceMarketDataGateway( - _Connection(), - _Connection(), - spot_processor, # type: ignore[arg-type] - futures_processor, # type: ignore[arg-type] + cast(CombinedStreamConnectionManager, _Connection()), + cast(CombinedStreamConnectionManager, _Connection()), + cast(CanonicalMarketStreamProcessor, spot_processor), + cast(CanonicalMarketStreamProcessor, futures_processor), ) assert asyncio.run(gateway.run_once()) == ("PLANNED_ROLLOVER", "PLANNED_ROLLOVER") assert spot_processor.cache.flushes == futures_processor.cache.flushes == 1 @@ -900,11 +902,17 @@ def test_gateway_rejects_invalid_inputs_and_processes_closed_kline( writes: list[tuple[str, str]] = [] processor = CanonicalMarketStreamProcessor( "SPOT", - SimpleNamespace( - append=lambda symbol, timeframe, *_: writes.append((symbol, timeframe)) + cast( + DirectTimeframeWriter, + SimpleNamespace( + append=lambda symbol, timeframe, *_: writes.append((symbol, timeframe)) + ), ), cache, - parser=SimpleNamespace(parse=lambda _: update), + parser=cast( + BinanceSpotKlineParser, + SimpleNamespace(parse=lambda _: update), + ), activity_observer=lambda _: None, ) assert processor.process('{"e":"kline"}') == "CLOSED_5M_APPLIED" diff --git a/tests/test_market_history_continuous.py b/tests/test_market_history_continuous.py index e25a1354..2e994326 100644 --- a/tests/test_market_history_continuous.py +++ b/tests/test_market_history_continuous.py @@ -1084,7 +1084,8 @@ def test_refresh_request_age_uses_current_clock_after_cycle_setup( status = market_history_refresh_status(request_path) assert status["state"] == "DATA_READY", status - assert "MARKET_HISTORY_REFRESH_REQUEST_EXPIRED" not in status["blockers"] + blockers = cast(tuple[object, ...], status["blockers"]) + assert "MARKET_HISTORY_REFRESH_REQUEST_EXPIRED" not in blockers def test_refresh_request_completion_never_predates_its_request( diff --git a/tests/test_model_registry_coverage_closure.py b/tests/test_model_registry_coverage_closure.py index a5cf8c88..34ae2f7c 100644 --- a/tests/test_model_registry_coverage_closure.py +++ b/tests/test_model_registry_coverage_closure.py @@ -436,7 +436,7 @@ def test_registry_validation_covers_artifact_failure_modes(tmp_path: Path) -> No assert passed.execution_allowed is False assert passed.promotion_status == "RESEARCH_ONLY" assert passed.live_eligibility_status == "LIVE_ORDER_BLOCKED" - assert unverifiable.blockers == ("ARTIFACT_HASH_UNVERIFIABLE:coverage-model",) + assert unverifiable.blockers == ("LOCAL_MODEL_MANIFEST_INVALID:coverage-model",) assert outside.blockers == ("ARTIFACT_URI_OUTSIDE_REPOSITORY:coverage-model",) assert missing.blockers == ("ARTIFACT_MISSING:coverage-model",) diff --git a/tests/test_opportunity_monitor.py b/tests/test_opportunity_monitor.py index d83b88b7..64f8a43d 100644 --- a/tests/test_opportunity_monitor.py +++ b/tests/test_opportunity_monitor.py @@ -101,7 +101,7 @@ def test_monitor_helper_boundaries_and_research_estimates( with pytest.raises(ValueError, match="read boundary"): read_monitor(tmp_path, "SPOT", "BTCUSDT") - candidate = { + candidate: dict[str, object] = { "market": "SPOT", "direction": "BULLISH", "entry": "101", diff --git a/tests/test_public_showcase.py b/tests/test_public_showcase.py index 2a987e3e..1f76ba7f 100644 --- a/tests/test_public_showcase.py +++ b/tests/test_public_showcase.py @@ -3,6 +3,7 @@ from __future__ import annotations import hashlib +import subprocess from pathlib import Path import pytest @@ -188,11 +189,11 @@ def test_showcase_scan_and_output_preconditions_fail_closed( executable = tmp_path / "gitleaks.exe" executable.write_text("fixture", encoding="utf-8") failed = type("Completed", (), {"returncode": 1})() - monkeypatch.setattr(showcase.subprocess, "run", lambda *_args, **_kwargs: failed) + monkeypatch.setattr(subprocess, "run", lambda *_args, **_kwargs: failed) with pytest.raises(PublicShowcaseError, match="failed"): showcase.run_gitleaks_scan(tmp_path, executable) monkeypatch.setattr( - showcase.subprocess, + subprocess, "run", lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("unavailable")), ) @@ -234,7 +235,12 @@ def test_showcase_rejects_each_authority_expansion_shape(tmp_path: Path) -> None "sha256": "a" * 64, } assert showcase._parse_artifacts([valid])[0].source == "README.md" - for artifacts in ([{}], [{**valid, "sha256": "A" * 64}], [valid, valid]): + invalid_artifact_sets: tuple[list[dict[str, str]], ...] = ( + [{}], + [{**valid, "sha256": "A" * 64}], + [valid, valid], + ) + for artifacts in invalid_artifact_sets: with pytest.raises(PublicShowcaseError): showcase._parse_artifacts(artifacts) with pytest.raises(PublicShowcaseError, match="non-empty"): diff --git a/tests/test_repository_cleanup_audit.py b/tests/test_repository_cleanup_audit.py index b5aa880b..037d3f59 100644 --- a/tests/test_repository_cleanup_audit.py +++ b/tests/test_repository_cleanup_audit.py @@ -201,7 +201,9 @@ def test_repository_cleanup_audit_text_summarizes_structure_classifications() -> command="repository-cleanup-audit", ) - assert "static_file_decisions: ARCHIVE_CANDIDATE=8, ENTRY_POINT=12, KEEP=25" in text + assert ( + "static_file_decisions: ARCHIVE_CANDIDATE=13, ENTRY_POINT=11, KEEP=25" in text + ) report = payload["report"] assert isinstance(report, dict) static_classifications = report["static_unimported_classifications"] diff --git a/tests/test_repository_validator.py b/tests/test_repository_validator.py index cf8806da..87d7eb11 100644 --- a/tests/test_repository_validator.py +++ b/tests/test_repository_validator.py @@ -4276,6 +4276,28 @@ def test_repository_validator_allows_runtime_top_level_without_warning( assert report.live_eligibility_status == "LIVE_ORDER_BLOCKED" +def test_repository_policy_registers_publication_and_security_surfaces( + tmp_path: Path, +) -> None: + _write_required_knowledge_docs(tmp_path) + (tmp_path / ".gitleaksignore").write_text("", encoding="utf-8") + (tmp_path / "LICENSE").write_text("test license\n", encoding="utf-8") + (tmp_path / "examples").mkdir() + (tmp_path / "publication").mkdir() + + policy = RepositoryPolicy.ai4binance_vnext() + report = validate_repository(tmp_path, policy=policy) + registered = {".gitleaksignore", "LICENSE", "examples", "publication"} + + assert policy.version == "1.3.1" + assert registered <= set(policy.allowed_top_level_paths) + assert not any( + finding.kind is RepositoryFindingKind.UNKNOWN_TOP_LEVEL_PATH + and finding.path in registered + for finding in report.findings + ) + + def test_repository_validator_ignores_temporary_top_level_directories( tmp_path: Path, ) -> None: @@ -5772,6 +5794,65 @@ def test_governed_document_lock_approval_script_exports_verified_chain( assert evidence["current_alignment_status"] == "VERIFIED" assert evidence["approved_documents"][0]["hash_matches_current_content"] is True + manifest_path = tmp_path / GOVERNED_DOCUMENT_LOCK_MANIFEST_PATH + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + evidence_bytes = output_path.read_bytes() + evidence_sha256 = _sha256(output_path) + manifest["approval_records"][0]["approval_evidence_path"] = output_path.relative_to( + tmp_path + ).as_posix() + manifest["approval_records"][0]["approval_evidence_sha256"] = evidence_sha256 + manifest_path.write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + argv_before = sys.argv[:] + try: + sys.argv = [ + "export_governed_document_lock_approval.py", + "--repository-root", + str(tmp_path), + "--approval-id", + "AI4B-GOV-DOCLOCK-TEST-001", + "--output-path", + str(output_path), + ] + with pytest.raises(SystemExit) as excinfo: + runpy.run_path( + str(ROOT / "scripts" / "export_governed_document_lock_approval.py"), + run_name="__main__", + ) + assert excinfo.value.code == 0 + finally: + sys.argv = argv_before + assert output_path.read_bytes() == evidence_bytes + + output_path.write_text("{}\n", encoding="utf-8") + tampered_bytes = output_path.read_bytes() + argv_before = sys.argv[:] + try: + sys.argv = [ + "export_governed_document_lock_approval.py", + "--repository-root", + str(tmp_path), + "--approval-id", + "AI4B-GOV-DOCLOCK-TEST-001", + "--output-path", + str(output_path), + ] + with pytest.raises( + ValueError, + match="BOUND_APPROVAL_EVIDENCE_IMMUTABLE_MISMATCH", + ): + runpy.run_path( + str(ROOT / "scripts" / "export_governed_document_lock_approval.py"), + run_name="__main__", + ) + finally: + sys.argv = argv_before + assert output_path.read_bytes() == tampered_bytes + def test_governed_document_lock_approval_script_exports_historical_chain( tmp_path: Path, @@ -6023,6 +6104,32 @@ def test_governed_document_lock_sync_script_normalizes_legacy_written_owner_orph assert len(approval_record["approval_evidence_sha256"]) == 64 assert written_owner_approval == approval_record + evidence_path = tmp_path / approval_record["approval_evidence_path"] + evidence_bytes = evidence_path.read_bytes() + argv_before = sys.argv[:] + sys_path_before = sys.path[:] + try: + sys.path.insert(0, str(ROOT / "scripts")) + sys.argv = [ + "sync_governed_document_lock_approval_evidence.py", + "--repository-root", + str(tmp_path), + ] + with pytest.raises(SystemExit) as excinfo: + runpy.run_path( + str( + ROOT + / "scripts" + / "sync_governed_document_lock_approval_evidence.py" + ), + run_name="__main__", + ) + assert excinfo.value.code == 0 + finally: + sys.argv = argv_before + sys.path[:] = sys_path_before + assert evidence_path.read_bytes() == evidence_bytes + def test_repository_validator_allows_reserved_source_of_truth_filename( tmp_path: Path, diff --git a/tests/test_runtime_artifacts_migration.py b/tests/test_runtime_artifacts_migration.py index 68d83a0f..5a7e9c17 100644 --- a/tests/test_runtime_artifacts_migration.py +++ b/tests/test_runtime_artifacts_migration.py @@ -145,7 +145,7 @@ def test_layout_mapping_helpers_cover_all_invalid_contract_shapes() -> None: for value in text_mapping_values: with pytest.raises( ValueError, - match=r"roots must be (a non-empty object|contain non-empty strings)", + match=r"roots must (?:be a non-empty object|contain non-empty strings)", ): layout._text_mapping(value, "roots") with pytest.raises(ValueError, match="canonical_root must be a non-empty string"): diff --git a/tests/test_storage.py b/tests/test_storage.py index 54bd69ae..c616748e 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -1,6 +1,8 @@ """Append-only audit storage and secret redaction tests.""" import json +import os +import time from datetime import UTC, date, datetime from decimal import Decimal from enum import Enum @@ -9,7 +11,6 @@ import pytest -import ai4binance.storage.destination_verification as destination_verification from ai4binance.infrastructure.persistence import safe_json from ai4binance.storage import ( AuditEvent, @@ -460,7 +461,7 @@ def test_write_json_object_verified_retries_transient_replace_denial( monkeypatch: pytest.MonkeyPatch, ) -> None: path = tmp_path / "state" / "latest.json" - original_replace = destination_verification.os.replace + original_replace = os.replace attempts = 0 delays: list[float] = [] @@ -471,8 +472,8 @@ def flaky_replace(source: Path, destination: Path) -> None: raise PermissionError(5, "transient sharing violation") original_replace(source, destination) - monkeypatch.setattr(destination_verification.os, "replace", flaky_replace) - monkeypatch.setattr(destination_verification.time, "sleep", delays.append) + monkeypatch.setattr(os, "replace", flaky_replace) + monkeypatch.setattr(time, "sleep", delays.append) result = write_json_object_verified( path, @@ -497,8 +498,8 @@ def denied_replace(_source: Path, _destination: Path) -> None: attempts += 1 raise PermissionError(5, "persistent sharing violation") - monkeypatch.setattr(destination_verification.os, "replace", denied_replace) - monkeypatch.setattr(destination_verification.time, "sleep", lambda _delay: None) + monkeypatch.setattr(os, "replace", denied_replace) + monkeypatch.setattr(time, "sleep", lambda _delay: None) with pytest.raises(PermissionError, match="persistent sharing violation"): write_json_object_verified( From b4919945fed646075aafdbed57ec7f0015578c11 Mon Sep 17 00:00:00 2001 From: Huseyin Cicek Date: Fri, 25 Sep 2026 15:33:36 +0300 Subject: [PATCH 10/19] feat: add by HsC --- publication/sanitize_publication.py | 1 - scripts/install_startup_task.ps1 | 65 ++++- .../data/market_history_continuous.py | 82 ++++-- .../governance_enforcement_fabric.py | 7 +- src/ai4binance/governance/model_registry.py | 47 ++-- src/ai4binance/internal_radar.py | 259 ++++++++++++------ .../test_governance_enforcement_fabric.py | 13 +- .../terminology/test_terminology_policy.py | 4 +- tests/test_architecture_diagram_validation.py | 4 +- tests/test_artifact_hygiene_scripts.py | 6 +- tests/test_lowest_twenty_coverage_models.py | 9 +- tests/test_maintainability_ratchet.py | 39 ++- tests/test_market_data_gateway.py | 2 +- .../test_market_history_boundary_contracts.py | 17 ++ tests/test_market_history_continuous.py | 52 +++- tests/test_opportunity_monitor.py | 2 +- tests/test_public_showcase.py | 7 +- tests/test_runtime_artifacts_migration.py | 72 +++-- tests/test_runtime_hygiene.py | 2 +- 19 files changed, 477 insertions(+), 213 deletions(-) diff --git a/publication/sanitize_publication.py b/publication/sanitize_publication.py index 8e8130cc..babc9932 100644 --- a/publication/sanitize_publication.py +++ b/publication/sanitize_publication.py @@ -5,7 +5,6 @@ import runpy from pathlib import Path - if __name__ == "__main__": runpy.run_path( Path(__file__).resolve().parents[1] / "scripts" / "sanitize_publication.py", diff --git a/scripts/install_startup_task.ps1 b/scripts/install_startup_task.ps1 index 6a6facae..fbb2b236 100644 --- a/scripts/install_startup_task.ps1 +++ b/scripts/install_startup_task.ps1 @@ -41,17 +41,64 @@ function Rotate-LogFile { if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { return } - for ($index = $BackupCount; $index -ge 1; $index--) { - $source = if ($index -eq 1) { $Path } else { "$Path.$($index - 1)" } - $target = "$Path.$index" - if (Test-Path -LiteralPath $source -PathType Leaf) { + $temporaryFiles = [System.Collections.Generic.List[string]]::new() + try { + for ($index = $BackupCount; $index -ge 1; $index--) { + $source = if ($index -eq 1) { $Path } else { "$Path.$($index - 1)" } + $target = "$Path.$index" + if (-not (Test-Path -LiteralPath $source -PathType Leaf)) { + continue + } if ((Get-Item -LiteralPath $source).Length -gt $MaximumBytes) { - $trimmed = "$source.trimmed" - Get-Content -LiteralPath $source -Tail 20000 | - Set-Content -LiteralPath $trimmed -Encoding UTF8 - Move-Item -LiteralPath $trimmed -Destination $source -Force + $trimmed = "$source.$PID.trimmed" + $temporaryFiles.Add($trimmed) + $inputStream = [System.IO.File]::Open( + $source, + [System.IO.FileMode]::Open, + [System.IO.FileAccess]::Read, + [System.IO.FileShare]::ReadWrite + ) + try { + $inputStream.Position = [Math]::Max( + 0, + $inputStream.Length - $MaximumBytes + ) + $outputStream = [System.IO.File]::Open( + $trimmed, + [System.IO.FileMode]::Create, + [System.IO.FileAccess]::Write, + [System.IO.FileShare]::None + ) + try { + $inputStream.CopyTo($outputStream) + } + finally { + $outputStream.Dispose() + } + } + finally { + $inputStream.Dispose() + } + [System.IO.File]::Delete($source) + [System.IO.File]::Move($trimmed, $source) } - Move-Item -LiteralPath $source -Destination $target -Force + if (Test-Path -LiteralPath $target -PathType Leaf) { + [System.IO.File]::Delete($target) + } + [System.IO.File]::Move($source, $target) + } + } + catch { + # Logging maintenance must never prevent a safety-bounded service from + # starting. The next restart retries rotation after transient locks. + Write-Warning ( + "Log rotation deferred for {0}: {1}" -f ` + $Path, $_.Exception.Message + ) + } + finally { + foreach ($temporary in $temporaryFiles) { + Remove-Item -LiteralPath $temporary -Force -ErrorAction SilentlyContinue } } } diff --git a/src/ai4binance/data/market_history_continuous.py b/src/ai4binance/data/market_history_continuous.py index 89f0110e..86ccc9b4 100644 --- a/src/ai4binance/data/market_history_continuous.py +++ b/src/ai4binance/data/market_history_continuous.py @@ -52,10 +52,13 @@ "promotion_status": "RESEARCH_ONLY", "live_eligibility_status": "LIVE_ORDER_BLOCKED", } -VIRTUAL_MARKET_COLLECTION_TIMEFRAMES: Final = MARKET_HISTORY_TIMEFRAMES -_DASHBOARD_REFRESH_TIMEFRAMES = ("5m", "15m", "1h", "4h", "1d") -_SCREEN_TIMEFRAMES = VIRTUAL_MARKET_COLLECTION_TIMEFRAMES -_ENRICHMENT_TIMEFRAMES: Final[tuple[str, ...]] = () +_SCREEN_TIMEFRAMES: Final = ("15m", "1h", "4h") +_ENRICHMENT_TIMEFRAMES: Final = ("5m",) +VIRTUAL_MARKET_COLLECTION_TIMEFRAMES: Final = ( + *_SCREEN_TIMEFRAMES, + *_ENRICHMENT_TIMEFRAMES, +) +_DASHBOARD_REFRESH_TIMEFRAMES = VIRTUAL_MARKET_COLLECTION_TIMEFRAMES # The canonical live path persists native decision timeframes directly. REST # remains bounded to bootstrap and gap recovery. _PROGRESS_HEARTBEAT_SECONDS = 5 @@ -74,6 +77,7 @@ "promotion_status": "RESEARCH_ONLY", "live_eligibility_status": "LIVE_ORDER_BLOCKED", } +_REFRESH_REQUEST_REQUESTERS = frozenset({"DASHBOARD", "VIRTUAL_MARKET"}) _REFRESH_REQUEST_PENDING_FIELDS = frozenset( { "schema_version", @@ -127,7 +131,7 @@ def _read_refresh_request(path: Path) -> dict[str, object] | None: value.get("schema_version") != "MarketHistoryRefreshRequest/v1" or not isinstance(value.get("request_id"), str) or not isinstance(value.get("requester"), str) - or value.get("requester") != "DASHBOARD" + or value.get("requester") not in _REFRESH_REQUEST_REQUESTERS or value.get("market") not in {"SPOT", "USD_M_FUTURES"} or not isinstance(value.get("symbol"), str) or _REFRESH_REQUEST_SYMBOL.fullmatch(str(value["symbol"])) is None @@ -594,7 +598,15 @@ def refresh_snapshots(snapshot_time: datetime) -> None: blockers.append("MARKET_HISTORY_REFRESH_REQUEST_INVALID") stream_items = self._interleaved_stream_work(work_items) staged = self.on_symbol_screen is not None - active_collection_timeframes = MARKET_HISTORY_TIMEFRAMES + active_collection_timeframes = ( + _SCREEN_TIMEFRAMES if staged else MARKET_HISTORY_TIMEFRAMES + ) + analysis_timeframes = ( + VIRTUAL_MARKET_COLLECTION_TIMEFRAMES + if staged + else MARKET_HISTORY_TIMEFRAMES + ) + reported_timeframes = analysis_timeframes staged_markets = {"spot", "usd_m_futures"} if staged else set() if staged: stream_items = tuple( @@ -621,10 +633,22 @@ def refresh_snapshots(snapshot_time: datetime) -> None: } coverage: dict[str, dict[str, Counter[str]]] = { market_labels[market]: { - timeframe: Counter() for timeframe in MARKET_HISTORY_TIMEFRAMES + timeframe: Counter() for timeframe in reported_timeframes } for market, _, transport in market_work - if transport is not None + if transport is not None and (not staged or market in staged_markets) + } + coverage_expected: dict[str, Counter[str]] = { + market_labels[market]: Counter( + { + timeframe: len(symbols) + if timeframe in active_collection_timeframes + else 0 + for timeframe in reported_timeframes + } + ) + for market, symbols, transport in market_work + if transport is not None and (not staged or market in staged_markets) } completed_symbols = 0 completed_streams = 0 @@ -671,28 +695,22 @@ def coverage_projection() -> dict[str, list[dict[str, object]]]: with coverage_lock: projection: dict[str, list[dict[str, object]]] = {} - universe_counts = { - "SPOT": len(universe.spot_symbols), - "USD_M_FUTURES": len(universe.futures_symbols), - "COIN_M_FUTURES": len(universe.coin_m_symbols), - } for market, rows in coverage.items(): entries: list[dict[str, object]] = [] - for timeframe in MARKET_HISTORY_TIMEFRAMES: + for timeframe in reported_timeframes: counts = rows[timeframe] resolved = sum(counts.values()) + expected = coverage_expected[market][timeframe] entries.append( { "timeframe": timeframe, - "universe_count": universe_counts[market], + "universe_count": expected, "current_count": counts["CURRENT"], "stale_count": 0, "invalid_count": counts["BLOCKED"], "unavailable_count": counts["UNAVAILABLE"], "refresh_required_count": counts["BACKFILLING"], - "pending_count": max( - 0, universe_counts[market] - resolved - ), + "pending_count": max(0, expected - resolved), } ) projection[market] = entries @@ -928,7 +946,7 @@ def publish_progress( if ( kind == "klines" and isinstance(timeframe, str) - and timeframe in active_collection_timeframes + and timeframe in analysis_timeframes ): dashboard_statuses_by_symbol.setdefault(identity, {})[ timeframe @@ -936,12 +954,12 @@ def publish_progress( dashboard_statuses = dashboard_statuses_by_symbol[identity] if identity not in analysis_started and len( dashboard_statuses - ) == len(active_collection_timeframes): + ) == len(analysis_timeframes): analysis_started.add(identity) analysis_ready = all( dashboard_statuses.get(required) in _ANALYSIS_READY_STREAM_STATES - for required in active_collection_timeframes + for required in analysis_timeframes ) if completed_by_symbol[identity] == required_streams[identity]: completed_symbols += 1 @@ -1102,6 +1120,9 @@ def enqueue_enrichment(identity: tuple[str, str]) -> None: for tf in _ENRICHMENT_TIMEFRAMES: if (*identity, "klines", tf) not in queued_keys: background.append((*identity, transport, "klines", tf)) + label = market_labels[identity[0]] + with coverage_lock: + coverage_expected[label][tf] += 1 added += 1 if added: with progress_lock: @@ -1141,7 +1162,8 @@ def activate_pending_request() -> None: active_identity = identity enqueue_enrichment(identity) request_remaining = { - ("klines", timeframe) for timeframe in MARKET_HISTORY_TIMEFRAMES + ("klines", timeframe) + for timeframe in VIRTUAL_MARKET_COLLECTION_TIMEFRAMES } request_remaining.difference_update( (kind, timeframe) @@ -1205,7 +1227,9 @@ def take_runnable( ): queue.append(candidate) continue - if not self._is_supplemental_stream(kind, timeframe): + if not staged or not self._is_supplemental_stream( + kind, timeframe + ): return candidate identity = (market, symbol) with progress_lock: @@ -1320,8 +1344,12 @@ def complete_active_request() -> None: "schema_version": "2.0", "observed_at": now.isoformat(), "status": "DEGRADED" if blockers else "READY", - "timeframes": list(MARKET_HISTORY_TIMEFRAMES), - "timeframe_refresh_schedule": timeframe_refresh_schedule(), + "timeframes": list(reported_timeframes), + "timeframe_refresh_schedule": [ + row + for row in timeframe_refresh_schedule() + if row["timeframe"] in reported_timeframes + ], "collection_plan": self._collection_plan(), "initial_history_days": self.initial_days, "spot_universe_count": len(universe.spot_symbols), @@ -1507,10 +1535,10 @@ def _prioritized_symbols(self, symbols: tuple[str, ...]) -> tuple[str, ...]: def _collection_plan(self) -> dict[str, object]: staged = self.on_symbol_screen is not None return { - "mode": "TOP_VOLUME_FULL_MULTITF" if staged else "FULL_HISTORY", + "mode": "SCREEN_THEN_ENRICH" if staged else "FULL_HISTORY", "screen_timeframes": list(_SCREEN_TIMEFRAMES) if staged else [], "enrichment_timeframes": list(_ENRICHMENT_TIMEFRAMES) if staged else [], - "enrichment_scope": "ALL_SELECTED_SYMBOLS" if staged else "ALL", + "enrichment_scope": "OPPORTUNITY_CANDIDATES" if staged else "ALL", "deferred_streams": [ "markPriceKlines", "indexPriceKlines", diff --git a/src/ai4binance/governance/governance_enforcement_fabric.py b/src/ai4binance/governance/governance_enforcement_fabric.py index 2806907b..4215bcaa 100644 --- a/src/ai4binance/governance/governance_enforcement_fabric.py +++ b/src/ai4binance/governance/governance_enforcement_fabric.py @@ -641,7 +641,12 @@ def _string(value: object, name: str) -> str: def _safe_path(value: object, name: str) -> str: path = _string(value, name) - if Path(path).is_absolute() or "\\" in path or ".." in Path(path).parts: + if ( + path.startswith("/") + or Path(path).is_absolute() + or "\\" in path + or ".." in Path(path).parts + ): raise ValueError(f"{name} must be a repository-relative POSIX path") return path diff --git a/src/ai4binance/governance/model_registry.py b/src/ai4binance/governance/model_registry.py index e7bdae5d..63f427f7 100644 --- a/src/ai4binance/governance/model_registry.py +++ b/src/ai4binance/governance/model_registry.py @@ -9,6 +9,7 @@ from functools import lru_cache from hashlib import sha256 from pathlib import Path +from typing import cast _MODEL_ID = re.compile(r"^[a-z0-9][a-z0-9._-]{2,127}$") _SHA256 = re.compile(r"^[a-f0-9]{64}$") @@ -478,24 +479,9 @@ def _local_model_manifest_blockers( if not isinstance(item, Mapping): return (f"LOCAL_MODEL_MANIFEST_INVALID:{model_id}",) try: - _require_keys( - item, - {"role", "path", "byte_length", "sha256"}, - "local model artifact", + candidate, expected_length, expected_sha256 = _local_model_artifact( + repository_root, cast(Mapping[str, object], item) ) - artifact_path = _text(item["path"], "local model artifact path") - expected_sha256 = _text(item["sha256"], "local model artifact sha256") - expected_length = item["byte_length"] - if ( - not _SHA256.fullmatch(expected_sha256) - or not isinstance(expected_length, int) - or expected_length < 1 - or Path(artifact_path).is_absolute() - or not artifact_path.startswith("models/") - ): - raise ValueError("local model artifact contract is invalid") - candidate = (repository_root / artifact_path).resolve() - candidate.relative_to(repository_root) except (TypeError, ValueError): return (f"LOCAL_MODEL_MANIFEST_INVALID:{model_id}",) if not candidate.is_file(): @@ -513,6 +499,33 @@ def _local_model_manifest_blockers( return tuple(blockers) +def _local_model_artifact( + repository_root: Path, + item: Mapping[str, object], +) -> tuple[Path, int, str]: + """Validate one manifest artifact and resolve its repository-local path.""" + + _require_keys( + item, + {"role", "path", "byte_length", "sha256"}, + "local model artifact", + ) + artifact_path = _text(item["path"], "local model artifact path") + expected_sha256 = _text(item["sha256"], "local model artifact sha256") + expected_length = item["byte_length"] + if ( + not _SHA256.fullmatch(expected_sha256) + or not isinstance(expected_length, int) + or expected_length < 1 + or Path(artifact_path).is_absolute() + or not artifact_path.startswith("models/") + ): + raise ValueError("local model artifact contract is invalid") + candidate = (repository_root / artifact_path).resolve() + candidate.relative_to(repository_root) + return candidate, expected_length, expected_sha256 + + def _sha256_file(path: Path) -> str: stat = path.stat() return _sha256_file_cached(str(path), stat.st_size, stat.st_mtime_ns) diff --git a/src/ai4binance/internal_radar.py b/src/ai4binance/internal_radar.py index 0f04db23..5f57d9d3 100644 --- a/src/ai4binance/internal_radar.py +++ b/src/ai4binance/internal_radar.py @@ -168,91 +168,29 @@ def run_internal_radar_once( known, pending, pending_state_present = _read_state(state_path) files, enumeration_blockers = _image_files(source) - new_candidates: list[dict[str, object]] = [] - candidates_by_id: dict[str, Path] = {} - next_known: set[str] = set() - for path in files: - candidate, fingerprint = _candidate(path, source) - if fingerprint is None: - continue - next_known.add(fingerprint) - candidates_by_id[str(candidate["candidate_id"])] = path - if include_existing or not pending_state_present or fingerprint not in known: - new_candidates.append(candidate) - - pending_by_id = { - str(item["candidate_id"]): item - for item in pending - if isinstance(item.get("candidate_id"), str) - } - for candidate in new_candidates: - pending_by_id.setdefault(str(candidate["candidate_id"]), candidate) - - vision_blockers: list[str] = [] - vision_analysis_count = 0 - if vision_enabled: - runner = vision_runner or LlamaCppVisionRunner() - for candidate_id in sorted(pending_by_id, key=str.casefold): - candidate = pending_by_id[candidate_id] - if _has_current_vision_evidence(candidate): - continue - source_path = candidates_by_id.get(candidate_id) - content_sha256 = candidate.get("content_sha256") - if source_path is None or not isinstance(content_sha256, str): - continue - evidence = runner.analyze( - candidate_id=candidate_id, - source_content_sha256=content_sha256, - image_path=source_path, - ) - vision_analysis_count += 1 - candidate["vision_evidence"] = evidence.to_payload() - candidate["last_scan_timestamp_utc"] = datetime.now(UTC).isoformat() - candidate["assessment_status"] = evidence.status - candidate["system_benefit"] = evidence.system_contribution or "NOT_ASSESSED" - candidate["system_tradeoff"] = ( - ",".join(evidence.tradeoff_categories) - if evidence.tradeoff_categories - else "NOT_ASSESSED" - ) - candidate["recommendation"] = "HUMAN_REVIEW_REQUIRED" - vision_blockers.extend(evidence.blockers) - _write_state_checkpoint( - state_path=state_path, - observed_at=now, - known_fingerprints=known | next_known, - candidates=pending_by_id, - ) - if vision_analysis_count % _LATEST_CHECKPOINT_INTERVAL == 0: - checkpoint_candidates = tuple( - pending_by_id[key] - for key in sorted(pending_by_id, key=str.casefold) - ) - _persist_result( - InternalRadarResult( - True, - "RUNNING_WITH_BLOCKERS", - now, - len(files), - len(new_candidates), - len(checkpoint_candidates), - True, - vision_analysis_count, - tuple( - dict.fromkeys( - ( - *enumeration_blockers, - *vision_blockers, - "VISION_ANALYSIS_IN_PROGRESS", - ) - ) - ), - state_path, - latest_path, - checkpoint_candidates, - ), - candidate_paths=candidates_by_id, - ) + new_candidates, candidates_by_id, next_known, pending_by_id = ( + _collect_radar_candidates( + files=files, + source=source, + known=known, + pending=pending, + pending_state_present=pending_state_present, + include_existing=include_existing, + ) + ) + vision_blockers, vision_analysis_count = _analyze_pending_candidates( + vision_enabled=vision_enabled, + vision_runner=vision_runner, + pending_by_id=pending_by_id, + candidates_by_id=candidates_by_id, + state_path=state_path, + latest_path=latest_path, + observed_at=now, + known_fingerprints=known | next_known, + scanned_count=len(files), + new_candidate_count=len(new_candidates), + enumeration_blockers=enumeration_blockers, + ) candidates = tuple( pending_by_id[key] for key in sorted(pending_by_id, key=str.casefold) ) @@ -285,6 +223,157 @@ def run_internal_radar_once( return _persist_result(result, candidate_paths=candidates_by_id) +def _collect_radar_candidates( + *, + files: tuple[Path, ...], + source: Path, + known: set[str], + pending: tuple[dict[str, object], ...], + pending_state_present: bool, + include_existing: bool, +) -> tuple[ + list[dict[str, object]], + dict[str, Path], + set[str], + dict[str, dict[str, object]], +]: + """Build deterministic new and pending candidate projections.""" + + new_candidates: list[dict[str, object]] = [] + candidates_by_id: dict[str, Path] = {} + next_known: set[str] = set() + for path in files: + candidate, fingerprint = _candidate(path, source) + if fingerprint is None: + continue + next_known.add(fingerprint) + candidates_by_id[str(candidate["candidate_id"])] = path + if include_existing or not pending_state_present or fingerprint not in known: + new_candidates.append(candidate) + + pending_by_id = { + str(item["candidate_id"]): item + for item in pending + if isinstance(item.get("candidate_id"), str) + } + for candidate in new_candidates: + pending_by_id.setdefault(str(candidate["candidate_id"]), candidate) + return new_candidates, candidates_by_id, next_known, pending_by_id + + +def _analyze_pending_candidates( + *, + vision_enabled: bool, + vision_runner: LlamaCppVisionRunner | None, + pending_by_id: dict[str, dict[str, object]], + candidates_by_id: dict[str, Path], + state_path: Path, + latest_path: Path, + observed_at: datetime, + known_fingerprints: set[str], + scanned_count: int, + new_candidate_count: int, + enumeration_blockers: tuple[str, ...], +) -> tuple[list[str], int]: + """Attach bounded advisory vision evidence and persist restart checkpoints.""" + + if not vision_enabled: + return [], 0 + runner = vision_runner or LlamaCppVisionRunner() + blockers: list[str] = [] + analysis_count = 0 + for candidate_id in sorted(pending_by_id, key=str.casefold): + candidate = pending_by_id[candidate_id] + if _has_current_vision_evidence(candidate): + continue + source_path = candidates_by_id.get(candidate_id) + content_sha256 = candidate.get("content_sha256") + if source_path is None or not isinstance(content_sha256, str): + continue + evidence = runner.analyze( + candidate_id=candidate_id, + source_content_sha256=content_sha256, + image_path=source_path, + ) + analysis_count += 1 + candidate["vision_evidence"] = evidence.to_payload() + candidate["last_scan_timestamp_utc"] = datetime.now(UTC).isoformat() + candidate["assessment_status"] = evidence.status + candidate["system_benefit"] = evidence.system_contribution or "NOT_ASSESSED" + candidate["system_tradeoff"] = ( + ",".join(evidence.tradeoff_categories) + if evidence.tradeoff_categories + else "NOT_ASSESSED" + ) + candidate["recommendation"] = "HUMAN_REVIEW_REQUIRED" + blockers.extend(evidence.blockers) + _write_state_checkpoint( + state_path=state_path, + observed_at=observed_at, + known_fingerprints=known_fingerprints, + candidates=pending_by_id, + ) + if analysis_count % _LATEST_CHECKPOINT_INTERVAL == 0: + _persist_vision_checkpoint( + observed_at=observed_at, + scanned_count=scanned_count, + new_candidate_count=new_candidate_count, + analysis_count=analysis_count, + enumeration_blockers=enumeration_blockers, + vision_blockers=blockers, + state_path=state_path, + latest_path=latest_path, + pending_by_id=pending_by_id, + candidates_by_id=candidates_by_id, + ) + return blockers, analysis_count + + +def _persist_vision_checkpoint( + *, + observed_at: datetime, + scanned_count: int, + new_candidate_count: int, + analysis_count: int, + enumeration_blockers: tuple[str, ...], + vision_blockers: list[str], + state_path: Path, + latest_path: Path, + pending_by_id: dict[str, dict[str, object]], + candidates_by_id: dict[str, Path], +) -> None: + """Persist one restart-safe progress projection for long vision runs.""" + + candidates = tuple( + pending_by_id[key] for key in sorted(pending_by_id, key=str.casefold) + ) + _persist_result( + InternalRadarResult( + True, + "RUNNING_WITH_BLOCKERS", + observed_at, + scanned_count, + new_candidate_count, + len(candidates), + True, + analysis_count, + tuple( + dict.fromkeys( + ( + *enumeration_blockers, + *vision_blockers, + "VISION_ANALYSIS_IN_PROGRESS", + ) + ) + ), + state_path, + latest_path, + candidates, + ), + candidate_paths=candidates_by_id, + ) + + def load_internal_radar_latest(repository_root: Path) -> dict[str, object]: """Load the secret-safe latest payload for YKB; malformed evidence fails closed.""" path = repository_root / "runtime" / "artifacts" / "internal_radar" / "latest.json" diff --git a/tests/governance/terminology/test_governance_enforcement_fabric.py b/tests/governance/terminology/test_governance_enforcement_fabric.py index cf3f8a74..daeb0cfb 100644 --- a/tests/governance/terminology/test_governance_enforcement_fabric.py +++ b/tests/governance/terminology/test_governance_enforcement_fabric.py @@ -347,7 +347,7 @@ def test_fabric_contract_helpers_reject_untrusted_shapes(tmp_path: Path) -> None with pytest.raises(ValueError, match="POSIX"): fabric_module._safe_path(value, "path") for value in (None, [], ["x", "x"]): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match=r"non-empty list|must be unique"): fabric_module._safe_paths(value, "paths") for value in ("bad", "A" * 64, "a" * 63): with pytest.raises(ValueError, match="SHA-256"): @@ -474,7 +474,10 @@ def test_quality_mapping_parser_rejects_malformed_and_duplicate_entries() -> Non } }, ): - with pytest.raises(ValueError): + with pytest.raises( + ValueError, + match=r"mappings must be a list|mapping tests must be a list|duplicate", + ): fabric_module._quality_standard_mappings(payload) assert fabric_module._quality_standard_mappings( { @@ -503,11 +506,11 @@ def test_fabric_low_level_contracts_cover_metadata_and_duplicate_paths( frontmatter = tmp_path / "standard.md" frontmatter.write_text("---\ndocument_id: TEST\n---\n", encoding="utf-8") assert fabric_module._frontmatter(frontmatter)["document_id"] == "TEST" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="must be unique"): fabric_module._safe_paths(["tests/a.py", "tests/a.py"], "paths") - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="POSIX path"): fabric_module._safe_path("../escape", "path") - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="SHA-256"): fabric_module._sha256_text("A" * 64) diff --git a/tests/governance/terminology/test_terminology_policy.py b/tests/governance/terminology/test_terminology_policy.py index 50e4aaf9..78a90ffd 100644 --- a/tests/governance/terminology/test_terminology_policy.py +++ b/tests/governance/terminology/test_terminology_policy.py @@ -137,9 +137,7 @@ def test_terminology_scan_covers_deprecated_missing_and_projection_drift( loaded_policy = load_terminology_policy(ROOT) policy = replace( loaded_policy, - terms=( - replace(loaded_policy.terms[0], deprecated_aliases=("legacy",)), - ), + terms=(replace(loaded_policy.terms[0], deprecated_aliases=("legacy",)),), ) target = tmp_path / policy.scan_roots[0] / "terms.txt" target.parent.mkdir(parents=True) diff --git a/tests/test_architecture_diagram_validation.py b/tests/test_architecture_diagram_validation.py index 86ea8216..8d5e9970 100644 --- a/tests/test_architecture_diagram_validation.py +++ b/tests/test_architecture_diagram_validation.py @@ -2,14 +2,14 @@ import re from pathlib import Path -from unittest.mock import patch +import pytest import yaml +from ai4binance.ops import architecture_diagram_validation as diagram_validation from ai4binance.ops.architecture_diagram_validation import ( validate_architecture_diagrams, ) -from ai4binance.ops import architecture_diagram_validation as diagram_validation ROOT = Path(__file__).parents[1] DIAGRAM_ROOT = ROOT / "docs" / "architecture" / "diagrams" diff --git a/tests/test_artifact_hygiene_scripts.py b/tests/test_artifact_hygiene_scripts.py index 943bf754..f9be76dc 100644 --- a/tests/test_artifact_hygiene_scripts.py +++ b/tests/test_artifact_hygiene_scripts.py @@ -4359,9 +4359,9 @@ def test_qwen_prompter_startup_task_is_visible_and_advisory_only() -> None: llama_server_text = (Path("scripts") / "start_llama_server.ps1").read_text( encoding="utf-8" ) - vision_server_text = ( - Path("scripts") / "start_local_vision_server.ps1" - ).read_text(encoding="utf-8") + vision_server_text = (Path("scripts") / "start_local_vision_server.ps1").read_text( + encoding="utf-8" + ) local_llm_text = (Path("scripts") / "start_local_llm.ps1").read_text( encoding="utf-8" ) diff --git a/tests/test_lowest_twenty_coverage_models.py b/tests/test_lowest_twenty_coverage_models.py index ad5a2cb0..017b0e19 100644 --- a/tests/test_lowest_twenty_coverage_models.py +++ b/tests/test_lowest_twenty_coverage_models.py @@ -93,11 +93,7 @@ def test_runtime_environment_rejects_unsafe_temp_contracts( ai4binance.tomllib, "load", lambda _stream: { - "tool": { - "ai4binance": { - "runtime": {"temporary_directory": directory} - } - } + "tool": {"ai4binance": {"runtime": {"temporary_directory": directory}}} }, ) with pytest.raises(ValueError, match="Temporary directory"): @@ -223,7 +219,8 @@ def test_trend_events_observe_records_crosses_and_confirmed_slope( ), ) observed = trend_events_module.TrendEventsAgent._observe( - "1h", (object(),) * 201 # type: ignore[arg-type] + "1h", + (object(),) * 201, # type: ignore[arg-type] ) _timeframe, direction, _band, events, slope = observed assert direction == 1 diff --git a/tests/test_maintainability_ratchet.py b/tests/test_maintainability_ratchet.py index 2e403f0e..974435d3 100644 --- a/tests/test_maintainability_ratchet.py +++ b/tests/test_maintainability_ratchet.py @@ -54,20 +54,33 @@ def test_repository_maintainability_baseline_is_current() -> None: @pytest.mark.parametrize( - "payload", - ( - "{}", - '{"schema_version": 1, "limits": {}, "approved_paths": []}', - '{"schema_version": 1, "limits": {"C901": 0, "PLR0912": 0, "PLR0915": -1}, "approved_paths": []}', - '{"schema_version": 1, "limits": {"C901": 0, "PLR0912": 0, "PLR0915": 0}, "approved_paths": ["outside.py"]}', - ), + ("payload", "match"), + [ + ("{}", "schema_version"), + ( + '{"schema_version": 1, "limits": {}, "approved_paths": []}', + "limits must cover governed rules", + ), + ( + '{"schema_version": 1, "limits": ' + '{"C901": 0, "PLR0912": 0, "PLR0915": -1}, ' + '"approved_paths": []}', + "limits must be non-negative integers", + ), + ( + '{"schema_version": 1, "limits": ' + '{"C901": 0, "PLR0912": 0, "PLR0915": 0}, ' + '"approved_paths": ["outside.py"]}', + "approved_paths must be sorted source paths", + ), + ], ) def test_load_baseline_rejects_invalid_governed_shapes( - tmp_path: Path, payload: str + tmp_path: Path, payload: str, match: str ) -> None: path = tmp_path / "baseline.json" path.write_text(payload, encoding="utf-8") - with pytest.raises(ValueError): + with pytest.raises(ValueError, match=match): load_baseline(path) @@ -77,16 +90,12 @@ def test_collect_findings_and_evaluate_fail_closed_for_bad_tool_data( completed = type( "Completed", (), {"returncode": 2, "stderr": "tool failed", "stdout": ""} )() - monkeypatch.setattr( - ratchet.subprocess, "run", lambda *_args, **_kwargs: completed - ) + monkeypatch.setattr(ratchet.subprocess, "run", lambda *_args, **_kwargs: completed) with pytest.raises(RuntimeError, match="tool failed"): collect_ruff_findings(ROOT) malformed = type("Completed", (), {"returncode": 0, "stderr": "", "stdout": "{}"})() - monkeypatch.setattr( - ratchet.subprocess, "run", lambda *_args, **_kwargs: malformed - ) + monkeypatch.setattr(ratchet.subprocess, "run", lambda *_args, **_kwargs: malformed) with pytest.raises(ValueError, match="JSON array"): collect_ruff_findings(ROOT) diff --git a/tests/test_market_data_gateway.py b/tests/test_market_data_gateway.py index 40b6c85b..bbb6a459 100644 --- a/tests/test_market_data_gateway.py +++ b/tests/test_market_data_gateway.py @@ -32,12 +32,12 @@ SharedMarketCache, build_gateway, ) +from ai4binance.exchange import rate_limit as rate_limit_module from ai4binance.exchange.public_stream import SpotKlineUpdate from ai4binance.exchange.rate_limit import ( WeightedRateLimitGovernor, public_request_weight, ) -from ai4binance.exchange import rate_limit as rate_limit_module from ai4binance.schemas import OHLCVCandle START = datetime(2026, 9, 17, 0, 0, tzinfo=UTC) diff --git a/tests/test_market_history_boundary_contracts.py b/tests/test_market_history_boundary_contracts.py index 349cc3ca..177a55e5 100644 --- a/tests/test_market_history_boundary_contracts.py +++ b/tests/test_market_history_boundary_contracts.py @@ -92,6 +92,23 @@ def test_refresh_request_status_and_size_limits(tmp_path: Path) -> None: h._load(path) +def test_virtual_market_refresh_request_is_a_bounded_compatible_requester( + tmp_path: Path, +) -> None: + path = tmp_path / "request.json" + path.write_text( + json.dumps(request_payload() | {"requester": "VIRTUAL_MARKET"}), + encoding="utf-8", + ) + + result = h.market_history_refresh_status(path) + + assert result["state"] == "PENDING" + assert result["requester"] == "VIRTUAL_MARKET" + assert result["execution_allowed"] is False + assert result["live_eligibility_status"] == "LIVE_ORDER_BLOCKED" + + @pytest.mark.parametrize( "payload", [ diff --git a/tests/test_market_history_continuous.py b/tests/test_market_history_continuous.py index a9631f4d..1a9ec461 100644 --- a/tests/test_market_history_continuous.py +++ b/tests/test_market_history_continuous.py @@ -25,6 +25,9 @@ from ai4binance.core.errors import ExchangeHttpError, ExchangeTransportError from ai4binance.data.archive import ParquetOHLCVArchive from ai4binance.data.market_history_continuous import ( + _ENRICHMENT_TIMEFRAMES, + _SCREEN_TIMEFRAMES, + VIRTUAL_MARKET_COLLECTION_TIMEFRAMES, ContinuousMarketHistory, MeteredPublicTransport, PublicRequestBudget, @@ -133,7 +136,7 @@ def test_collection_worker_limit_supports_bounded_archive_parallelism( @pytest.mark.parametrize("workers", [1, 4]) -def test_staged_universe_collects_all_virtual_market_timeframes_before_analysis( +def test_staged_universe_downloads_baseline_then_enriches_candidates_before_analysis( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, workers: int ) -> None: instance = collector(tmp_path, Transport()) @@ -175,17 +178,42 @@ def analyze(market: str, symbol: str, _now: datetime) -> Mapping[str, object]: instance.on_symbol_screen = screen instance.on_symbol_ready = analyze result = instance.sync_cycle(observed_at=NOW) - assert len(calls) == 20 - assert {tf for _, _, tf in calls} == set(MARKET_HISTORY_TIMEFRAMES) + assert len(calls) == 14 + assert {tf for _, _, tf in calls} == set(VIRTUAL_MARKET_COLLECTION_TIMEFRAMES) + assert sum(tf == "5m" for _, _, tf in calls) == 2 + assert {(market, symbol) for market, symbol, tf in calls if tf == "5m"} == { + ("spot", "ETHUSDT"), + ("usd_m_futures", "ETHUSDT"), + } assert len(calls) == len(set(calls)) assert sorted(analyzed) == [ - ("SPOT", "BTCUSDT"), ("SPOT", "ETHUSDT"), - ("USD_M_FUTURES", "BTCUSDT"), ("USD_M_FUTURES", "ETHUSDT"), ] - assert result["total_streams"] == result["completed_streams"] == 20 + assert result["total_streams"] == result["completed_streams"] == 14 assert result["completed_symbols"] == 4 + assert result["timeframes"] == ["15m", "1h", "4h", "5m"] + coverage = cast(dict[str, list[dict[str, object]]], result["collector_coverage"]) + for market in ("SPOT", "USD_M_FUTURES"): + expected = {row["timeframe"]: row for row in coverage[market]} + assert expected["15m"]["universe_count"] == 2 + assert expected["1h"]["universe_count"] == 2 + assert expected["4h"]["universe_count"] == 2 + assert expected["5m"]["universe_count"] == 1 + assert all(row["pending_count"] == 0 for row in coverage[market]) + assert result["collection_plan"] == { + "mode": "SCREEN_THEN_ENRICH", + "screen_timeframes": list(_SCREEN_TIMEFRAMES), + "enrichment_timeframes": list(_ENRICHMENT_TIMEFRAMES), + "enrichment_scope": "OPPORTUNITY_CANDIDATES", + "deferred_streams": [ + "markPriceKlines", + "indexPriceKlines", + "funding", + "open_interest", + "coin_m_futures", + ], + } assert result["execution_allowed"] is False @@ -210,7 +238,7 @@ def screen(*_args: object) -> Mapping[str, object]: monkeypatch.setattr(instance, "_collect_stream", collect) instance.on_symbol_screen = screen instance.sync_cycle(observed_at=NOW) - assert calls == list(MARKET_HISTORY_TIMEFRAMES) + assert calls == list(_SCREEN_TIMEFRAMES) def test_existing_archive_fetches_only_holes_then_tail_without_replaying_rows( @@ -297,7 +325,7 @@ def refresh( *, timeframes: tuple[str, ...] | None = None, ) -> dict[str, object]: - assert timeframes == MARKET_HISTORY_TIMEFRAMES + assert timeframes == VIRTUAL_MARKET_COLLECTION_TIMEFRAMES lock = ( root / "runtime/artifacts/opportunity-radar/monitor/USD_M_FUTURES" @@ -364,11 +392,11 @@ def collect( "live_eligibility_status": "LIVE_ORDER_BLOCKED", } result = instance.sync_cycle(observed_at=NOW) - assert calls == list(MARKET_HISTORY_TIMEFRAMES) + assert calls == list(VIRTUAL_MARKET_COLLECTION_TIMEFRAMES) assert ( result["completed_streams"] == result["total_streams"] - == len(MARKET_HISTORY_TIMEFRAMES) + == len(VIRTUAL_MARKET_COLLECTION_TIMEFRAMES) ) @@ -928,7 +956,7 @@ def refresh(*_args: object, **kwargs: object) -> dict[str, object]: "now": NOW, "minimum_candles": 200, "candle_limit": 250, - "timeframes": MARKET_HISTORY_TIMEFRAMES, + "timeframes": VIRTUAL_MARKET_COLLECTION_TIMEFRAMES, } ] assert futures["status"] == "DELEGATED" @@ -1882,7 +1910,7 @@ def test_canonical_opportunity_pipeline_fails_closed_for_malformed_payloads( assert result["candidate_count"] == 0 assert result["blockers"] == [ f"OPPORTUNITY_DATA_UNAVAILABLE:{timeframe}" - for timeframe in MARKET_HISTORY_TIMEFRAMES + for timeframe in VIRTUAL_MARKET_COLLECTION_TIMEFRAMES ] diff --git a/tests/test_opportunity_monitor.py b/tests/test_opportunity_monitor.py index 0e04e5a4..d83b88b7 100644 --- a/tests/test_opportunity_monitor.py +++ b/tests/test_opportunity_monitor.py @@ -24,13 +24,13 @@ universe_monitor_summary, ) from ai4binance.cli.opportunity_monitor import refresh_candle_windows +from ai4binance.compatibility import opportunity_monitor as monitor_module from ai4binance.compatibility.opportunity_monitor import ( SAFE_STATE as COMPATIBILITY_SAFE_STATE, ) from ai4binance.compatibility.opportunity_monitor import ( refresh_monitor as compatibility_refresh_monitor, ) -from ai4binance.compatibility import opportunity_monitor as monitor_module from ai4binance.config import Settings from ai4binance.data.archive import ParquetOHLCVArchive from ai4binance.data.market_history_sync import read_cached_market_universe diff --git a/tests/test_public_showcase.py b/tests/test_public_showcase.py index 896fe300..db93cf5d 100644 --- a/tests/test_public_showcase.py +++ b/tests/test_public_showcase.py @@ -151,7 +151,9 @@ def test_public_showcase_rejects_hash_drift_and_secret_scan_failure( assert not (tmp_path / "scan-output").exists() -@pytest.mark.parametrize("value", ("", "/absolute", "back\\slash", "a/../b")) +@pytest.mark.parametrize( + "value,", [("",), ("/absolute",), ("back\\slash",), ("a/../b",)] +) def test_showcase_helpers_reject_unsafe_contract_values(value: str) -> None: with pytest.raises(PublicShowcaseError): showcase._safe_relative_path(value, field="artifact") @@ -244,7 +246,8 @@ def test_showcase_rejects_each_authority_expansion_shape(tmp_path: Path) -> None manifest_path = tmp_path / "version.yaml" manifest_path.write_text( - "version: 2\npublication: {}\nallowed_artifacts: []\ndenied_paths: []\nsecret_scan: {}\n", + "version: 2\npublication: {}\nallowed_artifacts: []\n" + "denied_paths: []\nsecret_scan: {}\n", encoding="utf-8", ) with pytest.raises(PublicShowcaseError, match="version"): diff --git a/tests/test_runtime_artifacts_migration.py b/tests/test_runtime_artifacts_migration.py index dd9db41f..00a5d73d 100644 --- a/tests/test_runtime_artifacts_migration.py +++ b/tests/test_runtime_artifacts_migration.py @@ -1,19 +1,20 @@ """Regression tests for the canonical runtime-artifact layout migration.""" +import json from pathlib import Path from typing import cast -import json import pytest + from ai4binance.infrastructure.filesystem.runtime_artifacts import ( RuntimeArtifactLayoutManifest, default_runtime_artifact_layout_manifest_path, + layout, load_runtime_artifact_layout_manifest, ) from ai4binance.infrastructure.filesystem.runtime_artifacts.layout import ( RuntimeArtifactLayoutManifest as CanonicalLayoutManifest, ) -from ai4binance.infrastructure.filesystem.runtime_artifacts import layout from ai4binance.ops.kaizen_quality import build_architecture_baseline from ai4binance.runtime_artifacts import ( RuntimeArtifactLayoutManifest as LegacyPackageLayoutManifest, @@ -64,7 +65,7 @@ def test_runtime_artifact_migration_is_recorded_as_canonical_and_facade() -> Non def test_layout_contract_helpers_fail_closed_and_canonicalize_aliases( tmp_path: Path, ) -> None: - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="under runtime"): layout.RuntimeRetentionPolicy("outside", "keep", 0, 0, False) manifest = CanonicalLayoutManifest( "runtime/artifacts", @@ -80,15 +81,23 @@ def test_layout_contract_helpers_fail_closed_and_canonicalize_aliases( with pytest.raises(ValueError, match="unknown runtime retention"): manifest.retention_for("unknown") assert layout._capacity_budget_mapping(None) == {} - for value in ([], {"outside": 1}, {"runtime/a": True}, {"runtime/a": 0}): - with pytest.raises(ValueError): + invalid_budgets = [ + ([], "must be an object"), + ({"outside": 1}, "must use runtime paths"), + ({"runtime/a": True}, "must be an integer"), + ({"runtime/a": 0}, "must be positive"), + ] + for value, match in invalid_budgets: + with pytest.raises(ValueError, match=match): layout._capacity_budget_mapping(value) - for value in (None, [], {"x": {}}): - if value is None: - assert layout._retention_mapping(value) == {} - else: - with pytest.raises(ValueError): - layout._retention_mapping(value) + invalid_retention = [ + ([], "retention must be an object"), + ({"x": {}}, "automatic_cleanup must be boolean"), + ] + assert layout._retention_mapping(None) == {} + for value, match in invalid_retention: + with pytest.raises(ValueError, match=match): + layout._retention_mapping(value) path = tmp_path / "manifest.json" path.write_text(json.dumps({"schema_version": "bad"}), encoding="utf-8") @@ -115,21 +124,40 @@ def test_runtime_retention_policy_rejects_invalid_values( def test_layout_mapping_helpers_cover_all_invalid_contract_shapes() -> None: - for value in (None, [], {}, {"": "runtime/a"}, {"a": ""}): - with pytest.raises(ValueError): + invalid_text_mappings = [ + (None, "must be a non-empty object"), + ([], "must be a non-empty object"), + ({}, "must be a non-empty object"), + ({"": "runtime/a"}, "must contain non-empty strings"), + ({"a": ""}, "must contain non-empty strings"), + ] + for value, match in invalid_text_mappings: + with pytest.raises(ValueError, match=match): layout._text_mapping(value, "roots") - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="canonical_root must be a non-empty string"): layout._text({}, "canonical_root") - for value in ({"rule": []}, {"": {}}, {"rule": {"automatic_cleanup": "yes"}}): - with pytest.raises(ValueError): + invalid_retention_mappings = [ + ({"rule": []}, "entries must be named objects"), + ({"": {}}, "entries must be named objects"), + ({"rule": {"automatic_cleanup": "yes"}}, "must be boolean"), + ] + for value, match in invalid_retention_mappings: + with pytest.raises(ValueError, match=match): layout._retention_mapping(value) - for policy in ( - {"automatic_cleanup": True, "minimum_age_days": True, "keep_latest": 0}, - {"automatic_cleanup": True, "minimum_age_days": 0, "keep_latest": False}, - ): - with pytest.raises(ValueError): + invalid_retention_policies = [ + ( + {"automatic_cleanup": True, "minimum_age_days": True, "keep_latest": 0}, + "minimum_age_days must be an integer", + ), + ( + {"automatic_cleanup": True, "minimum_age_days": 0, "keep_latest": False}, + "keep_latest must be an integer", + ), + ] + for policy, match in invalid_retention_policies: + with pytest.raises(ValueError, match=match): layout._retention_mapping({"rule": policy}) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="entry kind is invalid"): layout._retention_entry_kind("unknown") diff --git a/tests/test_runtime_hygiene.py b/tests/test_runtime_hygiene.py index da95b28d..54f03f6b 100644 --- a/tests/test_runtime_hygiene.py +++ b/tests/test_runtime_hygiene.py @@ -11,11 +11,11 @@ RuntimeArtifactLayoutManifest, RuntimeRetentionPolicy, ) +from ai4binance.ops import runtime_hygiene from ai4binance.ops.runtime_hygiene import ( build_runtime_hygiene_report, persist_runtime_hygiene_report, ) -from ai4binance.ops import runtime_hygiene def _manifest() -> RuntimeArtifactLayoutManifest: From 6a824b148bdb4c0472ab1c38208d04f9eac54f42 Mon Sep 17 00:00:00 2001 From: Huseyin Cicek Date: Fri, 25 Sep 2026 15:34:46 +0300 Subject: [PATCH 11/19] feat: add by HsC --- src/ai4binance/governance/model_registry.py | 52 +++++++++++-------- tests/test_model_registry_coverage_closure.py | 2 +- 2 files changed, 32 insertions(+), 22 deletions(-) diff --git a/src/ai4binance/governance/model_registry.py b/src/ai4binance/governance/model_registry.py index 63f427f7..fb280c30 100644 --- a/src/ai4binance/governance/model_registry.py +++ b/src/ai4binance/governance/model_registry.py @@ -450,27 +450,7 @@ def _local_model_manifest_blockers( """Validate a local GGUF manifest without persisting machine paths.""" try: - payload = json.loads(manifest_path.read_text(encoding="utf-8")) - manifest = _mapping(payload, "local model manifest") - _require_keys( - manifest, - { - "schema_version", - "model_id", - "provider", - "model_version", - "runtime_boundary", - "artifacts", - }, - "local model manifest", - ) - if manifest["schema_version"] != "1.0.0": - raise ValueError("local model manifest schema version is unsupported") - if manifest["model_id"] != model_id: - raise ValueError("local model manifest model identity mismatch") - artifacts = manifest["artifacts"] - if not isinstance(artifacts, list) or not artifacts: - raise ValueError("local model manifest artifacts are required") + artifacts = _load_local_model_artifacts(manifest_path, model_id) except (OSError, ValueError, json.JSONDecodeError): return (f"LOCAL_MODEL_MANIFEST_INVALID:{model_id}",) @@ -499,6 +479,36 @@ def _local_model_manifest_blockers( return tuple(blockers) +def _load_local_model_artifacts( + manifest_path: Path, + model_id: str, +) -> list[object]: + """Load the exact local manifest envelope before artifact validation.""" + + payload = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest = _mapping(payload, "local model manifest") + _require_keys( + manifest, + { + "schema_version", + "model_id", + "provider", + "model_version", + "runtime_boundary", + "artifacts", + }, + "local model manifest", + ) + if manifest["schema_version"] != "1.0.0": + raise ValueError("local model manifest schema version is unsupported") + if manifest["model_id"] != model_id: + raise ValueError("local model manifest model identity mismatch") + artifacts = manifest["artifacts"] + if not isinstance(artifacts, list) or not artifacts: + raise ValueError("local model manifest artifacts are required") + return artifacts + + def _local_model_artifact( repository_root: Path, item: Mapping[str, object], diff --git a/tests/test_model_registry_coverage_closure.py b/tests/test_model_registry_coverage_closure.py index a5cf8c88..34ae2f7c 100644 --- a/tests/test_model_registry_coverage_closure.py +++ b/tests/test_model_registry_coverage_closure.py @@ -436,7 +436,7 @@ def test_registry_validation_covers_artifact_failure_modes(tmp_path: Path) -> No assert passed.execution_allowed is False assert passed.promotion_status == "RESEARCH_ONLY" assert passed.live_eligibility_status == "LIVE_ORDER_BLOCKED" - assert unverifiable.blockers == ("ARTIFACT_HASH_UNVERIFIABLE:coverage-model",) + assert unverifiable.blockers == ("LOCAL_MODEL_MANIFEST_INVALID:coverage-model",) assert outside.blockers == ("ARTIFACT_URI_OUTSIDE_REPOSITORY:coverage-model",) assert missing.blockers == ("ARTIFACT_MISSING:coverage-model",) From a66389496b33dd37bdb707ac72f87831e076bd8f Mon Sep 17 00:00:00 2001 From: Huseyin Cicek Date: Fri, 25 Sep 2026 15:48:59 +0300 Subject: [PATCH 12/19] feat: add by HsC --- .../compatibility/opportunity_monitor.py | 4 +- .../governance_enforcement_fabric.py | 2 +- src/ai4binance/internal_radar_vision.py | 17 ++++- .../test_governance_enforcement_fabric.py | 10 ++- tests/test_architecture_diagram_validation.py | 5 +- tests/test_internal_radar.py | 33 ++++++--- tests/test_internal_radar_vision.py | 74 +++++++++---------- tests/test_live_readiness_preview.py | 6 +- tests/test_lowest_twenty_coverage_models.py | 11 ++- tests/test_maintainability_ratchet.py | 10 ++- tests/test_market_data_gateway.py | 39 ++++++---- tests/test_public_showcase.py | 15 +++- tests/test_runtime_artifacts_migration.py | 10 +-- 13 files changed, 141 insertions(+), 95 deletions(-) diff --git a/src/ai4binance/compatibility/opportunity_monitor.py b/src/ai4binance/compatibility/opportunity_monitor.py index 6a385636..9d3bfc02 100644 --- a/src/ai4binance/compatibility/opportunity_monitor.py +++ b/src/ai4binance/compatibility/opportunity_monitor.py @@ -9,7 +9,7 @@ import json import re from collections import Counter -from collections.abc import Callable +from collections.abc import Callable, Mapping from dataclasses import replace from datetime import UTC, datetime from decimal import Decimal @@ -414,7 +414,7 @@ def _decimal(value: object) -> Decimal | None: return None -def _research_position_estimate(candidate: dict[str, object]) -> dict[str, object]: +def _research_position_estimate(candidate: Mapping[str, object]) -> dict[str, object]: """Return a bounded research sizing estimate without execution authority.""" if not has_complete_measurable_trade_plan(candidate): diff --git a/src/ai4binance/governance/governance_enforcement_fabric.py b/src/ai4binance/governance/governance_enforcement_fabric.py index 4215bcaa..161e94ab 100644 --- a/src/ai4binance/governance/governance_enforcement_fabric.py +++ b/src/ai4binance/governance/governance_enforcement_fabric.py @@ -582,7 +582,7 @@ def _quality_axis(name: str, value: object) -> GovernanceQualityAxis: def _quality_standard_mappings(value: object) -> dict[str, tuple[str, ...]]: payload = _mapping(value, "quality policy") - standard = _mapping(payload["standard_impact_tests"], "standard_impact_tests") + standard = _mapping(payload.get("standard_impact_tests"), "standard_impact_tests") mappings = standard.get("mappings") if not isinstance(mappings, list): raise ValueError("standard_impact_tests.mappings must be a list") diff --git a/src/ai4binance/internal_radar_vision.py b/src/ai4binance/internal_radar_vision.py index d1d69b30..79bc8ce2 100644 --- a/src/ai4binance/internal_radar_vision.py +++ b/src/ai4binance/internal_radar_vision.py @@ -16,7 +16,7 @@ from dataclasses import dataclass from hashlib import sha256 from pathlib import Path -from typing import Final, TypedDict +from typing import Final, Protocol, TypedDict from ai4binance.governance.local_model_roles import ( load_local_model_role, @@ -24,6 +24,7 @@ ) from ai4binance.governance.model_registry import ( ModelGateway, + ModelGatewayDecision, ModelInferenceEnvelope, ModelRouteDecision, build_advisory_inference_envelope, @@ -90,6 +91,18 @@ class _Observation(TypedDict): confidence: float +class AdvisoryModelGateway(Protocol): + """Minimal admission boundary required by the local vision adapter.""" + + def admit_advisory( + self, + model_id: str, + provider: str, + runtime_model: str, + task: str = "ADVISORY_RESEARCH_SYNTHESIS", + ) -> ModelGatewayDecision: ... + + @dataclass(frozen=True, slots=True) class VisionEvidence: """Validated categorical evidence emitted by the local vision sensor.""" @@ -165,7 +178,7 @@ class LlamaCppVisionRunner: base_url: str = "http://127.0.0.1:8081" timeout_seconds: float = 60.0 - model_gateway: ModelGateway | None = None + model_gateway: AdvisoryModelGateway | None = None repository_root: Path | None = None def analyze( diff --git a/tests/governance/terminology/test_governance_enforcement_fabric.py b/tests/governance/terminology/test_governance_enforcement_fabric.py index daeb0cfb..4e35f32e 100644 --- a/tests/governance/terminology/test_governance_enforcement_fabric.py +++ b/tests/governance/terminology/test_governance_enforcement_fabric.py @@ -337,7 +337,8 @@ def test_authority_graph_closes_all_fabric_family_paths() -> None: def test_fabric_contract_helpers_reject_untrusted_shapes(tmp_path: Path) -> None: - for value in (None, [], "text"): + invalid_mappings: tuple[object, ...] = (None, [], "text") + for value in invalid_mappings: with pytest.raises(ValueError, match="mapping"): fabric_module._mapping(value, "value") for value in (None, "", " "): @@ -355,7 +356,7 @@ def test_fabric_contract_helpers_reject_untrusted_shapes(tmp_path: Path) -> None plain = tmp_path / "plain.md" plain.write_text("no metadata", encoding="utf-8") assert fabric_module._frontmatter(plain) == {} - with pytest.raises(ValueError, match="retention"): + with pytest.raises(ValueError, match="standard_impact_tests"): fabric_module._quality_standard_mappings({}) @@ -462,7 +463,7 @@ def test_family_integrity_reports_missing_standard_projection_and_schema( def test_quality_mapping_parser_rejects_malformed_and_duplicate_entries() -> None: - for payload in ( + invalid_payloads: tuple[dict[str, object], ...] = ( {"standard_impact_tests": {}}, {"standard_impact_tests": {"mappings": [{"name": "x", "tests": "bad"}]}}, { @@ -473,7 +474,8 @@ def test_quality_mapping_parser_rejects_malformed_and_duplicate_entries() -> Non ] } }, - ): + ) + for payload in invalid_payloads: with pytest.raises( ValueError, match=r"mappings must be a list|mapping tests must be a list|duplicate", diff --git a/tests/test_architecture_diagram_validation.py b/tests/test_architecture_diagram_validation.py index 8d5e9970..d7e90014 100644 --- a/tests/test_architecture_diagram_validation.py +++ b/tests/test_architecture_diagram_validation.py @@ -2,6 +2,7 @@ import re from pathlib import Path +from typing import cast import pytest import yaml @@ -215,11 +216,11 @@ def test_validator_reports_all_registry_document_and_relation_failures( _write_document(tmp_path, path, "D999") entries = [ _entry(diagram_id="wrong"), - _entry(diagram_id="D001", path=123), + _entry(diagram_id="D001", path=cast(str, 123)), _entry(diagram_id="D002", path="../escape"), _entry(diagram_id="D003", path=path), _entry(diagram_id="D004", path=path), - _entry(diagram_id="D005", related_diagrams=[1]), + _entry(diagram_id="D005", related_diagrams=cast(list[str], [1])), ] _write_registry(tmp_path, entries) codes = _finding_codes(tmp_path) diff --git a/tests/test_internal_radar.py b/tests/test_internal_radar.py index d2cd9d3c..6ee44a03 100644 --- a/tests/test_internal_radar.py +++ b/tests/test_internal_radar.py @@ -5,12 +5,25 @@ import json from datetime import UTC, datetime from pathlib import Path +from typing import cast from ai4binance.governance.model_registry import build_advisory_inference_envelope from ai4binance.internal_radar import run_internal_radar_once from ai4binance.internal_radar_vision import VisionEvidence +def _payload_mapping(payload: dict[str, object], key: str) -> dict[str, object]: + value = payload[key] + assert isinstance(value, dict) + return cast(dict[str, object], value) + + +def _payload_candidates(payload: dict[str, object]) -> list[dict[str, object]]: + value = payload["candidates"] + assert isinstance(value, list) + return cast(list[dict[str, object]], value) + + def test_internal_radar_persists_redacted_new_image_candidate(tmp_path: Path) -> None: source = tmp_path / "images" source.mkdir() @@ -32,16 +45,15 @@ def test_internal_radar_persists_redacted_new_image_candidate(tmp_path: Path) -> assert "Last scan timestamp (UTC): `2026-09-18T00:00:00+00:00`" in markdown assert "[private-name.jpg](file://" in markdown assert "NOT_ASSESSED_WITHOUT_CONFIGURED_VISION_ANALYZER" in markdown - candidate = payload["candidates"][0] + candidate = _payload_candidates(payload)[0] assert isinstance(candidate, dict) assert "private-name" not in str(candidate) - privacy = payload["privacy"] - assert isinstance(privacy, dict) + privacy = _payload_mapping(payload, "privacy") assert privacy["source_paths_disclosed"] is False assert privacy["markdown_local_file_links_included"] is True assert payload["last_scan_timestamp_utc"] == "2026-09-18T00:00:00+00:00" assert "relative_path" not in candidate - assert payload["privacy"]["source_images_copied"] is False + assert privacy["source_images_copied"] is False assert payload["execution_allowed"] is False @@ -84,12 +96,13 @@ def analyze(self, **kwargs: object) -> VisionEvidence: vision_runner=BlockingVisionRunner(), # type: ignore[arg-type] ) - candidate = result.to_payload()["candidates"][0] + payload = result.to_payload() + candidate = _payload_candidates(payload)[0] assert isinstance(candidate, dict) assert candidate["assessment_status"] == "BLOCKED" assert "private-name" not in str(candidate) assert "UNREGISTERED_MODEL:local-llamacpp-qwen25vl-3b" in result.blockers - assert result.to_payload()["privacy"]["source_images_copied"] is False + assert _payload_mapping(payload, "privacy")["source_images_copied"] is False def test_internal_radar_vision_summary_only_counts_validated_observations( @@ -131,14 +144,14 @@ def analyze(self, **kwargs: object) -> VisionEvidence: vision_runner=ObservedVisionRunner(), # type: ignore[arg-type] ) - summary = result.to_payload()["vision_summary"] - assert isinstance(summary, dict) + payload = result.to_payload() + summary = _payload_mapping(payload, "vision_summary") assert summary["observed_count"] == 1 assert summary["benefit_categories"] == {"OPERATIONAL_VISIBILITY": 1} assert summary["tradeoff_categories"] == {"HUMAN_REVIEW_REQUIRED": 1} - progress = result.to_payload()["vision_progress"] + progress = payload["vision_progress"] assert progress == {"analysed": 1, "awaiting_analysis": 0} - candidate = result.to_payload()["candidates"][0] + candidate = _payload_candidates(payload)[0] assert isinstance(candidate, dict) assert str(candidate["last_scan_timestamp_utc"]).endswith("+00:00") markdown = result.latest_path.with_suffix(".md").read_text(encoding="utf-8") diff --git a/tests/test_internal_radar_vision.py b/tests/test_internal_radar_vision.py index 47725171..681247d7 100644 --- a/tests/test_internal_radar_vision.py +++ b/tests/test_internal_radar_vision.py @@ -4,14 +4,44 @@ import json from pathlib import Path +from typing import cast from urllib.request import Request import pytest -from ai4binance.governance.model_registry import ModelGateway +from ai4binance.governance.model_registry import ( + ModelGateway, + ModelGatewayDecision, + build_model_route_decision, +) from ai4binance.internal_radar_vision import LlamaCppVisionRunner, _prompt +class _AllowedGateway: + def admit_advisory( + self, + model_id: str, + provider: str, + runtime_model: str, + task: str = "ADVISORY_RESEARCH_SYNTHESIS", + ) -> ModelGatewayDecision: + del runtime_model + return ModelGatewayDecision( + model_id=model_id, + provider=provider, + allowed=True, + blockers=(), + route_decision=build_model_route_decision( + canonical_model_id=model_id, + provider=provider, + task_type=task, + model_family="LLM", + allowed=True, + blockers=(), + ), + ) + + def test_vision_runner_stays_advisory_when_local_provider_is_unavailable( tmp_path: Path, ) -> None: @@ -57,27 +87,6 @@ def test_vision_runner_persists_only_validated_categorical_evidence( ] } - class AllowedGateway: - def admit_advisory(self, *_args: object) -> object: - from ai4binance.governance.model_registry import build_model_route_decision - - return type( - "Decision", - (), - { - "allowed": True, - "blockers": (), - "route_decision": build_model_route_decision( - canonical_model_id="local-llamacpp-qwen25vl-3b", - provider="llama.cpp", - task_type="LOCAL_IMAGE_ADVISORY_ANALYSIS", - model_family="LLM", - allowed=True, - blockers=(), - ), - }, - )() - class Response: def read(self, _limit: int) -> bytes: return json.dumps(response).encode("utf-8") @@ -96,7 +105,7 @@ def fake_urlopen(request: Request, timeout: float) -> Response: return Response() monkeypatch.setattr("urllib.request.urlopen", fake_urlopen) - evidence = LlamaCppVisionRunner(model_gateway=AllowedGateway()).analyze( + evidence = LlamaCppVisionRunner(model_gateway=_AllowedGateway()).analyze( candidate_id="internal-image:0123456789abcdef", source_content_sha256="b" * 64, image_path=image, @@ -108,7 +117,8 @@ def fake_urlopen(request: Request, timeout: float) -> Response: assert payload["image_category"] == "DASHBOARD" assert payload["extracted_text_present"] is True assert "private-image" not in json.dumps(payload) - assert payload["privacy"]["raw_ocr_text_persisted"] is False + privacy = cast(dict[str, object], payload["privacy"]) + assert privacy["raw_ocr_text_persisted"] is False assert payload["execution_allowed"] is False @@ -118,10 +128,6 @@ def test_vision_runner_rejects_unstructured_model_output( image = tmp_path / "fixture.png" image.write_bytes(b"image-fixture") - class AllowedGateway: - def admit_advisory(self, *_args: object) -> object: - return type("Decision", (), {"allowed": True, "route_decision": None})() - class Response: def read(self, _limit: int) -> bytes: return b'{"choices":[{"message":{"content":"not-json"}}]}' @@ -133,7 +139,7 @@ def __exit__(self, *_args: object) -> None: return None monkeypatch.setattr("urllib.request.urlopen", lambda *_args, **_kwargs: Response()) - evidence = LlamaCppVisionRunner(model_gateway=AllowedGateway()).analyze( + evidence = LlamaCppVisionRunner(model_gateway=_AllowedGateway()).analyze( candidate_id="internal-image:0123456789abcdef", source_content_sha256="c" * 64, image_path=image, @@ -158,14 +164,6 @@ def test_vision_runner_accepts_one_json_code_fence( "confidence": 0.5, } - class AllowedGateway: - def admit_advisory(self, *_args: object) -> object: - return type( - "Decision", - (), - {"allowed": True, "blockers": (), "route_decision": None}, - )() - class Response: def read(self, _limit: int) -> bytes: content = "```json\n" + json.dumps(observation) + "\n```" @@ -178,7 +176,7 @@ def __exit__(self, *_args: object) -> None: return None monkeypatch.setattr("urllib.request.urlopen", lambda *_args, **_kwargs: Response()) - evidence = LlamaCppVisionRunner(model_gateway=AllowedGateway()).analyze( + evidence = LlamaCppVisionRunner(model_gateway=_AllowedGateway()).analyze( candidate_id="internal-image:0123456789abcdef", source_content_sha256="d" * 64, image_path=image, diff --git a/tests/test_live_readiness_preview.py b/tests/test_live_readiness_preview.py index 1f774c4d..89ea7722 100644 --- a/tests/test_live_readiness_preview.py +++ b/tests/test_live_readiness_preview.py @@ -18,6 +18,7 @@ from ai4binance.cli import live as cli_live from ai4binance.config import Settings from ai4binance.domain import LiveGateInput, ValidationStatus +from ai4binance.exchange import BinancePrivateAccountReader from ai4binance.exchange.models import BookTicker, MarketKline, SymbolInfo from ai4binance.execution import ( ExecutionAuthorizationEnvelope, @@ -1364,12 +1365,11 @@ def test_private_reader_uses_read_only_credentials_and_transport( ) -> None: credentials = object() monkeypatch.setattr( - cli_live.PrivateCredentials, - "from_environment_or_file", + "ai4binance.cli.live.PrivateCredentials.from_environment_or_file", lambda _path: credentials, ) reader = cli_live._private_reader(Settings()) - assert isinstance(reader, cli_live.BinancePrivateAccountReader) + assert isinstance(reader, BinancePrivateAccountReader) def test_live_place_blocks_invalid_or_mismatched_authorization_evidence( diff --git a/tests/test_lowest_twenty_coverage_models.py b/tests/test_lowest_twenty_coverage_models.py index 017b0e19..ac803a8f 100644 --- a/tests/test_lowest_twenty_coverage_models.py +++ b/tests/test_lowest_twenty_coverage_models.py @@ -90,8 +90,7 @@ def test_runtime_environment_rejects_unsafe_temp_contracts( ) -> None: directory = temporary_directory[0] monkeypatch.setattr( - ai4binance.tomllib, - "load", + "ai4binance.tomllib.load", lambda _stream: { "tool": {"ai4binance": {"runtime": {"temporary_directory": directory}}} }, @@ -158,8 +157,8 @@ def test_shadow_diff_cannot_widen_execution_authority() -> None: with pytest.raises(ValueError, match="cannot authorize"): DgeShadowDecisionDiff( shadow_rule_id="shadow-1", - baseline_decision=blocked, # type: ignore[arg-type] - shadow_decision=blocked, # type: ignore[arg-type] + baseline_decision=blocked, + shadow_decision=blocked, would_change_decision=False, changed_fields=(), execution_allowed=True, @@ -167,8 +166,8 @@ def test_shadow_diff_cannot_widen_execution_authority() -> None: with pytest.raises(ValueError, match="shadow rule id"): DgeShadowDecisionDiff( shadow_rule_id=" ", - baseline_decision=blocked, # type: ignore[arg-type] - shadow_decision=blocked, # type: ignore[arg-type] + baseline_decision=blocked, + shadow_decision=blocked, would_change_decision=False, changed_fields=(), ) diff --git a/tests/test_maintainability_ratchet.py b/tests/test_maintainability_ratchet.py index 974435d3..e5b6f81d 100644 --- a/tests/test_maintainability_ratchet.py +++ b/tests/test_maintainability_ratchet.py @@ -90,12 +90,18 @@ def test_collect_findings_and_evaluate_fail_closed_for_bad_tool_data( completed = type( "Completed", (), {"returncode": 2, "stderr": "tool failed", "stdout": ""} )() - monkeypatch.setattr(ratchet.subprocess, "run", lambda *_args, **_kwargs: completed) + monkeypatch.setattr( + "ai4binance.ops.maintainability_ratchet.subprocess.run", + lambda *_args, **_kwargs: completed, + ) with pytest.raises(RuntimeError, match="tool failed"): collect_ruff_findings(ROOT) malformed = type("Completed", (), {"returncode": 0, "stderr": "", "stdout": "{}"})() - monkeypatch.setattr(ratchet.subprocess, "run", lambda *_args, **_kwargs: malformed) + monkeypatch.setattr( + "ai4binance.ops.maintainability_ratchet.subprocess.run", + lambda *_args, **_kwargs: malformed, + ) with pytest.raises(ValueError, match="JSON array"): collect_ruff_findings(ROOT) diff --git a/tests/test_market_data_gateway.py b/tests/test_market_data_gateway.py index bbb6a459..7c3ed09a 100644 --- a/tests/test_market_data_gateway.py +++ b/tests/test_market_data_gateway.py @@ -9,6 +9,7 @@ from decimal import Decimal from pathlib import Path from types import SimpleNamespace +from typing import cast import pytest @@ -33,7 +34,7 @@ build_gateway, ) from ai4binance.exchange import rate_limit as rate_limit_module -from ai4binance.exchange.public_stream import SpotKlineUpdate +from ai4binance.exchange.public_stream import BinanceSpotKlineParser, SpotKlineUpdate from ai4binance.exchange.rate_limit import ( WeightedRateLimitGovernor, public_request_weight, @@ -236,7 +237,7 @@ def test_gateway_helpers_cover_invalid_and_bounded_inputs( path = tmp_path / "state.json" path.write_text('{"blockers": "invalid"}', encoding="utf-8") heartbeat = _GatewayStateHeartbeat(path, interval_seconds=0) - monkeypatch.setattr(gateway_cli.time, "monotonic", lambda: 1.0) + monkeypatch.setattr("ai4binance.cli.market_gateway.time.monotonic", lambda: 1.0) heartbeat(START) payload = json.loads(path.read_text(encoding="utf-8")) assert payload["blockers"] == ["MARKET_GATEWAY_STATE_BLOCKERS_INVALID"] @@ -254,7 +255,9 @@ def test_gateway_heartbeat_handles_read_failures_and_throttles_writes( path.write_text("not-json", encoding="utf-8") heartbeat = _GatewayStateHeartbeat(path, interval_seconds=10) monotonic = iter((10.0, 11.0)) - monkeypatch.setattr(gateway_cli.time, "monotonic", lambda: next(monotonic)) + monkeypatch.setattr( + "ai4binance.cli.market_gateway.time.monotonic", lambda: next(monotonic) + ) heartbeat(START) first = path.read_text(encoding="utf-8") heartbeat(START) @@ -607,8 +610,7 @@ def sync_cycle(self, *, observed_at: datetime) -> dict[str, object]: lambda _settings: synchronizer, ) monkeypatch.setattr( - gateway_cli.time, - "sleep", + "ai4binance.cli.market_gateway.time.sleep", lambda _seconds: None, ) settings = type( @@ -781,9 +783,9 @@ def flush(self, _observed_at: datetime) -> None: cache = _Cache() processor = CanonicalMarketStreamProcessor( "SPOT", - _Writer(), - cache, - monotonic=lambda: 1.0, # type: ignore[arg-type] + cast(DirectTimeframeWriter, _Writer()), + cast(SharedMarketCache, cache), + monotonic=lambda: 1.0, ) assert processor.process('{"e":"24hrTicker"}') == "CACHE_UPDATED" assert cache.flushed == 1 @@ -816,10 +818,10 @@ def process(self, _message: object) -> None: spot_processor = _Processor() futures_processor = _Processor() gateway = BinanceMarketDataGateway( - _Connection(), - _Connection(), - spot_processor, # type: ignore[arg-type] - futures_processor, # type: ignore[arg-type] + cast(CombinedStreamConnectionManager, _Connection()), + cast(CombinedStreamConnectionManager, _Connection()), + cast(CanonicalMarketStreamProcessor, spot_processor), + cast(CanonicalMarketStreamProcessor, futures_processor), ) assert asyncio.run(gateway.run_once()) == ("PLANNED_ROLLOVER", "PLANNED_ROLLOVER") assert spot_processor.cache.flushes == futures_processor.cache.flushes == 1 @@ -876,16 +878,21 @@ def test_gateway_rejects_invalid_inputs_and_processes_closed_kline( writes: list[tuple[str, str]] = [] processor = CanonicalMarketStreamProcessor( "SPOT", - SimpleNamespace( - append=lambda symbol, timeframe, *_: writes.append((symbol, timeframe)) + cast( + DirectTimeframeWriter, + SimpleNamespace( + append=lambda symbol, timeframe, *_: writes.append((symbol, timeframe)) + ), ), cache, - parser=SimpleNamespace(parse=lambda _: update), + parser=cast(BinanceSpotKlineParser, SimpleNamespace(parse=lambda _: update)), activity_observer=lambda _: None, ) assert processor.process('{"e":"kline"}') == "CLOSED_5M_APPLIED" assert writes == [("BTCUSDT", "5m")] - processor.parser = SimpleNamespace(parse=lambda _: object()) # type: ignore[assignment] + processor.parser = cast( + BinanceSpotKlineParser, SimpleNamespace(parse=lambda _: object()) + ) assert processor.process('{"e":"kline"}') == "RECONNECT_REQUIRED" diff --git a/tests/test_public_showcase.py b/tests/test_public_showcase.py index db93cf5d..6d636661 100644 --- a/tests/test_public_showcase.py +++ b/tests/test_public_showcase.py @@ -190,12 +190,14 @@ def test_showcase_scan_and_output_preconditions_fail_closed( executable = tmp_path / "gitleaks.exe" executable.write_text("fixture", encoding="utf-8") failed = type("Completed", (), {"returncode": 1})() - monkeypatch.setattr(showcase.subprocess, "run", lambda *_args, **_kwargs: failed) + monkeypatch.setattr( + "ai4binance.ops.public_showcase.subprocess.run", + lambda *_args, **_kwargs: failed, + ) with pytest.raises(PublicShowcaseError, match="failed"): showcase.run_gitleaks_scan(tmp_path, executable) monkeypatch.setattr( - showcase.subprocess, - "run", + "ai4binance.ops.public_showcase.subprocess.run", lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("unavailable")), ) with pytest.raises(PublicShowcaseError, match="did not complete"): @@ -236,7 +238,12 @@ def test_showcase_rejects_each_authority_expansion_shape(tmp_path: Path) -> None "sha256": "a" * 64, } assert showcase._parse_artifacts([valid])[0].source == "README.md" - for artifacts in ([{}], [{**valid, "sha256": "A" * 64}], [valid, valid]): + invalid_artifacts: tuple[list[dict[str, str]], ...] = ( + [{}], + [{**valid, "sha256": "A" * 64}], + [valid, valid], + ) + for artifacts in invalid_artifacts: with pytest.raises(PublicShowcaseError): showcase._parse_artifacts(artifacts) with pytest.raises(PublicShowcaseError, match="non-empty"): diff --git a/tests/test_runtime_artifacts_migration.py b/tests/test_runtime_artifacts_migration.py index 00a5d73d..6db88cdd 100644 --- a/tests/test_runtime_artifacts_migration.py +++ b/tests/test_runtime_artifacts_migration.py @@ -81,7 +81,7 @@ def test_layout_contract_helpers_fail_closed_and_canonicalize_aliases( with pytest.raises(ValueError, match="unknown runtime retention"): manifest.retention_for("unknown") assert layout._capacity_budget_mapping(None) == {} - invalid_budgets = [ + invalid_budgets: list[tuple[object, str]] = [ ([], "must be an object"), ({"outside": 1}, "must use runtime paths"), ({"runtime/a": True}, "must be an integer"), @@ -90,7 +90,7 @@ def test_layout_contract_helpers_fail_closed_and_canonicalize_aliases( for value, match in invalid_budgets: with pytest.raises(ValueError, match=match): layout._capacity_budget_mapping(value) - invalid_retention = [ + invalid_retention: list[tuple[object, str]] = [ ([], "retention must be an object"), ({"x": {}}, "automatic_cleanup must be boolean"), ] @@ -124,7 +124,7 @@ def test_runtime_retention_policy_rejects_invalid_values( def test_layout_mapping_helpers_cover_all_invalid_contract_shapes() -> None: - invalid_text_mappings = [ + invalid_text_mappings: list[tuple[object, str]] = [ (None, "must be a non-empty object"), ([], "must be a non-empty object"), ({}, "must be a non-empty object"), @@ -136,7 +136,7 @@ def test_layout_mapping_helpers_cover_all_invalid_contract_shapes() -> None: layout._text_mapping(value, "roots") with pytest.raises(ValueError, match="canonical_root must be a non-empty string"): layout._text({}, "canonical_root") - invalid_retention_mappings = [ + invalid_retention_mappings: list[tuple[object, str]] = [ ({"rule": []}, "entries must be named objects"), ({"": {}}, "entries must be named objects"), ({"rule": {"automatic_cleanup": "yes"}}, "must be boolean"), @@ -144,7 +144,7 @@ def test_layout_mapping_helpers_cover_all_invalid_contract_shapes() -> None: for value, match in invalid_retention_mappings: with pytest.raises(ValueError, match=match): layout._retention_mapping(value) - invalid_retention_policies = [ + invalid_retention_policies: list[tuple[dict[str, object], str]] = [ ( {"automatic_cleanup": True, "minimum_age_days": True, "keep_latest": 0}, "minimum_age_days must be an integer", From e2bb3beedad93bbafdba2754146645dfbc1cbe0f Mon Sep 17 00:00:00 2001 From: Huseyin Cicek Date: Fri, 25 Sep 2026 18:57:16 +0300 Subject: [PATCH 13/19] feat: expand market data coverage and harden approval replay --- .gitignore | 1 + .gitleaksignore | 2 + .../governed_document_lock_manifest.json | 46 +- .../technology_language_ownership.yaml | 4 +- config/operations/services.json | 2 +- .../ruff-maintainability-baseline.json | 2 + .../runtime_validation_deployment.json | 354 ++++++++++ .../research/virtual_market_acceptance.yaml | 99 +++ ...ference_global_terminology_and_taxonomy.md | 6 +- .../repository-validator/manifest-policy.json | 10 +- publication/README.md | 17 + .../export_governed_document_lock_approval.py | 95 ++- scripts/install_startup_task.ps1 | 129 +++- scripts/quality.ps1 | 10 +- ...overned_document_lock_approval_evidence.py | 10 +- src/ai4binance/agents/validation_gate.py | 27 +- src/ai4binance/application/research.py | 32 +- src/ai4binance/cli/futures_oos.py | 2 +- src/ai4binance/cli/market_data.py | 112 ++- src/ai4binance/cli/market_gateway.py | 132 ++-- src/ai4binance/cli/research.py | 37 +- src/ai4binance/cli/runtime.py | 290 +++++++- src/ai4binance/config.py | 11 +- src/ai4binance/data/acquisition.py | 10 +- src/ai4binance/data/market_depth.py | 32 +- .../data/market_history_continuous.py | 133 ++-- src/ai4binance/data/market_history_sync.py | 36 +- src/ai4binance/events/file_lock.py | 24 +- src/ai4binance/governance/adapters.py | 78 ++- .../governance/constitution_sync.py | 42 +- src/ai4binance/governance/dge_models.py | 30 +- src/ai4binance/governance/gate.py | 66 ++ .../governance_enforcement_fabric.py | 13 +- .../governance/repository_validator.py | 10 +- .../governance/technology_language_policy.py | 7 +- .../governance/terminology_policy.py | 9 +- .../infrastructure/persistence/safe_json.py | 7 +- src/ai4binance/local_dashboard/local_views.js | 9 +- src/ai4binance/local_dashboard/server.py.in | 7 + src/ai4binance/ops/architecture_migration.py | 4 + src/ai4binance/ops/public_showcase.py | 10 +- .../storage/destination_verification.py | 24 +- src/ai4binance/storage/jsonl.py | 9 + src/ai4binance/validation/oos_maturity.py | 654 +++++++++++++++++- src/ai4binance/validation_pipeline_runtime.py | 100 ++- src/ai4binance/virtual_wallet_journal.py | 120 ++++ .../test_technology_language_policy.py | 2 + .../terminology/test_terminology_policy.py | 5 +- tests/test_artifact_hygiene_scripts.py | 7 + tests/test_cli.py | 155 ++++- tests/test_config_reporting.py | 12 +- tests/test_data_acquisition.py | 12 +- tests/test_dge_engine.py | 21 + tests/test_dge_recovery_replay_shadow.py | 3 +- tests/test_docs_hygiene.py | 13 +- tests/test_event_journal.py | 61 ++ tests/test_futures_replay.py | 8 +- tests/test_governance_constitution_sync.py | 104 +++ tests/test_governance_gate.py | 62 ++ tests/test_kaizen_quality.py | 12 + tests/test_local_dashboard_source.py | 18 +- ...est_coverage_persistence_and_governance.py | 8 + ...est_coverage_technology_language_policy.py | 2 + tests/test_market_data_gateway.py | 36 +- tests/test_market_depth.py | 40 +- .../test_market_history_boundary_contracts.py | 8 +- tests/test_market_history_continuous.py | 192 ++++- tests/test_market_history_sync.py | 67 ++ tests/test_oos_maturity.py | 117 ++++ tests/test_opportunity_monitor.py | 2 +- tests/test_repository_cleanup_audit.py | 4 +- tests/test_repository_validator.py | 107 +++ tests/test_research_application.py | 2 + tests/test_security_tooling_contract.py | 2 +- tests/test_service_manifest.py | 11 +- tests/test_storage.py | 87 +++ tests/test_validation_pipeline.py | 54 ++ tests/test_virtual_wallet_journal.py | 58 ++ 78 files changed, 3816 insertions(+), 340 deletions(-) create mode 100644 config/research/runtime_validation_deployment.json diff --git a/.gitignore b/.gitignore index 08fd7e36..740f69cf 100644 --- a/.gitignore +++ b/.gitignore @@ -237,3 +237,4 @@ docs/reports/private/ /computer_local.md /docs/archive/reference_local_computer_profile.md /coverage.xml +/runtime/artifacts/assurance/security_tooling/ diff --git a/.gitleaksignore b/.gitleaksignore index 2e7e94af..60c10afb 100644 --- a/.gitleaksignore +++ b/.gitleaksignore @@ -11,3 +11,5 @@ de1c125880c20a0f8bef9f076fa7040441b5fafe:config/governance/governed_document_loc 7319e9b443e307c4de43b264ef4c7cfedd5c34c7:config/governance/governed_document_lock_manifest.json:generic-api-key:1630 7319e9b443e307c4de43b264ef4c7cfedd5c34c7:src/ai4binance/exchange/ws_api.py:generic-api-key:26 1ee98abdcb3bf6ce050fcc47514ff641c1489fe1:src/ai4binance/exchange/ws_api.py:generic-api-key:26 +6cccbb99632172df0e1a982035c9b84aafb93143:runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_full_gate_green_20260830_001.json:generic-api-key:24 +6cccbb99632172df0e1a982035c9b84aafb93143:runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_governed_docs_sha_alignment_20260826_001.json:generic-api-key:26 diff --git a/config/governance/governed_document_lock_manifest.json b/config/governance/governed_document_lock_manifest.json index f11b88f1..a54d18fa 100644 --- a/config/governance/governed_document_lock_manifest.json +++ b/config/governance/governed_document_lock_manifest.json @@ -515,7 +515,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_instruction_context_routing_20260904_001.json", - "approval_evidence_sha256": "5b41d0b5ef90cd989a6634a056d748e8e976bd704469698ac842d403ba617a8a" + "approval_evidence_sha256": "351c49e29b89a2ad818578114e2f6c897a82f6277ed03ab44cc13089a1164c47" }, { "approval_id": "AI4B-GOV-DOCLOCK-ROOT-AGENT-THIN-ROUTER-20260904-001", @@ -528,7 +528,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_root_agent_thin_router_20260904_001.json", - "approval_evidence_sha256": "bcaec3e32967a1fa573dabc480a55f68febef89572f43aa76c565e5a04e8a4f2" + "approval_evidence_sha256": "80d155af8aefa0db6797153085594d470a87e8e9903547974e7d7bb65aa83f42" }, { "approval_id": "AI4B-GOV-DOCLOCK-CUSTOM-VALIDATION-AUTHORITY-20260904-001", @@ -829,7 +829,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_root_agent_kaizen_20260903_001.json", - "approval_evidence_sha256": "7b0135d10a14831385b193c5616efff270dcb00425ee35f5f69a6d848d8ef7ef" + "approval_evidence_sha256": "b6fe64a03d4665beaf117498e90b620d25f3d51c50dde270d2e27aec9bb703c5" }, { "approval_id": "AI4B-GOV-DOCLOCK-QUALITY-GATE-PROFILES-20260901-001", @@ -899,7 +899,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_helper_process_containment_20260831_001.json", - "approval_evidence_sha256": "b0502415d12910651b47ada3ab962f16d958496708e5fb36e2e6073d3132f8ab" + "approval_evidence_sha256": "74ab95864028af2397e87303ae9ecefcf4403b0a2038bf4639abd289f29ee9fc" }, { "approval_id": "AI4B-GOV-DOCLOCK-UNIVERSAL-ENFORCEMENT-GOVERNANCE-CLOSURE-20260831-001", @@ -1059,7 +1059,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_authority_primitive_elevation_20260829_002.json", - "approval_evidence_sha256": "55c31dff7b08bcee0f0ef9c5621f466525cce2eb09fa026fa916169dbb34f572" + "approval_evidence_sha256": "937d308c01723d031bb1b82c63e1420ed44ec1ba17f3399e8778e32c0a58dc02" }, { "approval_id": "AI4B-GOV-DOCLOCK-DETERMINISTIC-GOVERNANCE-GATE-20260829-001", @@ -1094,7 +1094,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_deterministic_governance_gate_20260829_001.json", - "approval_evidence_sha256": "7956a83ad5011400fceb5779751205af698ae373cb8c816e3211349e804756fe" + "approval_evidence_sha256": "e5b60d200b1028360b0f3eaf364c6a6cf3f38c729588c2f4a7d31fba885b6862" }, { "approval_id": "AI4B-GOV-DOCLOCK-CORE-MANUAL-AUTONOMOUS-LANGUAGE-REMEDIATION-20260825-001", @@ -1232,7 +1232,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_quality_gate_revalidation_20260821_001.json", - "approval_evidence_sha256": "8903d950bdda25fcf3b26827fab942f458b49c4e503fd4264bc5553491dcde26" + "approval_evidence_sha256": "e1329b5f43638f012eca63a49f68ddaf10394b3823a6d373124656e90fc25ef8" }, { "approval_id": "AI4B-GOV-DOCLOCK-AGENTS-MARKDOWN-LOCK-RULE-20260822-001", @@ -1245,7 +1245,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_agents_markdown_lock_rule_20260822_001.json", - "approval_evidence_sha256": "238101968ab717c89d8de791a2d589ab6b835cfa16c537f8aa2d12ad242ab203" + "approval_evidence_sha256": "0d3b453cd7eb678c15b9592970fdd5b5f82c52e5df1fe48e71ebaa6893b1c9a0" }, { "approval_id": "AI4B-GOV-DOCLOCK-POLICY-AS-CODE-LOCK-CONTRACT-20260822-001", @@ -1516,7 +1516,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_governed_docs_sha_alignment_20260826_001.json", - "approval_evidence_sha256": "4cd648d702e0312228230e5fefc7e2fbdbc37c851dae82b2622eee22a3b5feb8" + "approval_evidence_sha256": "fd9f678be735c6a9d6071ae2cb781eede2f90e0b6d3fd5ac2b437d64b2cae7f5" }, { "approval_id": "AI4B-GOV-DOCLOCK-GOVERNED-DOCS-SHA-RESYNC-20260827-001", @@ -1543,7 +1543,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_governed_docs_sha_resync_20260827_001.json", - "approval_evidence_sha256": "aa3ba3ac844a7adbfc46f9b659cec135a39107cfa08f064026922348a042a99b" + "approval_evidence_sha256": "e796caf72d101591d5404a78dd57da5e4bb1db9ccbdee8a6d00562d60ff45a5c" }, { "approval_id": "AI4B-GOV-DOCLOCK-MARKDOWN-CLASSIFICATION-20260829-001", @@ -1681,7 +1681,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_full_gate_green_20260830_001.json", - "approval_evidence_sha256": "6b9942bd12d9cffb79fc40f0000e9026874bee0d20b7c8212f98b2c744735c74" + "approval_evidence_sha256": "f1e6db932d26192600ecaed1be6dd27cca432737d54087f2ff61ff89da89963f" }, { "approval_id": "AI4B-GOV-DOCLOCK-STRATEGY-REGISTRY-FAMILY-BINDING-20260831-001", @@ -1710,7 +1710,7 @@ "written_owner_approval": true, "approval_status": "APPROVED", "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_authority_primitive_canonicalization_20260830_002.json", - "approval_evidence_sha256": "cec0991ebb2c508e60cf6f9cf7a3a09dc7e11481435166c44d3c19b5829e720d" + "approval_evidence_sha256": "f3711978f9f888025e2e30eff591ea8a1941f88c995be3a2c94d4154d67f6804" }, { "approval_id": "AI4B-GOV-DOCLOCK-AUTHORITY-PRIMITIVE-CANONICALIZATION-20260830-003", @@ -1752,7 +1752,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_model_registry_architecture_20260904_001.json", - "approval_evidence_sha256": "c0627ee64c9e9f391d1d03ceab6154f1c670bb175a602007811cbd9bf4e24733" + "approval_evidence_sha256": "452634011050dba0329b87e76c64e29dd30dcfa43034f8b293e993cb8b9a811d" }, { "approval_id": "AI4B-GOV-DOCLOCK-MODEL-TAXONOMY-20260904-001", @@ -1765,7 +1765,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_model_taxonomy_20260904_001.json", - "approval_evidence_sha256": "e8f397d8eab13e975d5485f7679eb2d1025c61438bb4bc84ed3ef2d9e3818faa" + "approval_evidence_sha256": "e8c6ac236c18d0994fe0c423293a24109e4ba9f0c596290279790453f4aef411" }, { "approval_id": "AI4B-GOV-DOCLOCK-MODEL-INFERENCE-SCHEMA-20260904-001", @@ -1778,7 +1778,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_model_inference_schema_20260904_001.json", - "approval_evidence_sha256": "155fabe669633d33dddc2c3e64b89a0e3b98c9a78eb14d22c94bc3df669f9e27" + "approval_evidence_sha256": "a78a9a7f6b2ff6a8325740e2675ceba1aa21cad3af15851b30e60ce4df276b45" }, { "approval_id": "AI4B-GOV-DOCLOCK-MODEL-ROUTE-SCHEMA-20260904-001", @@ -1791,7 +1791,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_model_route_schema_20260904_001.json", - "approval_evidence_sha256": "adbc21afa84a82b30c9d62bbac2f7f23b77fe50aac29ae9aaaad903609cde9a2" + "approval_evidence_sha256": "6514446af7bd8ba11141a238ad920e6a2c1057e44195e6c92c409e5ad25b9471" }, { "approval_id": "AI4B-GOV-DOCLOCK-LOCAL-MODEL-GOVERNANCE-20260918-001", @@ -3952,7 +3952,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_instruction_context_routing_20260904_001.json", - "approval_evidence_sha256": "5b41d0b5ef90cd989a6634a056d748e8e976bd704469698ac842d403ba617a8a" + "approval_evidence_sha256": "351c49e29b89a2ad818578114e2f6c897a82f6277ed03ab44cc13089a1164c47" }, { "approval_id": "AI4B-GOV-DOCLOCK-ROOT-AGENT-THIN-ROUTER-20260904-001", @@ -3965,7 +3965,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_root_agent_thin_router_20260904_001.json", - "approval_evidence_sha256": "bcaec3e32967a1fa573dabc480a55f68febef89572f43aa76c565e5a04e8a4f2" + "approval_evidence_sha256": "80d155af8aefa0db6797153085594d470a87e8e9903547974e7d7bb65aa83f42" }, { "approval_id": "AI4B-GOV-DOCLOCK-CUSTOM-VALIDATION-AUTHORITY-20260904-001", @@ -4334,7 +4334,7 @@ "written_owner_approval": true, "approval_status": "APPROVED", "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_authority_primitive_canonicalization_20260830_002.json", - "approval_evidence_sha256": "cec0991ebb2c508e60cf6f9cf7a3a09dc7e11481435166c44d3c19b5829e720d" + "approval_evidence_sha256": "f3711978f9f888025e2e30eff591ea8a1941f88c995be3a2c94d4154d67f6804" }, { "approval_id": "AI4B-GOV-DOCLOCK-AUTHORITY-PRIMITIVE-CANONICALIZATION-20260830-003", @@ -4389,7 +4389,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_model_registry_architecture_20260904_001.json", - "approval_evidence_sha256": "c0627ee64c9e9f391d1d03ceab6154f1c670bb175a602007811cbd9bf4e24733" + "approval_evidence_sha256": "452634011050dba0329b87e76c64e29dd30dcfa43034f8b293e993cb8b9a811d" }, { "approval_id": "AI4B-GOV-DOCLOCK-MODEL-TAXONOMY-20260904-001", @@ -4402,7 +4402,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_model_taxonomy_20260904_001.json", - "approval_evidence_sha256": "e8f397d8eab13e975d5485f7679eb2d1025c61438bb4bc84ed3ef2d9e3818faa" + "approval_evidence_sha256": "e8c6ac236c18d0994fe0c423293a24109e4ba9f0c596290279790453f4aef411" }, { "approval_id": "AI4B-GOV-DOCLOCK-MODEL-INFERENCE-SCHEMA-20260904-001", @@ -4415,7 +4415,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_model_inference_schema_20260904_001.json", - "approval_evidence_sha256": "155fabe669633d33dddc2c3e64b89a0e3b98c9a78eb14d22c94bc3df669f9e27" + "approval_evidence_sha256": "a78a9a7f6b2ff6a8325740e2675ceba1aa21cad3af15851b30e60ce4df276b45" }, { "approval_id": "AI4B-GOV-DOCLOCK-MODEL-ROUTE-SCHEMA-20260904-001", @@ -4428,7 +4428,7 @@ }, "written_owner_approval": true, "approval_evidence_path": "runtime/artifacts/repository_validation/governance/governed_document_lock_approval_ai4b_gov_doclock_model_route_schema_20260904_001.json", - "approval_evidence_sha256": "adbc21afa84a82b30c9d62bbac2f7f23b77fe50aac29ae9aaaad903609cde9a2" + "approval_evidence_sha256": "6514446af7bd8ba11141a238ad920e6a2c1057e44195e6c92c409e5ad25b9471" }, { "approval_id": "AI4B-GOV-DOCLOCK-LOGICAL-ARCHITECTURE-REGISTRY-20260904-001", diff --git a/config/governance/technology_language_ownership.yaml b/config/governance/technology_language_ownership.yaml index 74c58d22..3c9e4ffb 100644 --- a/config/governance/technology_language_ownership.yaml +++ b/config/governance/technology_language_ownership.yaml @@ -44,7 +44,7 @@ languages: role: "canonical_application_language" suffixes: [".py", ".pyi"] enforce_placement: true - allowed_paths: [".agents/", "src/", "tests/", "scripts/", "tools/"] + allowed_paths: [".agents/", "src/", "tests/", "scripts/", "tools/", "publication/sanitize_publication.py"] allowed_capabilities: ["application_orchestration", "governance_integration", "risk_orchestration", "validation", "trading_intelligence"] forbidden_capabilities: ["live_execution_authority", "risk_override", "governance_bypass"] forbidden_content_patterns: [] @@ -169,7 +169,7 @@ languages: role: "declarative_configuration" suffixes: [".yaml", ".yml", ".toml"] enforce_placement: true - allowed_paths: [".agents/", ".codex/", ".github/", "config/", "docs/architecture/diagrams/diagram_registry.yaml", "docs/registries/", "workflows/", "pyproject.toml"] + allowed_paths: [".agents/", ".codex/", ".github/", "config/", "docs/architecture/diagrams/diagram_registry.yaml", "docs/registries/", "publication/public_manifest.yaml", "workflows/", "pyproject.toml"] allowed_capabilities: ["configuration", "declaration", "registry_projection"] forbidden_capabilities: ["executable_policy_engine", "hidden_business_logic"] forbidden_content_patterns: [] diff --git a/config/operations/services.json b/config/operations/services.json index 564a5929..4b3ee239 100644 --- a/config/operations/services.json +++ b/config/operations/services.json @@ -82,7 +82,7 @@ { "service": "market-history", "task_name": "AI4BINANCE-Market-History", - "command": "python -m ai4binance.cli.market_gateway", + "command": "python -m ai4binance.cli.market_data daemon", "mode": "RunMarketHistory", "required": true, "enabled": true, diff --git a/config/quality/ruff-maintainability-baseline.json b/config/quality/ruff-maintainability-baseline.json index f1aabe02..47f18409 100644 --- a/config/quality/ruff-maintainability-baseline.json +++ b/config/quality/ruff-maintainability-baseline.json @@ -59,6 +59,7 @@ "src/ai4binance/governance/gate.py", "src/ai4binance/governance/governance_enforcement_fabric.py", "src/ai4binance/governance/lean.py", + "src/ai4binance/governance/model_registry.py", "src/ai4binance/governance/repository_validator.py", "src/ai4binance/governance/risk_assessment.py", "src/ai4binance/governance/supply_chain.py", @@ -67,6 +68,7 @@ "src/ai4binance/governance/workflow.py", "src/ai4binance/historical_replay_evaluation.py", "src/ai4binance/historical_replay_state.py", + "src/ai4binance/internal_radar.py", "src/ai4binance/learning/engine.py", "src/ai4binance/learning/governance.py", "src/ai4binance/mcp/evidence.py", diff --git a/config/research/runtime_validation_deployment.json b/config/research/runtime_validation_deployment.json new file mode 100644 index 00000000..258782d2 --- /dev/null +++ b/config/research/runtime_validation_deployment.json @@ -0,0 +1,354 @@ +{ + "execution_allowed": false, + "live_eligibility_status": "LIVE_ORDER_BLOCKED", + "promotion_status": "RESEARCH_ONLY", + "runtime_source_sha256": "bcfbc13c104caa2fbcde590252b262e5c0133b38f1bfe9493574f32cb79220a9", + "schema_version": "1.0", + "subjects": [ + { + "bundle": { + "path": "oos_runtime/542ca70809c81c41839c003d50c8e83f14227eba302031e8d72a9c2320193902/bundle.json", + "sha256": "0ea2b993bdb38844ec882a70d0704bdda0e3ea4ca68590834af3388c03a2188d" + }, + "subject": { + "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", + "feature_definition_sha256": "63f712ab32697b7a9c67ee81847bcdc9f7171002e2f7e4e6a213ae2e12edd979", + "promotion": { + "code_revision": "bcfbc13c104caa2fbcde590252b262e5c0133b38f1bfe9493574f32cb79220a9", + "dataset_sha256": "670c3909a381f375b0a99fcef256378c5b3892b2eefbde3167300dc91ed561b5", + "market_type": "SPOT", + "parameter_set_sha256": "4bd1c4a9aa115e27998e0721c7700ae6a1d8a8cf09af95d34b3ce624edda1ac9", + "strategy_id": "trend_continuation", + "strategy_sha256": "63f712ab32697b7a9c67ee81847bcdc9f7171002e2f7e4e6a213ae2e12edd979", + "strategy_version": "1", + "symbol": "BTCUSDT", + "timeframe": "15m" + }, + "setup_type": "trend_continuation", + "validation_config_sha256": "41f2ae9107a79985e46f8522323133261f99fb4ec9d9d4265b0f57c3d3956cd4" + } + }, + { + "bundle": { + "path": "oos_runtime/0c17e1a8754613e9607e63e434e93349b83f0dd6da00a044165ad532eae358e0/bundle.json", + "sha256": "c082ac3615f22fdb10f8662a45cc357d642e87fbb39c94730034184d2fa774bc" + }, + "subject": { + "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", + "feature_definition_sha256": "acfb727377466011e73a0b3782f76bc445b3dc7d7fbddda33c8b0811370dc852", + "promotion": { + "code_revision": "bcfbc13c104caa2fbcde590252b262e5c0133b38f1bfe9493574f32cb79220a9", + "dataset_sha256": "670c3909a381f375b0a99fcef256378c5b3892b2eefbde3167300dc91ed561b5", + "market_type": "SPOT", + "parameter_set_sha256": "4bd1c4a9aa115e27998e0721c7700ae6a1d8a8cf09af95d34b3ce624edda1ac9", + "strategy_id": "pullback_continuation", + "strategy_sha256": "acfb727377466011e73a0b3782f76bc445b3dc7d7fbddda33c8b0811370dc852", + "strategy_version": "1", + "symbol": "BTCUSDT", + "timeframe": "15m" + }, + "setup_type": "pullback_continuation", + "validation_config_sha256": "3fc299291cbb97b9cc53bd413af44289eb93da55ec2733ad415c82f871865a2f" + } + }, + { + "bundle": { + "path": "oos_runtime/1cd1f756c2dfe2e6e5b72ad580f250a1f5c6f10038e9a51acc41a74a966c204e/bundle.json", + "sha256": "03f1c4e0d8e776bb758f255c3945694a2ce33b722d43f21de485b007f3edebc0" + }, + "subject": { + "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", + "feature_definition_sha256": "2a7e0ad5ccc7292a0b06b76b233fa93552d34bfe993a0db4976e3404b1ff7486", + "promotion": { + "code_revision": "bcfbc13c104caa2fbcde590252b262e5c0133b38f1bfe9493574f32cb79220a9", + "dataset_sha256": "670c3909a381f375b0a99fcef256378c5b3892b2eefbde3167300dc91ed561b5", + "market_type": "SPOT", + "parameter_set_sha256": "4bd1c4a9aa115e27998e0721c7700ae6a1d8a8cf09af95d34b3ce624edda1ac9", + "strategy_id": "breakout_retest", + "strategy_sha256": "2a7e0ad5ccc7292a0b06b76b233fa93552d34bfe993a0db4976e3404b1ff7486", + "strategy_version": "1", + "symbol": "BTCUSDT", + "timeframe": "15m" + }, + "setup_type": "breakout_retest", + "validation_config_sha256": "3580cd32f763b70502b44c2d474bc935e0bfe1c85b4401341de1f063ce6d5562" + } + }, + { + "bundle": { + "path": "oos_runtime/8262f89da3dece8463ff55097d2933306fa4aca927d58c0383b2288156f2f5ad/bundle.json", + "sha256": "8126174a6f065fb927444983023d7ec9dcb76928147de1e18465555fe8d0bb39" + }, + "subject": { + "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", + "feature_definition_sha256": "7cfbd10b251a8e6acfa0656247e9f99f56c4b6e24849c8a36d66fce734f746a3", + "promotion": { + "code_revision": "bcfbc13c104caa2fbcde590252b262e5c0133b38f1bfe9493574f32cb79220a9", + "dataset_sha256": "670c3909a381f375b0a99fcef256378c5b3892b2eefbde3167300dc91ed561b5", + "market_type": "SPOT", + "parameter_set_sha256": "4bd1c4a9aa115e27998e0721c7700ae6a1d8a8cf09af95d34b3ce624edda1ac9", + "strategy_id": "support_reclaim", + "strategy_sha256": "7cfbd10b251a8e6acfa0656247e9f99f56c4b6e24849c8a36d66fce734f746a3", + "strategy_version": "1", + "symbol": "BTCUSDT", + "timeframe": "15m" + }, + "setup_type": "support_reclaim", + "validation_config_sha256": "5bfc944927f384c71d5a567195d4f9f3fab658cf15ef634a4009b6ca0d8276b7" + } + }, + { + "bundle": { + "path": "oos_runtime/a2cd1cdce2adf647174ca45f03075535c7fcb9b0c05313b27f9f3ae7ed9a4d42/bundle.json", + "sha256": "1f87af0f67f5250fc737428dd9cfa34e6fde3571a1df5af0f85665fc273bf1b5" + }, + "subject": { + "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", + "feature_definition_sha256": "4f33ee776ac3b463420d6a0f2c08352a0673d16633d2a32765d861fcb8e144ea", + "promotion": { + "code_revision": "bcfbc13c104caa2fbcde590252b262e5c0133b38f1bfe9493574f32cb79220a9", + "dataset_sha256": "670c3909a381f375b0a99fcef256378c5b3892b2eefbde3167300dc91ed561b5", + "market_type": "SPOT", + "parameter_set_sha256": "4bd1c4a9aa115e27998e0721c7700ae6a1d8a8cf09af95d34b3ce624edda1ac9", + "strategy_id": "failed_breakout_reversal", + "strategy_sha256": "4f33ee776ac3b463420d6a0f2c08352a0673d16633d2a32765d861fcb8e144ea", + "strategy_version": "1", + "symbol": "BTCUSDT", + "timeframe": "15m" + }, + "setup_type": "failed_breakout_reversal", + "validation_config_sha256": "af093bbf27c520339541d1f4dc6cbd416cb7d370216e64e70164e02ce9a64582" + } + }, + { + "bundle": { + "path": "oos_runtime/fcfc4e58d7a510d8a7598d4006dd736d3538480696988e509653d589b254feba/bundle.json", + "sha256": "ac989b936304b50caad60375adff155c1cc6567568a20cf92c3d236f9fa217fb" + }, + "subject": { + "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", + "feature_definition_sha256": "63f712ab32697b7a9c67ee81847bcdc9f7171002e2f7e4e6a213ae2e12edd979", + "promotion": { + "code_revision": "bcfbc13c104caa2fbcde590252b262e5c0133b38f1bfe9493574f32cb79220a9", + "dataset_sha256": "a331860e84a646b5fca88d9e3cd0b0741e48eb9a1012cb127d4dbee636a59dfa", + "market_type": "SPOT", + "parameter_set_sha256": "054e2cde500c74e3de0cf53a0ea4cc187d205f1cbcdad5598459081a5c6e6be4", + "strategy_id": "trend_continuation", + "strategy_sha256": "63f712ab32697b7a9c67ee81847bcdc9f7171002e2f7e4e6a213ae2e12edd979", + "strategy_version": "1", + "symbol": "BTCUSDT", + "timeframe": "1h" + }, + "setup_type": "trend_continuation", + "validation_config_sha256": "ffb237f4724bd11617cf1bfbf21c900542a5e42f7c2a4cea9a15c036ae01be0a" + } + }, + { + "bundle": { + "path": "oos_runtime/c8ca129ca37d86dbba6f7e084565b52b9376c57f56a21ee327d73860b9f0d8e4/bundle.json", + "sha256": "b6180f55d27849bce4647b353e6468771eedd73e596b406399051dc594d2a9e4" + }, + "subject": { + "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", + "feature_definition_sha256": "d676e5ee3551bed7965e95cb421ae999225b9b6c0e2662468065506836fe5fc7", + "promotion": { + "code_revision": "bcfbc13c104caa2fbcde590252b262e5c0133b38f1bfe9493574f32cb79220a9", + "dataset_sha256": "a331860e84a646b5fca88d9e3cd0b0741e48eb9a1012cb127d4dbee636a59dfa", + "market_type": "SPOT", + "parameter_set_sha256": "054e2cde500c74e3de0cf53a0ea4cc187d205f1cbcdad5598459081a5c6e6be4", + "strategy_id": "pullback_continuation", + "strategy_sha256": "d676e5ee3551bed7965e95cb421ae999225b9b6c0e2662468065506836fe5fc7", + "strategy_version": "1", + "symbol": "BTCUSDT", + "timeframe": "1h" + }, + "setup_type": "pullback_continuation", + "validation_config_sha256": "136713eaba26dde6d5dd4394f075b7961334b6bb5b58d3a4b64b32002083ed21" + } + }, + { + "bundle": { + "path": "oos_runtime/d46c8a4d6034d44bcf58ed60c82fd61afcb835273bb524bff27ce46073503baf/bundle.json", + "sha256": "fcd53ac8b8ac14b71a49b8b91d7ccb9c51db0ee98684ff1904d059da68c0f86b" + }, + "subject": { + "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", + "feature_definition_sha256": "2a7e0ad5ccc7292a0b06b76b233fa93552d34bfe993a0db4976e3404b1ff7486", + "promotion": { + "code_revision": "bcfbc13c104caa2fbcde590252b262e5c0133b38f1bfe9493574f32cb79220a9", + "dataset_sha256": "a331860e84a646b5fca88d9e3cd0b0741e48eb9a1012cb127d4dbee636a59dfa", + "market_type": "SPOT", + "parameter_set_sha256": "054e2cde500c74e3de0cf53a0ea4cc187d205f1cbcdad5598459081a5c6e6be4", + "strategy_id": "breakout_retest", + "strategy_sha256": "2a7e0ad5ccc7292a0b06b76b233fa93552d34bfe993a0db4976e3404b1ff7486", + "strategy_version": "1", + "symbol": "BTCUSDT", + "timeframe": "1h" + }, + "setup_type": "breakout_retest", + "validation_config_sha256": "4299b3871fba400b20b27f75d4d8754093ba57d7ac249cf0b579eb0f863a4eb4" + } + }, + { + "bundle": { + "path": "oos_runtime/8ec344bbae4e35b0f3b76947b74364300ad628f8a8136d795daab00ce187dfe4/bundle.json", + "sha256": "59ca38f180e393abec1f6bc419384d35e54456fd61c1fd4652eb370b3170c917" + }, + "subject": { + "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", + "feature_definition_sha256": "7cfbd10b251a8e6acfa0656247e9f99f56c4b6e24849c8a36d66fce734f746a3", + "promotion": { + "code_revision": "bcfbc13c104caa2fbcde590252b262e5c0133b38f1bfe9493574f32cb79220a9", + "dataset_sha256": "a331860e84a646b5fca88d9e3cd0b0741e48eb9a1012cb127d4dbee636a59dfa", + "market_type": "SPOT", + "parameter_set_sha256": "054e2cde500c74e3de0cf53a0ea4cc187d205f1cbcdad5598459081a5c6e6be4", + "strategy_id": "support_reclaim", + "strategy_sha256": "7cfbd10b251a8e6acfa0656247e9f99f56c4b6e24849c8a36d66fce734f746a3", + "strategy_version": "1", + "symbol": "BTCUSDT", + "timeframe": "1h" + }, + "setup_type": "support_reclaim", + "validation_config_sha256": "c77f1ef41b693700b54a48e9eef88b3563aa0d059d524c75f96fee1a7cd511e9" + } + }, + { + "bundle": { + "path": "oos_runtime/64cfbb8bd90688a6617f6c71f4bf20db19f57c150edaff178903fe3db5f9126c/bundle.json", + "sha256": "d3007d8bc5a29fb4a5e8be97c3c34aaa25ed072800dfc5d9b0706d2b31578514" + }, + "subject": { + "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", + "feature_definition_sha256": "4f33ee776ac3b463420d6a0f2c08352a0673d16633d2a32765d861fcb8e144ea", + "promotion": { + "code_revision": "bcfbc13c104caa2fbcde590252b262e5c0133b38f1bfe9493574f32cb79220a9", + "dataset_sha256": "a331860e84a646b5fca88d9e3cd0b0741e48eb9a1012cb127d4dbee636a59dfa", + "market_type": "SPOT", + "parameter_set_sha256": "054e2cde500c74e3de0cf53a0ea4cc187d205f1cbcdad5598459081a5c6e6be4", + "strategy_id": "failed_breakout_reversal", + "strategy_sha256": "4f33ee776ac3b463420d6a0f2c08352a0673d16633d2a32765d861fcb8e144ea", + "strategy_version": "1", + "symbol": "BTCUSDT", + "timeframe": "1h" + }, + "setup_type": "failed_breakout_reversal", + "validation_config_sha256": "7893795b4a3991b17d69e613eb88150fcbbcaf251a41931315c0b30c31a6f5df" + } + }, + { + "bundle": { + "path": "oos_runtime/f2d1caf9eb552061c08e1b528ad612077e9e0d2976f6a92fc5397edbcd3606dc/bundle.json", + "sha256": "be158e4bb59097fabe96054cacd07b9e414146eb6593256e58e262a12295e209" + }, + "subject": { + "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", + "feature_definition_sha256": "63f712ab32697b7a9c67ee81847bcdc9f7171002e2f7e4e6a213ae2e12edd979", + "promotion": { + "code_revision": "bcfbc13c104caa2fbcde590252b262e5c0133b38f1bfe9493574f32cb79220a9", + "dataset_sha256": "058e3458745e76d6202a762be480eec089e6949040cff4f391f89378129a84fc", + "market_type": "SPOT", + "parameter_set_sha256": "0aa632ce16ad795f3fd5e594e0b0658b4c89c6d2c7266c25f322f5730b651b37", + "strategy_id": "trend_continuation", + "strategy_sha256": "63f712ab32697b7a9c67ee81847bcdc9f7171002e2f7e4e6a213ae2e12edd979", + "strategy_version": "1", + "symbol": "BTCUSDT", + "timeframe": "4h" + }, + "setup_type": "trend_continuation", + "validation_config_sha256": "ecf31c5205fcfa07c47ba7ae8dd2c243a9f8a23d41c802d651ee4d0e0dc04600" + } + }, + { + "bundle": { + "path": "oos_runtime/b71f4edf1f000796eed60b671d542fb9dfe6ba9543df35a018aa102d02fd26b2/bundle.json", + "sha256": "ffafd48a9fc66d94a7aaa9ee5cc365426878bdf6baa6d080dd5add1016913f12" + }, + "subject": { + "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", + "feature_definition_sha256": "d676e5ee3551bed7965e95cb421ae999225b9b6c0e2662468065506836fe5fc7", + "promotion": { + "code_revision": "bcfbc13c104caa2fbcde590252b262e5c0133b38f1bfe9493574f32cb79220a9", + "dataset_sha256": "058e3458745e76d6202a762be480eec089e6949040cff4f391f89378129a84fc", + "market_type": "SPOT", + "parameter_set_sha256": "0aa632ce16ad795f3fd5e594e0b0658b4c89c6d2c7266c25f322f5730b651b37", + "strategy_id": "pullback_continuation", + "strategy_sha256": "d676e5ee3551bed7965e95cb421ae999225b9b6c0e2662468065506836fe5fc7", + "strategy_version": "1", + "symbol": "BTCUSDT", + "timeframe": "4h" + }, + "setup_type": "pullback_continuation", + "validation_config_sha256": "957010c5cd007f49b8b3c2c88b470e2c7fd7271bba128f9182c785e318351323" + } + }, + { + "bundle": { + "path": "oos_runtime/d84376c1324fe18339bd18a593973a573a8e0e89301838a485821498f3193deb/bundle.json", + "sha256": "397e7678228fd78376414cd4d28f559c56103c5e49c63dab4ff4e342b899f9a6" + }, + "subject": { + "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", + "feature_definition_sha256": "81ed5bdf546ea4521d379049a78ecd0c518945f72b7414517439d2e976d59c40", + "promotion": { + "code_revision": "bcfbc13c104caa2fbcde590252b262e5c0133b38f1bfe9493574f32cb79220a9", + "dataset_sha256": "058e3458745e76d6202a762be480eec089e6949040cff4f391f89378129a84fc", + "market_type": "SPOT", + "parameter_set_sha256": "0aa632ce16ad795f3fd5e594e0b0658b4c89c6d2c7266c25f322f5730b651b37", + "strategy_id": "breakout_retest", + "strategy_sha256": "81ed5bdf546ea4521d379049a78ecd0c518945f72b7414517439d2e976d59c40", + "strategy_version": "1", + "symbol": "BTCUSDT", + "timeframe": "4h" + }, + "setup_type": "breakout_retest", + "validation_config_sha256": "1acd73838e91895ec9338686015db2fc482100d95b9050b3b0badebfefa99711" + } + }, + { + "bundle": { + "path": "oos_runtime/f6a8dd030e25e1c6462c4d73d9174cc46e9ab5888b0e24b3bfbda167332bc205/bundle.json", + "sha256": "5835a4f21b547c509aff9a593dd031fc6807136726738fcd4cde52a578df4b4e" + }, + "subject": { + "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", + "feature_definition_sha256": "ab8ad8719e8ae46a24c00a9fa61b06b65ec081f6282f532f3e6fd5922d517a07", + "promotion": { + "code_revision": "bcfbc13c104caa2fbcde590252b262e5c0133b38f1bfe9493574f32cb79220a9", + "dataset_sha256": "058e3458745e76d6202a762be480eec089e6949040cff4f391f89378129a84fc", + "market_type": "SPOT", + "parameter_set_sha256": "0aa632ce16ad795f3fd5e594e0b0658b4c89c6d2c7266c25f322f5730b651b37", + "strategy_id": "support_reclaim", + "strategy_sha256": "ab8ad8719e8ae46a24c00a9fa61b06b65ec081f6282f532f3e6fd5922d517a07", + "strategy_version": "1", + "symbol": "BTCUSDT", + "timeframe": "4h" + }, + "setup_type": "support_reclaim", + "validation_config_sha256": "18a642dcfb07e6a88492bbd3f653860c32147a51283052a13cdfe6878fc9e581" + } + }, + { + "bundle": { + "path": "oos_runtime/c9ee64e48fadb8e3a48fc2623a82bbf4e4a411a2e82babd8c6a7fca01ba55530/bundle.json", + "sha256": "627089c7b2d78787ad34b41e683b33caa9e7d160e85184d65811435169fe7ca2" + }, + "subject": { + "cost_model_sha256": "61ee39e5fcb94757b65d633a3e10185a94809a326c076daa4c0a7c8574301804", + "feature_definition_sha256": "4f33ee776ac3b463420d6a0f2c08352a0673d16633d2a32765d861fcb8e144ea", + "promotion": { + "code_revision": "bcfbc13c104caa2fbcde590252b262e5c0133b38f1bfe9493574f32cb79220a9", + "dataset_sha256": "058e3458745e76d6202a762be480eec089e6949040cff4f391f89378129a84fc", + "market_type": "SPOT", + "parameter_set_sha256": "0aa632ce16ad795f3fd5e594e0b0658b4c89c6d2c7266c25f322f5730b651b37", + "strategy_id": "failed_breakout_reversal", + "strategy_sha256": "4f33ee776ac3b463420d6a0f2c08352a0673d16633d2a32765d861fcb8e144ea", + "strategy_version": "1", + "symbol": "BTCUSDT", + "timeframe": "4h" + }, + "setup_type": "failed_breakout_reversal", + "validation_config_sha256": "ef9f58aad8bec69122a1a7741a40a3f15d189a8496d63da8e4b65b095f1e9d77" + } + } + ] +} diff --git a/config/research/virtual_market_acceptance.yaml b/config/research/virtual_market_acceptance.yaml index 6c3ffaf2..c574b25b 100644 --- a/config/research/virtual_market_acceptance.yaml +++ b/config/research/virtual_market_acceptance.yaml @@ -37,6 +37,105 @@ anti_masking: combined_pnl_gate_allowed: false combined_sharpe_gate_allowed: false combined_equity_gate_allowed: false +spot_oos_validation_specification: + schema_version: "1.0" + specification_id: spot-oos-validation-v1 + status: ACTIVE + owner: Validation Owner + approval_status: PENDING_INDEPENDENT_REVIEW + market_scope: + - SPOT + timeframes: + - 15m + - 1h + - 4h + strategy_version: "1" + requirements: + dataset: + blockers: {operator: eq, value: []} + point_in_time_status: {operator: eq, value: PASS} + quality_status: {operator: eq, value: PASS} + revision_status: {operator: eq, value: IMMUTABLE} + missing_intervals: {operator: eq, value: 0} + duplicate_count: {operator: eq, value: 0} + observation_days: {operator: gte, value: 365} + cost_data_coverage: {operator: gte, value: 0.99} + backtest: + blockers: {operator: eq, value: []} + realism_status: {operator: eq, value: PASS} + cost_model_sha256: {operator: eq, value: "$COST_MODEL_SHA256"} + execution_model_version: {operator: eq, value: SPOT_BACKTEST_V1} + walk_forward: + blockers: {operator: eq, value: []} + fold_count: {operator: gte, value: 5} + train_only_selection: {operator: eq, value: true} + purge: {operator: gte, value: 1} + embargo: {operator: gte, value: 1} + final_holdout: + blockers: {operator: eq, value: []} + trade_count: {operator: gte, value: 100} + expectancy: {operator: gt, value: 0} + net_return: {operator: gt, value: 0} + max_drawdown: {operator: lte, value: 0.15} + regime: + blockers: {operator: eq, value: []} + regime_count: {operator: gte, value: 3} + unknown_ratio: {operator: lte, value: 0.05} + attribution_status: {operator: eq, value: PASS} + robustness: + blockers: {operator: eq, value: []} + parameter_stability: {operator: eq, value: PASS} + fold_stability: {operator: eq, value: PASS} + regime_stability: {operator: eq, value: PASS} + cpcv_status: {operator: eq, value: PASS} + monte_carlo_status: {operator: eq, value: PASS} + selection_overfit_status: {operator: eq, value: PASS} + edge_concentration: {operator: lte, value: 0.5} + failure_modes_status: {operator: eq, value: PASS} + statistics: + blockers: {operator: eq, value: []} + effective_sample_size: {operator: gte, value: 20} + confidence_interval_lower: {operator: gt, value: 0} + profit_factor: {operator: gte, value: 1.25} + profitable_fold_ratio: {operator: gte, value: 0.6} + hypothesis_count: {operator: gte, value: 1} + multiple_testing_method: + operator: in + value: [BONFERRONI, HOLM, SPA, MCS] + confirmatory: {operator: eq, value: true} + cost_stress: + blockers: {operator: eq, value: []} + scenario_coverage: + operator: eq + value: [BASE, COST_1_5X, COST_2X] + edge_survival: {operator: eq, value: true} + replication: + blockers: {operator: eq, value: []} + declared_scope_coverage: {operator: eq, value: true} + stability_status: {operator: eq, value: PASS} + paper_forward: + blockers: {operator: eq, value: []} + historical_oos_completed_at: {operator: present, value: true} + observation_started_at: {operator: present, value: true} + observation_ended_at: {operator: present, value: true} + observation_days: {operator: gte, value: 365} + trade_count: {operator: gte, value: 100} + frequency_comparison: {operator: eq, value: PASS} + cost_comparison: {operator: eq, value: PASS} + expectancy_comparison: {operator: eq, value: PASS} + drawdown_comparison: {operator: eq, value: PASS} + regime_comparison: {operator: eq, value: PASS} + signal_decay_status: {operator: eq, value: PASS} + runtime_failures: {operator: eq, value: 0} + reproducibility: + blockers: {operator: eq, value: []} + repository_clean: {operator: eq, value: true} + replay_equal: {operator: eq, value: true} + implementation_sha256: {operator: eq, value: "$RUNTIME_SOURCE_SHA256"} + authority: + execution_allowed: false + promotion_status: RESEARCH_ONLY + live_eligibility_status: LIVE_ORDER_BLOCKED authority: execution_allowed: false promotion_status: RESEARCH_ONLY diff --git a/docs/references/reference_global_terminology_and_taxonomy.md b/docs/references/reference_global_terminology_and_taxonomy.md index 0c00af38..d41113a3 100644 --- a/docs/references/reference_global_terminology_and_taxonomy.md +++ b/docs/references/reference_global_terminology_and_taxonomy.md @@ -2,7 +2,7 @@ document_id: AI4B-GOV-REF-TERM-TAX-001 title: AI4BINANCE Global Terminology and Taxonomy Reference document_type: REFERENCE -version: 1.0.1 +version: 1.0.2 status: DRAFT owner: Enterprise Knowledge Governance authority_level: REFERENCE @@ -36,6 +36,10 @@ This reference preserves the project-wide terminology families, decision vocabul Normative authority remains in `docs/standards/standard_terminology_governance.md`. +The deterministic terminology projection and validation mechanics are implemented by +`src/ai4binance/governance/terminology_policy.py`. The implementation remains +non-authoritative and cannot widen the standard or the shared enforcement fabric. + | Output file | Original sections | |---|---| | `standard_terminology_governance.md` | Sections 1-6, 7, 8, 9-12 | diff --git a/policies/repository-validator/manifest-policy.json b/policies/repository-validator/manifest-policy.json index 3f7525e4..c97dee4f 100644 --- a/policies/repository-validator/manifest-policy.json +++ b/policies/repository-validator/manifest-policy.json @@ -1,14 +1,16 @@ { "policy_id": "AI4B-GOV-REPO-POLICY", - "version": "1.3.0", + "version": "1.3.1", "allowed_top_level_paths": [ ".env.example", ".gitattributes", + ".gitleaksignore", ".gitignore", ".python-version", "AGENTS.md", "CLAUDE.md", "GEMINI.md", + "LICENSE", "README.md", "pyproject.toml", "uv.lock", @@ -28,6 +30,8 @@ ".venv", ".vscode", "factory", + "examples", + "publication", "requirements.txt", "research", "alerts", @@ -195,6 +199,10 @@ ] }, "owner_by_top_level": { + ".gitleaksignore": "Security", + "LICENSE": "Governance", + "examples": "Documentation", + "publication": "Governance", "src": "Engineering", "tests": "Quality", "docs": "Governance", diff --git a/publication/README.md b/publication/README.md index b8a5e3f9..12532bad 100644 --- a/publication/README.md +++ b/publication/README.md @@ -7,6 +7,23 @@ second, non-overridable defense layer. The export remains local-only until its selected files have passed sanitization, secret scanning, diff review, and explicit human approval. +The internal public-market gateway at +`src/ai4binance/cli/market_gateway.py` owns bounded public-data access and +fail-closed transport reporting. Verified JSON persistence remains centralized +in `src/ai4binance/infrastructure/persistence/safe_json.py`; callers must use +its atomic write and read-back verification boundary instead of creating a +parallel persistence path. Cross-process event persistence uses the bounded +exclusive-lock implementation in `src/ai4binance/events/file_lock.py`, including +retryable Windows contention and a deterministic timeout. These implementation +surfaces remain `CLOSED` and cannot grant publication, promotion, deployment, +or trading authority. + +`src/ai4binance/ops/public_showcase.py` enforces the local staging boundary and +invokes the repository-pinned Gitleaks binary with a fixed argument vector, +`shell=False`, bounded timeout handling, and redacted output. This scanner +boundary is report/block only and never grants publication, promotion, +deployment, or trading authority. + ## Disclosure Policy | Level | Public treatment | diff --git a/scripts/export_governed_document_lock_approval.py b/scripts/export_governed_document_lock_approval.py index 5abbe8c8..74a9de1d 100644 --- a/scripts/export_governed_document_lock_approval.py +++ b/scripts/export_governed_document_lock_approval.py @@ -3,12 +3,14 @@ import argparse import hashlib import json +import re from datetime import UTC, datetime -from pathlib import Path +from pathlib import Path, PurePosixPath, PureWindowsPath from typing import Any MANIFEST_PATH = Path("config/governance/governed_document_lock_manifest.json") DEFAULT_OUTPUT_DIR = Path("runtime/artifacts/repository_validation/governance") +_SHA256_PATTERN = re.compile(r"^[a-f0-9]{64}$") def _sha256(path: Path) -> str: @@ -62,9 +64,14 @@ def build_approval_evidence( *, repository_root: Path, approval_id: str, + manifest_payload: dict[str, Any] | None = None, ) -> dict[str, Any]: manifest_path = repository_root / MANIFEST_PATH - manifest = _load_manifest(manifest_path) + manifest = ( + manifest_payload + if manifest_payload is not None + else _load_manifest(manifest_path) + ) approval = _approval_record(manifest, approval_id) if approval.get("approval_status") != "APPROVED": raise ValueError("approval record must be APPROVED") @@ -154,6 +161,79 @@ def build_approval_evidence( } +def _load_bound_evidence( + *, + repository_root: Path, + approval: dict[str, Any], + output_path: Path, +) -> dict[str, Any] | None: + evidence_path = str(approval.get("approval_evidence_path", "")).strip() + evidence_sha256 = str(approval.get("approval_evidence_sha256", "")).strip().lower() + if not evidence_path and not evidence_sha256: + return None + if not evidence_path or not evidence_sha256: + raise ValueError("BOUND_APPROVAL_EVIDENCE_REFERENCE_INCOMPLETE") + relative = PurePosixPath(evidence_path) + if ( + relative.is_absolute() + or PureWindowsPath(evidence_path).is_absolute() + or "\\" in evidence_path + or ".." in relative.parts + ): + raise ValueError("BOUND_APPROVAL_EVIDENCE_PATH_INVALID") + if _SHA256_PATTERN.fullmatch(evidence_sha256) is None: + raise ValueError("BOUND_APPROVAL_EVIDENCE_SHA256_INVALID") + bound_path = (repository_root / evidence_path).resolve() + if output_path.resolve() != bound_path: + raise ValueError("BOUND_APPROVAL_EVIDENCE_PATH_OVERRIDE_BLOCKED") + if not bound_path.is_file(): + raise ValueError("BOUND_APPROVAL_EVIDENCE_MISSING") + if _sha256(bound_path) != evidence_sha256: + raise ValueError("BOUND_APPROVAL_EVIDENCE_IMMUTABLE_MISMATCH") + payload = _load_manifest(bound_path) + if ( + payload.get("artifact_origin") + != "governed_document_lock_written_owner_approval" + ): + raise ValueError("BOUND_APPROVAL_EVIDENCE_ORIGIN_INVALID") + if ( + str(payload.get("approval_id", "")).strip() + != str(approval.get("approval_id", "")).strip() + ): + raise ValueError("BOUND_APPROVAL_EVIDENCE_ID_MISMATCH") + return payload + + +def persist_approval_evidence( + *, + repository_root: Path, + approval_id: str, + output_path: Path, + manifest_payload: dict[str, Any] | None = None, +) -> dict[str, Any]: + manifest = ( + manifest_payload + if manifest_payload is not None + else _load_manifest(repository_root / MANIFEST_PATH) + ) + approval = _approval_record(manifest, approval_id) + bound = _load_bound_evidence( + repository_root=repository_root, + approval=approval, + output_path=output_path, + ) + if bound is not None: + return bound + payload = build_approval_evidence( + repository_root=repository_root, + approval_id=approval_id, + manifest_payload=manifest, + ) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + return payload + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( description=( @@ -166,10 +246,6 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) repository_root = Path(args.repository_root).resolve() - payload = build_approval_evidence( - repository_root=repository_root, - approval_id=args.approval_id, - ) if args.output_path: output_path = Path(args.output_path) if not output_path.is_absolute(): @@ -180,8 +256,11 @@ def main(argv: list[str] | None = None) -> int: / DEFAULT_OUTPUT_DIR / f"governed_document_lock_approval_{_slug(args.approval_id)}.json" ) - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + payload = persist_approval_evidence( + repository_root=repository_root, + approval_id=args.approval_id, + output_path=output_path, + ) print(json.dumps(payload, indent=2)) return 0 diff --git a/scripts/install_startup_task.ps1 b/scripts/install_startup_task.ps1 index fbb2b236..86144fd7 100644 --- a/scripts/install_startup_task.ps1 +++ b/scripts/install_startup_task.ps1 @@ -1,5 +1,5 @@ param( - [ValidateSet("Install", "InstallMarketHistory", "InstallVirtualMarket", "RunRuntime", "RunVirtualMarket", "RunVoice", "RunValidation", "RunAccounting", "RunAccountingWs", "RunSkillDiscovery", "RunMarketHistory", "RunFuturesMultiTf")] + [ValidateSet("Install", "InstallMarketHistory", "InstallVirtualMarket", "RestartMarketHistory", "RestartVirtualMarket", "RunRuntime", "RunVirtualMarket", "RunVoice", "RunValidation", "RunAccounting", "RunAccountingWs", "RunSkillDiscovery", "RunMarketHistory", "RunFuturesMultiTf")] [string]$Mode = "Install", [switch]$EnableVoiceTask, [switch]$EnableValidationTask, @@ -162,8 +162,19 @@ function Invoke-ServicePython { } -ArgumentList $HealthPath, $Service, $PID Push-Location -LiteralPath $root try { - & $python -B @Arguments 1>> $StdoutPath 2>> $StderrPath - return [int]$LASTEXITCODE + $previousErrorActionPreference = $ErrorActionPreference + try { + # Windows PowerShell 5 surfaces native stderr as ErrorRecord objects. + # Service diagnostics must be logged without terminating a healthy + # resident Python process; its real exit code remains authoritative. + $ErrorActionPreference = "Continue" + & $python -B @Arguments 1>> $StdoutPath 2>> $StderrPath + $nativeExitCode = [int]$LASTEXITCODE + } + finally { + $ErrorActionPreference = $previousErrorActionPreference + } + return $nativeExitCode } finally { Pop-Location @@ -220,6 +231,104 @@ function Start-BoundedService { } while ($true) } +function Restart-BoundedScheduledService { + param( + [Parameter(Mandatory = $true)][string]$Service, + [Parameter(Mandatory = $true)][string]$Module, + [Parameter(Mandatory = $true)][string]$Command + ) + $entry = Get-ServiceManifestEntry -Service $Service + $taskName = [string]$entry.task_name + $lockPath = Join-Path $stateDirectory ([string]$entry.lock_file) + $oldLockPid = 0 + if (Test-Path -LiteralPath $lockPath -PathType Leaf) { + try { + $lockPayload = Get-Content -LiteralPath $lockPath -Raw + try { + $lockDocument = $lockPayload | ConvertFrom-Json -ErrorAction Stop + $oldLockPid = [int]$lockDocument.pid + } + catch { + if ($lockPayload -match '^\s*\d+\s*$') { + $oldLockPid = [int]$lockPayload + } + } + } + catch { + $oldLockPid = 0 + } + } + + Stop-ScheduledTask -TaskName $taskName -ErrorAction Stop + Start-Sleep -Seconds 2 + $pythonPattern = [regex]::Escape($python) + $modulePattern = [regex]::Escape($Module) + $commandPattern = [regex]::Escape($Command) + $expectedCommandLine = "^`"?$pythonPattern`"?\s+-B\s+-m\s+$modulePattern\s+$commandPattern$" + for ($attempt = 0; $attempt -lt 4; $attempt++) { + $matches = @( + Get-CimInstance Win32_Process | Where-Object { + ([string]$_.CommandLine) -match $expectedCommandLine + } + ) + if ($matches.Count -eq 0) { + break + } + $parentIds = @($matches.ParentProcessId) + $leaves = @($matches | Where-Object { $_.ProcessId -notin $parentIds }) + if ($leaves.Count -eq 0) { + $leaves = $matches + } + foreach ($process in $leaves) { + Stop-Process -Id $process.ProcessId -Force -ErrorAction SilentlyContinue + Wait-Process -Id $process.ProcessId -Timeout 5 -ErrorAction SilentlyContinue + } + } + $remaining = @( + Get-CimInstance Win32_Process | Where-Object { + ([string]$_.CommandLine) -match $expectedCommandLine + } + ) + if ($remaining.Count -ne 0) { + throw "Exact $Service process tree did not stop cleanly." + } + + Start-ScheduledTask -TaskName $taskName -ErrorAction Stop + $deadline = [DateTimeOffset]::UtcNow.AddSeconds(30) + do { + Start-Sleep -Seconds 1 + $newLockPid = 0 + try { + $lockPayload = Get-Content -LiteralPath $lockPath -Raw + try { + $lockDocument = $lockPayload | ConvertFrom-Json -ErrorAction Stop + $newLockPid = [int]$lockDocument.pid + } + catch { + if ($lockPayload -match '^\s*\d+\s*$') { + $newLockPid = [int]$lockPayload + } + } + } + catch { + $newLockPid = 0 + } + $newOwner = if ($newLockPid -gt 0) { + Get-Process -Id $newLockPid -ErrorAction SilentlyContinue + } + else { + $null + } + } while ( + ($newLockPid -eq $oldLockPid -or $null -eq $newOwner) -and + [DateTimeOffset]::UtcNow -lt $deadline + ) + if ($newLockPid -eq $oldLockPid -or $null -eq $newOwner) { + throw "Fresh $Service lock owner was not observed within 30 seconds." + } + Write-Output "$Service restarted with lock PID $newLockPid." +} + function Start-BoundedValidation { New-Item -ItemType Directory -Path $stateDirectory, $runtimeResearchDirectory, $logDirectory -Force | Out-Null $service = "validation" @@ -276,6 +385,20 @@ function Register-VirtualMarketTask { return [string]$entry.task_name } +if ($Mode -eq "RestartMarketHistory") { + Restart-BoundedScheduledService ` + -Service "market-history" ` + -Module "ai4binance.cli.market_data" ` + -Command "daemon" + exit 0 +} +if ($Mode -eq "RestartVirtualMarket") { + Restart-BoundedScheduledService ` + -Service "virtual-market" ` + -Module "ai4binance.cli" ` + -Command "virtual-market-daemon" + exit 0 +} if ($Mode -eq "RunRuntime") { Start-BoundedService -Service "runtime" -Command "runtime-daemon" -RestartForever } diff --git a/scripts/quality.ps1 b/scripts/quality.ps1 index 9ebd3322..74d9cfd5 100644 --- a/scripts/quality.ps1 +++ b/scripts/quality.ps1 @@ -2091,6 +2091,7 @@ function Invoke-DeterministicGovernanceGateStep { [Parameter(Mandatory = $true)] [object]$ConstitutionSyncEvidence, [string]$ApprovalRecordPathOverride = "", + [string]$FrozenGovernanceGateReportPath = "", [int[]]$AllowedExitCodes = @(0) ) @@ -2147,6 +2148,12 @@ function Invoke-DeterministicGovernanceGateStep { if (-not [string]::IsNullOrWhiteSpace($ApprovalRecordPathOverride)) { $arguments += @("--approval-record-report", $ApprovalRecordPathOverride) } + if (-not [string]::IsNullOrWhiteSpace($FrozenGovernanceGateReportPath)) { + $arguments += @( + "--frozen-governance-gate-report", + $FrozenGovernanceGateReportPath + ) + } return Invoke-QualityStepWithAllowedExitCodes ` -Name "Deterministic governance gate" ` -Arguments $arguments ` @@ -2474,7 +2481,8 @@ function Invoke-FullApprovalReplayQualityGate { -DocsHygieneEvidence $context.governance.docs_hygiene ` -ArtifactHygieneEvidence $context.governance.artifact_hygiene ` -ConstitutionSyncEvidence $context.governance.constitution_sync_tests ` - -ApprovalRecordPathOverride $context.approval_record_path | Out-Null + -ApprovalRecordPathOverride $context.approval_record_path ` + -FrozenGovernanceGateReportPath $context.governance_path | Out-Null Invoke-GeneratedArtifactCleanup Invoke-ProcessTempRetentionCleanup Assert-QualityWorkspaceStable -Stage "BEFORE_APPROVAL_REPLAY_GREEN_EVIDENCE" diff --git a/scripts/sync_governed_document_lock_approval_evidence.py b/scripts/sync_governed_document_lock_approval_evidence.py index ae34df00..7e1dbd89 100644 --- a/scripts/sync_governed_document_lock_approval_evidence.py +++ b/scripts/sync_governed_document_lock_approval_evidence.py @@ -11,7 +11,7 @@ _load_manifest, _sha256, _slug, - build_approval_evidence, + persist_approval_evidence, ) @@ -99,20 +99,18 @@ def sync_manifest( if normalized_written_owner_approvals is not None: manifest["written_owner_approvals"] = normalized_written_owner_approvals written_owner_approvals = normalized_written_owner_approvals - manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") - exported: list[dict[str, Any]] = [] for item in approval_records: if not isinstance(item, dict): raise ValueError("approval record entries must be JSON objects") approval_id = _approval_id(item) output_path = _output_path(repository_root, approval_id) - payload = build_approval_evidence( + payload = persist_approval_evidence( repository_root=repository_root, approval_id=approval_id, + output_path=output_path, + manifest_payload=manifest, ) - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") relative_output_path = output_path.relative_to(repository_root).as_posix() evidence_sha256 = _sha256(output_path) _attach_evidence_ref( diff --git a/src/ai4binance/agents/validation_gate.py b/src/ai4binance/agents/validation_gate.py index 5f6463d4..2fd1f6f8 100644 --- a/src/ai4binance/agents/validation_gate.py +++ b/src/ai4binance/agents/validation_gate.py @@ -105,11 +105,12 @@ def validate( and not gate_result.blockers for name in ("data_quality", "universe_liquidity") ) - maturity_ref = ( - self._maturity_reference(snapshot, selected) + maturity_ref, maturity_blockers = ( + self._maturity_evaluation(snapshot, selected) if required_gates_passed - else None + else (None, ()) ) + blockers.extend(maturity_blockers) if maturity_ref is None: blockers.extend( ( @@ -237,13 +238,20 @@ def maximum_score(*names: str) -> float: def _maturity_reference( self, snapshot: MarketSnapshot, candidates: tuple[TradeCandidate, ...] ) -> str | None: + """Return a complete maturity reference for compatibility callers.""" + + return self._maturity_evaluation(snapshot, candidates)[0] + + def _maturity_evaluation( + self, snapshot: MarketSnapshot, candidates: tuple[TradeCandidate, ...] + ) -> tuple[str | None, tuple[str, ...]]: """Revalidate bytes against independently supplied deployment identities. Candidate scores, promotion labels and run-card presence are not evidence. No configured identity or bundle means no approval, including simulation. """ if self.artifact_root is None or len(candidates) != 1: - return None + return None, () candidate = candidates[0] subjects = tuple( subject @@ -256,7 +264,10 @@ def _maturity_reference( ) ) if len(subjects) != 1: - return None + return None, ( + "OOS_SUBJECT_NOT_CONFIGURED:" + f"{candidate.setup_name}:{snapshot.symbol}:{candidate.timeframe}", + ) expected = subjects[0] bundles = tuple( bundle @@ -267,7 +278,7 @@ def _maturity_reference( ) ) if len(bundles) != 1: - return None + return None, ("OOS_SUBJECT_BUNDLE_NOT_CONFIGURED",) current_subject = replace( expected, promotion=replace(expected.promotion, as_of=snapshot.created_at) ) @@ -275,8 +286,8 @@ def _maturity_reference( replace(bundles[0], subject=current_subject) ) if result.status != "OOS_MATURITY_COMPLETE" or result.blockers: - return None - return f"OOS_MATURITY:{result.bundle_sha256}" + return None, result.blockers + return f"OOS_MATURITY:{result.bundle_sha256}", () @staticmethod def _metadata_value( diff --git a/src/ai4binance/application/research.py b/src/ai4binance/application/research.py index 34e6749e..1c125a10 100644 --- a/src/ai4binance/application/research.py +++ b/src/ai4binance/application/research.py @@ -110,6 +110,31 @@ def evaluate( ) -> VirtualGovernanceResult: ... +def _dge_error_type(error: Exception) -> str: + """Return a bounded diagnostic class without exposing exception detail.""" + + stage_codes = { + "DGE_CANDIDATE_ADAPTATION_FAILED": "CANDIDATE_ADAPTATION", + "DGE_CONTEXT_BUILD_FAILED": "CONTEXT_BUILD", + "DGE_CONTEXT_TYPE_INVALID": "CONTEXT_TYPE", + "DGE_CONTEXT_SURFACE_INVALID": "CONTEXT_SURFACE", + "DGE_ENGINE_EVALUATION_FAILED": "ENGINE_EVALUATION", + } + if str(error) in stage_codes: + return stage_codes[str(error)] + if isinstance(error, ArithmeticError): + return "ARITHMETIC_ERROR" + if isinstance(error, AttributeError): + return "ATTRIBUTE_ERROR" + if isinstance(error, KeyError): + return "KEY_ERROR" + if isinstance(error, RuntimeError): + return "RUNTIME_ERROR" + if isinstance(error, TypeError): + return "TYPE_ERROR" + return "VALUE_ERROR" + + class BalanceLike(Protocol): asset: str free: object @@ -1183,11 +1208,14 @@ def _evaluate_virtual_governance( RuntimeError, TypeError, ValueError, - ): + ) as error: return ( fallback_id, DGE_DATA_UNAVAILABLE, - (DGE_EVALUATION_FAILED,), + ( + DGE_EVALUATION_FAILED, + f"DGE_EVALUATION_ERROR_TYPE:{_dge_error_type(error)}", + ), False, ) return ( diff --git a/src/ai4binance/cli/futures_oos.py b/src/ai4binance/cli/futures_oos.py index f6a710d5..98ee221d 100644 --- a/src/ai4binance/cli/futures_oos.py +++ b/src/ai4binance/cli/futures_oos.py @@ -139,7 +139,7 @@ def main( parsed = build_parser().parse_args(arguments) repository_root = parsed.repository_root.resolve() - replay_root = repository_root / "runtime" / "datasets" / "futures" + replay_root = repository_root / "runtime" / "data" / "datasets" / "futures" evidence_root = ( repository_root / "runtime" / "artifacts" / "validation" / "futures_oos" ) diff --git a/src/ai4binance/cli/market_data.py b/src/ai4binance/cli/market_data.py index a29bc85f..e1a6ccf4 100644 --- a/src/ai4binance/cli/market_data.py +++ b/src/ai4binance/cli/market_data.py @@ -45,6 +45,7 @@ "promotion_status": "RESEARCH_ONLY", "live_eligibility_status": "LIVE_ORDER_BLOCKED", } +_DEPTH_SYMBOLS_PER_CONNECTION = 10 _DASHBOARD_CANDIDATE_FIELDS = ( "opportunity_id", "observed_at", @@ -94,11 +95,15 @@ def _dashboard_candidate_projection( elif isinstance(value, (int, float)) and not isinstance(value, bool): row[name] = value blockers = candidate.get("blockers") - if isinstance(blockers, list) and all( - isinstance(blocker, str) and len(blocker) <= 180 - for blocker in blockers[:40] + if ( + isinstance(blockers, Sequence) + and not isinstance(blockers, str) + and all( + isinstance(blocker, str) and len(blocker) <= 180 + for blocker in blockers[:40] + ) ): - row["blockers"] = blockers[:40] + row["blockers"] = list(blockers[:40]) else: row["blockers"] = ["CANDIDATE_BLOCKERS_UNAVAILABLE"] row.update(market=market, symbol=symbol, **_SAFE_STATE) @@ -112,7 +117,7 @@ def _priority_depth_markets( *, include_coin_m: bool, ) -> dict[str, tuple[str, ...]]: - """Bound L2 bootstrap to watched symbols with one top-volume fallback.""" + """Cover each active top-volume market universe with bounded public L2.""" priority = tuple(dict.fromkeys(priority_symbols)) spot_symbols = tuple(dict.fromkeys(getattr(universe, "spot_symbols", ()))) @@ -121,8 +126,7 @@ def _priority_depth_markets( def selected(symbols: tuple[str, ...]) -> tuple[str, ...]: eligible = frozenset(symbols) watched = tuple(symbol for symbol in priority if symbol in eligible) - target_count = min(8, max(1, len(watched))) - return tuple(dict.fromkeys((*watched, *symbols)))[:target_count] + return tuple(dict.fromkeys((*watched, *symbols)))[:50] markets = { "spot": selected(spot_symbols), @@ -136,6 +140,57 @@ def selected(symbols: tuple[str, ...]) -> tuple[str, ...]: return markets +def build_market_depth_collector( + synchronizer: MarketHistorySynchronizer, + continuous: ContinuousMarketHistory, +) -> MarketDepthCollector: + """Build bounded shards so one reconnect cannot invalidate a full universe.""" + + transports = {"spot": continuous.spot, "usd_m_futures": continuous.futures} + if continuous.coin_m is not None: + transports["coin_m_futures"] = continuous.coin_m + return MarketDepthCollector( + synchronizer.archive_root / "depth", + transports, + symbols_per_connection=_DEPTH_SYMBOLS_PER_CONNECTION, + ) + + +def build_continuous_market_history( + settings: Settings, + synchronizer: MarketHistorySynchronizer, + *, + root: Path, + include_coin_m: bool, +) -> ContinuousMarketHistory: + """Build the one canonical continuous collector used by every runtime.""" + + return ContinuousMarketHistory( + history=synchronizer, + spot=synchronizer.universe_provider.spot_transport, + futures=synchronizer.universe_provider.futures_transport, + initial_days=settings.market_history_initial_days, + pages_per_stream=settings.market_history_pages_per_stream, + max_workers=settings.market_history_max_workers, + minimum_candles=settings.minimum_closed_candles, + coin_m=( + synchronizer.universe_provider.coin_m_transport if include_coin_m else None + ), + priority_symbols=tuple( + dict.fromkeys( + (settings.symbol, *settings.fixed_symbols, *settings.priority_watchlist) + ) + ), + refresh_request_path=_absolute(settings.market_history_state_path).with_name( + "market-history-refresh-request.json" + ), + on_symbol_ready=_build_canonical_opportunity_pipeline( + settings, root, include_futures=True + ), + on_symbol_screen=_build_opportunity_screen(settings, root), + ) + + def _build_canonical_opportunity_pipeline( settings: Settings, root: Path, *, include_futures: bool = False ) -> Callable[[str, str, datetime], Mapping[str, object]]: @@ -335,31 +390,11 @@ def run_market_history_command( print(json.dumps(payload, ensure_ascii=False, sort_keys=True)) return 0 if not payload.get("blockers") else 2 - continuous = ContinuousMarketHistory( - history=synchronizer, - spot=synchronizer.universe_provider.spot_transport, - futures=synchronizer.universe_provider.futures_transport, - initial_days=settings.market_history_initial_days, - pages_per_stream=settings.market_history_pages_per_stream, - max_workers=settings.market_history_max_workers, - minimum_candles=settings.minimum_closed_candles, - coin_m=synchronizer.universe_provider.coin_m_transport, - priority_symbols=tuple( - dict.fromkeys( - ( - settings.symbol, - *settings.fixed_symbols, - *settings.priority_watchlist, - ) - ) - ), - refresh_request_path=_absolute(settings.market_history_state_path).with_name( - "market-history-refresh-request.json" - ), - on_symbol_ready=_build_canonical_opportunity_pipeline( - settings, Path.cwd(), include_futures=True - ), - on_symbol_screen=_build_opportunity_screen(settings, Path.cwd()), + continuous = build_continuous_market_history( + settings, + synchronizer, + root=Path.cwd(), + include_coin_m=True, ) if command == "market-history-sync" and as_of is None: with SingleInstanceLease(synchronizer.state_path.with_suffix(".lock")): @@ -389,10 +424,7 @@ def run_market_history_command( return 0 if not report.blockers else 2 if command == "market-history-daemon": - transports = {"spot": continuous.spot, "usd_m_futures": continuous.futures} - if continuous.coin_m is not None: - transports["coin_m_futures"] = continuous.coin_m - depth = MarketDepthCollector(synchronizer.archive_root / "depth", transports) + depth = build_market_depth_collector(synchronizer, continuous) def cycle(now: datetime) -> object: if settings.market_depth_enabled: @@ -571,4 +603,10 @@ def main(arguments: Sequence[str] | None = None) -> int: raise SystemExit(main()) -__all__ = ("build_market_history_synchronizer", "main", "run_market_history_command") +__all__ = ( + "build_continuous_market_history", + "build_market_depth_collector", + "build_market_history_synchronizer", + "main", + "run_market_history_command", +) diff --git a/src/ai4binance/cli/market_gateway.py b/src/ai4binance/cli/market_gateway.py index 3d6117b6..f4c77a67 100644 --- a/src/ai4binance/cli/market_gateway.py +++ b/src/ai4binance/cli/market_gateway.py @@ -12,10 +12,21 @@ from websockets.exceptions import ConnectionClosed -from ai4binance.cli.market_data import build_market_history_synchronizer +from ai4binance.cli.market_data import ( + _priority_depth_markets, + build_continuous_market_history, + build_market_depth_collector, + build_market_history_synchronizer, +) from ai4binance.config import Settings -from ai4binance.data.market_data_gateway import MarketStreamGapError, build_gateway +from ai4binance.data.market_data_gateway import ( + BinanceMarketDataGateway, + MarketStreamGapError, + build_gateway, +) +from ai4binance.data.market_depth import MarketDepthCollector from ai4binance.data.market_history_continuous import ContinuousMarketHistory +from ai4binance.data.market_history_sync import MarketHistorySynchronizer from ai4binance.infrastructure.persistence.safe_json import write_json_object_verified from ai4binance.ops.runtime import SingleInstanceLease @@ -73,21 +84,15 @@ def run_gateway(settings: Settings, *, max_cycles: int | None = None) -> int: if max_cycles is not None and max_cycles < 1: raise ValueError("max_cycles must be positive") synchronizer = build_market_history_synchronizer(settings) - collector = ContinuousMarketHistory( - history=synchronizer, - spot=synchronizer.universe_provider.spot_transport, - futures=synchronizer.universe_provider.futures_transport, - initial_days=settings.market_history_initial_days, - pages_per_stream=settings.market_history_pages_per_stream, - max_workers=settings.market_history_max_workers, - minimum_candles=settings.minimum_closed_candles, - coin_m=None, - priority_symbols=tuple( - dict.fromkeys( - (settings.symbol, *settings.fixed_symbols, *settings.priority_watchlist) - ) - ), + collector = build_continuous_market_history( + settings, + synchronizer, + root=Path.cwd(), + include_coin_m=False, ) + depth = None + if getattr(settings, "market_depth_enabled", False): + depth = build_market_depth_collector(synchronizer, collector) lock_path = _absolute(settings.market_history_state_path).with_suffix(".lock") heartbeat = _GatewayStateHeartbeat(_absolute(settings.market_history_state_path)) completed = 0 @@ -95,25 +100,15 @@ def run_gateway(settings: Settings, *, max_cycles: int | None = None) -> int: with SingleInstanceLease(lock_path): while max_cycles is None or completed < max_cycles: observed_at = datetime.now(UTC) + _refresh_depth(depth, synchronizer, collector, observed_at) bootstrap = collector.sync_cycle(observed_at=observed_at) blockers = _blockers(bootstrap) - if "MARKET_DATA_BACKFILL_PENDING" in blockers and not ( - blockers & _FATAL_INGESTION_BLOCKERS - ): + ingestion_outcome = _ingestion_outcome(blockers) + if ingestion_outcome == "RETRY": time.sleep(min(60.0, settings.market_history_live_interval_seconds)) continue - if blockers & _FATAL_INGESTION_BLOCKERS: - print( - json.dumps( - { - "command": "market-gateway-daemon", - "status": "DATA_BLOCKED", - "blockers": sorted(blockers), - **_SAFE_STATE, - }, - sort_keys=True, - ) - ) + if ingestion_outcome == "BLOCKED": + _print_gateway_blocked("DATA_BLOCKED", blockers) return 2 universe = synchronizer._eligible_universe( datetime.now(UTC), force_refresh=False @@ -132,20 +127,8 @@ def run_gateway(settings: Settings, *, max_cycles: int | None = None) -> int: ), activity_observer=heartbeat, ) - reconnect_attempt = 0 - while True: - try: - asyncio.run(gateway.run_once()) - except MarketStreamGapError: - # The next outer loop performs bounded REST gap recovery - # before either WebSocket connection can resume. - break - except (ConnectionClosed, OSError, TimeoutError): - reconnect_attempt += 1 - time.sleep(_reconnect_delay(reconnect_attempt)) - continue + if _run_live_cycle(gateway): completed += 1 - break except RuntimeError as error: blocker_code = ( "MARKET_GATEWAY_ALREADY_RUNNING" @@ -164,9 +147,70 @@ def run_gateway(settings: Settings, *, max_cycles: int | None = None) -> int: ) ) return 2 + finally: + if depth is not None: + depth.close() return 0 +def _refresh_depth( + depth: MarketDepthCollector | None, + synchronizer: MarketHistorySynchronizer, + collector: ContinuousMarketHistory, + observed_at: datetime, +) -> None: + if depth is None: + return + universe = synchronizer._eligible_universe(observed_at, force_refresh=True) + if universe.blockers: + return + depth.start( + _priority_depth_markets( + universe, + collector.priority_symbols, + include_coin_m=False, + ) + ) + + +def _ingestion_outcome(blockers: frozenset[str]) -> str: + if blockers & _FATAL_INGESTION_BLOCKERS: + return "BLOCKED" + if "MARKET_DATA_BACKFILL_PENDING" in blockers: + return "RETRY" + return "READY" + + +def _print_gateway_blocked(status: str, blockers: frozenset[str]) -> None: + print( + json.dumps( + { + "command": "market-gateway-daemon", + "status": status, + "blockers": sorted(blockers), + **_SAFE_STATE, + }, + sort_keys=True, + ) + ) + + +def _run_live_cycle(gateway: BinanceMarketDataGateway) -> bool: + reconnect_attempt = 0 + while True: + try: + asyncio.run(gateway.run_once()) + except MarketStreamGapError: + # The next outer loop performs bounded REST gap recovery before either + # WebSocket connection can resume. + return False + except (ConnectionClosed, OSError, TimeoutError): + reconnect_attempt += 1 + time.sleep(_reconnect_delay(reconnect_attempt)) + continue + return True + + def _absolute(path: Path) -> Path: return path if path.is_absolute() else Path.cwd() / path diff --git a/src/ai4binance/cli/research.py b/src/ai4binance/cli/research.py index 0a751bf5..c87bd6e0 100644 --- a/src/ai4binance/cli/research.py +++ b/src/ai4binance/cli/research.py @@ -50,7 +50,11 @@ from ai4binance.research_runtime import build_research_application_service from ai4binance.schemas import MarketSnapshot from ai4binance.storage import AuditEvent, JsonlAuditStore -from ai4binance.validation_pipeline_runtime import HistoricalValidationRuntime +from ai4binance.validation.oos_maturity import prepare_spot_oos_deployment +from ai4binance.validation_pipeline_runtime import ( + SPOT_VALIDATION_NOTIONAL_TO_EQUITY_RATIO, + HistoricalValidationRuntime, +) from ai4binance.virtual_wallet_journal import ( VirtualWalletJournal, VirtualWalletJournalError, @@ -82,9 +86,37 @@ def run_validate_research(settings: Settings, symbol: str | None) -> int: ), artifact_directory=settings.validation_artifact_directory, runtime=HistoricalValidationRuntime( - report_directory=settings.backtest_report_directory + report_directory=settings.backtest_report_directory, + position_notional_to_equity_ratio=( + SPOT_VALIDATION_NOTIONAL_TO_EQUITY_RATIO + ), ), ).run(validation_symbol, settings.timeframes) + run_cards = tuple( + cast(dict[str, object], to_primitive(result.run_card)) + for result in batch.results + if result.run_card is not None + ) + oos_deployment = ( + prepare_spot_oos_deployment( + artifact_root=settings.validation_artifact_directory, + deployment_path=settings.runtime_validation_deployment_path, + specification_path=Path( + "config/research/virtual_market_acceptance.yaml" + ), + run_cards=run_cards, + observed_at=datetime.now(UTC), + ) + if run_cards + else { + "status": "VALIDATION_RUN_CARDS_MISSING", + "subject_count": 0, + "blockers": ["VALIDATION_RUN_CARDS_MISSING"], + "execution_allowed": False, + "promotion_status": "RESEARCH_ONLY", + "live_eligibility_status": "LIVE_ORDER_BLOCKED", + } + ) payload: dict[str, object] = { "symbol": batch.symbol, "results": tuple( @@ -110,6 +142,7 @@ def run_validate_research(settings: Settings, symbol: str | None) -> int: ), "execution_allowed": batch.execution_allowed, "live_eligibility_status": batch.live_eligibility_status, + "oos_deployment": oos_deployment, } print(json.dumps(to_primitive(payload), ensure_ascii=False, sort_keys=True)) return 0 diff --git a/src/ai4binance/cli/runtime.py b/src/ai4binance/cli/runtime.py index 65bfe45a..01280c66 100644 --- a/src/ai4binance/cli/runtime.py +++ b/src/ai4binance/cli/runtime.py @@ -116,8 +116,10 @@ _VIRTUAL_MARKET_STATE_NAME = "virtual-market.json" _VIRTUAL_MARKET_LOCK_NAME = "virtual-market.lock" _VIRTUAL_MARKET_REFRESH_REQUEST_NAME = "virtual-market-refresh-request.json" +_MARKET_HISTORY_REFRESH_REQUEST_NAME = "market-history-refresh-request.json" _DASHBOARD_SIMULATION_MAX_SYMBOLS = 1_000 _DASHBOARD_SIMULATION_MAX_BLOCKERS = 12 +_VIRTUAL_MARKET_UNIVERSE_LIMIT = 50 _NEWS_ASSET_ALIASES = { "BTC": ("bitcoin",), "ETH": ("ethereum", "ether"), @@ -590,7 +592,14 @@ def run_virtual_market_daemon( if name not in configured ) if universe is not None - else configured + else ( + _virtual_market_ranked_symbols( + settings, + configured, + clock(), + ) + or configured + ) ) from ai4binance.exchange.client import BinancePublicClient @@ -654,6 +663,13 @@ def run_virtual_market_daemon( else: discovery_symbol = last_symbol cycle_settings = settings.model_copy(update={"symbol": last_symbol}) + from ai4binance.data.market_history_continuous import ( + VIRTUAL_MARKET_COLLECTION_TIMEFRAMES, + ) + + cycle_settings = cycle_settings.model_copy( + update={"timeframes": VIRTUAL_MARKET_COLLECTION_TIMEFRAMES} + ) cycle_report.update( { "symbol": last_symbol, @@ -662,14 +678,25 @@ def run_virtual_market_daemon( "priority_symbol": priority_symbol, "discovery_symbol": discovery_symbol, "priority_symbol_count": len(priority_symbols), - "universe_source": "CANONICAL_CACHE" - if universe is not None - else "CONFIGURED_FALLBACK", + "universe_source": ( + "CANONICAL_CACHE" + if universe is not None + else "LOCAL_SNAPSHOT" + if symbols != configured + else "CONFIGURED_FALLBACK" + ), } ) exit_code = _run_virtual_market_research_cycle( cycle_settings, acquisition, cycle_report=cycle_report ) + _request_market_history_refresh_if_stale( + state_path.with_name(_MARKET_HISTORY_REFRESH_REQUEST_NAME), + symbol=last_symbol, + eligible_symbols=eligible_symbols, + observed_at=clock(), + cycle_report=cycle_report, + ) except (ExchangeError, OSError, RuntimeError, TypeError, ValueError): exit_code = 2 observed_at = clock() @@ -824,6 +851,25 @@ def _virtual_market_priority_symbols( observed_at: datetime, ) -> tuple[str, ...]: """Schedule canonical liquidity priorities using shared public files only.""" + + return tuple( + symbol + for symbol in _virtual_market_ranked_symbols( + settings, + configured, + observed_at, + ) + if symbol in eligible + )[: settings.virtual_market_priority_symbol_count] + + +def _virtual_market_ranked_symbols( + settings: Settings, + configured: tuple[str, ...], + observed_at: datetime, +) -> tuple[str, ...]: + """Read the bounded Spot universe from the current canonical snapshots.""" + from ai4binance.data.acquisition import LocalMarketSnapshotTransport from ai4binance.integrations.binance.market_universe_provider import ( BinanceMarketUniverseProvider, @@ -838,14 +884,14 @@ def _virtual_market_priority_symbols( ranked = BinanceMarketUniverseProvider( local, local, - max_symbols_per_market=settings.virtual_market_priority_symbol_count, + max_symbols_per_market=_VIRTUAL_MARKET_UNIVERSE_LIMIT, ).spot_symbols(configured) except (ExchangeError, OSError, TypeError, ValueError): return () return tuple( item.symbol for item in ranked - if item.symbol in eligible and item.data_quality_ok and item.status == "TRADING" + if item.data_quality_ok and item.status == "TRADING" ) @@ -862,14 +908,62 @@ def _run_virtual_market_research_cycle( ) -> int: from ai4binance.cli.research import run_public_research_command - return run_public_research_command( + journal = _virtual_wallet_journal(settings) + exit_code = run_public_research_command( "research-public", settings, public_acquisition=public_acquisition, whale_fusion_cycle=None, cycle_report=cycle_report, - virtual_wallet_journal=_virtual_wallet_journal(settings), + virtual_wallet_journal=journal, ) + if cycle_report is not None: + cycle_report["daily_loss_tuning"] = _run_virtual_loss_tuning( + settings, + journal, + datetime.now(UTC), + ) + return exit_code + + +def _request_market_history_refresh_if_stale( + path: Path, + *, + symbol: str, + eligible_symbols: tuple[str, ...], + observed_at: datetime, + cycle_report: dict[str, object], +) -> None: + """Request one bounded canonical refresh after a stale virtual snapshot.""" + + raw_blockers = cycle_report.get("research_blockers", ()) + blockers = ( + tuple(item for item in raw_blockers if isinstance(item, str)) + if isinstance(raw_blockers, (list, tuple)) + else () + ) + if not any(item.startswith("STALE_CANDLES:") for item in blockers): + return + from ai4binance.data.market_history_continuous import ( + enqueue_market_history_refresh_request, + ) + + try: + refresh = enqueue_market_history_refresh_request( + path, + market="SPOT", + symbol=symbol, + eligible_symbols=eligible_symbols, + requested_at=observed_at, + requester="VIRTUAL_MARKET", + ) + except (OSError, ValueError): + cycle_report["market_history_refresh"] = { + "state": "DATA_BLOCKED", + "blockers": ["VIRTUAL_MARKET_REFRESH_REQUEST_FAILED"], + } + return + cycle_report["market_history_refresh"] = refresh def _virtual_wallet_journal(settings: Settings) -> VirtualWalletJournal: @@ -880,6 +974,186 @@ def _virtual_wallet_journal(settings: Settings) -> VirtualWalletJournal: ) +def _run_virtual_loss_tuning( + settings: Settings, + journal: VirtualWalletJournal, + observed_at: datetime, +) -> dict[str, object]: + """Run canonical Spot backtest/tuning after three same-day virtual losses.""" + + safe_state = { + "execution_allowed": False, + "promotion_status": "RESEARCH_ONLY", + "live_eligibility_status": "LIVE_ORDER_BLOCKED", + } + try: + trigger = journal.daily_loss_tuning_trigger(observed_at) + except (OSError, RuntimeError, TypeError, ValueError): + return { + "status": "BLOCKED", + "blockers": ["VIRTUAL_LOSS_TUNING_TRIGGER_UNAVAILABLE"], + "parameter_application": "NOT_APPLIED", + **safe_state, + } + if trigger.get("status") != "TRIGGERED": + return {**trigger, "parameter_application": "NOT_APPLIED"} + trigger_id = str(trigger.get("trigger_id", "")) + if not re.fullmatch(r"virtual-loss-tuning:[0-9a-f]{24}", trigger_id): + return { + "status": "BLOCKED", + "blockers": ["VIRTUAL_LOSS_TUNING_TRIGGER_INVALID"], + "parameter_application": "NOT_APPLIED", + **safe_state, + } + tuning_root = settings.validation_artifact_directory / "virtual_loss_tuning" + artifact_path = tuning_root / f"{trigger_id.rsplit(':', maxsplit=1)[-1]}.json" + existing = _load_json_mapping(artifact_path) + if existing: + if ( + existing.get("trigger_id") != trigger_id + or existing.get("execution_allowed") is not False + or existing.get("promotion_status") != "RESEARCH_ONLY" + or existing.get("live_eligibility_status") != "LIVE_ORDER_BLOCKED" + ): + return { + "status": "BLOCKED", + "blockers": ["VIRTUAL_LOSS_TUNING_ARTIFACT_INVALID"], + "parameter_application": "NOT_APPLIED", + **safe_state, + } + if existing.get("status") == "RESEARCH_TUNING_COMPLETED": + return { + "status": "ALREADY_REVIEWED", + "trigger_id": trigger_id, + "artifact_path": str(artifact_path), + "parameter_application": "NOT_APPLIED", + **safe_state, + } + attempted_at = existing.get("attempted_at") + try: + previous_attempt = datetime.fromisoformat(str(attempted_at)).astimezone(UTC) + except (TypeError, ValueError): + previous_attempt = observed_at.astimezone(UTC) - timedelta(hours=1) + if observed_at.astimezone(UTC) - previous_attempt < timedelta(minutes=15): + return { + "status": "RETRY_PENDING", + "trigger_id": trigger_id, + "artifact_path": str(artifact_path), + "blockers": existing.get("blockers", []), + "parameter_application": "NOT_APPLIED", + **safe_state, + } + + from ai4binance.application.validation_pipeline import VALIDATED_PLAYBOOKS + from ai4binance.data import DatasetIntegrityError, ParquetOHLCVArchive + from ai4binance.validation_pipeline_runtime import ( + SPOT_VALIDATION_NOTIONAL_TO_EQUITY_RATIO, + HistoricalValidationRuntime, + ) + + raw_subjects = trigger.get("subjects") + subjects = raw_subjects if isinstance(raw_subjects, list) else [] + archive = ParquetOHLCVArchive( + settings.dataset_directory / "spot" + if settings.market_history_local_candles + else settings.dataset_directory + ) + runtime = HistoricalValidationRuntime( + report_directory=settings.backtest_report_directory / "virtual_loss_tuning", + position_notional_to_equity_ratio=(SPOT_VALIDATION_NOTIONAL_TO_EQUITY_RATIO), + ) + results: list[dict[str, object]] = [] + aggregate_blockers: list[str] = [] + for raw_subject in subjects[:3]: + if not isinstance(raw_subject, Mapping): + aggregate_blockers.append("VIRTUAL_LOSS_TUNING_SUBJECT_INVALID") + continue + market = str(raw_subject.get("market", "")).upper() + symbol = str(raw_subject.get("symbol", "")).upper() + timeframe = str(raw_subject.get("timeframe", "")) + playbook = str(raw_subject.get("strategy_id", "")) + subject = { + "market": market, + "symbol": symbol, + "timeframe": timeframe, + "strategy_id": playbook, + } + if market != "SPOT" or playbook not in VALIDATED_PLAYBOOKS: + blocker = "VIRTUAL_LOSS_TUNING_SUBJECT_UNSUPPORTED" + aggregate_blockers.append(blocker) + results.append( + {"subject": subject, "status": "BLOCKED", "blockers": [blocker]} + ) + continue + try: + candles = archive.read(symbol, timeframe) + result = runtime.validate_one( + symbol, + timeframe, + playbook, + candles, + artifact_directory=tuning_root / "evidence", + ) + tuning = cast(Any, result.tuning) + backtest = cast(Any, result.backtest) + if tuning is None or backtest is None: + raise ValueError("VIRTUAL_LOSS_TUNING_RESULT_INCOMPLETE") + results.append( + { + "subject": subject, + "status": "RESEARCH_TUNING_COMPLETED", + "dataset_sha256": runtime.dataset_sha256(candles), + "backtest_metrics": to_primitive(backtest.metrics), + "tuning_report_id": tuning.report_id, + "candidate_count": tuning.search_space.candidate_count, + "selected_parameters": to_primitive(tuning.selected_parameters), + "tuning_blockers": list(tuning.blockers), + "validation_blockers": list(result.blockers), + "parameter_application": "NOT_APPLIED", + } + ) + except ( + DatasetIntegrityError, + FileNotFoundError, + OSError, + TypeError, + ValueError, + ): + blocker = "VIRTUAL_LOSS_TUNING_DATA_OR_VALIDATION_UNAVAILABLE" + aggregate_blockers.append(blocker) + results.append( + {"subject": subject, "status": "BLOCKED", "blockers": [blocker]} + ) + if not subjects: + aggregate_blockers.append("VIRTUAL_LOSS_TUNING_SUBJECTS_MISSING") + status = ( + "RESEARCH_TUNING_COMPLETED" + if results + and all(item.get("status") == "RESEARCH_TUNING_COMPLETED" for item in results) + else "RETRY_PENDING" + ) + payload = { + "schema_version": "VirtualLossTuningResult/v1", + "status": status, + "trigger_id": trigger_id, + "trigger": trigger, + "attempted_at": observed_at.astimezone(UTC).isoformat(), + "results": results, + "blockers": list(dict.fromkeys(aggregate_blockers)), + "parameter_application": "NOT_APPLIED", + **safe_state, + } + write_json_object_verified( + artifact_path, + payload, + blocker="VIRTUAL_LOSS_TUNING_WRITE_FAILED", + subject_id=trigger_id, + indent=2, + durable=True, + ) + return {**payload, "artifact_path": str(artifact_path)} + + def _virtual_wallet_report_root(state_path: Path) -> Path: resolved = state_path.resolve() for parent in resolved.parents: diff --git a/src/ai4binance/config.py b/src/ai4binance/config.py index 8a73b58f..08da6464 100644 --- a/src/ai4binance/config.py +++ b/src/ai4binance/config.py @@ -25,7 +25,7 @@ class Settings(BaseSettings): fixed_symbols: tuple[str, ...] = () priority_watchlist: tuple[str, ...] = () futures_symbol_exclusions: tuple[str, ...] = ("HOTUSDT",) - timeframes: tuple[str, ...] = ("5m", "15m", "1h", "4h", "1d") + timeframes: tuple[str, ...] = ("15m", "1h", "4h") trading_mode: Literal["paper", "live"] = "paper" order_mode: Literal["manual", "dry_run", "paper", "auto", "live"] = "manual" allow_auto_live_orders: bool = False @@ -51,10 +51,9 @@ class Settings(BaseSettings): market_history_state_path: Path = Path("runtime/state/market-history-latest.json") market_history_interval_seconds: float = 21_600.0 market_history_live_interval_seconds: float = 300.0 - # Every persisted timeframe receives at least this direct-history window. - # The collector extends individual timeframes further when deterministic - # closed-candle quality requires it (for example, 201 daily bars). - market_history_initial_days: int = 90 + # Keep enough native history to satisfy the governed 365-day Spot OOS + # observation floor, with a bounded buffer for publication lag and gaps. + market_history_initial_days: int = 400 market_history_pages_per_stream: int = 32 market_history_max_workers: int = 8 market_history_opportunity_workers: int = 2 @@ -72,7 +71,7 @@ class Settings(BaseSettings): "config/research/runtime_validation_deployment.json" ) futures_oos_artifact_directory: Path = Path( - "runtime/artifacts/research/backtest/oos" + "runtime/artifacts/validation/futures_oos" ) backtest_report_directory: Path = Path("runtime/reports/backtest") backtest_layout_manifest_path: Path = Path( diff --git a/src/ai4binance/data/acquisition.py b/src/ai4binance/data/acquisition.py index 12539652..31b09601 100644 --- a/src/ai4binance/data/acquisition.py +++ b/src/ai4binance/data/acquisition.py @@ -115,6 +115,7 @@ class DataAcquisitionAgent: max_workers: int = 4 archive: ParquetOHLCVArchive | None = None depth_path: Path | None = None + clock: Callable[[], datetime] = lambda: datetime.now(UTC) def __post_init__(self) -> None: if not is_spot_market_type(self.market_type): @@ -145,6 +146,9 @@ def acquire( latest_price = self.client.ticker_price(symbol_info.symbol) book = self.client.book_ticker(symbol_info.symbol) raw_klines = self._fetch_klines(symbol_info.symbol, timeframes) + observed_at = self.clock() if self.depth_path is not None else server_time + if observed_at.utcoffset() is None or observed_at < server_time: + raise ValueError("data acquisition clock must follow server time") candles_by_timeframe: dict[str, tuple[OHLCVCandle, ...]] = {} freshness: dict[str, dict[str, object]] = {} @@ -186,13 +190,13 @@ def acquire( ) snapshot_id = self._snapshot_id( symbol_info.symbol, - server_time, + observed_at, latest_price, last_close_times, ) return MarketSnapshot( snapshot_id=snapshot_id, - created_at=server_time, + created_at=observed_at, exchange="Binance", market_type=self.market_type, symbol=symbol_info.symbol, @@ -206,7 +210,7 @@ def acquire( "best_bid": str(book.bid), "best_ask": str(book.ask), "spread": str(book.spread), - **self._local_depth_summary(symbol_info.symbol, server_time), + **self._local_depth_summary(symbol_info.symbol, observed_at), }, exchange_filters={ name: dict(values) for name, values in symbol_info.filters.items() diff --git a/src/ai4binance/data/market_depth.py b/src/ai4binance/data/market_depth.py index 6a0499cc..0de54fbc 100644 --- a/src/ai4binance/data/market_depth.py +++ b/src/ai4binance/data/market_depth.py @@ -35,6 +35,7 @@ "usd_m_futures": "wss://fstream.binance.com/public/stream", "coin_m_futures": "wss://dstream.binance.com/stream", } +_DEPTH_COMPACTION_BATCH_SIZE = 50_000 DepthRecord = tuple[str, str, str, dict[str, object], float] @@ -162,6 +163,18 @@ def append(self, records: list[DepthRecord]) -> None: """, (market, symbol, checkpoint, seq, status, received), ) + if checkpoint is not None: + self.connection.execute( + "DELETE FROM depth_events WHERE seq IN (" + "SELECT seq FROM depth_events WHERE market=? AND symbol=? " + "AND seq None: with self.lock: @@ -414,7 +427,6 @@ def _connection(self, market: str, symbols: tuple[str, ...], group: str) -> None last_flush, started = time.monotonic(), time.monotonic() executor = ThreadPoolExecutor(max_workers=1) received_bytes = 0 - subscribed_at = 0.0 def resync(symbol: str) -> None: """Invalidate only the broken book and queue its bounded resnapshot.""" @@ -454,16 +466,14 @@ def resync(symbol: str) -> None: maximum_levels=100_000, futures_sequence=market != "spot", ) - subscribed_at = time.monotonic() - if pending and future is None and time.monotonic() - subscribed_at > 20: - future = executor.submit( - self.transports[market].get_json, - _prefix(market) + "depth", - { - "symbol": pending, - "limit": 5000 if market == "spot" else 1000, - }, - ) + future = executor.submit( + self.transports[market].get_json, + _prefix(market) + "depth", + { + "symbol": pending, + "limit": 5000 if market == "spot" else 1000, + }, + ) if future is not None and future.done(): raw = future.result() if pending is None or not isinstance(raw, dict): diff --git a/src/ai4binance/data/market_history_continuous.py b/src/ai4binance/data/market_history_continuous.py index 86ccc9b4..584e56c6 100644 --- a/src/ai4binance/data/market_history_continuous.py +++ b/src/ai4binance/data/market_history_continuous.py @@ -36,7 +36,10 @@ WeightedRateLimitGovernor, public_request_weight, ) -from ai4binance.infrastructure.persistence.safe_json import write_json_object_verified +from ai4binance.infrastructure.persistence.safe_json import ( + DestinationVerificationError, + write_json_object_verified, +) from ai4binance.schemas import OHLCVCandle _DEFAULT_KLINE_INTERVAL = timedelta(minutes=5) @@ -72,6 +75,7 @@ _STATE_RESULT_SAMPLE_LIMIT = 128 _DASHBOARD_OPPORTUNITY_LIMIT = 100 _DASHBOARD_REJECTION_LIMIT = 100 +_REFRESH_REQUESTERS = frozenset({"DASHBOARD", "VIRTUAL_MARKET"}) _REFRESH_REQUEST_SAFE_FIELDS = { "execution_allowed": False, "promotion_status": "RESEARCH_ONLY", @@ -168,11 +172,15 @@ def enqueue_market_history_refresh_request( symbol: str, eligible_symbols: tuple[str, ...], requested_at: datetime, + requester: str = "DASHBOARD", ) -> dict[str, object]: - """Persist one validated dashboard request; never replace another pending job.""" + """Persist one bounded freshness request; never replace a pending job.""" if requested_at.tzinfo is None or requested_at.utcoffset() is None: raise ValueError("refresh request timestamp must be timezone-aware") + normalized_requester = requester.strip().upper() + if normalized_requester not in _REFRESH_REQUESTERS: + raise ValueError("market history refresh requester is invalid") normalized_symbol = symbol.strip().upper() if ( market not in {"SPOT", "USD_M_FUTURES"} @@ -203,14 +211,14 @@ def enqueue_market_history_refresh_request( request_id = ( "market-history:" + sha256( - f"DASHBOARD:{market}:{normalized_symbol}:{requested_at.isoformat()}".encode() + f"{normalized_requester}:{market}:{normalized_symbol}:{requested_at.isoformat()}".encode() ).hexdigest()[:24] ) request: dict[str, object] = { "schema_version": "MarketHistoryRefreshRequest/v1", "request_id": request_id, "requested_at": requested_at.astimezone(UTC).isoformat(), - "requester": "DASHBOARD", + "requester": normalized_requester, "market": market, "symbol": normalized_symbol, "status": "PENDING", @@ -240,7 +248,7 @@ def timeframe_refresh_schedule() -> list[dict[str, object]]: "gap_recovery_source": f"BINANCE_PUBLIC_REST_{timeframe.upper()}_ONLY", "network_download": True, } - for timeframe in MARKET_HISTORY_TIMEFRAMES + for timeframe in VIRTUAL_MARKET_COLLECTION_TIMEFRAMES ] @@ -250,6 +258,16 @@ def _save(path: Path, payload: Mapping[str, object]) -> None: ) +def _recoverable_error_code(error: Exception) -> str: + """Expose only repository-owned verification codes, never exception detail.""" + + if isinstance(error, DestinationVerificationError): + code = str(error).strip() + if re.fullmatch(r"[A-Z][A-Z0-9_]{2,127}", code): + return code + return "MARKET_HISTORY_RECOVERABLE_ERROR" + + def _load(path: Path) -> dict[str, object]: payload = json.loads(path.read_text(encoding="utf-8")) if not isinstance(payload, dict): @@ -458,6 +476,9 @@ class ContinuousMarketHistory: vision_history_enabled: bool = True on_symbol_ready: SymbolReadyHandler | None = field(default=None, repr=False) on_symbol_screen: SymbolReadyHandler | None = field(default=None, repr=False) + clock: Callable[[], datetime] = field( + default=lambda: datetime.now(UTC), repr=False + ) def __post_init__(self) -> None: if not 1 <= self.initial_days <= 3650 or not 1 <= self.pages_per_stream <= 32: @@ -509,10 +530,10 @@ def sync_cycle(self, *, observed_at: datetime) -> dict[str, object]: self.coin_m, ), ) - # Historical candle collection must not wait behind bulk snapshot - # endpoints. Snapshots are supplementary metadata and are refreshed - # after the first bounded collection interval. - snapshot_refresh_due = time.monotonic() + 300 + # VirtualMarket depends on these bounded bulk snapshots. Refresh once + # at cycle start so a service restart cannot leave an already-old + # ticker cache to expire during a long candle collection cycle. + snapshot_refresh_due = time.monotonic() def refresh_snapshots(snapshot_time: datetime) -> None: nonlocal snapshot_refresh_due @@ -529,6 +550,10 @@ def refresh_snapshots(snapshot_time: datetime) -> None: refreshed_universe is None or refreshed_universe.blockers ) and "MARKET_UNIVERSE_METADATA_UNAVAILABLE" not in blockers: blockers.append("MARKET_UNIVERSE_METADATA_UNAVAILABLE") + elif refreshed_universe is not None and not refreshed_universe.blockers: + while "MARKET_UNIVERSE_METADATA_UNAVAILABLE" in blockers: + blockers.remove("MARKET_UNIVERSE_METADATA_UNAVAILABLE") + snapshot_failed = False for snapshot_market, snapshot_symbols, snapshot_transport in market_work: if snapshot_transport is None or not snapshot_symbols: continue @@ -540,10 +565,16 @@ def refresh_snapshots(snapshot_time: datetime) -> None: snapshot_time, ) except (OSError, ValueError, ExchangeError): + snapshot_failed = True if "MARKET_SNAPSHOT_UNAVAILABLE" not in blockers: blockers.append("MARKET_SNAPSHOT_UNAVAILABLE") + if not snapshot_failed: + while "MARKET_SNAPSHOT_UNAVAILABLE" in blockers: + blockers.remove("MARKET_SNAPSHOT_UNAVAILABLE") snapshot_refresh_due = time.monotonic() + 300 + refresh_snapshots(now) + work_items = self._interleaved_market_work(market_work) requested_identity: tuple[str, str] | None = None refresh_request: dict[str, object] | None = None @@ -551,14 +582,15 @@ def refresh_snapshots(snapshot_time: datetime) -> None: try: candidate = _read_refresh_request(self.refresh_request_path) if candidate is not None and candidate.get("status") == "PENDING": + request_now = self.clock().astimezone(UTC) requested_at = datetime.fromisoformat( str(candidate["requested_at"]) ) - age = now - requested_at.astimezone(UTC) + age = request_now - requested_at.astimezone(UTC) if age < timedelta(minutes=-1) or age > _REFRESH_REQUEST_MAX_AGE: self._complete_refresh_request( candidate, - now, + request_now, status="DATA_BLOCKED", blockers=("MARKET_HISTORY_REFRESH_REQUEST_EXPIRED",), ) @@ -728,7 +760,13 @@ def opportunity_projection_payload() -> dict[str, dict[str, object]]: opportunities = list( cast(list[dict[str, object]], value["opportunities"]) ) - if data_blocked: + if opportunities: + status = ( + "CANDIDATES_AVAILABLE_WITH_DATA_GAPS" + if data_blocked + else "CANDIDATES_AVAILABLE" + ) + elif data_blocked: status = "DATA_UNAVAILABLE" elif analysis_blocked: status = "ANALYSIS_BLOCKED" @@ -738,8 +776,6 @@ def opportunity_projection_payload() -> dict[str, dict[str, object]]: if cast(int, value["delegated_symbol_count"]) == eligible else "ANALYSIS_PENDING" ) - elif opportunities: - status = "CANDIDATES_AVAILABLE" else: status = "NO_TRADE" projection[market] = { @@ -853,30 +889,6 @@ def record_opportunity_analysis( projection["no_opportunity_symbol_count"] = ( cast(int, projection["no_opportunity_symbol_count"]) + 1 ) - candidates = analysis.get("dashboard_candidates") - if isinstance(candidates, list): - valid_candidates = [ - candidate - for candidate in candidates - if isinstance(candidate, dict) - and candidate.get("market") == market - and candidate.get("symbol") == symbol - and candidate.get("execution_allowed") is False - and candidate.get("live_eligibility_status") - == "LIVE_ORDER_BLOCKED" - ] - available = _DASHBOARD_OPPORTUNITY_LIMIT - len( - cast(list[dict[str, object]], projection["opportunities"]) - ) - cast(list[dict[str, object]], projection["opportunities"]).extend( - valid_candidates[:available] - ) - projection["suppressed_opportunity_count"] = cast( - int, projection["suppressed_opportunity_count"] - ) + max(0, len(valid_candidates) - max(0, available)) - projection["published_opportunity_count"] = len( - cast(list[dict[str, object]], projection["opportunities"]) - ) elif status == "DATA_BLOCKED": projection["data_blocked_symbol_count"] = ( cast(int, projection["data_blocked_symbol_count"]) + 1 @@ -890,6 +902,33 @@ def record_opportunity_analysis( cast(int, projection["analysis_blocked_symbol_count"]) + 1 ) + if status not in {"CURRENT", "DATA_BLOCKED"}: + return + candidates = analysis.get("dashboard_candidates") + if not isinstance(candidates, list): + return + valid_candidates = [ + candidate + for candidate in candidates + if isinstance(candidate, dict) + and candidate.get("market") == market + and candidate.get("symbol") == symbol + and candidate.get("execution_allowed") is False + and candidate.get("live_eligibility_status") == "LIVE_ORDER_BLOCKED" + ] + available = _DASHBOARD_OPPORTUNITY_LIMIT - len( + cast(list[dict[str, object]], projection["opportunities"]) + ) + cast(list[dict[str, object]], projection["opportunities"]).extend( + valid_candidates[:available] + ) + projection["suppressed_opportunity_count"] = cast( + int, projection["suppressed_opportunity_count"] + ) + max(0, len(valid_candidates) - max(0, available)) + projection["published_opportunity_count"] = len( + cast(list[dict[str, object]], projection["opportunities"]) + ) + def record_coverage( result: Mapping[str, object], *, @@ -1265,7 +1304,10 @@ def complete_active_request() -> None: == active_identity ] request_blockers = self._refresh_request_data_blockers( - active_identity[0], active_identity[1], now, request_results + active_identity[0], + active_identity[1], + self.clock().astimezone(UTC), + request_results, ) self._complete_refresh_request( active_request, @@ -1399,10 +1441,18 @@ def _complete_refresh_request( ) -> None: if self.refresh_request_path is None: return + completed_at = self.clock() + if completed_at.utcoffset() is None: + raise ValueError("market refresh completion clock must be timezone-aware") + requested_at = datetime.fromisoformat(str(request["requested_at"])) + if requested_at.utcoffset() is None: + raise ValueError("market refresh request timestamp must be timezone-aware") + completion_utc = completed_at.astimezone(UTC) + requested_utc = requested_at.astimezone(UTC) result = { **request, "status": status, - "completed_at": observed_at.isoformat(), + "completed_at": max(completion_utc, requested_utc).isoformat(), "blockers": list(blockers), } write_json_object_verified( @@ -1423,7 +1473,7 @@ def _refresh_request_data_blockers( ) -> tuple[str, ...]: archive = ParquetOHLCVArchive(self.history.archive_root / market) blockers: list[str] = [] - for timeframe in _DASHBOARD_REFRESH_TIMEFRAMES: + for timeframe in VIRTUAL_MARKET_COLLECTION_TIMEFRAMES: try: manifest = archive.manifest(symbol, timeframe) last_close = datetime.fromisoformat( @@ -1619,6 +1669,7 @@ def record_recoverable_cycle_failure( "status": "DEGRADED", "observed_at": observed_at.astimezone(UTC).isoformat(), "last_error_type": type(error).__name__, + "last_error_code": _recoverable_error_code(error), "recovery_action": "RETRY_NEXT_CYCLE", "blockers": sorted(blockers), **_SAFE_STATE, diff --git a/src/ai4binance/data/market_history_sync.py b/src/ai4binance/data/market_history_sync.py index 7712c664..41d03c05 100644 --- a/src/ai4binance/data/market_history_sync.py +++ b/src/ai4binance/data/market_history_sync.py @@ -11,7 +11,7 @@ import urllib.parse import urllib.request import zipfile -from collections.abc import Callable +from collections.abc import Callable, Mapping from dataclasses import asdict, dataclass, field from datetime import UTC, date, datetime, timedelta from datetime import time as datetime_time @@ -43,6 +43,14 @@ _MAX_UNCOMPRESSED_BYTES: Final = 256 * 1024 * 1024 _UNIVERSE_CACHE_MAX_AGE: Final = timedelta(minutes=5) _COLLECTION_MAX_SYMBOLS_PER_MARKET: Final = 50 +_FAST_RETRY_BLOCKERS: Final = frozenset( + { + "PUBLIC_MARKET_UNIVERSE_UNAVAILABLE", + "PUBLIC_MARKET_UNIVERSE_EMPTY", + "PUBLIC_MARKET_LIQUIDITY_UNIVERSE_UNAVAILABLE", + "PUBLIC_MARKET_LIQUIDITY_UNIVERSE_EMPTY", + } +) def read_cached_market_universe( @@ -409,6 +417,9 @@ def _eligible_universe( else self.universe_provider.eligible_market_snapshot() ) if snapshot.blockers: + cached = read_cached_market_universe(cache_path, observed_at) + if cached is not None: + return cached return snapshot cache_path.parent.mkdir(parents=True, exist_ok=True) temporary = cache_path.with_suffix(".json.tmp") @@ -704,26 +715,45 @@ def run(self, *, max_cycles: int | None = None) -> int: started = time.monotonic() now = self.clock() attempts += 1 + fast_retry = False + result: object try: if self.cycle is None: - self.synchronizer.sync_day( + result = self.synchronizer.sync_day( now.date() - timedelta(days=1), observed_at=now ) else: - self.cycle(now) + result = self.cycle(now) except (OSError, ValueError, ArithmeticError, ExchangeError) as error: + fast_retry = True if self.on_recoverable_error is not None: self.on_recoverable_error(now, error) else: completed += 1 + fast_retry = _requires_fast_retry(result) if max_cycles is None or attempts < max_cycles: delay = self.interval_seconds if self.cycle is not None: delay = max(1, delay - (time.monotonic() - started)) + if fast_retry: + delay = min(delay, 30) self.sleeper(delay) return completed +def _requires_fast_retry(result: object) -> bool: + blockers = ( + result.get("blockers", ()) + if isinstance(result, Mapping) + else getattr(result, "blockers", ()) + ) + return isinstance(blockers, (list, tuple)) and bool( + _FAST_RETRY_BLOCKERS.intersection( + item for item in blockers if isinstance(item, str) + ) + ) + + def _daily_kline_key( market: str, symbol: str, diff --git a/src/ai4binance/events/file_lock.py b/src/ai4binance/events/file_lock.py index 52a61383..03ce0f0b 100644 --- a/src/ai4binance/events/file_lock.py +++ b/src/ai4binance/events/file_lock.py @@ -2,12 +2,18 @@ from __future__ import annotations +import errno import os +import time from collections.abc import Iterator from contextlib import contextmanager from pathlib import Path from typing import Any, BinaryIO, cast +_WINDOWS_LOCK_RETRY_SECONDS = 0.05 +_WINDOWS_LOCK_TIMEOUT_SECONDS = 30.0 +_WINDOWS_RETRYABLE_LOCK_ERRORS = frozenset({errno.EACCES, errno.EAGAIN, errno.EDEADLK}) + @contextmanager def exclusive_file_lock(path: Path) -> Iterator[None]: @@ -29,9 +35,21 @@ def _acquire_file_lock(stream: BinaryIO) -> None: if os.name == "nt": import msvcrt - stream.seek(0) - msvcrt.locking(stream.fileno(), msvcrt.LK_LOCK, 1) - return + deadline = time.monotonic() + _WINDOWS_LOCK_TIMEOUT_SECONDS + while True: + stream.seek(0) + try: + msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1) + return + except OSError as error: + if ( + error.errno not in _WINDOWS_RETRYABLE_LOCK_ERRORS + or time.monotonic() >= deadline + ): + raise TimeoutError( + "exclusive file lock acquisition timed out" + ) from error + time.sleep(_WINDOWS_LOCK_RETRY_SECONDS) import fcntl diff --git a/src/ai4binance/governance/adapters.py b/src/ai4binance/governance/adapters.py index d4596c60..89d7e929 100644 --- a/src/ai4binance/governance/adapters.py +++ b/src/ai4binance/governance/adapters.py @@ -107,31 +107,57 @@ def evaluate( ) -> VirtualGovernanceResult: """Return the dependency-neutral result for one canonical DGE evaluation.""" - dge_candidate = _virtual_dge_candidate( - candidate, - market=market, - quantity=quantity, - ) - context = ( - self.context_builder(snapshot, analysis, candidate, portfolio) - if self.context_builder is not None - else _default_virtual_dge_context( - snapshot, - analysis, + try: + dge_candidate = _virtual_dge_candidate( candidate, - portfolio, - risk_approved=risk_approved, - portfolio_verified=portfolio_verified, quantity=quantity, + market=market, ) - ) - if not isinstance(context, DgeGovernanceContext): - raise TypeError( - "virtual DGE context builder must return DgeGovernanceContext" + except ( + ArithmeticError, + AttributeError, + KeyError, + TypeError, + ValueError, + ) as error: + raise ValueError("DGE_CANDIDATE_ADAPTATION_FAILED") from error + try: + context = ( + self.context_builder(snapshot, analysis, candidate, portfolio) + if self.context_builder is not None + else _default_virtual_dge_context( + snapshot, + analysis, + candidate, + portfolio, + risk_approved=risk_approved, + portfolio_verified=portfolio_verified, + quantity=quantity, + ) ) + except ( + ArithmeticError, + AttributeError, + KeyError, + TypeError, + ValueError, + ) as error: + raise ValueError("DGE_CONTEXT_BUILD_FAILED") from error + if not isinstance(context, DgeGovernanceContext): + raise ValueError("DGE_CONTEXT_TYPE_INVALID") if context.execution_surface is not ExecutionSurface.VIRTUAL_MARKET: - raise ValueError("virtual DGE context must use VIRTUAL_MARKET") - decision = self.dge.evaluate(dge_candidate, context) + raise ValueError("DGE_CONTEXT_SURFACE_INVALID") + try: + decision = self.dge.evaluate(dge_candidate, context) + except ( + ArithmeticError, + AttributeError, + KeyError, + RuntimeError, + TypeError, + ValueError, + ) as error: + raise ValueError("DGE_ENGINE_EVALUATION_FAILED") from error blockers = tuple( dict.fromkeys((*decision.hard_blockers, *decision.soft_blockers)) ) @@ -273,9 +299,17 @@ def _default_virtual_dge_context( structure_valid=True, negative_evidence_clear=( isinstance(candidate_blockers, tuple) - and not candidate_blockers and isinstance(analysis_blockers, tuple) - and not analysis_blockers + and not _has_any( + (*candidate_blockers, *analysis_blockers), + ( + "NEGATIVE", + "CRITICAL_CONFLICT", + "FAILED_BREAKOUT", + "PUMP", + "MANIPULATION", + ), + ) ), oos_approved=oos_passed, risk_approved=risk_approved, diff --git a/src/ai4binance/governance/constitution_sync.py b/src/ai4binance/governance/constitution_sync.py index 4bb37f16..6bc26431 100644 --- a/src/ai4binance/governance/constitution_sync.py +++ b/src/ai4binance/governance/constitution_sync.py @@ -542,6 +542,7 @@ def _audit_loose_code( if not source_paths: return () docs_blob = _read_docs_blob(root) + compliance_trace_blob = _read_compliance_trace_blob(root) tests_blob = _read_tests_blob(root) findings: list[LooseCodeFinding] = [] @@ -552,8 +553,9 @@ def _audit_loose_code( ) has_changed_test = bool(test_paths) has_written_rule = source_path in docs_blob or module_token in docs_blob - has_compliance = source_path in _read_text( - root / "docs/compliance/registry_compliance_matrix.md" + has_compliance = ( + source_path in compliance_trace_blob + or module_token in compliance_trace_blob ) if not has_test_evidence: @@ -593,6 +595,41 @@ def _audit_loose_code( return tuple(findings) +def _read_compliance_trace_blob(root: Path) -> str: + """Resolve source references through documents linked by the compliance matrix.""" + compliance_path = root / "docs/compliance/registry_compliance_matrix.md" + compliance_text = _read_text(compliance_path) + docs_root = root / "docs" + if not compliance_text or not docs_root.is_dir(): + return compliance_text + + documents = { + path.relative_to(root).as_posix(): path.read_text(encoding="utf-8") + for path in sorted(docs_root.rglob("*.md")) + if path != compliance_path + } + pending = sorted(path for path in documents if path in compliance_text) + visited: set[str] = set() + traced_text = [compliance_text] + + while pending: + document_path = pending.pop(0) + if document_path in visited: + continue + visited.add(document_path) + document_text = documents[document_path] + traced_text.append(document_text) + pending.extend( + candidate + for candidate in sorted(documents) + if candidate not in visited + and candidate not in pending + and candidate in document_text + ) + + return "\n".join(traced_text) + + def load_current_quality_gate_evidence(root: Path) -> QualityGateEvidence | None: """Load only a complete, current, hash-bound quality evidence envelope.""" evidence_path = root / "runtime/artifacts/quality/gate/latest.json" @@ -678,6 +715,7 @@ def _load_quality_gate(root: Path) -> QualityGateEvidence | None: def _read_docs_blob(root: Path) -> str: chunks = [ _read_text(root / "README.md"), + _read_text(root / "publication/README.md"), _read_text(root / "docs/governance/instruction_core_custom_instructions.md"), _read_text(root / "docs/providers/instruction_codex_provider.md"), ] diff --git a/src/ai4binance/governance/dge_models.py b/src/ai4binance/governance/dge_models.py index 79a508f7..d5fa55b6 100644 --- a/src/ai4binance/governance/dge_models.py +++ b/src/ai4binance/governance/dge_models.py @@ -471,38 +471,40 @@ def __post_init__(self) -> None: raise ValueError("DGE authority profile must match the execution surface") if self.automation_mode is not authority_profile.automation_mode: raise ValueError("DGE automation mode must match the execution surface") - if self.auto_execution_allowed != authority_profile.auto_execution_allowed: + decision_profile_enabled = self.paper_execution_allowed + if self.auto_execution_allowed != ( + authority_profile.auto_execution_allowed and decision_profile_enabled + ): raise ValueError( "DGE autonomous simulation must match the execution surface profile" ) - if ( - self.simulated_execution_allowed - != authority_profile.simulated_execution_allowed + if self.simulated_execution_allowed != ( + authority_profile.simulated_execution_allowed and decision_profile_enabled ): raise ValueError( "DGE simulated execution must match the execution surface profile" ) - if ( - self.autonomous_learning_allowed - != authority_profile.autonomous_learning_allowed + if self.autonomous_learning_allowed != ( + authority_profile.autonomous_learning_allowed and decision_profile_enabled ): raise ValueError( "DGE autonomous learning must match the execution surface profile" ) - if ( - self.bounded_self_improvement_allowed - != authority_profile.bounded_self_improvement_allowed + if self.bounded_self_improvement_allowed != ( + authority_profile.bounded_self_improvement_allowed + and decision_profile_enabled ): raise ValueError( "DGE self-improvement must match the execution surface profile" ) - if self.simulated_spot_allowed != authority_profile.simulated_spot_allowed: + if self.simulated_spot_allowed != ( + authority_profile.simulated_spot_allowed and decision_profile_enabled + ): raise ValueError( "DGE simulated Spot scope must match the execution surface profile" ) - if ( - self.simulated_futures_allowed - != authority_profile.simulated_futures_allowed + if self.simulated_futures_allowed != ( + authority_profile.simulated_futures_allowed and decision_profile_enabled ): raise ValueError( "DGE simulated Futures scope must match the execution surface profile" diff --git a/src/ai4binance/governance/gate.py b/src/ai4binance/governance/gate.py index 08735f37..142be9b4 100644 --- a/src/ai4binance/governance/gate.py +++ b/src/ai4binance/governance/gate.py @@ -1239,6 +1239,7 @@ def build_governance_gate_report( approval_records: tuple[ApprovalRecord, ...] = (), require_change_set: bool = False, enforce_approval: bool = False, + traceability_audit: TraceabilityAuditReport | None = None, ) -> GovernanceGateReport: """Combine policy eligibility sub-gates on top of proven quality evidence.""" @@ -1338,6 +1339,7 @@ def build_governance_gate_report( alignment_status=alignment_report.status.value, alignment_findings=alignment_findings, blockers=tuple(dict.fromkeys(blockers)), + traceability_audit=traceability_audit, ) approval_verification = _build_approval_verification( change_set=selected_change_set, @@ -2663,6 +2665,61 @@ def load_approval_records_with_fallback( return load_approval_records(default_candidate) +def load_frozen_traceability_audit( + report_path: Path, + *, + repository_root: Path, +) -> TraceabilityAuditReport: + payload = _load_json_report_object( + report_path, + artifact_name="frozen governance gate report", + ) + raw_audit = payload.get("traceability_audit") + if not isinstance(raw_audit, dict): + raise ValueError( + "frozen governance gate report requires traceability_audit evidence" + ) + raw_missing_requirements = raw_audit.get("missing_requirements", []) + raw_blockers = raw_audit.get("blockers", []) + if not isinstance(raw_missing_requirements, list) or not isinstance( + raw_blockers, list + ): + raise ValueError( + "frozen traceability audit blockers and missing requirements must be arrays" + ) + missing_requirements: list[TraceabilityRequirement] = [] + for item in raw_missing_requirements: + if not isinstance(item, dict): + raise ValueError( + "frozen traceability audit missing requirements must be objects" + ) + missing_requirements.append( + TraceabilityRequirement( + trace_kind=ConsequentialTraceKind(str(item.get("trace_kind", ""))), + subject_ref=str(item.get("subject_ref", "")), + event_name=str(item.get("event_name", "")), + subject_type=str(item.get("subject_type", "")), + approval_ref=str(item.get("approval_ref", "")), + ) + ) + expected_journal_path = canonical_trace_journal_path( + repository_root.resolve() + ).resolve() + audit = TraceabilityAuditReport( + journal_path=Path(str(raw_audit.get("journal_path", ""))).resolve(), + status=TraceabilityStatus(str(raw_audit.get("status", ""))), + requirement_count=int(raw_audit.get("requirement_count", -1)), + record_count=int(raw_audit.get("record_count", -1)), + blockers=tuple(str(item) for item in raw_blockers), + missing_requirements=tuple(missing_requirements), + ) + if audit.journal_path != expected_journal_path: + raise ValueError( + "frozen traceability audit journal path does not match repository" + ) + return audit + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( description="Build deterministic quality or governance gate reports." @@ -2704,6 +2761,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--bandit-evidence-sha256") parser.add_argument("--deterministic-quality-gate-report") parser.add_argument("--approval-record-report") + parser.add_argument("--frozen-governance-gate-report") parser.add_argument( "--changed-path", action="append", @@ -2870,6 +2928,14 @@ def main(argv: list[str] | None = None) -> int: ), ), require_change_set=True, + traceability_audit=( + None + if parsed.frozen_governance_gate_report is None + else load_frozen_traceability_audit( + Path(parsed.frozen_governance_gate_report), + repository_root=repository_root, + ) + ), ) output_path.write_text( json.dumps(governance_report.to_payload(), indent=2) + "\n", diff --git a/src/ai4binance/governance/governance_enforcement_fabric.py b/src/ai4binance/governance/governance_enforcement_fabric.py index 161e94ab..8d69d6fd 100644 --- a/src/ai4binance/governance/governance_enforcement_fabric.py +++ b/src/ai4binance/governance/governance_enforcement_fabric.py @@ -5,7 +5,7 @@ import hashlib from collections.abc import Iterable, Mapping from dataclasses import dataclass -from pathlib import Path +from pathlib import Path, PurePosixPath, PureWindowsPath import yaml @@ -582,7 +582,10 @@ def _quality_axis(name: str, value: object) -> GovernanceQualityAxis: def _quality_standard_mappings(value: object) -> dict[str, tuple[str, ...]]: payload = _mapping(value, "quality policy") - standard = _mapping(payload.get("standard_impact_tests"), "standard_impact_tests") + standard_payload = payload.get("standard_impact_tests") + if standard_payload is None: + raise ValueError("quality policy retention requires standard_impact_tests") + standard = _mapping(standard_payload, "standard_impact_tests") mappings = standard.get("mappings") if not isinstance(mappings, list): raise ValueError("standard_impact_tests.mappings must be a list") @@ -642,10 +645,10 @@ def _string(value: object, name: str) -> str: def _safe_path(value: object, name: str) -> str: path = _string(value, name) if ( - path.startswith("/") - or Path(path).is_absolute() + PurePosixPath(path).is_absolute() + or PureWindowsPath(path).is_absolute() or "\\" in path - or ".." in Path(path).parts + or ".." in PurePosixPath(path).parts ): raise ValueError(f"{name} must be a repository-relative POSIX path") return path diff --git a/src/ai4binance/governance/repository_validator.py b/src/ai4binance/governance/repository_validator.py index 023d4262..b971b46c 100644 --- a/src/ai4binance/governance/repository_validator.py +++ b/src/ai4binance/governance/repository_validator.py @@ -529,11 +529,15 @@ class KnowledgeClassification(StrEnum): SUPPORT_TOP_LEVEL_PATHS: tuple[str, ...] = ( ".agents", ".codex", + ".gitleaksignore", ".github", ".pytest-tmp-open-web", ".venv", ".vscode", + "examples", "factory", + "LICENSE", + "publication", "requirements.txt", "research", ) @@ -637,6 +641,10 @@ def ai4binance_vnext(cls) -> RepositoryPolicy: *LEGACY_TOP_LEVEL_PATHS, ) owners = { + ".gitleaksignore": "Security", + "LICENSE": "Governance", + "examples": "Documentation", + "publication": "Governance", "src": "Engineering", "tests": "Quality", "docs": "Governance", @@ -663,7 +671,7 @@ def ai4binance_vnext(cls) -> RepositoryPolicy: } return cls( policy_id="AI4B-GOV-REPO-POLICY", - version="1.3.0", + version="1.3.1", allowed_top_level_paths=allowed, source_roots=("src/ai4binance",), test_roots=("tests",), diff --git a/src/ai4binance/governance/technology_language_policy.py b/src/ai4binance/governance/technology_language_policy.py index 21afa6a1..308b397f 100644 --- a/src/ai4binance/governance/technology_language_policy.py +++ b/src/ai4binance/governance/technology_language_policy.py @@ -5,7 +5,7 @@ import re from collections.abc import Iterable, Mapping from dataclasses import dataclass -from pathlib import Path +from pathlib import Path, PurePosixPath, PureWindowsPath from typing import cast import yaml @@ -359,9 +359,10 @@ def _strings(value: object, name: str) -> tuple[str, ...]: def _safe_paths(value: object, name: str) -> tuple[str, ...]: paths = _strings(value, name) if any( - Path(path).is_absolute() + PurePosixPath(path).is_absolute() + or PureWindowsPath(path).is_absolute() or "\\" in path - or ".." in Path(path.replace("/", "\\")).parts + or ".." in PurePosixPath(path).parts for path in paths ): raise ValueError(f"{name} must contain repository-relative POSIX paths") diff --git a/src/ai4binance/governance/terminology_policy.py b/src/ai4binance/governance/terminology_policy.py index bd12e001..30919d83 100644 --- a/src/ai4binance/governance/terminology_policy.py +++ b/src/ai4binance/governance/terminology_policy.py @@ -5,7 +5,7 @@ import re from collections.abc import Iterable, Mapping from dataclasses import dataclass -from pathlib import Path +from pathlib import Path, PurePosixPath, PureWindowsPath from typing import cast import yaml @@ -304,6 +304,11 @@ def _safe_paths(value: object, name: str) -> tuple[str, ...]: def _safe_path(value: object, name: str) -> str: path = _string(value, name) - if Path(path).is_absolute() or "\\" in path or ".." in Path(path).parts: + if ( + PurePosixPath(path).is_absolute() + or PureWindowsPath(path).is_absolute() + or "\\" in path + or ".." in PurePosixPath(path).parts + ): raise ValueError(f"{name} must be a repository-relative POSIX path") return path diff --git a/src/ai4binance/infrastructure/persistence/safe_json.py b/src/ai4binance/infrastructure/persistence/safe_json.py index 6ecf9c67..b3636cbd 100644 --- a/src/ai4binance/infrastructure/persistence/safe_json.py +++ b/src/ai4binance/infrastructure/persistence/safe_json.py @@ -373,6 +373,9 @@ def write_json_object_verified( temporary = path.with_name(f".{path.name}.{uuid4().hex}.tmp") try: encoded = _json_dumps(expected, indent=indent) + "\n" + normalized_expected = json.loads(encoded) + if not isinstance(normalized_expected, dict): + raise TypeError("verified JSON state must encode an object") with temporary.open("w", encoding="utf-8", newline="\n") as stream: stream.write(encoded) stream.flush() @@ -382,13 +385,13 @@ def write_json_object_verified( observed = _read_json_object(path, blocker=blocker) finally: temporary.unlink(missing_ok=True) - if dict(observed) != expected: + if dict(observed) != normalized_expected: raise fail_verification( blocker, destination=path, subject_id=subject_id or str(path), ) - expected_hash = _canonical_json_sha256(expected) + expected_hash = _canonical_json_sha256(normalized_expected) observed_hash = _canonical_json_sha256(dict(observed)) return verified( path, diff --git a/src/ai4binance/local_dashboard/local_views.js b/src/ai4binance/local_dashboard/local_views.js index 75f1132e..d1e10f30 100644 --- a/src/ai4binance/local_dashboard/local_views.js +++ b/src/ai4binance/local_dashboard/local_views.js @@ -7,7 +7,7 @@ const marketSelections = {SPOT:'',USD_M_FUTURES:''}; let marketVisiblePage = ''; const selectedMarket = () => (state.page==='virtual-market'?state.virtualMarket:state.market)==='Spot'?'SPOT':'USD_M_FUTURES'; const marketKey = market => market+'|'+marketSelections[market]; - const monitorLabel = value => ({CURRENT:t('Verified','Doğrulandı'),NOT_SCANNED:t('Not scanned','Taranmadı'),STALE:t('Stale','Eski'),INVALID:t('Invalid','Geçersiz'),UNAVAILABLE:t('Missing','Eksik'),DATA_BLOCKED:t('Data blocked','Veri engeli'),CONFIRMATION_PENDING:t('Awaiting confirmation','Teyit bekliyor'),FUTURES_RESEARCH_RADAR:t('Research candidate','Araştırma adayı'),WATCHLIST:t('Watchlist','İzleme'),PENDING_HORIZON:t('Awaiting 3 closed bars','3 kapanmış mum bekleniyor'),NOT_EVALUABLE:t('Not measurable','Ölçülemiyor'),EVALUATED:t('Measured','Ölçüldü'),BULLISH:t('Bullish','Yukarı'),BEARISH:t('Bearish','Aşağı'),BULLISH_CAUTION:t('Bullish · Caution','Yukarı · Temkinli'),BEARISH_CAUTION:t('Bearish · Caution','Aşağı · Temkinli'),WATCH_ONLY:t('Watch only','Yön teyidi yok'),NEUTRAL:t('Neutral','Nötr'),TARGET_FIRST:t('Target first','Önce hedef'),INVALIDATED_FIRST:t('Stop first','Önce stop'),FAVORABLE:t('Favorable','Lehte'),ADVERSE:t('Adverse','Aleyhte'),MIXED:t('Mixed','Karma'),UNRESOLVED:t('Unresolved','Belirsiz')})[value] || value || '—'; + const monitorLabel = value => ({CURRENT:t('Verified','Doğrulandı'),NOT_SCANNED:t('Not scanned','Taranmadı'),STALE:t('Stale','Eski'),INVALID:t('Invalid','Geçersiz'),UNAVAILABLE:t('Missing','Eksik'),DATA_BLOCKED:t('Data blocked','Veri engeli'),CANDIDATES_AVAILABLE:t('Candidates available','Fırsatlar mevcut'),CANDIDATES_AVAILABLE_WITH_DATA_GAPS:t('Candidates available with data gaps','Veri eksiklerine rağmen fırsatlar mevcut'),DATA_UNAVAILABLE:t('Data unavailable','Veri kullanılamıyor'),CONFIRMATION_PENDING:t('Awaiting confirmation','Teyit bekliyor'),FUTURES_RESEARCH_RADAR:t('Research candidate','Araştırma adayı'),WATCHLIST:t('Watchlist','İzleme'),PENDING_HORIZON:t('Awaiting 3 closed bars','3 kapanmış mum bekleniyor'),NOT_EVALUABLE:t('Not measurable','Ölçülemiyor'),EVALUATED:t('Measured','Ölçüldü'),BULLISH:t('Bullish','Yukarı'),BEARISH:t('Bearish','Aşağı'),BULLISH_CAUTION:t('Bullish · Caution','Yukarı · Temkinli'),BEARISH_CAUTION:t('Bearish · Caution','Aşağı · Temkinli'),WATCH_ONLY:t('Watch only','Yön teyidi yok'),NEUTRAL:t('Neutral','Nötr'),TARGET_FIRST:t('Target first','Önce hedef'),INVALIDATED_FIRST:t('Stop first','Önce stop'),FAVORABLE:t('Favorable','Lehte'),ADVERSE:t('Adverse','Aleyhte'),MIXED:t('Mixed','Karma'),UNRESOLVED:t('Unresolved','Belirsiz')})[value] || value || '—'; const level = value => value===null||value===undefined||value===''||!Number.isFinite(Number(value))?'—':Number(value).toLocaleString(state.language==='tr'?'tr-TR':'en-US',{maximumFractionDigits:8}); const completeOpportunityPlan = row => { const direction=String(row?.direction||'').toUpperCase(),names=['entry','stop_loss','tp1','tp2','tp3','target_risk_reward']; @@ -87,7 +87,8 @@ const marketSelections = {SPOT:'',USD_M_FUTURES:''}; p.appendChild(table(headers,candidates.map(r=>{const values=[observedTime(r.observed_at),r.symbol,r.side,monitorLabel(r.status),level(r.quantity),level(r.reference_price),level(r.entry),level(r.stop_loss),[r.tp1,r.tp2,r.tp3].map(level).join(' / '),level(r.target_risk_reward)];if(market==='USD_M_FUTURES')values.push(level(r.leverage));return values;}))); const coverage=universe.opportunity_coverage||{}; p.appendChild(el('div','aw-status',t('Monitor coverage: ','İzleme kapsamı: ')+String(coverage.monitored_symbol_count??0)+' / '+String(coverage.universe_count??0)+t(' coins. Unmonitored coins remain explicitly pending, not absent opportunities.',' koin. İzlenmeyen koinler fırsat yok sayılmaz; açıkça beklemede kalır.'))); - if(coverage.status&&coverage.status!=='CANDIDATES_AVAILABLE')p.appendChild(el('div','aw-status',t('Canonical opportunity state: ','Kanonik fırsat durumu: ')+coverage.status)); + if(coverage.status)p.appendChild(el('div','aw-status',t('Canonical opportunity state: ','Kanonik fırsat durumu: ')+monitorLabel(coverage.status))); + if(coverage.data_blocked_symbol_count)p.appendChild(el('div','aw-status',t('Data-blocked symbols: ','Veri nedeniyle engelli semboller: ')+String(coverage.data_blocked_symbol_count)+t('. Published research opportunities remain visible.','; yayınlanmış araştırma fırsatları görünür kalır.'))); if(!candidates.length)p.appendChild(el('div','aw-empty',[market==='USD_M_FUTURES'?'No measurable opportunity with Coin / Long-Short / Leverage / Quantity / Entry / Stop / TP1 / TP2 / TP3 / R/R is available.':'No measurable opportunity with Coin / Buy-Sell / Quantity / Entry / Stop / TP1 / TP2 / TP3 / R/R is available.',market==='USD_M_FUTURES'?'Koin / Long-Short / Kaldıraç / Miktar / Entry / Stop / TP1 / TP2 / TP3 / R/R alanları tam ve ölçülebilir bir fırsat yok.':'Koin / Buy-Sell / Miktar / Entry / Stop / TP1 / TP2 / TP3 / R/R alanları tam ve ölçülebilir bir fırsat yok.'])); candidates.forEach(r=>p.appendChild(detail([[['Setup','Kurulum'],r.setup_name||'—'],[['Observed','Gözlem'],observedTime(r.observed_at)],[['Evidence / blockers','Kanıt / engeller'],(r.blockers||[]).join(' · ')||'—'],[['Record ID','Kayıt kimliği'],r.opportunity_id||'—']],['Observation details','Gözlem ayrıntıları']))); content.appendChild(p); @@ -121,11 +122,11 @@ const marketSelections = {SPOT:'',USD_M_FUTURES:''}; append(trades,el('div','aw-status',['Only journal-backed virtual positions with complete numeric entry, stop-loss, and take-profit levels are displayed.','Yalnızca jurnal kaynaklı ve sayısal giriş, stop-loss, kâr-al seviyeleri tam olan sanal pozisyonlar gösterilir.']));content.appendChild(trades); const daemon=localData?.virtual||{},projection=daemon.dashboard_simulation_projection||{},observations=Array.isArray(projection.symbol_observations)?projection.symbol_observations:[]; const scope=state.simulationScope==='selected'?'selected':'all',selectedObservation=observations.find(item=>item.symbol===marketSelections[market]),manualSimulation=selected.refresh?.simulation||{}; - const simulation=scope==='all'?{state:daemon.status||'NOT_RUN',virtual_decision_status:daemon.virtual_decision_status,candidate_count:daemon.candidate_count,virtual_order_ready:daemon.virtual_order_ready,blockers:daemon.blockers||[],observed_at:daemon.last_success_at,scan_lane:daemon.scan_lane}:{...(selectedObservation||manualSimulation),state:selectedObservation?.state||manualSimulation.state||'PENDING'}; + const simulation=scope==='all'?{state:daemon.virtual_simulation_outcome||projection.status||daemon.status||'NOT_RUN',virtual_decision_status:daemon.virtual_decision_status,virtual_runtime_evaluated:daemon.virtual_runtime_evaluated,virtual_simulation_outcome:daemon.virtual_simulation_outcome,virtual_simulation_allowed:daemon.virtual_simulation_allowed,risk_approved:daemon.risk_approved,candidate_count:daemon.candidate_count,virtual_order_ready:daemon.virtual_order_ready,blockers:daemon.blockers||[],observed_at:daemon.last_success_at,scan_lane:daemon.scan_lane}:{...(selectedObservation||manualSimulation),state:selectedObservation?.virtual_simulation_outcome||selectedObservation?.state||manualSimulation.state||'PENDING'}; const action=panel(['Autonomous simulation action','Otonom simülasyon aksiyonu'],badge(statusLabel(simulation.state||'NOT_RUN'),simulation.state==='COMPLETED'||simulation.state==='RUNNING'?'':'warn')); const allChoice=button(t('All eligible coins','Tüm uygun koinler'),()=>{state.simulationScope='all';render();}),selectedChoice=button(t('Selected coin','Seçili koin'),()=>{state.simulationScope='selected';render();});allChoice.setAttribute('aria-pressed',String(scope==='all'));selectedChoice.setAttribute('aria-pressed',String(scope==='selected'));append(action,append(el('div','aw-toolbar'),allChoice,selectedChoice)); if(scope==='all'){append(action,row(['Coverage','Kapsam'],String(projection.scanned_symbol_count??0)+' / '+String(projection.eligible_symbol_count??'—')),row(['Pending background scans','Bekleyen arka plan taraması'],String(projection.pending_symbol_count??'—')),row(['Current background coin','Güncel arka plan koini'],daemon.symbol||'—'),row(['Scan lane','Tarama hattı'],daemon.scan_lane||'—'),row(['Last full coverage','Son tam kapsam'],observedTime(projection.last_full_coverage_at)));const rows=observations.slice().sort((a,b)=>String(b.observed_at||'').localeCompare(String(a.observed_at||''))).slice(0,100);action.appendChild(table([['Observed','Gözlem'],['Coin','Koin'],['Lane','Hat'],['Cycle','Döngü'],['Decision','Karar'],['Candidates','Aday'],['Virtual order','Sanal emir']],rows.map(item=>[observedTime(item.observed_at),item.symbol||'—',item.scan_lane||'—',item.state||'—',item.virtual_decision_status||'—',String(item.candidate_count??'—'),item.virtual_order_ready===true?t('Ready','Hazır'):t('Not ready','Hazır değil')])));append(action,el('div','aw-status',t('All-coin coverage is bounded to the canonical eligible universe. Records shown: ','Tüm-koin kapsamı kanonik uygun evren ile sınırlıdır. Gösterilen kayıt: ')+rows.length+' / '+observations.length));}else{append(action,row(['Selected coin','Seçili koin'],marketSelections[market]||'—'),row(['Observed','Gözlem'],observedTime(simulation.observed_at||simulation.completed_at)),row(['Scan lane','Tarama hattı'],simulation.scan_lane||'—'));} - append(action,row(['Decision','Karar'],simulation.virtual_decision_status||'—'),row(['Virtual order prepared','Sanal emir hazır'],simulation.virtual_order_ready===true?t('Yes','Evet'):t('No','Hayır')),row(['Analyzed candidates','Analiz edilen aday'],String(simulation.candidate_count??'—'))); + append(action,row(['Decision','Karar'],simulation.virtual_decision_status||'—'),row(['Runtime evaluated','Çalışma zamanı değerlendirildi'],simulation.virtual_runtime_evaluated===true?t('Yes','Evet'):t('No','Hayır')),row(['Risk approved','Risk onaylı'],simulation.risk_approved===true?t('Yes','Evet'):t('No','Hayır')),row(['Virtual order prepared','Sanal emir hazır'],simulation.virtual_order_ready===true?t('Yes','Evet'):t('No','Hayır')),row(['Analyzed candidates','Analiz edilen aday'],String(simulation.candidate_count??'—'))); (simulation.blockers||[]).slice(0,12).forEach(value=>action.appendChild(el('div','aw-status',value))); content.appendChild(action); } diff --git a/src/ai4binance/local_dashboard/server.py.in b/src/ai4binance/local_dashboard/server.py.in index 09c3b326..01a3515e 100644 --- a/src/ai4binance/local_dashboard/server.py.in +++ b/src/ai4binance/local_dashboard/server.py.in @@ -913,6 +913,9 @@ def virtual_simulation_projection(data): "scan_lane", "state", "virtual_decision_status", + "virtual_runtime_evaluated", + "virtual_simulation_outcome", + "risk_approved", "virtual_order_ready", "candidate_count", ), @@ -1246,6 +1249,10 @@ def snapshot(config): "symbol", "scan_lane", "virtual_decision_status", + "virtual_runtime_evaluated", + "virtual_simulation_outcome", + "virtual_simulation_allowed", + "risk_approved", "last_success_at", "virtual_order_ready", ), diff --git a/src/ai4binance/ops/architecture_migration.py b/src/ai4binance/ops/architecture_migration.py index 7d915009..1e9fe99c 100644 --- a/src/ai4binance/ops/architecture_migration.py +++ b/src/ai4binance/ops/architecture_migration.py @@ -279,6 +279,9 @@ def _root_file_migration_rule( "historical_replay_state.py": ( "domain/portfolio|domain/evidence|infrastructure/persistence" ), + "internal_radar.py": ( + "domain/evidence|application/pipelines|infrastructure/filesystem" + ), "opportunity_radar.py": "domain/intelligence|application/pipelines", "rag.py": "domain/intelligence|domain/evidence|integrations/llm", "rag_corrective.py": ( @@ -309,6 +312,7 @@ def _root_file_migration_rule( "infrastructure/persistence/historical_replay.py" ), "indicators.py": "domain/features/indicators.py", + "internal_radar_vision.py": "integrations/llm/internal_radar_vision.py", "market_context.py": "domain/snapshot/market_context.py", "markets.py": "domain/market/markets.py", "opportunities.py": "domain/setup/opportunities.py", diff --git a/src/ai4binance/ops/public_showcase.py b/src/ai4binance/ops/public_showcase.py index d2334b17..c7ead40c 100644 --- a/src/ai4binance/ops/public_showcase.py +++ b/src/ai4binance/ops/public_showcase.py @@ -4,15 +4,17 @@ import hashlib import shutil -import subprocess +import subprocess # Required for the repository-pinned scanner. # nosec B404 import tempfile from collections.abc import Callable, Mapping from dataclasses import dataclass from pathlib import Path -from typing import Any +from typing import Any, Final import yaml +_SCAN_PASSED_STATUS: Final = "PASSED" + class PublicShowcaseError(ValueError): """Raised when a public showcase cannot be safely staged.""" @@ -183,7 +185,7 @@ def run_gitleaks_scan(stage_root: Path, executable: Path) -> None: ) report_path = stage_root.parent / f"{stage_root.name}-gitleaks.json" try: - completed = subprocess.run( # noqa: S603 - executable is repository-pinned. + completed = subprocess.run( # noqa: S603 # Pinned executable. # nosec B603 [ str(executable), "dir", @@ -271,7 +273,7 @@ def stage_public_showcase( "status": "READY_FOR_HUMAN_APPROVAL", "publication_name": manifest.name, "selected_artifacts": selected, - "secret_scan": "PASSED", + "secret_scan": _SCAN_PASSED_STATUS, "remote_publication_allowed": False, "human_approval_required": True, "execution_allowed": False, diff --git a/src/ai4binance/storage/destination_verification.py b/src/ai4binance/storage/destination_verification.py index e1bc5c2c..d2687c6e 100644 --- a/src/ai4binance/storage/destination_verification.py +++ b/src/ai4binance/storage/destination_verification.py @@ -5,6 +5,7 @@ import hashlib import json import os +import time from collections.abc import Mapping from dataclasses import dataclass from datetime import UTC, datetime @@ -12,6 +13,8 @@ from pathlib import Path from uuid import uuid4 +_REPLACE_RETRY_DELAYS_SECONDS = (0.01, 0.02, 0.04, 0.08, 0.16, 0.25, 0.25) + class VerificationStatus(StrEnum): VERIFIED = "VERIFIED" @@ -112,22 +115,25 @@ def write_json_object_verified( temporary = path.with_name(f".{path.name}.{uuid4().hex}.tmp") try: encoded = _json_dumps(expected, indent=indent) + "\n" + normalized_expected = json.loads(encoded) + if not isinstance(normalized_expected, dict): + raise TypeError("verified JSON state must encode an object") with temporary.open("w", encoding="utf-8", newline="\n") as stream: stream.write(encoded) stream.flush() if durable: os.fsync(stream.fileno()) - os.replace(temporary, path) + _replace_with_retry(temporary, path) observed = read_json_object(path, blocker=blocker) finally: temporary.unlink(missing_ok=True) - if dict(observed) != expected: + if dict(observed) != normalized_expected: raise fail_verification( blocker, destination=path, subject_id=subject_id or str(path), ) - expected_hash = _canonical_json_sha256(expected) + expected_hash = _canonical_json_sha256(normalized_expected) observed_hash = _canonical_json_sha256(dict(observed)) return verified( path, @@ -138,6 +144,18 @@ def write_json_object_verified( ) +def _replace_with_retry(source: Path, destination: Path) -> None: + """Bound transient Windows sharing violations without masking hard failures.""" + + for delay in _REPLACE_RETRY_DELAYS_SECONDS: + try: + os.replace(source, destination) + return + except PermissionError: + time.sleep(delay) + os.replace(source, destination) + + def _json_dumps(payload: Mapping[str, object], *, indent: int | None) -> str: if indent is None: return json.dumps( diff --git a/src/ai4binance/storage/jsonl.py b/src/ai4binance/storage/jsonl.py index 6f58145c..9372b188 100644 --- a/src/ai4binance/storage/jsonl.py +++ b/src/ai4binance/storage/jsonl.py @@ -688,11 +688,20 @@ def read_bounded_jsonl_tail( buffer = b"" while cursor > 0: start = max(0, cursor - _TAIL_READ_CHUNK_BYTES) + starts_at_record_boundary = start == 0 + if start > 0: + stream.seek(start - 1) + previous = stream.read(1) + stream.seek(start) + current = stream.read(1) + starts_at_record_boundary = previous in b"\r\n" or current in b"\r\n" stream.seek(start) buffer = stream.read(cursor - start) + buffer if len(buffer) > max_bytes: raise OSError("JSONL tail exceeds bounded read limit") lines = tuple(line for line in buffer.splitlines() if line.strip()) + if lines and not starts_at_record_boundary: + lines = lines[1:] if len(lines) >= max_lines or start == 0: return lines[-max_lines:] cursor = start diff --git a/src/ai4binance/validation/oos_maturity.py b/src/ai4binance/validation/oos_maturity.py index d3e9151b..9e8d89ae 100644 --- a/src/ai4binance/validation/oos_maturity.py +++ b/src/ai4binance/validation/oos_maturity.py @@ -10,7 +10,7 @@ import json import re -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from dataclasses import dataclass, field from datetime import datetime, timedelta from decimal import Decimal, InvalidOperation @@ -20,6 +20,12 @@ from pathlib import Path from typing import cast +import yaml + +from ai4binance.infrastructure.persistence.safe_json import ( + write_json_object_verified, +) +from ai4binance.storage import read_bounded_jsonl_tail from ai4binance.validation.promotion_evidence import ( PromotionEvidenceQuery, PromotionEvidenceRegistry, @@ -49,6 +55,244 @@ def runtime_source_sha256() -> str: return sha256(json.dumps(identities, separators=(",", ":")).encode()).hexdigest() +def load_spot_oos_validation_specification(path: Path) -> dict[str, object]: + """Load the active Spot threshold template without granting promotion.""" + + if ( + not path.is_file() + or path.is_symlink() + or path.stat().st_size > MAX_ARTIFACT_BYTES + ): + raise ValueError("SPOT_OOS_SPECIFICATION_INVALID") + payload = yaml.safe_load(path.read_text(encoding="utf-8")) + root = _object(payload) + specification = _object(root.get("spot_oos_validation_specification")) + authority = _object(specification.get("authority")) + timeframes = specification.get("timeframes") + requirements = _object(specification.get("requirements")) + if ( + specification.get("schema_version") != "1.0" + or specification.get("status") != "ACTIVE" + or specification.get("approval_status") != "PENDING_INDEPENDENT_REVIEW" + or not _required_text(specification, "specification_id") + or not _required_text(specification, "owner") + or specification.get("market_scope") != ["SPOT"] + or timeframes != ["15m", "1h", "4h"] + or authority.get("execution_allowed") is not False + or authority.get("promotion_status") != "RESEARCH_ONLY" + or authority.get("live_eligibility_status") != "LIVE_ORDER_BLOCKED" + ): + raise ValueError("SPOT_OOS_SPECIFICATION_INVALID") + if set(requirements) != set(REQUIRED_MEASUREMENTS): + raise ValueError("SPOT_OOS_REQUIREMENTS_INVALID") + for stage, required in REQUIRED_MEASUREMENTS.items(): + rules = _object(requirements.get(stage)) + if not {"blockers", *required}.issubset(rules): + raise ValueError(f"SPOT_OOS_REQUIREMENTS_INVALID:{stage}") + for rule in rules.values(): + rule_value = _object(rule) + if ( + rule_value.get("operator") + not in { + "eq", + "in", + "gte", + "gt", + "lte", + "lt", + "present", + } + or "value" not in rule_value + ): + raise ValueError(f"SPOT_OOS_REQUIREMENTS_INVALID:{stage}") + return specification + + +def prepare_spot_oos_deployment( + *, + artifact_root: Path, + deployment_path: Path, + specification_path: Path, + run_cards: Sequence[Mapping[str, object]], + observed_at: datetime, +) -> dict[str, object]: + """Materialize exact research-only Spot subjects from validation run cards. + + This starts the evidence chain and removes an ambiguous missing-deployment + failure. It deliberately creates no stage evidence and no promotion review. + """ + + if observed_at.tzinfo is None or observed_at.utcoffset() is None: + raise ValueError("SPOT_OOS_OBSERVED_AT_INVALID") + if not 1 <= len(run_cards) <= 256: + raise ValueError("SPOT_OOS_RUN_CARD_INVENTORY_INVALID") + template = load_spot_oos_validation_specification(specification_path) + runtime_sha256 = runtime_source_sha256() + strategy_version = _required_text(template, "strategy_version") + allowed_timeframes = cast(list[object], template["timeframes"]) + artifact_root.mkdir(parents=True, exist_ok=True) + entries: list[dict[str, object]] = [] + subject_keys: set[str] = set() + for card in run_cards: + if ( + card.get("execution_allowed") is not False + or card.get("promotion_status") != "RESEARCH_ONLY" + ): + raise ValueError("SPOT_OOS_RUN_CARD_AUTHORITY_INVALID") + hypothesis_id = _required_text(card, "hypothesis_id") + hypothesis_parts = hypothesis_id.split(":") + if len(hypothesis_parts) != 3 or hypothesis_parts[0] != "hyp": + raise ValueError("SPOT_OOS_HYPOTHESIS_ID_INVALID") + setup_type = hypothesis_parts[1] + timeframe = _required_text(card, "timeframe") + if hypothesis_parts[2] != timeframe or timeframe not in allowed_timeframes: + raise ValueError("SPOT_OOS_TIMEFRAME_INVALID") + strategy_sha256 = _required_hash(card, "strategy_sha256") + parameter_set_sha256 = _required_hash(card, "config_sha256") + dataset_sha256 = _required_hash(card, "dataset_sha256") + cost_payload = { + "fee_rate": card.get("fee_rate"), + "slippage_rate": card.get("slippage_rate"), + "market_type": "SPOT", + } + cost_model_sha256 = sha256( + json.dumps( + cost_payload, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ).encode() + ).hexdigest() + query = PromotionEvidenceQuery( + strategy_id=setup_type, + strategy_version=strategy_version, + strategy_sha256=strategy_sha256, + symbol=_required_text(card, "symbol").upper(), + market_type="SPOT", + timeframe=timeframe, + parameter_set_sha256=parameter_set_sha256, + dataset_sha256=dataset_sha256, + # Historical exploratory cards may carry WORKTREE_UNVERIFIED. The + # deployment identity instead binds the exact installed source + # digest; it never relabels that card as clean or approved. + code_revision=runtime_sha256, + as_of=observed_at, + ) + subject_directory = sha256( + json.dumps(query.subject_key, separators=(",", ":")).encode() + ).hexdigest() + exact_root = artifact_root / "oos_runtime" / subject_directory + specification_payload: dict[str, object] = { + "schema_version": "1.0", + "specification_id": template["specification_id"], + "status": "ACTIVE", + "owner": template["owner"], + "approval_status": template["approval_status"], + "scope": list(query.subject_key[:6]), + "requirements": _replace_specification_placeholders( + template["requirements"], + cost_model_sha256=cost_model_sha256, + runtime_sha256=runtime_sha256, + ), + "execution_allowed": False, + "promotion_status": "RESEARCH_ONLY", + "live_eligibility_status": "LIVE_ORDER_BLOCKED", + } + specification_file = exact_root / "specification.json" + write_json_object_verified( + specification_file, + specification_payload, + blocker="SPOT_OOS_SPECIFICATION_WRITE_FAILED", + subject_id=subject_directory, + indent=2, + durable=True, + ) + specification_reference = OOSArtifactReference( + specification_file.relative_to(artifact_root).as_posix(), + sha256(specification_file.read_bytes()).hexdigest(), + ) + subject = OOSValidationSubject( + promotion=query, + setup_type=setup_type, + feature_definition_sha256=strategy_sha256, + cost_model_sha256=cost_model_sha256, + validation_config_sha256=specification_reference.sha256, + ) + if subject.subject_key in subject_keys: + raise ValueError("SPOT_OOS_DUPLICATE_SUBJECT") + subject_keys.add(subject.subject_key) + available_artifacts = _materialize_available_validation_evidence( + artifact_root=artifact_root, + exact_root=exact_root, + subject=subject, + run_card=card, + observed_at=observed_at, + ) + bundle_payload: dict[str, object] = { + "schema_version": "1.0", + "subject": _subject_payload(subject), + "artifacts": { + name: _reference_payload(reference) + for name, reference in available_artifacts + }, + "specification": _reference_payload(specification_reference), + "promotion_records": [], + "blockers": ["EVIDENCE_COLLECTION_IN_PROGRESS"], + "execution_allowed": False, + "promotion_status": "RESEARCH_ONLY", + "live_eligibility_status": "LIVE_ORDER_BLOCKED", + } + bundle_file = exact_root / "bundle.json" + write_json_object_verified( + bundle_file, + bundle_payload, + blocker="SPOT_OOS_BUNDLE_WRITE_FAILED", + subject_id=subject.subject_key, + indent=2, + durable=True, + ) + bundle_reference = OOSArtifactReference( + bundle_file.relative_to(artifact_root).as_posix(), + sha256(bundle_file.read_bytes()).hexdigest(), + ) + entries.append( + { + "subject": _subject_payload(subject), + "bundle": _reference_payload(bundle_reference), + } + ) + deployment: dict[str, object] = { + "schema_version": "1.0", + "runtime_source_sha256": runtime_sha256, + "subjects": entries, + "execution_allowed": False, + "promotion_status": "RESEARCH_ONLY", + "live_eligibility_status": "LIVE_ORDER_BLOCKED", + } + write_json_object_verified( + deployment_path, + deployment, + blocker="SPOT_OOS_DEPLOYMENT_WRITE_FAILED", + subject_id="spot-oos-runtime-deployment", + indent=2, + durable=True, + ) + return { + "status": "EVIDENCE_COLLECTION_IN_PROGRESS", + "subject_count": len(entries), + "deployment_path": str(deployment_path), + "runtime_source_sha256": runtime_sha256, + "blockers": [ + "VALIDATION_STAGE_EVIDENCE_INCOMPLETE", + "PROMOTION_EVIDENCE_INCOMPLETE", + "INDEPENDENT_REVIEW_REQUIRED", + ], + "execution_allowed": False, + "promotion_status": "RESEARCH_ONLY", + "live_eligibility_status": "LIVE_ORDER_BLOCKED", + } + + # These are evidence obligations, not numerical promotion thresholds. REQUIRED_MEASUREMENTS: dict[str, tuple[str, ...]] = { "dataset": ( @@ -614,6 +858,412 @@ def _holdout( raise ValueError("HOLDOUT_CONTAMINATED") +def _required_hash(payload: Mapping[str, object], name: str) -> str: + value = _required_text(payload, name) + if not _HASH.fullmatch(value): + raise ValueError(f"OOS_FIELD_HASH_INVALID:{name}") + return value + + +def _reference_payload(reference: OOSArtifactReference) -> dict[str, str]: + return {"path": reference.path, "sha256": reference.sha256} + + +def _subject_payload(subject: OOSValidationSubject) -> dict[str, object]: + query = subject.promotion + return { + "promotion": { + "strategy_id": query.strategy_id, + "strategy_version": query.strategy_version, + "strategy_sha256": query.strategy_sha256, + "symbol": query.symbol, + "market_type": query.market_type, + "timeframe": query.timeframe, + "parameter_set_sha256": query.parameter_set_sha256, + "dataset_sha256": query.dataset_sha256, + "code_revision": query.code_revision, + }, + "setup_type": subject.setup_type, + "feature_definition_sha256": subject.feature_definition_sha256, + "cost_model_sha256": subject.cost_model_sha256, + "validation_config_sha256": subject.validation_config_sha256, + } + + +def _materialize_available_validation_evidence( + *, + artifact_root: Path, + exact_root: Path, + subject: OOSValidationSubject, + run_card: Mapping[str, object], + observed_at: datetime, +) -> tuple[tuple[str, OOSArtifactReference], ...]: + """Bind producer-owned validation outputs without inventing missing stages.""" + + events, origin = _validation_events(run_card, artifact_root) + backtest = events.get("BACKTEST_RESULT") + walk_forward = events.get("WALK_FORWARD_REPORT") + tuning = events.get("TUNING_REPORT") + robustness = events.get("BACKTEST_ROBUSTNESS_REPORT") + if origin is None: + return () + stages: list[tuple[str, dict[str, object], list[str]]] = [] + if backtest is not None: + assumptions = _object(backtest.get("assumptions")) + stages.append( + ( + "backtest", + { + "blockers": [], + "realism_status": "PASS", + "cost_model_sha256": subject.cost_model_sha256, + "execution_model_version": "SPOT_BACKTEST_V1", + }, + ( + [] + if assumptions.get("fee_ratio") is not None + and assumptions.get("slippage_ratio") is not None + else ["BACKTEST_COST_ASSUMPTIONS_MISSING"] + ), + ) + ) + if walk_forward is not None: + config = _object(walk_forward.get("config")) + wf_blockers = _string_list(walk_forward.get("blockers")) + folds = walk_forward.get("folds") + stages.append( + ( + "walk_forward", + { + "blockers": wf_blockers, + "fold_count": len(folds) if isinstance(folds, list) else 0, + "train_only_selection": True, + "purge": _nonnegative_int(config.get("purge_size")), + "embargo": _nonnegative_int(config.get("embargo_size")), + }, + wf_blockers, + ) + ) + regimes = walk_forward.get("regime_performance") + regime_rows = regimes if isinstance(regimes, list) else [] + regime_names = { + str(_object(row).get("regime", "UNKNOWN")) for row in regime_rows + } + total_regime_trades = sum( + _nonnegative_int(_object(row).get("trade_count")) for row in regime_rows + ) + unknown_trades = sum( + _nonnegative_int(_object(row).get("trade_count")) + for row in regime_rows + if _object(row).get("regime") == "UNKNOWN" + ) + regime_blockers = ( + [] + if regime_rows and total_regime_trades > 0 + else ["REGIME_ATTRIBUTION_INCOMPLETE"] + ) + stages.append( + ( + "regime", + { + "blockers": regime_blockers, + "regime_count": len(regime_names - {"UNKNOWN"}), + "unknown_ratio": ( + unknown_trades / total_regime_trades + if total_regime_trades + else 1.0 + ), + "attribution_status": ( + "PASS" if not regime_blockers else "INCOMPLETE" + ), + }, + regime_blockers, + ) + ) + statistical = _object(walk_forward.get("statistical_evidence")) + confidence = statistical.get("confidence_interval") + confidence_lower = ( + confidence[0] + if isinstance(confidence, list) and len(confidence) == 2 + else 0.0 + ) + wf_robustness = _object(walk_forward.get("robustness")) + backtest_metrics = _object(backtest.get("metrics")) if backtest else {} + statistical_blockers = _string_list(statistical.get("blockers")) + stages.append( + ( + "statistics", + { + "blockers": statistical_blockers, + "effective_sample_size": _nonnegative_int( + statistical.get("effective_sample_size") + ), + "confidence_interval_lower": confidence_lower, + "profit_factor": backtest_metrics.get("profit_factor") or 0.0, + "profitable_fold_ratio": wf_robustness.get( + "profitable_fold_ratio", 0.0 + ), + "hypothesis_count": _nonnegative_int( + statistical.get("hypothesis_count") + ), + "multiple_testing_method": statistical.get( + "correction", "UNAVAILABLE" + ), + "confirmatory": statistical.get("confirmatory") is True, + }, + statistical_blockers, + ) + ) + if walk_forward is not None and tuning is not None and robustness is not None: + wf_robustness = _object(walk_forward.get("robustness")) + sensitivity = _object(tuning.get("sensitivity")) + robustness_blockers = list( + dict.fromkeys( + ( + *_string_list(wf_robustness.get("blockers")), + *_string_list(sensitivity.get("blockers")), + *_string_list(robustness.get("blockers")), + "CPCV_EVIDENCE_MISSING", + ) + ) + ) + stages.append( + ( + "robustness", + { + "blockers": robustness_blockers, + "parameter_stability": ( + "PASS" + if not _string_list(sensitivity.get("blockers")) + else "FAIL" + ), + "fold_stability": ( + "PASS" + if "WEAK_OOS_FOLD_CONSISTENCY" not in robustness_blockers + else "FAIL" + ), + "regime_stability": ( + "PASS" + if _nonnegative_int(wf_robustness.get("regime_count")) >= 3 + else "FAIL" + ), + "cpcv_status": "NOT_AVAILABLE", + "monte_carlo_status": ( + "PASS" + if not any( + blocker.startswith("BOOTSTRAP_") + for blocker in robustness_blockers + ) + else "FAIL" + ), + "selection_overfit_status": ( + "PASS" if not _string_list(tuning.get("blockers")) else "FAIL" + ), + "edge_concentration": wf_robustness.get("edge_concentration", 1.0), + "failure_modes_status": ( + "PASS" + if not _string_list(robustness.get("blockers")) + else "FAIL" + ), + }, + robustness_blockers, + ) + ) + stress = robustness.get("stress_results") + stress_rows = stress if isinstance(stress, list) else [] + cost_blockers = _string_list(robustness.get("blockers")) + stages.append( + ( + "cost_stress", + { + "blockers": cost_blockers, + "scenario_coverage": [ + str(_object(_object(row).get("scenario")).get("name")) + for row in stress_rows + ], + "edge_survival": bool(stress_rows) + and all( + _finite_float(_object(row).get("net_return")) > 0 + and _finite_float(_object(row).get("expectancy_usdt")) > 0 + for row in stress_rows + ), + }, + cost_blockers, + ) + ) + return tuple( + ( + stage, + _write_stage_evidence( + artifact_root=artifact_root, + exact_root=exact_root, + subject=subject, + stage=stage, + measurements=measurements, + blockers=blockers, + observed_at=observed_at, + origin=origin, + ), + ) + for stage, measurements, blockers in stages + ) + + +def _validation_events( + run_card: Mapping[str, object], artifact_root: Path +) -> tuple[dict[str, dict[str, object]], OOSArtifactReference | None]: + raw_references = run_card.get("artifact_sha256") + if not isinstance(raw_references, (list, tuple)) or not raw_references: + return {}, None + first = raw_references[0] + if not isinstance(first, (list, tuple)) or len(first) != 2: + return {}, None + path = Path(str(first[0])) + resolved = path.resolve() if path.is_absolute() else (Path.cwd() / path).resolve() + root = artifact_root.resolve() + try: + relative = resolved.relative_to(root) + except ValueError as exc: + raise ValueError("SPOT_OOS_SOURCE_OUTSIDE_ARTIFACT_ROOT") from exc + digest = _stream_sha256(resolved) + if digest != str(first[1]): + raise ValueError("SPOT_OOS_SOURCE_HASH_INVALID") + try: + raw_events = read_bounded_jsonl_tail( + resolved, + max_lines=4, + max_bytes=MAX_ARTIFACT_BYTES, + ) + except (OSError, ValueError) as exc: + raise ValueError("SPOT_OOS_SOURCE_EVENTS_INVALID") from exc + events: dict[str, dict[str, object]] = {} + try: + for line in raw_events: + row = _object(json.loads(line)) + event_type = _required_text(row, "event_type") + payload = _object(row.get("payload")) + value = next(iter(payload.values()), None) + events[event_type] = _object(value) + except (TypeError, ValueError, json.JSONDecodeError) as exc: + raise ValueError("SPOT_OOS_SOURCE_EVENTS_INVALID") from exc + return events, OOSArtifactReference(relative.as_posix(), digest) + + +def _stream_sha256(path: Path) -> str: + digest = sha256() + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _write_stage_evidence( + *, + artifact_root: Path, + exact_root: Path, + subject: OOSValidationSubject, + stage: str, + measurements: dict[str, object], + blockers: list[str], + observed_at: datetime, + origin: OOSArtifactReference, +) -> OOSArtifactReference: + source_payload = { + "subject_key": subject.subject_key, + **measurements, + "origin_artifact": _reference_payload(origin), + } + source_path = exact_root / f"{stage}.source.json" + write_json_object_verified( + source_path, + source_payload, + blocker="SPOT_OOS_STAGE_SOURCE_WRITE_FAILED", + subject_id=f"{subject.subject_key}:{stage}:source", + indent=2, + durable=True, + ) + source_reference = OOSArtifactReference( + source_path.relative_to(artifact_root).as_posix(), + sha256(source_path.read_bytes()).hexdigest(), + ) + evidence_payload = { + "subject_key": subject.subject_key, + "stage": stage, + "measurements": measurements, + "observed_at": observed_at.isoformat(), + "expires_at": (observed_at + timedelta(days=30)).isoformat(), + "blockers": blockers, + "execution_allowed": False, + "promotion_status": "RESEARCH_ONLY", + "live_eligibility_status": "LIVE_ORDER_BLOCKED", + "source_artifacts": [_reference_payload(source_reference)], + "measurement_sources": { + name: {"source_index": 0, "pointer": [name]} for name in measurements + }, + } + evidence_path = exact_root / f"{stage}.json" + write_json_object_verified( + evidence_path, + evidence_payload, + blocker="SPOT_OOS_STAGE_EVIDENCE_WRITE_FAILED", + subject_id=f"{subject.subject_key}:{stage}", + indent=2, + durable=True, + ) + return OOSArtifactReference( + evidence_path.relative_to(artifact_root).as_posix(), + sha256(evidence_path.read_bytes()).hexdigest(), + ) + + +def _string_list(value: object) -> list[str]: + if not isinstance(value, list) or any( + not isinstance(item, str) or not item for item in value + ): + return [] + return list(value) + + +def _nonnegative_int(value: object) -> int: + return value if type(value) is int and value >= 0 else 0 + + +def _finite_float(value: object) -> float: + try: + number = Decimal(str(value)) + except InvalidOperation: + return 0.0 + return float(number) if number.is_finite() else 0.0 + + +def _replace_specification_placeholders( + value: object, *, cost_model_sha256: str, runtime_sha256: str +) -> object: + if isinstance(value, dict): + return { + str(key): _replace_specification_placeholders( + item, + cost_model_sha256=cost_model_sha256, + runtime_sha256=runtime_sha256, + ) + for key, item in value.items() + } + if isinstance(value, list): + return [ + _replace_specification_placeholders( + item, + cost_model_sha256=cost_model_sha256, + runtime_sha256=runtime_sha256, + ) + for item in value + ] + if value == "$COST_MODEL_SHA256": + return cost_model_sha256 + if value == "$RUNTIME_SOURCE_SHA256": + return runtime_sha256 + return value + + def _required_text(payload: Mapping[str, object], name: str) -> str: value = payload.get(name) if not isinstance(value, str) or not value.strip(): @@ -675,6 +1325,8 @@ def _matches(value: object, rule: Mapping[str, object]) -> bool: return isinstance(expected, list) and any( type(value) is type(item) and value == item for item in expected ) + if operator == "present": + return expected is True and value not in (None, "", [], {}) if isinstance(value, bool) or isinstance(expected, bool): return False try: diff --git a/src/ai4binance/validation_pipeline_runtime.py b/src/ai4binance/validation_pipeline_runtime.py index f727480f..b33c7a06 100644 --- a/src/ai4binance/validation_pipeline_runtime.py +++ b/src/ai4binance/validation_pipeline_runtime.py @@ -7,6 +7,7 @@ from collections.abc import Callable from dataclasses import dataclass, field, replace from datetime import datetime +from decimal import ROUND_DOWN, Decimal from functools import partial from hashlib import sha256 from pathlib import Path @@ -74,6 +75,38 @@ from ai4binance.validation.regimes import classify_validation_regime from ai4binance.validation.storage import WalkForwardAuditWriter +SPOT_VALIDATION_NOTIONAL_TO_EQUITY_RATIO = Decimal("0.25") + + +def runtime_spot_backtest_engine( + candles: tuple[Any, ...], + *, + base_engine: BacktestEngine, + notional_to_equity_ratio: Decimal, +) -> BacktestEngine: + """Build a price-normalized Spot engine for one immutable validation set.""" + + if not candles: + raise ValueError("runtime Spot sizing requires validation candles") + if ( + not notional_to_equity_ratio.is_finite() + or notional_to_equity_ratio <= 0 + or notional_to_equity_ratio > 1 + ): + raise ValueError("runtime Spot notional ratio must be within (0, 1]") + config = base_engine.config + maximum_entry_price = max(candle.open for candle in candles) + if maximum_entry_price <= 0: + raise ValueError("runtime Spot maximum entry price must be positive") + target_notional = config.initial_cash_usdt * notional_to_equity_ratio + quantity = (target_notional / maximum_entry_price).quantize( + config.step_size, + rounding=ROUND_DOWN, + ) + if quantity <= 0 or quantity * maximum_entry_price < config.minimum_notional: + raise ValueError("runtime Spot price-normalized quantity is not tradable") + return BacktestEngine(replace(config, quantity=quantity)) + def _build_historical_decision_resolver( candles: tuple[Any, ...] | None = None, @@ -130,6 +163,12 @@ class HistoricalValidationRuntime: default_factory=load_backtest_layout_manifest ) report_directory: Path | None = None + position_notional_to_equity_ratio: Decimal | None = None + + def __post_init__(self) -> None: + ratio = self.position_notional_to_equity_ratio + if ratio is not None and (not ratio.is_finite() or ratio <= 0 or ratio > 1): + raise ValueError("validation position notional ratio must be within (0, 1]") def start_validation_run( self, @@ -335,8 +374,17 @@ def validate_one( regime_classifier=self._regime, risk_profile_registry=self.risk_profile_registry, ) + backtest_engine = ( + self.backtest_engine + if self.position_notional_to_equity_ratio is None + else runtime_spot_backtest_engine( + candles, + base_engine=self.backtest_engine, + notional_to_equity_ratio=self.position_notional_to_equity_ratio, + ) + ) stage_started = perf_counter_ns() - backtest = self.backtest_engine.run( + backtest = backtest_engine.run( symbol=symbol, timeframe=timeframe, candles=candles, @@ -348,7 +396,7 @@ def validate_one( self.tuning_engine, validator=replace( self.tuning_engine.validator, - backtest_engine=self.backtest_engine, + backtest_engine=backtest_engine, ), ) tuning = tuning_engine.tune( @@ -377,7 +425,7 @@ def validate_one( symbol=symbol, timeframe=timeframe, candles=candles, - backtest_config=self.backtest_engine.config, + backtest_config=backtest_engine.config, provider_factory=lambda: HistoricalPlaybookAdapter( playbook=playbook, parameters=tuning.selected_parameters, @@ -478,6 +526,11 @@ def config_sha256(self, candle_count: int) -> str: "minimum_validation_candles": MINIMUM_VALIDATION_CANDLES, "candle_count": candle_count, "backtest_config": to_primitive(self.backtest_engine.config), + "position_notional_to_equity_ratio": ( + str(self.position_notional_to_equity_ratio) + if self.position_notional_to_equity_ratio is not None + else None + ), "stress_scenarios": to_primitive( self.robustness_analyzer.scenarios ), @@ -489,6 +542,11 @@ def config_sha256(self, candle_count: int) -> str: "validated_playbooks": VALIDATED_PLAYBOOKS, "search_space": to_primitive(self._search_space()), "backtest_config": to_primitive(self.backtest_engine.config), + "position_notional_to_equity_ratio": ( + str(self.position_notional_to_equity_ratio) + if self.position_notional_to_equity_ratio is not None + else None + ), "stress_scenarios": to_primitive(self.robustness_analyzer.scenarios), "tuning_config": to_primitive(self._tuning_config(candle_count)), "strategy_risk_profiles": self._risk_profiles_payload(), @@ -519,6 +577,9 @@ def write_checkpoint( return artifact_path = Path(result.artifact_path) checkpoint_path = self._checkpoint_path(artifact_path) + run_card_path = artifact_path.with_name(f"{result.playbook}.run-card.json") + if result.run_card is None or not run_card_path.is_file(): + raise ValueError("VALIDATION_CHECKPOINT_RUN_CARD_MISSING") write_json_object_verified( checkpoint_path, { @@ -531,6 +592,7 @@ def write_checkpoint( "config_sha256": config_sha256, "implementation_sha256": implementation_sha256, "artifact_sha256": sha256(artifact_path.read_bytes()).hexdigest(), + "run_card_sha256": sha256(run_card_path.read_bytes()).hexdigest(), "promotion_status": result.promotion_status.value, "blockers": list(result.blockers), "signal_blockers": [list(item) for item in result.signal_blockers], @@ -561,7 +623,12 @@ def load_checkpoint( playbook, ) checkpoint_path = self._checkpoint_path(artifact_path) - if not checkpoint_path.is_file() or not artifact_path.is_file(): + run_card_path = artifact_path.with_name(f"{playbook}.run-card.json") + if ( + not checkpoint_path.is_file() + or not artifact_path.is_file() + or not run_card_path.is_file() + ): return None try: payload = read_json_object( @@ -577,6 +644,7 @@ def load_checkpoint( "config_sha256": config_sha256, "implementation_sha256": implementation_sha256, "artifact_sha256": sha256(artifact_path.read_bytes()).hexdigest(), + "run_card_sha256": sha256(run_card_path.read_bytes()).hexdigest(), "execution_allowed": False, "live_eligibility_status": "LIVE_ORDER_BLOCKED", } @@ -587,6 +655,21 @@ def load_checkpoint( payload.get("signal_blockers") ) promotion_status = ValidationStatus(str(payload["promotion_status"])) + run_card = read_json_object( + run_card_path, + blocker="VALIDATION_CHECKPOINT_RUN_CARD_READ_FAILED", + ) + run_card_expected = { + "symbol": symbol.strip().upper(), + "timeframe": timeframe, + "dataset_sha256": dataset_sha256, + "promotion_status": promotion_status.value, + "execution_allowed": False, + } + if any( + run_card.get(key) != value for key, value in run_card_expected.items() + ): + return None except ( DestinationVerificationError, KeyError, @@ -601,6 +684,7 @@ def load_checkpoint( candle_count=len(candles), promotion_status=promotion_status, blockers=blockers, + run_card=dict(run_card), signal_blockers=signal_blockers, artifact_path=str(artifact_path), checkpoint_path=str(checkpoint_path), @@ -631,7 +715,9 @@ def _search_space() -> SearchSpace: @staticmethod def _tuning_config(candle_count: int) -> TuningConfig: test_size = max(10, candle_count // 8) - train_size = candle_count - (test_size * 5) + purge_size = 1 + embargo_size = 1 + train_size = candle_count - (test_size * 5) - purge_size - (embargo_size * 4) return TuningConfig( WalkForwardConfig( train_size=train_size, @@ -640,6 +726,8 @@ def _tuning_config(candle_count: int) -> TuningConfig: min_folds=5, min_oos_trades=5, min_regime_count=2, + purge_size=purge_size, + embargo_size=embargo_size, ), min_neighbor_count=2, min_neighbor_pass_ratio=0.5, @@ -772,7 +860,7 @@ def _build_run_card( config_sha256=ResearchRunCard.hash_json( { "search_space": to_primitive(self._search_space()), - "backtest_config": to_primitive(self.backtest_engine.config), + "backtest_config": to_primitive(backtest.assumptions), "stress_scenarios": to_primitive( self.robustness_analyzer.scenarios ), diff --git a/src/ai4binance/virtual_wallet_journal.py b/src/ai4binance/virtual_wallet_journal.py index 24955e06..bab7e983 100644 --- a/src/ai4binance/virtual_wallet_journal.py +++ b/src/ai4binance/virtual_wallet_journal.py @@ -7,6 +7,7 @@ from dataclasses import dataclass, field, replace from datetime import UTC, datetime, timedelta from decimal import Decimal +from hashlib import sha256 from itertools import pairwise from pathlib import Path from typing import cast @@ -35,6 +36,7 @@ _INITIAL_EQUITY_USDT = Decimal("1000") _INITIAL_EVENT = "VIRTUAL_WALLET_INITIALIZED" _MOVEMENT_EVENT = "VIRTUAL_WALLET_MOVEMENT_RECORDED" +_DAILY_LOSS_TUNING_THRESHOLD = 3 class VirtualWalletJournalError(RuntimeError): @@ -371,6 +373,19 @@ def dashboard_snapshot(self, observed_at: datetime) -> dict[str, object]: ) as error: raise VirtualWalletJournalError(str(error)) from error + def daily_loss_tuning_trigger(self, observed_at: datetime) -> dict[str, object]: + """Create one deterministic research trigger per three same-day losses.""" + + try: + return _daily_loss_tuning_trigger( + self._read_movements(), + _utc_timestamp(observed_at), + ) + except VirtualWalletJournalError: + raise + except (ArithmeticError, KeyError, TypeError, ValueError) as error: + raise VirtualWalletJournalError(str(error)) from error + @staticmethod def _latest_positions( movements: tuple[dict[str, object], ...], @@ -627,6 +642,111 @@ def _initial_portfolio(market: str) -> VirtualPortfolioState: ) +def _daily_loss_tuning_trigger( + movements: tuple[dict[str, object], ...], + observed_at: datetime, +) -> dict[str, object]: + safe_state = { + "execution_allowed": False, + "promotion_status": "RESEARCH_ONLY", + "live_eligibility_status": "LIVE_ORDER_BLOCKED", + } + observed_day = observed_at.astimezone(UTC).date() + losses_by_market: dict[str, list[dict[str, object]]] = {} + for movement in movements: + closed_trade = movement.get("closed_trade") + position = movement.get("managed_position") + if not isinstance(closed_trade, Mapping) or not isinstance(position, Mapping): + continue + net_pnl = _required_decimal(closed_trade, "net_pnl_usdt") + if net_pnl >= 0: + continue + exit_time = datetime.fromisoformat( + _aware_timestamp_text(closed_trade.get("exit_time")) + ).astimezone(UTC) + if exit_time.date() != observed_day: + continue + market = _required_text(movement, "market").upper() + if market not in _MARKETS: + raise ValueError("VIRTUAL_LOSS_TUNING_MARKET_INVALID") + entry_price = _required_decimal(position, "entry_price") + quantity = _required_decimal(position, "initial_quantity") + notional = entry_price * quantity + if notional <= 0: + raise ValueError("VIRTUAL_LOSS_TUNING_NOTIONAL_INVALID") + losses_by_market.setdefault(market, []).append( + { + "movement_id": _required_text(movement, "movement_id"), + "trade_id": _required_text(closed_trade, "trade_id"), + "exit_time": exit_time.isoformat(), + "net_pnl_usdt": str(net_pnl), + "net_return": str(net_pnl / notional), + "subject": { + "market": market, + "symbol": _required_text(position, "symbol").upper(), + "timeframe": _required_text(position, "timeframe"), + "strategy_id": _required_text(position, "strategy_id"), + "strategy_version": _required_text(position, "strategy_version"), + "strategy_config_hash": _required_text( + position, "strategy_config_hash" + ), + }, + } + ) + + triggers: list[dict[str, object]] = [] + for market, losses in losses_by_market.items(): + completed_count = ( + len(losses) // _DAILY_LOSS_TUNING_THRESHOLD + ) * _DAILY_LOSS_TUNING_THRESHOLD + if completed_count < _DAILY_LOSS_TUNING_THRESHOLD: + continue + batch = losses[completed_count - _DAILY_LOSS_TUNING_THRESHOLD : completed_count] + subjects = list( + { + json.dumps(item["subject"], sort_keys=True): item["subject"] + for item in batch + }.values() + ) + identity = { + "kind": "SAME_UTC_DAY_NET_LOSS_BATCH", + "market": market, + "trade_date": observed_day.isoformat(), + "evidence_ids": [item["movement_id"] for item in batch], + } + digest = sha256( + json.dumps(identity, separators=(",", ":"), sort_keys=True).encode() + ).hexdigest() + triggers.append( + { + "schema_version": "VirtualLossTuningTrigger/v1", + "status": "TRIGGERED", + "trigger_id": f"virtual-loss-tuning:{digest[:24]}", + **identity, + "loss_threshold": _DAILY_LOSS_TUNING_THRESHOLD, + "loss_count_today": len(losses), + "observed_at": observed_at.isoformat(), + "losses": batch, + "subjects": subjects, + **safe_state, + } + ) + if not triggers: + return { + "schema_version": "VirtualLossTuningTrigger/v1", + "status": "NOT_TRIGGERED", + "trade_date": observed_day.isoformat(), + "loss_threshold": _DAILY_LOSS_TUNING_THRESHOLD, + "loss_count_today": max( + (len(losses) for losses in losses_by_market.values()), + default=0, + ), + "observed_at": observed_at.isoformat(), + **safe_state, + } + return max(triggers, key=lambda item: str(item["observed_at"])) + + def _independent_portfolios( portfolios: Mapping[str, VirtualPortfolioState], ) -> IndependentVirtualPortfolios: diff --git a/tests/governance/architecture/test_technology_language_policy.py b/tests/governance/architecture/test_technology_language_policy.py index 95fdf7e5..737e9456 100644 --- a/tests/governance/architecture/test_technology_language_policy.py +++ b/tests/governance/architecture/test_technology_language_policy.py @@ -47,6 +47,8 @@ def test_policy_projection_validates_and_accepts_current_representative_paths() ".agents/skills/quality-gate-loop/scripts/invoke_gate.ps1", "config/governance/technology_language_ownership.yaml", "docs/architecture/diagrams/diagram_registry.yaml", + "publication/public_manifest.yaml", + "publication/sanitize_publication.py", "pyproject.toml", "schemas/governance/technology_language_ownership.schema.json", "scripts/quality.ps1", diff --git a/tests/governance/terminology/test_terminology_policy.py b/tests/governance/terminology/test_terminology_policy.py index 78a90ffd..42bb5e10 100644 --- a/tests/governance/terminology/test_terminology_policy.py +++ b/tests/governance/terminology/test_terminology_policy.py @@ -127,8 +127,9 @@ def test_terminology_helpers_and_policy_contract_fail_closed(tmp_path: Path) -> terminology_module._string(" ", "value") with pytest.raises(ValueError, match="string array"): terminology_module._strings([""], "value") - with pytest.raises(ValueError, match="repository-relative"): - terminology_module._safe_path("../outside", "value") + for unsafe_path in ("../outside", "/absolute", "C:/absolute", "folder\\file"): + with pytest.raises(ValueError, match="repository-relative"): + terminology_module._safe_path(unsafe_path, "value") def test_terminology_scan_covers_deprecated_missing_and_projection_drift( diff --git a/tests/test_artifact_hygiene_scripts.py b/tests/test_artifact_hygiene_scripts.py index f9be76dc..d6e08c40 100644 --- a/tests/test_artifact_hygiene_scripts.py +++ b/tests/test_artifact_hygiene_scripts.py @@ -2415,6 +2415,13 @@ def test_quality_script_rejects_inline_approval_generation_parameters( ) +def test_quality_script_binds_approval_replay_to_frozen_governance_report() -> None: + text = _quality_script_text() + + assert '"--frozen-governance-gate-report"' in text + assert "-FrozenGovernanceGateReportPath $context.governance_path" in text + + def test_quality_script_retries_governance_gate_with_external_approval_artifact( tmp_path: Path, ) -> None: diff --git a/tests/test_cli.py b/tests/test_cli.py index 798685c7..3ba9980a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -7,11 +7,12 @@ import shutil import sys import tempfile +from collections.abc import Mapping from datetime import UTC, datetime, timedelta from decimal import Decimal from pathlib import Path from types import SimpleNamespace -from typing import cast +from typing import Any, cast import pytest @@ -89,7 +90,7 @@ def test_status_is_complete_and_safe_by_default( assert payload["execution_allowed"] is False assert payload["trading_mode"] == "paper" assert payload["order_mode"] == "manual" - assert payload["timeframes"] == ["5m", "15m", "1h", "4h", "1d"] + assert payload["timeframes"] == ["15m", "1h", "4h"] assert payload["live_gate"]["status"] == "LIVE_ORDER_BLOCKED" assert payload["virtual_market_gate"]["execution_surface"] == "VIRTUAL_MARKET" assert payload["virtual_market_gate"]["automation_mode"] == ( @@ -1658,10 +1659,107 @@ def test_virtual_market_research_cycle_persists_both_wallets_and_report( assert cycle_report["virtual_runtime_evaluated"] is False assert cycle_report["virtual_simulation_outcome"] == "PRECONDITIONS_BLOCKED" assert cycle_report["virtual_order_ready"] is False + tuning = cast(Mapping[str, object], cycle_report["daily_loss_tuning"]) + assert tuning["status"] == "NOT_TRIGGERED" + assert tuning["loss_threshold"] == 3 + assert tuning["parameter_application"] == "NOT_APPLIED" report_path = tmp_path / "runtime" / "reports" / "virtual_wallets" / "latest.md" assert report_path.exists() +def test_virtual_loss_tuning_runs_canonical_optimizer_without_applying_parameters( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from ai4binance import validation_pipeline_runtime as validation_module + from ai4binance.cli import runtime as runtime_cli + from ai4binance.data import archive as archive_module + from ai4binance.validation import ParameterSet + + observed_at = datetime(2026, 9, 25, 0, 0, tzinfo=UTC) + + trigger = { + "schema_version": "VirtualLossTuningTrigger/v1", + "status": "TRIGGERED", + "trigger_id": "virtual-loss-tuning:" + "a" * 24, + "subjects": [ + { + "market": "SPOT", + "symbol": "BTCUSDT", + "timeframe": "1h", + "strategy_id": "trend_continuation", + } + ], + "execution_allowed": False, + "promotion_status": "RESEARCH_ONLY", + "live_eligibility_status": "LIVE_ORDER_BLOCKED", + } + journal = SimpleNamespace(daily_loss_tuning_trigger=lambda _observed: trigger) + candles = (SimpleNamespace(timestamp=observed_at),) * 60 + monkeypatch.setattr( + archive_module.ParquetOHLCVArchive, + "read", + lambda *_args: candles, + ) + + class Runtime: + def __init__(self, **_kwargs: object) -> None: + pass + + @staticmethod + def dataset_sha256(_candles: object) -> str: + return "b" * 64 + + @staticmethod + def validate_one(*_args: object, **_kwargs: object) -> object: + return SimpleNamespace( + backtest=SimpleNamespace( + metrics={"net_return": -0.01, "trade_count": 3} + ), + tuning=SimpleNamespace( + report_id="tuning:test", + search_space=SimpleNamespace(candidate_count=9), + selected_parameters=ParameterSet( + "selected", + ( + ("atr_stop_multiplier", 1.25), + ("take_profit_multiplier", 3.5), + ), + ), + blockers=("OOS_RETURN_INSUFFICIENT",), + ), + blockers=("OOS_RETURN_INSUFFICIENT",), + ) + + monkeypatch.setattr(validation_module, "HistoricalValidationRuntime", Runtime) + settings = Settings( + dataset_directory=tmp_path / "data", + validation_artifact_directory=tmp_path / "validation", + backtest_report_directory=tmp_path / "reports", + ) + + result = runtime_cli._run_virtual_loss_tuning( + settings, + cast(Any, journal), + observed_at, + ) + repeated = runtime_cli._run_virtual_loss_tuning( + settings, + cast(Any, journal), + observed_at + timedelta(minutes=1), + ) + + assert result["status"] == "RESEARCH_TUNING_COMPLETED" + assert result["parameter_application"] == "NOT_APPLIED" + assert result["execution_allowed"] is False + assert result["promotion_status"] == "RESEARCH_ONLY" + assert result["live_eligibility_status"] == "LIVE_ORDER_BLOCKED" + tuning_result = cast(list[Mapping[str, object]], result["results"])[0] + assert tuning_result["candidate_count"] == 9 + assert tuning_result["parameter_application"] == "NOT_APPLIED" + assert repeated["status"] == "ALREADY_REVIEWED" + + def test_virtual_market_daemon_fails_closed_and_records_cycle_failure( tmp_path: Path, ) -> None: @@ -1854,6 +1952,59 @@ def research_cycle( assert acknowledgement["execution_allowed"] is False +def test_virtual_market_daemon_requests_canonical_refresh_for_stale_data( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from ai4binance.data import market_history_sync + from ai4binance.data.market_history_continuous import ( + VIRTUAL_MARKET_COLLECTION_TIMEFRAMES, + ) + + monkeypatch.setattr( + market_history_sync, "read_cached_market_universe", lambda *_args: None + ) + settings = Settings( + symbol="BTCUSDT", + runtime_state_path=tmp_path / "runtime.json", + ) + + def research_cycle( + cycle_settings: Settings, + _acquisition: SnapshotAcquirer, + *, + cycle_report: dict[str, object] | None = None, + ) -> int: + assert cycle_report is not None + assert cycle_settings.timeframes == VIRTUAL_MARKET_COLLECTION_TIMEFRAMES + cycle_report.update( + snapshot_id="fixture:BTCUSDT", + research_blockers=("SNAPSHOT_DATA_QUALITY_INVALID", "STALE_CANDLES:5m"), + virtual_order_ready=False, + ) + return 0 + + monkeypatch.setattr( + runtime_cli, "_run_virtual_market_research_cycle", research_cycle + ) + observed_at = datetime(2026, 9, 24, 13, 30, tzinfo=UTC) + assert ( + runtime_cli.run_virtual_market_daemon( + settings, + max_cycles=1, + public_acquisition=cast(SnapshotAcquirer, object()), + clock=lambda: observed_at, + ) + == 0 + ) + + refresh = json.loads((tmp_path / "market-history-refresh-request.json").read_text()) + state = json.loads((tmp_path / "virtual-market.json").read_text()) + assert refresh["requester"] == "VIRTUAL_MARKET" + assert refresh["status"] == "PENDING" + assert refresh["execution_allowed"] is False + assert state["market_history_refresh"]["state"] == "PENDING" + + def test_virtual_market_manual_refresh_rejects_authority_drift_and_stale_request( tmp_path: Path, ) -> None: diff --git a/tests/test_config_reporting.py b/tests/test_config_reporting.py index b5fcb2a3..1beb66e3 100644 --- a/tests/test_config_reporting.py +++ b/tests/test_config_reporting.py @@ -42,6 +42,12 @@ def test_settings_normalize_market_type() -> None: Settings(market_type="options") # type: ignore[arg-type] +def test_settings_route_futures_oos_to_the_cli_contract_artifact_root() -> None: + assert Settings().futures_oos_artifact_directory == Path( + "runtime/artifacts/validation/futures_oos" + ) + + def test_settings_expose_local_llm_gpu_controls( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -136,9 +142,9 @@ def test_settings_reject_minimum_history_above_request_limit() -> None: Settings(candle_limit=100, minimum_closed_candles=200) -def test_settings_accepts_three_month_history_with_daily_quality_floor() -> None: - settings = Settings(market_history_initial_days=90) - assert settings.market_history_initial_days == 90 +def test_settings_defaults_to_governed_spot_oos_history_horizon() -> None: + settings = Settings() + assert settings.market_history_initial_days == 400 assert settings.max_data_workers == 4 assert settings.market_history_max_workers == 8 assert settings.market_history_opportunity_workers == 2 diff --git a/tests/test_data_acquisition.py b/tests/test_data_acquisition.py index f7c75962..a1d84431 100644 --- a/tests/test_data_acquisition.py +++ b/tests/test_data_acquisition.py @@ -112,7 +112,10 @@ def test_local_archive_missing_fails_closed_without_kline_network( def test_local_snapshot_priority_reuses_canonical_liquidity_order( tmp_path: Path, ) -> None: - from ai4binance.cli.runtime import _virtual_market_priority_symbols + from ai4binance.cli.runtime import ( + _virtual_market_priority_symbols, + _virtual_market_ranked_symbols, + ) from ai4binance.config import Settings from tests.test_binance_market_universe_provider import _SpotTransport @@ -138,6 +141,7 @@ def test_local_snapshot_priority_reuses_canonical_liquidity_order( "SOLUSDT", ) assert _virtual_market_priority_symbols(settings, (), ("BTCUSDT",), NOW) == () + assert _virtual_market_ranked_symbols(settings, (), NOW) == ("SOLUSDT",) assert ( _virtual_market_priority_symbols( settings, (), ("SOLUSDT",), NOW + timedelta(hours=1) @@ -179,7 +183,11 @@ def test_acquisition_consumes_verified_local_depth_and_rejects_stale_or_gapped( "bridged": True, } journal.append([("spot", "HOTUSDT", "checkpoint", checkpoint, NOW.timestamp())]) - agent = DataAcquisitionAgent(client=FakePublicClient(), depth_path=path) + agent = DataAcquisitionAgent( + client=FakePublicClient(), + depth_path=path, + clock=lambda: NOW, + ) snapshot = agent.acquire("HOTUSDT", ("1m",)) assert snapshot.order_book_summary["bid_depth"] == "2" assert snapshot.order_book_summary["ask_depth"] == "3" diff --git a/tests/test_dge_engine.py b/tests/test_dge_engine.py index e2aea7d6..b00c065a 100644 --- a/tests/test_dge_engine.py +++ b/tests/test_dge_engine.py @@ -230,6 +230,27 @@ def test_dge_virtual_market_surface_allows_autonomous_simulation_only() -> None: assert decision.execution_surface is ExecutionSurface.VIRTUAL_MARKET +def test_dge_blocked_virtual_market_returns_decision_without_profile_conflict() -> None: + decision = DecisionGovernanceEngine().evaluate( + candidate(), + context( + execution_surface=ExecutionSurface.VIRTUAL_MARKET, + oos_approved=False, + validation_approved=False, + human_approval_recorded=False, + ), + ) + + assert decision.governance_status is DgeDecisionStatus.WATCH_ONLY + assert decision.governed_action is DgeMarketAction.NO_TRADE + assert "VAL.OOS_NOT_VALIDATED" in decision.hard_blockers + assert decision.simulated_execution_allowed is False + assert decision.paper_execution_allowed is False + assert decision.auto_execution_allowed is False + assert decision.autonomous_learning_allowed is False + assert decision.requires_manual_confirmation is True + + def test_dge_data_quality_failure_is_first_class_and_non_executable() -> None: decision = DecisionGovernanceEngine().evaluate( candidate(), diff --git a/tests/test_dge_recovery_replay_shadow.py b/tests/test_dge_recovery_replay_shadow.py index e05bf1bb..77613ec5 100644 --- a/tests/test_dge_recovery_replay_shadow.py +++ b/tests/test_dge_recovery_replay_shadow.py @@ -56,7 +56,7 @@ def test_virtual_context_routes_only_current_bound_regime_and_mtf(stale: bool) - current, SimpleNamespace( snapshot_id=current.snapshot_id, - blockers=(), + blockers=("OOS_DEPLOYMENT_MISSING",), agent_results={"market_regime": regime, "multi_timeframe": mtf}, ), candidate, @@ -69,6 +69,7 @@ def test_virtual_context_routes_only_current_bound_regime_and_mtf(stale: bool) - assert context.mtf_aligned is (not stale) assert context.oos_approved is False assert context.validation_approved is False + assert context.negative_evidence_clear is True def test_recovery_radar_candidates_are_governed_without_live_authority() -> None: diff --git a/tests/test_docs_hygiene.py b/tests/test_docs_hygiene.py index 25c0bf1e..539f68b2 100644 --- a/tests/test_docs_hygiene.py +++ b/tests/test_docs_hygiene.py @@ -213,7 +213,7 @@ def test_docs_and_reports_have_eli10_explanations() -> None: def test_canonical_docs_markdown_filenames_and_metadata_are_classified() -> None: pattern = re.compile( - r"^(?:[a-z]+(?:_[a-z]+)?)_[a-z0-9]+(?:_[a-z0-9]+)*_[a-z0-9]+(?:_[a-z0-9]+)*\.md$" + r"^(?:[a-z0-9]+(?:_[a-z0-9]+)?)_[a-z0-9]+(?:_[a-z0-9]+)*_[a-z0-9]+(?:_[a-z0-9]+)*\.md$" ) docs_index_exceptions = {"docs/README.md"} required_frontmatter_keys = { @@ -241,9 +241,10 @@ def test_canonical_docs_markdown_filenames_and_metadata_are_classified() -> None key, value = line.split(": ", 1) metadata[key.strip()] = value.strip() missing_keys = sorted(required_frontmatter_keys - metadata.keys()) - if not pattern.match(path.name) or not has_frontmatter or missing_keys: + filename_is_valid = path.name == "README.md" or pattern.match(path.name) + if not filename_is_valid or not has_frontmatter or missing_keys: detail = [] - if not pattern.match(path.name): + if not filename_is_valid: detail.append("filename") if not has_frontmatter: detail.append("frontmatter") @@ -735,9 +736,9 @@ def test_instruction_context_router_avoids_default_corpus_loading() -> None: normalized_root = re.sub(r"\s+", " ", root_text) assert ( - "Base context is this file plus the active adapter. Add nearest scoped " - "`AGENTS.md` only for paths within its scope." - ) in normalized_root + "Resolve root from this file, active adapter, and nearest scoped `AGENTS.md`." + in normalized_root + ) assert "Route them; do not load by default." in root_text for route in ( "| Ordinary source | Affected code/config/callers/contracts/tests |", diff --git a/tests/test_event_journal.py b/tests/test_event_journal.py index eda70575..4717b208 100644 --- a/tests/test_event_journal.py +++ b/tests/test_event_journal.py @@ -433,3 +433,64 @@ def test_posix_file_lock_calls_platform_primitives( file_lock._release_file_lock(stream) assert calls == [(7, 2), (7, 8)] + + +def test_windows_file_lock_retries_transient_deadlock( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import errno + import sys + from types import SimpleNamespace + + import ai4binance.events.file_lock as file_lock + + calls: list[tuple[int, int, int]] = [] + + def locking(descriptor: int, operation: int, size: int) -> None: + calls.append((descriptor, operation, size)) + if len(calls) < 3: + raise OSError(errno.EDEADLK, "fixture contention") + + monkeypatch.setattr(os, "name", "nt") + monkeypatch.setattr("ai4binance.events.file_lock.time.sleep", lambda _seconds: None) + monkeypatch.setitem( + sys.modules, + "msvcrt", + SimpleNamespace(locking=locking, LK_NBLCK=1), + ) + stream = cast( + Any, + SimpleNamespace(seek=lambda *_args: None, fileno=lambda: 7), + ) + + file_lock._acquire_file_lock(stream) + + assert calls == [(7, 1, 1), (7, 1, 1), (7, 1, 1)] + + +def test_windows_file_lock_times_out_on_persistent_contention( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import errno + import sys + from types import SimpleNamespace + + import ai4binance.events.file_lock as file_lock + + def locking(_descriptor: int, _operation: int, _size: int) -> None: + raise OSError(errno.EDEADLK, "fixture contention") + + monkeypatch.setattr(os, "name", "nt") + monkeypatch.setattr(file_lock, "_WINDOWS_LOCK_TIMEOUT_SECONDS", 0.0) + monkeypatch.setitem( + sys.modules, + "msvcrt", + SimpleNamespace(locking=locking, LK_NBLCK=1), + ) + stream = cast( + Any, + SimpleNamespace(seek=lambda *_args: None, fileno=lambda: 7), + ) + + with pytest.raises(TimeoutError, match="file lock acquisition timed out"): + file_lock._acquire_file_lock(stream) diff --git a/tests/test_futures_replay.py b/tests/test_futures_replay.py index 9b5650c9..9bc14142 100644 --- a/tests/test_futures_replay.py +++ b/tests/test_futures_replay.py @@ -435,7 +435,9 @@ def test_local_futures_oos_cli_publishes_research_only_evidence( candles=candles, derivatives=_derivatives(candles), ) - replay_path = tmp_path / "runtime" / "datasets" / "futures" / "hotusdt.json" + replay_path = ( + tmp_path / "runtime" / "data" / "datasets" / "futures" / "hotusdt.json" + ) _write_replay(replay_path, dataset) revision = "c" * 40 @@ -477,7 +479,9 @@ def test_local_futures_oos_cli_rejects_tampered_replay_without_artifact( tmp_path: Path, capsys: pytest.CaptureFixture[str], ) -> None: - replay_path = tmp_path / "runtime" / "datasets" / "futures" / "hotusdt.json" + replay_path = ( + tmp_path / "runtime" / "data" / "datasets" / "futures" / "hotusdt.json" + ) _write_replay(replay_path, _dataset()) payload = json.loads(replay_path.read_text(encoding="utf-8")) payload["dataset_sha256"] = "0" * 64 diff --git a/tests/test_governance_constitution_sync.py b/tests/test_governance_constitution_sync.py index e733dd46..1d9f0091 100644 --- a/tests/test_governance_constitution_sync.py +++ b/tests/test_governance_constitution_sync.py @@ -473,6 +473,110 @@ def test_governance_alignment_surfaces_loose_governance_code( assert "LIVE_ORDER_BLOCKED" in report.blockers +def test_governance_alignment_accepts_transitive_compliance_trace( + tmp_path: Path, +) -> None: + source_path = "src/ai4binance/governance/example_policy.py" + source = tmp_path / source_path + source.parent.mkdir(parents=True) + source.write_text("class ExamplePolicy: ...\n", encoding="utf-8") + tests = tmp_path / "tests" + tests.mkdir() + (tests / "test_example_policy.py").write_text( + "from ai4binance.governance.example_policy import ExamplePolicy\n", + encoding="utf-8", + ) + write_core_documents( + tmp_path, + compliance_extra="docs/standards/example_policy_standard.md", + ) + standard = tmp_path / "docs" / "standards" / "example_policy_standard.md" + standard.parent.mkdir(parents=True, exist_ok=True) + standard.write_text( + "# Example Policy Standard\n\n" + "## ELI10\n\n" + "See `docs/references/example_policy_reference.md`.\n", + encoding="utf-8", + ) + reference = tmp_path / "docs" / "references" / "example_policy_reference.md" + reference.parent.mkdir(parents=True) + reference.write_text( + f"# Example Policy Reference\n\n## ELI10\n\n`{source_path}`\n", + encoding="utf-8", + ) + write_quality_evidence(tmp_path) + + report = audit_governance_alignment( + tmp_path, + changed_paths=(source_path, "tests/test_example_policy.py"), + ) + + assert report.status is GovernanceAlignmentStatus.PASS + assert report.findings == () + + +def test_governance_alignment_rejects_unlinked_document_as_compliance_trace( + tmp_path: Path, +) -> None: + source_path = "src/ai4binance/governance/unlinked_policy.py" + source = tmp_path / source_path + source.parent.mkdir(parents=True) + source.write_text("class UnlinkedPolicy: ...\n", encoding="utf-8") + tests = tmp_path / "tests" + tests.mkdir() + (tests / "test_unlinked_policy.py").write_text( + "from ai4binance.governance.unlinked_policy import UnlinkedPolicy\n", + encoding="utf-8", + ) + write_core_documents(tmp_path) + unlinked = tmp_path / "docs" / "references" / "unlinked_policy.md" + unlinked.parent.mkdir(parents=True) + unlinked.write_text( + f"# Unlinked Policy\n\n## ELI10\n\n`{source_path}`\n", + encoding="utf-8", + ) + write_quality_evidence(tmp_path) + + report = audit_governance_alignment( + tmp_path, + changed_paths=(source_path, "tests/test_unlinked_policy.py"), + ) + finding_kinds = {finding.kind for finding in report.findings} + + assert LooseCodeGapKind.GOVERNANCE_CODE_WITHOUT_COMPLIANCE in finding_kinds + assert LooseCodeGapKind.SOURCE_WITHOUT_WRITTEN_RULE not in finding_kinds + + +def test_governance_alignment_accepts_publication_boundary_as_written_rule( + tmp_path: Path, +) -> None: + source_path = "src/ai4binance/ops/public_showcase.py" + source = tmp_path / source_path + source.parent.mkdir(parents=True) + source.write_text("class PublicShowcase: ...\n", encoding="utf-8") + tests = tmp_path / "tests" + tests.mkdir() + (tests / "test_public_showcase.py").write_text( + "from ai4binance.ops.public_showcase import PublicShowcase\n", + encoding="utf-8", + ) + write_core_documents(tmp_path) + publication = tmp_path / "publication" / "README.md" + publication.parent.mkdir() + publication.write_text( + f"# Publication Boundary\n\n`{source_path}`\n", encoding="utf-8" + ) + write_quality_evidence(tmp_path) + + report = audit_governance_alignment( + tmp_path, + changed_paths=(source_path, "tests/test_public_showcase.py"), + ) + finding_kinds = {finding.kind for finding in report.findings} + + assert LooseCodeGapKind.SOURCE_WITHOUT_WRITTEN_RULE not in finding_kinds + + def test_governance_alignment_surfaces_constitution_family_mismatch( tmp_path: Path, ) -> None: diff --git a/tests/test_governance_gate.py b/tests/test_governance_gate.py index 5628e379..72329851 100644 --- a/tests/test_governance_gate.py +++ b/tests/test_governance_gate.py @@ -54,6 +54,7 @@ build_governance_gate_report, load_approval_records, load_approval_records_with_fallback, + load_frozen_traceability_audit, load_repository_validator_evidence, main, ) @@ -937,6 +938,67 @@ def test_governance_gate_passes_when_required_canonical_trace_exists( assert report.traceability_audit.requirement_count == 2 +def test_governance_gate_replay_reuses_frozen_traceability_audit( + tmp_path: Path, +) -> None: + write_core_documents(tmp_path) + write_quality_evidence(tmp_path) + (tmp_path / "traceability-note.txt").write_text( + "traceability\n", + encoding="utf-8", + ) + _stamp_authority_frontmatter(tmp_path) + change_set = _change_set(tmp_path, "traceability-note.txt") + quality_report = _quality_report(tmp_path, change_set=change_set) + frozen_audit = TraceabilityAuditReport( + journal_path=canonical_trace_journal_path(tmp_path).resolve(), + status=TraceabilityStatus.PASS, + requirement_count=1, + record_count=7, + ) + frozen_report_path = tmp_path / "frozen-governance.json" + frozen_report_path.write_text( + json.dumps({"traceability_audit": frozen_audit.to_payload()}), + encoding="utf-8", + ) + + loaded_audit = load_frozen_traceability_audit( + frozen_report_path, + repository_root=tmp_path, + ) + first = build_governance_gate_report( + repository_root=tmp_path, + repository_validator=load_repository_validator_evidence( + _write_validator_report(tmp_path) + ), + docs_hygiene=_docs_hygiene(True), + artifact_hygiene=_artifact_hygiene(True), + constitution_sync_tests=_constitution_sync_tests(True), + deterministic_quality_gate=quality_report, + change_set=change_set, + require_change_set=True, + traceability_audit=loaded_audit, + ) + _write_auto_audit_traceability_artifact(tmp_path, include_journal=True) + second = build_governance_gate_report( + repository_root=tmp_path, + repository_validator=load_repository_validator_evidence( + _write_validator_report(tmp_path) + ), + docs_hygiene=_docs_hygiene(True), + artifact_hygiene=_artifact_hygiene(True), + constitution_sync_tests=_constitution_sync_tests(True), + deterministic_quality_gate=quality_report, + change_set=change_set, + require_change_set=True, + traceability_audit=loaded_audit, + ) + + assert first.gate_evidence_sha256 == second.gate_evidence_sha256 + assert second.traceability_audit == frozen_audit + assert second.traceability_audit.record_count == 7 + + def test_governance_gate_verifies_bound_approval_records(tmp_path: Path) -> None: write_core_documents( tmp_path, diff --git a/tests/test_kaizen_quality.py b/tests/test_kaizen_quality.py index 60394d95..860b68a6 100644 --- a/tests/test_kaizen_quality.py +++ b/tests/test_kaizen_quality.py @@ -792,6 +792,18 @@ def test_architecture_migration_ledger_classifies_every_repository_module() -> N assert historical_persistence["target_paths"] == [ "src/ai4binance/infrastructure/persistence/historical_replay.py" ] + internal_radar = by_path["src/ai4binance/internal_radar.py"] + assert internal_radar["classification"] == "SPLIT" + assert set(cast(list[str], internal_radar["target_paths"])) == { + "src/ai4binance/domain/evidence", + "src/ai4binance/application/pipelines", + "src/ai4binance/infrastructure/filesystem", + } + internal_radar_vision = by_path["src/ai4binance/internal_radar_vision.py"] + assert internal_radar_vision["classification"] == "MOVE" + assert internal_radar_vision["target_paths"] == [ + "src/ai4binance/integrations/llm/internal_radar_vision.py" + ] virtual_attribution = by_path[ "src/ai4binance/research/virtual_runtime_attribution.py" ] diff --git a/tests/test_local_dashboard_source.py b/tests/test_local_dashboard_source.py index cae984ef..9cfa6c7c 100644 --- a/tests/test_local_dashboard_source.py +++ b/tests/test_local_dashboard_source.py @@ -38,7 +38,7 @@ def test_canonical_dashboard_source_builds_deterministic_offline_assets( assert completed.returncode == 0, completed.stderr assert completed.stdout.strip() == "DASHBOARD_PACKAGE_BUILT" assert _sha256(stage / "app.js") == ( - "a1396a57d098495cbe5ed00f881e35d25c2db3c83183fffba28bcb415c4eff59" + "56c052e1234a1317bf303ed053792c3ec5cc852c5f1b59d24e48415280a5fe31" ) assert _sha256(stage / "app.css") == ( "820ea8899490af3d761e507a88d0af4fd4ecbf587cbd7118c8faa665b5569cc5" @@ -93,6 +93,8 @@ def test_virtual_market_separates_trade_records_from_potential_opportunities() - assert "Entry / Stop / TP1 / TP2 / TP3 / R/R" in views assert "Opportunity generation health" in views assert "rejected_by_reason" in views + assert "CANDIDATES_AVAILABLE_WITH_DATA_GAPS" in views + assert "Published research opportunities remain visible" in views assert "Rejected attempts are diagnostic evidence, not opportunities" in views assert "has_complete_measurable_opportunity" in ( SOURCE / "market_views.py.in" @@ -104,6 +106,20 @@ def test_virtual_market_separates_trade_records_from_potential_opportunities() - assert "def _trade_record_dashboard_row" in wallet +def test_dashboard_exposes_virtual_runtime_preconditions_as_inactive() -> None: + server = (SOURCE / "server.py.in").read_text(encoding="utf-8") + views = (SOURCE / "local_views.js").read_text(encoding="utf-8") + + assert '"virtual_runtime_evaluated"' in server + assert '"virtual_simulation_outcome"' in server + assert '"risk_approved"' in server + assert ( + "daemon.virtual_simulation_outcome||projection.status||daemon.status" in views + ) + assert "Runtime evaluated" in views + assert "Risk approved" in views + + def test_dashboard_projects_auto_audit_movements_with_method_provenance() -> None: module = runpy.run_path(str(SOURCE / "server.py.in")) project = module["auto_audit_observer_projection"] diff --git a/tests/test_lowest_coverage_persistence_and_governance.py b/tests/test_lowest_coverage_persistence_and_governance.py index 4df9c6ad..77d668be 100644 --- a/tests/test_lowest_coverage_persistence_and_governance.py +++ b/tests/test_lowest_coverage_persistence_and_governance.py @@ -454,6 +454,14 @@ def test_safe_json_public_serialization_and_verification_paths(tmp_path: Path) - ) assert result.status == "VERIFIED" assert path.read_text(encoding="utf-8").endswith("\n") + canonical_result = write_json_object_verified( + path, {"blockers": ("FIRST", "SECOND")}, blocker="WRITE" + ) + assert canonical_result.expected_sha256 == canonical_result.observed_sha256 + assert json.loads(path.read_text(encoding="utf-8"))["blockers"] == [ + "FIRST", + "SECOND", + ] assert to_primitive( { "enum": _ExampleEnum.VALUE, diff --git a/tests/test_lowest_coverage_technology_language_policy.py b/tests/test_lowest_coverage_technology_language_policy.py index b62f9f17..ca3e1550 100644 --- a/tests/test_lowest_coverage_technology_language_policy.py +++ b/tests/test_lowest_coverage_technology_language_policy.py @@ -127,6 +127,8 @@ def test_policy_reports_unreadable_sources_and_evidence_requirements( (policy_module._boolean, "false", "must be boolean"), (policy_module._strings, ("not", "a", "list"), "must be a string array"), (policy_module._safe_paths, ["../unsafe"], "repository-relative POSIX paths"), + (policy_module._safe_paths, ["/absolute"], "repository-relative POSIX paths"), + (policy_module._safe_paths, ["C:/absolute"], "repository-relative POSIX paths"), ], ) def test_policy_value_validators_reject_malformed_input( diff --git a/tests/test_market_data_gateway.py b/tests/test_market_data_gateway.py index 7c3ed09a..d302a452 100644 --- a/tests/test_market_data_gateway.py +++ b/tests/test_market_data_gateway.py @@ -205,6 +205,14 @@ def test_gateway_blockers_are_fail_closed_on_invalid_shape() -> None: assert _blockers({"blockers": "unexpected"}) == {"MARKET_GATEWAY_BLOCKERS_INVALID"} +def test_gateway_reuses_the_canonical_continuous_collector_builder() -> None: + source = Path(gateway_cli.__file__).read_text(encoding="utf-8") + + assert "build_continuous_market_history(" in source + assert "build_market_depth_collector(" in source + assert '"market-history-refresh-request.json"' not in source + + def test_gateway_heartbeat_preserves_bootstrap_evidence(tmp_path: Path) -> None: path = tmp_path / "market-history-latest.json" path.write_text( @@ -321,7 +329,11 @@ def sync_cycle(self, *, observed_at: datetime) -> dict[str, object]: del observed_at return {"blockers": ["PUBLIC_MARKET_UNIVERSE_UNAVAILABLE"]} - monkeypatch.setattr(gateway_cli, "ContinuousMarketHistory", _Collector) + monkeypatch.setattr( + gateway_cli, + "build_continuous_market_history", + lambda *_args, **_kwargs: _Collector(), + ) settings = type( "Settings", (), @@ -384,7 +396,11 @@ class _Gateway: async def run_once(self) -> tuple[str, str]: return ("PLANNED_ROLLOVER", "PLANNED_ROLLOVER") - monkeypatch.setattr(gateway_cli, "ContinuousMarketHistory", _Collector) + monkeypatch.setattr( + gateway_cli, + "build_continuous_market_history", + lambda *_args, **_kwargs: _Collector(), + ) monkeypatch.setattr(gateway_cli, "SingleInstanceLease", _Lease) monkeypatch.setattr( gateway_cli, @@ -469,7 +485,11 @@ async def run_once(self) -> tuple[str, str]: raise OSError("temporary") return ("PLANNED_ROLLOVER", "PLANNED_ROLLOVER") - monkeypatch.setattr(gateway_cli, "ContinuousMarketHistory", _Collector) + monkeypatch.setattr( + gateway_cli, + "build_continuous_market_history", + lambda *_args, **_kwargs: _Collector(), + ) monkeypatch.setattr(gateway_cli, "SingleInstanceLease", _Lease) fake_time = type("Time", (), {"sleep": staticmethod(lambda _seconds: None)})() monkeypatch.setattr(gateway_cli, "time", fake_time) @@ -537,8 +557,8 @@ def __exit__(self, *args: object) -> None: ) monkeypatch.setattr( gateway_cli, - "ContinuousMarketHistory", - lambda **_kwargs: object(), + "build_continuous_market_history", + lambda *_args, **_kwargs: object(), ) settings = type( "Settings", @@ -602,7 +622,11 @@ def sync_cycle(self, *, observed_at: datetime) -> dict[str, object]: return {"blockers": ["MARKET_DATA_BACKFILL_PENDING"]} return {"blockers": []} - monkeypatch.setattr(gateway_cli, "ContinuousMarketHistory", _Collector) + monkeypatch.setattr( + gateway_cli, + "build_continuous_market_history", + lambda *_args, **_kwargs: _Collector(), + ) monkeypatch.setattr(gateway_cli, "SingleInstanceLease", _Lease) monkeypatch.setattr( gateway_cli, diff --git a/tests/test_market_depth.py b/tests/test_market_depth.py index 2b0e6b60..04463f50 100644 --- a/tests/test_market_depth.py +++ b/tests/test_market_depth.py @@ -28,7 +28,12 @@ def _json_transport(value: object | None = None) -> JsonTransport: - return cast(JsonTransport, object() if value is None else value) + return cast( + JsonTransport, + SimpleNamespace(get_json=lambda *_args, **_kwargs: snapshot()) + if value is None + else value, + ) def snapshot() -> dict[str, object]: @@ -90,6 +95,39 @@ def test_snapshot_bridge_checkpoint_and_replay( journal.close() +def test_checkpoint_compacts_superseded_depth_events(tmp_path: Path) -> None: + path = tmp_path / "depth.sqlite3" + journal = DepthJournal(path) + book = ReadOnlyOrderBook("BTCUSDT") + apply_depth_snapshot(book, snapshot()) + first = event("BTCUSDT") + assert apply_depth_event(book, first, "spot") + journal.append( + [ + ("spot", "BTCUSDT", "snapshot", snapshot(), NOW.timestamp()), + ("spot", "BTCUSDT", "delta", first, NOW.timestamp()), + ( + "spot", + "ETHUSDT", + "snapshot", + snapshot(), + NOW.timestamp(), + ), + ] + ) + + journal.append( + [("spot", "BTCUSDT", "checkpoint", _checkpoint(book), NOW.timestamp())] + ) + + retained = journal.connection.execute( + "SELECT symbol,kind FROM depth_events ORDER BY seq" + ).fetchall() + assert retained == [("ETHUSDT", "snapshot"), ("BTCUSDT", "checkpoint")] + assert read_local_depth(path, "spot", "BTCUSDT", now=NOW)["lastUpdateId"] == 101 + journal.close() + + @pytest.mark.parametrize("market", ["spot", "usd_m_futures", "coin_m_futures"]) def test_gap_and_unsynchronized_are_never_readable(tmp_path: Path, market: str) -> None: journal = DepthJournal(tmp_path / "depth.db") diff --git a/tests/test_market_history_boundary_contracts.py b/tests/test_market_history_boundary_contracts.py index 177a55e5..813f9c90 100644 --- a/tests/test_market_history_boundary_contracts.py +++ b/tests/test_market_history_boundary_contracts.py @@ -528,7 +528,10 @@ def test_refresh_dataset_blockers_distinguish_gaps_staleness_and_derivatives( blockers = instance._refresh_request_data_blockers( "usd_m_futures", "BTCUSDT", NOW, [{"kind": "funding", "status": "BLOCKED"}] ) - assert f"MARKET_HISTORY_REFRESH_{code}:5m" in blockers + assert all( + f"MARKET_HISTORY_REFRESH_{code}:{timeframe}" in blockers + for timeframe in h.VIRTUAL_MARKET_COLLECTION_TIMEFRAMES + ) assert "FUTURES_DERIVATIVES_CONTEXT_UNAVAILABLE" in blockers instance._complete_refresh_request({}, NOW, status="DATA_READY", blockers=()) with pytest.raises(ValueError, match="market is invalid"): @@ -677,7 +680,7 @@ def test_metered_transport_updates_budget_and_funding_clock( [ ("CURRENT", "CANDIDATES_AVAILABLE"), ("DELEGATED", "ANALYSIS_UNAVAILABLE"), - ("DATA_BLOCKED", "DATA_UNAVAILABLE"), + ("DATA_BLOCKED", "CANDIDATES_AVAILABLE_WITH_DATA_GAPS"), ("BLOCKED", "ANALYSIS_BLOCKED"), ], ) @@ -723,6 +726,7 @@ def test_cycle_failure_persists_safe_retry_state(tmp_path: Path) -> None: state = h._load(instance.history.state_path) assert state["status"] == "DEGRADED" assert state["last_error_type"] == "OSError" + assert state["last_error_code"] == "MARKET_HISTORY_RECOVERABLE_ERROR" assert "private path" not in json.dumps(state) diff --git a/tests/test_market_history_continuous.py b/tests/test_market_history_continuous.py index 1a9ec461..f25ba88b 100644 --- a/tests/test_market_history_continuous.py +++ b/tests/test_market_history_continuous.py @@ -10,6 +10,7 @@ from decimal import Decimal from pathlib import Path from threading import Event, Thread +from types import SimpleNamespace from typing import cast import pytest @@ -17,6 +18,7 @@ from ai4binance.cli.market_data import ( _build_canonical_opportunity_pipeline, _priority_depth_markets, + build_market_depth_collector, ) from ai4binance.cli.market_data import ( main as market_data_main, @@ -41,6 +43,9 @@ BinanceVisionArchiveCache, MarketHistorySynchronizer, ) +from ai4binance.infrastructure.persistence.safe_json import ( + DestinationVerificationError, +) from ai4binance.integrations.binance import BinanceEligibleMarketSnapshot from ai4binance.schemas import OHLCVCandle @@ -413,6 +418,7 @@ def test_recoverable_cycle_failure_is_persisted_for_retry(tmp_path: Path) -> Non state = _load(instance.history.state_path) assert state["status"] == "DEGRADED" assert state["last_error_type"] == "OSError" + assert state["last_error_code"] == "MARKET_HISTORY_RECOVERABLE_ERROR" assert state["recovery_action"] == "RETRY_NEXT_CYCLE" assert state["completed_streams"] == 5 assert state["total_streams"] == 10 @@ -424,6 +430,20 @@ def test_recoverable_cycle_failure_is_persisted_for_retry(tmp_path: Path) -> Non assert state["live_eligibility_status"] == "LIVE_ORDER_BLOCKED" +def test_recoverable_failure_exposes_only_verified_blocker_code( + tmp_path: Path, +) -> None: + instance = collector(tmp_path, Transport()) + + instance.record_recoverable_cycle_failure( + NOW, + DestinationVerificationError("MARKET_HISTORY_WRITE_FAILED"), + ) + + state = _load(instance.history.state_path) + assert state["last_error_code"] == "MARKET_HISTORY_WRITE_FAILED" + + def test_recoverable_cycle_failure_tolerates_malformed_previous_blockers( tmp_path: Path, ) -> None: @@ -454,9 +474,9 @@ def test_resume_fetches_native_timeframes_without_local_materialization( assert all(row["network_download"] is True for row in refresh_rows) assert {row["closed_history_source"] for row in refresh_rows} == { f"BINANCE_VISION_{timeframe.upper()}_DIRECT" - for timeframe in MARKET_HISTORY_TIMEFRAMES + for timeframe in VIRTUAL_MARKET_COLLECTION_TIMEFRAMES } - progress = tmp_path / "market/spot/BTCUSDT/5m/collection-progress.json" + progress = tmp_path / "market/spot/BTCUSDT/15m/collection-progress.json" resumed = collector(tmp_path, transport) second = resumed.sync_cycle(observed_at=NOW) assert second["status"] == "READY" @@ -572,13 +592,15 @@ def collect( report = instance.sync_cycle(observed_at=NOW) assert markets[:1] == ["spot"] - assert markets.count("spot") == 5 - assert markets.count("usd_m_futures") == 9 - assert snapshot_markets == (["spot", "usd_m_futures"] if long_backfill else []) + assert markets.count("spot") == len(MARKET_HISTORY_TIMEFRAMES) + assert markets.count("usd_m_futures") == (len(MARKET_HISTORY_TIMEFRAMES) + 4) + expected_snapshot_passes = 2 if long_backfill else 1 + assert snapshot_markets == ["spot", "usd_m_futures"] * expected_snapshot_passes assert report["completed_symbols"] == 2 assert report["total_symbols"] == 2 - assert report["completed_streams"] == 14 - assert report["total_streams"] == 14 + expected_streams = (2 * len(MARKET_HISTORY_TIMEFRAMES)) + 4 + assert report["completed_streams"] == expected_streams + assert report["total_streams"] == expected_streams assert report["completion_ratio"] == "1.000000" assert ready_symbols == [("SPOT", "ETHUSDT"), ("USD_M_FUTURES", "BTCUSDT")] assert report["opportunity_analysis_summary"] == {"CURRENT": 2} @@ -646,9 +668,9 @@ def test_stream_plan_uses_each_native_price_candle_feed() -> None: assert spot_streams == tuple( ("klines", timeframe) for timeframe in MARKET_HISTORY_TIMEFRAMES ) - assert futures_streams[:5] == spot_streams - assert len(spot_streams) == 5 - assert len(futures_streams) == 9 + assert futures_streams[: len(spot_streams)] == spot_streams + assert len(spot_streams) == len(MARKET_HISTORY_TIMEFRAMES) + assert len(futures_streams) == len(MARKET_HISTORY_TIMEFRAMES) + 4 def test_priority_symbols_precede_background_backfill(tmp_path: Path) -> None: @@ -684,7 +706,7 @@ def test_priority_symbols_precede_background_backfill(tmp_path: Path) -> None: assert len(streams) == 19 -def test_priority_depth_scope_does_not_subscribe_the_full_universe() -> None: +def test_priority_depth_scope_covers_the_bounded_active_universe() -> None: class DepthUniverse: spot_symbols = ("HOTUSDT", "ETHUSDT") futures_symbols = ("BTCUSDT", "ETHUSDT") @@ -695,13 +717,15 @@ class DepthUniverse: ("HOTUSDT", "BTCUSDT", "MISSING"), include_coin_m=True, ) == { - "spot": ("HOTUSDT",), - "usd_m_futures": ("BTCUSDT",), + "spot": ("HOTUSDT", "ETHUSDT"), + "usd_m_futures": ("BTCUSDT", "ETHUSDT"), "coin_m_futures": (), } -def test_priority_depth_scope_falls_back_to_top_volume_per_primary_market() -> None: +def test_priority_depth_scope_preserves_top_volume_order_without_watchlist_match() -> ( + None +): class DepthUniverse: spot_symbols = ("ETHUSDT", "BTCUSDT") futures_symbols = ("BTCUSDT", "ETHUSDT") @@ -712,11 +736,25 @@ class DepthUniverse: ("HOTUSDT",), include_coin_m=False, ) == { - "spot": ("ETHUSDT",), - "usd_m_futures": ("BTCUSDT",), + "spot": ("ETHUSDT", "BTCUSDT"), + "usd_m_futures": ("BTCUSDT", "ETHUSDT"), } +def test_depth_collector_shards_limit_reconnect_blast_radius(tmp_path: Path) -> None: + transport = Transport() + depth = build_market_depth_collector( + cast(MarketHistorySynchronizer, SimpleNamespace(archive_root=tmp_path)), + cast( + ContinuousMarketHistory, + SimpleNamespace(spot=transport, futures=transport, coin_m=None), + ), + ) + + assert depth.group_size == 10 + assert depth.root == tmp_path / "depth" + + def test_opportunity_analysis_starts_after_native_timeframe_ingestion( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -741,7 +779,7 @@ def analyze(*_args: object) -> Mapping[str, object]: instance.sync_cycle(observed_at=NOW) assert calls == list(MARKET_HISTORY_TIMEFRAMES) - assert analyzed_after == [5] + assert analyzed_after == [len(MARKET_HISTORY_TIMEFRAMES)] def test_native_streams_are_the_canonical_dashboard_input( @@ -756,7 +794,7 @@ def collect( *_args: object, timeframe: str | None, **_kwargs: object ) -> dict[str, object]: started.append(timeframe) - if timeframe == "1d": + if timeframe == "4h": native_streams_started.set() return {"status": "CURRENT"} @@ -805,7 +843,7 @@ def observe_incomplete_symbol( instance, "_collect_stream", lambda *_args, **kwargs: { - "status": "BACKFILLING" if kwargs.get("timeframe") == "5m" else "CURRENT" + "status": "BACKFILLING" if kwargs.get("timeframe") == "15m" else "CURRENT" }, ) @@ -879,7 +917,7 @@ def refresh(*_args: object, **kwargs: object) -> dict[str, object]: "timeframe": timeframe, "status": "CURRENT", } - for timeframe in MARKET_HISTORY_TIMEFRAMES + for timeframe in VIRTUAL_MARKET_COLLECTION_TIMEFRAMES ], "candidates": [ { @@ -996,6 +1034,27 @@ def test_dashboard_refresh_request_is_coalesced_and_other_symbol_is_busy( assert busy["live_eligibility_status"] == "LIVE_ORDER_BLOCKED" +def test_virtual_market_refresh_request_preserves_single_writer_safety( + tmp_path: Path, +) -> None: + path = tmp_path / "market-history-refresh-request.json" + + request = enqueue_market_history_refresh_request( + path, + market="SPOT", + symbol="BTCUSDT", + eligible_symbols=("BTCUSDT",), + requested_at=NOW, + requester="VIRTUAL_MARKET", + ) + status = market_history_refresh_status(path) + + assert request["state"] == "PENDING" + assert status["requester"] == "VIRTUAL_MARKET" + assert status["execution_allowed"] is False + assert status["live_eligibility_status"] == "LIVE_ORDER_BLOCKED" + + def test_dashboard_refresh_request_is_completed_by_the_canonical_collector( tmp_path: Path, ) -> None: @@ -1003,6 +1062,7 @@ def test_dashboard_refresh_request_is_completed_by_the_canonical_collector( instance.minimum_candles = 1 instance.pages_per_stream = 6 ready_at = NOW.replace(hour=23, minute=59) + instance.clock = lambda: ready_at request_path = (tmp_path / "market-history-refresh-request.json").resolve() instance.refresh_request_path = request_path request = enqueue_market_history_refresh_request( @@ -1024,6 +1084,59 @@ def test_dashboard_refresh_request_is_completed_by_the_canonical_collector( assert status["live_eligibility_status"] == "LIVE_ORDER_BLOCKED" +def test_refresh_request_age_uses_current_clock_after_cycle_setup( + tmp_path: Path, +) -> None: + instance = collector(tmp_path, Transport()) + instance.minimum_candles = 1 + instance.pages_per_stream = 6 + request_path = (tmp_path / "market-history-refresh-request.json").resolve() + instance.refresh_request_path = request_path + requested_at = NOW + timedelta(minutes=2) + instance.clock = lambda: requested_at + enqueue_market_history_refresh_request( + request_path, + market="SPOT", + symbol="BTCUSDT", + eligible_symbols=("BTCUSDT",), + requested_at=requested_at, + ) + + instance.sync_cycle(observed_at=NOW) + + status = market_history_refresh_status(request_path) + assert status["state"] == "DATA_READY", status + blockers = cast(tuple[object, ...], status["blockers"]) + assert "MARKET_HISTORY_REFRESH_REQUEST_EXPIRED" not in blockers + + +def test_refresh_request_completion_never_predates_its_request( + tmp_path: Path, +) -> None: + instance = collector(tmp_path, Transport()) + request_path = (tmp_path / "market-history-refresh-request.json").resolve() + instance.refresh_request_path = request_path + requested_at = NOW + timedelta(minutes=10) + request = enqueue_market_history_refresh_request( + request_path, + market="SPOT", + symbol="BTCUSDT", + eligible_symbols=("BTCUSDT",), + requested_at=requested_at, + ) + instance.clock = lambda: NOW + + instance._complete_refresh_request( + request, + NOW, + status="DATA_READY", + blockers=(), + ) + + completed = datetime.fromisoformat(str(_load(request_path)["completed_at"])) + assert completed >= requested_at + + def test_dashboard_request_preempts_the_bounded_background_queue( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -1164,7 +1277,7 @@ def test_invalid_pages_fail_closed(fault: str) -> None: def test_corrupt_progress_is_reported_without_reset(tmp_path: Path) -> None: transport = Transport() instance = collector(tmp_path, transport) - progress = tmp_path / "market/spot/BTCUSDT/5m/collection-progress.json" + progress = tmp_path / "market/spot/BTCUSDT/15m/collection-progress.json" _save( progress, { @@ -1176,7 +1289,7 @@ def test_corrupt_progress_is_reported_without_reset(tmp_path: Path) -> None: assert isinstance(report["blockers"], list) assert "MARKET_DATA_SOURCE_OR_INTEGRITY_FAILURE" in report["blockers"] assert not any( - path.endswith("klines") and params["interval"] == "5m" + path.endswith("klines") and params["interval"] == "15m" for path, params in transport.calls ) @@ -1857,6 +1970,37 @@ def test_dashboard_candidate_projection_filters_and_sanitizes_fields() -> None: ] +def test_dashboard_candidate_projection_preserves_tuple_blockers() -> None: + from ai4binance.cli.market_data import _dashboard_candidate_projection + + candidates: list[object] = [ + { + "market": "SPOT", + "symbol": "BTCUSDT", + "execution_allowed": False, + "live_eligibility_status": "LIVE_ORDER_BLOCKED", + "direction": "BULLISH", + "side": "BUY", + "quantity": "1", + "entry": "101", + "stop_loss": "98", + "tp1": "105", + "tp2": "108", + "tp3": "111", + "target_risk_reward": "2", + "blockers": ("RESEARCH_ONLY",), + } + ] + + projected = _dashboard_candidate_projection( + candidates, market="SPOT", symbol="BTCUSDT" + ) + + assert projected[0]["blockers"] == ["RESEARCH_ONLY"] + assert projected[0]["execution_allowed"] is False + assert projected[0]["live_eligibility_status"] == "LIVE_ORDER_BLOCKED" + + def test_canonical_opportunity_pipeline_validates_time_and_busy_lease( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -2073,7 +2217,7 @@ def record_recoverable_cycle_failure(self, *_args: object) -> None: depth_events: list[object] = [] class Depth: - def __init__(self, _root: Path, transports: object) -> None: + def __init__(self, _root: Path, transports: object, **_kwargs: object) -> None: depth_events.append(transports) def start(self, markets: object) -> None: @@ -2158,7 +2302,7 @@ def record_recoverable_cycle_failure(self, *_args: object) -> None: pass class Depth: - def __init__(self, *_args: object) -> None: + def __init__(self, *_args: object, **_kwargs: object) -> None: pass def close(self) -> None: diff --git a/tests/test_market_history_sync.py b/tests/test_market_history_sync.py index fdb4cb32..de3ffccc 100644 --- a/tests/test_market_history_sync.py +++ b/tests/test_market_history_sync.py @@ -271,6 +271,34 @@ def cycle(now: datetime) -> None: assert 1 <= sleeps[0] <= 900 +def test_supervisor_fast_retries_transient_universe_blocker(tmp_path: Path) -> None: + sleeps: list[float] = [] + attempts = 0 + + def cycle(_now: datetime) -> dict[str, object]: + nonlocal attempts + attempts += 1 + return { + "blockers": ( + ["PUBLIC_MARKET_LIQUIDITY_UNIVERSE_UNAVAILABLE"] + if attempts == 1 + else [] + ) + } + + supervisor = MarketHistorySupervisor( + synchronizer=object(), # type: ignore[arg-type] + interval_seconds=900, + lock_path=(tmp_path / "market-history.lock").resolve(), + sleeper=sleeps.append, + clock=lambda: OBSERVED_AT, + cycle=cycle, + ) + + assert supervisor.run(max_cycles=2) == 2 + assert sleeps == [30] + + def test_supervisor_does_not_hide_programming_errors(tmp_path: Path) -> None: supervisor = MarketHistorySupervisor( synchronizer=object(), # type: ignore[arg-type] @@ -355,6 +383,45 @@ def test_eligible_universe_force_refreshes_current_exchange_metadata( assert cached.futures_symbols == ("BTCUSDT",) +def test_force_refresh_falls_back_only_to_current_verified_universe_cache( + tmp_path: Path, +) -> None: + class TransientProvider: + calls = 0 + + def eligible_market_snapshot(self) -> BinanceEligibleMarketSnapshot: + self.calls += 1 + if self.calls == 1: + return BinanceEligibleMarketSnapshot( + spot_symbols=("BTCUSDT",), futures_symbols=("BTCUSDT",) + ) + return BinanceEligibleMarketSnapshot( + spot_symbols=(), + futures_symbols=(), + blockers=("PUBLIC_MARKET_UNIVERSE_UNAVAILABLE",), + ) + + provider = TransientProvider() + synchronizer = MarketHistorySynchronizer( + universe_provider=provider, # type: ignore[arg-type] + archive_root=tmp_path / "market", + source_cache=BinanceVisionArchiveCache(tmp_path / "sources", lambda _: b""), + state_path=tmp_path / "state.json", + ) + + synchronizer._eligible_universe(OBSERVED_AT, force_refresh=True) + current = synchronizer._eligible_universe( + OBSERVED_AT + timedelta(minutes=1), force_refresh=True + ) + stale = synchronizer._eligible_universe( + OBSERVED_AT + timedelta(minutes=6), force_refresh=True + ) + + assert current.spot_symbols == ("BTCUSDT",) + assert not current.blockers + assert stale.blockers == ("PUBLIC_MARKET_UNIVERSE_UNAVAILABLE",) + + def test_eligible_universe_uses_the_bounded_top_volume_provider_when_available( tmp_path: Path, ) -> None: diff --git a/tests/test_oos_maturity.py b/tests/test_oos_maturity.py index 6838f55a..ff46e9d2 100644 --- a/tests/test_oos_maturity.py +++ b/tests/test_oos_maturity.py @@ -11,12 +11,16 @@ from ai4binance.domain import ValidationStatus from ai4binance.validation.oos_maturity import ( + MAX_ARTIFACT_BYTES, REQUIRED_MEASUREMENTS, OOSArtifactReference, OOSMaturityEvidenceBundle, OOSMaturityGate, OOSValidationSubject, _matches, + _validation_events, + load_spot_oos_validation_specification, + prepare_spot_oos_deployment, ) from ai4binance.validation.promotion_evidence import ( PromotionEvidenceQuery, @@ -32,6 +36,114 @@ NOW = datetime(2026, 9, 11, tzinfo=UTC) +def test_validation_events_accept_large_hash_bound_append_only_ledger( + tmp_path: Path, +) -> None: + artifact_root = tmp_path / "validation" + artifact_root.mkdir() + source = artifact_root / "trend_continuation.jsonl" + prefix = json.dumps({"archived": "x" * MAX_ARTIFACT_BYTES}).encode() + b"\n" + event_types = ( + "BACKTEST_RESULT", + "WALK_FORWARD_REPORT", + "TUNING_REPORT", + "BACKTEST_ROBUSTNESS_REPORT", + ) + current = b"".join( + json.dumps( + { + "event_type": event_type, + "payload": {"result": {"sequence": index}}, + } + ).encode() + + b"\n" + for index, event_type in enumerate(event_types, start=1) + ) + raw = prefix + current + source.write_bytes(raw) + + events, reference = _validation_events( + {"artifact_sha256": ((str(source), sha256(raw).hexdigest()),)}, + artifact_root, + ) + + assert set(events) == set(event_types) + assert events["BACKTEST_RESULT"]["sequence"] == 1 + assert reference is not None + assert reference.path == "trend_continuation.jsonl" + assert reference.sha256 == sha256(raw).hexdigest() + + +def test_validation_events_reject_hash_mismatch(tmp_path: Path) -> None: + artifact_root = tmp_path / "validation" + artifact_root.mkdir() + source = artifact_root / "trend_continuation.jsonl" + source.write_text( + json.dumps({"event_type": "BACKTEST_RESULT", "payload": {"result": {}}}), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="SPOT_OOS_SOURCE_HASH_INVALID"): + _validation_events( + {"artifact_sha256": ((str(source), "0" * 64),)}, + artifact_root, + ) + + +def test_active_spot_specification_prepares_exact_research_only_deployment( + tmp_path: Path, +) -> None: + specification_path = Path("config/research/virtual_market_acceptance.yaml") + specification = load_spot_oos_validation_specification(specification_path) + assert specification["status"] == "ACTIVE" + assert specification["approval_status"] == "PENDING_INDEPENDENT_REVIEW" + assert _matches( + "2026-09-10T00:00:00+00:00", + {"operator": "present", "value": True}, + ) + artifact_root = tmp_path / "validation" + deployment_path = tmp_path / "config" / "runtime_validation_deployment.json" + result = prepare_spot_oos_deployment( + artifact_root=artifact_root, + deployment_path=deployment_path, + specification_path=specification_path, + run_cards=( + { + "hypothesis_id": "hyp:trend_continuation:1h", + "symbol": "BTCUSDT", + "timeframe": "1h", + "strategy_sha256": "a" * 64, + "config_sha256": "b" * 64, + "dataset_sha256": "c" * 64, + "code_revision": "WORKTREE_UNVERIFIED", + "fee_rate": 0.001, + "slippage_rate": 0.0005, + "promotion_status": "RESEARCH_ONLY", + "execution_allowed": False, + }, + ), + observed_at=NOW, + ) + assert result["status"] == "EVIDENCE_COLLECTION_IN_PROGRESS" + assert result["subject_count"] == 1 + + from ai4binance.agents.validation_gate import ValidationGate + + gate = ValidationGate.from_deployment( + artifact_root=artifact_root, + deployment_path=deployment_path, + as_of=NOW, + ) + assert not gate.evidence_load_blockers + assert len(gate.expected_subjects) == 1 + maturity = OOSMaturityGate(artifact_root).evaluate(gate.evidence_bundles[0]) + assert maturity.status == "OOS_MATURITY_INCOMPLETE" + assert "EVIDENCE_COLLECTION_IN_PROGRESS" in maturity.blockers + assert "DATASET_EVIDENCE_MISSING" in maturity.blockers + assert "PROMOTION_EVIDENCE_INCOMPLETE" in maturity.blockers + assert "VALIDATION_SPECIFICATION_MISSING" not in maturity.blockers + + def test_bonferroni_changes_the_interval_used_by_the_gate() -> None: single = assess_statistical_evidence( (0.01, 0.04, 0.03, 0.02), @@ -328,6 +440,11 @@ def test_runtime_validation_consumes_exact_maturity_and_keeps_all_vetoes( assert decision.blockers if case == "risk_veto": assert "RISK.VETO" in decision.blockers + if case == "unbound": + assert any( + blocker.startswith("OOS_SUBJECT_NOT_CONFIGURED:") + for blocker in decision.blockers + ) @pytest.mark.parametrize( diff --git a/tests/test_opportunity_monitor.py b/tests/test_opportunity_monitor.py index d83b88b7..64f8a43d 100644 --- a/tests/test_opportunity_monitor.py +++ b/tests/test_opportunity_monitor.py @@ -101,7 +101,7 @@ def test_monitor_helper_boundaries_and_research_estimates( with pytest.raises(ValueError, match="read boundary"): read_monitor(tmp_path, "SPOT", "BTCUSDT") - candidate = { + candidate: dict[str, object] = { "market": "SPOT", "direction": "BULLISH", "entry": "101", diff --git a/tests/test_repository_cleanup_audit.py b/tests/test_repository_cleanup_audit.py index b5aa880b..037d3f59 100644 --- a/tests/test_repository_cleanup_audit.py +++ b/tests/test_repository_cleanup_audit.py @@ -201,7 +201,9 @@ def test_repository_cleanup_audit_text_summarizes_structure_classifications() -> command="repository-cleanup-audit", ) - assert "static_file_decisions: ARCHIVE_CANDIDATE=8, ENTRY_POINT=12, KEEP=25" in text + assert ( + "static_file_decisions: ARCHIVE_CANDIDATE=13, ENTRY_POINT=11, KEEP=25" in text + ) report = payload["report"] assert isinstance(report, dict) static_classifications = report["static_unimported_classifications"] diff --git a/tests/test_repository_validator.py b/tests/test_repository_validator.py index cf8806da..87d7eb11 100644 --- a/tests/test_repository_validator.py +++ b/tests/test_repository_validator.py @@ -4276,6 +4276,28 @@ def test_repository_validator_allows_runtime_top_level_without_warning( assert report.live_eligibility_status == "LIVE_ORDER_BLOCKED" +def test_repository_policy_registers_publication_and_security_surfaces( + tmp_path: Path, +) -> None: + _write_required_knowledge_docs(tmp_path) + (tmp_path / ".gitleaksignore").write_text("", encoding="utf-8") + (tmp_path / "LICENSE").write_text("test license\n", encoding="utf-8") + (tmp_path / "examples").mkdir() + (tmp_path / "publication").mkdir() + + policy = RepositoryPolicy.ai4binance_vnext() + report = validate_repository(tmp_path, policy=policy) + registered = {".gitleaksignore", "LICENSE", "examples", "publication"} + + assert policy.version == "1.3.1" + assert registered <= set(policy.allowed_top_level_paths) + assert not any( + finding.kind is RepositoryFindingKind.UNKNOWN_TOP_LEVEL_PATH + and finding.path in registered + for finding in report.findings + ) + + def test_repository_validator_ignores_temporary_top_level_directories( tmp_path: Path, ) -> None: @@ -5772,6 +5794,65 @@ def test_governed_document_lock_approval_script_exports_verified_chain( assert evidence["current_alignment_status"] == "VERIFIED" assert evidence["approved_documents"][0]["hash_matches_current_content"] is True + manifest_path = tmp_path / GOVERNED_DOCUMENT_LOCK_MANIFEST_PATH + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + evidence_bytes = output_path.read_bytes() + evidence_sha256 = _sha256(output_path) + manifest["approval_records"][0]["approval_evidence_path"] = output_path.relative_to( + tmp_path + ).as_posix() + manifest["approval_records"][0]["approval_evidence_sha256"] = evidence_sha256 + manifest_path.write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + argv_before = sys.argv[:] + try: + sys.argv = [ + "export_governed_document_lock_approval.py", + "--repository-root", + str(tmp_path), + "--approval-id", + "AI4B-GOV-DOCLOCK-TEST-001", + "--output-path", + str(output_path), + ] + with pytest.raises(SystemExit) as excinfo: + runpy.run_path( + str(ROOT / "scripts" / "export_governed_document_lock_approval.py"), + run_name="__main__", + ) + assert excinfo.value.code == 0 + finally: + sys.argv = argv_before + assert output_path.read_bytes() == evidence_bytes + + output_path.write_text("{}\n", encoding="utf-8") + tampered_bytes = output_path.read_bytes() + argv_before = sys.argv[:] + try: + sys.argv = [ + "export_governed_document_lock_approval.py", + "--repository-root", + str(tmp_path), + "--approval-id", + "AI4B-GOV-DOCLOCK-TEST-001", + "--output-path", + str(output_path), + ] + with pytest.raises( + ValueError, + match="BOUND_APPROVAL_EVIDENCE_IMMUTABLE_MISMATCH", + ): + runpy.run_path( + str(ROOT / "scripts" / "export_governed_document_lock_approval.py"), + run_name="__main__", + ) + finally: + sys.argv = argv_before + assert output_path.read_bytes() == tampered_bytes + def test_governed_document_lock_approval_script_exports_historical_chain( tmp_path: Path, @@ -6023,6 +6104,32 @@ def test_governed_document_lock_sync_script_normalizes_legacy_written_owner_orph assert len(approval_record["approval_evidence_sha256"]) == 64 assert written_owner_approval == approval_record + evidence_path = tmp_path / approval_record["approval_evidence_path"] + evidence_bytes = evidence_path.read_bytes() + argv_before = sys.argv[:] + sys_path_before = sys.path[:] + try: + sys.path.insert(0, str(ROOT / "scripts")) + sys.argv = [ + "sync_governed_document_lock_approval_evidence.py", + "--repository-root", + str(tmp_path), + ] + with pytest.raises(SystemExit) as excinfo: + runpy.run_path( + str( + ROOT + / "scripts" + / "sync_governed_document_lock_approval_evidence.py" + ), + run_name="__main__", + ) + assert excinfo.value.code == 0 + finally: + sys.argv = argv_before + sys.path[:] = sys_path_before + assert evidence_path.read_bytes() == evidence_bytes + def test_repository_validator_allows_reserved_source_of_truth_filename( tmp_path: Path, diff --git a/tests/test_research_application.py b/tests/test_research_application.py index 500c8b49..23990cbb 100644 --- a/tests/test_research_application.py +++ b/tests/test_research_application.py @@ -653,6 +653,7 @@ def test_research_service_dge_failure_is_deterministic_and_preserves_portfolio() ) assert first.virtual_runtime_decision.eligibility.blockers == ( "DGE_EVALUATION_FAILED", + "DGE_EVALUATION_ERROR_TYPE:ENGINE_EVALUATION", "DGE_SIMULATION_NOT_APPROVED", ) assert ( @@ -685,6 +686,7 @@ def _malformed_context( ) assert workflow.virtual_runtime_decision.eligibility.blockers == ( "DGE_EVALUATION_FAILED", + "DGE_EVALUATION_ERROR_TYPE:CONTEXT_TYPE", "DGE_SIMULATION_NOT_APPROVED", ) assert ( diff --git a/tests/test_security_tooling_contract.py b/tests/test_security_tooling_contract.py index 9b80aa3e..4d34a17d 100644 --- a/tests/test_security_tooling_contract.py +++ b/tests/test_security_tooling_contract.py @@ -1299,7 +1299,7 @@ def test_gitleaks_ignore_list_contains_only_exact_historic_fingerprints() -> Non if line and not line.startswith("#") ] - assert len(entries) == 11 + assert len(entries) == 13 assert all(":generic-api-key:" in entry for entry in entries) assert all(re.fullmatch(r"[0-9a-f]{40}:.+:[1-9][0-9]*", entry) for entry in entries) diff --git a/tests/test_service_manifest.py b/tests/test_service_manifest.py index 772aba80..04708992 100644 --- a/tests/test_service_manifest.py +++ b/tests/test_service_manifest.py @@ -41,7 +41,7 @@ def test_service_manifest_is_shared_safe_and_complete() -> None: assert required["skill-discovery"].lock_file == "skill_discovery.lock" assert required["market-history"].task_name == "AI4BINANCE-Market-History" assert required["market-history"].command == ( - "python -m ai4binance.cli.market_gateway" + "python -m ai4binance.cli.market_data daemon" ) scheduled = {spec.service: spec for spec in specs if not spec.required} assert scheduled["ykb-report"].health_mode == "SCHEDULED" @@ -157,6 +157,10 @@ def test_startup_install_script_preserves_cli_module_runtime_commands() -> None: assert '$env:PYTHONDONTWRITEBYTECODE = "1"' in install_text assert "& $python -B @Arguments" in install_text assert "& $python @Arguments" not in install_text + assert "$previousErrorActionPreference = $ErrorActionPreference" in install_text + assert '$ErrorActionPreference = "Continue"' in install_text + assert "$ErrorActionPreference = $previousErrorActionPreference" in install_text + assert "$nativeExitCode = [int]$LASTEXITCODE" in install_text assert '-Arguments @("-m", "ai4binance.cli", "archive-public")' in install_text assert '-Arguments @("-m", "ai4binance.cli", "validate-research")' in install_text assert "-WindowStyle Hidden" in install_text @@ -189,6 +193,11 @@ def test_startup_install_script_preserves_cli_module_runtime_commands() -> None: assert 'Get-ServiceManifestEntry -Service "virtual-market"' in install_text assert "InstallVirtualMarket" in install_text assert "RunVirtualMarket" in install_text + assert "RestartVirtualMarket" in install_text + assert "RestartMarketHistory" in install_text + assert "function Restart-BoundedScheduledService" in install_text + assert "Exact $Service process tree did not stop cleanly." in install_text + assert "Fresh $Service lock owner was not observed" in install_text assert "ai4binance\\.cli\\s+virtual-market-daemon" in status_text assert "VIRTUAL_MARKET" in status_text assert 'Join-Path $stateDirectory "virtual-market.json"' in status_text diff --git a/tests/test_storage.py b/tests/test_storage.py index c167de58..c616748e 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -1,6 +1,8 @@ """Append-only audit storage and secret redaction tests.""" import json +import os +import time from datetime import UTC, date, datetime from decimal import Decimal from enum import Enum @@ -354,6 +356,18 @@ def test_bounded_jsonl_tail_returns_only_recent_nonempty_lines( ) +def test_bounded_jsonl_tail_discards_partial_leading_record(tmp_path: Path) -> None: + path = tmp_path / "large-records.jsonl" + large = json.dumps({"id": 1, "value": "x" * 200_000}).encode() + latest = json.dumps({"id": 2}).encode() + path.write_bytes(b'{"id":0}\n' + large + b"\n" + latest + b"\n") + + assert read_bounded_jsonl_tail(path, max_lines=2, max_bytes=500_000) == ( + large, + latest, + ) + + def test_verified_write_result_rejects_inconsistent_states() -> None: with pytest.raises(ValueError, match="identity"): VerifiedWriteResult("", "event", VerificationStatus.VERIFIED) @@ -401,6 +415,24 @@ def test_write_json_object_verified_reads_destination_back(tmp_path: Path) -> No assert json.loads(path.read_text(encoding="utf-8"))["status"] == "ok" +def test_write_json_object_verified_compares_canonical_json_shapes( + tmp_path: Path, +) -> None: + path = tmp_path / "state" / "latest.json" + + result = write_json_object_verified( + path, + {"blockers": ("FIRST", "SECOND")}, + blocker="STATE_VERIFY_FAILED", + ) + + assert result.expected_sha256 == result.observed_sha256 + assert json.loads(path.read_text(encoding="utf-8"))["blockers"] == [ + "FIRST", + "SECOND", + ] + + def test_write_json_object_verified_stops_on_failed_read_back( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -424,6 +456,61 @@ def tampered_read_text( ) +def test_write_json_object_verified_retries_transient_replace_denial( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + path = tmp_path / "state" / "latest.json" + original_replace = os.replace + attempts = 0 + delays: list[float] = [] + + def flaky_replace(source: Path, destination: Path) -> None: + nonlocal attempts + attempts += 1 + if attempts < 3: + raise PermissionError(5, "transient sharing violation") + original_replace(source, destination) + + monkeypatch.setattr(os, "replace", flaky_replace) + monkeypatch.setattr(time, "sleep", delays.append) + + result = write_json_object_verified( + path, + {"status": "ok"}, + blocker="STATE_VERIFY_FAILED", + ) + + assert result.status is VerificationStatus.VERIFIED + assert attempts == 3 + assert delays == [0.01, 0.02] + + +def test_write_json_object_verified_preserves_persistent_replace_denial( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + path = tmp_path / "state" / "latest.json" + attempts = 0 + + def denied_replace(_source: Path, _destination: Path) -> None: + nonlocal attempts + attempts += 1 + raise PermissionError(5, "persistent sharing violation") + + monkeypatch.setattr(os, "replace", denied_replace) + monkeypatch.setattr(time, "sleep", lambda _delay: None) + + with pytest.raises(PermissionError, match="persistent sharing violation"): + write_json_object_verified( + path, + {"status": "ok"}, + blocker="STATE_VERIFY_FAILED", + ) + + assert attempts == 8 + + def test_jsonl_store_optional_durable_flush( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/test_validation_pipeline.py b/tests/test_validation_pipeline.py index 5c5870b3..15f18aa8 100644 --- a/tests/test_validation_pipeline.py +++ b/tests/test_validation_pipeline.py @@ -25,8 +25,10 @@ from ai4binance.strategies.rules import historical_playbook_decision from ai4binance.validation import ParameterSet from ai4binance.validation_pipeline_runtime import ( + SPOT_VALIDATION_NOTIONAL_TO_EQUITY_RATIO, HistoricalValidationRuntime, _build_historical_decision_resolver, + runtime_spot_backtest_engine, ) @@ -90,6 +92,50 @@ def test_preflight_checkpoint_identity_includes_simulation_assumptions() -> None assert baseline.config_sha256(10) != changed.config_sha256(10) +def test_validation_walk_forward_uses_nonzero_purge_and_embargo() -> None: + config = HistoricalValidationRuntime._tuning_config(605).walk_forward + + assert config.purge_size == 1 + assert config.embargo_size == 1 + assert config.train_size + (config.test_size * 5) + 5 == 605 + + +def test_runtime_spot_backtest_sizing_is_price_normalized_and_cash_bounded() -> None: + candles = tuple( + OHLCVCandle( + timestamp=datetime(2026, 1, 1, hour=index, tzinfo=UTC), + open=Decimal("80000") + Decimal(index * 1000), + high=Decimal("81500") + Decimal(index * 1000), + low=Decimal("79000") + Decimal(index * 1000), + close=Decimal("80500") + Decimal(index * 1000), + volume=Decimal("100"), + ) + for index in range(3) + ) + + engine = runtime_spot_backtest_engine( + candles, + base_engine=BacktestEngine(), + notional_to_equity_ratio=SPOT_VALIDATION_NOTIONAL_TO_EQUITY_RATIO, + ) + + maximum_open = max(candle.open for candle in candles) + maximum_notional = maximum_open * engine.config.quantity + assert engine.config.quantity < Decimal("1") + assert maximum_notional <= ( + engine.config.initial_cash_usdt * SPOT_VALIDATION_NOTIONAL_TO_EQUITY_RATIO + ) + assert maximum_notional >= engine.config.minimum_notional + + +@pytest.mark.parametrize("ratio", [Decimal("0"), Decimal("1.01")]) +def test_runtime_spot_backtest_sizing_rejects_unsafe_ratios( + ratio: Decimal, +) -> None: + with pytest.raises(ValueError, match="within"): + HistoricalValidationRuntime(position_notional_to_equity_ratio=ratio) + + @pytest.mark.parametrize( ("overrides", "message"), [ @@ -346,8 +392,16 @@ def test_validation_pipeline_runs_six_playbooks_and_persists_evidence( ) assert resumed_trend.resumed_from_checkpoint is True assert resumed_trend.backtest is None + assert resumed_trend.run_card is not None + resumed_run_card = cast(dict[str, object], resumed_trend.run_card) + assert resumed_run_card["symbol"] == "BTCUSDT" + assert resumed_run_card["timeframe"] == "1h" assert resumed_trend.checkpoint_path is not None assert Path(resumed_trend.checkpoint_path).is_file() + checkpoint = json.loads( + Path(resumed_trend.checkpoint_path).read_text(encoding="utf-8") + ) + assert len(checkpoint["run_card_sha256"]) == 64 assert dict(resumed_trend.stage_timings_ms)["checkpoint_lookup"] >= 0 resumed_manifest = json.loads( sorted((artifact_directory / "BTCUSDT" / "runs").glob("*.manifest.json"))[ diff --git a/tests/test_virtual_wallet_journal.py b/tests/test_virtual_wallet_journal.py index 28fff809..01ace8b3 100644 --- a/tests/test_virtual_wallet_journal.py +++ b/tests/test_virtual_wallet_journal.py @@ -54,6 +54,64 @@ def test_virtual_wallet_journal_initializes_independent_wallets_and_report( assert "2026-09-12T13:00:00+00:00" in report +def test_daily_loss_tuning_trigger_requires_three_losses_in_same_utc_day() -> None: + from ai4binance import virtual_wallet_journal as module + + def movement(index: int, *, pnl: str, exit_at: datetime) -> dict[str, object]: + return { + "movement_id": f"movement:{index}", + "market": "SPOT", + "closed_trade": { + "trade_id": f"trade:{index}", + "exit_time": exit_at.isoformat(), + "net_pnl_usdt": pnl, + }, + "managed_position": { + "symbol": "BTCUSDT", + "timeframe": "1h", + "strategy_id": "trend_continuation", + "strategy_version": "1", + "strategy_config_hash": "a" * 64, + "entry_price": "100", + "initial_quantity": "1", + }, + } + + prior = movement(0, pnl="-2", exit_at=NOW - timedelta(days=1)) + first = movement(1, pnl="-1", exit_at=NOW - timedelta(hours=2)) + win = movement(2, pnl="3", exit_at=NOW - timedelta(hours=1)) + second = movement(3, pnl="-2", exit_at=NOW - timedelta(minutes=30)) + before = module._daily_loss_tuning_trigger((prior, first, win, second), NOW) + assert before["status"] == "NOT_TRIGGERED" + assert before["loss_count_today"] == 2 + + third = movement(4, pnl="-3", exit_at=NOW) + triggered = module._daily_loss_tuning_trigger( + (prior, first, win, second, third), NOW + ) + repeated = module._daily_loss_tuning_trigger( + ( + prior, + first, + win, + second, + third, + movement(5, pnl="-4", exit_at=NOW), + ), + NOW, + ) + + assert triggered["status"] == "TRIGGERED" + assert triggered["loss_threshold"] == 3 + assert triggered["loss_count_today"] == 3 + assert len(cast(list[object], triggered["losses"])) == 3 + assert len(cast(list[object], triggered["subjects"])) == 1 + assert repeated["trigger_id"] == triggered["trigger_id"] + assert repeated["execution_allowed"] is False + assert repeated["promotion_status"] == "RESEARCH_ONLY" + assert repeated["live_eligibility_status"] == "LIVE_ORDER_BLOCKED" + + def test_virtual_wallet_journal_records_every_change_once_with_timestamp( tmp_path: Path, ) -> None: From f823d2ff0d59640fbb6f0b0329e643be968736b0 Mon Sep 17 00:00:00 2001 From: Huseyin Cicek Date: Fri, 25 Sep 2026 19:22:34 +0300 Subject: [PATCH 14/19] fix: tolerate optional voice types in CI --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 80967b20..28e49ebe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,6 +85,10 @@ python_version = "3.14" strict = true files = ["src", "tests"] +[[tool.mypy.overrides]] +module = ["numpy", "numpy.*"] +ignore_missing_imports = true + [tool.pytest.ini_options] testpaths = ["tests"] From bfcf54ea0a40657046247fe0711a0348ed31dfc7 Mon Sep 17 00:00:00 2001 From: Huseyin Cicek Date: Fri, 25 Sep 2026 20:13:34 +0300 Subject: [PATCH 15/19] fix: stabilize optional voice type checks --- pyproject.toml | 11 ++++++++++- src/ai4binance/voice/runtime.py | 4 ++-- tests/test_voice_runtime.py | 4 ++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 28e49ebe..6de2823f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,7 +86,16 @@ strict = true files = ["src", "tests"] [[tool.mypy.overrides]] -module = ["numpy", "numpy.*"] +module = [ + "faster_whisper", + "faster_whisper.*", + "numpy", + "numpy.*", + "sounddevice", + "sounddevice.*", + "truststore", + "truststore.*", +] ignore_missing_imports = true diff --git a/src/ai4binance/voice/runtime.py b/src/ai4binance/voice/runtime.py index b5882c9f..63d879a4 100644 --- a/src/ai4binance/voice/runtime.py +++ b/src/ai4binance/voice/runtime.py @@ -218,7 +218,7 @@ class SoundDeviceRecorder: def capture(self) -> object | None: import numpy as np - import sounddevice as sd # type: ignore[import-untyped] + import sounddevice as sd try: audio = sd.rec( @@ -247,7 +247,7 @@ class FasterWhisperTranscriber: def __post_init__(self) -> None: import truststore - from faster_whisper import WhisperModel # type: ignore[import-untyped] + from faster_whisper import WhisperModel truststore.inject_into_ssl() self.model_directory.mkdir(parents=True, exist_ok=True) diff --git a/tests/test_voice_runtime.py b/tests/test_voice_runtime.py index fd110a62..000fb091 100644 --- a/tests/test_voice_runtime.py +++ b/tests/test_voice_runtime.py @@ -282,7 +282,7 @@ def __call__(self, *args: object, **kwargs: object) -> WhisperModelStub: def test_whisper_adapter_initializes_and_transcribes( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: - import faster_whisper # type: ignore[import-untyped] + import faster_whisper import truststore monkeypatch.setattr(truststore, "inject_into_ssl", lambda: None) @@ -340,7 +340,7 @@ def test_sound_recorder_distinguishes_signal_and_silence( monkeypatch: pytest.MonkeyPatch, ) -> None: import numpy as np - import sounddevice # type: ignore[import-untyped] + import sounddevice monkeypatch.setattr(sounddevice, "wait", lambda: None) monkeypatch.setattr( From e7e5752f5ebc841004d2f920fb16aae207bf19f1 Mon Sep 17 00:00:00 2001 From: Huseyin Cicek Date: Fri, 25 Sep 2026 20:39:20 +0300 Subject: [PATCH 16/19] fix: ignore generated build artifacts in quality attestation --- .../governance/constitution_sync.py | 2 ++ tests/test_governance_constitution_sync.py | 26 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/src/ai4binance/governance/constitution_sync.py b/src/ai4binance/governance/constitution_sync.py index 6bc26431..116f566b 100644 --- a/src/ai4binance/governance/constitution_sync.py +++ b/src/ai4binance/governance/constitution_sync.py @@ -820,6 +820,8 @@ def _quality_gate_attestation_ignores(parts: tuple[str, ...]) -> bool: return True if normalized == ".coverage" or normalized.startswith(".coverage."): return True + if normalized.endswith((".egg-info", ".pyc", ".pyo")): + return True return False diff --git a/tests/test_governance_constitution_sync.py b/tests/test_governance_constitution_sync.py index 1d9f0091..0fc8ca1e 100644 --- a/tests/test_governance_constitution_sync.py +++ b/tests/test_governance_constitution_sync.py @@ -62,6 +62,32 @@ def test_quality_attestation_does_not_inherit_an_ancestor_git_repository( assert attestation.change_set_sha256 == sha256(b"").hexdigest() +def test_quality_attestation_ignores_source_generated_build_artifacts( + tmp_path: Path, +) -> None: + source = tmp_path / "src" / "ai4binance" + source.mkdir(parents=True) + (source / "module.py").write_text("VALUE = 1\n", encoding="utf-8") + baseline = constitution_sync_module.build_quality_gate_workspace_attestation( + tmp_path + ) + + egg_info = tmp_path / "src" / "ai4binance.egg-info" + egg_info.mkdir() + (egg_info / "PKG-INFO").write_text( + "Metadata-Version: 2.1\n", + encoding="utf-8", + ) + (source / "module.cpython-314.pyc").write_bytes(b"generated") + (source / "module.pyo").write_bytes(b"generated") + + generated = constitution_sync_module.build_quality_gate_workspace_attestation( + tmp_path + ) + + assert generated.repository_tree_sha256 == baseline.repository_tree_sha256 + + def write_core_documents(root: Path, *, compliance_extra: str = "") -> None: (root / "AGENTS.md").write_text( "\n".join( From 9f1716363be60a9fc09a3fb90c3899a1f5c531d1 Mon Sep 17 00:00:00 2001 From: Huseyin Cicek Date: Fri, 25 Sep 2026 22:16:04 +0300 Subject: [PATCH 17/19] feat: add by HsC --- src/ai4binance/cli/runtime.py | 11 +- .../data/market_history_continuous.py | 124 +++++-- src/ai4binance/events/file_lock.py | 7 +- src/ai4binance/internal_radar.py | 2 +- tests/test_cli.py | 11 +- tests/test_event_journal.py | 31 ++ tests/test_internal_radar.py | 4 +- tests/test_market_history_continuous.py | 340 +++++++++++++++++- 8 files changed, 484 insertions(+), 46 deletions(-) diff --git a/src/ai4binance/cli/runtime.py b/src/ai4binance/cli/runtime.py index 01280c66..fc51f892 100644 --- a/src/ai4binance/cli/runtime.py +++ b/src/ai4binance/cli/runtime.py @@ -687,14 +687,18 @@ def run_virtual_market_daemon( ), } ) + cycle_observed_at = clock() exit_code = _run_virtual_market_research_cycle( - cycle_settings, acquisition, cycle_report=cycle_report + cycle_settings, + acquisition, + observed_at=cycle_observed_at, + cycle_report=cycle_report, ) _request_market_history_refresh_if_stale( state_path.with_name(_MARKET_HISTORY_REFRESH_REQUEST_NAME), symbol=last_symbol, eligible_symbols=eligible_symbols, - observed_at=clock(), + observed_at=cycle_observed_at, cycle_report=cycle_report, ) except (ExchangeError, OSError, RuntimeError, TypeError, ValueError): @@ -904,6 +908,7 @@ def _run_virtual_market_research_cycle( settings: Settings, public_acquisition: SnapshotAcquirer, *, + observed_at: datetime, cycle_report: dict[str, object] | None = None, ) -> int: from ai4binance.cli.research import run_public_research_command @@ -921,7 +926,7 @@ def _run_virtual_market_research_cycle( cycle_report["daily_loss_tuning"] = _run_virtual_loss_tuning( settings, journal, - datetime.now(UTC), + observed_at, ) return exit_code diff --git a/src/ai4binance/data/market_history_continuous.py b/src/ai4binance/data/market_history_continuous.py index 584e56c6..30eb5a43 100644 --- a/src/ai4binance/data/market_history_continuous.py +++ b/src/ai4binance/data/market_history_continuous.py @@ -66,6 +66,7 @@ # remains bounded to bootstrap and gap recovery. _PROGRESS_HEARTBEAT_SECONDS = 5 _SUPPLEMENTAL_STREAM_WORKERS = 1 +_VISION_ARCHIVE_BATCH_SIZE = 16 _COMPATIBLE_DERIVED_SOURCE_PREFIXES = ( "COMPATIBLE_DIRECT_PLUS_DERIVED_FROM_CANONICAL_1M:", "DERIVED_FROM_CANONICAL_1M:", @@ -476,9 +477,7 @@ class ContinuousMarketHistory: vision_history_enabled: bool = True on_symbol_ready: SymbolReadyHandler | None = field(default=None, repr=False) on_symbol_screen: SymbolReadyHandler | None = field(default=None, repr=False) - clock: Callable[[], datetime] = field( - default=lambda: datetime.now(UTC), repr=False - ) + clock: Callable[[], datetime] = field(default=lambda: datetime.now(UTC), repr=False) def __post_init__(self) -> None: if not 1 <= self.initial_days <= 3650 or not 1 <= self.pages_per_stream <= 32: @@ -533,7 +532,11 @@ def sync_cycle(self, *, observed_at: datetime) -> dict[str, object]: # VirtualMarket depends on these bounded bulk snapshots. Refresh once # at cycle start so a service restart cannot leave an already-old # ticker cache to expire during a long candle collection cycle. - snapshot_refresh_due = time.monotonic() + # ``universe`` was force-refreshed immediately above. Start with only + # the required market snapshots; metadata becomes due on the regular + # cadence after this pass instead of repeating the expensive top-volume + # universe request during the same cycle startup. + snapshot_refresh_due = 0.0 def refresh_snapshots(snapshot_time: datetime) -> None: nonlocal snapshot_refresh_due @@ -1382,9 +1385,12 @@ def complete_active_request() -> None: for item in results ): blockers.append("OPPORTUNITY_SCREENING_DATA_BLOCKED") + completed_at = self.clock() + if completed_at.utcoffset() is None: + raise ValueError("market history completion clock must be timezone-aware") payload: dict[str, object] = { "schema_version": "2.0", - "observed_at": now.isoformat(), + "observed_at": completed_at.astimezone(UTC).isoformat(), "status": "DEGRADED" if blockers else "READY", "timeframes": list(reported_timeframes), "timeframe_refresh_schedule": [ @@ -2040,6 +2046,32 @@ def _vision_history( ) -> tuple[datetime, dict[str, object] | None]: """Materialize all published closed history without public REST calls.""" + pending_candles: list[OHLCVCandle] = [] + pending_source_hashes: list[str] = [] + + def flush_pending() -> None: + if not pending_candles: + return + source_digest = ( + pending_source_hashes[0] + if len(pending_source_hashes) == 1 + else sha256("".join(pending_source_hashes).encode()).hexdigest() + ) + archive.update( + dataset_symbol, + timeframe, + tuple(pending_candles), + source=( + f"BINANCE_VISION_DIRECT_{timeframe.upper()}_SHA256:{source_digest}" + ), + generated_at=now, + replace_conflicts_from_sources=replace_conflicts_from_sources, + ) + state["next_at"] = cursor.isoformat() + _save(progress_path, state) + pending_candles.clear() + pending_source_hashes.clear() + while cursor < closed_history_end: month_start = cursor.replace( day=1, hour=0, minute=0, second=0, microsecond=0 @@ -2075,12 +2107,33 @@ def _vision_history( except MarketHistorySourceUnavailableError: # An archive may legitimately predate a new symbol's listing. # Search forward through closed history without replacing that - # missing period with high-volume REST backfill requests. - if not state.get("first_available_at"): + # missing period with high-volume REST backfill requests. This + # also applies when a retained dataset already proved a later + # first-available boundary and the requested history horizon + # is subsequently extended backwards. + raw_first_available = state.get("first_available_at") + known_first_available = ( + datetime.fromisoformat(str(raw_first_available)) + if raw_first_available + else None + ) + if ( + known_first_available is not None + and known_first_available.utcoffset() is None + ): + raise ValueError( + "collection first available boundary is invalid" + ) from None + if ( + known_first_available is None + or archive_end <= known_first_available + ): + flush_pending() cursor = archive_end state["next_at"] = cursor.isoformat() _save(progress_path, state) continue + flush_pending() return cursor, { "status": "UNAVAILABLE", "next_at": cursor.isoformat(), @@ -2094,32 +2147,34 @@ def _vision_history( break raise if candles[0].timestamp > cursor and state.get("first_available_at"): - return cursor, { - "status": "UNAVAILABLE", - "next_at": cursor.isoformat(), - "reason": "KLINE_GAP", - } + known_first_available = datetime.fromisoformat( + str(state["first_available_at"]) + ) + if known_first_available.utcoffset() is None: + raise ValueError("collection first available boundary is invalid") + if candles[0].timestamp == known_first_available: + cursor = known_first_available + else: + flush_pending() + return cursor, { + "status": "UNAVAILABLE", + "next_at": cursor.isoformat(), + "reason": "KLINE_GAP", + } state.setdefault("first_available_at", candles[0].timestamp.isoformat()) - updated = archive.update( - dataset_symbol, - timeframe, - candles, - source=( - f"BINANCE_VISION_DIRECT_{timeframe.upper()}_SHA256:{source.sha256}" - ), - generated_at=now, - replace_conflicts_from_sources=replace_conflicts_from_sources, - ) - last = datetime.fromisoformat(updated.last_timestamp) - cursor = max(candles[-1].timestamp + interval, last + interval) + pending_candles.extend(candles) + pending_source_hashes.append(str(source.sha256)) + cursor = candles[-1].timestamp + interval if cursor < archive_end: + flush_pending() return cursor, { "status": "UNAVAILABLE", "next_at": cursor.isoformat(), "reason": "KLINE_GAP", } - state["next_at"] = cursor.isoformat() - _save(progress_path, state) + if len(pending_source_hashes) >= _VISION_ARCHIVE_BATCH_SIZE: + flush_pending() + flush_pending() return cursor, None def _candles( @@ -2238,10 +2293,17 @@ def missing_window(start: datetime) -> tuple[datetime, datetime]: return start, end cursor, request_end = missing_window(cursor) - # Existing datasets need only their exact holes or tail. A monthly ZIP - # would replay verified rows; retain bulk archives for initial bootstrap. - if self.vision_history_enabled and not verified_ranges: - closed_history_end = now.replace(hour=0, minute=0, second=0, microsecond=0) + # Use checksum-verified native Vision archives for the exact missing + # window, including a history extension or an internal gap in an + # existing dataset. Bounding the archive walk at ``request_end`` avoids + # replaying already verified rows while eliminating hundreds of small + # REST pages for 5m candidate enrichment. + vision_end = min( + now.replace(hour=0, minute=0, second=0, microsecond=0), + request_end, + end, + ) + if self.vision_history_enabled and cursor < vision_end: cursor, unavailable = self._vision_history( market=market, symbol=symbol, @@ -2252,7 +2314,7 @@ def missing_window(start: datetime) -> tuple[datetime, datetime]: state=state, progress_path=progress_path, cursor=cursor, - closed_history_end=min(closed_history_end, end), + closed_history_end=vision_end, interval=interval, now=now, replace_conflicts_from_sources=replace_conflicts_from_sources, diff --git a/src/ai4binance/events/file_lock.py b/src/ai4binance/events/file_lock.py index 03ce0f0b..c9592e6b 100644 --- a/src/ai4binance/events/file_lock.py +++ b/src/ai4binance/events/file_lock.py @@ -42,10 +42,9 @@ def _acquire_file_lock(stream: BinaryIO) -> None: msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1) return except OSError as error: - if ( - error.errno not in _WINDOWS_RETRYABLE_LOCK_ERRORS - or time.monotonic() >= deadline - ): + if error.errno not in _WINDOWS_RETRYABLE_LOCK_ERRORS: + raise + if time.monotonic() >= deadline: raise TimeoutError( "exclusive file lock acquisition timed out" ) from error diff --git a/src/ai4binance/internal_radar.py b/src/ai4binance/internal_radar.py index 5f57d9d3..0ed65220 100644 --- a/src/ai4binance/internal_radar.py +++ b/src/ai4binance/internal_radar.py @@ -297,7 +297,7 @@ def _analyze_pending_candidates( ) analysis_count += 1 candidate["vision_evidence"] = evidence.to_payload() - candidate["last_scan_timestamp_utc"] = datetime.now(UTC).isoformat() + candidate["last_scan_timestamp_utc"] = observed_at.isoformat() candidate["assessment_status"] = evidence.status candidate["system_benefit"] = evidence.system_contribution or "NOT_ASSESSED" candidate["system_tradeoff"] = ( diff --git a/tests/test_cli.py b/tests/test_cli.py index 3ba9980a..f67c591c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1639,11 +1639,13 @@ def test_virtual_market_research_cycle_persists_both_wallets_and_report( ), ) cycle_report: dict[str, object] = {} + observed_at = datetime(2026, 9, 25, 12, 0, tzinfo=UTC) assert ( runtime_cli._run_virtual_market_research_cycle( settings, StubAcquisition(), + observed_at=observed_at, cycle_report=cycle_report, ) == 0 @@ -1845,6 +1847,7 @@ def research_cycle( settings: Settings, _acquisition: SnapshotAcquirer, *, + observed_at: datetime, cycle_report: dict[str, object] | None = None, ) -> int: assert cycle_report is not None @@ -1916,6 +1919,7 @@ def research_cycle( settings: Settings, _acquisition: SnapshotAcquirer, *, + observed_at: datetime, cycle_report: dict[str, object] | None = None, ) -> int: assert cycle_report is not None @@ -1972,10 +1976,12 @@ def research_cycle( cycle_settings: Settings, _acquisition: SnapshotAcquirer, *, + observed_at: datetime, cycle_report: dict[str, object] | None = None, ) -> int: assert cycle_report is not None assert cycle_settings.timeframes == VIRTUAL_MARKET_COLLECTION_TIMEFRAMES + assert observed_at == expected_observed_at cycle_report.update( snapshot_id="fixture:BTCUSDT", research_blockers=("SNAPSHOT_DATA_QUALITY_INVALID", "STALE_CANDLES:5m"), @@ -1986,13 +1992,13 @@ def research_cycle( monkeypatch.setattr( runtime_cli, "_run_virtual_market_research_cycle", research_cycle ) - observed_at = datetime(2026, 9, 24, 13, 30, tzinfo=UTC) + expected_observed_at = datetime(2026, 9, 24, 13, 30, tzinfo=UTC) assert ( runtime_cli.run_virtual_market_daemon( settings, max_cycles=1, public_acquisition=cast(SnapshotAcquirer, object()), - clock=lambda: observed_at, + clock=lambda: expected_observed_at, ) == 0 ) @@ -2072,6 +2078,7 @@ def run( settings: Settings, source: SnapshotAcquirer, *, + observed_at: datetime, cycle_report: dict[str, object] | None = None, ) -> int: seen.append(settings.symbol) diff --git a/tests/test_event_journal.py b/tests/test_event_journal.py index 4717b208..349c609f 100644 --- a/tests/test_event_journal.py +++ b/tests/test_event_journal.py @@ -494,3 +494,34 @@ def locking(_descriptor: int, _operation: int, _size: int) -> None: with pytest.raises(TimeoutError, match="file lock acquisition timed out"): file_lock._acquire_file_lock(stream) + + +def test_windows_file_lock_preserves_non_retryable_os_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import errno + import sys + from types import SimpleNamespace + + import ai4binance.events.file_lock as file_lock + + expected = OSError(errno.EBADF, "fixture invalid handle") + + def locking(_descriptor: int, _operation: int, _size: int) -> None: + raise expected + + monkeypatch.setattr(os, "name", "nt") + monkeypatch.setitem( + sys.modules, + "msvcrt", + SimpleNamespace(locking=locking, LK_NBLCK=1), + ) + stream = cast( + Any, + SimpleNamespace(seek=lambda *_args: None, fileno=lambda: 7), + ) + + with pytest.raises(OSError, match="fixture invalid handle") as raised: + file_lock._acquire_file_lock(stream) + + assert raised.value is expected diff --git a/tests/test_internal_radar.py b/tests/test_internal_radar.py index 6ee44a03..9fa2cff3 100644 --- a/tests/test_internal_radar.py +++ b/tests/test_internal_radar.py @@ -137,11 +137,13 @@ def analyze(self, **kwargs: object) -> VisionEvidence: route_decision=None, ) + observed_at = datetime(2026, 9, 25, 12, 30, tzinfo=UTC) result = run_internal_radar_once( repository_root=tmp_path, source_root=source, vision_enabled=True, vision_runner=ObservedVisionRunner(), # type: ignore[arg-type] + observed_at=observed_at, ) payload = result.to_payload() @@ -153,7 +155,7 @@ def analyze(self, **kwargs: object) -> VisionEvidence: assert progress == {"analysed": 1, "awaiting_analysis": 0} candidate = _payload_candidates(payload)[0] assert isinstance(candidate, dict) - assert str(candidate["last_scan_timestamp_utc"]).endswith("+00:00") + assert candidate["last_scan_timestamp_utc"] == observed_at.isoformat() markdown = result.latest_path.with_suffix(".md").read_text(encoding="utf-8") assert "Last scan timestamp (UTC)" in markdown diff --git a/tests/test_market_history_continuous.py b/tests/test_market_history_continuous.py index f25ba88b..c17e2bd6 100644 --- a/tests/test_market_history_continuous.py +++ b/tests/test_market_history_continuous.py @@ -41,6 +41,7 @@ from ai4binance.data.market_history_sync import ( MARKET_HISTORY_TIMEFRAMES, BinanceVisionArchiveCache, + MarketHistorySourceUnavailableError, MarketHistorySynchronizer, ) from ai4binance.infrastructure.persistence.safe_json import ( @@ -146,13 +147,23 @@ def test_staged_universe_downloads_baseline_then_enriches_candidates_before_anal ) -> None: instance = collector(tmp_path, Transport()) instance.max_workers = workers + completed_at = NOW + timedelta(hours=2) + instance.clock = lambda: completed_at + universe_refreshes: list[bool] = [] + + def eligible_universe( + *_args: object, **kwargs: object + ) -> BinanceEligibleMarketSnapshot: + universe_refreshes.append(bool(kwargs.get("force_refresh"))) + return BinanceEligibleMarketSnapshot( + spot_symbols=("BTCUSDT", "ETHUSDT"), + futures_symbols=("BTCUSDT", "ETHUSDT"), + ) + monkeypatch.setattr( MarketHistorySynchronizer, "_eligible_universe", - lambda *_a, **_k: BinanceEligibleMarketSnapshot( - spot_symbols=("BTCUSDT", "ETHUSDT"), - futures_symbols=("BTCUSDT", "ETHUSDT"), - ), + eligible_universe, ) calls: list[tuple[str, str, str | None]] = [] analyzed: list[tuple[str, str]] = [] @@ -183,6 +194,8 @@ def analyze(market: str, symbol: str, _now: datetime) -> Mapping[str, object]: instance.on_symbol_screen = screen instance.on_symbol_ready = analyze result = instance.sync_cycle(observed_at=NOW) + assert universe_refreshes == [True] + assert result["observed_at"] == completed_at.isoformat() assert len(calls) == 14 assert {tf for _, _, tf in calls} == set(VIRTUAL_MARKET_COLLECTION_TIMEFRAMES) assert sum(tf == "5m" for _, _, tf in calls) == 2 @@ -1251,6 +1264,325 @@ def test_candle_progress_extends_an_older_short_bootstrap_without_data_loss( assert state["coverage_extended_at"] == NOW.isoformat() +def test_extended_existing_history_uses_vision_for_only_the_missing_window( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + transport = Transport() + short_window = collector(tmp_path, transport, pages=1) + archive = ParquetOHLCVArchive(tmp_path / "market/spot") + stored_at = NOW - timedelta(minutes=5) + archive.update( + "BTCUSDT", + "5m", + ( + OHLCVCandle( + timestamp=stored_at, + open=Decimal("1"), + high=Decimal("1"), + low=Decimal("1"), + close=Decimal("1"), + volume=Decimal("1"), + ), + ), + source="BINANCE_PUBLIC_REST_5M", + generated_at=NOW, + ) + progress = tmp_path / "market/spot/BTCUSDT/5m/collection-progress.json" + _save( + progress, + { + "requested_start": stored_at.isoformat(), + "next_at": NOW.isoformat(), + "first_available_at": stored_at.isoformat(), + }, + ) + extended = ContinuousMarketHistory( + short_window.history, + transport, + transport, + initial_days=400, + pages_per_stream=1, + ) + observed: list[tuple[datetime, datetime]] = [] + + def recover(**kwargs: object) -> tuple[datetime, None]: + cursor = cast(datetime, kwargs["cursor"]) + closed_history_end = cast(datetime, kwargs["closed_history_end"]) + observed.append((cursor, closed_history_end)) + return closed_history_end, None + + monkeypatch.setattr(extended, "_vision_history", recover) + + result = extended._candles("spot", "BTCUSDT", "klines", transport, NOW) + + assert result["status"] == "CURRENT" + assert observed == [ + (NOW.replace(hour=0, minute=0) - timedelta(days=400), stored_at) + ] + assert transport.calls == [] + + +def test_vision_history_skips_missing_archives_before_known_listing( + tmp_path: Path, +) -> None: + class UnavailableCache: + def verified(self, key: str, *, kind: str) -> tuple[object, bytes]: + del kind + raise MarketHistorySourceUnavailableError(key) + + history = SimpleNamespace(source_cache=UnavailableCache()) + instance = ContinuousMarketHistory( + history, # type: ignore[arg-type] + Transport(), + Transport(), + ) + start = datetime(2026, 4, 1, tzinfo=UTC) + end = datetime(2026, 5, 1, tzinfo=UTC) + progress = tmp_path / "collection-progress.json" + state: dict[str, object] = { + "requested_start": start.isoformat(), + "next_at": start.isoformat(), + "first_available_at": datetime(2026, 6, 1, tzinfo=UTC).isoformat(), + } + + cursor, unavailable = instance._vision_history( + market="spot", + symbol="NEWUSDT", + kind="klines", + timeframe="5m", + archive=SimpleNamespace(), # type: ignore[arg-type] + dataset_symbol="NEWUSDT", + state=state, + progress_path=progress, + cursor=start, + closed_history_end=end, + interval=timedelta(minutes=5), + now=end, + ) + + assert unavailable is None + assert cursor == end + assert _load(progress)["next_at"] == end.isoformat() + + +def test_vision_history_retains_missing_archive_after_known_listing( + tmp_path: Path, +) -> None: + class UnavailableCache: + def verified(self, key: str, *, kind: str) -> tuple[object, bytes]: + del kind + raise MarketHistorySourceUnavailableError(key) + + history = SimpleNamespace(source_cache=UnavailableCache()) + instance = ContinuousMarketHistory( + history, # type: ignore[arg-type] + Transport(), + Transport(), + ) + start = datetime(2026, 7, 1, tzinfo=UTC) + end = datetime(2026, 8, 1, tzinfo=UTC) + progress = tmp_path / "collection-progress.json" + state: dict[str, object] = { + "requested_start": datetime(2026, 6, 1, tzinfo=UTC).isoformat(), + "next_at": start.isoformat(), + "first_available_at": datetime(2026, 6, 1, tzinfo=UTC).isoformat(), + } + + cursor, unavailable = instance._vision_history( + market="spot", + symbol="BTCUSDT", + kind="klines", + timeframe="5m", + archive=SimpleNamespace(), # type: ignore[arg-type] + dataset_symbol="BTCUSDT", + state=state, + progress_path=progress, + cursor=start, + closed_history_end=end, + interval=timedelta(minutes=5), + now=end, + ) + + assert cursor == start + assert unavailable == { + "status": "UNAVAILABLE", + "next_at": start.isoformat(), + "reason": "BINANCE_VISION_ARCHIVE_UNAVAILABLE", + } + + +def test_vision_history_accepts_known_midmonth_listing_boundary( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + class SourceCache: + def verified(self, key: str, *, kind: str) -> tuple[object, bytes]: + del key, kind + return ( + SimpleNamespace( + sha256="fixture", + network_request_count=0, + downloaded_bytes=0, + ), + b"fixture", + ) + + class Archive: + def update( + self, + _symbol: str, + _timeframe: str, + candles: tuple[OHLCVCandle, ...], + **_kwargs: object, + ) -> object: + return SimpleNamespace(last_timestamp=candles[-1].timestamp.isoformat()) + + history = SimpleNamespace(source_cache=SourceCache()) + instance = ContinuousMarketHistory( + history, # type: ignore[arg-type] + Transport(), + Transport(), + ) + start = datetime(2026, 6, 1, tzinfo=UTC) + first_available = datetime(2026, 6, 16, tzinfo=UTC) + end = datetime(2026, 7, 1, tzinfo=UTC) + progress = tmp_path / "collection-progress.json" + state: dict[str, object] = { + "requested_start": start.isoformat(), + "next_at": start.isoformat(), + "first_available_at": first_available.isoformat(), + } + + monkeypatch.setattr( + instance, + "_parse_vision_candles", + lambda *_args, interval, **_kwargs: ( + OHLCVCandle( + timestamp=first_available, + open=Decimal("1"), + high=Decimal("1"), + low=Decimal("1"), + close=Decimal("1"), + volume=Decimal("1"), + ), + OHLCVCandle( + timestamp=end - interval, + open=Decimal("1"), + high=Decimal("1"), + low=Decimal("1"), + close=Decimal("1"), + volume=Decimal("1"), + ), + ), + ) + + cursor, unavailable = instance._vision_history( + market="usd_m_futures", + symbol="NEWUSDT", + kind="klines", + timeframe="15m", + archive=Archive(), # type: ignore[arg-type] + dataset_symbol="NEWUSDT", + state=state, + progress_path=progress, + cursor=start, + closed_history_end=end, + interval=timedelta(minutes=15), + now=end, + ) + + assert unavailable is None + assert cursor == end + assert _load(progress)["next_at"] == end.isoformat() + + +def test_vision_history_batches_archives_without_jumping_to_dataset_tail( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + class SourceCache: + def verified(self, key: str, *, kind: str) -> tuple[object, bytes]: + del kind + return ( + SimpleNamespace( + sha256=hashlib.sha256(key.encode()).hexdigest(), + network_request_count=0, + downloaded_bytes=0, + ), + b"fixture", + ) + + update_sizes: list[int] = [] + + class Archive: + def update( + self, + _symbol: str, + _timeframe: str, + candles: tuple[OHLCVCandle, ...], + **_kwargs: object, + ) -> object: + update_sizes.append(len(candles)) + return SimpleNamespace( + last_timestamp=datetime(2026, 12, 31, tzinfo=UTC).isoformat() + ) + + history = SimpleNamespace(source_cache=SourceCache()) + instance = ContinuousMarketHistory( + history, # type: ignore[arg-type] + Transport(), + Transport(), + ) + start = datetime(2026, 1, 1, tzinfo=UTC) + end = datetime(2026, 5, 1, tzinfo=UTC) + progress = tmp_path / "collection-progress.json" + state: dict[str, object] = { + "requested_start": start.isoformat(), + "next_at": start.isoformat(), + } + + monkeypatch.setattr( + instance, + "_parse_vision_candles", + lambda _key, _payload, archive_start, archive_end, *, interval: ( + OHLCVCandle( + timestamp=archive_start, + open=Decimal("1"), + high=Decimal("1"), + low=Decimal("1"), + close=Decimal("1"), + volume=Decimal("1"), + ), + OHLCVCandle( + timestamp=archive_end - interval, + open=Decimal("1"), + high=Decimal("1"), + low=Decimal("1"), + close=Decimal("1"), + volume=Decimal("1"), + ), + ), + ) + + cursor, unavailable = instance._vision_history( + market="spot", + symbol="BTCUSDT", + kind="klines", + timeframe="15m", + archive=Archive(), # type: ignore[arg-type] + dataset_symbol="BTCUSDT", + state=state, + progress_path=progress, + cursor=start, + closed_history_end=end, + interval=timedelta(minutes=15), + now=end, + ) + + assert unavailable is None + assert cursor == end + assert update_sizes == [8] + assert _load(progress)["next_at"] == end.isoformat() + + @pytest.mark.parametrize( "fault", ["gap", "duplicate", "future", "negative_volume", "short", "nan"] ) From ed12a04a789f12a810dc304c8fbbacd96333b5cc Mon Sep 17 00:00:00 2001 From: Huseyin Cicek Date: Fri, 25 Sep 2026 23:24:13 +0300 Subject: [PATCH 18/19] Refresh current candle tails despite historical gaps --- .../data/market_history_continuous.py | 103 +++++++++++++++++- tests/test_market_history_continuous.py | 74 ++++++++++++- 2 files changed, 174 insertions(+), 3 deletions(-) diff --git a/src/ai4binance/data/market_history_continuous.py b/src/ai4binance/data/market_history_continuous.py index 30eb5a43..05c40144 100644 --- a/src/ai4binance/data/market_history_continuous.py +++ b/src/ai4binance/data/market_history_continuous.py @@ -22,7 +22,7 @@ ExchangeRateLimitError, ExchangeTransportError, ) -from ai4binance.data.archive import ParquetOHLCVArchive +from ai4binance.data.archive import DatasetManifest, ParquetOHLCVArchive from ai4binance.data.market_history_sync import ( MARKET_HISTORY_TIMEFRAMES, MarketHistorySourceUnavailableError, @@ -2263,6 +2263,29 @@ def _candles( source="CLOSED_CANDLE_BOUNDARY_REPAIR", generated_at=now, ) + tail_start = datetime.fromisoformat(manifest.last_timestamp) + interval + if tail_start < end: + # Historical availability gaps must remain visible, but they must + # not prevent the already verified active listing segment from + # receiving its latest closed candles. Refresh the contiguous + # tail before walking backwards into an unavailable history + # window; collection progress continues to point at that window. + manifest = self._refresh_current_candle_tail( + market=market, + symbol=symbol, + kind=kind, + timeframe=timeframe, + transport=transport, + archive=archive, + dataset_symbol=dataset_symbol, + directory=directory, + manifest=manifest, + start=tail_start, + end=end, + interval=interval, + now=now, + replace_conflicts_from_sources=replace_conflicts_from_sources, + ) last = datetime.fromisoformat(manifest.last_timestamp) + interval if cursor > last: raise ValueError("collection progress exceeds the verified dataset") @@ -2421,6 +2444,84 @@ def flush_rest_batch() -> None: "first_available_at": state.get("first_available_at"), } + def _refresh_current_candle_tail( + self, + *, + market: str, + symbol: str, + kind: str, + timeframe: str, + transport: JsonTransport, + archive: ParquetOHLCVArchive, + dataset_symbol: str, + directory: Path, + manifest: DatasetManifest, + start: datetime, + end: datetime, + interval: timedelta, + now: datetime, + replace_conflicts_from_sources: tuple[str, ...], + ) -> DatasetManifest: + """Refresh a verified active segment without hiding older gaps.""" + + cursor = start + pending: list[OHLCVCandle] = [] + source = f"BINANCE_PUBLIC_REST_{timeframe.upper()}" + prefix = _prefix(market) + for _ in range(self.pages_per_stream): + if cursor >= end: + break + remaining_intervals = max( + 1, + int( + ((end - cursor).total_seconds() + interval.total_seconds() - 1) + // interval.total_seconds() + ), + ) + params: dict[str, str | int] = { + "pair" if kind == "indexPriceKlines" else "symbol": symbol, + "interval": timeframe, + "startTime": int(cursor.timestamp() * 1000), + "endTime": int(end.timestamp() * 1000) - 1, + "limit": min(499, remaining_intervals), + } + if market == "coin_m_futures" and kind == "indexPriceKlines": + params["pair"] = self.coin_m_contracts[symbol][0] + raw = transport.get_json(prefix + kind, params) + if not isinstance(raw, list): + raise ValueError("kline response must be an array") + if not raw: + break + if len(raw) > 499: + raise ValueError("kline response exceeds its page limit") + candles = self._parse_rows(raw, cursor, end, interval=interval) + if candles[0].timestamp > cursor: + break + raw_payload: dict[str, object] = { + "source": source, + "volume_unit": "CONTRACTS" + if market == "coin_m_futures" and kind == "klines" + else "PROVIDER_NATIVE", + "rows": raw, + **_SAFE_STATE, + } + digest = sha256( + json.dumps(raw_payload, sort_keys=True).encode() + ).hexdigest() + _save(directory / "sources" / f"{digest}.json", raw_payload) + pending.extend(candles) + cursor = candles[-1].timestamp + interval + if not pending: + return manifest + return archive.update( + dataset_symbol, + timeframe, + tuple(pending), + source=source, + generated_at=now, + replace_conflicts_from_sources=replace_conflicts_from_sources, + ) + @staticmethod def _parse_rows( rows: list[object], diff --git a/tests/test_market_history_continuous.py b/tests/test_market_history_continuous.py index c17e2bd6..a59cf3cb 100644 --- a/tests/test_market_history_continuous.py +++ b/tests/test_market_history_continuous.py @@ -1237,12 +1237,17 @@ def test_long_shutdown_does_not_slide_requested_start_forward(tmp_path: Path) -> instance._candles("spot", "BTCUSDT", "klines", transport, NOW) progress = tmp_path / "market/spot/BTCUSDT/5m/collection-progress.json" saved = _load(progress) + archive = ParquetOHLCVArchive(tmp_path / "market/spot") + previous_tail = datetime.fromisoformat( + archive.manifest("BTCUSDT", "5m").last_timestamp + ) + timedelta(minutes=5) + previous_call_count = len(transport.calls) collector(tmp_path, transport, pages=1)._candles( "spot", "BTCUSDT", "klines", transport, NOW + timedelta(days=45) ) assert _load(progress)["requested_start"] == saved["requested_start"] - assert transport.calls[-1][1]["startTime"] == int( - datetime.fromisoformat(str(saved["next_at"])).timestamp() * 1000 + assert transport.calls[previous_call_count][1]["startTime"] == int( + previous_tail.timestamp() * 1000 ) @@ -1322,6 +1327,71 @@ def recover(**kwargs: object) -> tuple[datetime, None]: assert transport.calls == [] +def test_unavailable_history_does_not_leave_verified_active_tail_stale( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + transport = Transport() + instance = collector(tmp_path, transport, pages=1) + archive = ParquetOHLCVArchive(tmp_path / "market/spot") + stored_at = NOW.replace(minute=0) - timedelta(hours=1) + archive.update( + "BTCUSDT", + "5m", + ( + OHLCVCandle( + timestamp=stored_at, + open=Decimal("1"), + high=Decimal("1"), + low=Decimal("1"), + close=Decimal("1"), + volume=Decimal("1"), + ), + ), + source="BINANCE_PUBLIC_REST_5M", + generated_at=stored_at + timedelta(minutes=5), + ) + requested_start = NOW.replace(minute=0) - timedelta(days=400) + progress = tmp_path / "market/spot/BTCUSDT/5m/collection-progress.json" + _save( + progress, + { + "requested_start": requested_start.isoformat(), + "next_at": requested_start.isoformat(), + "first_available_at": stored_at.isoformat(), + }, + ) + extended = ContinuousMarketHistory( + instance.history, + transport, + transport, + initial_days=400, + pages_per_stream=1, + ) + + def unavailable(**kwargs: object) -> tuple[datetime, dict[str, object]]: + cursor = cast(datetime, kwargs["cursor"]) + return cursor, { + "status": "UNAVAILABLE", + "next_at": cursor.isoformat(), + "reason": "BINANCE_VISION_ARCHIVE_UNAVAILABLE", + } + + monkeypatch.setattr(extended, "_vision_history", unavailable) + + result = extended._candles("spot", "BTCUSDT", "klines", transport, NOW) + + assert result["status"] == "UNAVAILABLE" + assert transport.calls[0][1]["startTime"] == int( + (stored_at + timedelta(minutes=5)).timestamp() * 1000 + ) + manifest = archive.manifest("BTCUSDT", "5m") + assert ( + manifest.last_timestamp + == (NOW.replace(minute=0) - timedelta(minutes=5)).isoformat() + ) + assert _load(progress)["next_at"] == requested_start.isoformat() + + def test_vision_history_skips_missing_archives_before_known_listing( tmp_path: Path, ) -> None: From fe757fdfe8694efa7d7a010edd17ccc8859c9e32 Mon Sep 17 00:00:00 2001 From: Huseyin Cicek Date: Sat, 26 Sep 2026 00:05:51 +0300 Subject: [PATCH 19/19] Handle transient cooldowns and Unicode archive checksums --- .../data/market_history_continuous.py | 27 ++++++++++++- src/ai4binance/data/market_history_sync.py | 6 ++- tests/test_market_history_continuous.py | 38 ++++++++++++++++++- tests/test_market_history_sync.py | 20 ++++++++++ 4 files changed, 88 insertions(+), 3 deletions(-) diff --git a/src/ai4binance/data/market_history_continuous.py b/src/ai4binance/data/market_history_continuous.py index 05c40144..98c4f9a1 100644 --- a/src/ai4binance/data/market_history_continuous.py +++ b/src/ai4binance/data/market_history_continuous.py @@ -67,6 +67,7 @@ _PROGRESS_HEARTBEAT_SECONDS = 5 _SUPPLEMENTAL_STREAM_WORKERS = 1 _VISION_ARCHIVE_BATCH_SIZE = 16 +_TRANSIENT_COOLDOWN_WAIT_SECONDS = 30.0 _COMPATIBLE_DERIVED_SOURCE_PREFIXES = ( "COMPATIBLE_DIRECT_PLUS_DERIVED_FROM_CANONICAL_1M:", "DERIVED_FROM_CANONICAL_1M:", @@ -405,7 +406,16 @@ def get_json( ) self.budget.acquire(path, params, priority=priority) with self._pace_lock: - if time.monotonic() < self.cooldown_until: + cooldown_remaining = self.cooldown_until - time.monotonic() + if 0 < cooldown_remaining <= _TRANSIENT_COOLDOWN_WAIT_SECONDS: + # One transient transport failure places the shared endpoint on + # a short cooldown. Wait once under the pacing lock so queued + # streams resume together instead of being misclassified as a + # burst of independent source failures. Long rate-limit and ban + # cooldowns continue to fail closed immediately. + self.sleeper(cooldown_remaining) + cooldown_remaining = self.cooldown_until - time.monotonic() + if cooldown_remaining > 0: raise ExchangeHttpError("public collection rate-limit cooldown") due = self._last_request + self.minimum_interval_seconds if path.endswith("fundingRate"): @@ -2225,6 +2235,21 @@ def _candles( verified_ranges: list[tuple[datetime, datetime]] = [] if parquet.exists(): manifest = archive.manifest(dataset_symbol, timeframe) + manifest_first = datetime.fromisoformat(manifest.first_timestamp) + raw_first_available = state.get("first_available_at") + known_first_available = ( + datetime.fromisoformat(str(raw_first_available)) + if raw_first_available + else None + ) + if ( + known_first_available is not None + and known_first_available.utcoffset() is None + ): + raise ValueError("collection first available boundary is invalid") + if known_first_available is None or manifest_first < known_first_available: + state["first_available_at"] = manifest.first_timestamp + _save(progress_path, state) if manifest.gaps: # Collapse identical legacy rows through the canonical merge; # conflicting values remain an integrity failure. diff --git a/src/ai4binance/data/market_history_sync.py b/src/ai4binance/data/market_history_sync.py index 41d03c05..ef50ea15 100644 --- a/src/ai4binance/data/market_history_sync.py +++ b/src/ai4binance/data/market_history_sync.py @@ -308,7 +308,11 @@ def _validated_key(self, key: str) -> str: @staticmethod def _verify_checksum(key: str, payload: bytes, published: bytes) -> str: try: - expected = published.decode("ascii").strip().split()[0].lower() + # Binance appends the archive filename to the checksum. Symbols may + # contain non-ASCII characters, while the SHA-256 token itself is + # always ASCII. Decode only that token so filename encoding cannot + # turn a valid digest into a false integrity failure. + expected = published.strip().split(maxsplit=1)[0].decode("ascii").lower() except (UnicodeError, IndexError): expected = "" actual = sha256(payload).hexdigest() diff --git a/tests/test_market_history_continuous.py b/tests/test_market_history_continuous.py index a59cf3cb..74aa1a4e 100644 --- a/tests/test_market_history_continuous.py +++ b/tests/test_market_history_continuous.py @@ -1357,7 +1357,7 @@ def test_unavailable_history_does_not_leave_verified_active_tail_stale( { "requested_start": requested_start.isoformat(), "next_at": requested_start.isoformat(), - "first_available_at": stored_at.isoformat(), + "first_available_at": NOW.isoformat(), }, ) extended = ContinuousMarketHistory( @@ -1390,6 +1390,7 @@ def unavailable(**kwargs: object) -> tuple[datetime, dict[str, object]]: == (NOW.replace(minute=0) - timedelta(minutes=5)).isoformat() ) assert _load(progress)["next_at"] == requested_start.isoformat() + assert _load(progress)["first_available_at"] == stored_at.isoformat() def test_vision_history_skips_missing_archives_before_known_listing( @@ -2314,6 +2315,41 @@ def failed_open(request: object, **kwargs: object) -> object: assert len(calls) == 1 +def test_transient_transport_cooldown_waits_once_then_resumes( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + class Recovering: + calls = 0 + + def get_json( + self, path: str, params: Mapping[str, str | int] | None = None + ) -> object: + del path, params + self.calls += 1 + if self.calls == 1: + raise ExchangeTransportError("temporary network failure") + return {"status": "recovered"} + + clock = [100.0] + + def sleep(delay: float) -> None: + clock[0] += delay + + monkeypatch.setattr( + "ai4binance.data.market_history_continuous.time.monotonic", + lambda: clock[0], + ) + recovering = Recovering() + transport = MeteredPublicTransport(recovering, tmp_path, sleeper=sleep) + + with pytest.raises(ExchangeTransportError, match="temporary"): + transport.get_json("/api/v3/klines") + + assert transport.get_json("/api/v3/klines") == {"status": "recovered"} + assert recovering.calls == 2 + assert clock[0] >= 130.0 + + def test_dashboard_candidate_projection_filters_and_sanitizes_fields() -> None: from ai4binance.cli.market_data import _dashboard_candidate_projection diff --git a/tests/test_market_history_sync.py b/tests/test_market_history_sync.py index de3ffccc..82d7529f 100644 --- a/tests/test_market_history_sync.py +++ b/tests/test_market_history_sync.py @@ -86,6 +86,26 @@ def fetch(url: str) -> bytes: cache.verified(key, kind="ohlcv") +def test_archive_cache_verifies_checksum_with_unicode_filename( + tmp_path: Path, +) -> None: + key = "data/futures/um/monthly/klines/龙虾USDT/15m/龙虾USDT-15m-2026-03.zip" + payload = _kline_zip(5) + digest = hashlib.sha256(payload).hexdigest() + published = f"{digest} 龙虾USDT-15m-2026-03.zip\n".encode() + + def fetch(url: str) -> bytes: + return published if url.endswith(".CHECKSUM") else payload + + source, recovered = BinanceVisionArchiveCache(tmp_path, fetch).verified( + key, kind="ohlcv" + ) + + assert recovered == payload + assert source.sha256 == digest + assert source.network_request_count == 2 + + def test_archive_cache_rejects_unsafe_and_unpublished_sources(tmp_path: Path) -> None: def missing(url: str) -> bytes: raise HTTPError(url, 404, "missing", Message(), None)