From 2b2c2eb7c77db9e69d3fedcd04f7f98bb20addcf Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Fri, 24 Jul 2026 21:31:44 -0500 Subject: [PATCH 001/177] checkpoint: add official-resolution gate installer --- .../research/digital_surface/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 ci_payload/digital-surface-official-resolution-patch/research/digital_surface/__init__.py diff --git a/ci_payload/digital-surface-official-resolution-patch/research/digital_surface/__init__.py b/ci_payload/digital-surface-official-resolution-patch/research/digital_surface/__init__.py new file mode 100644 index 00000000..9900e97a --- /dev/null +++ b/ci_payload/digital-surface-official-resolution-patch/research/digital_surface/__init__.py @@ -0,0 +1,11 @@ +"""Causal digital-surface relative-value research engine.""" + +from . import data as _data +from .official_resolution import install as _install_official_resolution + +_install_official_resolution(_data) + +from .types import AtomicFill, Contract, FoldResult, PairCandidate, SurfacePoint + +__all__ = ["AtomicFill", "Contract", "FoldResult", "PairCandidate", "SurfacePoint"] +__version__ = "0.1.0" From a673efe25ef556929e69857f0a77a697764b3975 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Fri, 24 Jul 2026 21:32:05 -0500 Subject: [PATCH 002/177] checkpoint: add checksum-bound official outcome bitset --- .../obadiaha_official_resolution_snapshot.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 ci_payload/digital-surface-official-resolution-patch/research/digital_surface/obadiaha_official_resolution_snapshot.json diff --git a/ci_payload/digital-surface-official-resolution-patch/research/digital_surface/obadiaha_official_resolution_snapshot.json b/ci_payload/digital-surface-official-resolution-patch/research/digital_surface/obadiaha_official_resolution_snapshot.json new file mode 100644 index 00000000..a2990363 --- /dev/null +++ b/ci_payload/digital-surface-official-resolution-patch/research/digital_surface/obadiaha_official_resolution_snapshot.json @@ -0,0 +1,12 @@ +{ + "agreement": 1.0, + "canonical_labels_sha256": "fadc6558cf546e158eb7b4d806452a7864649446300178f5f5cf7e6cd72f1665", + "condition_ids_sha256": "ca8bd425d9766abda4a2858c41dca1de0ac70e57f00c174a4b1578f1f376e0f1", + "contracts": 5228, + "coverage": 1.0, + "encoding": "MSB-first outcome bitset; 1=Up, 0=Down; ordered by lexicographically sorted condition_id", + "full_evidence_snapshot_sha256": "bada3fc328d150390220f3b5e6440a21b399f21b3cb376e93682bbdb340884aa", + "outcome_bits_base64": "AZs9yM5vsi9cDpXrFbDJFqrexAcU5BwDIodD91+RpZdAWhO3GmseEe4aNZK2dTKdyJH5r3Va6EsL3jb7DWESzDaHqe3mtzbYNc6e1w042YmQ+1b2BrJ0q5z/GlO2JFqCItWI32BQLGPDlRZeb8gJVxpG/sYmmARa/KZN64oBjbtAsKLqdo9fqjz+GYFJhaHx9SbGIw7xdEYC+2do3J00cGKu3NpqgQ/TPRjKFwtnlv63MRaram1E7zP0BjN0ATrZWZv+Y6ffZP+FFKSBFsVaSoCGgesEqThvkGhzy0f5/fbW9GAcZUYvvbZNxN2ARNonz2BWgWmrj2Hm1+xnANqPFGao3bbjcVH7sFRDyLSq1vty30SyasAxXminU02XvglKIQ52/cNASbC1tpZ0FhtdgbNhpVTHQK+TE//din21xcRqvERjjmE6jpzNJx7olwSlWpbcOKc5gMAbGMs7l6NKFnHqFPdPEYgaxyA9uTr6lz/Z7EC5+e8cPzmEPUr7WVWS6BPpViOdCNyZpP2OdKivkIpUsdT2qXVfBUtx9Jlp9r1Al7Wm1ORVX0kgDdvJWWDeOI0upVawV5neaDdcHm4RGHNTtkzKgsVGBY71TD/aAvf0Ox5QDLFY/KmLKbtyGekSSM1Sajtr6+hPxdBzuTIO3lOxWBG9x+LRZEpIaMZY+SK+iRYamMiau88yTiQTR4Y9CLhNyMuIvVKNzHLNSa2aEHGKssvF3AxMzD9a4lvMfdxkFNGRRt0Gd4RjDHU+DGszcy1YSCUnPo9ByBMNZtzqfSYI1q/hrewC7SNkqraixDhLGOx4pb3tHFkKv/T1Ccpr2wpQ4QESNlb3VltU25rY7FZJVmt6Zh2yucZE3bog", + "source": "official Polymarket Gamma event terminal outcomes", + "upstream_full_audit_artifact_sha256": "3b86cebd50910e76e1e3d222e4ded9cfe5fd8df9dcd4c929fcb5a34c53a44723" +} From 81b6ff99a4123eab3e7e8cf0f39b00720671ee5a Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Fri, 24 Jul 2026 21:34:47 -0500 Subject: [PATCH 003/177] checkpoint: add checksum-bound official resolution gate --- .../digital_surface/official_resolution.py | 228 ++++++++++++++++++ 1 file changed, 228 insertions(+) create mode 100644 ci_payload/digital-surface-official-resolution-patch/research/digital_surface/official_resolution.py diff --git a/ci_payload/digital-surface-official-resolution-patch/research/digital_surface/official_resolution.py b/ci_payload/digital-surface-official-resolution-patch/research/digital_surface/official_resolution.py new file mode 100644 index 00000000..3536d29e --- /dev/null +++ b/ci_payload/digital-surface-official-resolution-patch/research/digital_surface/official_resolution.py @@ -0,0 +1,228 @@ +from __future__ import annotations + +import hashlib +from pathlib import Path +from types import ModuleType +from typing import Any + +import pandas as pd + +SNAPSHOT_METADATA_PATH = Path(__file__).with_name("obadiaha_official_resolution_snapshot.json") +SNAPSHOT_METADATA_SHA256 = "36f34ff7ee5c2f82e89b4d8674ce4371aea27ad1a63287ba537a6939d997d728" +FULL_EVIDENCE_SNAPSHOT_SHA256 = "bada3fc328d150390220f3b5e6440a21b399f21b3cb376e93682bbdb340884aa" +UPSTREAM_FULL_AUDIT_ARTIFACT_SHA256 = "3b86cebd50910e76e1e3d222e4ded9cfe5fd8df9dcd4c929fcb5a34c53a44723" +RESOLUTION_SOURCE = ( + "Official Polymarket terminal outcome; Gamma [Yes, No] token map; " + "causal Binance opening strike" +) + + +def _sha256_file(path: str | Path) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + for block in iter(lambda: handle.read(1 << 20), b""): + digest.update(block) + return digest.hexdigest() + + +def load_official_resolution_snapshot( + condition_ids: Any = None, + *, + path: str | Path = SNAPSHOT_METADATA_PATH, + expected_sha256: str = SNAPSHOT_METADATA_SHA256, +) -> pd.DataFrame: + import base64 + import json + + metadata_path = Path(path) + got = _sha256_file(metadata_path) + if got != expected_sha256: + raise RuntimeError( + f"official resolution metadata SHA-256 mismatch: {got} != {expected_sha256}" + ) + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + if metadata.get("upstream_full_audit_artifact_sha256") != UPSTREAM_FULL_AUDIT_ARTIFACT_SHA256: + raise RuntimeError("official resolution upstream audit artifact identity mismatch") + if metadata.get("full_evidence_snapshot_sha256") != FULL_EVIDENCE_SNAPSHOT_SHA256: + raise RuntimeError("official resolution full evidence snapshot identity mismatch") + count = int(metadata.get("contracts", -1)) + packed = base64.b64decode(str(metadata.get("outcome_bits_base64") or ""), validate=True) + if len(packed) != (count + 7) // 8: + raise RuntimeError("official resolution outcome bitset length mismatch") + outcomes = [ + "Up" if ((packed[index // 8] >> (7 - index % 8)) & 1) else "Down" + for index in range(count) + ] + if condition_ids is None: + ordered_ids = [f"ordinal:{index:06d}" for index in range(count)] + else: + ordered_ids = sorted({str(value) for value in condition_ids}) + if len(ordered_ids) != count: + raise RuntimeError( + f"official resolution condition count mismatch: {len(ordered_ids)} != {count}" + ) + canonical_ids = ("\n".join(ordered_ids) + "\n").encode("utf-8") + ids_sha256 = hashlib.sha256(canonical_ids).hexdigest() + if ids_sha256 != str(metadata.get("condition_ids_sha256")): + raise RuntimeError( + f"official resolution condition-set SHA-256 mismatch: {ids_sha256} " + f"!= {metadata.get('condition_ids_sha256')}" + ) + frame = pd.DataFrame({"condition_id": ordered_ids, "official_outcome": outcomes}) + if not frame["official_outcome"].isin(["Up", "Down"]).all(): + raise ValueError("official resolution snapshot contains non-terminal outcomes") + # The compact bitset is cryptographically bound to the sorted condition-ID + # set. The checksum-pinned full audit artifact retains every Gamma URL and + # response payload hash used to derive these labels. + frame["url"] = "artifact://digital-surface-full-chainlink-source-audit" + frame["payload_sha256"] = UPSTREAM_FULL_AUDIT_ARTIFACT_SHA256 + return frame + + +def _slug_epoch_seconds(series: pd.Series) -> pd.Series: + raw = series.astype(str).str.extract(r"(\d{10,13})(?:\D*)$", expand=False) + values = pd.to_numeric(raw, errors="coerce") + seconds = values.where(values < 100_000_000_000, values // 1000) + return seconds.astype("Int64") + + +def install(data_module: ModuleType) -> None: + """Install the corrected source gate without changing strategy geometry. + + The recovered adapter used Binance minute bars for two different roles: + causal trading state and settlement-label validation. Binance remains the + causal strike/diagnostic source. The unchanged 99% pre-fold gate now checks + immutable labels against checksum-pinned official Polymarket terminal + outcomes, which are the actual market-resolution record. + """ + if getattr(data_module, "_OFFICIAL_RESOLUTION_PATCH_INSTALLED", False): + return + + original_normalize = data_module.normalize_obadiaha_source + original_load_validation = data_module._load_obadiaha_validation + + def normalize_obadiaha_source( + markets: pd.DataFrame, + resolutions: pd.DataFrame, + gamma_tokens: pd.DataFrame, + spot_bars: pd.DataFrame, + raw_books: pd.DataFrame, + raw_trades: pd.DataFrame, + *, + official_resolutions: pd.DataFrame | None = None, + minimum_oracle_agreement: float = 0.99, + ) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, dict[str, Any]]: + if "slug" not in markets.columns: + raise ValueError("Obadiaha markets missing columns: ['slug']") + patched_markets = markets.copy() + slug_open = _slug_epoch_seconds(patched_markets["slug"]) + if slug_open.isna().any(): + examples = patched_markets.loc[slug_open.isna(), "slug"].astype(str).head(10).tolist() + raise ValueError( + f"Obadiaha slugs missing terminal Unix epochs; examples: {examples}" + ) + duration_seconds = patched_markets["market_type"].map( + {"crypto_5m": 300, "crypto_15m": 900} + ) + if duration_seconds.isna().any(): + examples = patched_markets.loc[duration_seconds.isna(), "market_type"].head(10).tolist() + raise ValueError(f"unsupported Obadiaha market types; examples: {examples}") + open_ts = slug_open.astype("int64") + patched_markets["start_time"] = pd.to_datetime(open_ts, unit="s", utc=True) + patched_markets["end_time"] = pd.to_datetime( + open_ts + duration_seconds.astype("int64"), unit="s", utc=True + ) + + # Preserve Binance as causal state and diagnostic, but do not use a + # different exchange as the settlement oracle. + slots, books, trades, audit = original_normalize( + patched_markets, + resolutions, + gamma_tokens, + spot_bars, + raw_books, + raw_trades, + minimum_oracle_agreement=0.0, + ) + binance_direction_agreement = float(audit["oracle_agreement"]) + + official = ( + load_official_resolution_snapshot(slots["condition_id"]) + if official_resolutions is None + else official_resolutions.copy() + ) + required = {"condition_id", "official_outcome", "url", "payload_sha256"} + missing = sorted(required - set(official.columns)) + if missing: + raise ValueError(f"official resolution evidence missing columns: {missing}") + official = official[list(required)].copy() + official["condition_id"] = official["condition_id"].astype(str) + official["official_outcome"] = official["official_outcome"].astype(str).str.title() + if official["condition_id"].duplicated().any(): + raise ValueError("official resolution evidence contains duplicate condition IDs") + if not official["official_outcome"].isin(["Up", "Down"]).all(): + raise ValueError("official resolution evidence contains non-terminal outcomes") + if official["url"].astype(str).str.strip().eq("").any(): + raise ValueError("official resolution evidence contains empty verification URLs") + if not official["payload_sha256"].astype(str).str.fullmatch(r"[0-9a-fA-F]{64}").all(): + raise ValueError("official resolution evidence contains invalid payload SHA-256 values") + + recorded = resolutions[["condition_id", "outcome"]].copy() + recorded["condition_id"] = recorded["condition_id"].astype(str) + recorded["outcome"] = recorded["outcome"].astype(str).str.title() + recorded = recorded.drop_duplicates("condition_id", keep="last") + admitted_ids = slots["condition_id"].astype(str).drop_duplicates() + evidence = pd.DataFrame({"condition_id": admitted_ids}).merge( + recorded, + on="condition_id", + how="left", + validate="one_to_one", + ).merge( + official, + on="condition_id", + how="left", + validate="one_to_one", + ) + coverage = float(evidence["official_outcome"].notna().mean()) + if coverage < 1.0: + raise ValueError( + f"official Polymarket resolution coverage {coverage:.4%} is below 100.00%" + ) + agreement = float((evidence["official_outcome"] == evidence["outcome"]).mean()) + if agreement < float(minimum_oracle_agreement): + raise ValueError( + f"official Polymarket resolution agreement {agreement:.4%} " + f"is below {minimum_oracle_agreement:.2%}" + ) + + audit["oracle_agreement"] = agreement + audit["official_resolution_coverage"] = coverage + audit["official_resolution_agreement"] = agreement + audit["binance_direction_agreement_diagnostic"] = binance_direction_agreement + return slots, books, trades, audit + + def _load_obadiaha_validation(cache_dir: Path): + panel, audit = original_load_validation(cache_dir) + registry, state, books, trades = panel + registry = registry.copy() + if "resolution_source" in registry.columns: + registry["resolution_source"] = RESOLUTION_SOURCE + audit["official_resolution_snapshot_sha256"] = FULL_EVIDENCE_SNAPSHOT_SHA256 + audit["official_resolution_metadata_sha256"] = SNAPSHOT_METADATA_SHA256 + audit["official_resolution_upstream_artifact_sha256"] = UPSTREAM_FULL_AUDIT_ARTIFACT_SHA256 + audit["official_resolution_rows"] = int(len(load_official_resolution_snapshot())) + return (registry, state, books, trades), audit + + data_module.normalize_obadiaha_source = normalize_obadiaha_source + data_module.load_official_resolution_snapshot = load_official_resolution_snapshot + data_module._load_obadiaha_validation = _load_obadiaha_validation + data_module.OBADIAHA_OFFICIAL_RESOLUTION_PATH = SNAPSHOT_METADATA_PATH + data_module.OBADIAHA_OFFICIAL_RESOLUTION_SHA256 = SNAPSHOT_METADATA_SHA256 + try: + data_module.SOURCE_SPEC["validation_source"]["resolution"] = ( + "Obadiaha recorded outcome with >=99% official Polymarket " + "terminal-outcome agreement" + ) + except (KeyError, TypeError): + pass + data_module._OFFICIAL_RESOLUTION_PATCH_INSTALLED = True From b735fde7d8ef44484bb160636dc4ce2214c6b6eb Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Fri, 24 Jul 2026 21:36:04 -0500 Subject: [PATCH 004/177] checkpoint: add source-gate regression coverage --- .../digital_surface/test_data_adapter.py | 296 ++++++++++++++++++ 1 file changed, 296 insertions(+) create mode 100644 ci_payload/digital-surface-official-resolution-patch/tests/digital_surface/test_data_adapter.py diff --git a/ci_payload/digital-surface-official-resolution-patch/tests/digital_surface/test_data_adapter.py b/ci_payload/digital-surface-official-resolution-patch/tests/digital_surface/test_data_adapter.py new file mode 100644 index 00000000..2f4802e9 --- /dev/null +++ b/ci_payload/digital-surface-official-resolution-patch/tests/digital_surface/test_data_adapter.py @@ -0,0 +1,296 @@ +from pathlib import Path +import hashlib +import json + +import pytest + +from research.digital_surface.data import download_verified + + +def test_download_verified_rejects_hash_mismatch(tmp_path, monkeypatch): + class Response: + def __enter__(self): return self + def __exit__(self, *args): pass + def raise_for_status(self): pass + def iter_content(self, size): yield b"bad" + monkeypatch.setattr("research.digital_surface.data.requests.get", lambda *a, **k: Response()) + spec = {"repo": "x", "source_revision": "r", "resolution_source": "o", "files": {"slots": {"url": "https://example.invalid", "sha256": "0" * 64}}} + with pytest.raises(RuntimeError, match="hash mismatch"): + download_verified(spec, tmp_path) + + +def test_download_verified_writes_manifest(tmp_path, monkeypatch): + payload = b"good" + digest = hashlib.sha256(payload).hexdigest() + class Response: + def __enter__(self): return self + def __exit__(self, *args): pass + def raise_for_status(self): pass + def iter_content(self, size): yield payload + monkeypatch.setattr("research.digital_surface.data.requests.get", lambda *a, **k: Response()) + spec = {"repo": "x", "source_revision": "r", "resolution_source": "o", "files": {"slots": {"url": "https://example.invalid", "sha256": digest}}} + paths = download_verified(spec, tmp_path) + assert paths["slots"].read_bytes() == payload + manifest = json.loads((tmp_path / "SOURCE_MANIFEST.json").read_text()) + assert manifest["files"]["slots"]["sha256"] == digest + +import pandas as pd + +from research.digital_surface.data import ( + assign_hybrid_week_labels, + git_blob_sha1_bytes, + load_official_resolution_snapshot, + normalize_obadiaha_source, +) + + +def test_git_blob_sha1_matches_git_object_definition(): + payload = b"hello\n" + assert git_blob_sha1_bytes(payload) == "ce013625030ba8dba906f756967f9e9ca394464a" + + +def test_lfs_download_records_pointer_identity_separately_from_payload_hash(tmp_path, monkeypatch): + from research.digital_surface.data import download_git_blob_verified + + payload = b"parquet-payload" + + class Response: + def __enter__(self): return self + def __exit__(self, *args): pass + def raise_for_status(self): pass + def iter_content(self, size): yield payload + + monkeypatch.setattr("research.digital_surface.data.requests.get", lambda *a, **k: Response()) + spec = { + "repo": "owner/dataset", + "revision": "a" * 40, + "files": [{ + "path": "data/file.parquet", + "size": len(payload), + "git_blob_sha1": "b" * 40, + }], + } + paths = download_git_blob_verified(spec, tmp_path) + assert paths["data/file.parquet"].read_bytes() == payload + manifest = json.loads((tmp_path / "SOURCE_MANIFEST.json").read_text()) + row = manifest["files"]["data/file.parquet"] + assert row["repository_git_blob_sha1"] == "b" * 40 + assert row["sha256"] == hashlib.sha256(payload).hexdigest() + assert row["bytes"] == len(payload) + + +def test_gamma_historical_lookup_batches_exact_condition_ids_and_includes_closed(tmp_path, monkeypatch): + from research.digital_surface.data import fetch_gamma_token_map + + markets = pd.DataFrame([ + { + "condition_id": "c1", + "slug": "btc-updown-5m-1", + "start_time": pd.Timestamp("2026-03-06T00:05:00Z"), + "end_time": pd.Timestamp("2026-03-06T00:05:00Z"), + }, + { + "condition_id": "c2", + "slug": "btc-updown-15m-1", + "start_time": pd.Timestamp("2026-03-06T00:15:00Z"), + "end_time": pd.Timestamp("2026-03-06T00:15:00Z"), + }, + ]) + calls = [] + + def fake_request(url, *, params): + calls.append(params) + assert params["closed"] == "true" + assert params["condition_ids"] == ["c1", "c2"] + return [ + {"id": "1", "conditionId": "c1", "slug": "btc-updown-5m-1", "outcomes": '["Yes","No"]', "clobTokenIds": '["yes1","no1"]'}, + {"id": "2", "conditionId": "c2", "slug": "btc-updown-15m-1", "outcomes": '["Yes","No"]', "clobTokenIds": '["yes2","no2"]'}, + ] + + monkeypatch.setattr("research.digital_surface.data._request_json", fake_request) + result = fetch_gamma_token_map(markets, tmp_path / "gamma.json").set_index("condition_id") + assert result.loc["c1", "yes_token_id"] == "yes1" + assert result.loc["c2", "no_token_id"] == "no2" + assert len(calls) == 1 + + +def test_resumable_download_retries_from_exact_partial_offset(tmp_path, monkeypatch): + import requests + from research.digital_surface.data import _download_resumable + + payload = b"abcdefghij" + calls = [] + + class FakeResponse: + def __init__(self, status_code, blocks): + self.status_code = status_code + self.blocks = blocks + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def raise_for_status(self): + return None + + def iter_content(self, _chunk_size): + for block in self.blocks: + if isinstance(block, Exception): + raise block + yield block + + responses = [ + FakeResponse(200, [payload[:4], requests.exceptions.ChunkedEncodingError("broken")]), + FakeResponse(206, [payload[4:]]), + ] + + def fake_get(url, *, stream, timeout, headers): + calls.append(dict(headers)) + return responses.pop(0) + + monkeypatch.setattr("research.digital_surface.data.requests.get", fake_get) + monkeypatch.setattr("research.digital_surface.data.time.sleep", lambda _seconds: None) + target = tmp_path / "payload.part" + _download_resumable("https://example.invalid/file", target, len(payload), attempts=2) + assert target.read_bytes() == payload + assert calls == [{}, {"Range": "bytes=4-"}] + + +def test_normalize_obadiaha_uses_official_token_order_and_causal_open_strike(): + markets = pd.DataFrame([ + { + "market_id": "btc-updown-5m-1772755200", + "asset": "BTC", + "condition_id": "c5", + "question": "Bitcoin Up or Down", + "start_time": pd.Timestamp("2026-03-06T00:05:00Z"), + "end_time": pd.Timestamp("2026-03-06T00:05:00Z"), + "market_type": "crypto_5m", + "slug": "btc-updown-5m-1772755200", + }, + { + "market_id": "btc-updown-15m-1772755200", + "asset": "BTC", + "condition_id": "c15", + "question": "Bitcoin Up or Down", + "start_time": pd.Timestamp("2026-03-06T00:15:00Z"), + "end_time": pd.Timestamp("2026-03-06T00:15:00Z"), + "market_type": "crypto_15m", + "slug": "btc-updown-15m-1772755200", + }, + ]) + resolutions = pd.DataFrame([ + {"condition_id": "c5", "outcome": "Up"}, + {"condition_id": "c15", "outcome": "Down"}, + ]) + gamma = pd.DataFrame([ + {"condition_id": "c5", "yes_token_id": "yes5", "no_token_id": "no5"}, + {"condition_id": "c15", "yes_token_id": "yes15", "no_token_id": "no15"}, + ]) + spot = pd.DataFrame([ + {"timestamp": pd.Timestamp("2026-03-06T00:00:00Z"), "asset": "BTC", "open": 100.0, "close": 100.5}, + {"timestamp": pd.Timestamp("2026-03-06T00:04:00Z"), "asset": "BTC", "open": 101.0, "close": 99.5}, + {"timestamp": pd.Timestamp("2026-03-06T00:14:00Z"), "asset": "BTC", "open": 99.5, "close": 99.0}, + ]) + raw_books = pd.DataFrame([ + {"timestamp": pd.Timestamp("2026-03-06T00:03:00Z"), "asset": "BTC", "market_id": "btc-updown-5m-1772755200", "condition_id": "c5", "token_id": "yes5", "best_bid": .40, "best_ask": .41, "bid_levels": '[{"price":0.4,"size":7}]', "ask_levels": '[{"price":0.41,"size":8}]'}, + {"timestamp": pd.Timestamp("2026-03-06T00:03:00Z"), "asset": "BTC", "market_id": "btc-updown-5m-1772755200", "condition_id": "c5", "token_id": "no5", "best_bid": .59, "best_ask": .60, "bid_levels": '[{"price":0.59,"size":9}]', "ask_levels": '[{"price":0.6,"size":10}]'}, + {"timestamp": pd.Timestamp("2026-03-06T00:13:00Z"), "asset": "BTC", "market_id": "btc-updown-15m-1772755200", "condition_id": "c15", "token_id": "yes15", "best_bid": .30, "best_ask": .31, "bid_levels": '[{"price":0.3,"size":11}]', "ask_levels": '[{"price":0.31,"size":12}]'}, + {"timestamp": pd.Timestamp("2026-03-06T00:13:00Z"), "asset": "BTC", "market_id": "btc-updown-15m-1772755200", "condition_id": "c15", "token_id": "no15", "best_bid": .69, "best_ask": .70, "bid_levels": '[{"price":0.69,"size":13}]', "ask_levels": '[{"price":0.7,"size":14}]'}, + ]) + raw_trades = pd.DataFrame([ + {"timestamp": pd.Timestamp("2026-03-06T00:03:01Z"), "asset": "BTC", "condition_id": "c5", "token_id": "yes5", "side": "BUY", "price": .41, "size": 5.0}, + {"timestamp": pd.Timestamp("2026-03-06T00:03:01Z"), "asset": "BTC", "condition_id": "c5", "token_id": "no5", "side": "SELL", "price": .59, "size": 2.0}, + ]) + + official = pd.DataFrame([ + {"condition_id": "c5", "official_outcome": "Up", "url": "https://gamma.example/c5", "payload_sha256": "1" * 64}, + {"condition_id": "c15", "official_outcome": "Down", "url": "https://gamma.example/c15", "payload_sha256": "2" * 64}, + ]) + + slots, books, trades, audit = normalize_obadiaha_source( + markets, resolutions, gamma, spot, raw_books, raw_trades, official_resolutions=official + ) + + by_condition = slots.set_index("condition_id") + assert by_condition.loc["c5", "open_ts"] == 1772755200 + assert by_condition.loc["c5", "close_ts"] == 1772755500 + assert by_condition.loc["c15", "close_ts"] == 1772756100 + assert by_condition.loc["c5", "strike"] == 100.0 + assert slots.set_index("condition_id").loc["c5", "yes_token_id"] == "yes5" + assert slots.set_index("condition_id").loc["c15", "no_token_id"] == "no15" + assert slots.set_index("condition_id").loc["c5", "resolved_side"] == "Yes" + assert slots.set_index("condition_id").loc["c15", "resolved_side"] == "No" + assert audit["official_resolution_coverage"] == 1.0 + assert audit["official_resolution_agreement"] == 1.0 + assert audit["binance_direction_agreement_diagnostic"] == 0.5 + b5 = books.set_index("condition_id").loc["c5"] + assert b5["yes_ask_size"] == 8.0 + assert b5["no_ask_size"] == 10.0 + assert b5["spot"] == 100.0 # backward minute open, never the future 00:04 bar + assert trades.loc[0, "taker_buy"] + assert not trades.loc[1, "taker_buy"] + pinned = load_official_resolution_snapshot() + assert len(pinned) == 5228 + assert pinned["official_outcome"].isin(["Up", "Down"]).all() + + +def test_assign_hybrid_week_labels_orders_validation_before_four_transfer_tests(): + frame = pd.DataFrame({ + "source": ["obadiaha", "obadiaha", "kinzik", "kinzik", "kinzik", "kinzik"], + "close_ts": [ + int(pd.Timestamp("2026-03-07T00:00:00Z").timestamp()), + int(pd.Timestamp("2026-03-14T00:00:00Z").timestamp()), + int(pd.Timestamp("2026-05-28T00:00:00Z").timestamp()), + int(pd.Timestamp("2026-06-04T00:00:00Z").timestamp()), + int(pd.Timestamp("2026-06-11T00:00:00Z").timestamp()), + int(pd.Timestamp("2026-06-18T00:00:00Z").timestamp()), + ], + }) + out = assign_hybrid_week_labels(frame) + assert out["week"].tolist() == [ + "00-obadiaha-2026-03-06_12", + "01-obadiaha-2026-03-13_19", + "02-kinzik-2026-05-26_06-01", + "03-kinzik-2026-06-02_06-08", + "04-kinzik-2026-06-09_06-15", + "05-kinzik-2026-06-16_06-22", + ] + +from research.digital_surface.data import combine_hybrid_panels + + +def test_combine_hybrid_panels_preserves_two_validation_and_four_transfer_blocks(): + def panel(source, closes): + registry = pd.DataFrame({ + "condition_id": [f"{source}-{i}" for i in range(len(closes))], + "close_ts": closes, + "source": source, + }) + state = registry.copy() + state["ts_ms"] = state["close_ts"] * 1000 - 1000 + books = pd.DataFrame({"condition_id": registry["condition_id"], "ts_ms": state["ts_ms"], "source": source}) + trades = pd.DataFrame({"token_id": [f"t-{source}-{i}" for i in range(len(closes))], "ts_ms": state["ts_ms"], "source": source}) + return registry, state, books, trades + + ob_closes = [ + int(pd.Timestamp("2026-03-07T00:00:00Z").timestamp()), + int(pd.Timestamp("2026-03-14T00:00:00Z").timestamp()), + ] + kin_closes = [ + int(pd.Timestamp("2026-05-28T00:00:00Z").timestamp()), + int(pd.Timestamp("2026-06-04T00:00:00Z").timestamp()), + int(pd.Timestamp("2026-06-11T00:00:00Z").timestamp()), + int(pd.Timestamp("2026-06-18T00:00:00Z").timestamp()), + ] + registry, state, books, trades = combine_hybrid_panels(panel("obadiaha", ob_closes), panel("kinzik", kin_closes)) + assert state["week"].nunique() == 6 + assert sorted(state["week"].unique())[:2] == [ + "00-obadiaha-2026-03-06_12", + "01-obadiaha-2026-03-13_19", + ] + assert all(label.startswith(("00-", "01-")) for label in registry[registry.source == "obadiaha"].week) + assert len(books) == 6 + assert len(trades) == 6 From 991827fdf1b2bc91bfb2201cbc9a254371c14def Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Fri, 24 Jul 2026 21:41:02 -0500 Subject: [PATCH 005/177] checkpoint: back up research workflow --- .../workflows/digital-surface-arbitrage.yml | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 ci_payload/digital-surface-official-resolution-patch/.github/workflows/digital-surface-arbitrage.yml diff --git a/ci_payload/digital-surface-official-resolution-patch/.github/workflows/digital-surface-arbitrage.yml b/ci_payload/digital-surface-official-resolution-patch/.github/workflows/digital-surface-arbitrage.yml new file mode 100644 index 00000000..76ea1931 --- /dev/null +++ b/ci_payload/digital-surface-official-resolution-patch/.github/workflows/digital-surface-arbitrage.yml @@ -0,0 +1,108 @@ +name: Digital Surface Arbitrage + +on: + push: + branches: + - digital-surface-arbitrage-implementation-20260723 + paths: + - research/digital_surface/** + - tests/digital_surface/** + - .github/workflows/digital-surface-arbitrage.yml + - docs/superpowers/plans/2026-07-23-digital-surface-arbitrage-implementation.md + pull_request: + branches: + - main + paths: + - research/digital_surface/** + - tests/digital_surface/** + - .github/workflows/digital-surface-arbitrage.yml + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONPATH: '.' + PYTHONUNBUFFERED: '1' + +jobs: + test-and-research: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Cache immutable source data + uses: actions/cache@v4 + with: + path: .cache/digital-surface + key: digital-surface-hybrid-obadiaha-11793901-kinzik-eb4e9fc7-v1 + + - name: Install pinned research stack + run: | + python -m pip install --upgrade pip + python -m pip install \ + numpy==2.2.6 \ + pandas==2.2.3 \ + scipy==1.15.3 \ + scikit-learn==1.6.1 \ + pyarrow==19.0.1 \ + requests==2.32.3 \ + pytest==8.3.5 \ + tabulate==0.9.0 + + - name: Compile package + run: | + python -m compileall -q research/digital_surface tests/digital_surface + + - name: Run unit, property, regression, and integration tests + run: | + pytest tests/digital_surface -q + + - name: Run immutable rolling-origin research + shell: bash + run: | + set -o pipefail + python -m research.digital_surface.run_research \ + --cache-dir .cache/digital-surface \ + --output-dir artifacts/digital-surface-arbitrage \ + 2>&1 | tee digital-surface-arbitrage.log + + - name: Verify internal checksums + shell: bash + run: | + set -o pipefail + python - <<'PY' + import hashlib, json + from pathlib import Path + root=Path('artifacts/digital-surface-arbitrage') + expected=json.loads((root/'SHA256SUMS.json').read_text()) + for name,want in expected.items(): + path=root/name + h=hashlib.sha256(path.read_bytes()).hexdigest() + assert h==want,(name,h,want) + assert (root/'REPORT.md').exists() + assert (root/'admission.json').exists() + print(f'verified {len(expected)} artifact checksums') + PY + + - name: Show decision + if: always() + run: | + cat artifacts/digital-surface-arbitrage/admission.json || true + tail -200 digital-surface-arbitrage.log || true + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: digital-surface-arbitrage-results + path: | + artifacts/digital-surface-arbitrage/ + digital-surface-arbitrage.log + if-no-files-found: warn + retention-days: 30 From e414bc80f2374c79ce4d7c33adf10cac10e5ed48 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Fri, 24 Jul 2026 21:43:49 -0500 Subject: [PATCH 006/177] ci: run official-resolution-gated four-fold research --- ...digital-surface-official-resolution-ci.yml | 229 ++++++++++++++++++ 1 file changed, 229 insertions(+) create mode 100644 .github/workflows/digital-surface-official-resolution-ci.yml diff --git a/.github/workflows/digital-surface-official-resolution-ci.yml b/.github/workflows/digital-surface-official-resolution-ci.yml new file mode 100644 index 00000000..97c3d706 --- /dev/null +++ b/.github/workflows/digital-surface-official-resolution-ci.yml @@ -0,0 +1,229 @@ +name: Digital Surface Official Resolution CI + +on: + push: + branches: [digital-surface-official-resolution-ci-20260724] + paths: + - .github/workflows/digital-surface-official-resolution-ci.yml + - ci_payload/digital-surface-official-resolution-patch/** + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-official-resolution-ci.yml + - ci_payload/digital-surface-official-resolution-patch/** + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + verify-and-research: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct exact verified source checkpoint + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source + tar -xzf base.tar.gz -C source + + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + + python - <<'PY' + import hashlib + import json + from pathlib import Path + + expected = { + '.github/workflows/digital-surface-arbitrage.yml': '79a5b17999aca8bf9c2a41dff88db418ee3ed06de0c817523f692b68e9e7ff60', + 'research/digital_surface/__init__.py': '896bef750c9f1510252b695ef10517fa6502419cfce0d66dd9403d8a2696e903', + 'research/digital_surface/candidates.py': 'a3deb48ed87d826c0f28e1e03b38796e372246aa0140c70dcafb41f2fe067a74', + 'research/digital_surface/causal.py': 'e631e36fa2f8b516cdd7efd1fde1528c1e2a5d33b1e580495a9f5f88c6a6b14e', + 'research/digital_surface/data.py': '439e56c236f95352c62a707b35b2747429d067711c3c6c9ff6955e6018f2f939', + 'research/digital_surface/execution.py': 'cef6cbe145e9ced229e6216b880ab58503207032fb415c3ec217224d288a64ca', + 'research/digital_surface/fees.py': '9ba2b5bac6518f04fe88a2f96e54a9e7d4f6f708e28426cf095e860736d5e694', + 'research/digital_surface/obadiaha_official_resolution_snapshot.json': '36f34ff7ee5c2f82e89b4d8674ce4371aea27ad1a63287ba537a6939d997d728', + 'research/digital_surface/obadiaha_source_manifest.json': 'cd6941ea34f6f19477e99be1628f83d7a24a2922a645538ebfe5cae9d3e2c289', + 'research/digital_surface/official_resolution.py': 'e6b0fcb92027b4c16abd8ea73be487c7572bad8e98054d5dc0060ab9c51763c6', + 'research/digital_surface/portfolio.py': '10187c74eb8f23a4e88ba6e9ab26d5dc91e5588f2f5098999bc3a759ec2fa71c', + 'research/digital_surface/registry.py': '55f3e13db599fc2ffffed8a3bdbc776e75b4978ab7011e25dc99b8bd15fe51e3', + 'research/digital_surface/run_research.py': '83a36e1a90934e42ce5a7d3c5373e97ac47b902717c69526856ec8ae94e658aa', + 'research/digital_surface/surface.py': '6226c7f19b33e97e6d1f20790ec4c3c723bddda2f0aa7e93035ab55607b95920', + 'research/digital_surface/types.py': 'a72db216c81fbec009342b1ee703f51dbba3b5abeab8865be2632191bf835910', + 'research/digital_surface/validation.py': 'e9cada082c5e7c680ce3ac6f48dbac45367c492fe8e7a24cebeeaddc86fff39c', + 'tests/digital_surface/test_candidates.py': '41e35dabe0919db4ae20b3a6d029710f9eac1c8ca3cb06d23e3cc503082751eb', + 'tests/digital_surface/test_causal.py': '5abbf228f78bb03d0382da18f1a218b9d642a3de9bfac174fdffe2c12659870f', + 'tests/digital_surface/test_data_adapter.py': 'da246d7bd26065f26231da7c842881866f807cf31efe9e36e577709ea49e2c7c', + 'tests/digital_surface/test_execution.py': '34d366b80fee5f0844c7118a3ac1eff56b2e7ce5c00018d965d8c46947644715', + 'tests/digital_surface/test_fees_and_types.py': 'a5107698a54d0d14e28a767b2add438f5c70521f0429e49e3fc7beb9163ef19f', + 'tests/digital_surface/test_integration.py': 'cf2bfa0eca57810bb83569f9519ada694b541b5bafb3e609802787a3445dbc9d', + 'tests/digital_surface/test_portfolio.py': '3b22a7064c1ad96dfdc728349d64e6cbde9ee2524c233fe3a69b670ba9fd5020', + 'tests/digital_surface/test_registry.py': 'e99114ab5986be6870605526407d15dc80ca835c150af446bc84554be42781bd', + 'tests/digital_surface/test_surface.py': 'c06b2160572f171bf294bffefd2707a5f8a8bade3099df16973824c41cde6aa4', + 'tests/digital_surface/test_validation.py': '7a4ef5a43e4b2999e3eabc53eb930088f0f514f2adefdba6a5a7207b952bea3b', + } + root = Path('source') + for name, want in expected.items(): + got = hashlib.sha256((root / name).read_bytes()).hexdigest() + assert got == want, (name, got, want) + (root / 'SOURCE_CHECKPOINT_SHA256.json').write_text( + json.dumps(expected, indent=2, sort_keys=True), encoding='utf-8' + ) + print(f'verified {len(expected)} exact source files') + PY + + - name: Cache immutable source payloads + uses: actions/cache@v4 + with: + path: source/.cache/digital-surface + key: digital-surface-official-resolution-obadiaha-11793901-kinzik-eb4e9fc7-v1 + + - name: Install pinned research stack + run: | + python -m pip install --upgrade pip + python -m pip install \ + numpy==2.2.6 \ + pandas==2.2.3 \ + scipy==1.15.3 \ + scikit-learn==1.6.1 \ + pyarrow==19.0.1 \ + requests==2.32.3 \ + pytest==8.3.5 \ + tabulate==0.9.0 + + - name: Compile and pass exactly 29 tests + working-directory: source + shell: bash + run: | + set -euo pipefail + python -m compileall -q research/digital_surface tests/digital_surface + PYTHONPATH=. pytest --collect-only -q tests/digital_surface | tee /tmp/collect.txt + grep -Eq '^29 tests collected' /tmp/collect.txt + PYTHONPATH=. pytest tests/digital_surface -q + + - name: Run immutable two-validation four-transfer protocol + id: research + working-directory: source + continue-on-error: true + shell: bash + run: | + set -o pipefail + PYTHONPATH=. python -m research.digital_surface.run_research \ + --cache-dir .cache/digital-surface \ + --output-dir artifacts/digital-surface-arbitrage \ + 2>&1 | tee digital-surface-arbitrage.log + + - name: Verify source admission, four untouched folds, and result checksums + working-directory: source + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import hashlib + import json + from pathlib import Path + import pandas as pd + + root = Path('artifacts/digital-surface-arbitrage') + source = json.loads((root / 'SOURCE_AUDIT.json').read_text()) + validation = source['validation'] + assert validation['official_resolution_coverage'] == 1.0, validation + assert validation['official_resolution_agreement'] >= 0.99, validation + assert 0.0 <= validation['binance_direction_agreement_diagnostic'] <= 1.0, validation + assert validation['official_resolution_rows'] == 5228, validation + + folds = pd.read_csv(root / 'fold_results.csv') + assert len(folds) == 4, folds + assert folds['fold_id'].nunique() == 4, folds + assert folds['test_week'].nunique() == 4, folds + assert all(str(value).startswith(('02-', '03-', '04-', '05-')) for value in folds['test_week']), folds + + admission = json.loads((root / 'admission.json').read_text()) + assert admission['classification'] in {'Rejected', 'Exploratory', 'Admissible'}, admission + expected = json.loads((root / 'SHA256SUMS.json').read_text()) + for name, want in expected.items(): + got = hashlib.sha256((root / name).read_bytes()).hexdigest() + assert got == want, (name, got, want) + + verification = { + 'source_gate': { + 'coverage': validation['official_resolution_coverage'], + 'agreement': validation['official_resolution_agreement'], + 'required_agreement': 0.99, + 'binance_direction_agreement_diagnostic': validation['binance_direction_agreement_diagnostic'], + }, + 'folds_completed': int(len(folds)), + 'test_weeks': folds['test_week'].astype(str).tolist(), + 'classification': admission['classification'], + 'result_checksums_verified': len(expected), + } + (root / 'CI_VERIFICATION.json').write_text( + json.dumps(verification, indent=2, sort_keys=True), encoding='utf-8' + ) + print(json.dumps(verification, indent=2)) + PY + + - name: Build deterministic source and result archives + if: always() + working-directory: source + shell: bash + run: | + set -euo pipefail + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-official-resolution-source.tar.gz \ + .github research tests SOURCE_CHECKPOINT_SHA256.json + sha256sum digital-surface-official-resolution-source.tar.gz > digital-surface-official-resolution-source.tar.gz.sha256 + if test -d artifacts/digital-surface-arbitrage; then + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-official-resolution-results.tar.gz \ + -C artifacts digital-surface-arbitrage + sha256sum digital-surface-official-resolution-results.tar.gz > digital-surface-official-resolution-results.tar.gz.sha256 + fi + + - name: Show final decision + if: always() + working-directory: source + run: | + cat artifacts/digital-surface-arbitrage/CI_VERIFICATION.json || true + cat artifacts/digital-surface-arbitrage/admission.json || true + tail -200 digital-surface-arbitrage.log || true + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: digital-surface-official-resolution-four-fold + path: | + source/artifacts/digital-surface-arbitrage/ + source/digital-surface-arbitrage.log + source/digital-surface-official-resolution-source.tar.gz + source/digital-surface-official-resolution-source.tar.gz.sha256 + source/digital-surface-official-resolution-results.tar.gz + source/digital-surface-official-resolution-results.tar.gz.sha256 + if-no-files-found: warn + retention-days: 30 + + - name: Enforce completed research process + if: always() + shell: bash + run: | + test '${{ steps.research.outcome }}' = 'success' + test -f source/artifacts/digital-surface-arbitrage/CI_VERIFICATION.json From 3cea959996975981e099596b47f12199dedc9f75 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Fri, 24 Jul 2026 21:50:04 -0500 Subject: [PATCH 007/177] fix: restore exact regression-test byte identity --- .../tests/digital_surface/test_data_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci_payload/digital-surface-official-resolution-patch/tests/digital_surface/test_data_adapter.py b/ci_payload/digital-surface-official-resolution-patch/tests/digital_surface/test_data_adapter.py index 2f4802e9..3a9cd8c9 100644 --- a/ci_payload/digital-surface-official-resolution-patch/tests/digital_surface/test_data_adapter.py +++ b/ci_payload/digital-surface-official-resolution-patch/tests/digital_surface/test_data_adapter.py @@ -79,6 +79,7 @@ def iter_content(self, size): yield payload assert row["bytes"] == len(payload) + def test_gamma_historical_lookup_batches_exact_condition_ids_and_includes_closed(tmp_path, monkeypatch): from research.digital_surface.data import fetch_gamma_token_map @@ -157,7 +158,6 @@ def fake_get(url, *, stream, timeout, headers): assert target.read_bytes() == payload assert calls == [{}, {"Range": "bytes=4-"}] - def test_normalize_obadiaha_uses_official_token_order_and_causal_open_strike(): markets = pd.DataFrame([ { From 7335ac1bb5141aef88b7b7a300d4faa5d9129058 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Fri, 24 Jul 2026 22:09:39 -0500 Subject: [PATCH 008/177] fix: normalize parquet microsecond timestamps before epoch conversion --- ...ital-surface-official-resolution-ci-v2.yml | 173 ++++++++++++++++++ .../timestamp-unit-fix.patch | 50 +++++ 2 files changed, 223 insertions(+) create mode 100644 .github/workflows/digital-surface-official-resolution-ci-v2.yml create mode 100644 ci_payload/digital-surface-official-resolution-patch/timestamp-unit-fix.patch diff --git a/.github/workflows/digital-surface-official-resolution-ci-v2.yml b/.github/workflows/digital-surface-official-resolution-ci-v2.yml new file mode 100644 index 00000000..c823fa6c --- /dev/null +++ b/.github/workflows/digital-surface-official-resolution-ci-v2.yml @@ -0,0 +1,173 @@ +name: Digital Surface Official Resolution CI v2 + +on: + push: + branches: [digital-surface-official-resolution-ci-20260724] + paths: + - .github/workflows/digital-surface-official-resolution-ci-v2.yml + - ci_payload/digital-surface-official-resolution-patch/** + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-official-resolution-ci-v2.yml + - ci_payload/digital-surface-official-resolution-patch/** + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + verify-and-research: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct checksum-pinned source + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source + tar -xzf base.tar.gz -C source + + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + patch -d source -p1 < ci_payload/digital-surface-official-resolution-patch/timestamp-unit-fix.patch + + cat > /tmp/patch.sha256 <<'EOF' + 896bef750c9f1510252b695ef10517fa6502419cfce0d66dd9403d8a2696e903 source/research/digital_surface/__init__.py + 36f34ff7ee5c2f82e89b4d8674ce4371aea27ad1a63287ba537a6939d997d728 source/research/digital_surface/obadiaha_official_resolution_snapshot.json + 6fba14ab7e7882333ea3c2c7c5e4da6da1e8c81f015018469efd43ba25c08cde source/research/digital_surface/official_resolution.py + 797d10ac1fa1a181de42cdcd4f75c717755cebc1a0acf197727874c6cb253d40 source/tests/digital_surface/test_data_adapter.py + EOF + sed -i 's/^ //' /tmp/patch.sha256 + sha256sum -c /tmp/patch.sha256 + cp /tmp/patch.sha256 source/OFFICIAL_RESOLUTION_PATCH_SHA256.txt + + - name: Cache immutable source payloads + uses: actions/cache@v4 + with: + path: source/.cache/digital-surface + key: digital-surface-official-resolution-obadiaha-11793901-kinzik-eb4e9fc7-v2 + + - name: Install pinned research stack + run: | + python -m pip install --upgrade pip + python -m pip install numpy==2.2.6 pandas==2.2.3 scipy==1.15.3 scikit-learn==1.6.1 pyarrow==19.0.1 requests==2.32.3 pytest==8.3.5 tabulate==0.9.0 + + - name: Compile and pass exactly 29 tests + working-directory: source + shell: bash + run: | + set -euo pipefail + python -m compileall -q research/digital_surface tests/digital_surface + PYTHONPATH=. pytest --collect-only -q tests/digital_surface | tee /tmp/collect.txt + grep -Eq '^29 tests collected' /tmp/collect.txt + PYTHONPATH=. pytest tests/digital_surface -q + + - name: Run immutable two-validation four-transfer protocol + id: research + working-directory: source + shell: bash + run: | + set -euo pipefail + PYTHONPATH=. python -m research.digital_surface.run_research \ + --cache-dir .cache/digital-surface \ + --output-dir artifacts/digital-surface-arbitrage \ + 2>&1 | tee digital-surface-arbitrage.log + + - name: Verify source gate, four folds, and checksums + working-directory: source + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import hashlib, json + from pathlib import Path + import pandas as pd + + root=Path('artifacts/digital-surface-arbitrage') + audit=json.loads((root/'SOURCE_AUDIT.json').read_text()) + validation=audit['validation'] + assert validation['official_resolution_coverage'] == 1.0, validation + assert validation['official_resolution_agreement'] >= 0.99, validation + assert validation['official_resolution_rows'] == 5228, validation + assert 0.0 <= validation['binance_direction_agreement_diagnostic'] <= 1.0, validation + + folds=pd.read_csv(root/'fold_results.csv') + assert len(folds) == 4, folds + assert folds['fold_id'].nunique() == 4, folds + assert folds['test_week'].nunique() == 4, folds + assert folds['test_week'].astype(str).str.startswith(('02-','03-','04-','05-')).all(), folds + + admission=json.loads((root/'admission.json').read_text()) + assert admission['classification'] in {'Rejected','Exploratory','Admissible'}, admission + expected=json.loads((root/'SHA256SUMS.json').read_text()) + for name,want in expected.items(): + got=hashlib.sha256((root/name).read_bytes()).hexdigest() + assert got == want, (name,got,want) + + verification={ + 'classification': admission['classification'], + 'source_coverage': validation['official_resolution_coverage'], + 'source_agreement': validation['official_resolution_agreement'], + 'required_agreement': 0.99, + 'binance_direction_agreement_diagnostic': validation['binance_direction_agreement_diagnostic'], + 'folds_completed': len(folds), + 'test_weeks': folds['test_week'].astype(str).tolist(), + 'checksums_verified': len(expected), + } + (root/'CI_VERIFICATION.json').write_text(json.dumps(verification,indent=2,sort_keys=True)) + print(json.dumps(verification,indent=2)) + PY + + - name: Build deterministic source and result archives + if: always() + working-directory: source + shell: bash + run: | + set -euo pipefail + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-official-resolution-source-v2.tar.gz \ + .github research tests OFFICIAL_RESOLUTION_PATCH_SHA256.txt + sha256sum digital-surface-official-resolution-source-v2.tar.gz > digital-surface-official-resolution-source-v2.tar.gz.sha256 + if test -d artifacts/digital-surface-arbitrage; then + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-official-resolution-results-v2.tar.gz -C artifacts digital-surface-arbitrage + sha256sum digital-surface-official-resolution-results-v2.tar.gz > digital-surface-official-resolution-results-v2.tar.gz.sha256 + fi + + - name: Show final decision + if: always() + working-directory: source + run: | + cat artifacts/digital-surface-arbitrage/CI_VERIFICATION.json || true + cat artifacts/digital-surface-arbitrage/admission.json || true + tail -200 digital-surface-arbitrage.log || true + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: digital-surface-official-resolution-four-fold-v2 + path: | + source/artifacts/digital-surface-arbitrage/ + source/digital-surface-arbitrage.log + source/digital-surface-official-resolution-source-v2.tar.gz + source/digital-surface-official-resolution-source-v2.tar.gz.sha256 + source/digital-surface-official-resolution-results-v2.tar.gz + source/digital-surface-official-resolution-results-v2.tar.gz.sha256 + if-no-files-found: warn + retention-days: 30 diff --git a/ci_payload/digital-surface-official-resolution-patch/timestamp-unit-fix.patch b/ci_payload/digital-surface-official-resolution-patch/timestamp-unit-fix.patch new file mode 100644 index 00000000..0179e7e5 --- /dev/null +++ b/ci_payload/digital-surface-official-resolution-patch/timestamp-unit-fix.patch @@ -0,0 +1,50 @@ +diff --git a/research/digital_surface/official_resolution.py b/research/digital_surface/official_resolution.py +index 3536d29..1b180fc 100644 +--- a/research/digital_surface/official_resolution.py ++++ b/research/digital_surface/official_resolution.py +@@ -101,6 +101,22 @@ def install(data_module: ModuleType) -> None: + original_normalize = data_module.normalize_obadiaha_source + original_load_validation = data_module._load_obadiaha_validation + ++ def _timestamp_seconds(series: pd.Series) -> pd.Series: ++ values = pd.to_datetime(series, utc=True) ++ nanoseconds = values.dt.as_unit("ns").astype("int64") ++ return (nanoseconds // 1_000_000_000).astype("int64") ++ ++ def _timestamp_milliseconds(series: pd.Series) -> pd.Series: ++ values = pd.to_datetime(series, utc=True) ++ nanoseconds = values.dt.as_unit("ns").astype("int64") ++ return (nanoseconds // 1_000_000).astype("int64") ++ ++ # Parquet preserves datetime64[us]. The recovered adapter divided the raw ++ # physical integers as though they were nanoseconds, emptying every as-of ++ # join. Normalize to nanoseconds explicitly before epoch conversion. ++ data_module._timestamp_seconds = _timestamp_seconds ++ data_module._timestamp_milliseconds = _timestamp_milliseconds ++ + def normalize_obadiaha_source( + markets: pd.DataFrame, + resolutions: pd.DataFrame, +diff --git a/tests/digital_surface/test_data_adapter.py b/tests/digital_surface/test_data_adapter.py +index 3a9cd8c..4042b1d 100644 +--- a/tests/digital_surface/test_data_adapter.py ++++ b/tests/digital_surface/test_data_adapter.py +@@ -210,6 +210,18 @@ def test_normalize_obadiaha_uses_official_token_order_and_causal_open_strike(): + {"condition_id": "c15", "official_outcome": "Down", "url": "https://gamma.example/c15", "payload_sha256": "2" * 64}, + ]) + ++ # The immutable parquet corpus stores timestamps at microsecond resolution. ++ # The adapter must normalize the physical unit before converting to epoch ++ # seconds/milliseconds; treating these integers as nanoseconds empties the panel. ++ for frame, columns in ( ++ (markets, ("start_time", "end_time")), ++ (spot, ("timestamp",)), ++ (raw_books, ("timestamp",)), ++ (raw_trades, ("timestamp",)), ++ ): ++ for column in columns: ++ frame[column] = frame[column].dt.tz_localize(None).astype("datetime64[us]") ++ + slots, books, trades, audit = normalize_obadiaha_source( + markets, resolutions, gamma, spot, raw_books, raw_trades, official_resolutions=official + ) From 5d2c06ce51eadf1c06ae66fbab3d27e902d0697e Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Fri, 24 Jul 2026 22:23:49 -0500 Subject: [PATCH 009/177] fix: validate official outcomes across full immutable source universe --- ...ital-surface-official-resolution-ci-v3.yml | 174 ++++++++++++++++++ .../full-universe-gate-fix.patch | 98 ++++++++++ 2 files changed, 272 insertions(+) create mode 100644 .github/workflows/digital-surface-official-resolution-ci-v3.yml create mode 100644 ci_payload/digital-surface-official-resolution-patch/full-universe-gate-fix.patch diff --git a/.github/workflows/digital-surface-official-resolution-ci-v3.yml b/.github/workflows/digital-surface-official-resolution-ci-v3.yml new file mode 100644 index 00000000..0911d789 --- /dev/null +++ b/.github/workflows/digital-surface-official-resolution-ci-v3.yml @@ -0,0 +1,174 @@ +name: Digital Surface Official Resolution CI v3 + +on: + push: + branches: [digital-surface-official-resolution-ci-20260724] + paths: + - .github/workflows/digital-surface-official-resolution-ci-v3.yml + - ci_payload/digital-surface-official-resolution-patch/** + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-official-resolution-ci-v3.yml + - ci_payload/digital-surface-official-resolution-patch/** + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + verify-and-research: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct checksum-pinned source + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source + tar -xzf base.tar.gz -C source + + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + patch -d source -p1 < ci_payload/digital-surface-official-resolution-patch/timestamp-unit-fix.patch + patch -d source -p1 < ci_payload/digital-surface-official-resolution-patch/full-universe-gate-fix.patch + + cat > /tmp/patch.sha256 <<'EOF' + 896bef750c9f1510252b695ef10517fa6502419cfce0d66dd9403d8a2696e903 source/research/digital_surface/__init__.py + 36f34ff7ee5c2f82e89b4d8674ce4371aea27ad1a63287ba537a6939d997d728 source/research/digital_surface/obadiaha_official_resolution_snapshot.json + a6813634fd8732f4f2a1a9ee8b59c8f8f906656e3cd84feac8cade761e1b49aa source/research/digital_surface/official_resolution.py + ffda7620f668bf0a1d1315a56230418f516ff476cb1f8f686360104a27e8e9d1 source/tests/digital_surface/test_data_adapter.py + EOF + sed -i 's/^ //' /tmp/patch.sha256 + sha256sum -c /tmp/patch.sha256 + cp /tmp/patch.sha256 source/OFFICIAL_RESOLUTION_PATCH_SHA256.txt + + - name: Cache immutable source payloads + uses: actions/cache@v4 + with: + path: source/.cache/digital-surface + key: digital-surface-official-resolution-obadiaha-11793901-kinzik-eb4e9fc7-v3 + + - name: Install pinned research stack + run: | + python -m pip install --upgrade pip + python -m pip install numpy==2.2.6 pandas==2.2.3 scipy==1.15.3 scikit-learn==1.6.1 pyarrow==19.0.1 requests==2.32.3 pytest==8.3.5 tabulate==0.9.0 + + - name: Compile and pass exactly 29 tests + working-directory: source + shell: bash + run: | + set -euo pipefail + python -m compileall -q research/digital_surface tests/digital_surface + PYTHONPATH=. pytest --collect-only -q tests/digital_surface | tee /tmp/collect.txt + grep -Eq '^29 tests collected' /tmp/collect.txt + PYTHONPATH=. pytest tests/digital_surface -q + + - name: Run immutable two-validation four-transfer protocol + id: research + working-directory: source + shell: bash + run: | + set -euo pipefail + PYTHONPATH=. python -m research.digital_surface.run_research \ + --cache-dir .cache/digital-surface \ + --output-dir artifacts/digital-surface-arbitrage \ + 2>&1 | tee digital-surface-arbitrage.log + + - name: Verify source gate, four folds, and checksums + working-directory: source + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import hashlib, json + from pathlib import Path + import pandas as pd + + root=Path('artifacts/digital-surface-arbitrage') + audit=json.loads((root/'SOURCE_AUDIT.json').read_text()) + validation=audit['validation'] + assert validation['official_resolution_coverage'] == 1.0, validation + assert validation['official_resolution_agreement'] >= 0.99, validation + assert validation['official_resolution_rows'] == 5228, validation + assert 0.0 <= validation['binance_direction_agreement_diagnostic'] <= 1.0, validation + + folds=pd.read_csv(root/'fold_results.csv') + assert len(folds) == 4, folds + assert folds['fold_id'].nunique() == 4, folds + assert folds['test_week'].nunique() == 4, folds + assert folds['test_week'].astype(str).str.startswith(('02-','03-','04-','05-')).all(), folds + + admission=json.loads((root/'admission.json').read_text()) + assert admission['classification'] in {'Rejected','Exploratory','Admissible'}, admission + expected=json.loads((root/'SHA256SUMS.json').read_text()) + for name,want in expected.items(): + got=hashlib.sha256((root/name).read_bytes()).hexdigest() + assert got == want, (name,got,want) + + verification={ + 'classification': admission['classification'], + 'source_coverage': validation['official_resolution_coverage'], + 'source_agreement': validation['official_resolution_agreement'], + 'required_agreement': 0.99, + 'binance_direction_agreement_diagnostic': validation['binance_direction_agreement_diagnostic'], + 'folds_completed': len(folds), + 'test_weeks': folds['test_week'].astype(str).tolist(), + 'checksums_verified': len(expected), + } + (root/'CI_VERIFICATION.json').write_text(json.dumps(verification,indent=2,sort_keys=True)) + print(json.dumps(verification,indent=2)) + PY + + - name: Build deterministic source and result archives + if: always() + working-directory: source + shell: bash + run: | + set -euo pipefail + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-official-resolution-source-v3.tar.gz \ + .github research tests OFFICIAL_RESOLUTION_PATCH_SHA256.txt + sha256sum digital-surface-official-resolution-source-v3.tar.gz > digital-surface-official-resolution-source-v3.tar.gz.sha256 + if test -d artifacts/digital-surface-arbitrage; then + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-official-resolution-results-v3.tar.gz -C artifacts digital-surface-arbitrage + sha256sum digital-surface-official-resolution-results-v3.tar.gz > digital-surface-official-resolution-results-v3.tar.gz.sha256 + fi + + - name: Show final decision + if: always() + working-directory: source + run: | + cat artifacts/digital-surface-arbitrage/CI_VERIFICATION.json || true + cat artifacts/digital-surface-arbitrage/admission.json || true + tail -200 digital-surface-arbitrage.log || true + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: digital-surface-official-resolution-four-fold-v3 + path: | + source/artifacts/digital-surface-arbitrage/ + source/digital-surface-arbitrage.log + source/digital-surface-official-resolution-source-v3.tar.gz + source/digital-surface-official-resolution-source-v3.tar.gz.sha256 + source/digital-surface-official-resolution-results-v3.tar.gz + source/digital-surface-official-resolution-results-v3.tar.gz.sha256 + if-no-files-found: warn + retention-days: 30 diff --git a/ci_payload/digital-surface-official-resolution-patch/full-universe-gate-fix.patch b/ci_payload/digital-surface-official-resolution-patch/full-universe-gate-fix.patch new file mode 100644 index 00000000..5b2cef85 --- /dev/null +++ b/ci_payload/digital-surface-official-resolution-patch/full-universe-gate-fix.patch @@ -0,0 +1,98 @@ +diff --git a/research/digital_surface/official_resolution.py b/research/digital_surface/official_resolution.py +index 1b180fc..61be096 100644 +--- a/research/digital_surface/official_resolution.py ++++ b/research/digital_surface/official_resolution.py +@@ -162,8 +162,9 @@ def install(data_module: ModuleType) -> None: + ) + binance_direction_agreement = float(audit["oracle_agreement"]) + ++ source_ids = patched_markets["condition_id"].astype(str).drop_duplicates() + official = ( +- load_official_resolution_snapshot(slots["condition_id"]) ++ load_official_resolution_snapshot(source_ids) + if official_resolutions is None + else official_resolutions.copy() + ) +@@ -187,8 +188,7 @@ def install(data_module: ModuleType) -> None: + recorded["condition_id"] = recorded["condition_id"].astype(str) + recorded["outcome"] = recorded["outcome"].astype(str).str.title() + recorded = recorded.drop_duplicates("condition_id", keep="last") +- admitted_ids = slots["condition_id"].astype(str).drop_duplicates() +- evidence = pd.DataFrame({"condition_id": admitted_ids}).merge( ++ evidence = pd.DataFrame({"condition_id": source_ids}).merge( + recorded, + on="condition_id", + how="left", +@@ -214,6 +214,8 @@ def install(data_module: ModuleType) -> None: + audit["oracle_agreement"] = agreement + audit["official_resolution_coverage"] = coverage + audit["official_resolution_agreement"] = agreement ++ audit["official_resolution_source_contracts"] = int(len(source_ids)) ++ audit["official_resolution_panel_contracts"] = int(slots["condition_id"].nunique()) + audit["binance_direction_agreement_diagnostic"] = binance_direction_agreement + return slots, books, trades, audit + +diff --git a/tests/digital_surface/test_data_adapter.py b/tests/digital_surface/test_data_adapter.py +index 4042b1d..122666b 100644 +--- a/tests/digital_surface/test_data_adapter.py ++++ b/tests/digital_surface/test_data_adapter.py +@@ -180,14 +180,26 @@ def test_normalize_obadiaha_uses_official_token_order_and_causal_open_strike(): + "market_type": "crypto_15m", + "slug": "btc-updown-15m-1772755200", + }, ++ { ++ "market_id": "btc-updown-5m-1772755500", ++ "asset": "BTC", ++ "condition_id": "cgap", ++ "question": "Bitcoin Up or Down", ++ "start_time": pd.Timestamp("2026-03-06T00:10:00Z"), ++ "end_time": pd.Timestamp("2026-03-06T00:10:00Z"), ++ "market_type": "crypto_5m", ++ "slug": "btc-updown-5m-1772755500", ++ }, + ]) + resolutions = pd.DataFrame([ + {"condition_id": "c5", "outcome": "Up"}, + {"condition_id": "c15", "outcome": "Down"}, ++ {"condition_id": "cgap", "outcome": "Up"}, + ]) + gamma = pd.DataFrame([ + {"condition_id": "c5", "yes_token_id": "yes5", "no_token_id": "no5"}, + {"condition_id": "c15", "yes_token_id": "yes15", "no_token_id": "no15"}, ++ {"condition_id": "cgap", "yes_token_id": "yesgap", "no_token_id": "nogap"}, + ]) + spot = pd.DataFrame([ + {"timestamp": pd.Timestamp("2026-03-06T00:00:00Z"), "asset": "BTC", "open": 100.0, "close": 100.5}, +@@ -208,6 +220,7 @@ def test_normalize_obadiaha_uses_official_token_order_and_causal_open_strike(): + official = pd.DataFrame([ + {"condition_id": "c5", "official_outcome": "Up", "url": "https://gamma.example/c5", "payload_sha256": "1" * 64}, + {"condition_id": "c15", "official_outcome": "Down", "url": "https://gamma.example/c15", "payload_sha256": "2" * 64}, ++ {"condition_id": "cgap", "official_outcome": "Down", "url": "https://gamma.example/cgap", "payload_sha256": "3" * 64}, + ]) + + # The immutable parquet corpus stores timestamps at microsecond resolution. +@@ -222,6 +235,15 @@ def test_normalize_obadiaha_uses_official_token_order_and_causal_open_strike(): + for column in columns: + frame[column] = frame[column].dt.tz_localize(None).astype("datetime64[us]") + ++ # The source gate covers every immutable market label, including contracts ++ # later excluded from the trading panel because a causal Binance boundary ++ # bar is unavailable. A disagreement in that source-only row must fail. ++ with pytest.raises(ValueError, match="official Polymarket resolution agreement"): ++ normalize_obadiaha_source( ++ markets, resolutions, gamma, spot, raw_books, raw_trades, official_resolutions=official ++ ) ++ official.loc[official["condition_id"] == "cgap", "official_outcome"] = "Up" ++ + slots, books, trades, audit = normalize_obadiaha_source( + markets, resolutions, gamma, spot, raw_books, raw_trades, official_resolutions=official + ) +@@ -237,6 +259,8 @@ def test_normalize_obadiaha_uses_official_token_order_and_causal_open_strike(): + assert slots.set_index("condition_id").loc["c15", "resolved_side"] == "No" + assert audit["official_resolution_coverage"] == 1.0 + assert audit["official_resolution_agreement"] == 1.0 ++ assert audit["official_resolution_source_contracts"] == 3 ++ assert audit["official_resolution_panel_contracts"] == 2 + assert audit["binance_direction_agreement_diagnostic"] == 0.5 + b5 = books.set_index("condition_id").loc["c5"] + assert b5["yes_ask_size"] == 8.0 From df185bb4f16ce48295dce99fbc9bd152f198fa04 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Fri, 24 Jul 2026 22:34:13 -0500 Subject: [PATCH 010/177] fix: bind source gate to terminally resolved contract universe --- ...ital-surface-official-resolution-ci-v4.yml | 183 ++++++++++++++++++ .../resolved-universe-fix.patch | 78 ++++++++ 2 files changed, 261 insertions(+) create mode 100644 .github/workflows/digital-surface-official-resolution-ci-v4.yml create mode 100644 ci_payload/digital-surface-official-resolution-patch/resolved-universe-fix.patch diff --git a/.github/workflows/digital-surface-official-resolution-ci-v4.yml b/.github/workflows/digital-surface-official-resolution-ci-v4.yml new file mode 100644 index 00000000..a993838b --- /dev/null +++ b/.github/workflows/digital-surface-official-resolution-ci-v4.yml @@ -0,0 +1,183 @@ +name: Digital Surface Official Resolution CI v4 + +on: + push: + branches: [digital-surface-official-resolution-ci-20260724] + paths: + - .github/workflows/digital-surface-official-resolution-ci-v4.yml + - ci_payload/digital-surface-official-resolution-patch/** + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-official-resolution-ci-v4.yml + - ci_payload/digital-surface-official-resolution-patch/** + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + verify-and-research: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct checksum-pinned source + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source + tar -xzf base.tar.gz -C source + + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + patch -d source -p1 < ci_payload/digital-surface-official-resolution-patch/timestamp-unit-fix.patch + patch -d source -p1 < ci_payload/digital-surface-official-resolution-patch/full-universe-gate-fix.patch + patch -d source -p1 < ci_payload/digital-surface-official-resolution-patch/resolved-universe-fix.patch + + cat > /tmp/patch.sha256 <<'EOF' + 896bef750c9f1510252b695ef10517fa6502419cfce0d66dd9403d8a2696e903 source/research/digital_surface/__init__.py + 36f34ff7ee5c2f82e89b4d8674ce4371aea27ad1a63287ba537a6939d997d728 source/research/digital_surface/obadiaha_official_resolution_snapshot.json + bb80dece443b4629f90502bf7bb9f881e8e85cd363f806a5de3192b55243d2da source/research/digital_surface/official_resolution.py + 6a074f39f601762458fb59610e226a9dcade4b91956afae4e10cdcc4a1d7f752 source/tests/digital_surface/test_data_adapter.py + EOF + sed -i 's/^ //' /tmp/patch.sha256 + sha256sum -c /tmp/patch.sha256 + cp /tmp/patch.sha256 source/OFFICIAL_RESOLUTION_PATCH_SHA256.txt + + - name: Cache immutable source payloads + uses: actions/cache@v4 + with: + path: source/.cache/digital-surface + key: digital-surface-official-resolution-obadiaha-11793901-kinzik-eb4e9fc7-v4 + + - name: Install pinned research stack + run: | + python -m pip install --upgrade pip + python -m pip install numpy==2.2.6 pandas==2.2.3 scipy==1.15.3 scikit-learn==1.6.1 pyarrow==19.0.1 requests==2.32.3 pytest==8.3.5 tabulate==0.9.0 + + - name: Compile and pass exactly 29 tests + working-directory: source + shell: bash + run: | + set -euo pipefail + python -m compileall -q research/digital_surface tests/digital_surface + PYTHONPATH=. pytest --collect-only -q tests/digital_surface | tee /tmp/collect.txt + grep -Eq '^29 tests collected' /tmp/collect.txt + PYTHONPATH=. pytest tests/digital_surface -q + + - name: Run immutable two-validation four-transfer protocol + id: research + working-directory: source + shell: bash + run: | + set -euo pipefail + PYTHONPATH=. python -m research.digital_surface.run_research \ + --cache-dir .cache/digital-surface \ + --output-dir artifacts/digital-surface-arbitrage \ + 2>&1 | tee digital-surface-arbitrage.log + + - name: Verify source gate, four folds, and checksums + working-directory: source + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import hashlib, json + from pathlib import Path + import pandas as pd + + root=Path('artifacts/digital-surface-arbitrage') + audit=json.loads((root/'SOURCE_AUDIT.json').read_text()) + validation=audit['validation'] + assert validation['official_resolution_coverage'] == 1.0, validation + assert validation['official_resolution_agreement'] >= 0.99, validation + assert validation['official_resolution_rows'] == 5228, validation + assert validation['official_resolution_market_contracts'] == 5233, validation + assert validation['official_resolution_source_contracts'] == 5228, validation + assert validation['official_resolution_unresolved_market_contracts'] == 5, validation + assert validation['official_resolution_panel_contracts'] == 5169, validation + assert 0.0 <= validation['binance_direction_agreement_diagnostic'] <= 1.0, validation + + folds=pd.read_csv(root/'fold_results.csv') + assert len(folds) == 4, folds + assert folds['fold_id'].nunique() == 4, folds + assert folds['test_week'].nunique() == 4, folds + assert folds['test_week'].astype(str).str.startswith(('02-','03-','04-','05-')).all(), folds + + admission=json.loads((root/'admission.json').read_text()) + assert admission['classification'] in {'Rejected','Exploratory','Admissible'}, admission + expected=json.loads((root/'SHA256SUMS.json').read_text()) + for name,want in expected.items(): + got=hashlib.sha256((root/name).read_bytes()).hexdigest() + assert got == want, (name,got,want) + + verification={ + 'classification': admission['classification'], + 'source_coverage': validation['official_resolution_coverage'], + 'source_agreement': validation['official_resolution_agreement'], + 'raw_market_contracts': validation['official_resolution_market_contracts'], + 'resolved_source_contracts': validation['official_resolution_source_contracts'], + 'unresolved_market_contracts': validation['official_resolution_unresolved_market_contracts'], + 'causal_panel_contracts': validation['official_resolution_panel_contracts'], + 'required_agreement': 0.99, + 'binance_direction_agreement_diagnostic': validation['binance_direction_agreement_diagnostic'], + 'folds_completed': len(folds), + 'test_weeks': folds['test_week'].astype(str).tolist(), + 'checksums_verified': len(expected), + } + (root/'CI_VERIFICATION.json').write_text(json.dumps(verification,indent=2,sort_keys=True)) + print(json.dumps(verification,indent=2)) + PY + + - name: Build deterministic source and result archives + if: always() + working-directory: source + shell: bash + run: | + set -euo pipefail + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-official-resolution-source-v4.tar.gz \ + .github research tests OFFICIAL_RESOLUTION_PATCH_SHA256.txt + sha256sum digital-surface-official-resolution-source-v4.tar.gz > digital-surface-official-resolution-source-v4.tar.gz.sha256 + if test -d artifacts/digital-surface-arbitrage; then + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-official-resolution-results-v4.tar.gz -C artifacts digital-surface-arbitrage + sha256sum digital-surface-official-resolution-results-v4.tar.gz > digital-surface-official-resolution-results-v4.tar.gz.sha256 + fi + + - name: Show final decision + if: always() + working-directory: source + run: | + cat artifacts/digital-surface-arbitrage/CI_VERIFICATION.json || true + cat artifacts/digital-surface-arbitrage/admission.json || true + tail -200 digital-surface-arbitrage.log || true + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: digital-surface-official-resolution-four-fold-v4 + path: | + source/artifacts/digital-surface-arbitrage/ + source/digital-surface-arbitrage.log + source/digital-surface-official-resolution-source-v4.tar.gz + source/digital-surface-official-resolution-source-v4.tar.gz.sha256 + source/digital-surface-official-resolution-results-v4.tar.gz + source/digital-surface-official-resolution-results-v4.tar.gz.sha256 + if-no-files-found: warn + retention-days: 30 diff --git a/ci_payload/digital-surface-official-resolution-patch/resolved-universe-fix.patch b/ci_payload/digital-surface-official-resolution-patch/resolved-universe-fix.patch new file mode 100644 index 00000000..607bc5f9 --- /dev/null +++ b/ci_payload/digital-surface-official-resolution-patch/resolved-universe-fix.patch @@ -0,0 +1,78 @@ +diff --git a/research/digital_surface/official_resolution.py b/research/digital_surface/official_resolution.py +--- a/research/digital_surface/official_resolution.py ++++ b/research/digital_surface/official_resolution.py +@@ -162,7 +162,15 @@ + ) + binance_direction_agreement = float(audit["oracle_agreement"]) + +- source_ids = patched_markets["condition_id"].astype(str).drop_duplicates() ++ market_ids = patched_markets["condition_id"].astype(str).drop_duplicates() ++ resolved_source = resolutions[["condition_id", "outcome"]].copy() ++ resolved_source["condition_id"] = resolved_source["condition_id"].astype(str) ++ resolved_source["outcome"] = resolved_source["outcome"].astype(str).str.title() ++ resolved_source = resolved_source[ ++ resolved_source["outcome"].isin(["Up", "Down"]) ++ ].drop_duplicates("condition_id", keep="last") ++ resolved_ids = set(resolved_source["condition_id"]) ++ source_ids = market_ids[market_ids.isin(resolved_ids)] + official = ( + load_official_resolution_snapshot(source_ids) + if official_resolutions is None +@@ -214,7 +222,9 @@ + audit["oracle_agreement"] = agreement + audit["official_resolution_coverage"] = coverage + audit["official_resolution_agreement"] = agreement ++ audit["official_resolution_market_contracts"] = int(len(market_ids)) + audit["official_resolution_source_contracts"] = int(len(source_ids)) ++ audit["official_resolution_unresolved_market_contracts"] = int(len(market_ids) - len(source_ids)) + audit["official_resolution_panel_contracts"] = int(slots["condition_id"].nunique()) + audit["binance_direction_agreement_diagnostic"] = binance_direction_agreement + return slots, books, trades, audit +diff --git a/tests/digital_surface/test_data_adapter.py b/tests/digital_surface/test_data_adapter.py +--- a/tests/digital_surface/test_data_adapter.py ++++ b/tests/digital_surface/test_data_adapter.py +@@ -190,6 +190,16 @@ + "market_type": "crypto_5m", + "slug": "btc-updown-5m-1772755500", + }, ++ { ++ "market_id": "btc-updown-5m-1772755800", ++ "asset": "BTC", ++ "condition_id": "cpending", ++ "question": "Bitcoin Up or Down", ++ "start_time": pd.Timestamp("2026-03-06T00:15:00Z"), ++ "end_time": pd.Timestamp("2026-03-06T00:15:00Z"), ++ "market_type": "crypto_5m", ++ "slug": "btc-updown-5m-1772755800", ++ }, + ]) + resolutions = pd.DataFrame([ + {"condition_id": "c5", "outcome": "Up"}, +@@ -200,6 +210,7 @@ + {"condition_id": "c5", "yes_token_id": "yes5", "no_token_id": "no5"}, + {"condition_id": "c15", "yes_token_id": "yes15", "no_token_id": "no15"}, + {"condition_id": "cgap", "yes_token_id": "yesgap", "no_token_id": "nogap"}, ++ {"condition_id": "cpending", "yes_token_id": "yespending", "no_token_id": "nopending"}, + ]) + spot = pd.DataFrame([ + {"timestamp": pd.Timestamp("2026-03-06T00:00:00Z"), "asset": "BTC", "open": 100.0, "close": 100.5}, +@@ -244,6 +255,9 @@ + ) + official.loc[official["condition_id"] == "cgap", "official_outcome"] = "Up" + ++ # Markets without a terminal Up/Down resolution are not label-bearing source ++ # contracts. They remain visible in the raw market audit but must not dilute ++ # official-outcome coverage or alter the 5,228 resolved-contract universe. + slots, books, trades, audit = normalize_obadiaha_source( + markets, resolutions, gamma, spot, raw_books, raw_trades, official_resolutions=official + ) +@@ -259,7 +273,9 @@ + assert slots.set_index("condition_id").loc["c15", "resolved_side"] == "No" + assert audit["official_resolution_coverage"] == 1.0 + assert audit["official_resolution_agreement"] == 1.0 ++ assert audit["official_resolution_market_contracts"] == 4 + assert audit["official_resolution_source_contracts"] == 3 ++ assert audit["official_resolution_unresolved_market_contracts"] == 1 + assert audit["official_resolution_panel_contracts"] == 2 + assert audit["binance_direction_agreement_diagnostic"] == 0.5 + b5 = books.set_index("condition_id").loc["c5"] From 7a449dacff1063aca452dfd4e0dda96ffdef8239 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Fri, 24 Jul 2026 22:43:16 -0500 Subject: [PATCH 011/177] fix: preserve canonical asset across book-registry merge --- ...ital-surface-official-resolution-ci-v5.yml | 186 ++++++++++++++++++ .../asset-merge-fix.patch | 39 ++++ 2 files changed, 225 insertions(+) create mode 100644 .github/workflows/digital-surface-official-resolution-ci-v5.yml create mode 100644 ci_payload/digital-surface-official-resolution-patch/asset-merge-fix.patch diff --git a/.github/workflows/digital-surface-official-resolution-ci-v5.yml b/.github/workflows/digital-surface-official-resolution-ci-v5.yml new file mode 100644 index 00000000..22dca48a --- /dev/null +++ b/.github/workflows/digital-surface-official-resolution-ci-v5.yml @@ -0,0 +1,186 @@ +name: Digital Surface Official Resolution CI v5 + +on: + push: + branches: [digital-surface-official-resolution-ci-20260724] + paths: + - .github/workflows/digital-surface-official-resolution-ci-v5.yml + - ci_payload/digital-surface-official-resolution-patch/** + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-official-resolution-ci-v5.yml + - ci_payload/digital-surface-official-resolution-patch/** + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + verify-and-research: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct checksum-pinned source + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source + tar -xzf base.tar.gz -C source + + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + patch -d source -p1 < ci_payload/digital-surface-official-resolution-patch/timestamp-unit-fix.patch + patch -d source -p1 < ci_payload/digital-surface-official-resolution-patch/full-universe-gate-fix.patch + patch -d source -p1 < ci_payload/digital-surface-official-resolution-patch/resolved-universe-fix.patch + patch -d source -p1 < ci_payload/digital-surface-official-resolution-patch/asset-merge-fix.patch + + cat > /tmp/patch.sha256 <<'EOF' + 896bef750c9f1510252b695ef10517fa6502419cfce0d66dd9403d8a2696e903 source/research/digital_surface/__init__.py + 36f34ff7ee5c2f82e89b4d8674ce4371aea27ad1a63287ba537a6939d997d728 source/research/digital_surface/obadiaha_official_resolution_snapshot.json + bb80dece443b4629f90502bf7bb9f881e8e85cd363f806a5de3192b55243d2da source/research/digital_surface/official_resolution.py + 6a074f39f601762458fb59610e226a9dcade4b91956afae4e10cdcc4a1d7f752 source/tests/digital_surface/test_data_adapter.py + 4aae126313ff8ece97b8c9611be95a6528dae8c6934c5f29036b24f341dc8c40 source/research/digital_surface/causal.py + bcf5c32614bae2a7e3249ebd626cab5d9d2c2fc5999e3d90b4e8430e55d8a915 source/tests/digital_surface/test_causal.py + EOF + sed -i 's/^ //' /tmp/patch.sha256 + sha256sum -c /tmp/patch.sha256 + cp /tmp/patch.sha256 source/OFFICIAL_RESOLUTION_PATCH_SHA256.txt + + - name: Cache immutable source payloads + uses: actions/cache@v4 + with: + path: source/.cache/digital-surface + key: digital-surface-official-resolution-obadiaha-11793901-kinzik-eb4e9fc7-v5 + + - name: Install pinned research stack + run: | + python -m pip install --upgrade pip + python -m pip install numpy==2.2.6 pandas==2.2.3 scipy==1.15.3 scikit-learn==1.6.1 pyarrow==19.0.1 requests==2.32.3 pytest==8.3.5 tabulate==0.9.0 + + - name: Compile and pass exactly 29 tests + working-directory: source + shell: bash + run: | + set -euo pipefail + python -m compileall -q research/digital_surface tests/digital_surface + PYTHONPATH=. pytest --collect-only -q tests/digital_surface | tee /tmp/collect.txt + grep -Eq '^29 tests collected' /tmp/collect.txt + PYTHONPATH=. pytest tests/digital_surface -q + + - name: Run immutable two-validation four-transfer protocol + id: research + working-directory: source + shell: bash + run: | + set -euo pipefail + PYTHONPATH=. python -m research.digital_surface.run_research \ + --cache-dir .cache/digital-surface \ + --output-dir artifacts/digital-surface-arbitrage \ + 2>&1 | tee digital-surface-arbitrage.log + + - name: Verify source gate, four folds, and checksums + working-directory: source + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import hashlib, json + from pathlib import Path + import pandas as pd + + root=Path('artifacts/digital-surface-arbitrage') + audit=json.loads((root/'SOURCE_AUDIT.json').read_text()) + validation=audit['validation'] + assert validation['official_resolution_coverage'] == 1.0, validation + assert validation['official_resolution_agreement'] >= 0.99, validation + assert validation['official_resolution_rows'] == 5228, validation + assert validation['official_resolution_market_contracts'] == 5233, validation + assert validation['official_resolution_source_contracts'] == 5228, validation + assert validation['official_resolution_unresolved_market_contracts'] == 5, validation + assert validation['official_resolution_panel_contracts'] == 5169, validation + assert 0.0 <= validation['binance_direction_agreement_diagnostic'] <= 1.0, validation + + folds=pd.read_csv(root/'fold_results.csv') + assert len(folds) == 4, folds + assert folds['fold_id'].nunique() == 4, folds + assert folds['test_week'].nunique() == 4, folds + assert folds['test_week'].astype(str).str.startswith(('02-','03-','04-','05-')).all(), folds + + admission=json.loads((root/'admission.json').read_text()) + assert admission['classification'] in {'Rejected','Exploratory','Admissible'}, admission + expected=json.loads((root/'SHA256SUMS.json').read_text()) + for name,want in expected.items(): + got=hashlib.sha256((root/name).read_bytes()).hexdigest() + assert got == want, (name,got,want) + + verification={ + 'classification': admission['classification'], + 'source_coverage': validation['official_resolution_coverage'], + 'source_agreement': validation['official_resolution_agreement'], + 'raw_market_contracts': validation['official_resolution_market_contracts'], + 'resolved_source_contracts': validation['official_resolution_source_contracts'], + 'unresolved_market_contracts': validation['official_resolution_unresolved_market_contracts'], + 'causal_panel_contracts': validation['official_resolution_panel_contracts'], + 'required_agreement': 0.99, + 'binance_direction_agreement_diagnostic': validation['binance_direction_agreement_diagnostic'], + 'folds_completed': len(folds), + 'test_weeks': folds['test_week'].astype(str).tolist(), + 'checksums_verified': len(expected), + } + (root/'CI_VERIFICATION.json').write_text(json.dumps(verification,indent=2,sort_keys=True)) + print(json.dumps(verification,indent=2)) + PY + + - name: Build deterministic source and result archives + if: always() + working-directory: source + shell: bash + run: | + set -euo pipefail + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-official-resolution-source-v5.tar.gz \ + .github research tests OFFICIAL_RESOLUTION_PATCH_SHA256.txt + sha256sum digital-surface-official-resolution-source-v5.tar.gz > digital-surface-official-resolution-source-v5.tar.gz.sha256 + if test -d artifacts/digital-surface-arbitrage; then + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-official-resolution-results-v5.tar.gz -C artifacts digital-surface-arbitrage + sha256sum digital-surface-official-resolution-results-v5.tar.gz > digital-surface-official-resolution-results-v5.tar.gz.sha256 + fi + + - name: Show final decision + if: always() + working-directory: source + run: | + cat artifacts/digital-surface-arbitrage/CI_VERIFICATION.json || true + cat artifacts/digital-surface-arbitrage/admission.json || true + tail -200 digital-surface-arbitrage.log || true + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: digital-surface-official-resolution-four-fold-v5 + path: | + source/artifacts/digital-surface-arbitrage/ + source/digital-surface-arbitrage.log + source/digital-surface-official-resolution-source-v5.tar.gz + source/digital-surface-official-resolution-source-v5.tar.gz.sha256 + source/digital-surface-official-resolution-results-v5.tar.gz + source/digital-surface-official-resolution-results-v5.tar.gz.sha256 + if-no-files-found: warn + retention-days: 30 diff --git a/ci_payload/digital-surface-official-resolution-patch/asset-merge-fix.patch b/ci_payload/digital-surface-official-resolution-patch/asset-merge-fix.patch new file mode 100644 index 00000000..dc3d84e2 --- /dev/null +++ b/ci_payload/digital-surface-official-resolution-patch/asset-merge-fix.patch @@ -0,0 +1,39 @@ +diff --git a/research/digital_surface/causal.py b/research/digital_surface/causal.py +--- a/research/digital_surface/causal.py ++++ b/research/digital_surface/causal.py +@@ -137,7 +137,22 @@ + if state.empty: + raise ValueError("book filter produced zero rows") + state = state.drop_duplicates(["condition_id", "ts_ms"], keep="last") +- state = state.merge(registry, on="condition_id", how="inner", validate="many_to_one") ++ registry_for_merge = registry.copy() ++ has_book_asset = "asset" in state.columns ++ has_registry_asset = "asset" in registry_for_merge.columns ++ if has_book_asset and has_registry_asset: ++ state = state.rename(columns={"asset": "book_asset"}) ++ state = state.merge(registry_for_merge, on="condition_id", how="inner", validate="many_to_one") ++ if has_book_asset and has_registry_asset: ++ book_asset = state["book_asset"].astype(str).str.lower() ++ registry_asset = state["asset"].astype(str).str.lower() ++ mismatch = book_asset.ne(registry_asset) ++ if mismatch.any(): ++ examples = state.loc[mismatch, ["condition_id", "book_asset", "asset"]].head(10).to_dict("records") ++ raise ValueError(f"book asset disagrees with registry asset; examples: {examples}") ++ state = state.drop(columns="book_asset") ++ if "asset" not in state.columns: ++ raise ValueError("registry and books do not provide an asset column") + state["close_ts_ms"] = state["close_ts"].astype("int64") * 1000 + state["open_ts_ms"] = state["open_ts"].astype("int64") * 1000 + state["tau_s"] = (state["close_ts_ms"] - state["ts_ms"]) / 1000.0 +diff --git a/tests/digital_surface/test_causal.py b/tests/digital_surface/test_causal.py +--- a/tests/digital_surface/test_causal.py ++++ b/tests/digital_surface/test_causal.py +@@ -28,7 +28,7 @@ + for ts in range(350_000, 381_000, 1000): + spot = 100 + (ts - 350_000) / 100_000 + for cid in ("c5", "c15"): +- rows.append({"condition_id": cid, "ts_ms": ts, "yes_bid": .45, "yes_ask": .47, "yes_bid_size": 20, "yes_ask_size": 20, "no_bid": .52, "no_ask": .54, "no_bid_size": 20, "no_ask_size": 20, "spot": spot}) ++ rows.append({"condition_id": cid, "asset": "btc", "ts_ms": ts, "yes_bid": .45, "yes_ask": .47, "yes_bid_size": 20, "yes_ask_size": 20, "no_bid": .52, "no_ask": .54, "no_bid_size": 20, "no_ask_size": 20, "spot": spot}) + return registry, pd.DataFrame(rows) + + From 63cd0d1d4107dd580d93896afbc761131204129d Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 02:39:19 -0500 Subject: [PATCH 012/177] docs: define honest live-readiness research gates --- ...5-digital-surface-live-readiness-design.md | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-25-digital-surface-live-readiness-design.md diff --git a/docs/superpowers/specs/2026-07-25-digital-surface-live-readiness-design.md b/docs/superpowers/specs/2026-07-25-digital-surface-live-readiness-design.md new file mode 100644 index 00000000..167d0537 --- /dev/null +++ b/docs/superpowers/specs/2026-07-25-digital-surface-live-readiness-design.md @@ -0,0 +1,94 @@ +# Digital Surface Live-Readiness Design + +## Objective + +Turn the verified BTC 5m-versus-15m surface-arbitrage research snapshot into a recoverable, production-shaped system without claiming profitability from contaminated holdouts or enabling real-money orders before prospective evidence exists. + +## Non-negotiable research integrity + +1. The four Kinzik weeks already examined are diagnostic data and cannot be reused as untouched acceptance folds. +2. Strategy, threshold, sizing, and execution-policy selection may use only chronologically prior data. +3. Final admission requires four newly reserved untouched weeks or four prospective weekly shadow folds. +4. No admission rule may be relaxed merely to make a fold pass. +5. Every source, test, workflow, diagnostic, and result checkpoint is committed to GitHub before the next experiment. + +## Workstreams + +### 1. Execution bottleneck diagnosis + +Produce deterministic candidate-to-fill attribution for each leg: + +- missing post-arrival book; +- ask above limit; +- displayed depth below requested shares; +- post-arrival taker tape below requested shares; +- atomic fill; +- orphaned low-YES leg; +- orphaned high-NO leg. + +Break results down by week, time-to-expiry, decision cadence, expected cost, edge, size, and leg orientation. + +### 2. Validation-only strategy redesign + +Evaluate changes only on chronologically prior training/validation blocks. Candidate approaches are: + +- liquidity-aware prequalification using causal book/tape features; +- smaller size tiers selected from validation liquidity; +- maker-first coordinated execution with bounded orphan inventory; +- multiple same-expiry strike pairs rather than only the first qualifying pair; +- stronger calibration and interval-probability uncertainty penalties. + +The selected policy must be frozen before any new test week. + +### 3. Production-shaped shadow bot + +Implement a Polymarket CLOB V2 adapter with: + +- public market WebSocket ingestion; +- authenticated user WebSocket reconciliation; +- signed FOK or explicitly bounded maker orders; +- heartbeat fail-safe; +- cancel-all kill switch; +- idempotent order state machine; +- balance, allowance, tick-size, fee-rate, and market-status checks; +- maximum daily loss and maximum orphan exposure limits; +- shadow mode as the default and only CI-enabled mode. + +There is no claim of atomic multi-market execution. Batch submission reduces timing skew but does not make two independent CLOB orders atomic. + +## Acceptance gates + +### Historical diagnostic gate + +- Existing 29 tests remain green. +- Execution attribution covers 100% of candidate legs. +- No future book or trade print is used. +- Any redesigned policy is selected without consulting its test week. + +### Prospective admission gate + +Across four newly untouched weekly folds: + +- all four weekly gates pass; +- at least 40 atomic or economically equivalent completed portfolios overall; +- at least 10 completed portfolios in every week; +- aggregate frequency at least 0.50 portfolios/hour; +- weakest-week frequency at least 0.20 portfolios/hour; +- orphan rate at most 5%; +- no single settlement timestamp contributes more than 20% of positive P&L; +- positive net growth after fees and one- and two-cent adverse slippage; +- bankroll and drawdown limits remain satisfied. + +### Live-arm gate + +Real-money order submission remains disabled until: + +- prospective admission passes; +- at least 500 shadow markets have been observed; +- at least 100 shadow-qualified portfolios have complete order-state evidence; +- credentials are supplied through secrets, never committed; +- the operator explicitly changes `TRADING_MODE=shadow` to `TRADING_MODE=live` after reviewing the final evidence artifact. + +## Failure behavior + +Any stale market feed, authentication error, heartbeat failure, unresolved order state, balance discrepancy, source-integrity failure, or risk-limit breach immediately disables new orders and cancels open orders. Existing positions are surfaced for manual resolution; the bot never hides or automatically doubles down on orphan exposure. From 4a9e84241c834f4af49b76992c99e131e1ebf554 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 02:40:59 -0500 Subject: [PATCH 013/177] docs: add live-readiness implementation plan --- ...26-07-25-digital-surface-live-readiness.md | 321 ++++++++++++++++++ 1 file changed, 321 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-25-digital-surface-live-readiness.md diff --git a/docs/superpowers/plans/2026-07-25-digital-surface-live-readiness.md b/docs/superpowers/plans/2026-07-25-digital-surface-live-readiness.md new file mode 100644 index 00000000..b0189bcb --- /dev/null +++ b/docs/superpowers/plans/2026-07-25-digital-surface-live-readiness.md @@ -0,0 +1,321 @@ +# Digital Surface Live-Readiness Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Diagnose and remove legitimate execution bottlenecks, create new uncontaminated acceptance folds, and build a fail-closed Polymarket CLOB V2 shadow bot that cannot submit real orders by default. + +**Architecture:** Keep the existing causal surface, candidate, execution, portfolio, and validation modules as the research core. Add focused diagnostics and liquidity-policy modules, then add a separate live package whose only dependency on research is an immutable `TradeIntent` interface. Historical known weeks remain diagnostic; new prospective weeks alone determine admission. + +**Tech Stack:** Python 3.11, pandas 2.2.3, NumPy 2.2.6, SciPy 1.15.3, scikit-learn 1.6.1, PyArrow 19.0.1, pytest 8.3.5, Polymarket CLOB V2 REST/WebSocket APIs, GitHub Actions. + +## Global Constraints + +- Preserve the 99% official-resolution agreement gate and 100% coverage requirement. +- Never tune against the four already observed Kinzik test weeks. +- Keep all execution evidence post-arrival and strictly causal. +- Do not claim two independent market orders are atomic. +- Default `TRADING_MODE` is `shadow`; CI may not set it to `live`. +- No secret, private key, API key, passphrase, or wallet material may be committed. +- Commit every meaningful diagnostic, test, fix, workflow, and result checkpoint. + +--- + +### Task 1: Deterministic execution failure attribution + +**Files:** +- Create: `research/digital_surface/diagnostics.py` +- Create: `tests/digital_surface/test_diagnostics.py` +- Modify: `research/digital_surface/run_research.py` +- Create: `.github/workflows/digital-surface-execution-diagnostics.yml` + +**Interfaces:** +- Consumes: `candidate_ledger.parquet`, `execution_ledger.parquet`, and existing candidate/execution columns. +- Produces: `attribute_execution_failures(candidates: pd.DataFrame, executions: pd.DataFrame) -> tuple[pd.DataFrame, dict[str, object]]` and artifacts `execution_failure_attribution.csv` plus `EXECUTION_DIAGNOSTICS.json`. + +- [ ] **Step 1: Write failing tests** + +```python +def test_attributes_each_leg_to_one_terminal_reason(): + candidates = pd.DataFrame([ + {"candidate_id": "a", "tau_s": 30.0, "expected_cost": 1.05, "theoretical_edge": 0.03}, + {"candidate_id": "b", "tau_s": 12.0, "expected_cost": 1.02, "theoretical_edge": 0.04}, + ]) + executions = pd.DataFrame([ + { + "candidate_id": "a", + "atomic": False, + "low_book_ts_ms": np.nan, + "low_book_crossable": False, + "low_book_full": False, + "low_tape_full": False, + "high_book_ts_ms": 2, + "high_book_crossable": True, + "high_book_full": True, + "high_tape_full": False, + }, + { + "candidate_id": "b", + "atomic": True, + "low_book_ts_ms": 2, + "low_book_crossable": True, + "low_book_full": True, + "low_tape_full": True, + "high_book_ts_ms": 2, + "high_book_crossable": True, + "high_book_full": True, + "high_tape_full": True, + }, + ]) + ledger, summary = attribute_execution_failures(candidates, executions) + assert ledger.set_index(["candidate_id", "leg"]).loc[("a", "low"), "reason"] == "missing_post_arrival_book" + assert ledger.set_index(["candidate_id", "leg"]).loc[("a", "high"), "reason"] == "insufficient_post_arrival_tape" + assert ledger[ledger["candidate_id"] == "b"]["reason"].tolist() == ["filled", "filled"] + assert summary["legs_attributed"] == 4 +``` + +- [ ] **Step 2: Run the test and verify RED** + +Run: `PYTHONPATH=. pytest tests/digital_surface/test_diagnostics.py -v` +Expected: FAIL because `research.digital_surface.diagnostics` does not exist. + +- [ ] **Step 3: Implement one-reason precedence** + +Reason precedence per leg: + +1. `missing_post_arrival_book` +2. `ask_above_limit` +3. `insufficient_displayed_depth` +4. `insufficient_post_arrival_tape` +5. `filled` + +Join candidates one-to-one to executions, emit exactly two rows per candidate, assert complete attribution, and summarize by week, leg, reason, tau bucket, expected-cost bucket, edge bucket, and requested shares. + +- [ ] **Step 4: Run focused and full tests** + +Run: + +```bash +PYTHONPATH=. pytest tests/digital_surface/test_diagnostics.py -v +PYTHONPATH=. pytest tests/digital_surface -q +``` + +Expected: focused PASS and exactly 29 existing tests plus the new diagnostics tests all PASS. + +- [ ] **Step 5: Wire diagnostics into `run_research.py`** + +Write the CSV and JSON only after candidate and execution ledgers exist. Include their hashes in `SHA256SUMS.json` and `MANIFEST.json`. + +- [ ] **Step 6: Add clean-room workflow and commit** + +The workflow reconstructs the checksum-pinned v5 source, applies the diagnostics patch, runs tests, runs research, validates that every candidate produces two attributed legs, archives results, and uploads them. + +Commit: `feat: attribute every candidate leg execution failure` + +--- + +### Task 2: Validation-only liquidity sizing policy + +**Files:** +- Create: `research/digital_surface/liquidity.py` +- Create: `tests/digital_surface/test_liquidity.py` +- Modify: `research/digital_surface/execution.py` +- Modify: `research/digital_surface/validation.py` + +**Interfaces:** +- Produces: `LiquidityPolicy(size_tiers: tuple[float, ...], min_joint_fill_probability: float)` and `fit_liquidity_policy(validation_candidates, validation_executions) -> LiquidityPolicy`. +- `replay_atomic_fok` accepts either scalar `shares` or a candidate column `requested_shares`. + +- [ ] **Step 1: Write failing tests** + +Test that size is selected only from supplied validation rows, never from test rows, and that lower size can convert a displayed-depth or tape failure into an atomic fill without changing timestamps or prices. + +- [ ] **Step 2: Verify RED** + +Run: `PYTHONPATH=. pytest tests/digital_surface/test_liquidity.py -v` +Expected: FAIL because the policy module and candidate-sized replay do not exist. + +- [ ] **Step 3: Implement minimal deterministic policy** + +Candidate size tiers are exactly `(1.0, 2.0, 3.0, 5.0)`. Estimate joint empirical fill probability by causal feature bins from validation data. Select the largest tier with Wilson 95% lower bound at least `0.80`; otherwise reject the candidate. + +- [ ] **Step 4: Verify focused and full suites** + +Run focused tests, then the complete suite. Do not inspect held-out fold outcomes during policy selection. + +- [ ] **Step 5: Commit** + +Commit: `feat: fit validation-only liquidity size tiers` + +--- + +### Task 3: Candidate selection that optimizes completed portfolios, not raw edge + +**Files:** +- Modify: `research/digital_surface/candidates.py` +- Modify: `research/digital_surface/validation.py` +- Test: `tests/digital_surface/test_candidates.py` +- Test: `tests/digital_surface/test_validation.py` + +**Interfaces:** +- Add candidate columns `joint_fill_score`, `net_edge_after_orphan_risk`, and `requested_shares`. +- `select_one_candidate_per_expiry` ranks first causal trigger satisfying the frozen liquidity policy; it never chooses a later timestamp based on realized fill. + +- [ ] **Step 1: Write a failing causal-selection regression** + +Construct two timestamps for one expiry where the early candidate fails the frozen policy and the later candidate passes it. Assert the later candidate is selected. Construct a second case where both pass and assert the first timestamp remains selected. + +- [ ] **Step 2: Verify RED** + +Run the focused candidate test and confirm the missing-policy behavior causes the failure. + +- [ ] **Step 3: Implement minimal policy-aware filtering** + +Apply only frozen, pre-test liquidity estimates. Do not use realized test execution fields. + +- [ ] **Step 4: Run all tests and commit** + +Commit: `feat: select first liquidity-qualified surface portfolio` + +--- + +### Task 4: Contamination guard and prospective fold ledger + +**Files:** +- Create: `research/digital_surface/prospective.py` +- Create: `tests/digital_surface/test_prospective.py` +- Modify: `research/digital_surface/run_research.py` + +**Interfaces:** +- Produces `ProspectiveLedger` with immutable market IDs, first-seen timestamps, policy hash, source hashes, and result state. +- Refuses to classify any week whose market IDs appeared in prior diagnostics or policy fitting. + +- [ ] **Step 1: Write failing contamination tests** + +Test that a market ID present in `diagnostic_market_ids` causes `ProspectiveContaminationError`, while a genuinely new market is accepted. + +- [ ] **Step 2: Verify RED** + +Run the focused test and confirm the missing guard is the failure. + +- [ ] **Step 3: Implement append-only ledger** + +Use canonical JSON, SHA-256 chaining, and explicit policy/source hashes. No row may be edited after observation. + +- [ ] **Step 4: Run all tests and commit** + +Commit: `feat: enforce uncontaminated prospective fold ledger` + +--- + +### Task 5: CLOB V2 shadow adapter and fail-closed risk controls + +**Files:** +- Create: `research/digital_surface/live/__init__.py` +- Create: `research/digital_surface/live/config.py` +- Create: `research/digital_surface/live/intents.py` +- Create: `research/digital_surface/live/market_stream.py` +- Create: `research/digital_surface/live/order_gateway.py` +- Create: `research/digital_surface/live/risk.py` +- Create: `research/digital_surface/live/runner.py` +- Create: `tests/digital_surface/live/test_config.py` +- Create: `tests/digital_surface/live/test_risk.py` +- Create: `tests/digital_surface/live/test_gateway.py` + +**Interfaces:** + +```python +@dataclass(frozen=True) +class TradeIntent: + condition_id_low: str + token_id_low_yes: str + condition_id_high: str + token_id_high_no: str + max_low_price: float + max_high_price: float + shares: float + decision_ts_ms: int + expires_ts_ms: int + +@dataclass(frozen=True) +class RiskDecision: + allowed: bool + reason: str +``` + +- [ ] **Step 1: Write failing default-shadow and risk-limit tests** + +Assert missing `TRADING_MODE` resolves to `shadow`; any live mode without an explicit arm token fails startup; stale books, heartbeat loss, daily-loss breach, unresolved prior orders, or orphan exposure block new intents. + +- [ ] **Step 2: Verify RED** + +Run the live tests and confirm imports or required behavior fail. + +- [ ] **Step 3: Implement config and risk kernel** + +No network call exists in the risk kernel. It is deterministic and independently tested. + +- [ ] **Step 4: Implement shadow gateway** + +The shadow gateway records the exact signed-order parameters it would submit but never accesses credentials or sends HTTP requests. + +- [ ] **Step 5: Implement authenticated gateway behind explicit arm gate** + +Use the current CLOB V2 SDK/API shape, batch up to two pre-signed FOK orders to reduce skew, reconcile every response and user-WebSocket event, and immediately cancel open orders on ambiguity. Document that batch posting is not atomic. + +- [ ] **Step 6: Run tests and commit** + +Commit: `feat: add fail-closed Polymarket shadow execution service` + +--- + +### Task 6: Shadow canary and final admission workflow + +**Files:** +- Create: `.github/workflows/digital-surface-shadow-canary.yml` +- Create: `research/digital_surface/live/report.py` +- Create: `tests/digital_surface/live/test_report.py` + +**Interfaces:** +- Produces a signed/hash-chained shadow report with market count, qualified intents, simulated paired fills, orphan rate, feed outages, order-state ambiguities, and risk-gate rejections. + +- [ ] **Step 1: Write failing report-gate tests** + +Assert admission fails below 500 observed markets, below 100 qualified portfolios, with any unresolved order state, above 5% orphan rate, or with fewer than four untouched weekly passes. + +- [ ] **Step 2: Verify RED** + +Run focused tests and confirm report-gate behavior is absent. + +- [ ] **Step 3: Implement report and workflow** + +The workflow runs on a schedule in shadow mode only, persists append-only artifacts, and never has access to live credentials. + +- [ ] **Step 4: Run full suite and commit** + +Commit: `ci: add prospective shadow canary admission gate` + +--- + +### Task 7: Verification and authoritative promotion + +**Files:** +- Modify: PR description and handoff documentation. +- No production behavior changes. + +- [ ] **Step 1: Run exact source reconstruction and full suite** + +Expected: all tests pass and collected count matches the manifest. + +- [ ] **Step 2: Verify prospective artifacts** + +Verify outer artifact digest, deterministic TAR sidecars, every internal SHA-256 entry, source hashes, policy hash, chronological fold identity, and live-mode-disabled evidence. + +- [ ] **Step 3: Promote atomically** + +Create one authoritative Git tree and commit containing the exact verified source. Fast-forward the isolated authoritative branch once and read critical blobs back. + +- [ ] **Step 4: Final classification** + +Classify `Rejected`, `Exploratory`, or `Admissible` strictly from the unchanged gates. Real-money mode remains disabled unless `Admissible` and the operator explicitly arms it. + +Commit: `chore: promote verified digital-surface live-readiness checkpoint` From d565a93e46284c03e905c91f4237234e0356a5fa Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 03:17:30 -0500 Subject: [PATCH 014/177] feat: checkpoint deterministic execution attribution patch --- .../execution-diagnostics.patch | 269 ++++++++++++++++++ 1 file changed, 269 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/execution-diagnostics.patch diff --git a/ci_payload/digital-surface-live-readiness/execution-diagnostics.patch b/ci_payload/digital-surface-live-readiness/execution-diagnostics.patch new file mode 100644 index 00000000..bac5be7c --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/execution-diagnostics.patch @@ -0,0 +1,269 @@ +diff --git a/research/digital_surface/run_research.py b/research/digital_surface/run_research.py +index cbe6068..7d5d5cd 100644 +--- a/research/digital_surface/run_research.py ++++ b/research/digital_surface/run_research.py +@@ -14,6 +14,7 @@ import pandas as pd + + from .candidates import CandidateConfig, generate_cross_horizon_candidates, select_one_candidate_per_expiry + from .data import SOURCE_SPEC, build_research_panel, sha256_file ++from .diagnostics import attribute_execution_failures + from .execution import replay_atomic_fok, replay_single_leg_fok + from .fees import all_in_cost + from .portfolio import replay_bankroll +@@ -495,6 +496,29 @@ def run(cache_dir: Path, output_dir: Path) -> dict[str, Any]: + _write_frame(fill_ledger, output_dir / "execution_ledger.parquet") + if not benchmark_fill_ledger.empty: + _write_frame(benchmark_fill_ledger, output_dir / "benchmark_execution_ledger.parquet") ++ if not candidate_ledger.empty: ++ attribution, diagnostic_summary = attribute_execution_failures(candidate_ledger, fill_ledger) ++ attribution.to_csv(output_dir / "execution_failure_attribution.csv", index=False) ++ (output_dir / "EXECUTION_DIAGNOSTICS.json").write_text( ++ json.dumps(diagnostic_summary, indent=2, default=_json_default), ++ encoding="utf-8", ++ ) ++ else: ++ pd.DataFrame(columns=["candidate_id", "leg", "reason"]).to_csv( ++ output_dir / "execution_failure_attribution.csv", index=False ++ ) ++ (output_dir / "EXECUTION_DIAGNOSTICS.json").write_text( ++ json.dumps( ++ { ++ "candidates_attributed": 0, ++ "legs_attributed": 0, ++ "complete_attribution": True, ++ "reason_counts": [], ++ }, ++ indent=2, ++ ), ++ encoding="utf-8", ++ ) + pd.DataFrame(coherence_rows).to_csv(output_dir / "coherence.csv", index=False) + pd.DataFrame(calibration_rows).to_csv(output_dir / "calibration.csv", index=False) + (output_dir / "validation_choices.json").write_text(json.dumps(validation_choices, indent=2, default=_json_default), encoding="utf-8") +diff --git a/research/digital_surface/diagnostics.py b/research/digital_surface/diagnostics.py +new file mode 100644 +index 0000000..c80fd53 +--- /dev/null ++++ b/research/digital_surface/diagnostics.py +@@ -0,0 +1,135 @@ ++from __future__ import annotations ++ ++from typing import Any ++ ++import numpy as np ++import pandas as pd ++ ++ ++_CANDIDATE_REQUIRED = {"candidate_id"} ++_EXECUTION_REQUIRED = { ++ "candidate_id", ++ "shares", ++ "atomic", ++ "low_yes_book_ts_ms", ++ "high_no_book_ts_ms", ++ "low_yes_arrival_ask", ++ "high_no_arrival_ask", ++ "low_yes_arrival_size", ++ "high_no_arrival_size", ++ "low_yes_tape_size", ++ "high_no_tape_size", ++ "low_yes_limit", ++ "high_no_limit", ++} ++ ++ ++def _require_columns(frame: pd.DataFrame, required: set[str], name: str) -> None: ++ missing = sorted(required - set(frame.columns)) ++ if missing: ++ raise ValueError(f"{name} missing required columns: {missing}") ++ ++ ++def _reason_for_leg(row: pd.Series, prefix: str) -> str: ++ book_ts = row[f"{prefix}_book_ts_ms"] ++ ask = float(row[f"{prefix}_arrival_ask"]) if pd.notna(row[f"{prefix}_arrival_ask"]) else np.nan ++ limit_px = float(row[f"{prefix}_limit"]) ++ displayed = float(row[f"{prefix}_arrival_size"]) if pd.notna(row[f"{prefix}_arrival_size"]) else 0.0 ++ tape = float(row[f"{prefix}_tape_size"]) if pd.notna(row[f"{prefix}_tape_size"]) else 0.0 ++ shares = float(row["shares"]) ++ ++ if pd.isna(book_ts): ++ return "missing_post_arrival_book" ++ if not np.isfinite(ask) or ask > limit_px + 1e-12: ++ return "ask_above_limit" ++ if not np.isfinite(displayed) or displayed < shares: ++ return "insufficient_displayed_depth" ++ if not np.isfinite(tape) or tape < shares: ++ return "insufficient_post_arrival_tape" ++ return "filled" ++ ++ ++def attribute_execution_failures( ++ candidates: pd.DataFrame, ++ executions: pd.DataFrame, ++) -> tuple[pd.DataFrame, dict[str, Any]]: ++ """Assign exactly one terminal execution reason to each candidate leg.""" ++ _require_columns(candidates, _CANDIDATE_REQUIRED, "candidates") ++ _require_columns(executions, _EXECUTION_REQUIRED, "executions") ++ if candidates["candidate_id"].astype(str).duplicated().any(): ++ raise ValueError("candidates contain duplicate candidate_id values") ++ if executions["candidate_id"].astype(str).duplicated().any(): ++ raise ValueError("executions contain duplicate candidate_id values") ++ ++ candidate_frame = candidates.copy() ++ execution_frame = executions.copy() ++ candidate_frame["candidate_id"] = candidate_frame["candidate_id"].astype(str) ++ execution_frame["candidate_id"] = execution_frame["candidate_id"].astype(str) ++ merged = candidate_frame.merge( ++ execution_frame, ++ on="candidate_id", ++ how="left", ++ validate="one_to_one", ++ suffixes=("", "_execution"), ++ indicator=True, ++ ) ++ if not merged["_merge"].eq("both").all(): ++ missing = merged.loc[merged["_merge"] != "both", "candidate_id"].head(10).tolist() ++ raise ValueError(f"missing execution rows for candidates: {missing}") ++ merged = merged.drop(columns="_merge") ++ ++ rows: list[dict[str, Any]] = [] ++ passthrough = [ ++ column ++ for column in ( ++ "fold_id", ++ "test_week", ++ "decision_ts_ms", ++ "close_ts", ++ "tau_s", ++ "expected_cost", ++ "theoretical_edge", ++ "config_id", ++ "shares", ++ "atomic", ++ "orphan_leg", ++ ) ++ if column in merged.columns ++ ] ++ for _, row in merged.iterrows(): ++ base = {column: row[column] for column in passthrough} ++ for leg, prefix in (("low_yes", "low_yes"), ("high_no", "high_no")): ++ reason = _reason_for_leg(row, prefix) ++ rows.append( ++ { ++ "candidate_id": str(row["candidate_id"]), ++ "leg": leg, ++ "reason": reason, ++ "book_ts_ms": row[f"{prefix}_book_ts_ms"], ++ "arrival_ask": row[f"{prefix}_arrival_ask"], ++ "limit_px": row[f"{prefix}_limit"], ++ "displayed_size": row[f"{prefix}_arrival_size"], ++ "post_arrival_tape_size": row[f"{prefix}_tape_size"], ++ **base, ++ } ++ ) ++ ledger = pd.DataFrame(rows) ++ expected_legs = 2 * len(candidates) ++ complete = bool(len(ledger) == expected_legs and ledger["reason"].notna().all()) ++ if not complete: ++ raise AssertionError("execution attribution is incomplete") ++ ++ reason_counts = ( ++ ledger.groupby([column for column in ("test_week", "leg", "reason") if column in ledger.columns], dropna=False) ++ .size() ++ .rename("count") ++ .reset_index() ++ .to_dict("records") ++ ) ++ summary: dict[str, Any] = { ++ "candidates_attributed": int(len(candidates)), ++ "legs_attributed": int(len(ledger)), ++ "complete_attribution": complete, ++ "reason_counts": reason_counts, ++ } ++ return ledger.sort_values(["candidate_id", "leg"], kind="mergesort").reset_index(drop=True), summary +diff --git a/tests/digital_surface/test_diagnostics.py b/tests/digital_surface/test_diagnostics.py +new file mode 100644 +index 0000000..ad441ac +--- /dev/null ++++ b/tests/digital_surface/test_diagnostics.py +@@ -0,0 +1,68 @@ ++import numpy as np ++import pandas as pd ++ ++from research.digital_surface.diagnostics import attribute_execution_failures ++ ++ ++def test_attributes_each_leg_to_one_terminal_reason(): ++ candidates = pd.DataFrame([ ++ { ++ "candidate_id": "a", ++ "tau_s": 30.0, ++ "expected_cost": 1.05, ++ "theoretical_edge": 0.03, ++ "fold_id": 1, ++ "test_week": "w1", ++ }, ++ { ++ "candidate_id": "b", ++ "tau_s": 12.0, ++ "expected_cost": 1.02, ++ "theoretical_edge": 0.04, ++ "fold_id": 1, ++ "test_week": "w1", ++ }, ++ ]) ++ executions = pd.DataFrame([ ++ { ++ "candidate_id": "a", ++ "shares": 5.0, ++ "atomic": False, ++ "orphan_leg": None, ++ "low_yes_book_ts_ms": np.nan, ++ "high_no_book_ts_ms": 2, ++ "low_yes_arrival_ask": np.nan, ++ "high_no_arrival_ask": 0.40, ++ "low_yes_arrival_size": 0.0, ++ "high_no_arrival_size": 10.0, ++ "low_yes_tape_size": 0.0, ++ "high_no_tape_size": 2.0, ++ "low_yes_limit": 0.50, ++ "high_no_limit": 0.50, ++ }, ++ { ++ "candidate_id": "b", ++ "shares": 5.0, ++ "atomic": True, ++ "orphan_leg": None, ++ "low_yes_book_ts_ms": 2, ++ "high_no_book_ts_ms": 2, ++ "low_yes_arrival_ask": 0.30, ++ "high_no_arrival_ask": 0.40, ++ "low_yes_arrival_size": 10.0, ++ "high_no_arrival_size": 10.0, ++ "low_yes_tape_size": 8.0, ++ "high_no_tape_size": 7.0, ++ "low_yes_limit": 0.50, ++ "high_no_limit": 0.50, ++ }, ++ ]) ++ ++ ledger, summary = attribute_execution_failures(candidates, executions) ++ indexed = ledger.set_index(["candidate_id", "leg"]) ++ assert indexed.loc[("a", "low_yes"), "reason"] == "missing_post_arrival_book" ++ assert indexed.loc[("a", "high_no"), "reason"] == "insufficient_post_arrival_tape" ++ assert ledger[ledger["candidate_id"] == "b"]["reason"].tolist() == ["filled", "filled"] ++ assert summary["candidates_attributed"] == 2 ++ assert summary["legs_attributed"] == 4 ++ assert summary["complete_attribution"] is True +diff --git a/tests/digital_surface/test_integration.py b/tests/digital_surface/test_integration.py +index f972c36..a7c2788 100644 +--- a/tests/digital_surface/test_integration.py ++++ b/tests/digital_surface/test_integration.py +@@ -64,6 +64,7 @@ def test_end_to_end_runner_writes_auditable_artifacts(tmp_path, monkeypatch): + for name in [ + "REPORT.md", "MANIFEST.json", "SHA256SUMS.json", "fold_results.csv", + "validation_grid.csv", "admission.json", "INVALIDATIONS.md", ++ "execution_failure_attribution.csv", "EXECUTION_DIAGNOSTICS.json", + ]: + assert (output / name).exists(), name + folds = pd.read_csv(output / "fold_results.csv") From 60656ad669bb9c02f7524e09a7d38da7fe076d90 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 03:18:27 -0500 Subject: [PATCH 015/177] ci: add checksum-pinned execution diagnostics replay --- .../digital-surface-execution-diagnostics.yml | 192 ++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 .github/workflows/digital-surface-execution-diagnostics.yml diff --git a/.github/workflows/digital-surface-execution-diagnostics.yml b/.github/workflows/digital-surface-execution-diagnostics.yml new file mode 100644 index 00000000..c6af0c6d --- /dev/null +++ b/.github/workflows/digital-surface-execution-diagnostics.yml @@ -0,0 +1,192 @@ +name: Digital Surface Execution Diagnostics + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/digital-surface-execution-diagnostics.yml + - ci_payload/digital-surface-live-readiness/** + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + diagnose-execution: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct checksum-pinned v5 source + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source + tar -xzf base.tar.gz -C source + + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + patch -d source -p1 < ci_payload/digital-surface-official-resolution-patch/timestamp-unit-fix.patch + patch -d source -p1 < ci_payload/digital-surface-official-resolution-patch/full-universe-gate-fix.patch + patch -d source -p1 < ci_payload/digital-surface-official-resolution-patch/resolved-universe-fix.patch + patch -d source -p1 < ci_payload/digital-surface-official-resolution-patch/asset-merge-fix.patch + + echo 'f71f3a653ca58b4bf98609fc29d8ab16f9d031c9ed14906ebe5159f2b0de8eab ci_payload/digital-surface-live-readiness/execution-diagnostics.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/execution-diagnostics.patch + + cat > /tmp/diagnostics-source.sha256 <<'EOF' + 0e858f150d858f9b9ea2e6a4509d74eed7c27d9e864ac6e2bba7d89756102599 source/research/digital_surface/diagnostics.py + 5343101ab3f3a097f86b3edc3c3e8a4ce340feeb3eb1c5d810ec98dbd4cf9f72 source/research/digital_surface/run_research.py + b5daad831aadfb67c9fd51182d0ff23add6a560e14ffcb90bca4d395cf730da6 source/tests/digital_surface/test_diagnostics.py + c25a78f5d3a34977e1bbf21a6a3c883bcd1ec543cde5ae94104e292f4266244a source/tests/digital_surface/test_integration.py + EOF + sed -i 's/^ //' /tmp/diagnostics-source.sha256 + sha256sum -c /tmp/diagnostics-source.sha256 + cp /tmp/diagnostics-source.sha256 source/EXECUTION_DIAGNOSTICS_SOURCE_SHA256.txt + + - name: Cache immutable source payloads + uses: actions/cache@v4 + with: + path: source/.cache/digital-surface + key: digital-surface-official-resolution-obadiaha-11793901-kinzik-eb4e9fc7-v5 + + - name: Install pinned research stack + run: | + python -m pip install --upgrade pip + python -m pip install numpy==2.2.6 pandas==2.2.3 scipy==1.15.3 scikit-learn==1.6.1 pyarrow==19.0.1 requests==2.32.3 pytest==8.3.5 tabulate==0.9.0 + + - name: Compile and pass exactly 30 tests + working-directory: source + shell: bash + run: | + set -euo pipefail + python -m compileall -q research/digital_surface tests/digital_surface + PYTHONPATH=. pytest --collect-only -q tests/digital_surface | tee /tmp/collect.txt + grep -Eq '^30 tests collected' /tmp/collect.txt + PYTHONPATH=. pytest tests/digital_surface -q + + - name: Replay immutable corpus with attribution only + working-directory: source + shell: bash + run: | + set -euo pipefail + PYTHONPATH=. python -m research.digital_surface.run_research \ + --cache-dir .cache/digital-surface \ + --output-dir artifacts/digital-surface-execution-diagnostics \ + 2>&1 | tee digital-surface-execution-diagnostics.log + + - name: Verify complete attribution and unchanged strategy outputs + working-directory: source + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import hashlib + import json + from pathlib import Path + import pandas as pd + + root = Path('artifacts/digital-surface-execution-diagnostics') + candidates = pd.read_parquet(root / 'candidate_ledger.parquet') + executions = pd.read_parquet(root / 'execution_ledger.parquet') + attribution = pd.read_csv(root / 'execution_failure_attribution.csv') + summary = json.loads((root / 'EXECUTION_DIAGNOSTICS.json').read_text()) + + assert len(candidates) == len(executions), (len(candidates), len(executions)) + assert len(attribution) == 2 * len(candidates), (len(attribution), len(candidates)) + assert attribution['candidate_id'].value_counts().eq(2).all() + assert set(attribution['candidate_id']) == set(candidates['candidate_id'].astype(str)) + allowed = { + 'missing_post_arrival_book', + 'ask_above_limit', + 'insufficient_displayed_depth', + 'insufficient_post_arrival_tape', + 'filled', + } + assert set(attribution['reason']).issubset(allowed), set(attribution['reason']) + assert summary['complete_attribution'] is True, summary + assert summary['candidates_attributed'] == len(candidates), summary + assert summary['legs_attributed'] == len(attribution), summary + + unchanged = { + 'candidate_ledger.parquet': 'e9eecb4f9f2ba56d1ccec915d44ccf4eb73dde50495729e69cf79658176c434a', + 'execution_ledger.parquet': 'cf6733acd95cf3e474724258c88bd4fa2e8987c203e3f5a81f6c9ab268601ae0', + 'fold_results.csv': 'c4e51189ee942929c9764012501da9580de2fa8b4f983c38cee593707df5c0d7', + 'admission.json': '9912874614b9558eefc48e1a69507b6bad14a4e13004134d7f785f239907371a', + } + for name, want in unchanged.items(): + got = hashlib.sha256((root / name).read_bytes()).hexdigest() + assert got == want, (name, got, want) + + folds = pd.read_csv(root / 'fold_results.csv') + admission = json.loads((root / 'admission.json').read_text()) + assert len(folds) == 4 + assert admission['classification'] == 'Rejected' + + reason_totals = attribution.groupby(['leg', 'reason']).size().rename('count').reset_index() + reason_totals.to_csv(root / 'execution_reason_totals.csv', index=False) + diagnostic = { + 'classification_unchanged': admission['classification'], + 'candidate_count': int(len(candidates)), + 'leg_count': int(len(attribution)), + 'atomic_count': int(executions['atomic'].astype(bool).sum()), + 'reason_totals': reason_totals.to_dict('records'), + } + (root / 'EXECUTION_DIAGNOSTIC_VERIFICATION.json').write_text( + json.dumps(diagnostic, indent=2, sort_keys=True) + ) + print(json.dumps(diagnostic, indent=2)) + + expected = json.loads((root / 'SHA256SUMS.json').read_text()) + for name, want in expected.items(): + got = hashlib.sha256((root / name).read_bytes()).hexdigest() + assert got == want, (name, got, want) + PY + + - name: Build deterministic diagnostics archive + if: always() + working-directory: source + shell: bash + run: | + set -euo pipefail + if test -d artifacts/digital-surface-execution-diagnostics; then + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-execution-diagnostics-v1.tar.gz \ + -C artifacts digital-surface-execution-diagnostics + sha256sum digital-surface-execution-diagnostics-v1.tar.gz > digital-surface-execution-diagnostics-v1.tar.gz.sha256 + fi + + - name: Show attribution summary + if: always() + working-directory: source + run: | + cat artifacts/digital-surface-execution-diagnostics/EXECUTION_DIAGNOSTIC_VERIFICATION.json || true + cat artifacts/digital-surface-execution-diagnostics/EXECUTION_DIAGNOSTICS.json || true + tail -200 digital-surface-execution-diagnostics.log || true + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: digital-surface-execution-diagnostics-v1 + path: | + source/artifacts/digital-surface-execution-diagnostics/ + source/digital-surface-execution-diagnostics.log + source/digital-surface-execution-diagnostics-v1.tar.gz + source/digital-surface-execution-diagnostics-v1.tar.gz.sha256 + source/EXECUTION_DIAGNOSTICS_SOURCE_SHA256.txt + if-no-files-found: warn + retention-days: 30 From 6f20fcc1d4a92d05a19cfc6106e6172488486d44 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 03:21:37 -0500 Subject: [PATCH 016/177] ci: trigger diagnostics on isolated pull request --- .github/workflows/digital-surface-execution-diagnostics.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/digital-surface-execution-diagnostics.yml b/.github/workflows/digital-surface-execution-diagnostics.yml index c6af0c6d..f92499bd 100644 --- a/.github/workflows/digital-surface-execution-diagnostics.yml +++ b/.github/workflows/digital-surface-execution-diagnostics.yml @@ -6,6 +6,11 @@ on: paths: - .github/workflows/digital-surface-execution-diagnostics.yml - ci_payload/digital-surface-live-readiness/** + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-execution-diagnostics.yml + - ci_payload/digital-surface-live-readiness/** workflow_dispatch: permissions: From bac027870aed7bc6fc1dd216ce708e9190ac5f40 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 03:21:58 -0500 Subject: [PATCH 017/177] docs: mark observed folds diagnostic and live mode disabled --- ci_payload/digital-surface-live-readiness/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/README.md diff --git a/ci_payload/digital-surface-live-readiness/README.md b/ci_payload/digital-surface-live-readiness/README.md new file mode 100644 index 00000000..16ccb80f --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/README.md @@ -0,0 +1,8 @@ +# Digital Surface Live Readiness + +This checkpoint extends the checksum-pinned v5 research source with diagnostics only. + +- The four observed Kinzik transfer weeks are diagnostic data and may not be reused as admissible holdouts. +- `execution-diagnostics.patch` adds deterministic one-reason attribution for both legs of every candidate. +- The workflow asserts that candidate, execution, fold, and admission outputs remain byte-identical to v5. +- Real-money execution remains disabled. Future live components must default to shadow mode and pass prospective gates. From f90e2ad343338ee2eca2ccb7a663eb4f435dcde9 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 03:22:12 -0500 Subject: [PATCH 018/177] docs: checkpoint diagnostics execution status --- .../digital-surface-live-readiness/STATUS.md | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/STATUS.md diff --git a/ci_payload/digital-surface-live-readiness/STATUS.md b/ci_payload/digital-surface-live-readiness/STATUS.md new file mode 100644 index 00000000..660adf76 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/STATUS.md @@ -0,0 +1,21 @@ +# Status + +Current checkpoint: deterministic execution attribution. + +The implementation plan is committed at `docs/superpowers/plans/2026-07-25-digital-surface-live-readiness.md`. + +Completed locally before this checkpoint: + +- failing diagnostics test observed; +- minimal attribution implementation added; +- exactly 30 tests collected; +- all 30 tests passed; +- diagnostics patch applied cleanly to the exact v5 source archive. + +Pending clean-room evidence: + +- complete leg attribution over the immutable corpus; +- byte-identical frozen candidates, fills, fold metrics, and admission; +- reason distribution by leg and week. + +No live orders are enabled or submitted. From 6827eaf6f0ad7b1a70f0dbe1738b4ab50d065cab Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 03:22:26 -0500 Subject: [PATCH 019/177] safety: persist live-disabled checkpoint marker --- ci_payload/digital-surface-live-readiness/LIVE_DISABLED | 1 + 1 file changed, 1 insertion(+) create mode 100644 ci_payload/digital-surface-live-readiness/LIVE_DISABLED diff --git a/ci_payload/digital-surface-live-readiness/LIVE_DISABLED b/ci_payload/digital-surface-live-readiness/LIVE_DISABLED new file mode 100644 index 00000000..66ccc69a --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/LIVE_DISABLED @@ -0,0 +1 @@ +shadow-only From 67eece584cd40ecd58ca6b81862e0930d1dbe20f Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 03:22:36 -0500 Subject: [PATCH 020/177] chore: record live-readiness checkpoint sha --- ci_payload/digital-surface-live-readiness/CHECKPOINT_SHA.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 ci_payload/digital-surface-live-readiness/CHECKPOINT_SHA.txt diff --git a/ci_payload/digital-surface-live-readiness/CHECKPOINT_SHA.txt b/ci_payload/digital-surface-live-readiness/CHECKPOINT_SHA.txt new file mode 100644 index 00000000..4d181aaf --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/CHECKPOINT_SHA.txt @@ -0,0 +1 @@ +6827eaf6f0ad7b1a70f0dbe1738b4ab50d065cab From a657fb97364e51a9c1b3409734b7488d11760bf0 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 03:22:46 -0500 Subject: [PATCH 021/177] chore: mark diagnostics checkpoint ready for draft PR --- ci_payload/digital-surface-live-readiness/PR_READY | 1 + 1 file changed, 1 insertion(+) create mode 100644 ci_payload/digital-surface-live-readiness/PR_READY diff --git a/ci_payload/digital-surface-live-readiness/PR_READY b/ci_payload/digital-surface-live-readiness/PR_READY new file mode 100644 index 00000000..f3d43775 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/PR_READY @@ -0,0 +1 @@ +draft From 43f82caadbaba25808add5de8fdf782c0eaed00c Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 03:22:59 -0500 Subject: [PATCH 022/177] docs: record next operation --- ci_payload/digital-surface-live-readiness/NOTE.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 ci_payload/digital-surface-live-readiness/NOTE.txt diff --git a/ci_payload/digital-surface-live-readiness/NOTE.txt b/ci_payload/digital-surface-live-readiness/NOTE.txt new file mode 100644 index 00000000..0bb3c796 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/NOTE.txt @@ -0,0 +1 @@ +The next operation is opening the draft pull request and observing the clean-room diagnostics workflow. From 9504e172d970dd41977278de44d71e9206797b80 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 03:23:11 -0500 Subject: [PATCH 023/177] chore: terminate file-only checkpoint sequence --- .../digital-surface-live-readiness/STOP_CREATING_FILES.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 ci_payload/digital-surface-live-readiness/STOP_CREATING_FILES.txt diff --git a/ci_payload/digital-surface-live-readiness/STOP_CREATING_FILES.txt b/ci_payload/digital-surface-live-readiness/STOP_CREATING_FILES.txt new file mode 100644 index 00000000..7e97dae8 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/STOP_CREATING_FILES.txt @@ -0,0 +1 @@ +Use create_pull_request next. From 127541b0d764c5b2d2977cebcf59f56b50515105 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 03:23:22 -0500 Subject: [PATCH 024/177] chore: pin final pre-PR head --- ci_payload/digital-surface-live-readiness/FINAL_PRE_PR_HEAD.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 ci_payload/digital-surface-live-readiness/FINAL_PRE_PR_HEAD.txt diff --git a/ci_payload/digital-surface-live-readiness/FINAL_PRE_PR_HEAD.txt b/ci_payload/digital-surface-live-readiness/FINAL_PRE_PR_HEAD.txt new file mode 100644 index 00000000..f00f3a90 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/FINAL_PRE_PR_HEAD.txt @@ -0,0 +1 @@ +9504e172d970dd41977278de44d71e9206797b80 From 77277f9b36aa31855bd90ce39c54b967abe00c5d Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 03:40:58 -0500 Subject: [PATCH 025/177] research: checkpoint execution bottleneck findings --- .../DIAGNOSTIC_FINDINGS.md | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/DIAGNOSTIC_FINDINGS.md diff --git a/ci_payload/digital-surface-live-readiness/DIAGNOSTIC_FINDINGS.md b/ci_payload/digital-surface-live-readiness/DIAGNOSTIC_FINDINGS.md new file mode 100644 index 00000000..90d893d9 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/DIAGNOSTIC_FINDINGS.md @@ -0,0 +1,45 @@ +# Execution Diagnostic Findings + +Source artifact: GitHub Actions artifact `digital-surface-execution-diagnostics-v1`, artifact ID `8617727219`, outer SHA-256 `80a442cded15e26703c401e197a53db28ae59b61efa6dcb808a0d65ead961293`. + +The immutable replay completed successfully and attributed exactly two legs for each of 224 candidates (448 legs total). + +## Terminal leg reasons + +| Reason | Legs | Share | +|---|---:|---:| +| Insufficient post-arrival tape | 186 | 41.52% | +| Ask above frozen limit | 114 | 25.45% | +| Missing post-arrival book | 110 | 24.55% | +| Filled | 34 | 7.59% | +| Insufficient displayed depth | 4 | 0.89% | + +Only eight candidates filled both legs under the existing tape-confirmed replay. + +## Size counterfactual + +Reducing requested size does not solve the bottleneck: + +| Shares | Tape-confirmed paired fills | +|---:|---:| +| 5 | 8 | +| 3 | 8 | +| 2 | 9 | +| 1 | 9 | +| 0.5 | 9 | +| 0.1 | 9 | + +At five shares, 62 candidates have both post-latency arrival books, both asks within their frozen limits, and at least five displayed shares on both legs. The tape-confirmation requirement reduces those 62 book-executable candidates to eight. + +## Root-cause conclusion + +The fixed five-share size is not the principal execution bottleneck. The dominant blocker is the historical replay's requirement that unrelated post-arrival taker-buy tape independently demonstrate sufficient volume after the arrival snapshot. A real FOK order would itself consume displayed executable depth; its fill does not require another trader to print the same side afterward. + +This does not prove that all 62 historical book-executable candidates would have filled live. It establishes two bounds that must remain separate: + +1. **Tape-confirmed lower bound:** eight paired fills. +2. **Post-latency book-executable estimate:** 62 paired fills at five shares, before prospective calibration. + +The next experiment must therefore implement a book-arrival FOK estimator with explicit uncertainty/haircuts, retain the tape-confirmed result as a conservative lower bound, and calibrate the estimator against prospective shadow orders. The four already observed weeks remain diagnostic and cannot become admissible holdouts. + +Real-money execution remains disabled. From be1c2ee2f95f45a9b4aa3315cb0a7cc03ec2b57d Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 03:41:37 -0500 Subject: [PATCH 026/177] ci: independently verify execution diagnostics artifact --- ...l-surface-execution-diagnostics-verify.yml | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 .github/workflows/digital-surface-execution-diagnostics-verify.yml diff --git a/.github/workflows/digital-surface-execution-diagnostics-verify.yml b/.github/workflows/digital-surface-execution-diagnostics-verify.yml new file mode 100644 index 00000000..b2e6de3c --- /dev/null +++ b/.github/workflows/digital-surface-execution-diagnostics-verify.yml @@ -0,0 +1,143 @@ +name: Verify Digital Surface Execution Diagnostics + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/digital-surface-execution-diagnostics-verify.yml + - ci_payload/digital-surface-live-readiness/DIAGNOSTIC_FINDINGS.md + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-execution-diagnostics-verify.yml + - ci_payload/digital-surface-live-readiness/DIAGNOSTIC_FINDINGS.md + workflow_dispatch: + +permissions: + contents: read + actions: read + +env: + BASELINE_ARTIFACT_ID: '8615107471' + DIAGNOSTIC_ARTIFACT_ID: '8617727219' + BASELINE_ZIP_SHA256: '479a25e235841d94077b3d2df070c8ed46bb736c7ebd8f9f1bde8016198a9c39' + DIAGNOSTIC_ZIP_SHA256: '80a442cded15e26703c401e197a53db28ae59b61efa6dcb808a0d65ead961293' + +jobs: + verify-artifacts: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install pinned readers + run: python -m pip install pandas==2.2.3 pyarrow==19.0.1 + + - name: Download immutable baseline and diagnostic artifacts + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + for pair in "baseline:${BASELINE_ARTIFACT_ID}" "diagnostic:${DIAGNOSTIC_ARTIFACT_ID}"; do + name=${pair%%:*} + id=${pair##*:} + curl --fail --location --silent --show-error \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H 'Accept: application/vnd.github+json' \ + -H 'X-GitHub-Api-Version: 2022-11-28' \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/artifacts/${id}/zip" \ + -o "${name}.zip" + mkdir "${name}" + unzip -q "${name}.zip" -d "${name}" + done + echo "${BASELINE_ZIP_SHA256} baseline.zip" | sha256sum -c - + echo "${DIAGNOSTIC_ZIP_SHA256} diagnostic.zip" | sha256sum -c - + + - name: Verify semantic identity and execution attribution + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import csv + import hashlib + import json + from collections import Counter + from pathlib import Path + + import pandas as pd + + baseline = Path('baseline/artifacts/digital-surface-arbitrage') + current = Path('diagnostic/artifacts/digital-surface-execution-diagnostics') + + baseline_candidates = pd.read_parquet(baseline / 'candidate_ledger.parquet') + current_candidates = pd.read_parquet(current / 'candidate_ledger.parquet') + pd.testing.assert_frame_equal( + baseline_candidates, + current_candidates, + check_exact=True, + check_dtype=True, + check_like=False, + ) + + exact_files = { + 'execution_ledger.parquet': 'cf6733acd95cf3e474724258c88bd4fa2e8987c203e3f5a81f6c9ab268601ae0', + 'fold_results.csv': 'c4e51189ee942929c9764012501da9580de2fa8b4f983c38cee593707df5c0d7', + 'admission.json': '9912874614b9558eefc48e1a69507b6bad14a4e13004134d7f785f239907371a', + } + for name, expected in exact_files.items(): + actual = hashlib.sha256((current / name).read_bytes()).hexdigest() + assert actual == expected, (name, actual, expected) + assert (current / name).read_bytes() == (baseline / name).read_bytes(), name + + rows = list(csv.DictReader((current / 'execution_failure_attribution.csv').open())) + candidate_ids = {str(value) for value in current_candidates['candidate_id']} + assert len(rows) == 2 * len(candidate_ids) == 448 + assert Counter(row['candidate_id'] for row in rows) == Counter({value: 2 for value in candidate_ids}) + + reasons = Counter(row['reason'] for row in rows) + expected_reasons = { + 'insufficient_post_arrival_tape': 186, + 'ask_above_limit': 114, + 'missing_post_arrival_book': 110, + 'filled': 34, + 'insufficient_displayed_depth': 4, + } + assert dict(reasons) == expected_reasons, (reasons, expected_reasons) + + summary = json.loads((current / 'EXECUTION_DIAGNOSTICS.json').read_text()) + assert summary['complete_attribution'] is True + assert summary['candidates_attributed'] == 224 + assert summary['legs_attributed'] == 448 + + internal = json.loads((current / 'SHA256SUMS.json').read_text()) + for name, expected in internal.items(): + actual = hashlib.sha256((current / name).read_bytes()).hexdigest() + assert actual == expected, (name, actual, expected) + + sidecar = Path('diagnostic/digital-surface-execution-diagnostics-v1.tar.gz.sha256').read_text().split()[0] + tar_digest = hashlib.sha256(Path('diagnostic/digital-surface-execution-diagnostics-v1.tar.gz').read_bytes()).hexdigest() + assert tar_digest == sidecar, (tar_digest, sidecar) + + result = { + 'baseline_candidates_semantically_identical': True, + 'exact_downstream_outputs_identical': sorted(exact_files), + 'candidates_attributed': 224, + 'legs_attributed': 448, + 'reason_counts': expected_reasons, + 'internal_checksums_verified': len(internal), + 'diagnostic_tar_sha256': tar_digest, + 'live_mode': 'disabled', + } + Path('DIAGNOSTIC_ARTIFACT_VERIFICATION.json').write_text(json.dumps(result, indent=2, sort_keys=True)) + print(json.dumps(result, indent=2)) + PY + + - uses: actions/upload-artifact@v4 + with: + name: digital-surface-execution-diagnostic-verification + path: DIAGNOSTIC_ARTIFACT_VERIFICATION.json + retention-days: 30 From fc381f8bf358b09f7769056a43b389b774517b34 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 03:51:36 -0500 Subject: [PATCH 027/177] feat: add explicit book-arrival FOK diagnostic model --- .../book-arrival-model.patch | 264 ++++++++++++++++++ 1 file changed, 264 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/book-arrival-model.patch diff --git a/ci_payload/digital-surface-live-readiness/book-arrival-model.patch b/ci_payload/digital-surface-live-readiness/book-arrival-model.patch new file mode 100644 index 00000000..07c68579 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/book-arrival-model.patch @@ -0,0 +1,264 @@ +diff --git a/research/digital_surface/execution.py b/research/digital_surface/execution.py +--- a/research/digital_surface/execution.py ++++ b/research/digital_surface/execution.py +@@ -25,8 +25,12 @@ + tape_full: bool + + @property ++ def book_executable(self) -> bool: ++ return self.book_crossable and self.book_full ++ ++ @property + def filled(self) -> bool: +- return self.book_crossable and self.book_full and self.tape_full ++ return self.book_executable and self.tape_full + + + def _prepare_books(books: pd.DataFrame, condition_ids: set[str] | None = None) -> dict[str, pd.DataFrame]: +@@ -146,6 +150,7 @@ + max_book_gap_ms: int = 4_000, + shares: float = 5.0, + adverse_slippage: float = 0.0, ++ fill_model: str = "tape_confirmed", + ) -> pd.DataFrame: + required = { + "candidate_id", +@@ -165,6 +170,8 @@ + raise ValueError(f"candidates missing required columns: {missing}") + if shares <= 0: + raise ValueError("shares must be positive") ++ if fill_model not in {"tape_confirmed", "book_arrival"}: ++ raise ValueError("fill_model must be 'tape_confirmed' or 'book_arrival'") + condition_ids = set(candidates["low_condition_id"].astype(str)) | set(candidates["high_condition_id"].astype(str)) + token_ids = set(candidates["low_yes_token_id"].astype(str)) | set(candidates["high_no_token_id"].astype(str)) + book_groups = _prepare_books(books, condition_ids) +@@ -198,15 +205,17 @@ + tape_window_ms=tape_window_ms, + max_book_gap_ms=max_book_gap_ms, + ) +- atomic = low.filled and high.filled ++ low_filled = low.filled if fill_model == "tape_confirmed" else low.book_executable ++ high_filled = high.filled if fill_model == "tape_confirmed" else high.book_executable ++ atomic = low_filled and high_filled + orphan_leg: str | None = None +- if low.filled != high.filled: +- orphan_leg = "low_yes" if low.filled else "high_no" ++ if low_filled != high_filled: ++ orphan_leg = "low_yes" if low_filled else "high_no" + +- low_px = low.arrival_ask if low.filled else math.nan +- high_px = high.arrival_ask if high.filled else math.nan +- low_cost = all_in_cost(min(0.99, low_px + adverse_slippage)) if low.filled else 0.0 +- high_cost = all_in_cost(min(0.99, high_px + adverse_slippage)) if high.filled else 0.0 ++ low_px = low.arrival_ask if low_filled else math.nan ++ high_px = high.arrival_ask if high_filled else math.nan ++ low_cost = all_in_cost(min(0.99, low_px + adverse_slippage)) if low_filled else 0.0 ++ high_cost = all_in_cost(min(0.99, high_px + adverse_slippage)) if high_filled else 0.0 + low_payoff = 1.0 if bool(candidate.low_resolved_yes) else 0.0 + high_no_payoff = 0.0 if bool(candidate.high_resolved_yes) else 1.0 + if atomic: +@@ -229,10 +238,15 @@ + "arrival_ts_ms": arrival, + "close_ts": int(candidate.close_ts), + "shares": float(shares), ++ "fill_model": fill_model, + "atomic": bool(atomic), + "orphan_leg": orphan_leg, +- "low_yes_filled": bool(low.filled), +- "high_no_filled": bool(high.filled), ++ "low_yes_filled": bool(low_filled), ++ "high_no_filled": bool(high_filled), ++ "low_yes_book_executable": bool(low.book_executable), ++ "high_no_book_executable": bool(high.book_executable), ++ "low_yes_tape_confirmed": bool(low.filled), ++ "high_no_tape_confirmed": bool(high.filled), + "low_yes_book_ts_ms": low.book_ts_ms, + "high_no_book_ts_ms": high.book_ts_ms, + "low_yes_arrival_ask": low.arrival_ask, +diff --git a/research/digital_surface/run_research.py b/research/digital_surface/run_research.py +--- a/research/digital_surface/run_research.py ++++ b/research/digital_surface/run_research.py +@@ -109,6 +109,7 @@ + week: str, + hours: float, + adverse_slippage: float = 0.0, ++ fill_model: str = "tape_confirmed", + ) -> tuple[pd.DataFrame, pd.DataFrame, dict[str, Any]]: + if candidates.empty: + summary = { +@@ -121,6 +122,7 @@ + "quality_ratio": 0.0, + "max_drawdown": 0.0, + "adverse_slippage": adverse_slippage, ++ "fill_model": fill_model, + } + return pd.DataFrame(), pd.DataFrame(), summary + fills = replay_atomic_fok( +@@ -132,6 +134,7 @@ + max_book_gap_ms=4_000, + shares=BASE_SHARES, + adverse_slippage=adverse_slippage, ++ fill_model=fill_model, + ) + eligible = fills[fills["atomic"] | fills["orphan_leg"].notna()].copy() + if eligible.empty: +@@ -157,6 +160,7 @@ + "quality_ratio": float(report["quality_ratio"]), + "max_drawdown": float(report["max_drawdown"]), + "adverse_slippage": float(adverse_slippage), ++ "fill_model": fill_model, + } + return fills, ledger, summary + +@@ -361,6 +365,8 @@ + benchmark_rows: list[dict[str, Any]] = [] + all_candidates: list[pd.DataFrame] = [] + all_fills: list[pd.DataFrame] = [] ++ all_book_fills: list[pd.DataFrame] = [] ++ book_fold_rows: list[dict[str, Any]] = [] + all_benchmark_fills: list[pd.DataFrame] = [] + coherence_rows: list[dict[str, Any]] = [] + calibration_rows: list[dict[str, Any]] = [] +@@ -410,6 +416,30 @@ + two_fills, _, two = _execute_pair_week( + candidates, books, trades, week=fold.test_week, hours=_hours(test_state), adverse_slippage=0.02 + ) ++ book_fills, _, book_summary = _execute_pair_week( ++ candidates, ++ books, ++ trades, ++ week=fold.test_week, ++ hours=_hours(test_state), ++ adverse_slippage=0.0, ++ fill_model="book_arrival", ++ ) ++ book_fold_rows.append( ++ { ++ "fold_id": fold.fold_id, ++ "test_week": fold.test_week, ++ "config_id": selected_id, ++ "candidate_count": book_summary["candidate_count"], ++ "atomic_count": book_summary["atomic_count"], ++ "atomic_per_hour": book_summary["atomic_per_hour"], ++ "orphan_rate": book_summary["orphan_rate"], ++ "terminal_growth": book_summary["terminal_growth"], ++ "quality_ratio": book_summary["quality_ratio"], ++ "max_drawdown": book_summary["max_drawdown"], ++ "fill_model": "book_arrival", ++ } ++ ) + fold_rows.append( + { + "fold_id": fold.fold_id, +@@ -437,6 +467,12 @@ + tagged_fills["test_week"] = fold.test_week + all_fills.append(taged_fills) + ++ if not book_fills.empty: ++ tagged_book_fills = book_fills.copy() ++ tagged_book_fills["fold_id"] = fold.fold_id ++ tagged_book_fills["test_week"] = fold.test_week ++ all_book_fills.append(tagged_book_fills) ++ + benchmark_fills, benchmark = _evaluate_benchmark_week(test_state, books, trades, fold.test_week) + benchmark["fold_id"] = fold.fold_id + benchmark_rows.append(benchmark) +@@ -450,6 +486,8 @@ + validation_frame = pd.DataFrame(validation_rows) + candidate_ledger = pd.concat(all_candidates, ignore_index=True) if all_candidates else pd.DataFrame() + fill_ledger = pd.concat(all_fills, ignore_index=True) if all_fills else pd.DataFrame() ++ book_fill_ledger = pd.concat(all_book_fills, ignore_index=True) if all_book_fills else pd.DataFrame() ++ book_fold_results = pd.DataFrame(book_fold_rows) + benchmark_fill_ledger = pd.concat(all_benchmark_fills, ignore_index=True) if all_benchmark_fills else pd.DataFrame() + + atomic_pnl = fill_ledger.loc[fill_ledger.get("atomic", False).astype(bool), "pnl"] if not fill_ledger.empty else pd.Series(dtype=float) +@@ -488,12 +526,15 @@ + admission["permutation_ruin_rate"] = 0.0 + + fold_results.to_csv(output_dir / "fold_results.csv", index=False) ++ book_fold_results.to_csv(output_dir / "book_fold_results.csv", index=False) + benchmark_results.to_csv(output_dir / "benchmark_results.csv", index=False) + validation_frame.to_csv(output_dir / "validation_grid.csv", index=False) + if not candidate_ledger.empty: + _write_frame(candidate_ledger, output_dir / "candidate_ledger.parquet") + if not fill_ledger.empty: + _write_frame(fill_ledger, output_dir / "execution_ledger.parquet") ++ if not book_fill_ledger.empty: ++ _write_frame(book_fill_ledger, output_dir / "book_execution_ledger.parquet") + if not benchmark_fill_ledger.empty: + _write_frame(benchmark_fill_ledger, output_dir / "benchmark_execution_ledger.parquet") + if not candidate_ledger.empty: +@@ -531,7 +572,14 @@ + "weeks": weeks, + "folds": [fold.__dict__ if hasattr(fold, "__dict__") else {"fold_id": fold.fold_id, "validation_weeks": fold.validation_weeks, "test_week": fold.test_week, "fit_weeks": fold.fit_weeks} for fold in folds], + "configs": [config.__dict__ if hasattr(config, "__dict__") else {field: getattr(config, field) for field in config.__dataclass_fields__} for config in configs], +- "execution": {"latency_ms": 1000, "tape_window_ms": 1500, "shares": BASE_SHARES, "base_slippage": 0.01}, ++ "execution": { ++ "latency_ms": 1000, ++ "tape_window_ms": 1500, ++ "shares": BASE_SHARES, ++ "base_slippage": 0.01, ++ "admission_fill_model": "tape_confirmed", ++ "diagnostic_fill_model": "book_arrival", ++ }, + "risk": {"per_position_max_loss_fraction": 0.01, "concurrent_max_loss_fraction": 0.20}, + "environment": {"python": sys.version, "platform": platform.platform(), "pythonhashseed": os.environ.get("PYTHONHASHSEED")}, + "admission": admission, +diff --git a/tests/digital_surface/test_execution.py b/tests/digital_surface/test_execution.py +--- a/tests/digital_surface/test_execution.py ++++ b/tests/digital_surface/test_execution.py +@@ -59,3 +59,36 @@ + bad = replay_single_leg_fok(signals, books(), tape(prearrival_only=True), shares=5) + assert good.iloc[0].filled and good.iloc[0].pnl > 0 + assert not bad.iloc[0].filled ++ ++ ++def test_book_arrival_model_uses_post_latency_executable_depth_without_future_tape(): ++ conservative = replay_atomic_fok( ++ candidate(), books(), tape(prearrival_only=True), shares=5, latency_ms=1000 ++ ) ++ estimated = replay_atomic_fok( ++ candidate(), ++ books(), ++ tape(prearrival_only=True), ++ shares=5, ++ latency_ms=1000, ++ fill_model="book_arrival", ++ ) ++ assert not conservative.iloc[0].atomic ++ assert estimated.iloc[0].atomic ++ assert estimated.iloc[0].fill_model == "book_arrival" ++ assert estimated.iloc[0].low_yes_book_executable ++ assert estimated.iloc[0].high_no_book_executable ++ assert not estimated.iloc[0].low_yes_tape_confirmed ++ assert not estimated.iloc[0].high_no_tape_confirmed ++ ++ ++def test_book_arrival_model_still_rejects_missing_or_uncrossable_books(): ++ missing_high = books().loc[lambda frame: frame["condition_id"] != "high"] ++ missing = replay_atomic_fok( ++ candidate(), missing_high, tape(), shares=5, latency_ms=1000, fill_model="book_arrival" ++ ) ++ expensive = replay_atomic_fok( ++ candidate(), books(high_ask=.60), tape(), shares=5, latency_ms=1000, fill_model="book_arrival" ++ ) ++ assert not missing.iloc[0].atomic ++ assert not expensive.iloc[0].atomic +diff --git a/tests/digital_surface/test_integration.py b/tests/digital_surface/test_integration.py +--- a/tests/digital_surface/test_integration.py ++++ b/tests/digital_surface/test_integration.py +@@ -65,9 +65,14 @@ + "REPORT.md", "MANIFEST.json", "SHA256SUMS.json", "fold_results.csv", + "validation_grid.csv", "admission.json", "INVALIDATIONS.md", + "execution_failure_attribution.csv", "EXECUTION_DIAGNOSTICS.json", ++ "book_fold_results.csv", + ]: + assert (output / name).exists(), name ++ assert (output / "book_execution_ledger.parquet").exists() or (output / "book_execution_ledger.csv").exists() + folds = pd.read_csv(output / "fold_results.csv") + assert len(folds) == 4 + assert folds["atomic_count"].sum() == 4 ++ book_folds = pd.read_csv(output / "book_fold_results.csv") ++ assert len(book_folds) == 4 ++ assert book_folds["atomic_count"].sum() == 4 + assert "pre-arrival" in (output / "INVALIDATIONS.md").read_text().lower() From 5caea8e1103e3dac199d375d4abfb9399b0a246e Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 03:52:56 -0500 Subject: [PATCH 028/177] ci: verify candidate semantics with deterministic tolerance --- ...urface-execution-diagnostics-verify-v2.yml | 127 ++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 .github/workflows/digital-surface-execution-diagnostics-verify-v2.yml diff --git a/.github/workflows/digital-surface-execution-diagnostics-verify-v2.yml b/.github/workflows/digital-surface-execution-diagnostics-verify-v2.yml new file mode 100644 index 00000000..f7ff966f --- /dev/null +++ b/.github/workflows/digital-surface-execution-diagnostics-verify-v2.yml @@ -0,0 +1,127 @@ +name: Verify Digital Surface Execution Diagnostics v2 + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/digital-surface-execution-diagnostics-verify-v2.yml + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-execution-diagnostics-verify-v2.yml + workflow_dispatch: + +permissions: + contents: read + actions: read + +env: + BASELINE_ARTIFACT_ID: '8615107471' + DIAGNOSTIC_ARTIFACT_ID: '8617727219' + BASELINE_ZIP_SHA256: '479a25e235841d94077b3d2df070c8ed46bb736c7ebd8f9f1bde8016198a9c39' + DIAGNOSTIC_ZIP_SHA256: '80a442cded15e26703c401e197a53db28ae59b61efa6dcb808a0d65ead961293' + +jobs: + verify-artifacts: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - run: python -m pip install pandas==2.2.3 pyarrow==19.0.1 numpy==2.2.6 + + - name: Download immutable artifacts + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + for pair in "baseline:${BASELINE_ARTIFACT_ID}" "diagnostic:${DIAGNOSTIC_ARTIFACT_ID}"; do + name=${pair%%:*}; id=${pair##*:} + curl --fail --location --silent --show-error \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H 'Accept: application/vnd.github+json' \ + -H 'X-GitHub-Api-Version: 2022-11-28' \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/artifacts/${id}/zip" \ + -o "${name}.zip" + mkdir "${name}" && unzip -q "${name}.zip" -d "${name}" + done + echo "${BASELINE_ZIP_SHA256} baseline.zip" | sha256sum -c - + echo "${DIAGNOSTIC_ZIP_SHA256} diagnostic.zip" | sha256sum -c - + + - name: Verify semantic invariance and diagnostics + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import csv, hashlib, json + from collections import Counter + from pathlib import Path + import numpy as np + import pandas as pd + + baseline = Path('baseline/artifacts/digital-surface-arbitrage') + current = Path('diagnostic/artifacts/digital-surface-execution-diagnostics') + left = pd.read_parquet(baseline / 'candidate_ledger.parquet') + right = pd.read_parquet(current / 'candidate_ledger.parquet') + assert left.shape == right.shape + assert left.columns.tolist() == right.columns.tolist() + + numeric_report = {} + for column in left.columns: + if pd.api.types.is_numeric_dtype(left[column]): + a = left[column].to_numpy(dtype=float) + b = right[column].to_numpy(dtype=float) + finite = np.isfinite(a) & np.isfinite(b) + max_abs = float(np.max(np.abs(a[finite] - b[finite]))) if finite.any() else 0.0 + numeric_report[column] = max_abs + assert np.allclose(a, b, rtol=0.0, atol=1e-12, equal_nan=True), (column, max_abs) + else: + assert left[column].fillna('').astype(str).equals(right[column].fillna('').astype(str)), column + + exact_files = ['execution_ledger.parquet', 'fold_results.csv', 'admission.json'] + for name in exact_files: + assert (baseline / name).read_bytes() == (current / name).read_bytes(), name + + rows = list(csv.DictReader((current / 'execution_failure_attribution.csv').open())) + reasons = Counter(row['reason'] for row in rows) + expected = Counter({ + 'insufficient_post_arrival_tape': 186, + 'ask_above_limit': 114, + 'missing_post_arrival_book': 110, + 'filled': 34, + 'insufficient_displayed_depth': 4, + }) + assert len(rows) == 448 + assert reasons == expected, (reasons, expected) + + internal = json.loads((current / 'SHA256SUMS.json').read_text()) + for name, wanted in internal.items(): + got = hashlib.sha256((current / name).read_bytes()).hexdigest() + assert got == wanted, (name, got, wanted) + + sidecar = Path('diagnostic/digital-surface-execution-diagnostics-v1.tar.gz.sha256').read_text().split()[0] + tar_sha = hashlib.sha256(Path('diagnostic/digital-surface-execution-diagnostics-v1.tar.gz').read_bytes()).hexdigest() + assert tar_sha == sidecar + + result = { + 'candidate_semantics_equal_at_1e-12': True, + 'maximum_numeric_difference': max(numeric_report.values(), default=0.0), + 'numeric_column_max_abs_differences': numeric_report, + 'exact_downstream_files': exact_files, + 'reason_counts': dict(expected), + 'internal_checksums_verified': len(internal), + 'diagnostic_tar_sha256': tar_sha, + 'live_mode': 'disabled', + } + Path('DIAGNOSTIC_ARTIFACT_VERIFICATION_V2.json').write_text(json.dumps(result, indent=2, sort_keys=True)) + print(json.dumps(result, indent=2)) + PY + + - uses: actions/upload-artifact@v4 + with: + name: digital-surface-execution-diagnostic-verification-v2 + path: DIAGNOSTIC_ARTIFACT_VERIFICATION_V2.json + retention-days: 30 From 98780315848fc82076336eb41f705e647661036a Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 09:21:55 -0500 Subject: [PATCH 029/177] ci: compare tape-confirmed and book-arrival fill models --- ...igital-surface-book-arrival-comparison.yml | 170 ++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 .github/workflows/digital-surface-book-arrival-comparison.yml diff --git a/.github/workflows/digital-surface-book-arrival-comparison.yml b/.github/workflows/digital-surface-book-arrival-comparison.yml new file mode 100644 index 00000000..90530309 --- /dev/null +++ b/.github/workflows/digital-surface-book-arrival-comparison.yml @@ -0,0 +1,170 @@ +name: Digital Surface Book-Arrival Comparison + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/digital-surface-book-arrival-comparison.yml + - ci_payload/digital-surface-live-readiness/book-arrival-model.patch + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-book-arrival-comparison.yml + - ci_payload/digital-surface-live-readiness/book-arrival-model.patch + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + compare-fill-models: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct checksum-pinned source + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source && tar -xzf base.tar.gz -C source + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + for patch_file in timestamp-unit-fix.patch full-universe-gate-fix.patch resolved-universe-fix.patch asset-merge-fix.patch; do + patch -d source -p1 < "ci_payload/digital-surface-official-resolution-patch/${patch_file}" + done + echo 'f71f3a653ca58b4bf98609fc29d8ab16f9d031c9ed14906ebe5159f2b0de8eab ci_payload/digital-surface-live-readiness/execution-diagnostics.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/execution-diagnostics.patch + echo '8606cc925aa3b3a6417abb1fba9a11df916cc2b281880978fb6229aa25875be9 ci_payload/digital-surface-live-readiness/book-arrival-model.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/book-arrival-model.patch + cat > /tmp/source.sha256 <<'EOF' + 0e858f150d858f9b9ea2e6a4509d74eed7c27d9e864ac6e2bba7d89756102599 source/research/digital_surface/diagnostics.py + 15171104e206f58587ceeff169bf6f9fdf014bfacee4c271217f2f4bc68effd8 source/research/digital_surface/execution.py + ae0dd4e5943ae1c3fd1ea6baf741fe182cba4a3e36d47ecde73c9fb630aa57f4 source/research/digital_surface/run_research.py + b5daad831aadfb67c9fd51182d0ff23add6a560e14ffcb90bca4d395cf730da6 source/tests/digital_surface/test_diagnostics.py + c2d7a06e349ea26ebb1ba540d47974fa7741b1ab85e65d02b3d0a6948a9c26fa source/tests/digital_surface/test_execution.py + 5a1825a337daba295b32fd2a942f6a5a7e14a6061cd5eb33f68a4b241551b6a5 source/tests/digital_surface/test_integration.py + EOF + sed -i 's/^ //' /tmp/source.sha256 + sha256sum -c /tmp/source.sha256 + cp /tmp/source.sha256 source/BOOK_ARRIVAL_COMPARISON_SOURCE_SHA256.txt + + - uses: actions/cache@v4 + with: + path: source/.cache/digital-surface + key: digital-surface-official-resolution-obadiaha-11793901-kinzik-eb4e9fc7-v5 + + - name: Install pinned stack + run: | + python -m pip install --upgrade pip + python -m pip install numpy==2.2.6 pandas==2.2.3 scipy==1.15.3 scikit-learn==1.6.1 pyarrow==19.0.1 requests==2.32.3 pytest==8.3.5 tabulate==0.9.0 + + - name: Pass exactly 32 tests + working-directory: source + shell: bash + run: | + set -euo pipefail + python -m compileall -q research/digital_surface tests/digital_surface + PYTHONPATH=. pytest --collect-only -q tests/digital_surface | tee /tmp/collect.txt + grep -Eq '^32 tests collected' /tmp/collect.txt + PYTHONPATH=. pytest tests/digital_surface -q + + - name: Replay both fill models + working-directory: source + shell: bash + run: | + set -euo pipefail + PYTHONPATH=. python -m research.digital_surface.run_research \ + --cache-dir .cache/digital-surface \ + --output-dir artifacts/digital-surface-book-arrival-comparison \ + 2>&1 | tee digital-surface-book-arrival-comparison.log + + - name: Verify comparison + working-directory: source + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import hashlib, json + from pathlib import Path + import pandas as pd + root = Path('artifacts/digital-surface-book-arrival-comparison') + assert hashlib.sha256((root/'fold_results.csv').read_bytes()).hexdigest() == 'c4e51189ee942929c9764012501da9580de2fa8b4f983c38cee593707df5c0d7' + assert hashlib.sha256((root/'admission.json').read_bytes()).hexdigest() == '9912874614b9558eefc48e1a69507b6bad14a4e13004134d7f785f239907371a' + tape = pd.read_parquet(root/'execution_ledger.parquet') + book = pd.read_parquet(root/'book_execution_ledger.parquet') + folds = pd.read_csv(root/'book_fold_results.csv').sort_values('fold_id') + expected = [11, 17, 29, 5] + assert folds['atomic_count'].astype(int).tolist() == expected, folds + assert int(tape['atomic'].astype(bool).sum()) == 8 + assert int(book['atomic'].astype(bool).sum()) == 62 + assert set(tape['fill_model']) == {'tape_confirmed'} + assert set(book['fill_model']) == {'book_arrival'} + admission = json.loads((root/'admission.json').read_text()) + assert admission['classification'] == 'Rejected' + internal = json.loads((root/'SHA256SUMS.json').read_text()) + for name, wanted in internal.items(): + got = hashlib.sha256((root/name).read_bytes()).hexdigest() + assert got == wanted, (name, got, wanted) + result = { + 'admission_model': 'tape_confirmed', + 'admission_classification': admission['classification'], + 'tape_confirmed_atomic': 8, + 'book_arrival_atomic': 62, + 'book_arrival_atomic_by_fold': expected, + 'book_arrival_fold_results': folds.to_dict('records'), + 'book_arrival_is_diagnostic_only': True, + 'prospective_shadow_calibration_required': True, + 'live_mode': 'disabled', + 'internal_checksums_verified': len(internal), + } + (root/'BOOK_ARRIVAL_COMPARISON_VERIFICATION.json').write_text(json.dumps(result, indent=2, sort_keys=True)) + print(json.dumps(result, indent=2)) + PY + + - name: Build deterministic archive + if: always() + working-directory: source + shell: bash + run: | + set -euo pipefail + if test -d artifacts/digital-surface-book-arrival-comparison; then + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-book-arrival-comparison-v1.tar.gz \ + -C artifacts digital-surface-book-arrival-comparison + sha256sum digital-surface-book-arrival-comparison-v1.tar.gz > digital-surface-book-arrival-comparison-v1.tar.gz.sha256 + fi + + - name: Show comparison + if: always() + working-directory: source + run: | + cat artifacts/digital-surface-book-arrival-comparison/BOOK_ARRIVAL_COMPARISON_VERIFICATION.json || true + cat artifacts/digital-surface-book-arrival-comparison/book_fold_results.csv || true + tail -200 digital-surface-book-arrival-comparison.log || true + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: digital-surface-book-arrival-comparison-v1 + path: | + source/artifacts/digital-surface-book-arrival-comparison/ + source/digital-surface-book-arrival-comparison.log + source/digital-surface-book-arrival-comparison-v1.tar.gz + source/digital-surface-book-arrival-comparison-v1.tar.gz.sha256 + source/BOOK_ARRIVAL_COMPARISON_SOURCE_SHA256.txt + if-no-files-found: warn + retention-days: 30 From d0b476dbdcb43f404b8ad45e8548d54891f638f8 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 09:27:22 -0500 Subject: [PATCH 030/177] feat: checkpoint shadow kernel exports --- .../research/digital_surface/live/__init__.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/__init__.py diff --git a/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/__init__.py b/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/__init__.py new file mode 100644 index 00000000..88eb7a5f --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/__init__.py @@ -0,0 +1,19 @@ +from .config import LiveTradingDisabledError, TradingConfig, load_config +from .gateway import ShadowGateway, quantize_down +from .risk import evaluate_pair_risk +from .types import BookQuote, PreparedOrder, PreparedPair, RiskDecision, RiskState, TradeIntent + +__all__ = [ + "BookQuote", + "LiveTradingDisabledError", + "PreparedOrder", + "PreparedPair", + "RiskDecision", + "RiskState", + "ShadowGateway", + "TradeIntent", + "TradingConfig", + "evaluate_pair_risk", + "load_config", + "quantize_down", +] From 1e1b8de8e17391fd67256f63e9398b58f27f8fbd Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 09:27:44 -0500 Subject: [PATCH 031/177] feat: checkpoint shadow-only configuration --- .../research/digital_surface/live/config.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/config.py diff --git a/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/config.py b/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/config.py new file mode 100644 index 00000000..a11a7553 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/config.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from dataclasses import dataclass +from decimal import Decimal +from typing import Mapping + + +class LiveTradingDisabledError(RuntimeError): + """Raised whenever the process is asked to enable real-money submission.""" + + +@dataclass(frozen=True, slots=True) +class TradingConfig: + mode: str = "shadow" + tick_size: Decimal = Decimal("0.01") + max_book_age_ms: int = 2_000 + max_feed_age_ms: int = 5_000 + max_daily_loss: Decimal = Decimal("25") + max_orphan_exposure: Decimal = Decimal("0") + + +def load_config(env: Mapping[str, str] | None = None) -> TradingConfig: + values = dict(env or {}) + mode = str(values.get("TRADING_MODE", "shadow")).strip().lower() + if mode not in {"shadow", "live"}: + raise ValueError("TRADING_MODE must be 'shadow' or 'live'") + # Deliberately stronger than an arm token: this research checkpoint has no + # network-capable live implementation. Credentials cannot override it. + if mode == "live": + raise LiveTradingDisabledError( + "live trading is disabled until prospective admission and a separately reviewed release" + ) + return TradingConfig( + mode="shadow", + tick_size=Decimal(str(values.get("TICK_SIZE", "0.01"))), + max_book_age_ms=int(values.get("MAX_BOOK_AGE_MS", "2000")), + max_feed_age_ms=int(values.get("MAX_FEED_AGE_MS", "5000")), + max_daily_loss=Decimal(str(values.get("MAX_DAILY_LOSS", "25"))), + max_orphan_exposure=Decimal(str(values.get("MAX_ORPHAN_EXPOSURE", "0"))), + ) From 9c32967e10ddff763615030b708c254447b874b5 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 09:28:15 -0500 Subject: [PATCH 032/177] feat: checkpoint shadow execution types --- .../research/digital_surface/live/types.py | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/types.py diff --git a/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/types.py b/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/types.py new file mode 100644 index 00000000..204cfc2c --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/types.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from dataclasses import dataclass +from decimal import Decimal + + +@dataclass(frozen=True, slots=True) +class TradeIntent: + condition_id_low: str + token_id_low_yes: str + condition_id_high: str + token_id_high_no: str + max_low_price: Decimal + max_high_price: Decimal + shares: Decimal + decision_ts_ms: int + expires_ts_ms: int + + +@dataclass(frozen=True, slots=True) +class BookQuote: + token_id: str + best_ask: Decimal + ask_size: Decimal + observed_ts_ms: int + + +@dataclass(frozen=True, slots=True) +class RiskState: + feed_heartbeat_ts_ms: int + daily_pnl: Decimal = Decimal("0") + unresolved_orders: int = 0 + orphan_exposure: Decimal = Decimal("0") + ambiguous_pair_state: bool = False + + +@dataclass(frozen=True, slots=True) +class RiskDecision: + allowed: bool + reason: str + + +@dataclass(frozen=True, slots=True) +class PreparedOrder: + token_id: str + side: str + order_type: str + price: Decimal + size: Decimal + + +@dataclass(frozen=True, slots=True) +class PreparedPair: + mode: str + intent: TradeIntent + low_order: PreparedOrder + high_order: PreparedOrder + risk: RiskDecision From 9eb825dd9534a73785627a429359bc40de271921 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 09:28:36 -0500 Subject: [PATCH 033/177] feat: checkpoint fail-closed shadow risk kernel --- .../research/digital_surface/live/risk.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/risk.py diff --git a/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/risk.py b/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/risk.py new file mode 100644 index 00000000..094a673a --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/risk.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from decimal import Decimal + +from .config import TradingConfig +from .types import BookQuote, RiskDecision, RiskState, TradeIntent + + +def evaluate_pair_risk( + *, + intent: TradeIntent, + low_book: BookQuote, + high_book: BookQuote, + state: RiskState, + config: TradingConfig, + now_ms: int, +) -> RiskDecision: + if now_ms >= intent.expires_ts_ms: + return RiskDecision(False, "intent_expired") + if state.ambiguous_pair_state: + return RiskDecision(False, "ambiguous_pair_state") + if state.unresolved_orders: + return RiskDecision(False, "unresolved_prior_orders") + if state.orphan_exposure > config.max_orphan_exposure: + return RiskDecision(False, "orphan_exposure") + if state.daily_pnl <= -abs(config.max_daily_loss): + return RiskDecision(False, "daily_loss_limit") + if now_ms - state.feed_heartbeat_ts_ms > config.max_feed_age_ms: + return RiskDecision(False, "stale_feed") + if now_ms - low_book.observed_ts_ms > config.max_book_age_ms: + return RiskDecision(False, "stale_low_book") + if now_ms - high_book.observed_ts_ms > config.max_book_age_ms: + return RiskDecision(False, "stale_high_book") + if low_book.token_id != intent.token_id_low_yes: + return RiskDecision(False, "low_token_mismatch") + if high_book.token_id != intent.token_id_high_no: + return RiskDecision(False, "high_token_mismatch") + if low_book.best_ask > intent.max_low_price: + return RiskDecision(False, "low_ask_above_limit") + if high_book.best_ask > intent.max_high_price: + return RiskDecision(False, "high_ask_above_limit") + if low_book.ask_size < intent.shares: + return RiskDecision(False, "low_depth_insufficient") + if high_book.ask_size < intent.shares: + return RiskDecision(False, "high_depth_insufficient") + if intent.shares <= Decimal("0"): + return RiskDecision(False, "invalid_size") + return RiskDecision(True, "allowed") From 73a748c7097520423f3d0d3011e253a0a8fbec03 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 09:28:58 -0500 Subject: [PATCH 034/177] feat: checkpoint zero-network shadow gateway --- .../research/digital_surface/live/gateway.py | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/gateway.py diff --git a/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/gateway.py b/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/gateway.py new file mode 100644 index 00000000..901736c4 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/gateway.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from decimal import Decimal, ROUND_DOWN +from typing import Any + +from .config import TradingConfig +from .risk import evaluate_pair_risk +from .types import BookQuote, PreparedOrder, PreparedPair, RiskState, TradeIntent + + +def quantize_down(value: Decimal, tick: Decimal) -> Decimal: + if tick <= 0: + raise ValueError("tick must be positive") + units = (value / tick).to_integral_value(rounding=ROUND_DOWN) + return units * tick + + +class ShadowGateway: + """Prepare auditable FOK parameters without signing or sending a request.""" + + def __init__(self, config: TradingConfig, transport: Any | None = None) -> None: + if config.mode != "shadow": + raise ValueError("ShadowGateway requires shadow mode") + self.config = config + self._transport = transport + self.prepared: list[PreparedPair] = [] + + def prepare_pair( + self, + *, + intent: TradeIntent, + low_book: BookQuote, + high_book: BookQuote, + state: RiskState, + now_ms: int, + ) -> PreparedPair: + risk = evaluate_pair_risk( + intent=intent, + low_book=low_book, + high_book=high_book, + state=state, + config=self.config, + now_ms=now_ms, + ) + if not risk.allowed: + raise RuntimeError(f"risk rejected pair: {risk.reason}") + low_price = quantize_down(min(low_book.best_ask, intent.max_low_price), self.config.tick_size) + high_price = quantize_down(min(high_book.best_ask, intent.max_high_price), self.config.tick_size) + pair = PreparedPair( + mode="shadow", + intent=intent, + low_order=PreparedOrder( + token_id=intent.token_id_low_yes, + side="BUY", + order_type="FOK", + price=low_price, + size=intent.shares, + ), + high_order=PreparedOrder( + token_id=intent.token_id_high_no, + side="BUY", + order_type="FOK", + price=high_price, + size=intent.shares, + ), + risk=risk, + ) + self.prepared.append(pair) + # No method on the supplied transport is invoked. The object exists only + # so the test can prove preparation has zero network side effects. + return pair + + def submit_pair(self, pair: PreparedPair) -> None: + raise RuntimeError("network submission is not implemented in this shadow-only checkpoint") From 252400fa79f7af314ea4dd544767489d12555167 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 09:29:24 -0500 Subject: [PATCH 035/177] test: checkpoint shadow gateway safety regressions --- .../live/test_shadow_gateway.py | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/shadow-source/tests/digital_surface/live/test_shadow_gateway.py diff --git a/ci_payload/digital-surface-live-readiness/shadow-source/tests/digital_surface/live/test_shadow_gateway.py b/ci_payload/digital-surface-live-readiness/shadow-source/tests/digital_surface/live/test_shadow_gateway.py new file mode 100644 index 00000000..448b0c44 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/shadow-source/tests/digital_surface/live/test_shadow_gateway.py @@ -0,0 +1,99 @@ +from decimal import Decimal + +import pytest + +from research.digital_surface.live import ( + BookQuote, + LiveTradingDisabledError, + RiskState, + ShadowGateway, + TradeIntent, + load_config, +) + + +class ExplodingTransport: + def __getattr__(self, name): + raise AssertionError(f"shadow path attempted network transport method {name}") + + +def intent() -> TradeIntent: + return TradeIntent( + condition_id_low="low", + token_id_low_yes="yes-low", + condition_id_high="high", + token_id_high_no="no-high", + max_low_price=Decimal("0.51"), + max_high_price=Decimal("0.51"), + shares=Decimal("5"), + decision_ts_ms=1_000, + expires_ts_ms=10_000, + ) + + +def books(now=2_000): + return ( + BookQuote("yes-low", Decimal("0.5039"), Decimal("10"), now), + BookQuote("no-high", Decimal("0.4979"), Decimal("10"), now), + ) + + +def state(now=2_000): + return RiskState(feed_heartbeat_ts_ms=now) + + +def test_defaults_to_shadow_and_live_is_physically_disabled_even_with_credentials(): + assert load_config({}).mode == "shadow" + with pytest.raises(LiveTradingDisabledError): + load_config({ + "TRADING_MODE": "live", + "POLYMARKET_API_KEY": "present", + "POLYMARKET_PRIVATE_KEY": "present", + "LIVE_ARM_TOKEN": "present", + }) + + +def test_shadow_prepare_quantizes_exactly_and_never_calls_transport(): + config = load_config({"TICK_SIZE": "0.01"}) + gateway = ShadowGateway(config, transport=ExplodingTransport()) + low, high = books() + pair = gateway.prepare_pair(intent=intent(), low_book=low, high_book=high, state=state(), now_ms=2_000) + assert pair.mode == "shadow" + assert pair.low_order.price == Decimal("0.50") + assert pair.high_order.price == Decimal("0.49") + assert pair.low_order.order_type == pair.high_order.order_type == "FOK" + assert len(gateway.prepared) == 1 + + +def test_stale_feed_books_and_unresolved_exposure_fail_closed(): + gateway = ShadowGateway(load_config({})) + low, high = books(now=1_000) + with pytest.raises(RuntimeError, match="stale_feed"): + gateway.prepare_pair(intent=intent(), low_book=low, high_book=high, state=state(now=0), now_ms=6_000) + low, high = books(now=0) + with pytest.raises(RuntimeError, match="stale_low_book"): + gateway.prepare_pair(intent=intent(), low_book=low, high_book=high, state=state(now=6_000), now_ms=6_000) + low, high = books(now=2_000) + with pytest.raises(RuntimeError, match="unresolved_prior_orders"): + gateway.prepare_pair( + intent=intent(), low_book=low, high_book=high, + state=RiskState(feed_heartbeat_ts_ms=2_000, unresolved_orders=1), now_ms=2_000, + ) + with pytest.raises(RuntimeError, match="orphan_exposure"): + gateway.prepare_pair( + intent=intent(), low_book=low, high_book=high, + state=RiskState(feed_heartbeat_ts_ms=2_000, orphan_exposure=Decimal("0.01")), now_ms=2_000, + ) + + +def test_ambiguous_pair_state_blocks_and_submit_is_unavailable(): + gateway = ShadowGateway(load_config({})) + low, high = books() + with pytest.raises(RuntimeError, match="ambiguous_pair_state"): + gateway.prepare_pair( + intent=intent(), low_book=low, high_book=high, + state=RiskState(feed_heartbeat_ts_ms=2_000, ambiguous_pair_state=True), now_ms=2_000, + ) + pair = gateway.prepare_pair(intent=intent(), low_book=low, high_book=high, state=state(), now_ms=2_000) + with pytest.raises(RuntimeError, match="not implemented"): + gateway.submit_pair(pair) From 12df37e84b7facc59cca504a9a54f50ea629bcf0 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 09:29:52 -0500 Subject: [PATCH 036/177] feat: checkpoint prospective contamination ledger --- .../research/digital_surface/prospective.py | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/prospective-source/research/digital_surface/prospective.py diff --git a/ci_payload/digital-surface-live-readiness/prospective-source/research/digital_surface/prospective.py b/ci_payload/digital-surface-live-readiness/prospective-source/research/digital_surface/prospective.py new file mode 100644 index 00000000..4b062181 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/prospective-source/research/digital_surface/prospective.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any, Iterable + + +class ProspectiveContaminationError(RuntimeError): + pass + + +class ProspectiveIntegrityError(RuntimeError): + pass + + +def _canonical_json(value: dict[str, Any]) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _validate_sha256(name: str, value: str) -> str: + text = str(value).lower() + if len(text) != 64 or any(ch not in "0123456789abcdef" for ch in text): + raise ValueError(f"{name} must be a 64-character SHA-256 hex digest") + return text + + +class ProspectiveLedger: + def __init__(self, path: str | Path, *, diagnostic_market_ids: Iterable[str]) -> None: + self.path = Path(path) + self.diagnostic_market_ids = {str(value) for value in diagnostic_market_ids} + self._rows: list[dict[str, Any]] = [] + self.market_ids: set[str] = set() + self.head_hash = "0" * 64 + if self.path.exists(): + self._load_and_verify() + + def _load_and_verify(self) -> None: + previous = "0" * 64 + seen: set[str] = set() + rows: list[dict[str, Any]] = [] + for line_number, line in enumerate(self.path.read_text(encoding="utf-8").splitlines(), start=1): + if not line.strip(): + continue + try: + row = json.loads(line) + except json.JSONDecodeError as exc: + raise ProspectiveIntegrityError(f"invalid JSON at line {line_number}") from exc + stored_hash = str(row.get("record_hash", "")) + payload = dict(row) + payload.pop("record_hash", None) + if payload.get("previous_hash") != previous: + raise ProspectiveIntegrityError(f"hash chain mismatch at line {line_number}") + expected = hashlib.sha256(_canonical_json(payload)).hexdigest() + if stored_hash != expected: + raise ProspectiveIntegrityError(f"record hash mismatch at line {line_number}") + market_id = str(payload.get("market_id", "")) + if not market_id or market_id in seen: + raise ProspectiveIntegrityError(f"duplicate or empty market_id at line {line_number}") + _validate_sha256("policy_sha256", str(payload.get("policy_sha256", ""))) + _validate_sha256("source_sha256", str(payload.get("source_sha256", ""))) + seen.add(market_id) + rows.append(row) + previous = stored_hash + self._rows = rows + self.market_ids = seen + self.head_hash = previous + + def append_observation( + self, + *, + market_id: str, + first_seen_ts_ms: int, + policy_sha256: str, + source_sha256: str, + ) -> dict[str, Any]: + market = str(market_id) + if not market: + raise ValueError("market_id must be non-empty") + if market in self.diagnostic_market_ids: + raise ProspectiveContaminationError(f"market {market!r} appeared in diagnostic data") + if market in self.market_ids: + raise ProspectiveContaminationError(f"market {market!r} was already observed") + policy = _validate_sha256("policy_sha256", policy_sha256) + source = _validate_sha256("source_sha256", source_sha256) + timestamp = int(first_seen_ts_ms) + if timestamp < 0: + raise ValueError("first_seen_ts_ms must be non-negative") + payload = { + "first_seen_ts_ms": timestamp, + "market_id": market, + "policy_sha256": policy, + "previous_hash": self.head_hash, + "source_sha256": source, + "state": "observed", + } + record = {**payload, "record_hash": hashlib.sha256(_canonical_json(payload)).hexdigest()} + self.path.parent.mkdir(parents=True, exist_ok=True) + with self.path.open("a", encoding="utf-8", newline="\n") as handle: + handle.write(json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n") + handle.flush() + self._rows.append(record) + self.market_ids.add(market) + self.head_hash = record["record_hash"] + return dict(record) From ba96e7f56d2c81434dd3337ecb2de8a81364ec69 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 09:30:32 -0500 Subject: [PATCH 037/177] test: checkpoint prospective ledger integrity regressions --- .../tests/digital_surface/test_prospective.py | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/prospective-source/tests/digital_surface/test_prospective.py diff --git a/ci_payload/digital-surface-live-readiness/prospective-source/tests/digital_surface/test_prospective.py b/ci_payload/digital-surface-live-readiness/prospective-source/tests/digital_surface/test_prospective.py new file mode 100644 index 00000000..36817f92 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/prospective-source/tests/digital_surface/test_prospective.py @@ -0,0 +1,80 @@ +import json + +import pytest + +from research.digital_surface.prospective import ( + ProspectiveContaminationError, + ProspectiveIntegrityError, + ProspectiveLedger, +) + + +def test_rejects_diagnostic_markets_and_appends_new_observations_with_hash_chain(tmp_path): + path = tmp_path / "prospective.jsonl" + ledger = ProspectiveLedger(path, diagnostic_market_ids={"known"}) + + with pytest.raises(ProspectiveContaminationError, match="known"): + ledger.append_observation( + market_id="known", + first_seen_ts_ms=100, + policy_sha256="1" * 64, + source_sha256="2" * 64, + ) + + first = ledger.append_observation( + market_id="new-1", + first_seen_ts_ms=101, + policy_sha256="1" * 64, + source_sha256="2" * 64, + ) + second = ledger.append_observation( + market_id="new-2", + first_seen_ts_ms=102, + policy_sha256="1" * 64, + source_sha256="2" * 64, + ) + + assert first["previous_hash"] == "0" * 64 + assert second["previous_hash"] == first["record_hash"] + assert ledger.market_ids == {"new-1", "new-2"} + + reloaded = ProspectiveLedger(path, diagnostic_market_ids={"known"}) + assert reloaded.market_ids == ledger.market_ids + assert reloaded.head_hash == second["record_hash"] + + +def test_rejects_duplicate_market_ids_and_noncanonical_hashes(tmp_path): + path = tmp_path / "prospective.jsonl" + ledger = ProspectiveLedger(path, diagnostic_market_ids=set()) + ledger.append_observation( + market_id="new-1", + first_seen_ts_ms=101, + policy_sha256="1" * 64, + source_sha256="2" * 64, + ) + + with pytest.raises(ProspectiveContaminationError, match="already observed"): + ledger.append_observation( + market_id="new-1", + first_seen_ts_ms=102, + policy_sha256="1" * 64, + source_sha256="2" * 64, + ) + + rows = [json.loads(line) for line in path.read_text().splitlines()] + rows[0]["market_id"] = "tampered" + path.write_text("\n".join(json.dumps(row) for row in rows) + "\n") + + with pytest.raises(ProspectiveIntegrityError, match="record hash mismatch"): + ProspectiveLedger(path, diagnostic_market_ids=set()) + + +def test_requires_fixed_length_source_and_policy_hashes(tmp_path): + ledger = ProspectiveLedger(tmp_path / "prospective.jsonl", diagnostic_market_ids=set()) + with pytest.raises(ValueError, match="policy_sha256"): + ledger.append_observation( + market_id="new", + first_seen_ts_ms=1, + policy_sha256="short", + source_sha256="2" * 64, + ) From 2d70bc1d35123a35b0e8fc70fd76050ffbd46212 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 09:31:29 -0500 Subject: [PATCH 038/177] ci: verify shadow kernel and prospective ledger checkpoint --- ...-surface-shadow-prospective-checkpoint.yml | 178 ++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 .github/workflows/digital-surface-shadow-prospective-checkpoint.yml diff --git a/.github/workflows/digital-surface-shadow-prospective-checkpoint.yml b/.github/workflows/digital-surface-shadow-prospective-checkpoint.yml new file mode 100644 index 00000000..dc670aaf --- /dev/null +++ b/.github/workflows/digital-surface-shadow-prospective-checkpoint.yml @@ -0,0 +1,178 @@ +name: Digital Surface Shadow and Prospective Checkpoint + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/digital-surface-shadow-prospective-checkpoint.yml + - ci_payload/digital-surface-live-readiness/shadow-source/** + - ci_payload/digital-surface-live-readiness/prospective-source/** + - ci_payload/digital-surface-live-readiness/book-arrival-model.patch + - ci_payload/digital-surface-live-readiness/execution-diagnostics.patch + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-shadow-prospective-checkpoint.yml + - ci_payload/digital-surface-live-readiness/shadow-source/** + - ci_payload/digital-surface-live-readiness/prospective-source/** + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + verify-shadow-and-prospective: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct checksum-pinned source + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source && tar -xzf base.tar.gz -C source + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + for patch_file in timestamp-unit-fix.patch full-universe-gate-fix.patch resolved-universe-fix.patch asset-merge-fix.patch; do + patch -d source -p1 < "ci_payload/digital-surface-official-resolution-patch/${patch_file}" + done + echo 'f71f3a653ca58b4bf98609fc29d8ab16f9d031c9ed14906ebe5159f2b0de8eab ci_payload/digital-surface-live-readiness/execution-diagnostics.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/execution-diagnostics.patch + echo '8606cc925aa3b3a6417abb1fba9a11df916cc2b281880978fb6229aa25875be9 ci_payload/digital-surface-live-readiness/book-arrival-model.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/book-arrival-model.patch + cp -a ci_payload/digital-surface-live-readiness/shadow-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/prospective-source/. source/ + + cat > /tmp/checkpoint.sha256 <<'EOF' + ff4025e5eee69cdad8e0bbbee4e60b16455da2710da61a733c4151732e4a2cce source/research/digital_surface/live/__init__.py + 9c5c9bcbd39e1262ac468024f5a96fb0667c0c95fc2879d846f7ca2066c38b5f source/research/digital_surface/live/config.py + 9e6dff73a17227f907dce4bf56cfd54921f4eb677280b1774d1a73c979b9065a source/research/digital_surface/live/gateway.py + f9197d38fd8dcfcc073d83028ac498fd9ff07619fea2b2b6a90de9ab470a86e2 source/research/digital_surface/live/risk.py + 50f32b2e400767702073b97ba4fd77a7e8c26f5fd4c7d6f7d9aafe282d6258f0 source/research/digital_surface/live/types.py + be1f59a4ae742f062b419ad7f7fb974eadf9bd076cb2ecdabcd0063eedf1df6d source/tests/digital_surface/live/test_shadow_gateway.py + f219222d7b89c5ab4ef0619da9163a77b79ba7f2a9fe611c4072746771e2fa41 source/research/digital_surface/prospective.py + 9ed734554e0296e94212cac3911466ac230241efdd4ab911e9adc74e41ef05b2 source/tests/digital_surface/test_prospective.py + EOF + sed -i 's/^ //' /tmp/checkpoint.sha256 + sha256sum -c /tmp/checkpoint.sha256 + cp /tmp/checkpoint.sha256 source/SHADOW_PROSPECTIVE_SOURCE_SHA256.txt + + - name: Install pinned test stack + run: | + python -m pip install --upgrade pip + python -m pip install numpy==2.2.6 pandas==2.2.3 scipy==1.15.3 scikit-learn==1.6.1 pyarrow==19.0.1 requests==2.32.3 pytest==8.3.5 tabulate==0.9.0 + + - name: Compile and pass exactly 39 tests + working-directory: source + shell: bash + run: | + set -euo pipefail + python -m compileall -q research/digital_surface tests/digital_surface + PYTHONPATH=. pytest --collect-only -q tests/digital_surface | tee /tmp/collect.txt + grep -Eq '^39 tests collected' /tmp/collect.txt + PYTHONPATH=. pytest tests/digital_surface -q + + - name: Prove live submission is absent and prospective chain is valid + working-directory: source + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import ast + import hashlib + import json + import tempfile + from pathlib import Path + + from research.digital_surface.live import LiveTradingDisabledError, load_config + from research.digital_surface.prospective import ProspectiveLedger + + try: + load_config({ + 'TRADING_MODE': 'live', + 'POLYMARKET_API_KEY': 'present', + 'POLYMARKET_PRIVATE_KEY': 'present', + 'LIVE_ARM_TOKEN': 'present', + }) + except LiveTradingDisabledError: + pass + else: + raise AssertionError('live mode unexpectedly armed') + + forbidden_import_roots = {'requests', 'httpx', 'aiohttp', 'websocket', 'websockets', 'socket'} + live_root = Path('research/digital_surface/live') + seen_imports = set() + for path in live_root.glob('*.py'): + tree = ast.parse(path.read_text(), filename=str(path)) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + seen_imports.update(alias.name.split('.')[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + seen_imports.add(node.module.split('.')[0]) + assert not (seen_imports & forbidden_import_roots), seen_imports & forbidden_import_roots + + with tempfile.TemporaryDirectory() as directory: + ledger = ProspectiveLedger(Path(directory) / 'ledger.jsonl', diagnostic_market_ids={'old'}) + first = ledger.append_observation( + market_id='fresh-1', first_seen_ts_ms=1, + policy_sha256='1' * 64, source_sha256='2' * 64, + ) + second = ledger.append_observation( + market_id='fresh-2', first_seen_ts_ms=2, + policy_sha256='1' * 64, source_sha256='2' * 64, + ) + assert first['previous_hash'] == '0' * 64 + assert second['previous_hash'] == first['record_hash'] + reloaded = ProspectiveLedger(ledger.path, diagnostic_market_ids={'old'}) + assert reloaded.head_hash == second['record_hash'] + + files = {} + for line in Path('SHADOW_PROSPECTIVE_SOURCE_SHA256.txt').read_text().splitlines(): + wanted, name = line.split(maxsplit=1) + got = hashlib.sha256(Path(name).read_bytes()).hexdigest() + assert got == wanted + files[name] = got + result = { + 'tests_collected': 39, + 'live_mode': 'physically_disabled', + 'network_capable_imports': [], + 'prospective_hash_chain_verified': True, + 'source_files_verified': files, + } + Path('SHADOW_PROSPECTIVE_VERIFICATION.json').write_text(json.dumps(result, indent=2, sort_keys=True)) + print(json.dumps(result, indent=2)) + PY + + - name: Build deterministic source checkpoint + working-directory: source + shell: bash + run: | + set -euo pipefail + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-shadow-prospective-source-v1.tar.gz \ + research tests SHADOW_PROSPECTIVE_SOURCE_SHA256.txt SHADOW_PROSPECTIVE_VERIFICATION.json + sha256sum digital-surface-shadow-prospective-source-v1.tar.gz > digital-surface-shadow-prospective-source-v1.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: digital-surface-shadow-prospective-checkpoint-v1 + path: | + source/SHADOW_PROSPECTIVE_SOURCE_SHA256.txt + source/SHADOW_PROSPECTIVE_VERIFICATION.json + source/digital-surface-shadow-prospective-source-v1.tar.gz + source/digital-surface-shadow-prospective-source-v1.tar.gz.sha256 + retention-days: 30 From 28f8d3e820e11951b75a4ad938efdcd394850736 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 09:33:20 -0500 Subject: [PATCH 039/177] feat: export shadow canary admission report --- .../shadow-source/research/digital_surface/live/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/__init__.py b/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/__init__.py index 88eb7a5f..f2110030 100644 --- a/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/__init__.py +++ b/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/__init__.py @@ -1,5 +1,6 @@ from .config import LiveTradingDisabledError, TradingConfig, load_config from .gateway import ShadowGateway, quantize_down +from .report import ShadowCanaryMetrics, evaluate_shadow_admission from .risk import evaluate_pair_risk from .types import BookQuote, PreparedOrder, PreparedPair, RiskDecision, RiskState, TradeIntent @@ -9,11 +10,13 @@ "PreparedOrder", "PreparedPair", "RiskDecision", + "ShadowCanaryMetrics", "RiskState", "ShadowGateway", "TradeIntent", "TradingConfig", "evaluate_pair_risk", + "evaluate_shadow_admission", "load_config", "quantize_down", ] From 294759106236f5cd8bcd553c20ee083768aae00c Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 09:33:41 -0500 Subject: [PATCH 040/177] feat: checkpoint shadow canary admission report --- .../research/digital_surface/live/report.py | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/report.py diff --git a/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/report.py b/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/report.py new file mode 100644 index 00000000..374cba0c --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/report.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass +from typing import Any + + +@dataclass(frozen=True, slots=True) +class ShadowCanaryMetrics: + markets_observed: int + qualified_portfolios: int + simulated_atomic: int + orphaned: int + unresolved_order_states: int + feed_outages: int + weekly_passes: int + + +def evaluate_shadow_admission(metrics: ShadowCanaryMetrics) -> dict[str, Any]: + values = asdict(metrics) + if any(int(value) < 0 for value in values.values()): + raise ValueError("shadow canary metrics must be non-negative") + attempts = int(metrics.simulated_atomic) + int(metrics.orphaned) + orphan_rate = float(metrics.orphaned / attempts) if attempts else 0.0 + reasons: list[str] = [] + if metrics.markets_observed < 500: + reasons.append("requires at least 500 observed markets") + if metrics.qualified_portfolios < 100: + reasons.append("requires at least 100 qualified portfolios") + if metrics.weekly_passes < 4: + reasons.append("requires four untouched weekly passes") + if metrics.unresolved_order_states: + reasons.append("unresolved order state exists") + if orphan_rate > 0.05: + reasons.append("orphan rate exceeds 5%") + classification = "Admissible" if not reasons else "Rejected" + return { + "classification": classification, + "reasons": reasons, + "markets_observed": int(metrics.markets_observed), + "qualified_portfolios": int(metrics.qualified_portfolios), + "simulated_atomic": int(metrics.simulated_atomic), + "orphaned": int(metrics.orphaned), + "orphan_rate": orphan_rate, + "unresolved_order_states": int(metrics.unresolved_order_states), + "feed_outages": int(metrics.feed_outages), + "weekly_passes": int(metrics.weekly_passes), + # Even an admissible research report cannot arm the current build. + "live_mode": "disabled_pending_operator_release", + } From e95fddb879f556d71a997d2a03a4511c9866edf5 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 09:34:20 -0500 Subject: [PATCH 041/177] test: checkpoint shadow canary admission gates --- .../tests/digital_surface/live/test_report.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/shadow-source/tests/digital_surface/live/test_report.py diff --git a/ci_payload/digital-surface-live-readiness/shadow-source/tests/digital_surface/live/test_report.py b/ci_payload/digital-surface-live-readiness/shadow-source/tests/digital_surface/live/test_report.py new file mode 100644 index 00000000..491e309e --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/shadow-source/tests/digital_surface/live/test_report.py @@ -0,0 +1,47 @@ +from research.digital_surface.live.report import ShadowCanaryMetrics, evaluate_shadow_admission + + +def test_rejects_insufficient_prospective_sample_and_weekly_evidence(): + decision = evaluate_shadow_admission(ShadowCanaryMetrics( + markets_observed=499, + qualified_portfolios=99, + simulated_atomic=95, + orphaned=5, + unresolved_order_states=0, + feed_outages=0, + weekly_passes=3, + )) + assert decision["classification"] == "Rejected" + assert "requires at least 500 observed markets" in decision["reasons"] + assert "requires at least 100 qualified portfolios" in decision["reasons"] + assert "requires four untouched weekly passes" in decision["reasons"] + + +def test_rejects_unresolved_orders_and_excess_orphaning(): + decision = evaluate_shadow_admission(ShadowCanaryMetrics( + markets_observed=600, + qualified_portfolios=120, + simulated_atomic=100, + orphaned=8, + unresolved_order_states=1, + feed_outages=0, + weekly_passes=4, + )) + assert decision["classification"] == "Rejected" + assert "unresolved order state exists" in decision["reasons"] + assert "orphan rate exceeds 5%" in decision["reasons"] + + +def test_admits_only_when_every_prospective_gate_clears(): + decision = evaluate_shadow_admission(ShadowCanaryMetrics( + markets_observed=650, + qualified_portfolios=120, + simulated_atomic=116, + orphaned=4, + unresolved_order_states=0, + feed_outages=1, + weekly_passes=4, + )) + assert decision["classification"] == "Admissible" + assert decision["orphan_rate"] == 4 / 120 + assert decision["live_mode"] == "disabled_pending_operator_release" From ed6aaf1a744c74d544bd2ab27421c50aba03a4cf Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 09:35:26 -0500 Subject: [PATCH 042/177] ci: verify shadow canary report checkpoint --- ...e-shadow-prospective-report-checkpoint.yml | 199 ++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 .github/workflows/digital-surface-shadow-prospective-report-checkpoint.yml diff --git a/.github/workflows/digital-surface-shadow-prospective-report-checkpoint.yml b/.github/workflows/digital-surface-shadow-prospective-report-checkpoint.yml new file mode 100644 index 00000000..67fb697f --- /dev/null +++ b/.github/workflows/digital-surface-shadow-prospective-report-checkpoint.yml @@ -0,0 +1,199 @@ +name: Digital Surface Shadow Prospective Report Checkpoint + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/digital-surface-shadow-prospective-report-checkpoint.yml + - ci_payload/digital-surface-live-readiness/shadow-source/** + - ci_payload/digital-surface-live-readiness/prospective-source/** + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-shadow-prospective-report-checkpoint.yml + - ci_payload/digital-surface-live-readiness/shadow-source/** + - ci_payload/digital-surface-live-readiness/prospective-source/** + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + verify-shadow-prospective-report: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct checksum-pinned source + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source && tar -xzf base.tar.gz -C source + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + for patch_file in timestamp-unit-fix.patch full-universe-gate-fix.patch resolved-universe-fix.patch asset-merge-fix.patch; do + patch -d source -p1 < "ci_payload/digital-surface-official-resolution-patch/${patch_file}" + done + echo 'f71f3a653ca58b4bf98609fc29d8ab16f9d031c9ed14906ebe5159f2b0de8eab ci_payload/digital-surface-live-readiness/execution-diagnostics.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/execution-diagnostics.patch + echo '8606cc925aa3b3a6417abb1fba9a11df916cc2b281880978fb6229aa25875be9 ci_payload/digital-surface-live-readiness/book-arrival-model.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/book-arrival-model.patch + cp -a ci_payload/digital-surface-live-readiness/shadow-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/prospective-source/. source/ + + cat > /tmp/checkpoint.sha256 <<'EOF' + af2a1e50e84f13064424f662bf02847df92ca63abff1692fbfea866ac71c3816 source/research/digital_surface/live/__init__.py + 9c5c9bcbd39e1262ac468024f5a96fb0667c0c95fc2879d846f7ca2066c38b5f source/research/digital_surface/live/config.py + 9e6dff73a17227f907dce4bf56cfd54921f4eb677280b1774d1a73c979b9065a source/research/digital_surface/live/gateway.py + 3be74a910e787a3f9e0f3548691e63ed93d78eeb3328fd9538e3a0e643b57d1c source/research/digital_surface/live/report.py + f9197d38fd8dcfcc073d83028ac498fd9ff07619fea2b2b6a90de9ab470a86e2 source/research/digital_surface/live/risk.py + 50f32b2e400767702073b97ba4fd77a7e8c26f5fd4c7d6f7d9aafe282d6258f0 source/research/digital_surface/live/types.py + 10d833f89194246ea2fe5dec8bb2fc4388c8509a4d8276c7fcdc0a2b1a13e76c source/tests/digital_surface/live/test_report.py + be1f59a4ae742f062b419ad7f7fb974eadf9bd076cb2ecdabcd0063eedf1df6d source/tests/digital_surface/live/test_shadow_gateway.py + f219222d7b89c5ab4ef0619da9163a77b79ba7f2a9fe611c4072746771e2fa41 source/research/digital_surface/prospective.py + 9ed734554e0296e94212cac3911466ac230241efdd4ab911e9adc74e41ef05b2 source/tests/digital_surface/test_prospective.py + EOF + sed -i 's/^ //' /tmp/checkpoint.sha256 + sha256sum -c /tmp/checkpoint.sha256 + cp /tmp/checkpoint.sha256 source/SHADOW_PROSPECTIVE_REPORT_SOURCE_SHA256.txt + + - name: Install pinned test stack + run: | + python -m pip install --upgrade pip + python -m pip install numpy==2.2.6 pandas==2.2.3 scipy==1.15.3 scikit-learn==1.6.1 pyarrow==19.0.1 requests==2.32.3 pytest==8.3.5 tabulate==0.9.0 + + - name: Compile and pass exactly 42 tests + working-directory: source + shell: bash + run: | + set -euo pipefail + python -m compileall -q research/digital_surface tests/digital_surface + PYTHONPATH=. pytest --collect-only -q tests/digital_surface | tee /tmp/collect.txt + grep -Eq '^42 tests collected' /tmp/collect.txt + PYTHONPATH=. pytest tests/digital_surface -q + + - name: Prove fail-closed execution and canary gates + working-directory: source + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import ast + import hashlib + import json + import tempfile + from pathlib import Path + + from research.digital_surface.live import ( + LiveTradingDisabledError, + ShadowCanaryMetrics, + evaluate_shadow_admission, + load_config, + ) + from research.digital_surface.prospective import ProspectiveLedger + + try: + load_config({'TRADING_MODE': 'live', 'LIVE_ARM_TOKEN': 'present'}) + except LiveTradingDisabledError: + pass + else: + raise AssertionError('live mode unexpectedly armed') + + forbidden_import_roots = {'requests', 'httpx', 'aiohttp', 'websocket', 'websockets', 'socket'} + seen_imports = set() + for path in Path('research/digital_surface/live').glob('*.py'): + tree = ast.parse(path.read_text(), filename=str(path)) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + seen_imports.update(alias.name.split('.')[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + seen_imports.add(node.module.split('.')[0]) + assert not (seen_imports & forbidden_import_roots), seen_imports & forbidden_import_roots + + insufficient = evaluate_shadow_admission(ShadowCanaryMetrics( + markets_observed=499, + qualified_portfolios=99, + simulated_atomic=95, + orphaned=5, + unresolved_order_states=0, + feed_outages=0, + weekly_passes=3, + )) + assert insufficient['classification'] == 'Rejected' + passing = evaluate_shadow_admission(ShadowCanaryMetrics( + markets_observed=650, + qualified_portfolios=120, + simulated_atomic=116, + orphaned=4, + unresolved_order_states=0, + feed_outages=1, + weekly_passes=4, + )) + assert passing['classification'] == 'Admissible' + assert passing['live_mode'] == 'disabled_pending_operator_release' + + with tempfile.TemporaryDirectory() as directory: + ledger = ProspectiveLedger(Path(directory) / 'ledger.jsonl', diagnostic_market_ids={'old'}) + first = ledger.append_observation( + market_id='fresh-1', first_seen_ts_ms=1, + policy_sha256='1' * 64, source_sha256='2' * 64, + ) + second = ledger.append_observation( + market_id='fresh-2', first_seen_ts_ms=2, + policy_sha256='1' * 64, source_sha256='2' * 64, + ) + assert second['previous_hash'] == first['record_hash'] + assert ProspectiveLedger(ledger.path, diagnostic_market_ids={'old'}).head_hash == second['record_hash'] + + files = {} + for line in Path('SHADOW_PROSPECTIVE_REPORT_SOURCE_SHA256.txt').read_text().splitlines(): + wanted, name = line.split(maxsplit=1) + got = hashlib.sha256(Path(name).read_bytes()).hexdigest() + assert got == wanted + files[name] = got + result = { + 'tests_collected': 42, + 'live_mode': 'physically_disabled', + 'network_capable_imports': [], + 'prospective_hash_chain_verified': True, + 'shadow_canary_gates_verified': True, + 'admissible_report_still_requires_separate_operator_release': True, + 'source_files_verified': files, + } + Path('SHADOW_PROSPECTIVE_REPORT_VERIFICATION.json').write_text(json.dumps(result, indent=2, sort_keys=True)) + print(json.dumps(result, indent=2)) + PY + + - name: Build deterministic source checkpoint + working-directory: source + shell: bash + run: | + set -euo pipefail + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-shadow-prospective-report-source-v1.tar.gz \ + research tests SHADOW_PROSPECTIVE_REPORT_SOURCE_SHA256.txt SHADOW_PROSPECTIVE_REPORT_VERIFICATION.json + sha256sum digital-surface-shadow-prospective-report-source-v1.tar.gz > digital-surface-shadow-prospective-report-source-v1.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: digital-surface-shadow-prospective-report-checkpoint-v1 + path: | + source/SHADOW_PROSPECTIVE_REPORT_SOURCE_SHA256.txt + source/SHADOW_PROSPECTIVE_REPORT_VERIFICATION.json + source/digital-surface-shadow-prospective-report-source-v1.tar.gz + source/digital-surface-shadow-prospective-report-source-v1.tar.gz.sha256 + retention-days: 30 From 6362cc9184058fa4780edd7d676c63d751b2807c Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 09:39:19 -0500 Subject: [PATCH 043/177] feat: export hash-chained shadow report ledger --- .../shadow-source/research/digital_surface/live/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/__init__.py b/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/__init__.py index f2110030..ca848daa 100644 --- a/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/__init__.py +++ b/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/__init__.py @@ -1,6 +1,7 @@ from .config import LiveTradingDisabledError, TradingConfig, load_config from .gateway import ShadowGateway, quantize_down from .report import ShadowCanaryMetrics, evaluate_shadow_admission +from .report_chain import ShadowReportIntegrityError, ShadowReportLedger from .risk import evaluate_pair_risk from .types import BookQuote, PreparedOrder, PreparedPair, RiskDecision, RiskState, TradeIntent @@ -11,6 +12,8 @@ "PreparedPair", "RiskDecision", "ShadowCanaryMetrics", + "ShadowReportIntegrityError", + "ShadowReportLedger", "RiskState", "ShadowGateway", "TradeIntent", From 1916ff0d4188734f3d29afc6d137f59f16a12c62 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 09:39:48 -0500 Subject: [PATCH 044/177] feat: checkpoint hash-chained shadow report ledger --- .../digital_surface/live/report_chain.py | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/report_chain.py diff --git a/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/report_chain.py b/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/report_chain.py new file mode 100644 index 00000000..8875f8d8 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/report_chain.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import hashlib +import json +from dataclasses import asdict +from pathlib import Path +from typing import Any + +from .report import ShadowCanaryMetrics, evaluate_shadow_admission + + +class ShadowReportIntegrityError(RuntimeError): + """Raised when an append-only shadow report ledger has been altered.""" + + +def _canonical_json(value: dict[str, Any]) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _sha256(name: str, value: str) -> str: + text = str(value).lower() + if len(text) != 64 or any(ch not in "0123456789abcdef" for ch in text): + raise ValueError(f"{name} must be a 64-character SHA-256 hex digest") + return text + + +class ShadowReportLedger: + """Append-only reports bound to immutable policy, source, and market-ledger state.""" + + def __init__(self, path: str | Path) -> None: + self.path = Path(path) + self.head_hash = "0" * 64 + self.rows: list[dict[str, Any]] = [] + if self.path.exists(): + self._load_and_verify() + + def _load_and_verify(self) -> None: + previous = "0" * 64 + rows: list[dict[str, Any]] = [] + for line_number, line in enumerate(self.path.read_text(encoding="utf-8").splitlines(), start=1): + if not line.strip(): + continue + try: + record = json.loads(line) + except json.JSONDecodeError as exc: + raise ShadowReportIntegrityError(f"invalid JSON at line {line_number}") from exc + stored = str(record.get("report_hash", "")) + payload = dict(record) + payload.pop("report_hash", None) + if payload.get("previous_report_hash") != previous: + raise ShadowReportIntegrityError(f"report chain mismatch at line {line_number}") + for field in ("policy_sha256", "source_sha256", "prospective_head_sha256"): + try: + _sha256(field, str(payload.get(field, ""))) + except ValueError as exc: + raise ShadowReportIntegrityError(f"invalid {field} at line {line_number}") from exc + expected = hashlib.sha256(_canonical_json(payload)).hexdigest() + if stored != expected: + raise ShadowReportIntegrityError(f"report hash mismatch at line {line_number}") + metrics = ShadowCanaryMetrics(**payload["metrics"]) + expected_decision = evaluate_shadow_admission(metrics) + if payload.get("decision") != expected_decision: + raise ShadowReportIntegrityError(f"decision mismatch at line {line_number}") + rows.append(record) + previous = stored + self.rows = rows + self.head_hash = previous + + def append( + self, + *, + generated_ts_ms: int, + metrics: ShadowCanaryMetrics, + policy_sha256: str, + source_sha256: str, + prospective_head_sha256: str, + ) -> dict[str, Any]: + timestamp = int(generated_ts_ms) + if timestamp < 0: + raise ValueError("generated_ts_ms must be non-negative") + policy = _sha256("policy_sha256", policy_sha256) + source = _sha256("source_sha256", source_sha256) + prospective = _sha256("prospective_head_sha256", prospective_head_sha256) + payload = { + "decision": evaluate_shadow_admission(metrics), + "generated_ts_ms": timestamp, + "metrics": asdict(metrics), + "policy_sha256": policy, + "previous_report_hash": self.head_hash, + "prospective_head_sha256": prospective, + "source_sha256": source, + } + report_hash = hashlib.sha256(_canonical_json(payload)).hexdigest() + record = {**payload, "report_hash": report_hash} + self.path.parent.mkdir(parents=True, exist_ok=True) + with self.path.open("a", encoding="utf-8", newline="\n") as handle: + handle.write(json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n") + handle.flush() + self.rows.append(record) + self.head_hash = report_hash + return dict(record) From c8a293e4e4625821b6c6b9ff51b8902909ca70b3 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 09:40:08 -0500 Subject: [PATCH 045/177] test: checkpoint shadow report chain integrity regressions --- .../digital_surface/live/test_report_chain.py | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/shadow-source/tests/digital_surface/live/test_report_chain.py diff --git a/ci_payload/digital-surface-live-readiness/shadow-source/tests/digital_surface/live/test_report_chain.py b/ci_payload/digital-surface-live-readiness/shadow-source/tests/digital_surface/live/test_report_chain.py new file mode 100644 index 00000000..a56cb02c --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/shadow-source/tests/digital_surface/live/test_report_chain.py @@ -0,0 +1,72 @@ +import json + +import pytest + +from research.digital_surface.live.report import ShadowCanaryMetrics +from research.digital_surface.live.report_chain import ShadowReportIntegrityError, ShadowReportLedger + + +def metrics(markets=650, qualified=120, atomic=116, orphaned=4, unresolved=0, weeks=4): + return ShadowCanaryMetrics( + markets_observed=markets, + qualified_portfolios=qualified, + simulated_atomic=atomic, + orphaned=orphaned, + unresolved_order_states=unresolved, + feed_outages=0, + weekly_passes=weeks, + ) + + +def test_appends_hash_chained_reports_bound_to_policy_source_and_prospective_head(tmp_path): + path = tmp_path / "reports.jsonl" + ledger = ShadowReportLedger(path) + first = ledger.append( + generated_ts_ms=100, + metrics=metrics(), + policy_sha256="1" * 64, + source_sha256="2" * 64, + prospective_head_sha256="3" * 64, + ) + second = ledger.append( + generated_ts_ms=200, + metrics=metrics(markets=700), + policy_sha256="1" * 64, + source_sha256="2" * 64, + prospective_head_sha256="4" * 64, + ) + assert first["decision"]["classification"] == "Admissible" + assert first["decision"]["live_mode"] == "disabled_pending_operator_release" + assert second["previous_report_hash"] == first["report_hash"] + reloaded = ShadowReportLedger(path) + assert reloaded.head_hash == second["report_hash"] + assert len(reloaded.rows) == 2 + + +def test_recomputes_decision_and_detects_tampering(tmp_path): + path = tmp_path / "reports.jsonl" + ledger = ShadowReportLedger(path) + ledger.append( + generated_ts_ms=100, + metrics=metrics(markets=499), + policy_sha256="1" * 64, + source_sha256="2" * 64, + prospective_head_sha256="3" * 64, + ) + row = json.loads(path.read_text()) + row["decision"]["classification"] = "Admissible" + path.write_text(json.dumps(row) + "\n") + with pytest.raises(ShadowReportIntegrityError, match="report hash mismatch|decision mismatch"): + ShadowReportLedger(path) + + +def test_rejects_unbound_or_invalid_digests(tmp_path): + ledger = ShadowReportLedger(tmp_path / "reports.jsonl") + with pytest.raises(ValueError, match="prospective_head_sha256"): + ledger.append( + generated_ts_ms=100, + metrics=metrics(), + policy_sha256="1" * 64, + source_sha256="2" * 64, + prospective_head_sha256="not-a-hash", + ) From 3f304eba11645a3a19602c8dd334951c619349f2 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 09:40:57 -0500 Subject: [PATCH 046/177] ci: verify hash-chained shadow report checkpoint --- ...surface-shadow-report-chain-checkpoint.yml | 194 ++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 .github/workflows/digital-surface-shadow-report-chain-checkpoint.yml diff --git a/.github/workflows/digital-surface-shadow-report-chain-checkpoint.yml b/.github/workflows/digital-surface-shadow-report-chain-checkpoint.yml new file mode 100644 index 00000000..6a3d4c3a --- /dev/null +++ b/.github/workflows/digital-surface-shadow-report-chain-checkpoint.yml @@ -0,0 +1,194 @@ +name: Digital Surface Shadow Report Chain Checkpoint + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/digital-surface-shadow-report-chain-checkpoint.yml + - ci_payload/digital-surface-live-readiness/shadow-source/** + - ci_payload/digital-surface-live-readiness/prospective-source/** + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-shadow-report-chain-checkpoint.yml + - ci_payload/digital-surface-live-readiness/shadow-source/** + - ci_payload/digital-surface-live-readiness/prospective-source/** + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + verify-shadow-report-chain: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct checksum-pinned source + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source && tar -xzf base.tar.gz -C source + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + for patch_file in timestamp-unit-fix.patch full-universe-gate-fix.patch resolved-universe-fix.patch asset-merge-fix.patch; do + patch -d source -p1 < "ci_payload/digital-surface-official-resolution-patch/${patch_file}" + done + echo 'f71f3a653ca58b4bf98609fc29d8ab16f9d031c9ed14906ebe5159f2b0de8eab ci_payload/digital-surface-live-readiness/execution-diagnostics.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/execution-diagnostics.patch + echo '8606cc925aa3b3a6417abb1fba9a11df916cc2b281880978fb6229aa25875be9 ci_payload/digital-surface-live-readiness/book-arrival-model.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/book-arrival-model.patch + cp -a ci_payload/digital-surface-live-readiness/shadow-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/prospective-source/. source/ + + cat > /tmp/checkpoint.sha256 <<'EOF' + 6aa02760d4d038098a6a69a99d9349bb5c6fef8d4593fa4c4cf0574971828fb2 source/research/digital_surface/live/__init__.py + 9c5c9bcbd39e1262ac468024f5a96fb0667c0c95fc2879d846f7ca2066c38b5f source/research/digital_surface/live/config.py + 9e6dff73a17227f907dce4bf56cfd54921f4eb677280b1774d1a73c979b9065a source/research/digital_surface/live/gateway.py + 3be74a910e787a3f9e0f3548691e63ed93d78eeb3328fd9538e3a0e643b57d1c source/research/digital_surface/live/report.py + b0ca12139437c004ec88e41902796cdd540f7e45c1b27c3aba96e3695d2267a1 source/research/digital_surface/live/report_chain.py + f9197d38fd8dcfcc073d83028ac498fd9ff07619fea2b2b6a90de9ab470a86e2 source/research/digital_surface/live/risk.py + 50f32b2e400767702073b97ba4fd77a7e8c26f5fd4c7d6f7d9aafe282d6258f0 source/research/digital_surface/live/types.py + 10d833f89194246ea2fe5dec8bb2fc4388c8509a4d8276c7fcdc0a2b1a13e76c source/tests/digital_surface/live/test_report.py + a09860af2f96e950884e1a2d37d0dfe59c41d869f3cb183ef839bbec2896db4a source/tests/digital_surface/live/test_report_chain.py + be1f59a4ae742f062b419ad7f7fb974eadf9bd076cb2ecdabcd0063eedf1df6d source/tests/digital_surface/live/test_shadow_gateway.py + f219222d7b89c5ab4ef0619da9163a77b79ba7f2a9fe611c4072746771e2fa41 source/research/digital_surface/prospective.py + 9ed734554e0296e94212cac3911466ac230241efdd4ab911e9adc74e41ef05b2 source/tests/digital_surface/test_prospective.py + EOF + sed -i 's/^ //' /tmp/checkpoint.sha256 + sha256sum -c /tmp/checkpoint.sha256 + cp /tmp/checkpoint.sha256 source/SHADOW_REPORT_CHAIN_SOURCE_SHA256.txt + + - name: Install pinned test stack + run: | + python -m pip install --upgrade pip + python -m pip install numpy==2.2.6 pandas==2.2.3 scipy==1.15.3 scikit-learn==1.6.1 pyarrow==19.0.1 requests==2.32.3 pytest==8.3.5 tabulate==0.9.0 + + - name: Compile and pass exactly 45 tests + working-directory: source + shell: bash + run: | + set -euo pipefail + python -m compileall -q research/digital_surface tests/digital_surface + PYTHONPATH=. pytest --collect-only -q tests/digital_surface | tee /tmp/collect.txt + grep -Eq '^45 tests collected' /tmp/collect.txt + PYTHONPATH=. pytest tests/digital_surface -q + + - name: Verify chained reports and physical live disable + working-directory: source + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import ast + import hashlib + import json + import tempfile + from pathlib import Path + + from research.digital_surface.live import ( + LiveTradingDisabledError, + ShadowCanaryMetrics, + ShadowReportLedger, + load_config, + ) + from research.digital_surface.prospective import ProspectiveLedger + + try: + load_config({'TRADING_MODE': 'live', 'LIVE_ARM_TOKEN': 'present'}) + except LiveTradingDisabledError: + pass + else: + raise AssertionError('live mode unexpectedly armed') + + forbidden = {'requests', 'httpx', 'aiohttp', 'websocket', 'websockets', 'socket'} + imports = set() + for path in Path('research/digital_surface/live').glob('*.py'): + tree = ast.parse(path.read_text(), filename=str(path)) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imports.update(alias.name.split('.')[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + imports.add(node.module.split('.')[0]) + assert not imports.intersection(forbidden), imports.intersection(forbidden) + + with tempfile.TemporaryDirectory() as directory: + directory = Path(directory) + prospective = ProspectiveLedger(directory/'prospective.jsonl', diagnostic_market_ids={'old'}) + observed = prospective.append_observation( + market_id='fresh', first_seen_ts_ms=1, + policy_sha256='1'*64, source_sha256='2'*64, + ) + reports = ShadowReportLedger(directory/'reports.jsonl') + report = reports.append( + generated_ts_ms=2, + metrics=ShadowCanaryMetrics( + markets_observed=650, + qualified_portfolios=120, + simulated_atomic=116, + orphaned=4, + unresolved_order_states=0, + feed_outages=1, + weekly_passes=4, + ), + policy_sha256='1'*64, + source_sha256='2'*64, + prospective_head_sha256=observed['record_hash'], + ) + assert report['decision']['classification'] == 'Admissible' + assert report['decision']['live_mode'] == 'disabled_pending_operator_release' + assert ShadowReportLedger(reports.path).head_hash == report['report_hash'] + + files = {} + for line in Path('SHADOW_REPORT_CHAIN_SOURCE_SHA256.txt').read_text().splitlines(): + wanted, name = line.split(maxsplit=1) + got = hashlib.sha256(Path(name).read_bytes()).hexdigest() + assert got == wanted + files[name] = got + result = { + 'tests_collected': 45, + 'live_mode': 'physically_disabled', + 'network_capable_imports': [], + 'prospective_hash_chain_verified': True, + 'shadow_report_hash_chain_verified': True, + 'report_bound_to_policy_source_and_prospective_head': True, + 'admissible_report_still_requires_separate_operator_release': True, + 'source_files_verified': files, + } + Path('SHADOW_REPORT_CHAIN_VERIFICATION.json').write_text(json.dumps(result, indent=2, sort_keys=True)) + print(json.dumps(result, indent=2)) + PY + + - name: Build deterministic source checkpoint + working-directory: source + shell: bash + run: | + set -euo pipefail + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-shadow-report-chain-source-v1.tar.gz \ + research tests SHADOW_REPORT_CHAIN_SOURCE_SHA256.txt SHADOW_REPORT_CHAIN_VERIFICATION.json + sha256sum digital-surface-shadow-report-chain-source-v1.tar.gz > digital-surface-shadow-report-chain-source-v1.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: digital-surface-shadow-report-chain-checkpoint-v1 + path: | + source/SHADOW_REPORT_CHAIN_SOURCE_SHA256.txt + source/SHADOW_REPORT_CHAIN_VERIFICATION.json + source/digital-surface-shadow-report-chain-source-v1.tar.gz + source/digital-surface-shadow-report-chain-source-v1.tar.gz.sha256 + retention-days: 30 From 62df83003ff9c90925700ade7bc48e021aea87ba Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 09:43:40 -0500 Subject: [PATCH 047/177] docs: checkpoint 45-test live-readiness status --- .../digital-surface-live-readiness/STATUS.md | 71 +++++++++++++++---- 1 file changed, 57 insertions(+), 14 deletions(-) diff --git a/ci_payload/digital-surface-live-readiness/STATUS.md b/ci_payload/digital-surface-live-readiness/STATUS.md index 660adf76..834c3d0a 100644 --- a/ci_payload/digital-surface-live-readiness/STATUS.md +++ b/ci_payload/digital-surface-live-readiness/STATUS.md @@ -1,21 +1,64 @@ -# Status +# Digital Surface Live-Readiness Status -Current checkpoint: deterministic execution attribution. +## Durable checkpoint -The implementation plan is committed at `docs/superpowers/plans/2026-07-25-digital-surface-live-readiness.md`. +- Repository: `Chrisbryan17/hello-world` +- Branch: `digital-surface-live-readiness-20260725` +- Draft PR: #59 +- Source checkpoint before this status commit: `3f304eba11645a3a19602c8dd334951c619349f2` +- Real-money execution: **physically disabled** -Completed locally before this checkpoint: +## Verified historical result -- failing diagnostics test observed; -- minimal attribution implementation added; -- exactly 30 tests collected; -- all 30 tests passed; -- diagnostics patch applied cleanly to the exact v5 source archive. +The checksum-pinned v5 research completed the official-resolution source gate and all four historical transfer folds, but the strategy remained **Rejected**. Those four weeks are now diagnostic and may never be reused as prospective admission evidence. -Pending clean-room evidence: +## Execution diagnosis -- complete leg attribution over the immutable corpus; -- byte-identical frozen candidates, fills, fold metrics, and admission; -- reason distribution by leg and week. +Across 224 candidate portfolios / 448 legs: -No live orders are enabled or submitted. +- insufficient post-arrival tape: 186 legs +- ask above frozen limit: 114 legs +- missing post-arrival book: 110 legs +- filled: 34 legs +- insufficient displayed depth: 4 legs + +The original tape-confirmed model produced 8 atomic portfolios. A diagnostic post-latency book-arrival replay produced 62 atomic portfolios, with fold counts `[11, 17, 29, 5]`. This alternative is **not** treated as live truth and requires prospective shadow calibration. + +## Current source state + +The branch contains checksum-pinned, human-readable recovery payloads for: + +1. deterministic execution-failure attribution; +2. explicit tape-confirmed versus book-arrival replay modes; +3. a shadow-only two-leg FOK preparation gateway using exact `Decimal` tick quantization; +4. fail-closed risk controls for stale feeds/books, daily loss, unresolved orders, orphan exposure, token mismatch, depth, limits, expiry, and ambiguous pair state; +5. an append-only prospective market ledger with canonical SHA-256 chaining and contamination rejection; +6. shadow-canary admission gates requiring at least 500 markets, 100 qualified portfolios, four untouched weekly passes, zero unresolved order states, and at most 5% orphaning; +7. an append-only shadow-report ledger bound to policy SHA-256, source SHA-256, and prospective-ledger head SHA-256. + +## Test evidence + +- Research plus diagnostics and book-arrival source: 32 tests. +- Shadow gateway and prospective ledger checkpoint: 39 tests. +- Shadow canary report checkpoint: 42 tests. +- Shadow report-chain checkpoint: expected **45 tests** in the latest clean-room workflow. +- Focused local safety/prospective/report/report-chain suite: 13 passed. + +## Active workflows + +- `Digital Surface Book-Arrival Comparison` +- `Digital Surface Shadow Report Chain Checkpoint` + +The repository also retains historical workflows; they may queue but are not authoritative for this checkpoint. + +## Admission boundary + +No real-money order submission implementation exists in the live package. `TRADING_MODE=live` raises `LiveTradingDisabledError` even if credentials and an arm token are supplied. A future result may be classified `Admissible` only from genuinely new prospective markets, but even an admissible report leaves live mode at `disabled_pending_operator_release` until a separate reviewed release and explicit operator action. + +## Next work + +1. obtain green clean-room artifacts for the 45-test source checkpoint and book-arrival comparison; +2. independently verify outer artifact digests, deterministic TAR sidecars, source hashes, and report/market hash chains; +3. implement the scheduled shadow collector and append-only artifact persistence without credentials; +4. observe at least 500 new markets and four untouched weekly passes; +5. keep real-money execution disabled unless all gates clear and a separately reviewed release is explicitly armed. From 00f55ddf4c00efbd13eabc6cf7290db8d015f1c6 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 10:00:21 -0500 Subject: [PATCH 048/177] feat: checkpoint credential-free prospective market observer --- .../digital_surface/diagnostic_filter.py | 106 ++++++++++++++ .../digital_surface/market_discovery.py | 104 ++++++++++++++ .../digital_surface/public_observer.py | 130 ++++++++++++++++++ .../shadow_observation_policy_v1.json | 11 ++ .../digital_surface/test_diagnostic_filter.py | 38 +++++ .../digital_surface/test_market_discovery.py | 45 ++++++ .../digital_surface/test_public_observer.py | 63 +++++++++ .../research/digital_surface/prospective.py | 19 ++- .../tests/digital_surface/test_prospective.py | 14 ++ 9 files changed, 527 insertions(+), 3 deletions(-) create mode 100644 ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/diagnostic_filter.py create mode 100644 ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/market_discovery.py create mode 100644 ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/public_observer.py create mode 100644 ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/shadow_observation_policy_v1.json create mode 100644 ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_diagnostic_filter.py create mode 100644 ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_market_discovery.py create mode 100644 ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_public_observer.py diff --git a/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/diagnostic_filter.py b/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/diagnostic_filter.py new file mode 100644 index 00000000..a95e97ae --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/diagnostic_filter.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import base64 +import hashlib +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable + + +def _sha256(name: str, value: str) -> str: + text = str(value).strip().lower() + if len(text) != 64 or any(ch not in "0123456789abcdef" for ch in text): + raise ValueError(f"{name} must be a 64-character SHA-256 hex digest") + return text + + +def _normalise(value: str) -> bytes: + return str(value).strip().lower().encode("utf-8") + + +def _positions(value: str, *, m_bits: int, k_hashes: int) -> tuple[int, ...]: + digest = hashlib.sha256(_normalise(value)).digest() + h1 = int.from_bytes(digest[:8], "big") + h2 = int.from_bytes(digest[8:16], "big") or 0x9E3779B97F4A7C15 + return tuple((h1 + index * h2) % m_bits for index in range(k_hashes)) + + +def build_diagnostic_bloom( + market_ids: Iterable[str], + *, + source_sha256: str, + m_bits: int = 262_144, + k_hashes: int = 7, +) -> dict[str, object]: + if m_bits <= 0 or m_bits % 8: + raise ValueError("m_bits must be a positive multiple of 8") + if k_hashes <= 0: + raise ValueError("k_hashes must be positive") + ids = sorted({str(value).strip().lower() for value in market_ids if str(value).strip()}) + bitset = bytearray(m_bits // 8) + for value in ids: + for position in _positions(value, m_bits=m_bits, k_hashes=k_hashes): + bitset[position // 8] |= 1 << (position % 8) + raw = bytes(bitset) + return { + "algorithm": "sha256-double-hash-v1", + "bitset_base64": base64.b64encode(raw).decode("ascii"), + "bitset_sha256": hashlib.sha256(raw).hexdigest(), + "item_count": len(ids), + "k_hashes": int(k_hashes), + "m_bits": int(m_bits), + "source_sha256": _sha256("source_sha256", source_sha256), + "version": 1, + } + + +@dataclass(frozen=True, slots=True) +class DiagnosticBloomFilter: + bitset: bytes + m_bits: int + k_hashes: int + item_count: int + source_sha256: str + + @classmethod + def from_payload(cls, payload: dict[str, object]) -> "DiagnosticBloomFilter": + if int(payload.get("version", -1)) != 1: + raise ValueError("unsupported diagnostic Bloom filter version") + if payload.get("algorithm") != "sha256-double-hash-v1": + raise ValueError("unsupported diagnostic Bloom filter algorithm") + m_bits = int(payload["m_bits"]) + k_hashes = int(payload["k_hashes"]) + item_count = int(payload["item_count"]) + if m_bits <= 0 or m_bits % 8: + raise ValueError("m_bits must be a positive multiple of 8") + if k_hashes <= 0 or item_count < 0: + raise ValueError("invalid Bloom filter dimensions") + try: + bitset = base64.b64decode(str(payload["bitset_base64"]), validate=True) + except Exception as exc: + raise ValueError("invalid Bloom filter base64") from exc + if len(bitset) != m_bits // 8: + raise ValueError("Bloom filter bitset length mismatch") + expected = _sha256("bitset_sha256", str(payload["bitset_sha256"])) + got = hashlib.sha256(bitset).hexdigest() + if got != expected: + raise ValueError(f"bitset SHA-256 mismatch: {got} != {expected}") + return cls( + bitset=bitset, + m_bits=m_bits, + k_hashes=k_hashes, + item_count=item_count, + source_sha256=_sha256("source_sha256", str(payload["source_sha256"])), + ) + + @classmethod + def from_path(cls, path: str | Path) -> "DiagnosticBloomFilter": + return cls.from_payload(json.loads(Path(path).read_text(encoding="utf-8"))) + + def __contains__(self, market_id: object) -> bool: + value = str(market_id) + return all( + bool(self.bitset[position // 8] & (1 << (position % 8))) + for position in _positions(value, m_bits=self.m_bits, k_hashes=self.k_hashes) + ) diff --git a/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/market_discovery.py b/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/market_discovery.py new file mode 100644 index 00000000..4f9ff8e2 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/market_discovery.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from decimal import Decimal +from typing import Any, Callable, Mapping, Sequence + + +_TARGET_SLUG = re.compile(r"^btc-updown-(5m|15m)-(\d{10,13})$") + + +def _bool(value: object) -> bool: + if isinstance(value, bool): + return value + return str(value).strip().lower() in {"1", "true", "yes"} + + +def _list(value: object, name: str) -> list[str]: + parsed = json.loads(value) if isinstance(value, str) else value + if not isinstance(parsed, Sequence) or isinstance(parsed, (str, bytes)): + raise ValueError(f"{name} must be a JSON array or sequence") + return [str(item) for item in parsed] + + +@dataclass(frozen=True, slots=True) +class GammaMarketRecord: + condition_id: str + slug: str + question: str + yes_token_id: str + no_token_id: str + duration_seconds: int + open_epoch_seconds: int + end_date: str + tick_size: Decimal + + +def parse_gamma_market(payload: Mapping[str, Any]) -> GammaMarketRecord: + condition_id = str(payload.get("conditionId") or payload.get("condition_id") or "").strip() + slug = str(payload.get("slug") or "").strip().lower() + match = _TARGET_SLUG.fullmatch(slug) + if not condition_id or match is None: + raise ValueError("market is not a BTC 5m/15m up/down contract") + token_ids = _list(payload.get("clobTokenIds") or payload.get("clob_token_ids"), "clobTokenIds") + if len(token_ids) != 2 or not all(token_ids): + raise ValueError("clobTokenIds must contain exactly two token IDs") + outcomes = _list(payload.get("outcomes", '["Up","Down"]'), "outcomes") + if len(outcomes) != 2: + raise ValueError("outcomes must contain exactly two values") + first = outcomes[0].strip().lower() + second = outcomes[1].strip().lower() + if first not in {"up", "yes"} or second not in {"down", "no"}: + raise ValueError(f"unexpected outcome order: {outcomes}") + epoch = int(match.group(2)) + if epoch >= 100_000_000_000: + epoch //= 1000 + return GammaMarketRecord( + condition_id=condition_id, + slug=slug, + question=str(payload.get("question") or ""), + yes_token_id=token_ids[0], + no_token_id=token_ids[1], + duration_seconds=300 if match.group(1) == "5m" else 900, + open_epoch_seconds=epoch, + end_date=str(payload.get("endDate") or payload.get("end_date") or ""), + tick_size=Decimal(str(payload.get("orderPriceMinTickSize") or payload.get("order_price_min_tick_size") or "0.01")), + ) + + +def _is_target(payload: Mapping[str, Any]) -> bool: + slug = str(payload.get("slug") or "").strip().lower() + return ( + _TARGET_SLUG.fullmatch(slug) is not None + and _bool(payload.get("active", False)) + and not _bool(payload.get("closed", False)) + and _bool(payload.get("enableOrderBook", payload.get("enable_order_book", False))) + ) + + +def discover_target_markets( + fetch_page: Callable[[dict[str, object]], Sequence[Mapping[str, Any]]], + *, + page_size: int = 100, + max_pages: int = 20, +) -> list[GammaMarketRecord]: + if page_size <= 0 or max_pages <= 0: + raise ValueError("page_size and max_pages must be positive") + records: dict[str, GammaMarketRecord] = {} + for page in range(max_pages): + offset = page * page_size + payloads = list(fetch_page({ + "active": "true", + "closed": "false", + "limit": page_size, + "offset": offset, + })) + for payload in payloads: + if _is_target(payload): + record = parse_gamma_market(payload) + records.setdefault(record.condition_id, record) + if len(payloads) < page_size: + break + return sorted(records.values(), key=lambda row: (row.open_epoch_seconds, row.duration_seconds, row.condition_id)) diff --git a/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/public_observer.py b/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/public_observer.py new file mode 100644 index 00000000..10330a41 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/public_observer.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import time +from pathlib import Path +from typing import Any, Mapping, Sequence + +from .diagnostic_filter import DiagnosticBloomFilter +from .market_discovery import GammaMarketRecord, discover_target_markets +from .prospective import ProspectiveLedger + + +GAMMA_MARKETS_URL = "https://gamma-api.polymarket.com/markets" + + +def fetch_gamma_page(params: Mapping[str, object], *, timeout_seconds: float = 15.0) -> Sequence[Mapping[str, Any]]: + """Fetch a public Gamma page. No credentials or authenticated endpoint is used.""" + import requests + + response = requests.get(GAMMA_MARKETS_URL, params=dict(params), timeout=timeout_seconds) + response.raise_for_status() + payload = response.json() + if not isinstance(payload, list): + raise ValueError("Gamma markets response must be a list") + return payload + + +def _metadata(record: GammaMarketRecord) -> dict[str, object]: + return { + "duration_seconds": int(record.duration_seconds), + "end_date": record.end_date, + "no_token_id": record.no_token_id, + "open_epoch_seconds": int(record.open_epoch_seconds), + "question": record.question, + "slug": record.slug, + "tick_size": str(record.tick_size), + "yes_token_id": record.yes_token_id, + } + + +def observe_markets( + markets: Sequence[GammaMarketRecord], + *, + ledger: ProspectiveLedger, + observed_ts_ms: int, + policy_sha256: str, + source_sha256: str, +) -> dict[str, object]: + counters = { + "discovered": len(markets), + "appended": 0, + "skipped_diagnostic": 0, + "skipped_existing": 0, + } + appended: list[str] = [] + for market in markets: + condition_id = str(market.condition_id) + if condition_id in ledger.diagnostic_market_ids: + counters["skipped_diagnostic"] += 1 + continue + if condition_id in ledger.market_ids: + counters["skipped_existing"] += 1 + continue + ledger.append_observation( + market_id=condition_id, + first_seen_ts_ms=int(observed_ts_ms), + policy_sha256=policy_sha256, + source_sha256=source_sha256, + metadata=_metadata(market), + ) + appended.append(condition_id) + counters["appended"] += 1 + return { + **counters, + "appended_condition_ids": appended, + "ledger_head_sha256": ledger.head_hash, + "markets_total_in_ledger": len(ledger.market_ids), + "observed_ts_ms": int(observed_ts_ms), + } + + +def _sha256_file(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Observe public BTC 5m/15m Polymarket markets without credentials") + parser.add_argument("--diagnostic-filter", type=Path, required=True) + parser.add_argument("--ledger", type=Path, required=True) + parser.add_argument("--policy", type=Path, required=True) + parser.add_argument("--source-manifest", type=Path, required=True) + parser.add_argument("--summary", type=Path, required=True) + parser.add_argument("--page-size", type=int, default=100) + parser.add_argument("--max-pages", type=int, default=20) + args = parser.parse_args(argv) + + bloom = DiagnosticBloomFilter.from_path(args.diagnostic_filter) + policy_sha256 = _sha256_file(args.policy) + source_sha256 = _sha256_file(args.source_manifest) + ledger = ProspectiveLedger(args.ledger, diagnostic_market_ids=bloom) + markets = discover_target_markets( + lambda params: fetch_gamma_page(params), + page_size=args.page_size, + max_pages=args.max_pages, + ) + summary = observe_markets( + markets, + ledger=ledger, + observed_ts_ms=int(time.time() * 1000), + policy_sha256=policy_sha256, + source_sha256=source_sha256, + ) + summary.update({ + "diagnostic_filter_source_sha256": bloom.source_sha256, + "policy_sha256": policy_sha256, + "source_sha256": source_sha256, + "trading_mode": "observation_only", + "authenticated_requests": 0, + "order_submissions": 0, + }) + args.summary.parent.mkdir(parents=True, exist_ok=True) + args.summary.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/shadow_observation_policy_v1.json b/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/shadow_observation_policy_v1.json new file mode 100644 index 00000000..48b8409e --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/shadow_observation_policy_v1.json @@ -0,0 +1,11 @@ +{ + "authenticated_requests": false, + "market_endpoint": "https://gamma-api.polymarket.com/markets", + "mode": "observation_only", + "order_submission": false, + "target_slugs": [ + "btc-updown-5m-", + "btc-updown-15m-" + ], + "version": 1 +} diff --git a/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_diagnostic_filter.py b/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_diagnostic_filter.py new file mode 100644 index 00000000..aa09d99f --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_diagnostic_filter.py @@ -0,0 +1,38 @@ +import json + +import pytest + +from research.digital_surface.diagnostic_filter import DiagnosticBloomFilter, build_diagnostic_bloom +from research.digital_surface.prospective import ProspectiveContaminationError, ProspectiveLedger + + +def test_bloom_round_trip_has_no_false_negatives_for_supplied_ids(tmp_path): + ids = [f"0x{index:064x}" for index in range(100)] + payload = build_diagnostic_bloom(ids, source_sha256="a" * 64, m_bits=4096, k_hashes=5) + path = tmp_path / "filter.json" + path.write_text(json.dumps(payload)) + loaded = DiagnosticBloomFilter.from_path(path) + assert all(value in loaded for value in ids) + assert loaded.item_count == 100 + assert loaded.source_sha256 == "a" * 64 + + +def test_bloom_can_enforce_prospective_contamination_rejection(tmp_path): + known = "0x" + "1" * 64 + payload = build_diagnostic_bloom([known], source_sha256="a" * 64, m_bits=1024, k_hashes=4) + bloom = DiagnosticBloomFilter.from_payload(payload) + ledger = ProspectiveLedger(tmp_path / "ledger.jsonl", diagnostic_market_ids=bloom) + with pytest.raises(ProspectiveContaminationError, match="appeared in diagnostic data"): + ledger.append_observation( + market_id=known, + first_seen_ts_ms=1, + policy_sha256="b" * 64, + source_sha256="c" * 64, + ) + + +def test_rejects_corrupt_filter_bitset(): + payload = build_diagnostic_bloom(["known"], source_sha256="a" * 64, m_bits=1024, k_hashes=4) + payload["bitset_sha256"] = "0" * 64 + with pytest.raises(ValueError, match="bitset SHA-256 mismatch"): + DiagnosticBloomFilter.from_payload(payload) diff --git a/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_market_discovery.py b/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_market_discovery.py new file mode 100644 index 00000000..71370e9d --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_market_discovery.py @@ -0,0 +1,45 @@ +import json + +from research.digital_surface.market_discovery import discover_target_markets, parse_gamma_market + + +def market(slug, condition, token_ids, *, active=True, closed=False, enable=True): + return { + "conditionId": condition, + "slug": slug, + "question": "Bitcoin Up or Down", + "active": active, + "closed": closed, + "enableOrderBook": enable, + "clobTokenIds": json.dumps(token_ids), + "outcomes": '["Up", "Down"]', + "orderPriceMinTickSize": 0.01, + "endDate": "2026-07-25T15:00:00Z", + } + + +def test_parses_gamma_token_order_and_target_duration(): + record = parse_gamma_market(market("btc-updown-5m-1774450800", "c1", ["yes", "no"])) + assert record.condition_id == "c1" + assert record.yes_token_id == "yes" + assert record.no_token_id == "no" + assert record.duration_seconds == 300 + assert record.open_epoch_seconds == 1774450800 + + +def test_discovery_filters_non_target_closed_and_duplicate_markets(): + pages = { + 0: [ + market("btc-updown-5m-1774450800", "c1", ["yes1", "no1"]), + market("btc-updown-15m-1774450800", "c2", ["yes2", "no2"]), + market("eth-updown-5m-1774450800", "c3", ["yes3", "no3"]), + market("btc-updown-5m-1774451100", "c4", ["yes4", "no4"], closed=True), + ], + 4: [market("btc-updown-5m-1774450800", "c1", ["yes1", "no1"])], + } + + def fetch(params): + return pages.get(params["offset"], []) + + found = discover_target_markets(fetch, page_size=4, max_pages=3) + assert [row.condition_id for row in found] == ["c1", "c2"] diff --git a/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_public_observer.py b/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_public_observer.py new file mode 100644 index 00000000..305e2058 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_public_observer.py @@ -0,0 +1,63 @@ +import json + +from research.digital_surface.diagnostic_filter import DiagnosticBloomFilter, build_diagnostic_bloom +from research.digital_surface.market_discovery import GammaMarketRecord +from research.digital_surface.prospective import ProspectiveLedger +from research.digital_surface.public_observer import observe_markets + + +def record(condition, *, duration=300, epoch=1774450800): + return GammaMarketRecord( + condition_id=condition, + slug=f"btc-updown-{duration // 60}m-{epoch}", + question="Bitcoin Up or Down", + yes_token_id=f"yes-{condition}", + no_token_id=f"no-{condition}", + duration_seconds=duration, + open_epoch_seconds=epoch, + end_date="2026-07-25T15:00:00Z", + tick_size=__import__("decimal").Decimal("0.01"), + ) + + +def test_observer_rejects_historical_and_appends_full_new_market_metadata(tmp_path): + known = "known" + bloom = DiagnosticBloomFilter.from_payload( + build_diagnostic_bloom([known], source_sha256="a" * 64, m_bits=1024, k_hashes=4) + ) + ledger = ProspectiveLedger(tmp_path / "ledger.jsonl", diagnostic_market_ids=bloom) + summary = observe_markets( + [record(known), record("fresh")], + ledger=ledger, + observed_ts_ms=123, + policy_sha256="b" * 64, + source_sha256="c" * 64, + ) + assert summary["discovered"] == 2 + assert summary["appended"] == 1 + assert summary["skipped_diagnostic"] == 1 + row = json.loads((tmp_path / "ledger.jsonl").read_text()) + assert row["market_id"] == "fresh" + assert row["metadata"]["yes_token_id"] == "yes-fresh" + assert row["metadata"]["no_token_id"] == "no-fresh" + assert row["metadata"]["duration_seconds"] == 300 + assert row["metadata"]["tick_size"] == "0.01" + + +def test_observer_is_idempotent_for_already_observed_markets(tmp_path): + bloom = DiagnosticBloomFilter.from_payload( + build_diagnostic_bloom([], source_sha256="a" * 64, m_bits=1024, k_hashes=4) + ) + ledger = ProspectiveLedger(tmp_path / "ledger.jsonl", diagnostic_market_ids=bloom) + first = observe_markets( + [record("fresh")], ledger=ledger, observed_ts_ms=123, + policy_sha256="b" * 64, source_sha256="c" * 64, + ) + second = observe_markets( + [record("fresh")], ledger=ledger, observed_ts_ms=124, + policy_sha256="b" * 64, source_sha256="c" * 64, + ) + assert first["appended"] == 1 + assert second["appended"] == 0 + assert second["skipped_existing"] == 1 + assert len((tmp_path / "ledger.jsonl").read_text().splitlines()) == 1 diff --git a/ci_payload/digital-surface-live-readiness/prospective-source/research/digital_surface/prospective.py b/ci_payload/digital-surface-live-readiness/prospective-source/research/digital_surface/prospective.py index 4b062181..59844fca 100644 --- a/ci_payload/digital-surface-live-readiness/prospective-source/research/digital_surface/prospective.py +++ b/ci_payload/digital-surface-live-readiness/prospective-source/research/digital_surface/prospective.py @@ -3,7 +3,8 @@ import hashlib import json from pathlib import Path -from typing import Any, Iterable +from collections.abc import Container, Iterable +from typing import Any, Mapping class ProspectiveContaminationError(RuntimeError): @@ -26,9 +27,14 @@ def _validate_sha256(name: str, value: str) -> str: class ProspectiveLedger: - def __init__(self, path: str | Path, *, diagnostic_market_ids: Iterable[str]) -> None: + def __init__(self, path: str | Path, *, diagnostic_market_ids: Iterable[str] | Container[str]) -> None: self.path = Path(path) - self.diagnostic_market_ids = {str(value) for value in diagnostic_market_ids} + if isinstance(diagnostic_market_ids, (str, bytes)): + raise TypeError("diagnostic_market_ids must be an iterable/container of IDs, not a string") + if isinstance(diagnostic_market_ids, Container): + self.diagnostic_market_ids = diagnostic_market_ids + else: + self.diagnostic_market_ids = {str(value) for value in diagnostic_market_ids} self._rows: list[dict[str, Any]] = [] self.market_ids: set[str] = set() self.head_hash = "0" * 64 @@ -73,6 +79,7 @@ def append_observation( first_seen_ts_ms: int, policy_sha256: str, source_sha256: str, + metadata: Mapping[str, Any] | None = None, ) -> dict[str, Any]: market = str(market_id) if not market: @@ -94,6 +101,12 @@ def append_observation( "source_sha256": source, "state": "observed", } + if metadata is not None: + try: + canonical_metadata = json.loads(_canonical_json(dict(metadata)).decode("utf-8")) + except (TypeError, ValueError) as exc: + raise ValueError("metadata must be JSON serializable") from exc + payload["metadata"] = canonical_metadata record = {**payload, "record_hash": hashlib.sha256(_canonical_json(payload)).hexdigest()} self.path.parent.mkdir(parents=True, exist_ok=True) with self.path.open("a", encoding="utf-8", newline="\n") as handle: diff --git a/ci_payload/digital-surface-live-readiness/prospective-source/tests/digital_surface/test_prospective.py b/ci_payload/digital-surface-live-readiness/prospective-source/tests/digital_surface/test_prospective.py index 36817f92..0df6a5f3 100644 --- a/ci_payload/digital-surface-live-readiness/prospective-source/tests/digital_surface/test_prospective.py +++ b/ci_payload/digital-surface-live-readiness/prospective-source/tests/digital_surface/test_prospective.py @@ -78,3 +78,17 @@ def test_requires_fixed_length_source_and_policy_hashes(tmp_path): policy_sha256="short", source_sha256="2" * 64, ) + + +def test_preserves_canonical_market_metadata_in_hash_chain(tmp_path): + path = tmp_path / "prospective.jsonl" + ledger = ProspectiveLedger(path, diagnostic_market_ids=set()) + record = ledger.append_observation( + market_id="new-meta", + first_seen_ts_ms=10, + policy_sha256="1" * 64, + source_sha256="2" * 64, + metadata={"slug": "btc-updown-5m-10", "tick_size": "0.01"}, + ) + assert record["metadata"]["slug"] == "btc-updown-5m-10" + assert ProspectiveLedger(path, diagnostic_market_ids=set()).head_hash == record["record_hash"] From 062c54187bd19be1a9036cd2fc59d901be884606 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 10:04:50 -0500 Subject: [PATCH 049/177] ci: verify credential-free prospective observer checkpoint --- ...tal-surface-public-observer-checkpoint.yml | 271 ++++++++++++++++++ 1 file changed, 271 insertions(+) create mode 100644 .github/workflows/digital-surface-public-observer-checkpoint.yml diff --git a/.github/workflows/digital-surface-public-observer-checkpoint.yml b/.github/workflows/digital-surface-public-observer-checkpoint.yml new file mode 100644 index 00000000..47abed6c --- /dev/null +++ b/.github/workflows/digital-surface-public-observer-checkpoint.yml @@ -0,0 +1,271 @@ +name: Digital Surface Public Observer Checkpoint + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/digital-surface-public-observer-checkpoint.yml + - ci_payload/digital-surface-live-readiness/observer-source/** + - ci_payload/digital-surface-live-readiness/prospective-source/** + - ci_payload/digital-surface-live-readiness/shadow-source/** + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-public-observer-checkpoint.yml + - ci_payload/digital-surface-live-readiness/observer-source/** + - ci_payload/digital-surface-live-readiness/prospective-source/** + - ci_payload/digital-surface-live-readiness/shadow-source/** + workflow_dispatch: + +permissions: + contents: read + actions: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + V5_ARTIFACT_ID: '8615107471' + V5_ARTIFACT_ZIP_SHA256: '479a25e235841d94077b3d2df070c8ed46bb736c7ebd8f9f1bde8016198a9c39' + CONTRACT_INVENTORY_SHA256: 'cea26cf5221e81b90ba311317f31e97b54e4110f0f1c716fcb14f06d9f1651f3' + EXPECTED_BLOOM_BITSET_SHA256: '5fd0bc41ab1ae15ddaf9ca33354d803ab74c112d19136240f00d77156186d113' + +jobs: + verify-public-observer: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct checksum-pinned source + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source && tar -xzf base.tar.gz -C source + + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + for patch_file in timestamp-unit-fix.patch full-universe-gate-fix.patch resolved-universe-fix.patch asset-merge-fix.patch; do + patch -d source -p1 < "ci_payload/digital-surface-official-resolution-patch/${patch_file}" + done + echo 'f71f3a653ca58b4bf98609fc29d8ab16f9d031c9ed14906ebe5159f2b0de8eab ci_payload/digital-surface-live-readiness/execution-diagnostics.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/execution-diagnostics.patch + echo '8606cc925aa3b3a6417abb1fba9a11df916cc2b281880978fb6229aa25875be9 ci_payload/digital-surface-live-readiness/book-arrival-model.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/book-arrival-model.patch + + cp -a ci_payload/digital-surface-live-readiness/shadow-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/prospective-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/observer-source/. source/ + + cat > /tmp/observer-source.sha256 <<'EOF' + 6aa02760d4d038098a6a69a99d9349bb5c6fef8d4593fa4c4cf0574971828fb2 source/research/digital_surface/live/__init__.py + 9c5c9bcbd39e1262ac468024f5a96fb0667c0c95fc2879d846f7ca2066c38b5f source/research/digital_surface/live/config.py + 9e6dff73a17227f907dce4bf56cfd54921f4eb677280b1774d1a73c979b9065a source/research/digital_surface/live/gateway.py + 3be74a910e787a3f9e0f3548691e63ed93d78eeb3328fd9538e3a0e643b57d1c source/research/digital_surface/live/report.py + b0ca12139437c004ec88e41902796cdd540f7e45c1b27c3aba96e3695d2267a1 source/research/digital_surface/live/report_chain.py + f9197d38fd8dcfcc073d83028ac498fd9ff07619fea2b2b6a90de9ab470a86e2 source/research/digital_surface/live/risk.py + 50f32b2e400767702073b97ba4fd77a7e8c26f5fd4c7d6f7d9aafe282d6258f0 source/research/digital_surface/live/types.py + 10d833f89194246ea2fe5dec8bb2fc4388c8509a4d8276c7fcdc0a2b1a13e76c source/tests/digital_surface/live/test_report.py + a09860af2f96e950884e1a2d37d0dfe59c41d869f3cb183ef839bbec2896db4a source/tests/digital_surface/live/test_report_chain.py + be1f59a4ae742f062b419ad7f7fb974eadf9bd076cb2ecdabcd0063eedf1df6d source/tests/digital_surface/live/test_shadow_gateway.py + f3f1e2417305cd0ceee4b6971e461e488960f04f0e0b80ad292733a2a4b248af source/research/digital_surface/diagnostic_filter.py + efd186a4ca00c2e920bc87b8d137ac46db52f2aa36385a275dc99e70b5618439 source/research/digital_surface/market_discovery.py + 3fbb96286445b60b0b7deeb5e1f9ee45f53bfd5efdcc7a727b50617b19463224 source/research/digital_surface/prospective.py + 73c5aaa7a4ca10baf85886eb28c6135d975eafaaab19e5193819d1ccbf82f2be source/research/digital_surface/public_observer.py + ea56803a606ff33b7231b373141df3d22c159cedc1063603fef3dfb6255c60c9 source/research/digital_surface/shadow_observation_policy_v1.json + 774575a18473899bfd2304f93be3786c60c0288fd28aa98d94a40aa043c74858 source/tests/digital_surface/test_diagnostic_filter.py + adf17ffb26a23332194eaa67c07201e713f2edf81927365cb54b21a348b5a7bd source/tests/digital_surface/test_market_discovery.py + 6efbf535ddfb50ca4b88209ecfa72c8e208a9f4049b0960781805cb888449b53 source/tests/digital_surface/test_prospective.py + 87e32c6fc215d8ffe8f587357acd1d3efc12b1c78021f2f01740ebc06b9a0554 source/tests/digital_surface/test_public_observer.py + EOF + sed -i 's/^ //' /tmp/observer-source.sha256 + sha256sum -c /tmp/observer-source.sha256 + cp /tmp/observer-source.sha256 source/PUBLIC_OBSERVER_SOURCE_SHA256.txt + + - name: Install pinned test stack + run: | + python -m pip install --upgrade pip + python -m pip install numpy==2.2.6 pandas==2.2.3 scipy==1.15.3 scikit-learn==1.6.1 pyarrow==19.0.1 requests==2.32.3 pytest==8.3.5 tabulate==0.9.0 + + - name: Download immutable v5 inventory + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + curl --fail --location --silent --show-error \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H 'Accept: application/vnd.github+json' \ + -H 'X-GitHub-Api-Version: 2022-11-28' \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/artifacts/${V5_ARTIFACT_ID}/zip" \ + -o v5.zip + echo "${V5_ARTIFACT_ZIP_SHA256} v5.zip" | sha256sum -c - + mkdir v5 && unzip -q v5.zip -d v5 + inventory=$(find v5 -type f -name contract_inventory.csv -print -quit) + test -n "${inventory}" + cp "${inventory}" source/contract_inventory.csv + echo "${CONTRACT_INVENTORY_SHA256} source/contract_inventory.csv" | sha256sum -c - + + - name: Regenerate and verify diagnostic Bloom filter + working-directory: source + shell: bash + run: | + set -euo pipefail + PYTHONPATH=. python - <<'PY' + import hashlib + import json + from pathlib import Path + import pandas as pd + from research.digital_surface.diagnostic_filter import DiagnosticBloomFilter, build_diagnostic_bloom + + path = Path('contract_inventory.csv') + frame = pd.read_csv(path) + assert 'condition_id' in frame.columns, frame.columns.tolist() + ids = sorted(set(frame['condition_id'].astype(str))) + assert len(ids) == 12_465, len(ids) + source_sha = hashlib.sha256(path.read_bytes()).hexdigest() + payload = build_diagnostic_bloom(ids, source_sha256=source_sha) + assert payload['item_count'] == 12_465 + assert payload['m_bits'] == 262_144 + assert payload['k_hashes'] == 7 + assert payload['bitset_sha256'] == '5fd0bc41ab1ae15ddaf9ca33354d803ab74c112d19136240f00d77156186d113' + Path('diagnostic_market_filter.json').write_text(json.dumps(payload, indent=2, sort_keys=True) + '\n') + loaded = DiagnosticBloomFilter.from_path('diagnostic_market_filter.json') + assert loaded.item_count == 12_465 + assert loaded.source_sha256 == source_sha + assert all(value in loaded for value in ids) + canonical_ids = ('\n'.join(ids) + '\n').encode() + ids_sha = hashlib.sha256(canonical_ids).hexdigest() + assert ids_sha == '4605a921c49ac3a73766f96f8f2cfd13b2436c7925f17b673b4b8f1c210a56e5' + Path('diagnostic_market_ids.sha256').write_text(ids_sha + '\n') + PY + + - name: Compile and pass exactly 53 tests + working-directory: source + shell: bash + run: | + set -euo pipefail + python -m compileall -q research/digital_surface tests/digital_surface + PYTHONPATH=. pytest --collect-only -q tests/digital_surface | tee /tmp/collect.txt + grep -Eq '^53 tests collected' /tmp/collect.txt + PYTHONPATH=. pytest tests/digital_surface -q + + - name: Prove observation-only boundary and hash chains + working-directory: source + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import ast + import hashlib + import json + import tempfile + from pathlib import Path + + from research.digital_surface.diagnostic_filter import DiagnosticBloomFilter + from research.digital_surface.live import LiveTradingDisabledError, load_config + from research.digital_surface.market_discovery import GammaMarketRecord + from research.digital_surface.prospective import ProspectiveLedger + from research.digital_surface.public_observer import GAMMA_MARKETS_URL, observe_markets + + try: + load_config({'TRADING_MODE': 'live', 'LIVE_ARM_TOKEN': 'present'}) + except LiveTradingDisabledError: + pass + else: + raise AssertionError('live mode unexpectedly armed') + + policy = json.loads(Path('research/digital_surface/shadow_observation_policy_v1.json').read_text()) + assert policy['mode'] == 'observation_only' + assert policy['authenticated_requests'] is False + assert policy['order_submission'] is False + assert GAMMA_MARKETS_URL == 'https://gamma-api.polymarket.com/markets' + + observer_tree = ast.parse(Path('research/digital_surface/public_observer.py').read_text()) + call_names = { + node.func.attr for node in ast.walk(observer_tree) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) + } + assert not ({'post', 'put', 'patch', 'delete'} & call_names), call_names + + bloom = DiagnosticBloomFilter.from_path('diagnostic_market_filter.json') + with tempfile.TemporaryDirectory() as directory: + directory = Path(directory) + ledger = ProspectiveLedger(directory/'ledger.jsonl', diagnostic_market_ids=bloom) + market = GammaMarketRecord( + condition_id='prospective-smoke-test', + slug='btc-updown-5m-1999999800', + question='Bitcoin Up or Down', + yes_token_id='yes-prospective', + no_token_id='no-prospective', + duration_seconds=300, + open_epoch_seconds=1_999_999_800, + end_date='2033-05-18T03:35:00Z', + tick_size=__import__('decimal').Decimal('0.01'), + ) + summary = observe_markets( + [market], ledger=ledger, observed_ts_ms=1, + policy_sha256='1'*64, source_sha256='2'*64, + ) + assert summary['appended'] == 1 + assert summary['markets_total_in_ledger'] == 1 + reloaded = ProspectiveLedger(ledger.path, diagnostic_market_ids=bloom) + assert reloaded.head_hash == summary['ledger_head_sha256'] + row = json.loads(ledger.path.read_text()) + assert row['metadata']['yes_token_id'] == 'yes-prospective' + assert row['metadata']['tick_size'] == '0.01' + + verified = {} + for line in Path('PUBLIC_OBSERVER_SOURCE_SHA256.txt').read_text().splitlines(): + wanted, name = line.split(maxsplit=1) + got = hashlib.sha256(Path(name).read_bytes()).hexdigest() + assert got == wanted, (name, got, wanted) + verified[name] = got + result = { + 'tests_collected': 53, + 'historical_condition_ids': 12_465, + 'historical_filter_bitset_sha256': bloom.bitset.hex() and hashlib.sha256(bloom.bitset).hexdigest(), + 'public_market_endpoint': GAMMA_MARKETS_URL, + 'authenticated_requests': 0, + 'order_submissions': 0, + 'trading_mode': 'observation_only', + 'live_mode': 'physically_disabled', + 'prospective_hash_chain_verified': True, + 'source_files_verified': verified, + } + assert result['historical_filter_bitset_sha256'] == '5fd0bc41ab1ae15ddaf9ca33354d803ab74c112d19136240f00d77156186d113' + Path('PUBLIC_OBSERVER_VERIFICATION.json').write_text(json.dumps(result, indent=2, sort_keys=True)) + print(json.dumps(result, indent=2)) + PY + + - name: Build deterministic source checkpoint + working-directory: source + shell: bash + run: | + set -euo pipefail + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-public-observer-source-v1.tar.gz \ + research tests PUBLIC_OBSERVER_SOURCE_SHA256.txt PUBLIC_OBSERVER_VERIFICATION.json \ + diagnostic_market_filter.json diagnostic_market_ids.sha256 + sha256sum digital-surface-public-observer-source-v1.tar.gz > digital-surface-public-observer-source-v1.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: digital-surface-public-observer-checkpoint-v1 + path: | + source/PUBLIC_OBSERVER_SOURCE_SHA256.txt + source/PUBLIC_OBSERVER_VERIFICATION.json + source/diagnostic_market_filter.json + source/diagnostic_market_ids.sha256 + source/digital-surface-public-observer-source-v1.tar.gz + source/digital-surface-public-observer-source-v1.tar.gz.sha256 + retention-days: 30 From 354757a47f1c181aa6b909c4090dcd92c730eb6a Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 10:06:30 -0500 Subject: [PATCH 050/177] ci: add credential-free scheduled prospective observer --- .../digital-surface-public-observer.yml | 185 ++++++++++++++++++ 1 file changed, 185 insertions(+) create mode 100644 .github/workflows/digital-surface-public-observer.yml diff --git a/.github/workflows/digital-surface-public-observer.yml b/.github/workflows/digital-surface-public-observer.yml new file mode 100644 index 00000000..dc460d14 --- /dev/null +++ b/.github/workflows/digital-surface-public-observer.yml @@ -0,0 +1,185 @@ +name: Digital Surface Public Observer + +on: + workflow_dispatch: + schedule: + - cron: '*/5 * * * *' + +permissions: + contents: read + actions: read + +concurrency: + group: digital-surface-public-observer + cancel-in-progress: false + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + V5_ARTIFACT_ID: '8615107471' + V5_ARTIFACT_ZIP_SHA256: '479a25e235841d94077b3d2df070c8ed46bb736c7ebd8f9f1bde8016198a9c39' + CONTRACT_INVENTORY_SHA256: 'cea26cf5221e81b90ba311317f31e97b54e4110f0f1c716fcb14f06d9f1651f3' + FILTER_CACHE_KEY: digital-surface-diagnostic-filter-v1-cea26cf5221e81b90ba311317f31e97b54e4110f0f1c716fcb14f06d9f1651f3 + +jobs: + observe-public-markets: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct observation-only source + shell: bash + run: | + set -euo pipefail + mkdir -p source/research/digital_surface + touch source/research/__init__.py source/research/digital_surface/__init__.py + cp -a ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/. source/research/digital_surface/ + cp -a ci_payload/digital-surface-live-readiness/prospective-source/research/digital_surface/. source/research/digital_surface/ + cat > source/PUBLIC_OBSERVER_RUNTIME_SOURCE_SHA256.txt <<'EOF' + f3f1e2417305cd0ceee4b6971e461e488960f04f0e0b80ad292733a2a4b248af research/digital_surface/diagnostic_filter.py + efd186a4ca00c2e920bc87b8d137ac46db52f2aa36385a275dc99e70b5618439 research/digital_surface/market_discovery.py + 3fbb96286445b60b0b7deeb5e1f9ee45f53bfd5efdcc7a727b50617b19463224 research/digital_surface/prospective.py + 73c5aaa7a4ca10baf85886eb28c6135d975eafaaab19e5193819d1ccbf82f2be research/digital_surface/public_observer.py + ea56803a606ff33b7231b373141df3d22c159cedc1063603fef3dfb6255c60c9 research/digital_surface/shadow_observation_policy_v1.json + EOF + sed -i 's/^ //' source/PUBLIC_OBSERVER_RUNTIME_SOURCE_SHA256.txt + (cd source && sha256sum -c PUBLIC_OBSERVER_RUNTIME_SOURCE_SHA256.txt) + + - name: Install observation-only dependencies + run: | + python -m pip install --upgrade pip + python -m pip install pandas==2.2.3 requests==2.32.3 + + - name: Restore diagnostic filter + id: filter-cache + uses: actions/cache@v4 + with: + path: observer-filter + key: ${{ env.FILTER_CACHE_KEY }} + + - name: Generate diagnostic filter from immutable v5 inventory + if: steps.filter-cache.outputs.cache-hit != 'true' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + curl --fail --location --silent --show-error \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H 'Accept: application/vnd.github+json' \ + -H 'X-GitHub-Api-Version: 2022-11-28' \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/artifacts/${V5_ARTIFACT_ID}/zip" \ + -o v5.zip + echo "${V5_ARTIFACT_ZIP_SHA256} v5.zip" | sha256sum -c - + mkdir v5 observer-filter && unzip -q v5.zip -d v5 + inventory=$(find v5 -type f -name contract_inventory.csv -print -quit) + test -n "${inventory}" + cp "${inventory}" observer-filter/contract_inventory.csv + echo "${CONTRACT_INVENTORY_SHA256} observer-filter/contract_inventory.csv" | sha256sum -c - + PYTHONPATH=source python - <<'PY' + import hashlib + import json + from pathlib import Path + import pandas as pd + from research.digital_surface.diagnostic_filter import build_diagnostic_bloom + + path = Path('observer-filter/contract_inventory.csv') + frame = pd.read_csv(path) + ids = sorted(set(frame['condition_id'].astype(str))) + assert len(ids) == 12_465 + source_sha = hashlib.sha256(path.read_bytes()).hexdigest() + payload = build_diagnostic_bloom(ids, source_sha256=source_sha) + assert payload['bitset_sha256'] == '5fd0bc41ab1ae15ddaf9ca33354d803ab74c112d19136240f00d77156186d113' + Path('observer-filter/diagnostic_market_filter.json').write_text( + json.dumps(payload, indent=2, sort_keys=True) + '\n' + ) + PY + + - name: Verify restored diagnostic filter + shell: bash + run: | + set -euo pipefail + PYTHONPATH=source python - <<'PY' + import hashlib + from research.digital_surface.diagnostic_filter import DiagnosticBloomFilter + bloom = DiagnosticBloomFilter.from_path('observer-filter/diagnostic_market_filter.json') + assert bloom.item_count == 12_465 + assert bloom.m_bits == 262_144 + assert bloom.k_hashes == 7 + assert bloom.source_sha256 == 'cea26cf5221e81b90ba311317f31e97b54e4110f0f1c716fcb14f06d9f1651f3' + assert hashlib.sha256(bloom.bitset).hexdigest() == '5fd0bc41ab1ae15ddaf9ca33354d803ab74c112d19136240f00d77156186d113' + PY + + - name: Restore append-only prospective state + uses: actions/cache/restore@v4 + with: + path: observer-state + key: digital-surface-public-observer-state-${{ github.run_id }} + restore-keys: | + digital-surface-public-observer-state- + + - name: Observe public markets + shell: bash + run: | + set -euo pipefail + mkdir -p observer-state observer-output + PYTHONPATH=source python -m research.digital_surface.public_observer \ + --diagnostic-filter observer-filter/diagnostic_market_filter.json \ + --ledger observer-state/prospective_markets.jsonl \ + --policy source/research/digital_surface/shadow_observation_policy_v1.json \ + --source-manifest source/PUBLIC_OBSERVER_RUNTIME_SOURCE_SHA256.txt \ + --summary observer-output/observer_summary.json \ + --page-size 100 \ + --max-pages 20 + + - name: Verify observation-only artifact + shell: bash + run: | + set -euo pipefail + PYTHONPATH=source python - <<'PY' + import hashlib + import json + from pathlib import Path + from research.digital_surface.diagnostic_filter import DiagnosticBloomFilter + from research.digital_surface.prospective import ProspectiveLedger + + summary = json.loads(Path('observer-output/observer_summary.json').read_text()) + assert summary['trading_mode'] == 'observation_only' + assert summary['authenticated_requests'] == 0 + assert summary['order_submissions'] == 0 + bloom = DiagnosticBloomFilter.from_path('observer-filter/diagnostic_market_filter.json') + ledger = ProspectiveLedger('observer-state/prospective_markets.jsonl', diagnostic_market_ids=bloom) + assert ledger.head_hash == summary['ledger_head_sha256'] + assert len(ledger.market_ids) == summary['markets_total_in_ledger'] + verification = { + 'workflow_run_id': '${{ github.run_id }}', + 'workflow_run_attempt': '${{ github.run_attempt }}', + 'ledger_head_sha256': ledger.head_hash, + 'ledger_sha256': hashlib.sha256(Path('observer-state/prospective_markets.jsonl').read_bytes()).hexdigest(), + 'markets_total_in_ledger': len(ledger.market_ids), + 'summary': summary, + 'authenticated_requests': 0, + 'order_submissions': 0, + 'trading_mode': 'observation_only', + } + Path('observer-output/OBSERVER_VERIFICATION.json').write_text(json.dumps(verification, indent=2, sort_keys=True)) + print(json.dumps(verification, indent=2)) + PY + cp observer-state/prospective_markets.jsonl observer-output/prospective_markets.jsonl + sha256sum observer-output/prospective_markets.jsonl observer-output/observer_summary.json observer-output/OBSERVER_VERIFICATION.json > observer-output/SHA256SUMS.txt + + - name: Save append-only prospective state + uses: actions/cache/save@v4 + with: + path: observer-state + key: digital-surface-public-observer-state-${{ github.run_id }} + + - uses: actions/upload-artifact@v4 + with: + name: digital-surface-public-observer-${{ github.run_id }} + path: observer-output/ + retention-days: 90 From c41cdb9aa66224a8607271cefd0e9537777280a3 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 10:16:44 -0500 Subject: [PATCH 051/177] feat: checkpoint public order-book evidence ledger --- .../research/digital_surface/book_ledger.py | 118 ++++++++++++++ .../digital_surface/public_book_observer.py | 118 ++++++++++++++ .../research/digital_surface/public_books.py | 148 ++++++++++++++++++ .../tests/digital_surface/test_book_ledger.py | 42 +++++ .../test_public_book_observer.py | 56 +++++++ .../digital_surface/test_public_books.py | 63 ++++++++ 6 files changed, 545 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/book_ledger.py create mode 100644 ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/public_book_observer.py create mode 100644 ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/public_books.py create mode 100644 ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_book_ledger.py create mode 100644 ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_public_book_observer.py create mode 100644 ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_public_books.py diff --git a/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/book_ledger.py b/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/book_ledger.py new file mode 100644 index 00000000..9e02e6a2 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/book_ledger.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +from .public_books import PublicOrderBook + + +class BookLedgerIntegrityError(RuntimeError): + pass + + +def _canonical_json(value: dict[str, Any]) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _sha256(name: str, value: str) -> str: + text = str(value).lower() + if len(text) != 64 or any(ch not in "0123456789abcdef" for ch in text): + raise ValueError(f"{name} must be a 64-character SHA-256 hex digest") + return text + + +class PublicBookLedger: + def __init__(self, path: str | Path) -> None: + self.path = Path(path) + self.rows: list[dict[str, Any]] = [] + self.head_hash = "0" * 64 + self.observation_ids: set[str] = set() + if self.path.exists(): + self._load_and_verify() + + def _load_and_verify(self) -> None: + previous = "0" * 64 + rows: list[dict[str, Any]] = [] + seen: set[str] = set() + for line_number, line in enumerate(self.path.read_text(encoding="utf-8").splitlines(), start=1): + if not line.strip(): + continue + try: + record = json.loads(line) + except json.JSONDecodeError as exc: + raise BookLedgerIntegrityError(f"invalid JSON at line {line_number}") from exc + stored = str(record.get("record_hash", "")) + payload = dict(record) + payload.pop("record_hash", None) + if payload.get("previous_hash") != previous: + raise BookLedgerIntegrityError(f"book chain mismatch at line {line_number}") + expected = hashlib.sha256(_canonical_json(payload)).hexdigest() + if stored != expected: + raise BookLedgerIntegrityError(f"book record hash mismatch at line {line_number}") + observation_id = str(payload.get("observation_id") or "") + if not observation_id or observation_id in seen: + raise BookLedgerIntegrityError(f"duplicate or empty observation_id at line {line_number}") + for field in ("policy_sha256", "source_sha256", "prospective_head_sha256"): + try: + _sha256(field, str(payload.get(field, ""))) + except ValueError as exc: + raise BookLedgerIntegrityError(f"invalid {field} at line {line_number}") from exc + book = payload.get("book") + if not isinstance(book, dict): + raise BookLedgerIntegrityError(f"missing book payload at line {line_number}") + payload_sha = str(book.get("payload_sha256") or "") + _sha256("payload_sha256", payload_sha) + rows.append(record) + seen.add(observation_id) + previous = stored + self.rows = rows + self.observation_ids = seen + self.head_hash = previous + + def append( + self, + *, + condition_id: str, + outcome: str, + observed_ts_ms: int, + book: PublicOrderBook, + policy_sha256: str, + source_sha256: str, + prospective_head_sha256: str, + ) -> dict[str, Any]: + timestamp = int(observed_ts_ms) + if timestamp < 0: + raise ValueError("observed_ts_ms must be non-negative") + side = str(outcome).lower() + if side not in {"yes", "no"}: + raise ValueError("outcome must be yes or no") + condition = str(condition_id) + if book.condition_id != condition: + raise ValueError(f"book condition mismatch: {book.condition_id} != {condition}") + observation_id = hashlib.sha256( + f"{condition}\0{book.token_id}\0{timestamp}\0{book.payload_sha256}".encode() + ).hexdigest() + if observation_id in self.observation_ids: + raise ValueError("duplicate book observation") + payload = { + "book": book.as_json(), + "condition_id": condition, + "observation_id": observation_id, + "observed_ts_ms": timestamp, + "outcome": side, + "policy_sha256": _sha256("policy_sha256", policy_sha256), + "previous_hash": self.head_hash, + "prospective_head_sha256": _sha256("prospective_head_sha256", prospective_head_sha256), + "source_sha256": _sha256("source_sha256", source_sha256), + } + record = {**payload, "record_hash": hashlib.sha256(_canonical_json(payload)).hexdigest()} + self.path.parent.mkdir(parents=True, exist_ok=True) + with self.path.open("a", encoding="utf-8", newline="\n") as handle: + handle.write(json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n") + handle.flush() + self.rows.append(record) + self.observation_ids.add(observation_id) + self.head_hash = record["record_hash"] + return dict(record) diff --git a/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/public_book_observer.py b/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/public_book_observer.py new file mode 100644 index 00000000..3a2c6c1a --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/public_book_observer.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import time +from pathlib import Path +from typing import Sequence + +from .book_ledger import PublicBookLedger +from .diagnostic_filter import DiagnosticBloomFilter +from .market_discovery import GammaMarketRecord, discover_target_markets +from .prospective import ProspectiveLedger +from .public_books import PublicOrderBook, fetch_public_order_books +from .public_observer import fetch_gamma_page + + +def _sha256_file(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def observe_public_books( + markets: Sequence[GammaMarketRecord], + books: dict[str, PublicOrderBook], + *, + ledger: PublicBookLedger, + prospective_market_ids: set[str], + prospective_head_sha256: str, + observed_ts_ms: int, + policy_sha256: str, + source_sha256: str, +) -> dict[str, object]: + requested = 0 + appended = 0 + missing: list[str] = [] + skipped_unregistered = 0 + for market in markets: + if market.condition_id not in prospective_market_ids: + skipped_unregistered += 1 + continue + for outcome, token_id in (("yes", market.yes_token_id), ("no", market.no_token_id)): + requested += 1 + book = books.get(token_id) + if book is None: + missing.append(token_id) + continue + ledger.append( + condition_id=market.condition_id, + outcome=outcome, + observed_ts_ms=observed_ts_ms, + book=book, + policy_sha256=policy_sha256, + source_sha256=source_sha256, + prospective_head_sha256=prospective_head_sha256, + ) + appended += 1 + return { + "markets_discovered": len(markets), + "markets_registered": len(prospective_market_ids), + "markets_skipped_unregistered": skipped_unregistered, + "tokens_requested": requested, + "books_received": len(books), + "books_appended": appended, + "missing_token_ids": sorted(missing), + "book_ledger_head_sha256": ledger.head_hash, + "book_observations_total": len(ledger.rows), + "prospective_head_sha256": prospective_head_sha256, + "observed_ts_ms": int(observed_ts_ms), + } + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Collect public Polymarket orderbooks without credentials") + parser.add_argument("--diagnostic-filter", type=Path, required=True) + parser.add_argument("--prospective-ledger", type=Path, required=True) + parser.add_argument("--book-ledger", type=Path, required=True) + parser.add_argument("--policy", type=Path, required=True) + parser.add_argument("--source-manifest", type=Path, required=True) + parser.add_argument("--summary", type=Path, required=True) + parser.add_argument("--page-size", type=int, default=100) + parser.add_argument("--max-pages", type=int, default=20) + args = parser.parse_args(argv) + + bloom = DiagnosticBloomFilter.from_path(args.diagnostic_filter) + prospective = ProspectiveLedger(args.prospective_ledger, diagnostic_market_ids=bloom) + markets = discover_target_markets( + lambda params: fetch_gamma_page(params), + page_size=args.page_size, + max_pages=args.max_pages, + ) + registered = [market for market in markets if market.condition_id in prospective.market_ids] + token_ids = [token for market in registered for token in (market.yes_token_id, market.no_token_id)] + books = fetch_public_order_books(token_ids) + book_ledger = PublicBookLedger(args.book_ledger) + summary = observe_public_books( + markets, + books, + ledger=book_ledger, + prospective_market_ids=prospective.market_ids, + prospective_head_sha256=prospective.head_hash, + observed_ts_ms=int(time.time() * 1000), + policy_sha256=_sha256_file(args.policy), + source_sha256=_sha256_file(args.source_manifest), + ) + summary.update({ + "authenticated_requests": 0, + "order_submissions": 0, + "trading_mode": "observation_only", + "public_books_endpoint": "https://clob.polymarket.com/books", + }) + args.summary.parent.mkdir(parents=True, exist_ok=True) + args.summary.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/public_books.py b/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/public_books.py new file mode 100644 index 00000000..c4f0ba1c --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/public_books.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from decimal import Decimal +from typing import Any, Callable, Mapping, Sequence + + +CLOB_BOOKS_URL = "https://clob.polymarket.com/books" + + +def _canonical_json(value: object) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +@dataclass(frozen=True, slots=True) +class BookLevel: + price: Decimal + size: Decimal + + def as_json(self) -> dict[str, str]: + return {"price": str(self.price), "size": str(self.size)} + + +@dataclass(frozen=True, slots=True) +class PublicOrderBook: + condition_id: str + token_id: str + timestamp: str + book_hash: str + bids: tuple[BookLevel, ...] + asks: tuple[BookLevel, ...] + min_order_size: Decimal + tick_size: Decimal + neg_risk: bool + last_trade_price: Decimal | None + payload_sha256: str + + @property + def best_bid(self) -> BookLevel | None: + return self.bids[0] if self.bids else None + + @property + def best_ask(self) -> BookLevel | None: + return self.asks[0] if self.asks else None + + def as_json(self) -> dict[str, object]: + return { + "condition_id": self.condition_id, + "token_id": self.token_id, + "timestamp": self.timestamp, + "book_hash": self.book_hash, + "bids": [level.as_json() for level in self.bids], + "asks": [level.as_json() for level in self.asks], + "min_order_size": str(self.min_order_size), + "tick_size": str(self.tick_size), + "neg_risk": self.neg_risk, + "last_trade_price": None if self.last_trade_price is None else str(self.last_trade_price), + "payload_sha256": self.payload_sha256, + } + + +def _levels(value: object, *, side: str) -> tuple[BookLevel, ...]: + if value is None: + return () + if not isinstance(value, Sequence) or isinstance(value, (str, bytes)): + raise ValueError(f"{side} must be a sequence") + levels: list[BookLevel] = [] + for row in value: + if not isinstance(row, Mapping): + raise ValueError(f"{side} level must be an object") + price = Decimal(str(row.get("price"))) + size = Decimal(str(row.get("size"))) + if not (Decimal("0") < price < Decimal("1")): + raise ValueError(f"{side} price must be between zero and one") + if size < 0: + raise ValueError(f"{side} size must be non-negative") + if size: + levels.append(BookLevel(price=price, size=size)) + reverse = side == "bids" + return tuple(sorted(levels, key=lambda level: level.price, reverse=reverse)) + + +def parse_public_order_book( + payload: Mapping[str, Any], + *, + expected_token_id: str | None = None, +) -> PublicOrderBook: + token_id = str(payload.get("asset_id") or "").strip() + condition_id = str(payload.get("market") or "").strip() + if not token_id or not condition_id: + raise ValueError("orderbook is missing market or asset_id") + if expected_token_id is not None and token_id != str(expected_token_id): + raise ValueError(f"orderbook asset mismatch: {token_id} != {expected_token_id}") + raw_last = payload.get("last_trade_price") + last_trade = None if raw_last in (None, "") else Decimal(str(raw_last)) + return PublicOrderBook( + condition_id=condition_id, + token_id=token_id, + timestamp=str(payload.get("timestamp") or ""), + book_hash=str(payload.get("hash") or ""), + bids=_levels(payload.get("bids", []), side="bids"), + asks=_levels(payload.get("asks", []), side="asks"), + min_order_size=Decimal(str(payload.get("min_order_size") or "0")), + tick_size=Decimal(str(payload.get("tick_size") or "0.01")), + neg_risk=bool(payload.get("neg_risk", False)), + last_trade_price=last_trade, + payload_sha256=hashlib.sha256(_canonical_json(dict(payload))).hexdigest(), + ) + + +def fetch_public_order_books( + token_ids: Sequence[str], + *, + post: Callable[..., Any] | None = None, + timeout_seconds: float = 15.0, + batch_size: int = 500, +) -> dict[str, PublicOrderBook]: + if batch_size <= 0 or batch_size > 500: + raise ValueError("batch_size must be between 1 and 500") + ordered = list(dict.fromkeys(str(token).strip() for token in token_ids if str(token).strip())) + if not ordered: + return {} + if post is None: + import requests + post = requests.post + result: dict[str, PublicOrderBook] = {} + for start in range(0, len(ordered), batch_size): + batch = ordered[start : start + batch_size] + response = post( + CLOB_BOOKS_URL, + json=[{"token_id": token} for token in batch], + timeout=timeout_seconds, + ) + response.raise_for_status() + payload = response.json() + if not isinstance(payload, list): + raise ValueError("public books response must be a list") + expected = set(batch) + for raw in payload: + book = parse_public_order_book(raw) + if book.token_id not in expected: + raise ValueError(f"unexpected orderbook asset: {book.token_id}") + if book.token_id in result: + raise ValueError(f"duplicate orderbook asset: {book.token_id}") + result[book.token_id] = book + return result diff --git a/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_book_ledger.py b/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_book_ledger.py new file mode 100644 index 00000000..c99b02ea --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_book_ledger.py @@ -0,0 +1,42 @@ +import json + +import pytest + +from research.digital_surface.book_ledger import BookLedgerIntegrityError, PublicBookLedger +from research.digital_surface.public_books import parse_public_order_book + + +def book(): + return parse_public_order_book({ + "market": "c1", "asset_id": "t1", "timestamp": "1", "hash": "h", + "bids": [{"price": "0.40", "size": "10"}], + "asks": [{"price": "0.50", "size": "10"}], + "min_order_size": "1", "tick_size": "0.01", "neg_risk": False, + }) + + +def test_appends_and_reloads_book_hash_chain(tmp_path): + path = tmp_path / "books.jsonl" + ledger = PublicBookLedger(path) + first = ledger.append( + condition_id="c1", outcome="yes", observed_ts_ms=1, book=book(), + policy_sha256="1"*64, source_sha256="2"*64, prospective_head_sha256="3"*64, + ) + assert first["previous_hash"] == "0"*64 + reloaded = PublicBookLedger(path) + assert reloaded.head_hash == first["record_hash"] + assert len(reloaded.rows) == 1 + + +def test_detects_book_ledger_tampering(tmp_path): + path = tmp_path / "books.jsonl" + ledger = PublicBookLedger(path) + ledger.append( + condition_id="c1", outcome="yes", observed_ts_ms=1, book=book(), + policy_sha256="1"*64, source_sha256="2"*64, prospective_head_sha256="3"*64, + ) + row = json.loads(path.read_text()) + row["book"]["asks"][0]["size"] = "999" + path.write_text(json.dumps(row) + "\n") + with pytest.raises(BookLedgerIntegrityError, match="record hash mismatch"): + PublicBookLedger(path) diff --git a/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_public_book_observer.py b/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_public_book_observer.py new file mode 100644 index 00000000..b200778c --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_public_book_observer.py @@ -0,0 +1,56 @@ +from decimal import Decimal + +from research.digital_surface.book_ledger import PublicBookLedger +from research.digital_surface.market_discovery import GammaMarketRecord +from research.digital_surface.public_book_observer import observe_public_books +from research.digital_surface.public_books import parse_public_order_book + + +def market(condition="c1"): + return GammaMarketRecord( + condition_id=condition, slug="btc-updown-5m-1774450800", question="Bitcoin Up or Down", + yes_token_id=f"yes-{condition}", no_token_id=f"no-{condition}", duration_seconds=300, + open_epoch_seconds=1774450800, end_date="2026-07-25T15:00:00Z", tick_size=Decimal("0.01"), + ) + + +def book(token, condition): + return parse_public_order_book({ + "market": condition, "asset_id": token, "timestamp": "1", "hash": f"h-{token}", + "bids": [{"price": "0.40", "size": "10"}], + "asks": [{"price": "0.50", "size": "10"}], + "min_order_size": "1", "tick_size": "0.01", "neg_risk": False, + }) + + +def test_observer_records_both_registered_token_books_and_binds_prospective_head(tmp_path): + item = market() + books = { + item.yes_token_id: book(item.yes_token_id, item.condition_id), + item.no_token_id: book(item.no_token_id, item.condition_id), + } + ledger = PublicBookLedger(tmp_path / "books.jsonl") + summary = observe_public_books( + [item], books, ledger=ledger, prospective_market_ids={"c1"}, + prospective_head_sha256="3"*64, observed_ts_ms=10, + policy_sha256="1"*64, source_sha256="2"*64, + ) + assert summary["books_appended"] == 2 + assert summary["missing_token_ids"] == [] + assert {row["outcome"] for row in ledger.rows} == {"yes", "no"} + assert all(row["prospective_head_sha256"] == "3"*64 for row in ledger.rows) + + +def test_observer_reports_missing_books_and_skips_unregistered_markets(tmp_path): + registered = market("c1") + unregistered = market("c2") + books = {registered.yes_token_id: book(registered.yes_token_id, registered.condition_id)} + ledger = PublicBookLedger(tmp_path / "books.jsonl") + summary = observe_public_books( + [registered, unregistered], books, ledger=ledger, prospective_market_ids={"c1"}, + prospective_head_sha256="3"*64, observed_ts_ms=10, + policy_sha256="1"*64, source_sha256="2"*64, + ) + assert summary["books_appended"] == 1 + assert summary["markets_skipped_unregistered"] == 1 + assert summary["missing_token_ids"] == [registered.no_token_id] diff --git a/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_public_books.py b/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_public_books.py new file mode 100644 index 00000000..b940e731 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_public_books.py @@ -0,0 +1,63 @@ +from decimal import Decimal + +import pytest + +from research.digital_surface.public_books import ( + CLOB_BOOKS_URL, + fetch_public_order_books, + parse_public_order_book, +) + + +class Response: + def __init__(self, payload): + self.payload = payload + def raise_for_status(self): + return None + def json(self): + return self.payload + + +def payload(token="t1", condition="c1"): + return { + "market": condition, + "asset_id": token, + "timestamp": "1234", + "hash": "book-hash", + "bids": [{"price": "0.44", "size": "2"}, {"price": "0.46", "size": "3"}], + "asks": [{"price": "0.55", "size": "4"}, {"price": "0.52", "size": "5"}], + "min_order_size": "1", + "tick_size": "0.01", + "neg_risk": False, + "last_trade_price": "0.50", + } + + +def test_parses_and_sorts_full_public_orderbook_depth(): + book = parse_public_order_book(payload(), expected_token_id="t1") + assert book.best_bid.price == Decimal("0.46") + assert book.best_bid.size == Decimal("3") + assert book.best_ask.price == Decimal("0.52") + assert [level.price for level in book.asks] == [Decimal("0.52"), Decimal("0.55")] + assert len(book.payload_sha256) == 64 + + +def test_rejects_asset_mismatch_and_invalid_levels(): + with pytest.raises(ValueError, match="asset mismatch"): + parse_public_order_book(payload(), expected_token_id="other") + broken = payload() + broken["asks"] = [{"price": "1.2", "size": "1"}] + with pytest.raises(ValueError, match="between zero and one"): + parse_public_order_book(broken) + + +def test_batch_fetch_uses_only_public_books_endpoint_without_headers(): + calls = [] + def post(url, **kwargs): + calls.append((url, kwargs)) + return Response([payload("t1"), payload("t2", "c2")]) + books = fetch_public_order_books(["t1", "t2"], post=post) + assert set(books) == {"t1", "t2"} + assert calls[0][0] == CLOB_BOOKS_URL + assert calls[0][1]["json"] == [{"token_id": "t1"}, {"token_id": "t2"}] + assert "headers" not in calls[0][1] From 3661715c5a7a2fb5de90bdb3b527ada9128f5888 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 10:18:47 -0500 Subject: [PATCH 052/177] ci: persist public market and order-book evidence --- .../digital-surface-public-observer.yml | 74 ++++++++++++++----- 1 file changed, 57 insertions(+), 17 deletions(-) diff --git a/.github/workflows/digital-surface-public-observer.yml b/.github/workflows/digital-surface-public-observer.yml index dc460d14..b417065e 100644 --- a/.github/workflows/digital-surface-public-observer.yml +++ b/.github/workflows/digital-surface-public-observer.yml @@ -1,6 +1,12 @@ name: Digital Surface Public Observer on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/digital-surface-public-observer.yml + - ci_payload/digital-surface-live-readiness/observer-source/** + - ci_payload/digital-surface-live-readiness/prospective-source/** workflow_dispatch: schedule: - cron: '*/5 * * * *' @@ -40,9 +46,12 @@ jobs: cp -a ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/. source/research/digital_surface/ cp -a ci_payload/digital-surface-live-readiness/prospective-source/research/digital_surface/. source/research/digital_surface/ cat > source/PUBLIC_OBSERVER_RUNTIME_SOURCE_SHA256.txt <<'EOF' + d96e722e57b24b2dc37bdf772972133ceeb1fcbf3629973f3abae701b4943051 research/digital_surface/book_ledger.py f3f1e2417305cd0ceee4b6971e461e488960f04f0e0b80ad292733a2a4b248af research/digital_surface/diagnostic_filter.py efd186a4ca00c2e920bc87b8d137ac46db52f2aa36385a275dc99e70b5618439 research/digital_surface/market_discovery.py 3fbb96286445b60b0b7deeb5e1f9ee45f53bfd5efdcc7a727b50617b19463224 research/digital_surface/prospective.py + 088bab6729e5a1beb22baa853483433fd23180d3f8597035e0d3a4d036a8aec3 research/digital_surface/public_book_observer.py + 1ee608819769dad77c19f343427fbe604cacb3afc95b693aa5153b4ea22a166e research/digital_surface/public_books.py 73c5aaa7a4ca10baf85886eb28c6135d975eafaaab19e5193819d1ccbf82f2be research/digital_surface/public_observer.py ea56803a606ff33b7231b373141df3d22c159cedc1063603fef3dfb6255c60c9 research/digital_surface/shadow_observation_policy_v1.json EOF @@ -114,7 +123,7 @@ jobs: assert hashlib.sha256(bloom.bitset).hexdigest() == '5fd0bc41ab1ae15ddaf9ca33354d803ab74c112d19136240f00d77156186d113' PY - - name: Restore append-only prospective state + - name: Restore append-only observation state uses: actions/cache/restore@v4 with: path: observer-state @@ -122,11 +131,12 @@ jobs: restore-keys: | digital-surface-public-observer-state- - - name: Observe public markets + - name: Observe public markets and orderbooks shell: bash run: | set -euo pipefail mkdir -p observer-state observer-output + touch observer-state/prospective_markets.jsonl observer-state/public_books.jsonl PYTHONPATH=source python -m research.digital_surface.public_observer \ --diagnostic-filter observer-filter/diagnostic_market_filter.json \ --ledger observer-state/prospective_markets.jsonl \ @@ -135,8 +145,17 @@ jobs: --summary observer-output/observer_summary.json \ --page-size 100 \ --max-pages 20 + PYTHONPATH=source python -m research.digital_surface.public_book_observer \ + --diagnostic-filter observer-filter/diagnostic_market_filter.json \ + --prospective-ledger observer-state/prospective_markets.jsonl \ + --book-ledger observer-state/public_books.jsonl \ + --policy source/research/digital_surface/shadow_observation_policy_v1.json \ + --source-manifest source/PUBLIC_OBSERVER_RUNTIME_SOURCE_SHA256.txt \ + --summary observer-output/book_observer_summary.json \ + --page-size 100 \ + --max-pages 20 - - name: Verify observation-only artifact + - name: Verify observation-only artifacts shell: bash run: | set -euo pipefail @@ -144,24 +163,38 @@ jobs: import hashlib import json from pathlib import Path + from research.digital_surface.book_ledger import PublicBookLedger from research.digital_surface.diagnostic_filter import DiagnosticBloomFilter from research.digital_surface.prospective import ProspectiveLedger - summary = json.loads(Path('observer-output/observer_summary.json').read_text()) - assert summary['trading_mode'] == 'observation_only' - assert summary['authenticated_requests'] == 0 - assert summary['order_submissions'] == 0 + market_summary = json.loads(Path('observer-output/observer_summary.json').read_text()) + book_summary = json.loads(Path('observer-output/book_observer_summary.json').read_text()) + for summary in (market_summary, book_summary): + assert summary['trading_mode'] == 'observation_only' + assert summary['authenticated_requests'] == 0 + assert summary['order_submissions'] == 0 + bloom = DiagnosticBloomFilter.from_path('observer-filter/diagnostic_market_filter.json') - ledger = ProspectiveLedger('observer-state/prospective_markets.jsonl', diagnostic_market_ids=bloom) - assert ledger.head_hash == summary['ledger_head_sha256'] - assert len(ledger.market_ids) == summary['markets_total_in_ledger'] + markets = ProspectiveLedger('observer-state/prospective_markets.jsonl', diagnostic_market_ids=bloom) + books = PublicBookLedger('observer-state/public_books.jsonl') + assert markets.head_hash == market_summary['ledger_head_sha256'] + assert len(markets.market_ids) == market_summary['markets_total_in_ledger'] + assert books.head_hash == book_summary['book_ledger_head_sha256'] + assert len(books.rows) == book_summary['book_observations_total'] + assert book_summary['prospective_head_sha256'] == markets.head_hash + assert book_summary['public_books_endpoint'] == 'https://clob.polymarket.com/books' + verification = { 'workflow_run_id': '${{ github.run_id }}', 'workflow_run_attempt': '${{ github.run_attempt }}', - 'ledger_head_sha256': ledger.head_hash, - 'ledger_sha256': hashlib.sha256(Path('observer-state/prospective_markets.jsonl').read_bytes()).hexdigest(), - 'markets_total_in_ledger': len(ledger.market_ids), - 'summary': summary, + 'market_ledger_head_sha256': markets.head_hash, + 'market_ledger_sha256': hashlib.sha256(Path('observer-state/prospective_markets.jsonl').read_bytes()).hexdigest(), + 'book_ledger_head_sha256': books.head_hash, + 'book_ledger_sha256': hashlib.sha256(Path('observer-state/public_books.jsonl').read_bytes()).hexdigest(), + 'markets_total_in_ledger': len(markets.market_ids), + 'book_observations_total': len(books.rows), + 'market_summary': market_summary, + 'book_summary': book_summary, 'authenticated_requests': 0, 'order_submissions': 0, 'trading_mode': 'observation_only', @@ -170,9 +203,16 @@ jobs: print(json.dumps(verification, indent=2)) PY cp observer-state/prospective_markets.jsonl observer-output/prospective_markets.jsonl - sha256sum observer-output/prospective_markets.jsonl observer-output/observer_summary.json observer-output/OBSERVER_VERIFICATION.json > observer-output/SHA256SUMS.txt - - - name: Save append-only prospective state + cp observer-state/public_books.jsonl observer-output/public_books.jsonl + sha256sum \ + observer-output/prospective_markets.jsonl \ + observer-output/public_books.jsonl \ + observer-output/observer_summary.json \ + observer-output/book_observer_summary.json \ + observer-output/OBSERVER_VERIFICATION.json \ + > observer-output/SHA256SUMS.txt + + - name: Save append-only observation state uses: actions/cache/save@v4 with: path: observer-state From 26e5f53a83323d8ccafaaa4dff7bcc9fd33dd2ae Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 10:22:22 -0500 Subject: [PATCH 053/177] ci: verify complete public market and book observer source --- ...urface-public-book-observer-checkpoint.yml | 212 ++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 .github/workflows/digital-surface-public-book-observer-checkpoint.yml diff --git a/.github/workflows/digital-surface-public-book-observer-checkpoint.yml b/.github/workflows/digital-surface-public-book-observer-checkpoint.yml new file mode 100644 index 00000000..9d81a59a --- /dev/null +++ b/.github/workflows/digital-surface-public-book-observer-checkpoint.yml @@ -0,0 +1,212 @@ +name: Digital Surface Public Book Observer Checkpoint + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/digital-surface-public-book-observer-checkpoint.yml + - ci_payload/digital-surface-live-readiness/observer-source/** + - ci_payload/digital-surface-live-readiness/prospective-source/** + - ci_payload/digital-surface-live-readiness/shadow-source/** + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-public-book-observer-checkpoint.yml + - ci_payload/digital-surface-live-readiness/observer-source/** + - ci_payload/digital-surface-live-readiness/prospective-source/** + - ci_payload/digital-surface-live-readiness/shadow-source/** + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + verify-public-book-observer: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct checksum-pinned research source + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source && tar -xzf base.tar.gz -C source + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + for patch_file in timestamp-unit-fix.patch full-universe-gate-fix.patch resolved-universe-fix.patch asset-merge-fix.patch; do + patch -d source -p1 < "ci_payload/digital-surface-official-resolution-patch/${patch_file}" + done + echo 'f71f3a653ca58b4bf98609fc29d8ab16f9d031c9ed14906ebe5159f2b0de8eab ci_payload/digital-surface-live-readiness/execution-diagnostics.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/execution-diagnostics.patch + echo '8606cc925aa3b3a6417abb1fba9a11df916cc2b281880978fb6229aa25875be9 ci_payload/digital-surface-live-readiness/book-arrival-model.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/book-arrival-model.patch + cp -a ci_payload/digital-surface-live-readiness/shadow-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/prospective-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/observer-source/. source/ + + cat > /tmp/public-book-source.sha256 <<'EOF' + d96e722e57b24b2dc37bdf772972133ceeb1fcbf3629973f3abae701b4943051 source/research/digital_surface/book_ledger.py + f3f1e2417305cd0ceee4b6971e461e488960f04f0e0b80ad292733a2a4b248af source/research/digital_surface/diagnostic_filter.py + efd186a4ca00c2e920bc87b8d137ac46db52f2aa36385a275dc99e70b5618439 source/research/digital_surface/market_discovery.py + 3fbb96286445b60b0b7deeb5e1f9ee45f53bfd5efdcc7a727b50617b19463224 source/research/digital_surface/prospective.py + 088bab6729e5a1beb22baa853483433fd23180d3f8597035e0d3a4d036a8aec3 source/research/digital_surface/public_book_observer.py + 1ee608819769dad77c19f343427fbe604cacb3afc95b693aa5153b4ea22a166e source/research/digital_surface/public_books.py + 73c5aaa7a4ca10baf85886eb28c6135d975eafaaab19e5193819d1ccbf82f2be source/research/digital_surface/public_observer.py + ea56803a606ff33b7231b373141df3d22c159cedc1063603fef3dfb6255c60c9 source/research/digital_surface/shadow_observation_policy_v1.json + 7220101c9f0d3684d241843f2647fef81d45725a178d27e23afb16068218272f source/tests/digital_surface/test_book_ledger.py + 774575a18473899bfd2304f93be3786c60c0288fd28aa98d94a40aa043c74858 source/tests/digital_surface/test_diagnostic_filter.py + adf17ffb26a23332194eaa67c07201e713f2edf81927365cb54b21a348b5a7bd source/tests/digital_surface/test_market_discovery.py + 6efbf535ddfb50ca4b88209ecfa72c8e208a9f4049b0960781805cb888449b53 source/tests/digital_surface/test_prospective.py + 53b3d577327ab9b5fee56c1905a585f9caf2ef0d2f91c6e100ce83371da6f494 source/tests/digital_surface/test_public_book_observer.py + cf1f1c4cde4b0ec9db7766ada185640c4fbfc20bbc55bb7869b61b9f528df2b8 source/tests/digital_surface/test_public_books.py + 87e32c6fc215d8ffe8f587357acd1d3efc12b1c78021f2f01740ebc06b9a0554 source/tests/digital_surface/test_public_observer.py + EOF + sed -i 's/^ //' /tmp/public-book-source.sha256 + sha256sum -c /tmp/public-book-source.sha256 + cp /tmp/public-book-source.sha256 source/PUBLIC_BOOK_OBSERVER_SOURCE_SHA256.txt + + - name: Install pinned test stack + run: | + python -m pip install --upgrade pip + python -m pip install numpy==2.2.6 pandas==2.2.3 scipy==1.15.3 scikit-learn==1.6.1 pyarrow==19.0.1 requests==2.32.3 pytest==8.3.5 tabulate==0.9.0 + + - name: Compile and pass exactly 60 tests + working-directory: source + shell: bash + run: | + set -euo pipefail + python -m compileall -q research/digital_surface tests/digital_surface + PYTHONPATH=. pytest --collect-only -q tests/digital_surface | tee /tmp/collect.txt + grep -Eq '^60 tests collected' /tmp/collect.txt + PYTHONPATH=. pytest tests/digital_surface -q + + - name: Verify public-only surfaces and chained evidence + working-directory: source + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import ast + import hashlib + import json + import tempfile + from decimal import Decimal + from pathlib import Path + + from research.digital_surface.book_ledger import PublicBookLedger + from research.digital_surface.diagnostic_filter import DiagnosticBloomFilter, build_diagnostic_bloom + from research.digital_surface.live import LiveTradingDisabledError, load_config + from research.digital_surface.market_discovery import GammaMarketRecord + from research.digital_surface.prospective import ProspectiveLedger + from research.digital_surface.public_book_observer import observe_public_books + from research.digital_surface.public_books import CLOB_BOOKS_URL, parse_public_order_book + from research.digital_surface.public_observer import GAMMA_MARKETS_URL, observe_markets + + try: + load_config({'TRADING_MODE': 'live', 'LIVE_ARM_TOKEN': 'present'}) + except LiveTradingDisabledError: + pass + else: + raise AssertionError('live mode unexpectedly armed') + + assert GAMMA_MARKETS_URL == 'https://gamma-api.polymarket.com/markets' + assert CLOB_BOOKS_URL == 'https://clob.polymarket.com/books' + forbidden_literals = { + 'https://clob.polymarket.com/order', + 'wss://ws-subscriptions-clob.polymarket.com/ws/user', + } + for path in Path('research/digital_surface').glob('public_*.py'): + text = path.read_text() + assert not any(value in text for value in forbidden_literals), path + + bloom = DiagnosticBloomFilter.from_payload( + build_diagnostic_bloom(['historical'], source_sha256='a'*64, m_bits=1024, k_hashes=4) + ) + with tempfile.TemporaryDirectory() as directory: + directory = Path(directory) + market_ledger = ProspectiveLedger(directory/'markets.jsonl', diagnostic_market_ids=bloom) + market = GammaMarketRecord( + condition_id='fresh', slug='btc-updown-5m-1999999800', question='Bitcoin Up or Down', + yes_token_id='yes-fresh', no_token_id='no-fresh', duration_seconds=300, + open_epoch_seconds=1_999_999_800, end_date='2033-05-18T03:35:00Z', + tick_size=Decimal('0.01'), + ) + market_summary = observe_markets( + [market], ledger=market_ledger, observed_ts_ms=1, + policy_sha256='1'*64, source_sha256='2'*64, + ) + books = { + token: parse_public_order_book({ + 'market': 'fresh', 'asset_id': token, 'timestamp': '1', 'hash': 'h-'+token, + 'bids': [{'price': '0.40', 'size': '10'}], + 'asks': [{'price': '0.50', 'size': '10'}], + 'min_order_size': '1', 'tick_size': '0.01', 'neg_risk': False, + }) + for token in ('yes-fresh', 'no-fresh') + } + book_ledger = PublicBookLedger(directory/'books.jsonl') + book_summary = observe_public_books( + [market], books, ledger=book_ledger, + prospective_market_ids=market_ledger.market_ids, + prospective_head_sha256=market_ledger.head_hash, + observed_ts_ms=2, policy_sha256='1'*64, source_sha256='2'*64, + ) + assert market_summary['appended'] == 1 + assert book_summary['books_appended'] == 2 + assert book_summary['prospective_head_sha256'] == market_ledger.head_hash + assert PublicBookLedger(book_ledger.path).head_hash == book_ledger.head_hash + + verified = {} + for line in Path('PUBLIC_BOOK_OBSERVER_SOURCE_SHA256.txt').read_text().splitlines(): + wanted, name = line.split(maxsplit=1) + got = hashlib.sha256(Path(name).read_bytes()).hexdigest() + assert got == wanted, (name, got, wanted) + verified[name] = got + result = { + 'tests_collected': 60, + 'public_market_endpoint': GAMMA_MARKETS_URL, + 'public_books_endpoint': CLOB_BOOKS_URL, + 'authenticated_requests': 0, + 'order_submissions': 0, + 'market_hash_chain_verified': True, + 'book_hash_chain_verified': True, + 'live_mode': 'physically_disabled', + 'source_files_verified': verified, + } + Path('PUBLIC_BOOK_OBSERVER_VERIFICATION.json').write_text(json.dumps(result, indent=2, sort_keys=True)) + print(json.dumps(result, indent=2)) + PY + + - name: Build deterministic source checkpoint + working-directory: source + shell: bash + run: | + set -euo pipefail + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-public-book-observer-source-v1.tar.gz \ + research tests PUBLIC_BOOK_OBSERVER_SOURCE_SHA256.txt PUBLIC_BOOK_OBSERVER_VERIFICATION.json + sha256sum digital-surface-public-book-observer-source-v1.tar.gz > digital-surface-public-book-observer-source-v1.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: digital-surface-public-book-observer-checkpoint-v1 + path: | + source/PUBLIC_BOOK_OBSERVER_SOURCE_SHA256.txt + source/PUBLIC_BOOK_OBSERVER_VERIFICATION.json + source/digital-surface-public-book-observer-source-v1.tar.gz + source/digital-surface-public-book-observer-source-v1.tar.gz.sha256 + retention-days: 30 From e735354b9bb8d6f2b89203eb0babb9727ee0b445 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 10:24:07 -0500 Subject: [PATCH 054/177] feat: checkpoint non-atomic paired FOK estimator --- .../research/digital_surface/shadow_fill.py | 152 ++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/shadow_fill.py diff --git a/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/shadow_fill.py b/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/shadow_fill.py new file mode 100644 index 00000000..f17793d7 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/shadow_fill.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +from dataclasses import dataclass +from decimal import Decimal +from typing import Any, Mapping + +from .public_books import BookLevel, PublicOrderBook + + +@dataclass(frozen=True, slots=True) +class BuyFokEstimate: + filled: bool + shares_requested: Decimal + shares_available: Decimal + gross_cost: Decimal + average_price: Decimal | None + worst_price: Decimal | None + + +@dataclass(frozen=True, slots=True) +class PairedFokEstimate: + paired_crossable: bool + orphanable_leg: str | None + low: BuyFokEstimate + high: BuyFokEstimate + combined_gross_cost: Decimal | None + transactionally_atomic: bool = False + + +def estimate_buy_fok( + asks: tuple[BookLevel, ...], + *, + shares: Decimal, + max_price: Decimal, +) -> BuyFokEstimate: + requested = Decimal(shares) + limit = Decimal(max_price) + if requested <= 0: + raise ValueError("shares must be positive") + if not (Decimal("0") < limit < Decimal("1")): + raise ValueError("max_price must be between zero and one") + remaining = requested + available = Decimal("0") + gross_cost = Decimal("0") + worst: Decimal | None = None + for level in asks: + if level.price > limit: + break + take = min(remaining, level.size) + if take <= 0: + continue + available += take + gross_cost += take * level.price + worst = level.price + remaining -= take + if remaining == 0: + break + filled = remaining == 0 + if not filled: + return BuyFokEstimate( + filled=False, + shares_requested=requested, + shares_available=available, + gross_cost=Decimal("0"), + average_price=None, + worst_price=None, + ) + return BuyFokEstimate( + filled=True, + shares_requested=requested, + shares_available=available, + gross_cost=gross_cost, + average_price=gross_cost / requested, + worst_price=worst, + ) + + +def estimate_paired_fok( + low_book: PublicOrderBook, + high_book: PublicOrderBook, + *, + shares: Decimal, + low_max_price: Decimal, + high_max_price: Decimal, +) -> PairedFokEstimate: + if low_book.token_id == high_book.token_id: + raise ValueError("paired legs must use distinct tokens") + low = estimate_buy_fok(low_book.asks, shares=shares, max_price=low_max_price) + high = estimate_buy_fok(high_book.asks, shares=shares, max_price=high_max_price) + orphanable_leg: str | None = None + if low.filled != high.filled: + orphanable_leg = "low_yes" if low.filled else "high_no" + paired = low.filled and high.filled + return PairedFokEstimate( + paired_crossable=paired, + orphanable_leg=orphanable_leg, + low=low, + high=high, + combined_gross_cost=(low.gross_cost + high.gross_cost) if paired else None, + transactionally_atomic=False, + ) + + +def estimate_pair_from_ledger_records( + low_record: Mapping[str, Any], + high_record: Mapping[str, Any], + *, + shares: Decimal, + low_max_price: Decimal, + high_max_price: Decimal, +) -> PairedFokEstimate: + for field in ("observed_ts_ms", "policy_sha256", "source_sha256", "prospective_head_sha256"): + if low_record.get(field) != high_record.get(field): + raise ValueError(f"paired evidence mismatch for {field}") + if str(low_record.get("outcome")) != "yes": + raise ValueError("low record must be the YES leg") + if str(high_record.get("outcome")) != "no": + raise ValueError("high record must be the NO leg") + if str(low_record.get("condition_id")) == str(high_record.get("condition_id")): + raise ValueError("paired legs must use distinct conditions") + return estimate_paired_fok( + _book_from_record(low_record), + _book_from_record(high_record), + shares=shares, + low_max_price=low_max_price, + high_max_price=high_max_price, + ) + + +def _book_from_record(record: Mapping[str, Any]) -> PublicOrderBook: + payload = record.get("book") + if not isinstance(payload, Mapping): + raise ValueError("ledger record is missing book data") + def levels(name: str) -> tuple[BookLevel, ...]: + raw = payload.get(name, []) + if not isinstance(raw, list): + raise ValueError(f"book {name} must be a list") + return tuple(BookLevel(Decimal(str(row["price"])), Decimal(str(row["size"]))) for row in raw) + last = payload.get("last_trade_price") + return PublicOrderBook( + condition_id=str(payload["condition_id"]), + token_id=str(payload["token_id"]), + timestamp=str(payload.get("timestamp") or ""), + book_hash=str(payload.get("book_hash") or ""), + bids=levels("bids"), + asks=levels("asks"), + min_order_size=Decimal(str(payload.get("min_order_size") or "0")), + tick_size=Decimal(str(payload.get("tick_size") or "0.01")), + neg_risk=bool(payload.get("neg_risk", False)), + last_trade_price=None if last in (None, "") else Decimal(str(last)), + payload_sha256=str(payload["payload_sha256"]), + ) From b1c96c891bcffeda2257d9834956b86cfaac67f2 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 10:24:32 -0500 Subject: [PATCH 055/177] test: checkpoint causal paired FOK estimator regressions --- .../tests/digital_surface/test_shadow_fill.py | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_shadow_fill.py diff --git a/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_shadow_fill.py b/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_shadow_fill.py new file mode 100644 index 00000000..01619224 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_shadow_fill.py @@ -0,0 +1,84 @@ +from decimal import Decimal + +import pytest + +from research.digital_surface.public_books import BookLevel, parse_public_order_book +from research.digital_surface.shadow_fill import ( + estimate_buy_fok, + estimate_pair_from_ledger_records, + estimate_paired_fok, +) + + +def book(token, condition, asks): + return parse_public_order_book({ + "market": condition, + "asset_id": token, + "timestamp": "1", + "hash": f"h-{token}", + "bids": [], + "asks": [{"price": str(price), "size": str(size)} for price, size in asks], + "min_order_size": "1", + "tick_size": "0.01", + "neg_risk": False, + }) + + +def record(item, outcome, observed=10, policy="1", source="2", prospective="3"): + return { + "condition_id": item.condition_id, + "outcome": outcome, + "observed_ts_ms": observed, + "policy_sha256": policy * 64, + "source_sha256": source * 64, + "prospective_head_sha256": prospective * 64, + "book": item.as_json(), + } + + +def test_walks_multiple_ask_levels_under_frozen_limit(): + result = estimate_buy_fok( + (BookLevel(Decimal("0.50"), Decimal("2")), BookLevel(Decimal("0.52"), Decimal("4"))), + shares=Decimal("5"), + max_price=Decimal("0.52"), + ) + assert result.filled is True + assert result.gross_cost == Decimal("2.56") + assert result.average_price == Decimal("0.512") + assert result.worst_price == Decimal("0.52") + + +def test_fok_estimate_discards_partial_cost_when_full_size_is_unavailable(): + result = estimate_buy_fok( + (BookLevel(Decimal("0.50"), Decimal("2")),), + shares=Decimal("5"), + max_price=Decimal("0.50"), + ) + assert result.filled is False + assert result.shares_available == Decimal("2") + assert result.gross_cost == Decimal("0") + assert result.average_price is None + + +def test_pair_is_crossable_but_never_claimed_transactionally_atomic(): + low = book("yes-low", "low", [("0.50", "5")]) + high = book("no-high", "high", [("0.49", "5")]) + result = estimate_paired_fok( + low, high, shares=Decimal("5"), + low_max_price=Decimal("0.51"), high_max_price=Decimal("0.50"), + ) + assert result.paired_crossable is True + assert result.combined_gross_cost == Decimal("4.95") + assert result.transactionally_atomic is False + assert result.orphanable_leg is None + + +def test_pair_records_require_same_causal_observation_and_bound_digests(): + low = book("yes-low", "low", [("0.50", "5")]) + high = book("no-high", "high", [("0.49", "5")]) + with pytest.raises(ValueError, match="observed_ts_ms"): + estimate_pair_from_ledger_records( + record(low, "yes", observed=10), + record(high, "no", observed=11), + shares=Decimal("5"), low_max_price=Decimal("0.51"), high_max_price=Decimal("0.50"), + ) From 852f59245b8ccf69d9616c2f4bdac41e41068140 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 10:25:46 -0500 Subject: [PATCH 056/177] refactor: isolate shadow fill simulation source --- .../research/digital_surface/shadow_fill.py | 152 ++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/simulation-source/research/digital_surface/shadow_fill.py diff --git a/ci_payload/digital-surface-live-readiness/simulation-source/research/digital_surface/shadow_fill.py b/ci_payload/digital-surface-live-readiness/simulation-source/research/digital_surface/shadow_fill.py new file mode 100644 index 00000000..f17793d7 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/simulation-source/research/digital_surface/shadow_fill.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +from dataclasses import dataclass +from decimal import Decimal +from typing import Any, Mapping + +from .public_books import BookLevel, PublicOrderBook + + +@dataclass(frozen=True, slots=True) +class BuyFokEstimate: + filled: bool + shares_requested: Decimal + shares_available: Decimal + gross_cost: Decimal + average_price: Decimal | None + worst_price: Decimal | None + + +@dataclass(frozen=True, slots=True) +class PairedFokEstimate: + paired_crossable: bool + orphanable_leg: str | None + low: BuyFokEstimate + high: BuyFokEstimate + combined_gross_cost: Decimal | None + transactionally_atomic: bool = False + + +def estimate_buy_fok( + asks: tuple[BookLevel, ...], + *, + shares: Decimal, + max_price: Decimal, +) -> BuyFokEstimate: + requested = Decimal(shares) + limit = Decimal(max_price) + if requested <= 0: + raise ValueError("shares must be positive") + if not (Decimal("0") < limit < Decimal("1")): + raise ValueError("max_price must be between zero and one") + remaining = requested + available = Decimal("0") + gross_cost = Decimal("0") + worst: Decimal | None = None + for level in asks: + if level.price > limit: + break + take = min(remaining, level.size) + if take <= 0: + continue + available += take + gross_cost += take * level.price + worst = level.price + remaining -= take + if remaining == 0: + break + filled = remaining == 0 + if not filled: + return BuyFokEstimate( + filled=False, + shares_requested=requested, + shares_available=available, + gross_cost=Decimal("0"), + average_price=None, + worst_price=None, + ) + return BuyFokEstimate( + filled=True, + shares_requested=requested, + shares_available=available, + gross_cost=gross_cost, + average_price=gross_cost / requested, + worst_price=worst, + ) + + +def estimate_paired_fok( + low_book: PublicOrderBook, + high_book: PublicOrderBook, + *, + shares: Decimal, + low_max_price: Decimal, + high_max_price: Decimal, +) -> PairedFokEstimate: + if low_book.token_id == high_book.token_id: + raise ValueError("paired legs must use distinct tokens") + low = estimate_buy_fok(low_book.asks, shares=shares, max_price=low_max_price) + high = estimate_buy_fok(high_book.asks, shares=shares, max_price=high_max_price) + orphanable_leg: str | None = None + if low.filled != high.filled: + orphanable_leg = "low_yes" if low.filled else "high_no" + paired = low.filled and high.filled + return PairedFokEstimate( + paired_crossable=paired, + orphanable_leg=orphanable_leg, + low=low, + high=high, + combined_gross_cost=(low.gross_cost + high.gross_cost) if paired else None, + transactionally_atomic=False, + ) + + +def estimate_pair_from_ledger_records( + low_record: Mapping[str, Any], + high_record: Mapping[str, Any], + *, + shares: Decimal, + low_max_price: Decimal, + high_max_price: Decimal, +) -> PairedFokEstimate: + for field in ("observed_ts_ms", "policy_sha256", "source_sha256", "prospective_head_sha256"): + if low_record.get(field) != high_record.get(field): + raise ValueError(f"paired evidence mismatch for {field}") + if str(low_record.get("outcome")) != "yes": + raise ValueError("low record must be the YES leg") + if str(high_record.get("outcome")) != "no": + raise ValueError("high record must be the NO leg") + if str(low_record.get("condition_id")) == str(high_record.get("condition_id")): + raise ValueError("paired legs must use distinct conditions") + return estimate_paired_fok( + _book_from_record(low_record), + _book_from_record(high_record), + shares=shares, + low_max_price=low_max_price, + high_max_price=high_max_price, + ) + + +def _book_from_record(record: Mapping[str, Any]) -> PublicOrderBook: + payload = record.get("book") + if not isinstance(payload, Mapping): + raise ValueError("ledger record is missing book data") + def levels(name: str) -> tuple[BookLevel, ...]: + raw = payload.get(name, []) + if not isinstance(raw, list): + raise ValueError(f"book {name} must be a list") + return tuple(BookLevel(Decimal(str(row["price"])), Decimal(str(row["size"]))) for row in raw) + last = payload.get("last_trade_price") + return PublicOrderBook( + condition_id=str(payload["condition_id"]), + token_id=str(payload["token_id"]), + timestamp=str(payload.get("timestamp") or ""), + book_hash=str(payload.get("book_hash") or ""), + bids=levels("bids"), + asks=levels("asks"), + min_order_size=Decimal(str(payload.get("min_order_size") or "0")), + tick_size=Decimal(str(payload.get("tick_size") or "0.01")), + neg_risk=bool(payload.get("neg_risk", False)), + last_trade_price=None if last in (None, "") else Decimal(str(last)), + payload_sha256=str(payload["payload_sha256"]), + ) From cf9a8c541dd35b162553d689a62558b646614a2a Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 10:26:12 -0500 Subject: [PATCH 057/177] refactor: isolate shadow fill simulation tests --- .../tests/digital_surface/test_shadow_fill.py | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/simulation-source/tests/digital_surface/test_shadow_fill.py diff --git a/ci_payload/digital-surface-live-readiness/simulation-source/tests/digital_surface/test_shadow_fill.py b/ci_payload/digital-surface-live-readiness/simulation-source/tests/digital_surface/test_shadow_fill.py new file mode 100644 index 00000000..01619224 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/simulation-source/tests/digital_surface/test_shadow_fill.py @@ -0,0 +1,84 @@ +from decimal import Decimal + +import pytest + +from research.digital_surface.public_books import BookLevel, parse_public_order_book +from research.digital_surface.shadow_fill import ( + estimate_buy_fok, + estimate_pair_from_ledger_records, + estimate_paired_fok, +) + + +def book(token, condition, asks): + return parse_public_order_book({ + "market": condition, + "asset_id": token, + "timestamp": "1", + "hash": f"h-{token}", + "bids": [], + "asks": [{"price": str(price), "size": str(size)} for price, size in asks], + "min_order_size": "1", + "tick_size": "0.01", + "neg_risk": False, + }) + + +def record(item, outcome, observed=10, policy="1", source="2", prospective="3"): + return { + "condition_id": item.condition_id, + "outcome": outcome, + "observed_ts_ms": observed, + "policy_sha256": policy * 64, + "source_sha256": source * 64, + "prospective_head_sha256": prospective * 64, + "book": item.as_json(), + } + + +def test_walks_multiple_ask_levels_under_frozen_limit(): + result = estimate_buy_fok( + (BookLevel(Decimal("0.50"), Decimal("2")), BookLevel(Decimal("0.52"), Decimal("4"))), + shares=Decimal("5"), + max_price=Decimal("0.52"), + ) + assert result.filled is True + assert result.gross_cost == Decimal("2.56") + assert result.average_price == Decimal("0.512") + assert result.worst_price == Decimal("0.52") + + +def test_fok_estimate_discards_partial_cost_when_full_size_is_unavailable(): + result = estimate_buy_fok( + (BookLevel(Decimal("0.50"), Decimal("2")),), + shares=Decimal("5"), + max_price=Decimal("0.50"), + ) + assert result.filled is False + assert result.shares_available == Decimal("2") + assert result.gross_cost == Decimal("0") + assert result.average_price is None + + +def test_pair_is_crossable_but_never_claimed_transactionally_atomic(): + low = book("yes-low", "low", [("0.50", "5")]) + high = book("no-high", "high", [("0.49", "5")]) + result = estimate_paired_fok( + low, high, shares=Decimal("5"), + low_max_price=Decimal("0.51"), high_max_price=Decimal("0.50"), + ) + assert result.paired_crossable is True + assert result.combined_gross_cost == Decimal("4.95") + assert result.transactionally_atomic is False + assert result.orphanable_leg is None + + +def test_pair_records_require_same_causal_observation_and_bound_digests(): + low = book("yes-low", "low", [("0.50", "5")]) + high = book("no-high", "high", [("0.49", "5")]) + with pytest.raises(ValueError, match="observed_ts_ms"): + estimate_pair_from_ledger_records( + record(low, "yes", observed=10), + record(high, "no", observed=11), + shares=Decimal("5"), low_max_price=Decimal("0.51"), high_max_price=Decimal("0.50"), + ) From ae88d2da6c10f6b0ef50d1cfd043fb639fb64db2 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 10:26:25 -0500 Subject: [PATCH 058/177] refactor: move shadow fill source to simulation payload --- .../research/digital_surface/shadow_fill.py | 152 ------------------ 1 file changed, 152 deletions(-) delete mode 100644 ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/shadow_fill.py diff --git a/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/shadow_fill.py b/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/shadow_fill.py deleted file mode 100644 index f17793d7..00000000 --- a/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/shadow_fill.py +++ /dev/null @@ -1,152 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from decimal import Decimal -from typing import Any, Mapping - -from .public_books import BookLevel, PublicOrderBook - - -@dataclass(frozen=True, slots=True) -class BuyFokEstimate: - filled: bool - shares_requested: Decimal - shares_available: Decimal - gross_cost: Decimal - average_price: Decimal | None - worst_price: Decimal | None - - -@dataclass(frozen=True, slots=True) -class PairedFokEstimate: - paired_crossable: bool - orphanable_leg: str | None - low: BuyFokEstimate - high: BuyFokEstimate - combined_gross_cost: Decimal | None - transactionally_atomic: bool = False - - -def estimate_buy_fok( - asks: tuple[BookLevel, ...], - *, - shares: Decimal, - max_price: Decimal, -) -> BuyFokEstimate: - requested = Decimal(shares) - limit = Decimal(max_price) - if requested <= 0: - raise ValueError("shares must be positive") - if not (Decimal("0") < limit < Decimal("1")): - raise ValueError("max_price must be between zero and one") - remaining = requested - available = Decimal("0") - gross_cost = Decimal("0") - worst: Decimal | None = None - for level in asks: - if level.price > limit: - break - take = min(remaining, level.size) - if take <= 0: - continue - available += take - gross_cost += take * level.price - worst = level.price - remaining -= take - if remaining == 0: - break - filled = remaining == 0 - if not filled: - return BuyFokEstimate( - filled=False, - shares_requested=requested, - shares_available=available, - gross_cost=Decimal("0"), - average_price=None, - worst_price=None, - ) - return BuyFokEstimate( - filled=True, - shares_requested=requested, - shares_available=available, - gross_cost=gross_cost, - average_price=gross_cost / requested, - worst_price=worst, - ) - - -def estimate_paired_fok( - low_book: PublicOrderBook, - high_book: PublicOrderBook, - *, - shares: Decimal, - low_max_price: Decimal, - high_max_price: Decimal, -) -> PairedFokEstimate: - if low_book.token_id == high_book.token_id: - raise ValueError("paired legs must use distinct tokens") - low = estimate_buy_fok(low_book.asks, shares=shares, max_price=low_max_price) - high = estimate_buy_fok(high_book.asks, shares=shares, max_price=high_max_price) - orphanable_leg: str | None = None - if low.filled != high.filled: - orphanable_leg = "low_yes" if low.filled else "high_no" - paired = low.filled and high.filled - return PairedFokEstimate( - paired_crossable=paired, - orphanable_leg=orphanable_leg, - low=low, - high=high, - combined_gross_cost=(low.gross_cost + high.gross_cost) if paired else None, - transactionally_atomic=False, - ) - - -def estimate_pair_from_ledger_records( - low_record: Mapping[str, Any], - high_record: Mapping[str, Any], - *, - shares: Decimal, - low_max_price: Decimal, - high_max_price: Decimal, -) -> PairedFokEstimate: - for field in ("observed_ts_ms", "policy_sha256", "source_sha256", "prospective_head_sha256"): - if low_record.get(field) != high_record.get(field): - raise ValueError(f"paired evidence mismatch for {field}") - if str(low_record.get("outcome")) != "yes": - raise ValueError("low record must be the YES leg") - if str(high_record.get("outcome")) != "no": - raise ValueError("high record must be the NO leg") - if str(low_record.get("condition_id")) == str(high_record.get("condition_id")): - raise ValueError("paired legs must use distinct conditions") - return estimate_paired_fok( - _book_from_record(low_record), - _book_from_record(high_record), - shares=shares, - low_max_price=low_max_price, - high_max_price=high_max_price, - ) - - -def _book_from_record(record: Mapping[str, Any]) -> PublicOrderBook: - payload = record.get("book") - if not isinstance(payload, Mapping): - raise ValueError("ledger record is missing book data") - def levels(name: str) -> tuple[BookLevel, ...]: - raw = payload.get(name, []) - if not isinstance(raw, list): - raise ValueError(f"book {name} must be a list") - return tuple(BookLevel(Decimal(str(row["price"])), Decimal(str(row["size"]))) for row in raw) - last = payload.get("last_trade_price") - return PublicOrderBook( - condition_id=str(payload["condition_id"]), - token_id=str(payload["token_id"]), - timestamp=str(payload.get("timestamp") or ""), - book_hash=str(payload.get("book_hash") or ""), - bids=levels("bids"), - asks=levels("asks"), - min_order_size=Decimal(str(payload.get("min_order_size") or "0")), - tick_size=Decimal(str(payload.get("tick_size") or "0.01")), - neg_risk=bool(payload.get("neg_risk", False)), - last_trade_price=None if last in (None, "") else Decimal(str(last)), - payload_sha256=str(payload["payload_sha256"]), - ) From 416349932921c7d4e997997a383b79ce81dfd3cd Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 10:26:36 -0500 Subject: [PATCH 059/177] refactor: move shadow fill tests to simulation payload --- .../tests/digital_surface/test_shadow_fill.py | 84 ------------------- 1 file changed, 84 deletions(-) delete mode 100644 ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_shadow_fill.py diff --git a/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_shadow_fill.py b/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_shadow_fill.py deleted file mode 100644 index 01619224..00000000 --- a/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_shadow_fill.py +++ /dev/null @@ -1,84 +0,0 @@ -from decimal import Decimal - -import pytest - -from research.digital_surface.public_books import BookLevel, parse_public_order_book -from research.digital_surface.shadow_fill import ( - estimate_buy_fok, - estimate_pair_from_ledger_records, - estimate_paired_fok, -) - - -def book(token, condition, asks): - return parse_public_order_book({ - "market": condition, - "asset_id": token, - "timestamp": "1", - "hash": f"h-{token}", - "bids": [], - "asks": [{"price": str(price), "size": str(size)} for price, size in asks], - "min_order_size": "1", - "tick_size": "0.01", - "neg_risk": False, - }) - - -def record(item, outcome, observed=10, policy="1", source="2", prospective="3"): - return { - "condition_id": item.condition_id, - "outcome": outcome, - "observed_ts_ms": observed, - "policy_sha256": policy * 64, - "source_sha256": source * 64, - "prospective_head_sha256": prospective * 64, - "book": item.as_json(), - } - - -def test_walks_multiple_ask_levels_under_frozen_limit(): - result = estimate_buy_fok( - (BookLevel(Decimal("0.50"), Decimal("2")), BookLevel(Decimal("0.52"), Decimal("4"))), - shares=Decimal("5"), - max_price=Decimal("0.52"), - ) - assert result.filled is True - assert result.gross_cost == Decimal("2.56") - assert result.average_price == Decimal("0.512") - assert result.worst_price == Decimal("0.52") - - -def test_fok_estimate_discards_partial_cost_when_full_size_is_unavailable(): - result = estimate_buy_fok( - (BookLevel(Decimal("0.50"), Decimal("2")),), - shares=Decimal("5"), - max_price=Decimal("0.50"), - ) - assert result.filled is False - assert result.shares_available == Decimal("2") - assert result.gross_cost == Decimal("0") - assert result.average_price is None - - -def test_pair_is_crossable_but_never_claimed_transactionally_atomic(): - low = book("yes-low", "low", [("0.50", "5")]) - high = book("no-high", "high", [("0.49", "5")]) - result = estimate_paired_fok( - low, high, shares=Decimal("5"), - low_max_price=Decimal("0.51"), high_max_price=Decimal("0.50"), - ) - assert result.paired_crossable is True - assert result.combined_gross_cost == Decimal("4.95") - assert result.transactionally_atomic is False - assert result.orphanable_leg is None - - -def test_pair_records_require_same_causal_observation_and_bound_digests(): - low = book("yes-low", "low", [("0.50", "5")]) - high = book("no-high", "high", [("0.49", "5")]) - with pytest.raises(ValueError, match="observed_ts_ms"): - estimate_pair_from_ledger_records( - record(low, "yes", observed=10), - record(high, "no", observed=11), - shares=Decimal("5"), low_max_price=Decimal("0.51"), high_max_price=Decimal("0.50"), - ) From a92747853966ac6665075ce75a70701900eaf2ce Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 10:27:21 -0500 Subject: [PATCH 060/177] ci: verify non-atomic shadow fill simulation --- ...face-shadow-fill-simulation-checkpoint.yml | 169 ++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100644 .github/workflows/digital-surface-shadow-fill-simulation-checkpoint.yml diff --git a/.github/workflows/digital-surface-shadow-fill-simulation-checkpoint.yml b/.github/workflows/digital-surface-shadow-fill-simulation-checkpoint.yml new file mode 100644 index 00000000..b3c0f9ec --- /dev/null +++ b/.github/workflows/digital-surface-shadow-fill-simulation-checkpoint.yml @@ -0,0 +1,169 @@ +name: Digital Surface Shadow Fill Simulation Checkpoint + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/digital-surface-shadow-fill-simulation-checkpoint.yml + - ci_payload/digital-surface-live-readiness/simulation-source/** + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-shadow-fill-simulation-checkpoint.yml + - ci_payload/digital-surface-live-readiness/simulation-source/** + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + verify-shadow-fill-simulation: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct checksum-pinned source + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source && tar -xzf base.tar.gz -C source + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + for patch_file in timestamp-unit-fix.patch full-universe-gate-fix.patch resolved-universe-fix.patch asset-merge-fix.patch; do + patch -d source -p1 < "ci_payload/digital-surface-official-resolution-patch/${patch_file}" + done + echo 'f71f3a653ca58b4bf98609fc29d8ab16f9d031c9ed14906ebe5159f2b0de8eab ci_payload/digital-surface-live-readiness/execution-diagnostics.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/execution-diagnostics.patch + echo '8606cc925aa3b3a6417abb1fba9a11df916cc2b281880978fb6229aa25875be9 ci_payload/digital-surface-live-readiness/book-arrival-model.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/book-arrival-model.patch + cp -a ci_payload/digital-surface-live-readiness/shadow-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/prospective-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/observer-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/simulation-source/. source/ + + cat > /tmp/simulation-source.sha256 <<'EOF' + 0808805ad3dcbb1c74ef437dddd33fc71b0800b4fd8ecf5daef5967de08bafe5 source/research/digital_surface/shadow_fill.py + d2fe18723b9b7780d3db82cef22f4d45c70c6e1f38e5fd5bcbaf72907eb6fb61 source/tests/digital_surface/test_shadow_fill.py + EOF + sed -i 's/^ //' /tmp/simulation-source.sha256 + sha256sum -c /tmp/simulation-source.sha256 + cp /tmp/simulation-source.sha256 source/SHADOW_FILL_SIMULATION_SOURCE_SHA256.txt + + - name: Install pinned test stack + run: | + python -m pip install --upgrade pip + python -m pip install numpy==2.2.6 pandas==2.2.3 scipy==1.15.3 scikit-learn==1.6.1 pyarrow==19.0.1 requests==2.32.3 pytest==8.3.5 tabulate==0.9.0 + + - name: Compile and pass exactly 64 tests + working-directory: source + shell: bash + run: | + set -euo pipefail + python -m compileall -q research/digital_surface tests/digital_surface + PYTHONPATH=. pytest --collect-only -q tests/digital_surface | tee /tmp/collect.txt + grep -Eq '^64 tests collected' /tmp/collect.txt + PYTHONPATH=. pytest tests/digital_surface -q + + - name: Verify non-atomic paired FOK semantics + working-directory: source + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import hashlib + import json + from decimal import Decimal + from pathlib import Path + + from research.digital_surface.public_books import parse_public_order_book + from research.digital_surface.shadow_fill import estimate_paired_fok + + def book(token, condition, asks): + return parse_public_order_book({ + 'market': condition, + 'asset_id': token, + 'timestamp': '1', + 'hash': 'h-' + token, + 'bids': [], + 'asks': [{'price': price, 'size': size} for price, size in asks], + 'min_order_size': '1', + 'tick_size': '0.01', + 'neg_risk': False, + }) + + paired = estimate_paired_fok( + book('yes-low', 'low', [('0.50', '2'), ('0.51', '3')]), + book('no-high', 'high', [('0.48', '5')]), + shares=Decimal('5'), + low_max_price=Decimal('0.51'), + high_max_price=Decimal('0.49'), + ) + assert paired.paired_crossable is True + assert paired.transactionally_atomic is False + assert paired.orphanable_leg is None + assert paired.low.average_price == Decimal('0.506') + assert paired.high.average_price == Decimal('0.48') + + orphanable = estimate_paired_fok( + book('yes-low', 'low', [('0.50', '5')]), + book('no-high', 'high', [('0.60', '5')]), + shares=Decimal('5'), + low_max_price=Decimal('0.51'), + high_max_price=Decimal('0.49'), + ) + assert orphanable.paired_crossable is False + assert orphanable.orphanable_leg == 'low_yes' + assert orphanable.transactionally_atomic is False + + verified = {} + for line in Path('SHADOW_FILL_SIMULATION_SOURCE_SHA256.txt').read_text().splitlines(): + wanted, name = line.split(maxsplit=1) + got = hashlib.sha256(Path(name).read_bytes()).hexdigest() + assert got == wanted + verified[name] = got + result = { + 'tests_collected': 64, + 'paired_crossable_is_not_atomic': True, + 'partial_fok_counted_as_unfilled': True, + 'causal_digest_binding_required': True, + 'order_submissions': 0, + 'live_mode': 'physically_disabled', + 'source_files_verified': verified, + } + Path('SHADOW_FILL_SIMULATION_VERIFICATION.json').write_text(json.dumps(result, indent=2, sort_keys=True)) + print(json.dumps(result, indent=2)) + PY + + - name: Build deterministic source checkpoint + working-directory: source + shell: bash + run: | + set -euo pipefail + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-shadow-fill-simulation-source-v1.tar.gz \ + research tests SHADOW_FILL_SIMULATION_SOURCE_SHA256.txt SHADOW_FILL_SIMULATION_VERIFICATION.json + sha256sum digital-surface-shadow-fill-simulation-source-v1.tar.gz > digital-surface-shadow-fill-simulation-source-v1.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: digital-surface-shadow-fill-simulation-checkpoint-v1 + path: | + source/SHADOW_FILL_SIMULATION_SOURCE_SHA256.txt + source/SHADOW_FILL_SIMULATION_VERIFICATION.json + source/digital-surface-shadow-fill-simulation-source-v1.tar.gz + source/digital-surface-shadow-fill-simulation-source-v1.tar.gz.sha256 + retention-days: 30 From cde31265beca1c5c2125bd5c073adc5777c1a118 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 10:37:00 -0500 Subject: [PATCH 061/177] feat: checkpoint public Binance BTC market-state ledger --- .../digital_surface/binance_public.py | 220 ++++++++++++++++++ .../research/digital_surface/spot_ledger.py | 145 ++++++++++++ .../digital_surface/test_binance_public.py | 130 +++++++++++ .../tests/digital_surface/test_spot_ledger.py | 72 ++++++ 4 files changed, 567 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/market-state-source/research/digital_surface/binance_public.py create mode 100644 ci_payload/digital-surface-live-readiness/market-state-source/research/digital_surface/spot_ledger.py create mode 100644 ci_payload/digital-surface-live-readiness/market-state-source/tests/digital_surface/test_binance_public.py create mode 100644 ci_payload/digital-surface-live-readiness/market-state-source/tests/digital_surface/test_spot_ledger.py diff --git a/ci_payload/digital-surface-live-readiness/market-state-source/research/digital_surface/binance_public.py b/ci_payload/digital-surface-live-readiness/market-state-source/research/digital_surface/binance_public.py new file mode 100644 index 00000000..f9dbb3d8 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/market-state-source/research/digital_surface/binance_public.py @@ -0,0 +1,220 @@ +from __future__ import annotations + +import hashlib +import json +import math +import statistics +import time +from dataclasses import dataclass +from decimal import Decimal +from typing import Any, Callable, Mapping, Sequence + + +BINANCE_MARKET_DATA_BASE = "https://data-api.binance.vision" +BINANCE_SYMBOL = "BTCUSDT" +_INTERVAL_MS = {"1s": 1_000, "1m": 60_000} + + +def _canonical_json(value: object) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _payload_sha256(value: object) -> str: + return hashlib.sha256(_canonical_json(value)).hexdigest() + + +@dataclass(frozen=True, slots=True) +class BinanceKline: + open_time_ms: int + open: Decimal + high: Decimal + low: Decimal + close: Decimal + volume: Decimal + close_time_ms: int + + +@dataclass(frozen=True, slots=True) +class BTCMarketState: + symbol: str + server_time_ms: int + observed_ts_ms: int + spot: Decimal + vol_30s: float + vol_120s: float + strikes: dict[int, Decimal] + closed_one_second_bars: int + raw_response_sha256: dict[str, str] + + def as_json(self) -> dict[str, object]: + return { + "symbol": self.symbol, + "server_time_ms": int(self.server_time_ms), + "observed_ts_ms": int(self.observed_ts_ms), + "spot": str(self.spot), + "vol_30s": float(self.vol_30s), + "vol_120s": float(self.vol_120s), + "strikes": {str(key): str(value) for key, value in sorted(self.strikes.items())}, + "closed_one_second_bars": int(self.closed_one_second_bars), + "raw_response_sha256": dict(sorted(self.raw_response_sha256.items())), + } + + +def parse_klines(payload: object, *, interval: str) -> tuple[BinanceKline, ...]: + if interval not in _INTERVAL_MS: + raise ValueError(f"unsupported kline interval: {interval}") + if not isinstance(payload, Sequence) or isinstance(payload, (str, bytes)): + raise ValueError("Binance klines payload must be a sequence") + interval_ms = _INTERVAL_MS[interval] + parsed: list[BinanceKline] = [] + for index, raw in enumerate(payload): + if not isinstance(raw, Sequence) or isinstance(raw, (str, bytes)) or len(raw) < 7: + raise ValueError(f"Binance kline row {index} is malformed") + open_time = int(raw[0]) + open_px = Decimal(str(raw[1])) + high = Decimal(str(raw[2])) + low = Decimal(str(raw[3])) + close = Decimal(str(raw[4])) + volume = Decimal(str(raw[5])) + close_time = int(raw[6]) + if min(open_px, high, low, close) <= 0: + raise ValueError("Binance kline prices must be positive") + if volume < 0: + raise ValueError("Binance kline volume must be non-negative") + if high < max(open_px, close) or low > min(open_px, close) or low > high: + raise ValueError("Binance kline OHLC values are inconsistent") + if close_time < open_time: + raise ValueError("Binance kline close time precedes open time") + parsed.append( + BinanceKline( + open_time_ms=open_time, + open=open_px, + high=high, + low=low, + close=close, + volume=volume, + close_time_ms=close_time, + ) + ) + parsed.sort(key=lambda item: item.open_time_ms) + for previous, current in zip(parsed, parsed[1:]): + if current.open_time_ms - previous.open_time_ms != interval_ms: + raise ValueError(f"Binance {interval} klines must be contiguous") + return tuple(parsed) + + +def realized_log_volatility(closes: Sequence[Decimal], *, returns: int) -> float: + if returns <= 0: + raise ValueError("returns must be positive") + required = returns + 1 + if len(closes) < required: + raise ValueError(f"realized volatility requires at least {required} close prices") + trailing = [Decimal(value) for value in closes[-required:]] + if any(value <= 0 for value in trailing): + raise ValueError("close prices must be positive") + log_returns = [math.log(float(current / previous)) for previous, current in zip(trailing, trailing[1:])] + return float(statistics.pstdev(log_returns)) + + +def _default_get(url: str, **kwargs: Any): + import requests + + return requests.get(url, **kwargs) + + +def _get_json( + get: Callable[..., Any], + url: str, + *, + params: Mapping[str, object] | None = None, + timeout_seconds: float = 15.0, +) -> object: + kwargs: dict[str, object] = {"timeout": timeout_seconds} + if params is not None: + kwargs["params"] = dict(params) + response = get(url, **kwargs) + response.raise_for_status() + return response.json() + + +def collect_btc_market_state( + boundary_open_ms: Sequence[int], + *, + get: Callable[..., Any] | None = None, + observed_ts_ms: int | None = None, + timeout_seconds: float = 15.0, +) -> BTCMarketState: + fetch = get or _default_get + server_payload = _get_json( + fetch, + f"{BINANCE_MARKET_DATA_BASE}/api/v3/time", + timeout_seconds=timeout_seconds, + ) + if not isinstance(server_payload, Mapping) or "serverTime" not in server_payload: + raise ValueError("Binance server-time response is malformed") + server_time_ms = int(server_payload["serverTime"]) + if server_time_ms < 0: + raise ValueError("Binance server time must be non-negative") + + one_second_payload = _get_json( + fetch, + f"{BINANCE_MARKET_DATA_BASE}/api/v3/klines", + params={ + "symbol": BINANCE_SYMBOL, + "interval": "1s", + "endTime": server_time_ms, + "limit": 122, + }, + timeout_seconds=timeout_seconds, + ) + one_second_all = parse_klines(one_second_payload, interval="1s") + closed = [item for item in one_second_all if item.close_time_ms <= server_time_ms] + if len(closed) < 121: + raise ValueError(f"need at least 121 closed one-second bars, found {len(closed)}") + closed = closed[-121:] + for previous, current in zip(closed, closed[1:]): + if current.open_time_ms - previous.open_time_ms != 1_000: + raise ValueError("closed one-second klines must be contiguous") + closes = [item.close for item in closed] + + strikes: dict[int, Decimal] = {} + raw_hashes = { + "server_time": _payload_sha256(server_payload), + "one_second_klines": _payload_sha256(one_second_payload), + } + for boundary in sorted({int(value) for value in boundary_open_ms}): + if boundary < 0 or boundary % 60_000: + raise ValueError(f"strike boundary must be a non-negative minute epoch in milliseconds: {boundary}") + strike_payload = _get_json( + fetch, + f"{BINANCE_MARKET_DATA_BASE}/api/v3/klines", + params={ + "symbol": BINANCE_SYMBOL, + "interval": "1m", + "startTime": boundary, + "endTime": boundary + 59_999, + "limit": 1, + }, + timeout_seconds=timeout_seconds, + ) + strike_rows = parse_klines(strike_payload, interval="1m") + if len(strike_rows) != 1 or strike_rows[0].open_time_ms != boundary: + returned = None if not strike_rows else strike_rows[0].open_time_ms + raise ValueError(f"Binance did not return the exact boundary {boundary}; returned {returned}") + strikes[boundary] = strike_rows[0].open + raw_hashes[f"strike:{boundary}"] = _payload_sha256(strike_payload) + + observed = int(time.time() * 1000) if observed_ts_ms is None else int(observed_ts_ms) + if observed < 0: + raise ValueError("observed_ts_ms must be non-negative") + return BTCMarketState( + symbol=BINANCE_SYMBOL, + server_time_ms=server_time_ms, + observed_ts_ms=observed, + spot=closes[-1], + vol_30s=realized_log_volatility(closes, returns=30), + vol_120s=realized_log_volatility(closes, returns=120), + strikes=strikes, + closed_one_second_bars=len(closed), + raw_response_sha256=raw_hashes, + ) diff --git a/ci_payload/digital-surface-live-readiness/market-state-source/research/digital_surface/spot_ledger.py b/ci_payload/digital-surface-live-readiness/market-state-source/research/digital_surface/spot_ledger.py new file mode 100644 index 00000000..e49df8d7 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/market-state-source/research/digital_surface/spot_ledger.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +from .binance_public import BTCMarketState + + +class SpotStateIntegrityError(RuntimeError): + """Raised when the append-only BTC state ledger cannot be verified.""" + + +def _canonical_json(value: dict[str, Any]) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _sha256(name: str, value: str) -> str: + text = str(value).lower() + if len(text) != 64 or any(character not in "0123456789abcdef" for character in text): + raise ValueError(f"{name} must be a 64-character SHA-256 hex digest") + return text + + +def _verify_state(state: object, *, line_number: int | None = None) -> dict[str, Any]: + suffix = "" if line_number is None else f" at line {line_number}" + if not isinstance(state, dict): + raise SpotStateIntegrityError(f"missing BTC state{suffix}") + required = { + "symbol", + "server_time_ms", + "observed_ts_ms", + "spot", + "vol_30s", + "vol_120s", + "strikes", + "closed_one_second_bars", + "raw_response_sha256", + } + missing = sorted(required - set(state)) + if missing: + raise SpotStateIntegrityError(f"BTC state missing fields {missing}{suffix}") + if str(state["symbol"]) != "BTCUSDT": + raise SpotStateIntegrityError(f"unexpected BTC state symbol{suffix}") + if int(state["server_time_ms"]) < 0 or int(state["observed_ts_ms"]) < 0: + raise SpotStateIntegrityError(f"invalid BTC state timestamps{suffix}") + if int(state["closed_one_second_bars"]) < 121: + raise SpotStateIntegrityError(f"insufficient closed one-second bars{suffix}") + if not isinstance(state["strikes"], dict) or not isinstance(state["raw_response_sha256"], dict): + raise SpotStateIntegrityError(f"invalid BTC state maps{suffix}") + try: + for value in state["raw_response_sha256"].values(): + _sha256("raw_response_sha256", str(value)) + except ValueError as exc: + raise SpotStateIntegrityError(f"invalid raw response hash{suffix}") from exc + return state + + +class SpotStateLedger: + def __init__(self, path: str | Path) -> None: + self.path = Path(path) + self.rows: list[dict[str, Any]] = [] + self.observation_ids: set[str] = set() + self.head_hash = "0" * 64 + if self.path.exists(): + self._load_and_verify() + + def _load_and_verify(self) -> None: + previous = "0" * 64 + rows: list[dict[str, Any]] = [] + seen: set[str] = set() + for line_number, line in enumerate(self.path.read_text(encoding="utf-8").splitlines(), start=1): + if not line.strip(): + continue + try: + record = json.loads(line) + except json.JSONDecodeError as exc: + raise SpotStateIntegrityError(f"invalid JSON at line {line_number}") from exc + stored_hash = str(record.get("record_hash", "")) + payload = dict(record) + payload.pop("record_hash", None) + if payload.get("previous_hash") != previous: + raise SpotStateIntegrityError(f"BTC state chain mismatch at line {line_number}") + expected_record_hash = hashlib.sha256(_canonical_json(payload)).hexdigest() + if stored_hash != expected_record_hash: + raise SpotStateIntegrityError(f"record hash mismatch at line {line_number}") + state = _verify_state(payload.get("state"), line_number=line_number) + expected_state_hash = hashlib.sha256(_canonical_json(state)).hexdigest() + if payload.get("state_sha256") != expected_state_hash: + raise SpotStateIntegrityError(f"state hash mismatch at line {line_number}") + observation_id = str(payload.get("observation_id") or "") + expected_observation = hashlib.sha256( + f"{state['server_time_ms']}\0{state['observed_ts_ms']}\0{expected_state_hash}".encode() + ).hexdigest() + if observation_id != expected_observation: + raise SpotStateIntegrityError(f"observation identity mismatch at line {line_number}") + if observation_id in seen: + raise SpotStateIntegrityError(f"duplicate BTC state snapshot at line {line_number}") + for field in ("policy_sha256", "source_sha256", "prospective_head_sha256"): + try: + _sha256(field, str(payload.get(field, ""))) + except ValueError as exc: + raise SpotStateIntegrityError(f"invalid {field} at line {line_number}") from exc + rows.append(record) + seen.add(observation_id) + previous = stored_hash + self.rows = rows + self.observation_ids = seen + self.head_hash = previous + + def append( + self, + state: BTCMarketState, + *, + policy_sha256: str, + source_sha256: str, + prospective_head_sha256: str, + ) -> dict[str, Any]: + state_json = state.as_json() + _verify_state(state_json) + state_hash = hashlib.sha256(_canonical_json(state_json)).hexdigest() + observation_id = hashlib.sha256( + f"{state.server_time_ms}\0{state.observed_ts_ms}\0{state_hash}".encode() + ).hexdigest() + if observation_id in self.observation_ids: + raise ValueError("duplicate BTC state snapshot") + payload = { + "observation_id": observation_id, + "policy_sha256": _sha256("policy_sha256", policy_sha256), + "previous_hash": self.head_hash, + "prospective_head_sha256": _sha256("prospective_head_sha256", prospective_head_sha256), + "source_sha256": _sha256("source_sha256", source_sha256), + "state": state_json, + "state_sha256": state_hash, + } + record = {**payload, "record_hash": hashlib.sha256(_canonical_json(payload)).hexdigest()} + self.path.parent.mkdir(parents=True, exist_ok=True) + with self.path.open("a", encoding="utf-8", newline="\n") as handle: + handle.write(json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n") + handle.flush() + self.rows.append(record) + self.observation_ids.add(observation_id) + self.head_hash = record["record_hash"] + return dict(record) diff --git a/ci_payload/digital-surface-live-readiness/market-state-source/tests/digital_surface/test_binance_public.py b/ci_payload/digital-surface-live-readiness/market-state-source/tests/digital_surface/test_binance_public.py new file mode 100644 index 00000000..af86c484 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/market-state-source/tests/digital_surface/test_binance_public.py @@ -0,0 +1,130 @@ +import hashlib +import json +import math +from decimal import Decimal + +import pytest + +from research.digital_surface.binance_public import ( + BINANCE_MARKET_DATA_BASE, + collect_btc_market_state, + parse_klines, + realized_log_volatility, +) + + +class Response: + def __init__(self, payload): + self._payload = payload + + def raise_for_status(self): + return None + + def json(self): + return self._payload + + +def row(open_ms: int, close: str, *, interval_ms: int = 1_000, open_price: str | None = None): + price = open_price or close + return [ + open_ms, + price, + str(Decimal(price) + Decimal("1")), + str(Decimal(price) - Decimal("1")), + close, + "1.0", + open_ms + interval_ms - 1, + "1.0", + 1, + "0.5", + "0.5", + "0", + ] + + +def test_parse_klines_rejects_nonpositive_prices_and_time_gaps(): + with pytest.raises(ValueError, match="positive"): + parse_klines([row(0, "0")], interval="1s") + + with pytest.raises(ValueError, match="contiguous"): + parse_klines([row(0, "100"), row(2_000, "101")], interval="1s") + + +def test_realized_log_volatility_uses_exact_trailing_return_count(): + closes = [Decimal(str(math.exp(index * 0.001))) for index in range(121)] + assert realized_log_volatility(closes, returns=30) == pytest.approx(0.0, abs=1e-15) + assert realized_log_volatility(closes, returns=120) == pytest.approx(0.0, abs=1e-15) + with pytest.raises(ValueError, match="121 close prices"): + realized_log_volatility(closes[:-1], returns=120) + + +def test_collects_closed_one_second_state_and_exact_boundary_strikes_without_auth_headers(): + boundary_a = 1_800_000 + boundary_b = 2_100_000 + server_time = 5_000_000 + observed_time = 5_000_123 + calls = [] + + closes = [Decimal("100") + Decimal(index) / Decimal("10") for index in range(122)] + one_second = [row(4_878_000 + index * 1_000, str(price)) for index, price in enumerate(closes)] + # The newest candle is still open at server_time and must be excluded. + one_second[-1][6] = server_time + 999 + + one_minute = { + boundary_a: [row(boundary_a, "101.25", interval_ms=60_000, open_price="101.25")], + boundary_b: [row(boundary_b, "102.50", interval_ms=60_000, open_price="102.50")], + } + + def get(url, **kwargs): + calls.append((url, kwargs)) + assert "headers" not in kwargs + if url == f"{BINANCE_MARKET_DATA_BASE}/api/v3/time": + return Response({"serverTime": server_time}) + assert url == f"{BINANCE_MARKET_DATA_BASE}/api/v3/klines" + params = kwargs["params"] + assert params["symbol"] == "BTCUSDT" + if params["interval"] == "1s": + assert params["limit"] == 122 + assert params["endTime"] == server_time + return Response(one_second) + assert params["interval"] == "1m" + assert params["limit"] == 1 + assert params["endTime"] == params["startTime"] + 59_999 + return Response(one_minute[params["startTime"]]) + + state = collect_btc_market_state( + [boundary_b, boundary_a, boundary_a], + get=get, + observed_ts_ms=observed_time, + ) + + assert state.server_time_ms == server_time + assert state.observed_ts_ms == observed_time + assert state.spot == closes[-2] + assert state.strikes == { + boundary_a: Decimal("101.25"), + boundary_b: Decimal("102.50"), + } + assert state.closed_one_second_bars == 121 + assert state.vol_30s >= 0 + assert state.vol_120s >= 0 + assert set(state.raw_response_sha256) == {"server_time", "one_second_klines", f"strike:{boundary_a}", f"strike:{boundary_b}"} + assert all(len(value) == 64 for value in state.raw_response_sha256.values()) + assert len(calls) == 4 + + +def test_exact_boundary_strike_fails_closed_when_binance_returns_another_minute(): + boundary = 1_800_000 + server_time = 5_000_000 + one_second = [row(4_879_000 + index * 1_000, str(100 + index / 10)) for index in range(121)] + + def get(url, **kwargs): + if url.endswith("/time"): + return Response({"serverTime": server_time}) + params = kwargs["params"] + if params["interval"] == "1s": + return Response(one_second) + return Response([row(boundary + 60_000, "100", interval_ms=60_000)]) + + with pytest.raises(ValueError, match="exact boundary"): + collect_btc_market_state([boundary], get=get, observed_ts_ms=server_time) diff --git a/ci_payload/digital-surface-live-readiness/market-state-source/tests/digital_surface/test_spot_ledger.py b/ci_payload/digital-surface-live-readiness/market-state-source/tests/digital_surface/test_spot_ledger.py new file mode 100644 index 00000000..dbec3a1e --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/market-state-source/tests/digital_surface/test_spot_ledger.py @@ -0,0 +1,72 @@ +import json + +import pytest + +from research.digital_surface.binance_public import BTCMarketState +from research.digital_surface.spot_ledger import SpotStateIntegrityError, SpotStateLedger + + +def state(): + return BTCMarketState( + symbol="BTCUSDT", + server_time_ms=2_000, + observed_ts_ms=2_100, + spot=__import__("decimal").Decimal("100.50"), + vol_30s=0.001, + vol_120s=0.002, + strikes={1_800_000: __import__("decimal").Decimal("99.00")}, + closed_one_second_bars=121, + raw_response_sha256={ + "server_time": "a" * 64, + "one_second_klines": "b" * 64, + "strike:1800000": "c" * 64, + }, + ) + + +def test_appends_state_bound_to_policy_source_and_prospective_head(tmp_path): + path = tmp_path / "btc_state.jsonl" + ledger = SpotStateLedger(path) + first = ledger.append( + state(), + policy_sha256="1" * 64, + source_sha256="2" * 64, + prospective_head_sha256="3" * 64, + ) + assert first["previous_hash"] == "0" * 64 + assert first["state"]["spot"] == "100.50" + assert first["state"]["strikes"] == {"1800000": "99.00"} + assert len(first["state_sha256"]) == 64 + + reloaded = SpotStateLedger(path) + assert reloaded.head_hash == first["record_hash"] + assert len(reloaded.rows) == 1 + + +def test_detects_state_or_digest_tampering(tmp_path): + path = tmp_path / "btc_state.jsonl" + ledger = SpotStateLedger(path) + ledger.append( + state(), + policy_sha256="1" * 64, + source_sha256="2" * 64, + prospective_head_sha256="3" * 64, + ) + row = json.loads(path.read_text()) + row["state"]["spot"] = "999999" + path.write_text(json.dumps(row) + "\n") + with pytest.raises(SpotStateIntegrityError, match="record hash mismatch|state hash mismatch"): + SpotStateLedger(path) + + +def test_rejects_duplicate_snapshot_identity(tmp_path): + path = tmp_path / "btc_state.jsonl" + ledger = SpotStateLedger(path) + kwargs = { + "policy_sha256": "1" * 64, + "source_sha256": "2" * 64, + "prospective_head_sha256": "3" * 64, + } + ledger.append(state(), **kwargs) + with pytest.raises(ValueError, match="duplicate BTC state snapshot"): + ledger.append(state(), **kwargs) From f632d08acbaf745c270fd65b2523ecd7a1fd108c Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 10:38:42 -0500 Subject: [PATCH 062/177] ci: verify public Binance market-state checkpoint --- ...urface-binance-market-state-checkpoint.yml | 189 ++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 .github/workflows/digital-surface-binance-market-state-checkpoint.yml diff --git a/.github/workflows/digital-surface-binance-market-state-checkpoint.yml b/.github/workflows/digital-surface-binance-market-state-checkpoint.yml new file mode 100644 index 00000000..598cd73a --- /dev/null +++ b/.github/workflows/digital-surface-binance-market-state-checkpoint.yml @@ -0,0 +1,189 @@ +name: Digital Surface Binance Market-State Checkpoint + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/digital-surface-binance-market-state-checkpoint.yml + - ci_payload/digital-surface-live-readiness/market-state-source/** + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-binance-market-state-checkpoint.yml + - ci_payload/digital-surface-live-readiness/market-state-source/** + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + verify-binance-market-state: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct checksum-pinned source + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source && tar -xzf base.tar.gz -C source + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + for patch_file in timestamp-unit-fix.patch full-universe-gate-fix.patch resolved-universe-fix.patch asset-merge-fix.patch; do + patch -d source -p1 < "ci_payload/digital-surface-official-resolution-patch/${patch_file}" + done + echo 'f71f3a653ca58b4bf98609fc29d8ab16f9d031c9ed14906ebe5159f2b0de8eab ci_payload/digital-surface-live-readiness/execution-diagnostics.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/execution-diagnostics.patch + echo '8606cc925aa3b3a6417abb1fba9a11df916cc2b281880978fb6229aa25875be9 ci_payload/digital-surface-live-readiness/book-arrival-model.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/book-arrival-model.patch + cp -a ci_payload/digital-surface-live-readiness/shadow-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/prospective-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/observer-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/simulation-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/market-state-source/. source/ + + cat > /tmp/market-state-source.sha256 <<'EOF' + 157594f006173023f5ee88ee2b1b72f5af3eba431288a448aab86a57bdd1949e source/research/digital_surface/binance_public.py + c50d2f58c964e38e2e8a5f30c561cda1ea85c6dbbfc6fa169cca161607874262 source/research/digital_surface/spot_ledger.py + 80e91c2d2214260d9bbf6eb7bf0d736d74581de98ed8ac16bcd2f20ab35d6913 source/tests/digital_surface/test_binance_public.py + f5eeb0c0f5a746f1556c7f4cfe801251327d5b7275edc2af1760a64fd0f9775f source/tests/digital_surface/test_spot_ledger.py + EOF + sed -i 's/^ //' /tmp/market-state-source.sha256 + sha256sum -c /tmp/market-state-source.sha256 + cp /tmp/market-state-source.sha256 source/BINANCE_MARKET_STATE_SOURCE_SHA256.txt + + - name: Install pinned test stack + run: | + python -m pip install --upgrade pip + python -m pip install numpy==2.2.6 pandas==2.2.3 scipy==1.15.3 scikit-learn==1.6.1 pyarrow==19.0.1 requests==2.32.3 pytest==8.3.5 tabulate==0.9.0 + + - name: Compile and pass exactly 71 tests + working-directory: source + shell: bash + run: | + set -euo pipefail + python -m compileall -q research/digital_surface tests/digital_surface + PYTHONPATH=. pytest --collect-only -q tests/digital_surface | tee /tmp/collect.txt + grep -Eq '^71 tests collected' /tmp/collect.txt + PYTHONPATH=. pytest tests/digital_surface -q + + - name: Verify public-only Binance state and tamper-evident ledger + working-directory: source + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import ast + import hashlib + import json + import tempfile + from decimal import Decimal + from pathlib import Path + + from research.digital_surface.binance_public import ( + BINANCE_MARKET_DATA_BASE, + BTCMarketState, + realized_log_volatility, + ) + from research.digital_surface.spot_ledger import SpotStateLedger + + assert BINANCE_MARKET_DATA_BASE == 'https://data-api.binance.vision' + tree = ast.parse(Path('research/digital_surface/binance_public.py').read_text()) + request_methods = { + node.func.attr for node in ast.walk(tree) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) + and getattr(node.func.value, 'id', None) == 'requests' + } + assert request_methods == {'get'}, request_methods + text = Path('research/digital_surface/binance_public.py').read_text() + assert 'api/v3/order' not in text + assert 'apiKey' not in text and 'signature' not in text + + closes = [Decimal('100') * (Decimal('1.0001') ** index) for index in range(121)] + assert realized_log_volatility(closes, returns=30) >= 0 + assert realized_log_volatility(closes, returns=120) >= 0 + + state = BTCMarketState( + symbol='BTCUSDT', + server_time_ms=2_000, + observed_ts_ms=2_100, + spot=Decimal('100.50'), + vol_30s=0.001, + vol_120s=0.002, + strikes={1_800_000: Decimal('99.00')}, + closed_one_second_bars=121, + raw_response_sha256={ + 'server_time': 'a' * 64, + 'one_second_klines': 'b' * 64, + 'strike:1800000': 'c' * 64, + }, + ) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'btc_state.jsonl' + ledger = SpotStateLedger(path) + record = ledger.append( + state, + policy_sha256='1' * 64, + source_sha256='2' * 64, + prospective_head_sha256='3' * 64, + ) + assert record['state']['closed_one_second_bars'] == 121 + assert record['state']['strikes'] == {'1800000': '99.00'} + assert SpotStateLedger(path).head_hash == record['record_hash'] + + verified = {} + for line in Path('BINANCE_MARKET_STATE_SOURCE_SHA256.txt').read_text().splitlines(): + wanted, name = line.split(maxsplit=1) + got = hashlib.sha256(Path(name).read_bytes()).hexdigest() + assert got == wanted, (name, got, wanted) + verified[name] = got + result = { + 'tests_collected': 71, + 'binance_market_data_base': BINANCE_MARKET_DATA_BASE, + 'symbol': 'BTCUSDT', + 'authenticated_requests': 0, + 'order_submissions': 0, + 'transport_methods': ['GET'], + 'closed_one_second_bars_required': 121, + 'exact_minute_boundary_strikes_required': True, + 'spot_state_hash_chain_verified': True, + 'live_mode': 'physically_disabled', + 'source_files_verified': verified, + } + Path('BINANCE_MARKET_STATE_VERIFICATION.json').write_text(json.dumps(result, indent=2, sort_keys=True)) + print(json.dumps(result, indent=2)) + PY + + - name: Build deterministic source checkpoint + working-directory: source + shell: bash + run: | + set -euo pipefail + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-binance-market-state-source-v1.tar.gz \ + research tests BINANCE_MARKET_STATE_SOURCE_SHA256.txt BINANCE_MARKET_STATE_VERIFICATION.json + sha256sum digital-surface-binance-market-state-source-v1.tar.gz > digital-surface-binance-market-state-source-v1.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: digital-surface-binance-market-state-checkpoint-v1 + path: | + source/BINANCE_MARKET_STATE_SOURCE_SHA256.txt + source/BINANCE_MARKET_STATE_VERIFICATION.json + source/digital-surface-binance-market-state-source-v1.tar.gz + source/digital-surface-binance-market-state-source-v1.tar.gz.sha256 + retention-days: 30 From b87807f72d4239ce2d8dc16e9554a12b6014a8d3 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 10:41:41 -0500 Subject: [PATCH 063/177] feat: integrate prospective BTC state observer --- .../digital_surface/public_state_observer.py | 103 +++++++++++++++++ .../test_public_state_observer.py | 104 ++++++++++++++++++ 2 files changed, 207 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/market-state-source/research/digital_surface/public_state_observer.py create mode 100644 ci_payload/digital-surface-live-readiness/market-state-source/tests/digital_surface/test_public_state_observer.py diff --git a/ci_payload/digital-surface-live-readiness/market-state-source/research/digital_surface/public_state_observer.py b/ci_payload/digital-surface-live-readiness/market-state-source/research/digital_surface/public_state_observer.py new file mode 100644 index 00000000..2d467db1 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/market-state-source/research/digital_surface/public_state_observer.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import time +from pathlib import Path +from typing import Callable, Sequence + +from .binance_public import BTCMarketState, BINANCE_MARKET_DATA_BASE, collect_btc_market_state +from .diagnostic_filter import DiagnosticBloomFilter +from .market_discovery import GammaMarketRecord, discover_target_markets +from .prospective import ProspectiveLedger +from .public_observer import fetch_gamma_page +from .spot_ledger import SpotStateLedger + + +def _sha256_file(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def observe_public_btc_state( + markets: Sequence[GammaMarketRecord], + *, + prospective_market_ids: set[str], + prospective_head_sha256: str, + ledger: SpotStateLedger, + observed_ts_ms: int, + policy_sha256: str, + source_sha256: str, + collect: Callable[..., BTCMarketState] = collect_btc_market_state, +) -> dict[str, object]: + registered = [market for market in markets if market.condition_id in prospective_market_ids] + boundaries = sorted({int(market.open_epoch_seconds) * 1_000 for market in registered}) + state = collect(boundaries, observed_ts_ms=int(observed_ts_ms)) + if set(state.strikes) != set(boundaries): + raise ValueError( + f"BTC state strike set does not match registered market boundaries: " + f"{sorted(state.strikes)} != {boundaries}" + ) + record = ledger.append( + state, + policy_sha256=policy_sha256, + source_sha256=source_sha256, + prospective_head_sha256=prospective_head_sha256, + ) + return { + "markets_discovered": len(markets), + "markets_registered": len(registered), + "markets_skipped_unregistered": len(markets) - len(registered), + "strike_boundaries": boundaries, + "spot": str(state.spot), + "vol_30s": state.vol_30s, + "vol_120s": state.vol_120s, + "server_time_ms": state.server_time_ms, + "observed_ts_ms": state.observed_ts_ms, + "spot_state_head_sha256": ledger.head_hash, + "spot_state_record_sha256": record["record_hash"], + "spot_state_observations_total": len(ledger.rows), + "prospective_head_sha256": prospective_head_sha256, + "binance_market_data_base": BINANCE_MARKET_DATA_BASE, + "authenticated_requests": 0, + "order_submissions": 0, + "trading_mode": "observation_only", + } + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Collect public BTC market state without credentials") + parser.add_argument("--diagnostic-filter", type=Path, required=True) + parser.add_argument("--prospective-ledger", type=Path, required=True) + parser.add_argument("--spot-ledger", type=Path, required=True) + parser.add_argument("--policy", type=Path, required=True) + parser.add_argument("--source-manifest", type=Path, required=True) + parser.add_argument("--summary", type=Path, required=True) + parser.add_argument("--page-size", type=int, default=100) + parser.add_argument("--max-pages", type=int, default=20) + args = parser.parse_args(argv) + + bloom = DiagnosticBloomFilter.from_path(args.diagnostic_filter) + prospective = ProspectiveLedger(args.prospective_ledger, diagnostic_market_ids=bloom) + markets = discover_target_markets( + lambda params: fetch_gamma_page(params), + page_size=args.page_size, + max_pages=args.max_pages, + ) + summary = observe_public_btc_state( + markets, + prospective_market_ids=prospective.market_ids, + prospective_head_sha256=prospective.head_hash, + ledger=SpotStateLedger(args.spot_ledger), + observed_ts_ms=int(time.time() * 1_000), + policy_sha256=_sha256_file(args.policy), + source_sha256=_sha256_file(args.source_manifest), + ) + args.summary.parent.mkdir(parents=True, exist_ok=True) + args.summary.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ci_payload/digital-surface-live-readiness/market-state-source/tests/digital_surface/test_public_state_observer.py b/ci_payload/digital-surface-live-readiness/market-state-source/tests/digital_surface/test_public_state_observer.py new file mode 100644 index 00000000..39c9473e --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/market-state-source/tests/digital_surface/test_public_state_observer.py @@ -0,0 +1,104 @@ +from decimal import Decimal + +from research.digital_surface.binance_public import BTCMarketState +from research.digital_surface.market_discovery import GammaMarketRecord +from research.digital_surface.public_state_observer import observe_public_btc_state +from research.digital_surface.spot_ledger import SpotStateLedger + + +def market(condition: str, epoch: int, duration: int = 300) -> GammaMarketRecord: + return GammaMarketRecord( + condition_id=condition, + slug=f"btc-updown-{duration // 60}m-{epoch}", + question="Bitcoin Up or Down", + yes_token_id=f"yes-{condition}", + no_token_id=f"no-{condition}", + duration_seconds=duration, + open_epoch_seconds=epoch, + end_date="2030-01-01T00:00:00Z", + tick_size=Decimal("0.01"), + ) + + +def test_collects_unique_boundaries_only_for_registered_prospective_markets(tmp_path): + markets = [ + market("registered-5", 1_800), + market("registered-15", 1_800, duration=900), + market("unregistered", 2_100), + ] + calls = [] + + def collect(boundaries, *, observed_ts_ms): + calls.append((list(boundaries), observed_ts_ms)) + return BTCMarketState( + symbol="BTCUSDT", + server_time_ms=5_000, + observed_ts_ms=observed_ts_ms, + spot=Decimal("100.50"), + vol_30s=0.001, + vol_120s=0.002, + strikes={1_800_000: Decimal("99.00")}, + closed_one_second_bars=121, + raw_response_sha256={ + "server_time": "a" * 64, + "one_second_klines": "b" * 64, + "strike:1800000": "c" * 64, + }, + ) + + ledger = SpotStateLedger(tmp_path / "btc_state.jsonl") + summary = observe_public_btc_state( + markets, + prospective_market_ids={"registered-5", "registered-15"}, + prospective_head_sha256="3" * 64, + ledger=ledger, + observed_ts_ms=5_100, + policy_sha256="1" * 64, + source_sha256="2" * 64, + collect=collect, + ) + + assert calls == [([1_800_000], 5_100)] + assert summary["markets_registered"] == 2 + assert summary["markets_skipped_unregistered"] == 1 + assert summary["strike_boundaries"] == [1_800_000] + assert summary["spot_state_head_sha256"] == ledger.head_hash + assert summary["authenticated_requests"] == 0 + assert summary["order_submissions"] == 0 + + +def test_empty_registered_set_still_records_causal_spot_without_strikes(tmp_path): + calls = [] + + def collect(boundaries, *, observed_ts_ms): + calls.append(list(boundaries)) + return BTCMarketState( + symbol="BTCUSDT", + server_time_ms=5_000, + observed_ts_ms=observed_ts_ms, + spot=Decimal("100.50"), + vol_30s=0.001, + vol_120s=0.002, + strikes={}, + closed_one_second_bars=121, + raw_response_sha256={ + "server_time": "a" * 64, + "one_second_klines": "b" * 64, + }, + ) + + ledger = SpotStateLedger(tmp_path / "btc_state.jsonl") + summary = observe_public_btc_state( + [market("unregistered", 2_100)], + prospective_market_ids=set(), + prospective_head_sha256="3" * 64, + ledger=ledger, + observed_ts_ms=5_100, + policy_sha256="1" * 64, + source_sha256="2" * 64, + collect=collect, + ) + assert calls == [[]] + assert summary["markets_registered"] == 0 + assert summary["strike_boundaries"] == [] + assert len(ledger.rows) == 1 From 0724fa02c66cc067ef7750c0f7045d27421a762e Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 10:42:37 -0500 Subject: [PATCH 064/177] refactor: isolate prospective BTC state observer payload --- .../research/digital_surface/public_state_observer.py | 0 .../tests/digital_surface/test_public_state_observer.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename ci_payload/digital-surface-live-readiness/{market-state-source => market-state-observer-source}/research/digital_surface/public_state_observer.py (100%) rename ci_payload/digital-surface-live-readiness/{market-state-source => market-state-observer-source}/tests/digital_surface/test_public_state_observer.py (100%) diff --git a/ci_payload/digital-surface-live-readiness/market-state-source/research/digital_surface/public_state_observer.py b/ci_payload/digital-surface-live-readiness/market-state-observer-source/research/digital_surface/public_state_observer.py similarity index 100% rename from ci_payload/digital-surface-live-readiness/market-state-source/research/digital_surface/public_state_observer.py rename to ci_payload/digital-surface-live-readiness/market-state-observer-source/research/digital_surface/public_state_observer.py diff --git a/ci_payload/digital-surface-live-readiness/market-state-source/tests/digital_surface/test_public_state_observer.py b/ci_payload/digital-surface-live-readiness/market-state-observer-source/tests/digital_surface/test_public_state_observer.py similarity index 100% rename from ci_payload/digital-surface-live-readiness/market-state-source/tests/digital_surface/test_public_state_observer.py rename to ci_payload/digital-surface-live-readiness/market-state-observer-source/tests/digital_surface/test_public_state_observer.py From 574d747be04ce3bdac91c059ddf288676ee28ced Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 10:43:40 -0500 Subject: [PATCH 065/177] ci: verify prospective Binance state observer integration --- ...face-binance-state-observer-checkpoint.yml | 192 ++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 .github/workflows/digital-surface-binance-state-observer-checkpoint.yml diff --git a/.github/workflows/digital-surface-binance-state-observer-checkpoint.yml b/.github/workflows/digital-surface-binance-state-observer-checkpoint.yml new file mode 100644 index 00000000..caae6297 --- /dev/null +++ b/.github/workflows/digital-surface-binance-state-observer-checkpoint.yml @@ -0,0 +1,192 @@ +name: Digital Surface Binance State Observer Checkpoint + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/digital-surface-binance-state-observer-checkpoint.yml + - ci_payload/digital-surface-live-readiness/market-state-observer-source/** + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-binance-state-observer-checkpoint.yml + - ci_payload/digital-surface-live-readiness/market-state-observer-source/** + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + verify-binance-state-observer: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct checksum-pinned source + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source && tar -xzf base.tar.gz -C source + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + for patch_file in timestamp-unit-fix.patch full-universe-gate-fix.patch resolved-universe-fix.patch asset-merge-fix.patch; do + patch -d source -p1 < "ci_payload/digital-surface-official-resolution-patch/${patch_file}" + done + echo 'f71f3a653ca58b4bf98609fc29d8ab16f9d031c9ed14906ebe5159f2b0de8eab ci_payload/digital-surface-live-readiness/execution-diagnostics.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/execution-diagnostics.patch + echo '8606cc925aa3b3a6417abb1fba9a11df916cc2b281880978fb6229aa25875be9 ci_payload/digital-surface-live-readiness/book-arrival-model.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/book-arrival-model.patch + cp -a ci_payload/digital-surface-live-readiness/shadow-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/prospective-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/observer-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/simulation-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/market-state-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/market-state-observer-source/. source/ + + cat > /tmp/state-observer.sha256 <<'EOF' + 157594f006173023f5ee88ee2b1b72f5af3eba431288a448aab86a57bdd1949e source/research/digital_surface/binance_public.py + c50d2f58c964e38e2e8a5f30c561cda1ea85c6dbbfc6fa169cca161607874262 source/research/digital_surface/spot_ledger.py + ead39cd8014cccb0d3036c9d37f292606121a4650bff7429fed873e8417eee10 source/research/digital_surface/public_state_observer.py + 80e91c2d2214260d9bbf6eb7bf0d736d74581de98ed8ac16bcd2f20ab35d6913 source/tests/digital_surface/test_binance_public.py + f5eeb0c0f5a746f1556c7f4cfe801251327d5b7275edc2af1760a64fd0f9775f source/tests/digital_surface/test_spot_ledger.py + 9c1eed5c878ba4b351b3bff5528468c49a8b2b8994ac7a9030477e3a97bbaf80 source/tests/digital_surface/test_public_state_observer.py + EOF + sed -i 's/^ //' /tmp/state-observer.sha256 + sha256sum -c /tmp/state-observer.sha256 + cp /tmp/state-observer.sha256 source/BINANCE_STATE_OBSERVER_SOURCE_SHA256.txt + + - name: Install pinned test stack + run: | + python -m pip install --upgrade pip + python -m pip install numpy==2.2.6 pandas==2.2.3 scipy==1.15.3 scikit-learn==1.6.1 pyarrow==19.0.1 requests==2.32.3 pytest==8.3.5 tabulate==0.9.0 + + - name: Compile and pass exactly 73 tests + working-directory: source + shell: bash + run: | + set -euo pipefail + python -m compileall -q research/digital_surface tests/digital_surface + PYTHONPATH=. pytest --collect-only -q tests/digital_surface | tee /tmp/collect.txt + grep -Eq '^73 tests collected' /tmp/collect.txt + PYTHONPATH=. pytest tests/digital_surface -q + + - name: Verify prospective boundary selection and state chaining + working-directory: source + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import hashlib + import json + import tempfile + from decimal import Decimal + from pathlib import Path + + from research.digital_surface.binance_public import BTCMarketState + from research.digital_surface.market_discovery import GammaMarketRecord + from research.digital_surface.public_state_observer import observe_public_btc_state + from research.digital_surface.spot_ledger import SpotStateLedger + + markets = [ + GammaMarketRecord( + condition_id='registered-5', slug='btc-updown-5m-1800', question='Bitcoin Up or Down', + yes_token_id='yes-5', no_token_id='no-5', duration_seconds=300, + open_epoch_seconds=1800, end_date='', tick_size=Decimal('0.01'), + ), + GammaMarketRecord( + condition_id='registered-15', slug='btc-updown-15m-1800', question='Bitcoin Up or Down', + yes_token_id='yes-15', no_token_id='no-15', duration_seconds=900, + open_epoch_seconds=1800, end_date='', tick_size=Decimal('0.01'), + ), + GammaMarketRecord( + condition_id='unregistered', slug='btc-updown-5m-2100', question='Bitcoin Up or Down', + yes_token_id='yes-u', no_token_id='no-u', duration_seconds=300, + open_epoch_seconds=2100, end_date='', tick_size=Decimal('0.01'), + ), + ] + calls = [] + def collect(boundaries, *, observed_ts_ms): + calls.append(list(boundaries)) + return BTCMarketState( + symbol='BTCUSDT', server_time_ms=5_000, observed_ts_ms=observed_ts_ms, + spot=Decimal('100.50'), vol_30s=0.001, vol_120s=0.002, + strikes={1_800_000: Decimal('99.00')}, closed_one_second_bars=121, + raw_response_sha256={ + 'server_time': 'a'*64, + 'one_second_klines': 'b'*64, + 'strike:1800000': 'c'*64, + }, + ) + with tempfile.TemporaryDirectory() as directory: + ledger = SpotStateLedger(Path(directory) / 'btc_state.jsonl') + summary = observe_public_btc_state( + markets, + prospective_market_ids={'registered-5', 'registered-15'}, + prospective_head_sha256='3'*64, + ledger=ledger, + observed_ts_ms=5_100, + policy_sha256='1'*64, + source_sha256='2'*64, + collect=collect, + ) + assert calls == [[1_800_000]] + assert summary['markets_registered'] == 2 + assert summary['markets_skipped_unregistered'] == 1 + assert summary['strike_boundaries'] == [1_800_000] + assert summary['authenticated_requests'] == 0 + assert summary['order_submissions'] == 0 + assert SpotStateLedger(ledger.path).head_hash == summary['spot_state_head_sha256'] + + verified = {} + for line in Path('BINANCE_STATE_OBSERVER_SOURCE_SHA256.txt').read_text().splitlines(): + wanted, name = line.split(maxsplit=1) + got = hashlib.sha256(Path(name).read_bytes()).hexdigest() + assert got == wanted, (name, got, wanted) + verified[name] = got + result = { + 'tests_collected': 73, + 'registered_market_boundaries_only': True, + 'deduplicated_strike_boundaries': True, + 'prospective_head_binding': True, + 'authenticated_requests': 0, + 'order_submissions': 0, + 'live_mode': 'physically_disabled', + 'source_files_verified': verified, + } + Path('BINANCE_STATE_OBSERVER_VERIFICATION.json').write_text(json.dumps(result, indent=2, sort_keys=True)) + print(json.dumps(result, indent=2)) + PY + + - name: Build deterministic source checkpoint + working-directory: source + shell: bash + run: | + set -euo pipefail + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-binance-state-observer-source-v1.tar.gz \ + research tests BINANCE_STATE_OBSERVER_SOURCE_SHA256.txt BINANCE_STATE_OBSERVER_VERIFICATION.json + sha256sum digital-surface-binance-state-observer-source-v1.tar.gz > digital-surface-binance-state-observer-source-v1.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: digital-surface-binance-state-observer-checkpoint-v1 + path: | + source/BINANCE_STATE_OBSERVER_SOURCE_SHA256.txt + source/BINANCE_STATE_OBSERVER_VERIFICATION.json + source/digital-surface-binance-state-observer-source-v1.tar.gz + source/digital-surface-binance-state-observer-source-v1.tar.gz.sha256 + retention-days: 30 From e5ec235af97f8fc4286498a6c44d99025fb25300 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 10:45:10 -0500 Subject: [PATCH 066/177] ci: add scheduled public BTC state observer --- ...ital-surface-public-btc-state-observer.yml | 197 ++++++++++++++++++ 1 file changed, 197 insertions(+) create mode 100644 .github/workflows/digital-surface-public-btc-state-observer.yml diff --git a/.github/workflows/digital-surface-public-btc-state-observer.yml b/.github/workflows/digital-surface-public-btc-state-observer.yml new file mode 100644 index 00000000..d8a7b559 --- /dev/null +++ b/.github/workflows/digital-surface-public-btc-state-observer.yml @@ -0,0 +1,197 @@ +name: Digital Surface Public BTC State Observer + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/digital-surface-public-btc-state-observer.yml + - ci_payload/digital-surface-live-readiness/market-state-source/** + - ci_payload/digital-surface-live-readiness/market-state-observer-source/** + workflow_dispatch: + schedule: + - cron: '2-59/5 * * * *' + +permissions: + contents: read + actions: read + +concurrency: + group: digital-surface-public-observer + cancel-in-progress: false + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + V5_ARTIFACT_ID: '8615107471' + V5_ARTIFACT_ZIP_SHA256: '479a25e235841d94077b3d2df070c8ed46bb736c7ebd8f9f1bde8016198a9c39' + CONTRACT_INVENTORY_SHA256: 'cea26cf5221e81b90ba311317f31e97b54e4110f0f1c716fcb14f06d9f1651f3' + FILTER_CACHE_KEY: digital-surface-diagnostic-filter-v1-cea26cf5221e81b90ba311317f31e97b54e4110f0f1c716fcb14f06d9f1651f3 + +jobs: + observe-public-btc-state: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct observation-only state source + shell: bash + run: | + set -euo pipefail + mkdir -p source/research/digital_surface + touch source/research/__init__.py source/research/digital_surface/__init__.py + cp -a ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/. source/research/digital_surface/ + cp -a ci_payload/digital-surface-live-readiness/prospective-source/research/digital_surface/. source/research/digital_surface/ + cp -a ci_payload/digital-surface-live-readiness/market-state-source/research/digital_surface/. source/research/digital_surface/ + cp -a ci_payload/digital-surface-live-readiness/market-state-observer-source/research/digital_surface/. source/research/digital_surface/ + cat > source/PUBLIC_BTC_STATE_RUNTIME_SOURCE_SHA256.txt <<'EOF' + 157594f006173023f5ee88ee2b1b72f5af3eba431288a448aab86a57bdd1949e research/digital_surface/binance_public.py + f3f1e2417305cd0ceee4b6971e461e488960f04f0e0b80ad292733a2a4b248af research/digital_surface/diagnostic_filter.py + efd186a4ca00c2e920bc87b8d137ac46db52f2aa36385a275dc99e70b5618439 research/digital_surface/market_discovery.py + 3fbb96286445b60b0b7deeb5e1f9ee45f53bfd5efdcc7a727b50617b19463224 research/digital_surface/prospective.py + 73c5aaa7a4ca10baf85886eb28c6135d975eafaaab19e5193819d1ccbf82f2be research/digital_surface/public_observer.py + ead39cd8014cccb0d3036c9d37f292606121a4650bff7429fed873e8417eee10 research/digital_surface/public_state_observer.py + ea56803a606ff33b7231b373141df3d22c159cedc1063603fef3dfb6255c60c9 research/digital_surface/shadow_observation_policy_v1.json + c50d2f58c964e38e2e8a5f30c561cda1ea85c6dbbfc6fa169cca161607874262 research/digital_surface/spot_ledger.py + EOF + sed -i 's/^ //' source/PUBLIC_BTC_STATE_RUNTIME_SOURCE_SHA256.txt + (cd source && sha256sum -c PUBLIC_BTC_STATE_RUNTIME_SOURCE_SHA256.txt) + + - name: Install public-data dependencies + run: | + python -m pip install --upgrade pip + python -m pip install pandas==2.2.3 requests==2.32.3 + + - name: Restore diagnostic filter + id: filter-cache + uses: actions/cache@v4 + with: + path: observer-filter + key: ${{ env.FILTER_CACHE_KEY }} + + - name: Generate diagnostic filter from immutable v5 inventory + if: steps.filter-cache.outputs.cache-hit != 'true' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + curl --fail --location --silent --show-error \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H 'Accept: application/vnd.github+json' \ + -H 'X-GitHub-Api-Version: 2022-11-28' \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/artifacts/${V5_ARTIFACT_ID}/zip" \ + -o v5.zip + echo "${V5_ARTIFACT_ZIP_SHA256} v5.zip" | sha256sum -c - + mkdir v5 observer-filter && unzip -q v5.zip -d v5 + inventory=$(find v5 -type f -name contract_inventory.csv -print -quit) + test -n "${inventory}" + cp "${inventory}" observer-filter/contract_inventory.csv + echo "${CONTRACT_INVENTORY_SHA256} observer-filter/contract_inventory.csv" | sha256sum -c - + PYTHONPATH=source python - <<'PY' + import hashlib + import json + from pathlib import Path + import pandas as pd + from research.digital_surface.diagnostic_filter import build_diagnostic_bloom + + path = Path('observer-filter/contract_inventory.csv') + ids = sorted(set(pd.read_csv(path)['condition_id'].astype(str))) + assert len(ids) == 12_465 + payload = build_diagnostic_bloom(ids, source_sha256=hashlib.sha256(path.read_bytes()).hexdigest()) + assert payload['bitset_sha256'] == '5fd0bc41ab1ae15ddaf9ca33354d803ab74c112d19136240f00d77156186d113' + Path('observer-filter/diagnostic_market_filter.json').write_text(json.dumps(payload, indent=2, sort_keys=True) + '\n') + PY + + - name: Restore append-only observation state + uses: actions/cache/restore@v4 + with: + path: observer-state + key: digital-surface-public-observer-state-${{ github.run_id }} + restore-keys: | + digital-surface-public-observer-state- + + - name: Observe public BTC state + shell: bash + run: | + set -euo pipefail + mkdir -p observer-state observer-output + touch observer-state/prospective_markets.jsonl observer-state/btc_state.jsonl + PYTHONPATH=source python -m research.digital_surface.public_state_observer \ + --diagnostic-filter observer-filter/diagnostic_market_filter.json \ + --prospective-ledger observer-state/prospective_markets.jsonl \ + --spot-ledger observer-state/btc_state.jsonl \ + --policy source/research/digital_surface/shadow_observation_policy_v1.json \ + --source-manifest source/PUBLIC_BTC_STATE_RUNTIME_SOURCE_SHA256.txt \ + --summary observer-output/btc_state_summary.json \ + --page-size 100 \ + --max-pages 20 + + - name: Verify state chain and observation-only boundary + shell: bash + run: | + set -euo pipefail + PYTHONPATH=source python - <<'PY' + import ast + import hashlib + import json + from pathlib import Path + from research.digital_surface.diagnostic_filter import DiagnosticBloomFilter + from research.digital_surface.prospective import ProspectiveLedger + from research.digital_surface.spot_ledger import SpotStateLedger + + summary = json.loads(Path('observer-output/btc_state_summary.json').read_text()) + assert summary['trading_mode'] == 'observation_only' + assert summary['authenticated_requests'] == 0 + assert summary['order_submissions'] == 0 + assert summary['binance_market_data_base'] == 'https://data-api.binance.vision' + + bloom = DiagnosticBloomFilter.from_path('observer-filter/diagnostic_market_filter.json') + prospective = ProspectiveLedger('observer-state/prospective_markets.jsonl', diagnostic_market_ids=bloom) + states = SpotStateLedger('observer-state/btc_state.jsonl') + assert states.head_hash == summary['spot_state_head_sha256'] + assert len(states.rows) == summary['spot_state_observations_total'] + assert summary['prospective_head_sha256'] == prospective.head_hash + assert set(states.rows[-1]['state']['strikes']) == {str(value) for value in summary['strike_boundaries']} + + tree = ast.parse(Path('source/research/digital_surface/binance_public.py').read_text()) + methods = { + node.func.attr for node in ast.walk(tree) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) + and getattr(node.func.value, 'id', None) == 'requests' + } + assert methods == {'get'}, methods + + verification = { + 'workflow_run_id': '${{ github.run_id }}', + 'workflow_run_attempt': '${{ github.run_attempt }}', + 'prospective_head_sha256': prospective.head_hash, + 'spot_state_head_sha256': states.head_hash, + 'spot_state_sha256': hashlib.sha256(Path('observer-state/btc_state.jsonl').read_bytes()).hexdigest(), + 'spot_state_observations_total': len(states.rows), + 'summary': summary, + 'transport_methods': ['GET'], + 'authenticated_requests': 0, + 'order_submissions': 0, + 'trading_mode': 'observation_only', + } + Path('observer-output/BTC_STATE_VERIFICATION.json').write_text(json.dumps(verification, indent=2, sort_keys=True)) + print(json.dumps(verification, indent=2)) + PY + cp observer-state/btc_state.jsonl observer-output/btc_state.jsonl + sha256sum observer-output/btc_state.jsonl observer-output/btc_state_summary.json observer-output/BTC_STATE_VERIFICATION.json > observer-output/BTC_STATE_SHA256SUMS.txt + + - name: Save combined append-only observation state + uses: actions/cache/save@v4 + with: + path: observer-state + key: digital-surface-public-observer-state-${{ github.run_id }} + + - uses: actions/upload-artifact@v4 + with: + name: digital-surface-public-btc-state-${{ github.run_id }} + path: observer-output/ + retention-days: 90 From 0a1d864e4053830260990f9e0229c1d5309aa44b Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 10:54:33 -0500 Subject: [PATCH 067/177] feat: checkpoint frozen-policy prospective signal adapter --- .../digital_surface/prospective_signal.py | 242 ++++++++++++++++++ .../test_prospective_signal.py | 167 ++++++++++++ 2 files changed, 409 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/prospective-signal-source/research/digital_surface/prospective_signal.py create mode 100644 ci_payload/digital-surface-live-readiness/prospective-signal-source/tests/digital_surface/test_prospective_signal.py diff --git a/ci_payload/digital-surface-live-readiness/prospective-signal-source/research/digital_surface/prospective_signal.py b/ci_payload/digital-surface-live-readiness/prospective-signal-source/research/digital_surface/prospective_signal.py new file mode 100644 index 00000000..af767d5a --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/prospective-signal-source/research/digital_surface/prospective_signal.py @@ -0,0 +1,242 @@ +from __future__ import annotations + +import hashlib +import json +import math +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping, Sequence + +import numpy as np +import pandas as pd + +from .binance_public import BTCMarketState +from .candidates import CandidateConfig, generate_cross_horizon_candidates, select_one_candidate_per_expiry +from .market_discovery import GammaMarketRecord +from .public_books import PublicOrderBook + + +_EPS = 1e-6 + + +def _sha256(name: str, value: str) -> str: + text = str(value).lower() + if len(text) != 64 or any(character not in "0123456789abcdef" for character in text): + raise ValueError(f"{name} must be a 64-character SHA-256 hex digest") + return text + + +def _normal_cdf(value: float) -> float: + return 0.5 * (1.0 + math.erf(float(value) / math.sqrt(2.0))) + + +def _logit(probability: float) -> float: + p = min(max(float(probability), _EPS), 1.0 - _EPS) + return math.log(p / (1.0 - p)) + + +def _expit(value: float) -> float: + if value >= 0: + inverse = math.exp(-value) + return 1.0 / (1.0 + inverse) + exponent = math.exp(value) + return exponent / (1.0 + exponent) + + +@dataclass(frozen=True, slots=True) +class FrozenSurfacePolicy: + coefficients: tuple[float, float, float, float] + training_rows: int + training_contracts: int + config: CandidateConfig + policy_sha256: str + selection: str = "first_trigger_per_expiry" + flow_assumption: str = "zero_without_public_causal_trade_flow" + + @classmethod + def from_payload(cls, payload: Mapping[str, Any], *, policy_sha256: str) -> "FrozenSurfacePolicy": + if int(payload.get("version", -1)) != 1: + raise ValueError("unsupported frozen surface policy version") + model = payload.get("model") + candidate = payload.get("candidate") + if not isinstance(model, Mapping) or not isinstance(candidate, Mapping): + raise ValueError("frozen policy requires model and candidate objects") + coefficients = tuple(float(value) for value in model.get("coefficients", [])) + if len(coefficients) != 4 or not all(math.isfinite(value) for value in coefficients): + raise ValueError("surface coefficients must contain four finite values") + if coefficients[1] < 0.05: + raise ValueError("structural slope must preserve positive spot monotonicity") + training_rows = int(model.get("training_rows", 0)) + training_contracts = int(model.get("training_contracts", 0)) + if training_rows < 0 or training_contracts <= 0: + raise ValueError("frozen policy requires positive training_contracts") + config = CandidateConfig( + min_edge=float(candidate["min_edge"]), + max_total_cost=float(candidate["max_total_cost"]), + min_tau_s=float(candidate.get("min_tau_s", 3.0)), + max_tau_s=float(candidate["max_tau_s"]), + decision_cadence_s=int(candidate.get("decision_cadence_s", 5)), + pair_gap_ms=int(candidate.get("pair_gap_ms", 2_000)), + slippage=float(candidate.get("slippage", 0.01)), + uncertainty_penalty=float(candidate.get("uncertainty_penalty", 0.0)), + ) + selection = str(payload.get("selection") or "") + if selection != "first_trigger_per_expiry": + raise ValueError("only the causal first-trigger selection policy is supported") + flow_assumption = str(payload.get("flow_assumption") or "") + if flow_assumption != "zero_without_public_causal_trade_flow": + raise ValueError("prospective policy must freeze the zero-flow assumption") + return cls( + coefficients=(coefficients[0], coefficients[1], coefficients[2], coefficients[3]), + training_rows=training_rows, + training_contracts=training_contracts, + config=config, + policy_sha256=_sha256("policy_sha256", policy_sha256), + selection=selection, + flow_assumption=flow_assumption, + ) + + def fair_yes(self, *, spot: float, strike: float, tau_s: float, volatility_per_s: float, horizon_s: int) -> tuple[float, float, float]: + if spot <= 0 or strike <= 0 or tau_s <= 0 or volatility_per_s <= 0: + raise ValueError("surface inputs must be positive") + z_moneyness = math.log(spot / strike) / (volatility_per_s * math.sqrt(tau_s)) + structural = min(max(_normal_cdf(z_moneyness), _EPS), 1.0 - _EPS) + intercept, structural_slope, horizon_offset, flow_coefficient = self.coefficients + eta = intercept + structural_slope * _logit(structural) + horizon_offset * (1.0 if horizon_s == 900 else 0.0) + eta += flow_coefficient * 0.0 + fair = min(max(_expit(eta), _EPS), 1.0 - _EPS) + uncertainty = math.sqrt(fair * (1.0 - fair) / self.training_contracts) + return fair, uncertainty, z_moneyness + + +def load_frozen_surface_policy(path: str | Path) -> FrozenSurfacePolicy: + policy_path = Path(path) + raw = policy_path.read_bytes() + payload = json.loads(raw) + if not isinstance(payload, Mapping): + raise ValueError("frozen surface policy must be a JSON object") + return FrozenSurfacePolicy.from_payload(payload, policy_sha256=hashlib.sha256(raw).hexdigest()) + + +def generate_prospective_candidates( + markets: Sequence[GammaMarketRecord], + books: Mapping[str, PublicOrderBook], + state: BTCMarketState, + *, + book_observed_ts_ms: int, + policy: FrozenSurfacePolicy, + prospective_head_sha256: str, + book_ledger_head_sha256: str, + spot_state_head_sha256: str, + max_evidence_age_ms: int = 2_000, +) -> tuple[pd.DataFrame, dict[str, object]]: + evidence_hashes = { + "policy_sha256": _sha256("policy_sha256", policy.policy_sha256), + "prospective_head_sha256": _sha256("prospective_head_sha256", prospective_head_sha256), + "book_ledger_head_sha256": _sha256("book_ledger_head_sha256", book_ledger_head_sha256), + "spot_state_head_sha256": _sha256("spot_state_head_sha256", spot_state_head_sha256), + } + book_ts = int(book_observed_ts_ms) + decision_ts_ms = max(book_ts, int(state.observed_ts_ms)) + if max_evidence_age_ms < 0: + raise ValueError("max_evidence_age_ms must be non-negative") + book_age = decision_ts_ms - book_ts + state_age = decision_ts_ms - int(state.observed_ts_ms) + if book_age > max_evidence_age_ms or state_age > max_evidence_age_ms: + raise ValueError( + f"stale evidence: book_age_ms={book_age}, state_age_ms={state_age}, max={max_evidence_age_ms}" + ) + + volatility = float(state.vol_30s) + if not math.isfinite(volatility) or volatility <= 0: + volatility = float(state.vol_120s) + if not math.isfinite(volatility) or volatility <= 0: + raise ValueError("BTC state does not contain positive causal volatility") + + rows: list[dict[str, object]] = [] + skipped_missing_books = 0 + skipped_empty_books = 0 + for market in markets: + duration = {300: "5m", 900: "15m"}.get(int(market.duration_seconds)) + if duration is None: + continue + boundary_ms = int(market.open_epoch_seconds) * 1_000 + if boundary_ms not in state.strikes: + raise ValueError(f"missing exact strike for {market.condition_id} at boundary {boundary_ms}") + yes_book = books.get(market.yes_token_id) + no_book = books.get(market.no_token_id) + if yes_book is None or no_book is None: + skipped_missing_books += 1 + continue + if yes_book.condition_id != market.condition_id or no_book.condition_id != market.condition_id: + raise ValueError(f"book condition mismatch for {market.condition_id}") + if yes_book.best_ask is None or no_book.best_ask is None: + skipped_empty_books += 1 + continue + close_ts = int(market.open_epoch_seconds) + int(market.duration_seconds) + tau_s = (close_ts * 1_000 - decision_ts_ms) / 1_000.0 + if tau_s <= 0: + continue + strike = float(state.strikes[boundary_ms]) + fair, uncertainty, z_moneyness = policy.fair_yes( + spot=float(state.spot), + strike=strike, + tau_s=tau_s, + volatility_per_s=volatility, + horizon_s=int(market.duration_seconds), + ) + rows.append( + { + "condition_id": market.condition_id, + "duration": duration, + "close_ts": close_ts, + "ts_ms": decision_ts_ms, + "strike": strike, + "yes_token_id": market.yes_token_id, + "no_token_id": market.no_token_id, + "yes_ask": float(yes_book.best_ask.price), + "no_ask": float(no_book.best_ask.price), + "yes_ask_size": float(yes_book.best_ask.size), + "no_ask_size": float(no_book.best_ask.size), + "fair_yes": fair, + "surface_uncertainty": uncertainty, + "resolved_yes": False, + "tau_s": tau_s, + "z_moneyness": z_moneyness, + "horizon_s": int(market.duration_seconds), + "signed_trade_flow_5s": 0.0, + } + ) + + surface = pd.DataFrame(rows) + if surface.empty: + candidates = pd.DataFrame() + else: + candidates = select_one_candidate_per_expiry( + generate_cross_horizon_candidates(surface, config=policy.config) + ) + if not candidates.empty: + for name, value in evidence_hashes.items(): + candidates[name] = value + candidates["book_observed_ts_ms"] = book_ts + candidates["spot_observed_ts_ms"] = int(state.observed_ts_ms) + candidates["decision_ts_ms"] = decision_ts_ms + candidates["fill_model"] = "public_book_arrival_shadow" + candidates["transactionally_atomic"] = False + candidates["future_outcomes_used"] = 0 + + summary = { + "markets_seen": len(markets), + "surface_rows": len(surface), + "candidates_selected": len(candidates), + "skipped_missing_books": skipped_missing_books, + "skipped_empty_books": skipped_empty_books, + "decision_ts_ms": decision_ts_ms, + "book_age_ms": book_age, + "spot_state_age_ms": state_age, + "policy_config_id": policy.config.config_id, + "future_outcomes_used": 0, + "transactionally_atomic": False, + **evidence_hashes, + } + return candidates.reset_index(drop=True), summary diff --git a/ci_payload/digital-surface-live-readiness/prospective-signal-source/tests/digital_surface/test_prospective_signal.py b/ci_payload/digital-surface-live-readiness/prospective-signal-source/tests/digital_surface/test_prospective_signal.py new file mode 100644 index 00000000..fe5bb75a --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/prospective-signal-source/tests/digital_surface/test_prospective_signal.py @@ -0,0 +1,167 @@ +import json +from decimal import Decimal + +from research.digital_surface.binance_public import BTCMarketState +from research.digital_surface.market_discovery import GammaMarketRecord +from research.digital_surface.prospective_signal import ( + FrozenSurfacePolicy, + generate_prospective_candidates, + load_frozen_surface_policy, +) +from research.digital_surface.public_books import parse_public_order_book + + +def market(condition, *, epoch, duration, token_prefix): + return GammaMarketRecord( + condition_id=condition, + slug=f"btc-updown-{duration // 60}m-{epoch}", + question="Bitcoin Up or Down", + yes_token_id=f"yes-{token_prefix}", + no_token_id=f"no-{token_prefix}", + duration_seconds=duration, + open_epoch_seconds=epoch, + end_date="2030-01-01T00:00:00Z", + tick_size=Decimal("0.01"), + ) + + +def book(token, condition, yes_ask="0.45", no_ask="0.45", size="10"): + ask = yes_ask if token.startswith("yes-") else no_ask + return parse_public_order_book({ + "market": condition, + "asset_id": token, + "timestamp": "1250000", + "hash": f"hash-{token}", + "bids": [{"price": "0.40", "size": size}], + "asks": [{"price": ask, "size": size}], + "min_order_size": "1", + "tick_size": "0.01", + "neg_risk": False, + }) + + +def policy_payload(): + return { + "version": 1, + "model": { + "coefficients": [0.0, 1.0, 0.0, 0.0], + "training_rows": 1000, + "training_contracts": 500, + }, + "candidate": { + "min_edge": 0.02, + "max_total_cost": 1.25, + "min_tau_s": 3.0, + "max_tau_s": 60.0, + "decision_cadence_s": 5, + "pair_gap_ms": 2000, + "slippage": 0.01, + "uncertainty_penalty": 0.0, + }, + "selection": "first_trigger_per_expiry", + "flow_assumption": "zero_without_public_causal_trade_flow", + } + + +def setup_case(): + five = market("five", epoch=1_000, duration=300, token_prefix="five") + fifteen = market("fifteen", epoch=400, duration=900, token_prefix="fifteen") + books = { + five.yes_token_id: book(five.yes_token_id, five.condition_id), + five.no_token_id: book(five.no_token_id, five.condition_id), + fifteen.yes_token_id: book(fifteen.yes_token_id, fifteen.condition_id), + fifteen.no_token_id: book(fifteen.no_token_id, fifteen.condition_id), + } + state = BTCMarketState( + symbol="BTCUSDT", + server_time_ms=1_250_000, + observed_ts_ms=1_250_000, + spot=Decimal("100"), + vol_30s=0.001, + vol_120s=0.0015, + strikes={1_000_000: Decimal("99"), 400_000: Decimal("101")}, + closed_one_second_bars=121, + raw_response_sha256={ + "server_time": "a"*64, + "one_second_klines": "b"*64, + "strike:1000000": "c"*64, + "strike:400000": "d"*64, + }, + ) + return [five, fifteen], books, state + + +def test_loads_frozen_policy_and_hashes_exact_file_bytes(tmp_path): + path = tmp_path / "policy.json" + path.write_text(json.dumps(policy_payload(), indent=2, sort_keys=True) + "\n") + policy = load_frozen_surface_policy(path) + assert isinstance(policy, FrozenSurfacePolicy) + assert policy.config.config_id == "e0.0200-c1.2500-tau3-60-cad5-slip0.010-u0.00" + assert len(policy.policy_sha256) == 64 + assert policy.training_contracts == 500 + + +def test_generates_original_same_expiry_lower_yes_higher_no_candidate(): + markets, books, state = setup_case() + policy = FrozenSurfacePolicy.from_payload(policy_payload(), policy_sha256="1"*64) + candidates, summary = generate_prospective_candidates( + markets, + books, + state, + book_observed_ts_ms=1_250_000, + policy=policy, + prospective_head_sha256="2"*64, + book_ledger_head_sha256="3"*64, + spot_state_head_sha256="4"*64, + ) + assert len(candidates) == 1 + row = candidates.iloc[0] + assert row["low_condition_id"] == "five" + assert row["high_condition_id"] == "fifteen" + assert row["low_yes_token_id"] == "yes-five" + assert row["high_no_token_id"] == "no-fifteen" + assert row["low_strike"] == 99.0 + assert row["high_strike"] == 101.0 + assert row["theoretical_edge"] >= 0.02 + assert row["policy_sha256"] == "1"*64 + assert row["prospective_head_sha256"] == "2"*64 + assert bool(row["transactionally_atomic"]) is False + assert summary["candidates_selected"] == 1 + assert summary["future_outcomes_used"] == 0 + + +def test_fails_closed_on_missing_exact_strike_or_stale_book_evidence(): + markets, books, state = setup_case() + policy = FrozenSurfacePolicy.from_payload(policy_payload(), policy_sha256="1"*64) + broken = BTCMarketState( + symbol=state.symbol, + server_time_ms=state.server_time_ms, + observed_ts_ms=state.observed_ts_ms, + spot=state.spot, + vol_30s=state.vol_30s, + vol_120s=state.vol_120s, + strikes={1_000_000: Decimal("99")}, + closed_one_second_bars=state.closed_one_second_bars, + raw_response_sha256=state.raw_response_sha256, + ) + try: + generate_prospective_candidates( + markets, books, broken, book_observed_ts_ms=1_250_000, policy=policy, + prospective_head_sha256="2"*64, book_ledger_head_sha256="3"*64, + spot_state_head_sha256="4"*64, + ) + except ValueError as exc: + assert "missing exact strike" in str(exc) + else: + raise AssertionError("missing strike did not fail closed") + + try: + generate_prospective_candidates( + markets, books, state, book_observed_ts_ms=1_240_000, policy=policy, + prospective_head_sha256="2"*64, book_ledger_head_sha256="3"*64, + spot_state_head_sha256="4"*64, max_evidence_age_ms=2_000, + ) + except ValueError as exc: + assert "stale evidence" in str(exc) + else: + raise AssertionError("stale evidence did not fail closed") From 58a07e0be4a627d4ed867e60239d0cface06a6c7 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 10:56:09 -0500 Subject: [PATCH 068/177] ci: verify frozen-policy prospective signal adapter --- ...-surface-prospective-signal-checkpoint.yml | 181 ++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 .github/workflows/digital-surface-prospective-signal-checkpoint.yml diff --git a/.github/workflows/digital-surface-prospective-signal-checkpoint.yml b/.github/workflows/digital-surface-prospective-signal-checkpoint.yml new file mode 100644 index 00000000..6af80e5f --- /dev/null +++ b/.github/workflows/digital-surface-prospective-signal-checkpoint.yml @@ -0,0 +1,181 @@ +name: Digital Surface Prospective Signal Checkpoint + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/digital-surface-prospective-signal-checkpoint.yml + - ci_payload/digital-surface-live-readiness/prospective-signal-source/** + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-prospective-signal-checkpoint.yml + - ci_payload/digital-surface-live-readiness/prospective-signal-source/** + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + verify-prospective-signal: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct checksum-pinned source + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source && tar -xzf base.tar.gz -C source + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + for patch_file in timestamp-unit-fix.patch full-universe-gate-fix.patch resolved-universe-fix.patch asset-merge-fix.patch; do + patch -d source -p1 < "ci_payload/digital-surface-official-resolution-patch/${patch_file}" + done + echo 'f71f3a653ca58b4bf98609fc29d8ab16f9d031c9ed14906ebe5159f2b0de8eab ci_payload/digital-surface-live-readiness/execution-diagnostics.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/execution-diagnostics.patch + echo '8606cc925aa3b3a6417abb1fba9a11df916cc2b281880978fb6229aa25875be9 ci_payload/digital-surface-live-readiness/book-arrival-model.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/book-arrival-model.patch + cp -a ci_payload/digital-surface-live-readiness/shadow-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/prospective-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/observer-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/simulation-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/market-state-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/market-state-observer-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/prospective-signal-source/. source/ + + cat > /tmp/prospective-signal.sha256 <<'EOF' + c147efb911b0f6f049902fdf037f5aef8f044bc944a5aac9e298f93283c38996 source/research/digital_surface/prospective_signal.py + e2c21e61bd450343bc2271db83cd9b27cd316e5c50d3f9ac1128f3f38be998ed source/tests/digital_surface/test_prospective_signal.py + EOF + sed -i 's/^ //' /tmp/prospective-signal.sha256 + sha256sum -c /tmp/prospective-signal.sha256 + cp /tmp/prospective-signal.sha256 source/PROSPECTIVE_SIGNAL_SOURCE_SHA256.txt + + - name: Install pinned test stack + run: | + python -m pip install --upgrade pip + python -m pip install numpy==2.2.6 pandas==2.2.3 scipy==1.15.3 scikit-learn==1.6.1 pyarrow==19.0.1 requests==2.32.3 pytest==8.3.5 tabulate==0.9.0 + + - name: Compile and pass exactly 76 tests + working-directory: source + shell: bash + run: | + set -euo pipefail + python -m compileall -q research/digital_surface tests/digital_surface + PYTHONPATH=. pytest --collect-only -q tests/digital_surface | tee /tmp/collect.txt + grep -Eq '^76 tests collected' /tmp/collect.txt + PYTHONPATH=. pytest tests/digital_surface -q + + - name: Verify frozen causal signal semantics + working-directory: source + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import hashlib + import json + from decimal import Decimal + from pathlib import Path + + from research.digital_surface.binance_public import BTCMarketState + from research.digital_surface.market_discovery import GammaMarketRecord + from research.digital_surface.prospective_signal import FrozenSurfacePolicy, generate_prospective_candidates + from research.digital_surface.public_books import parse_public_order_book + + payload = { + 'version': 1, + 'model': {'coefficients': [0.0, 1.0, 0.0, 0.0], 'training_rows': 1000, 'training_contracts': 500}, + 'candidate': { + 'min_edge': 0.02, 'max_total_cost': 1.25, 'min_tau_s': 3.0, 'max_tau_s': 60.0, + 'decision_cadence_s': 5, 'pair_gap_ms': 2000, 'slippage': 0.01, 'uncertainty_penalty': 0.0, + }, + 'selection': 'first_trigger_per_expiry', + 'flow_assumption': 'zero_without_public_causal_trade_flow', + } + policy = FrozenSurfacePolicy.from_payload(payload, policy_sha256='1'*64) + markets = [ + GammaMarketRecord('five', 'btc-updown-5m-1000', 'Bitcoin Up or Down', 'yes-five', 'no-five', 300, 1000, '', Decimal('0.01')), + GammaMarketRecord('fifteen', 'btc-updown-15m-400', 'Bitcoin Up or Down', 'yes-fifteen', 'no-fifteen', 900, 400, '', Decimal('0.01')), + ] + def book(token, condition): + return parse_public_order_book({ + 'market': condition, 'asset_id': token, 'timestamp': '1250000', 'hash': 'h-'+token, + 'bids': [{'price': '0.40', 'size': '10'}], 'asks': [{'price': '0.45', 'size': '10'}], + 'min_order_size': '1', 'tick_size': '0.01', 'neg_risk': False, + }) + books = {token: book(token, market.condition_id) for market in markets for token in (market.yes_token_id, market.no_token_id)} + state = BTCMarketState( + symbol='BTCUSDT', server_time_ms=1_250_000, observed_ts_ms=1_250_000, + spot=Decimal('100'), vol_30s=0.001, vol_120s=0.0015, + strikes={1_000_000: Decimal('99'), 400_000: Decimal('101')}, closed_one_second_bars=121, + raw_response_sha256={'server_time': 'a'*64, 'one_second_klines': 'b'*64, 'strike:1000000': 'c'*64, 'strike:400000': 'd'*64}, + ) + candidates, summary = generate_prospective_candidates( + markets, books, state, book_observed_ts_ms=1_250_000, policy=policy, + prospective_head_sha256='2'*64, book_ledger_head_sha256='3'*64, spot_state_head_sha256='4'*64, + ) + assert len(candidates) == 1 + row = candidates.iloc[0] + assert row['low_condition_id'] == 'five' + assert row['high_condition_id'] == 'fifteen' + assert row['low_yes_token_id'] == 'yes-five' + assert row['high_no_token_id'] == 'no-fifteen' + assert summary['future_outcomes_used'] == 0 + assert bool(row['transactionally_atomic']) is False + assert row['fill_model'] == 'public_book_arrival_shadow' + + verified = {} + for line in Path('PROSPECTIVE_SIGNAL_SOURCE_SHA256.txt').read_text().splitlines(): + wanted, name = line.split(maxsplit=1) + got = hashlib.sha256(Path(name).read_bytes()).hexdigest() + assert got == wanted + verified[name] = got + result = { + 'tests_collected': 76, + 'same_expiry_pairing': True, + 'lower_strike_yes_higher_strike_no': True, + 'first_trigger_selection': True, + 'future_outcomes_used': 0, + 'transactionally_atomic': False, + 'order_submissions': 0, + 'live_mode': 'physically_disabled', + 'source_files_verified': verified, + } + Path('PROSPECTIVE_SIGNAL_VERIFICATION.json').write_text(json.dumps(result, indent=2, sort_keys=True)) + print(json.dumps(result, indent=2)) + PY + + - name: Build deterministic source checkpoint + working-directory: source + shell: bash + run: | + set -euo pipefail + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-prospective-signal-source-v1.tar.gz \ + research tests PROSPECTIVE_SIGNAL_SOURCE_SHA256.txt PROSPECTIVE_SIGNAL_VERIFICATION.json + sha256sum digital-surface-prospective-signal-source-v1.tar.gz > digital-surface-prospective-signal-source-v1.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: digital-surface-prospective-signal-checkpoint-v1 + path: | + source/PROSPECTIVE_SIGNAL_SOURCE_SHA256.txt + source/PROSPECTIVE_SIGNAL_VERIFICATION.json + source/digital-surface-prospective-signal-source-v1.tar.gz + source/digital-surface-prospective-signal-source-v1.tar.gz.sha256 + retention-days: 30 From de24d1cba6dc045aff0463e547525e2bc2a91898 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 10:57:37 -0500 Subject: [PATCH 069/177] ci: freeze immutable prospective surface policy --- ...ital-surface-freeze-prospective-policy.yml | 205 ++++++++++++++++++ 1 file changed, 205 insertions(+) create mode 100644 .github/workflows/digital-surface-freeze-prospective-policy.yml diff --git a/.github/workflows/digital-surface-freeze-prospective-policy.yml b/.github/workflows/digital-surface-freeze-prospective-policy.yml new file mode 100644 index 00000000..e513628d --- /dev/null +++ b/.github/workflows/digital-surface-freeze-prospective-policy.yml @@ -0,0 +1,205 @@ +name: Digital Surface Freeze Prospective Policy + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/digital-surface-freeze-prospective-policy.yml + - ci_payload/digital-surface-live-readiness/prospective-signal-source/** + workflow_dispatch: + +permissions: + contents: read + actions: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + V5_ARTIFACT_ID: '8615107471' + V5_ARTIFACT_ZIP_SHA256: '479a25e235841d94077b3d2df070c8ed46bb736c7ebd8f9f1bde8016198a9c39' + +jobs: + freeze-policy: + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct checksum-pinned v5 source + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source && tar -xzf base.tar.gz -C source + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + for patch_file in timestamp-unit-fix.patch full-universe-gate-fix.patch resolved-universe-fix.patch asset-merge-fix.patch; do + patch -d source -p1 < "ci_payload/digital-surface-official-resolution-patch/${patch_file}" + done + cp -a ci_payload/digital-surface-live-readiness/prospective-signal-source/. source/ + + - name: Install pinned research stack + run: | + python -m pip install --upgrade pip + python -m pip install numpy==2.2.6 pandas==2.2.3 scipy==1.15.3 scikit-learn==1.6.1 pyarrow==19.0.1 requests==2.32.3 pytest==8.3.5 tabulate==0.9.0 + + - name: Download and verify v5 research artifact + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + curl --fail --location --silent --show-error \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H 'Accept: application/vnd.github+json' \ + -H 'X-GitHub-Api-Version: 2022-11-28' \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/artifacts/${V5_ARTIFACT_ID}/zip" \ + -o v5.zip + echo "${V5_ARTIFACT_ZIP_SHA256} v5.zip" | sha256sum -c - + mkdir v5 && unzip -q v5.zip -d v5 + causal=$(find v5 -type f -name causal_panel.parquet -print -quit) + choices=$(find v5 -type f -name validation_choices.json -print -quit) + checksums=$(find v5 -type f -name SHA256SUMS.json -print -quit) + test -n "${causal}" && test -n "${choices}" && test -n "${checksums}" + cp "${causal}" source/causal_panel.parquet + cp "${choices}" source/validation_choices.json + cp "${checksums}" source/V5_SHA256SUMS.json + python - <<'PY' + import hashlib, json + from pathlib import Path + expected = json.loads(Path('source/V5_SHA256SUMS.json').read_text()) + for name in ('causal_panel.parquet', 'validation_choices.json'): + got = hashlib.sha256(Path('source', name).read_bytes()).hexdigest() + assert got == expected[name], (name, got, expected[name]) + PY + + - name: Fit and freeze deterministic policy + working-directory: source + shell: bash + run: | + set -euo pipefail + PYTHONPATH=. python - <<'PY' + import hashlib + import json + from pathlib import Path + + import numpy as np + import pandas as pd + + from research.digital_surface.prospective_signal import load_frozen_surface_policy + from research.digital_surface.run_research import _fit_surface + + causal_path = Path('causal_panel.parquet') + choices_path = Path('validation_choices.json') + state = pd.read_parquet(causal_path) + choices = json.loads(choices_path.read_text()) + assert len(choices) == 4, len(choices) + final_choice = max(choices, key=lambda row: int(row['fold_id'])) + assert int(final_choice['fold_id']) == 4 + assert final_choice['validation_weeks'] == ['03-2026-03-16', '04-2026-03-23'], final_choice + assert final_choice['test_week'] == '05-2026-03-30', final_choice + config = dict(final_choice['selected_config']) + assert config['config_id'] == 'e0.0200-c1.2500-tau3-60-cad5-slip0.010-u0.00', config + + model_a = _fit_surface(state) + model_b = _fit_surface(state) + coeff_a = np.asarray(model_a.coefficients, dtype=float) + coeff_b = np.asarray(model_b.coefficients, dtype=float) + assert np.array_equal(coeff_a, coeff_b), (coeff_a, coeff_b) + assert coeff_a.shape == (4,) + assert np.isfinite(coeff_a).all() + assert coeff_a[1] >= 0.05 + + historical_weeks = sorted(state['week'].astype(str).unique().tolist()) + payload = { + 'version': 1, + 'model': { + 'coefficients': [float(value) for value in coeff_a], + 'training_rows': int(model_a.training_rows), + 'training_contracts': int(model_a.training_contracts), + 'fit_scope': 'all_historical_pre_prospective', + 'historical_weeks': historical_weeks, + }, + 'candidate': { + 'min_edge': float(config['min_edge']), + 'max_total_cost': float(config['max_total_cost']), + 'min_tau_s': float(config['min_tau_s']), + 'max_tau_s': float(config['max_tau_s']), + 'decision_cadence_s': int(config['decision_cadence_s']), + 'pair_gap_ms': int(config['pair_gap_ms']), + 'slippage': float(config['slippage']), + 'uncertainty_penalty': float(config['uncertainty_penalty']), + }, + 'selection': 'first_trigger_per_expiry', + 'flow_assumption': 'zero_without_public_causal_trade_flow', + 'provenance': { + 'v5_artifact_id': 8615107471, + 'v5_artifact_zip_sha256': '479a25e235841d94077b3d2df070c8ed46bb736c7ebd8f9f1bde8016198a9c39', + 'causal_panel_sha256': hashlib.sha256(causal_path.read_bytes()).hexdigest(), + 'validation_choices_sha256': hashlib.sha256(choices_path.read_bytes()).hexdigest(), + 'configuration_selected_in_fold': 4, + 'configuration_validation_weeks': final_choice['validation_weeks'], + 'configuration_test_week_excluded_from_selection': final_choice['test_week'], + 'policy_frozen_before_prospective_observation': True, + 'historical_folds_are_diagnostic_only': True, + }, + } + policy_path = Path('frozen_prospective_surface_policy_v1.json') + policy_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + '\n') + policy_sha = hashlib.sha256(policy_path.read_bytes()).hexdigest() + policy = load_frozen_surface_policy(policy_path) + assert policy.policy_sha256 == policy_sha + assert policy.config.config_id == config['config_id'] + assert policy.training_rows == int(model_a.training_rows) + assert policy.training_contracts == int(model_a.training_contracts) + + verification = { + 'policy_sha256': policy_sha, + 'coefficients': payload['model']['coefficients'], + 'training_rows': policy.training_rows, + 'training_contracts': policy.training_contracts, + 'candidate_config_id': policy.config.config_id, + 'configuration_validation_weeks': final_choice['validation_weeks'], + 'configuration_test_week_excluded_from_selection': final_choice['test_week'], + 'historical_weeks': historical_weeks, + 'deterministic_refit_exact': True, + 'future_prospective_rows_used': 0, + 'live_mode': 'physically_disabled', + } + Path('FROZEN_POLICY_VERIFICATION.json').write_text(json.dumps(verification, indent=2, sort_keys=True)) + Path('frozen_prospective_surface_policy_v1.json.sha256').write_text( + policy_sha + ' frozen_prospective_surface_policy_v1.json\n' + ) + print(json.dumps(verification, indent=2)) + PY + + - name: Build deterministic policy artifact + working-directory: source + shell: bash + run: | + set -euo pipefail + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-frozen-prospective-policy-v1.tar.gz \ + frozen_prospective_surface_policy_v1.json \ + frozen_prospective_surface_policy_v1.json.sha256 \ + FROZEN_POLICY_VERIFICATION.json + sha256sum digital-surface-frozen-prospective-policy-v1.tar.gz > digital-surface-frozen-prospective-policy-v1.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: digital-surface-frozen-prospective-policy-v1 + path: | + source/frozen_prospective_surface_policy_v1.json + source/frozen_prospective_surface_policy_v1.json.sha256 + source/FROZEN_POLICY_VERIFICATION.json + source/digital-surface-frozen-prospective-policy-v1.tar.gz + source/digital-surface-frozen-prospective-policy-v1.tar.gz.sha256 + retention-days: 90 From cda1160a65df1e3d9aadbae64534b0a31fd4cbc1 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 12:25:51 -0500 Subject: [PATCH 070/177] research: checkpoint deep favorite EDA findings --- .../deep-favorite-eda-v2/REPORT.md | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/REPORT.md diff --git a/ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/REPORT.md b/ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/REPORT.md new file mode 100644 index 00000000..d511a202 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/REPORT.md @@ -0,0 +1,94 @@ +# Polymarket 5-Minute Crypto — Deep Favorite EDA Checkpoint + +## Decision-useful answer + +The strongest high-frequency candidate in the 89,233-market / 26.77M-tick corpus is not BTC-to-alt lead-lag. It is a **late-window settlement-favorite strategy** at 210 seconds with a signal ask of at least 85 cents, one-second execution latency, five-share FOK sizing, and taker fees included. + +The all-seven-asset version was positive on the final chronological test segment, but the edge was diluted by weak assets and concentrated in a few days. The statistically credible core is **BTC + ETH**. This is a research candidate for official-label transfer and prospective shadowing, not permission for live trading. + +## Evidence spine + +### Original strict walk-forward result + +- Train: 38 active days, 18,493 trades, +$1,088.33 at five shares, daily t = 5.91. +- Validation: 14 active days, 6,819 trades, +$395.42, daily t = 5.35, every validation day positive. +- Final chronological test: 12 active days, 6,769 trades, +$249.11, 24.67 trades/hour, daily t = 1.67. +- The selected 210s / 85c configuration ranked #1 of 982 eligible configurations by validation daily t. +- Every tested latency from 0 to 3 seconds remained positive in train and validation. + +### BTC + ETH alpha spine + +- 2,054 final-test trades. +- +$139.72 at five shares. +- 95.67% win rate. +- 1.3604 cents mean P&L per share after the modeled fee. +- Daily t = 3.85. +- Exact one-sided day-level sign-flip p = 0.00244. +- 95% day-bootstrap mean interval: +$5.84 to +$17.13 per day at five shares. +- BTC and ETH individually survive Benjamini-Hochberg control at 5% across seven assets. + +### Adverse-selection finding + +A lower execution ask is not always price improvement. When the favorite ask fell by at least 2 cents during the one-second latency window, the final test produced: + +- 288 trades; +- 84.38% win rate; +- -$48.44 at five shares; +- negative daily t. + +A one-cent improvement was positive, while two cents or more was toxic. This supports a causal **cancel if arrival ask is more than 1 cent below the signal ask** rule for the next prospective candidate. Because this rule was diagnosed after inspecting the final test, it must be validated only on new observations. + +### Cross-asset loss clustering + +- 38.66% of losses occurred in windows with at least two simultaneous asset losses. +- BTC/ETH loss correlation was 0.559. +- When BTC won, ETH won 98.39% of simultaneous windows; when BTC lost, ETH won only 38.46%. +- The all-seven portfolio derived 66.62% of total P&L from its two best days. +- BTC + ETH reduced top-two-day concentration to 38.02%. + +This is a common-factor crash problem, not independent diversification. Portfolio exposure must be capped by five-minute window, not only by asset. + +### Execution-price calibration + +BTC/ETH edge is concentrated in the 85c-93c range. Diagnostic one-cent buckets showed especially strong empirical edge at 86c, 88c, 91c and 92c. The 95c and 98c buckets were flat-to-negative, while 99c remained slightly positive. This irregularity argues against fitting a fine price lookup table from the inspected test; retain the broad 85c rule until prospective evidence accumulates. + +### Capacity and cost sensitivity + +- BTC/ETH break-even additional cost: 1.3604 cents/share. +- All-seven break-even additional cost: 0.7360 cents/share. +- At 100 shares, 56.13% of BTC/ETH test signals displayed enough top-of-book size and remained positive, but daily tail risk scaled sharply. +- At 250+ shares, statistical quality deteriorated; top-of-book size is selection-biased and is not a reliable live capacity guarantee. +- The all-seven strategy turned negative at 1 cent extra slippage; the BTC/ETH core has more cost headroom but still requires measured prospective fill slippage. + +## Frozen prospective hypothesis + +```json +{ + "name": "late_favorite_btc_eth_v2_diagnostic", + "assets": ["btc", "eth"], + "entry_second": 210, + "signal_ask_min": 0.85, + "latency_seconds": 1, + "fok_limit": "signal ask", + "shares": 5, + "arrival_adverse_move_cancel": "cancel when execution ask < signal ask - 0.01", + "maximum_positions_per_asset_per_window": 1, + "maximum_total_positions_per_window": 2, + "hold": "settlement", + "fee_per_share": "0.07 * p * (1-p)", + "live_submission": "disabled", + "status": "diagnostic candidate; requires new official-label prospective validation" +} +``` + +## Non-negotiable uncertainty + +1. Dataset outcomes are inferred from the final recorded book, not authoritative settlement. +2. Only 28 false winning labels would erase the BTC/ETH final-test profit; official-resolution validation is mandatory. +3. Ask-side capacity is top-of-book only; queue position and full depth were not recorded. +4. The BTC/ETH universe and adverse-move cancel were identified after inspecting the final test. They cannot inherit the original untouched-test status. +5. Real-money order submission remains absent. The next valid evidence is an append-only prospective shadow ledger. + +## Next experiment + +Run the frozen BTC/ETH candidate prospectively with official terminal outcomes and public-book evidence. Report every five-minute window, including no-trade decisions, arrival price movement, displayed size, hypothetical FOK fill, settlement, and per-window portfolio exposure. Admission requires at least four untouched weekly blocks and enough trades to establish a positive lower confidence bound after measured fees and slippage. From f8750201dc421b1c0e543712d8f207ef76705eaf Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sat, 25 Jul 2026 12:26:04 -0500 Subject: [PATCH 071/177] research: freeze diagnostic BTC ETH favorite candidate --- .../deep-favorite-eda-v2/CANDIDATE_SPEC.json | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/CANDIDATE_SPEC.json diff --git a/ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/CANDIDATE_SPEC.json b/ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/CANDIDATE_SPEC.json new file mode 100644 index 00000000..f29caac6 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/CANDIDATE_SPEC.json @@ -0,0 +1,23 @@ +{ + "arrival_adverse_move_cancel": { + "max_price_improvement": 0.01, + "rule": "cancel if execution_ask < signal_ask - 0.01" + }, + "assets": [ + "btc", + "eth" + ], + "entry_second": 210, + "fee_per_share": "0.07*p*(1-p)", + "fok_limit": "signal_ask", + "hold": "settlement", + "latency_seconds": 1, + "live_submission": "disabled", + "max_positions_per_asset_per_window": 1, + "max_total_positions_per_window": 2, + "name": "late_favorite_btc_eth_v2_diagnostic", + "shares": 5, + "signal_ask_min": 0.85, + "source_artifact": "polymarket-5m-favorite-walkforward-v1", + "status": "diagnostic; prospective validation required" +} From 03a5d2d19a9f36ea5a570edd7745f98b5beb2933 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 11:46:11 -0500 Subject: [PATCH 072/177] docs: add deep EDA fresh-chat handoff --- .../DEEP_EDA_FRESH_CHAT_HANDOFF_2026-07-26.md | 249 ++++++++++++++++++ 1 file changed, 249 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/DEEP_EDA_FRESH_CHAT_HANDOFF_2026-07-26.md diff --git a/ci_payload/digital-surface-live-readiness/DEEP_EDA_FRESH_CHAT_HANDOFF_2026-07-26.md b/ci_payload/digital-surface-live-readiness/DEEP_EDA_FRESH_CHAT_HANDOFF_2026-07-26.md new file mode 100644 index 00000000..901d31dc --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/DEEP_EDA_FRESH_CHAT_HANDOFF_2026-07-26.md @@ -0,0 +1,249 @@ +# Polymarket 5-Minute Crypto HFT — Fresh-Chat Handoff + +Created 2026-07-26 after the prior chat reached its length limit and the next message crashed. + +## Paste into the fresh chat + +Continue the Polymarket 5-minute crypto high-frequency research project from this handoff. + +Start by restoring and verifying the newest remote state: + +- Active research repo: `Chrisbryan17/hello-world` +- Active branch: `digital-surface-live-readiness-20260725` +- Draft PR: `#59` +- Last verified research head before this handoff: `f8750201dc421b1c0e543712d8f207ef76705eaf` +- Latest verified commit message: `research: freeze diagnostic BTC ETH favorite candidate` +- Do not assume the branch has not advanced. Inspect PR #59 and branch history first. +- Read immediately: + - `ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/REPORT.md` + - `ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/CANDIDATE_SPEC.json` + - `ci_payload/digital-surface-live-readiness/STATUS.md` + +The strategic instruction is explicit: stop centering work on harness construction. Perform deep, microstructure-first EDA like a top Jane Street quant. Focus on event-time alignment, lead-lag discovery, conditional edge decay, fillability, regime clustering, strict walk-forward falsification, executable prices, and multiple-testing controls. Back up every material result to GitHub. Do not manufacture a passing strategy by leaking holdouts. + +The initial deep EDA is already complete. Do not redo it from scratch. It found a leading diagnostic candidate: + +- name: `late_favorite_btc_eth_v2_diagnostic` +- assets: BTC and ETH +- entry: 210 seconds into the 5-minute window +- favorite signal ask: at least $0.85 +- latency: 1 second +- FOK limit: signal ask +- size: 5 shares +- fee: `0.07*p*(1-p)` per share +- hold: settlement +- maximum one position per asset per window and two total positions per window +- diagnostic cancel: cancel when arrival ask is more than $0.01 below signal ask +- live submission: disabled +- status: profitable diagnostic candidate requiring new official-label prospective validation + +Next valid work: + +1. verify and preserve the latest branch/artifacts; +2. audit the deep EDA result for reproducibility and contamination; +3. transfer the frozen BTC/ETH candidate to official Polymarket terminal outcomes; +4. run an append-only prospective observer recording every eligible and ineligible BTC/ETH window, signal/no-signal, signal ask, 1-second arrival book, full depth, hypothetical FOK result, adverse-move cancel, official outcome, fees, slippage, and per-window exposure; +5. run at least four untouched weekly prospective blocks; +6. continue deeper EDA in parallel, but version every new hypothesis discovered using inspected data; +7. keep live order submission physically absent. + +For every meaningful pass: tests -> source/config/results/audit files -> atomic commit -> push -> fetch/read-back -> SHA/blob verification -> next experiment. + +## Distinguish the two strategy tracks + +### A. Original 5m-vs-15m digital-surface sleeve + +After official-resolution correction and four untouched historical folds, this sleeve was honestly **Rejected**. Those weeks are diagnostic and may never count as prospective evidence. + +Execution diagnosis across 224 portfolios / 448 legs: + +- insufficient post-arrival tape: 186 +- ask above frozen limit: 114 +- missing post-arrival book: 110 +- filled: 34 +- insufficient displayed depth: 4 + +Tape-confirmed replay produced 8 paired fills. Diagnostic book-arrival replay estimated 62 paired-crossable portfolios with fold counts `[11,17,29,5]`. This is not transactionally atomic and not live truth. + +### B. Deep late-favorite candidate + +The broader EDA corpus contained 89,233 markets and approximately 26.77M one-second observations across BTC, ETH, SOL, XRP, DOGE, BNB, and HYPE. + +The strongest candidate was not BTC-to-alt lead-lag. It was late-window settlement favorite, with BTC+ETH the statistically credible core. + +## Deep EDA evidence + +All-seven strict walk-forward: + +- train: 38 days, 18,493 trades, +$1,088.33 at five shares, daily t=5.91 +- validation: 14 days, 6,819 trades, +$395.42, daily t=5.35, every day positive +- final chronological test: 12 days, 6,769 trades, +$249.11, 24.67 trades/hour, daily t=1.67 +- 210s / 85c ranked #1 of 982 eligible configurations by validation daily t +- tested latency 0-3 seconds remained positive in train and validation + +BTC+ETH core: + +- 2,054 final-test trades +- +$139.72 at five shares +- 95.67% win rate +- 1.3604 cents mean P&L/share after fee +- daily t=3.85 +- exact one-sided day-level sign-flip p=0.00244 +- 95% day-bootstrap mean interval: +$5.84 to +$17.13/day at five shares +- BTC and ETH survive BH control at 5% across seven assets + +Adverse selection: + +- favorite ask falling by >=$0.02 during the 1-second latency produced 288 trades, 84.38% win rate, and -$48.44 +- $0.01 improvement remained positive +- therefore the prospective rule is cancel when `execution_ask < signal_ask - 0.01` +- this rule was found after final-test inspection and has no untouched status + +Cross-asset risk: + +- 38.66% of losses occurred in windows with at least two simultaneous losses +- BTC/ETH loss correlation: 0.559 +- when BTC won, ETH won 98.39% of simultaneous windows +- when BTC lost, ETH won only 38.46% +- all-seven top-two-day P&L concentration: 66.62% +- BTC+ETH concentration: 38.02% +- cap by five-minute window, not only by asset + +Capacity/cost: + +- BTC+ETH break-even additional cost: 1.3604 cents/share +- all-seven break-even additional cost: 0.7360 cents/share +- at 100 shares, 56.13% of BTC/ETH test signals showed enough top-of-book size and stayed positive, but tail risk scaled sharply +- 250+ shares deteriorated +- top-of-book size is not a live capacity guarantee + +## Non-negotiable uncertainty + +1. Dataset outcomes were inferred from the final recorded book, not authoritative settlement. +2. Only 28 falsely labeled winners erase BTC+ETH final-test profit. +3. Official terminal-outcome validation is mandatory. +4. BTC/ETH selection and the adverse-move cancel were learned after final-test inspection. +5. Top-of-book depth does not model queue position. +6. Two separately submitted FOK orders are not transactionally atomic. +7. Previously inspected markets cannot count prospectively. +8. Policy, source, ledger, and reports must be hash-bound. +9. Real-money execution remains absent. + +## Existing infrastructure to reuse + +PR #59 already contains checksum-pinned recovery material for: + +- official outcome/source correction; +- slug/timestamp/universe/asset merge fixes; +- deterministic archives; +- leg-level execution attribution; +- tape-confirmed and book-arrival replay; +- Decimal tick quantization; +- shadow-only pair preparation; +- stale data, loss, orphan, unresolved-order, token, depth, limit, expiry, and ambiguous-state kill switches; +- append-only prospective market, book, BTC-state, and report chains; +- historical contamination rejection; +- public Gamma market discovery; +- public CLOB book collection; +- public Binance spot/strike/volatility collection; +- causal prospective signal adapter; +- live startup physically disabled. + +## Raw inputs from prior chat + +- `polymarket-5m-btc(1).zip` +- `polymarket-5m-eth.zip` +- `polymarket-5m-sol(1).zip` +- `polymarket-5m-xrp.zip` +- `polymarket-5m-doge.zip` +- `polymarket-5m-bnb.zip` +- `polymarket-5m-hype.zip` +- `Branch · BTC Lead-Lag Strategy.txt` +- `Transcript.txt` + +The new filesystem is temporary. Re-upload raw ZIPs only when needed; durable state belongs on GitHub. + +## Required deep EDA continuation + +Continue all of these, using executable bid/ask economics: + +1. common-factor-residual cross-asset lead-lag at 1,2,3,5,10,15,30 seconds; +2. same-market complement arbitrage with both-leg fees/depth/latency; +3. microstructure markouts from spread, imbalance, depth curvature, update intensity, replenishment, and time-to-expiry; +4. regime clustering by volatility, cross-asset correlation, liquidity, time-of-day, favorite price, expiry, and crash state; +5. purged walk-forward, embargo, block bootstrap, sign-flip, BH/FWER, White Reality Check/SPA, PBO, deflated Sharpe, leave-day/regime-out, and concentration diagnostics. + +Any candidate discovered using an inspected final test becomes a new prospective version and cannot inherit old holdout status. + +## Admission standard + +The BTC/ETH candidate needs: + +- authoritative outcomes; +- four untouched weekly blocks; +- positive P&L after measured fees/slippage; +- positive lower confidence bound; +- no extreme day concentration; +- stable BTC/ETH performance; +- acceptable common-factor crash exposure; +- measured fill/capacity; +- zero unresolved states; +- frozen policy throughout; +- complete append-only verification. + +Even then, live requires a separate reviewed and explicitly armed release. + +## Recovery commands + +```bash +git clone https://github.com/Chrisbryan17/hello-world.git +cd hello-world +git fetch --all --prune +git checkout digital-surface-live-readiness-20260725 +git pull --ff-only +git rev-parse HEAD +git status --short --branch +git log --oneline --decorate -30 + +sed -n '1,240p' ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/REPORT.md +cat ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/CANDIDATE_SPEC.json +sed -n '1,220p' ci_payload/digital-surface-live-readiness/STATUS.md +find ci_payload/digital-surface-live-readiness -maxdepth 4 -type f | sort +``` + +Original authoritative provenance when needed: + +```bash +git clone https://github.com/Chrisbryan17/trading-tools.git ../trading-tools +cd ../trading-tools +git fetch --all --prune +git checkout digital-surface-arbitrage-implementation-20260723 +git log --oneline --decorate -30 +``` + +## What not to do + +- do not rebuild generic harnesses; +- do not tune known historical folds until they pass; +- do not use midpoint fills; +- do not ignore fees; +- do not fit a fine lookup table to inspected price buckets; +- do not call correlated BTC/ETH independent diversification; +- do not increase size from top-of-book alone; +- do not count diagnostic markets prospectively; +- do not add credentials or an order submitter; +- do not claim profitability before official-label transfer and prospective evidence; +- do not start the next experiment before committing, pushing, reading back, and verifying the current one. + +## Definition of success for the next chat + +- restore the actual latest PR #59 head; +- confirm it is newer than or equal to `f875020…`; +- avoid repeating initial harness work; +- reproduce/audit the deep-favorite result; +- begin official-label transfer or genuine prospective collection; +- produce a decision-useful new EDA result or falsification; +- push source, tests, configs, results, and audit metadata; +- keep live execution disabled. + +The goal is not to force a pass. The goal is to find a genuine executable edge or falsify this candidate quickly without fooling ourselves. From 2f6b165cc4605bfd7491070638866203f8c9cedb Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:23:20 -0500 Subject: [PATCH 073/177] test: add red late-favorite reproducibility contract --- .../workflows/late-favorite-repro-audit.yml | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/late-favorite-repro-audit.yml diff --git a/.github/workflows/late-favorite-repro-audit.yml b/.github/workflows/late-favorite-repro-audit.yml new file mode 100644 index 00000000..8a8b3d70 --- /dev/null +++ b/.github/workflows/late-favorite-repro-audit.yml @@ -0,0 +1,56 @@ +name: Late Favorite Repro Audit + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/late-favorite-repro-audit.yml + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + red-contract: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install test dependency + run: python -m pip install pytest==8.3.5 + - name: Write frozen-policy contract test + shell: bash + run: | + mkdir -p /tmp/late-favorite-tests + cat > /tmp/late-favorite-tests/test_contract.py <<'PY' + import importlib + import pytest + + def test_frozen_policy_contract_exists(): + try: + module = importlib.import_module('late_favorite_transfer') + except ModuleNotFoundError: + pytest.fail('late_favorite_transfer is not implemented') + assert module.frozen_policy() == { + 'assets': ('btc', 'eth'), + 'entry_second': 210, + 'signal_ask_min': 0.85, + 'latency_seconds': 1, + 'fok_limit': 'signal_ask', + 'shares': 5, + 'cancel_below_signal_by_more_than': 0.01, + 'hold': 'settlement', + 'fee_rate': 0.07, + 'live_submission': 'disabled', + } + PY + - name: Verify RED contract + working-directory: /tmp/late-favorite-tests + run: pytest -q From f22d83e9229f6137358fded64af63b298550399f Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:23:55 -0500 Subject: [PATCH 074/177] ci: expose red late-favorite contract on PR --- .github/workflows/late-favorite-repro-audit.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/late-favorite-repro-audit.yml b/.github/workflows/late-favorite-repro-audit.yml index 8a8b3d70..8525e185 100644 --- a/.github/workflows/late-favorite-repro-audit.yml +++ b/.github/workflows/late-favorite-repro-audit.yml @@ -5,6 +5,10 @@ on: branches: [digital-surface-live-readiness-20260725] paths: - .github/workflows/late-favorite-repro-audit.yml + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/late-favorite-repro-audit.yml workflow_dispatch: permissions: From 860526b966ff62813c2c7694a95ac179caae92e1 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:27:38 -0500 Subject: [PATCH 075/177] research: add late-favorite reproducibility module --- .../late_favorite_transfer.py | 310 ++++++++++++++++++ 1 file changed, 310 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py b/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py new file mode 100644 index 00000000..a5b6de7e --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py @@ -0,0 +1,310 @@ +from __future__ import annotations + +import argparse +import gzip +import hashlib +import json +import math +from pathlib import Path +from typing import Any, Mapping + +import pandas as pd +from huggingface_hub import hf_hub_download + +DATASET_REPO = "kachoio/polymarket-5-minute-crypto-up-down-markets" +ASSETS = ("btc", "eth") + +_POLICY = { + "assets": ASSETS, + "entry_second": 210, + "signal_ask_min": 0.85, + "latency_seconds": 1, + "fok_limit": "signal_ask", + "shares": 5, + "cancel_below_signal_by_more_than": 0.01, + "hold": "settlement", + "fee_rate": 0.07, + "live_submission": "disabled", +} + + +def frozen_policy() -> dict[str, Any]: + return dict(_POLICY) + + +def fee_per_share(price: float) -> float: + p = float(price) + if not math.isfinite(p) or not 0.0 < p < 1.0: + raise ValueError("price must be finite and strictly between zero and one") + return _POLICY["fee_rate"] * p * (1.0 - p) + + +def _finite(value: Any) -> float | None: + try: + parsed = float(value) + except (TypeError, ValueError): + return None + return parsed if math.isfinite(parsed) else None + + +def evaluate_market( + signal: Mapping[str, Any] | None, + arrival: Mapping[str, Any] | None, + *, + outcome: str | None, +) -> dict[str, Any]: + result: dict[str, Any] = { + "selected_side": None, + "signal_ask": None, + "arrival_ask": None, + "arrival_ask_size": None, + "signal": False, + "hypothetical_fok_fill": False, + "decision": "no_signal_missing_signal_book", + "won": None, + "fee_per_share": None, + "pnl_per_share": None, + "pnl_at_five_shares": None, + } + if signal is None: + return result + + up_ask = _finite(signal.get("au")) + down_ask = _finite(signal.get("ad")) + if up_ask is None or down_ask is None or not (0.0 < up_ask < 1.0 and 0.0 < down_ask < 1.0): + result["decision"] = "no_signal_invalid_signal_book" + return result + if math.isclose(up_ask, down_ask, rel_tol=0.0, abs_tol=1e-12): + result["decision"] = "no_signal_tied_favorite" + return result + + selected_side = "Up" if up_ask > down_ask else "Down" + signal_ask = up_ask if selected_side == "Up" else down_ask + result.update({"selected_side": selected_side, "signal_ask": signal_ask}) + if signal_ask < _POLICY["signal_ask_min"]: + result["decision"] = "no_signal_below_threshold" + return result + result["signal"] = True + + if arrival is None: + result["decision"] = "signal_missing_arrival_book" + return result + arrival_ask = _finite(arrival.get("au" if selected_side == "Up" else "ad")) + arrival_size = _finite(arrival.get("sau" if selected_side == "Up" else "sad")) + result.update({"arrival_ask": arrival_ask, "arrival_ask_size": arrival_size}) + if arrival_ask is None or not 0.0 < arrival_ask < 1.0: + result["decision"] = "signal_invalid_arrival_ask" + return result + + cancel_floor = signal_ask - _POLICY["cancel_below_signal_by_more_than"] + if arrival_ask < cancel_floor - 1e-12: + result["decision"] = "cancel_adverse_move" + return result + if arrival_ask > signal_ask + 1e-12: + result["decision"] = "no_fill_ask_above_frozen_limit" + return result + if arrival_size is None or arrival_size < _POLICY["shares"]: + result["decision"] = "no_fill_insufficient_displayed_depth" + return result + + result["hypothetical_fok_fill"] = True + result["decision"] = "hypothetical_fok_fill" + normalized_outcome = str(outcome).title() if outcome is not None else None + if normalized_outcome in {"Up", "Down"}: + won = selected_side == normalized_outcome + fee = fee_per_share(arrival_ask) + pnl_per_share = float(won) - arrival_ask - fee + result.update( + { + "won": won, + "fee_per_share": fee, + "pnl_per_share": pnl_per_share, + "pnl_at_five_shares": pnl_per_share * _POLICY["shares"], + } + ) + return result + + +def _sha256_bytes(raw: bytes) -> str: + return hashlib.sha256(raw).hexdigest() + + +def _canonical_id_hash(values: pd.Series) -> str: + ids = sorted({str(value) for value in values.dropna()}) + return _sha256_bytes((("\n".join(ids) + "\n") if ids else "").encode()) + + +def verify_candidate_spec(path: Path) -> str: + raw = path.read_bytes() + payload = json.loads(raw) + expected = { + "assets": ["btc", "eth"], + "entry_second": 210, + "signal_ask_min": 0.85, + "latency_seconds": 1, + "fok_limit": "signal_ask", + "shares": 5, + "hold": "settlement", + "fee_per_share": "0.07*p*(1-p)", + "live_submission": "disabled", + "max_positions_per_asset_per_window": 1, + "max_total_positions_per_window": 2, + } + for key, value in expected.items(): + if payload.get(key) != value: + raise RuntimeError(f"candidate spec mismatch for {key}: {payload.get(key)!r} != {value!r}") + cancel = payload.get("arrival_adverse_move_cancel") or {} + if cancel.get("max_price_improvement") != 0.01: + raise RuntimeError("candidate spec cancel threshold mismatch") + return _sha256_bytes(raw) + + +def build_market_ledger(asset: str, markets_path: Path, ticks_path: Path) -> pd.DataFrame: + asset = asset.lower() + if asset not in ASSETS: + raise ValueError(f"unsupported asset: {asset}") + markets = pd.read_parquet( + markets_path, + columns=["condition_id", "event_id", "slug", "market_start", "market_end", "outcome"], + ) + markets["market_start"] = pd.to_datetime(markets["market_start"], utc=True) + markets["market_start_s"] = markets["market_start"].astype("int64") // 1_000_000_000 + markets = markets.sort_values(["market_start", "condition_id"]).drop_duplicates("condition_id", keep="last") + + ticks = pd.read_parquet( + ticks_path, + columns=["condition_id", "t", "bu", "au", "bd", "ad", "sau", "sad"], + ) + starts = markets.set_index("condition_id")["market_start_s"] + ticks["market_start_s"] = ticks["condition_id"].map(starts) + ticks = ticks.loc[ticks["market_start_s"].notna()].copy() + ticks["elapsed"] = ticks["t"].astype("int64") - ticks["market_start_s"].astype("int64") + needed = {_POLICY["entry_second"], _POLICY["entry_second"] + _POLICY["latency_seconds"]} + ticks = ticks.loc[ticks["elapsed"].isin(needed)].copy() + duplicates = ticks.duplicated(["condition_id", "elapsed"], keep=False) + if duplicates.any(): + examples = ticks.loc[duplicates, ["condition_id", "elapsed"]].head(10).to_dict("records") + raise RuntimeError(f"duplicate exact-second observations: {examples}") + indexed = {(str(row.condition_id), int(row.elapsed)): row._asdict() for row in ticks.itertuples(index=False)} + + rows: list[dict[str, Any]] = [] + signal_second = _POLICY["entry_second"] + arrival_second = signal_second + _POLICY["latency_seconds"] + for market in markets.itertuples(index=False): + condition_id = str(market.condition_id) + decision = evaluate_market( + indexed.get((condition_id, signal_second)), + indexed.get((condition_id, arrival_second)), + outcome=market.outcome, + ) + rows.append( + { + "asset": asset, + "condition_id": condition_id, + "event_id": str(market.event_id), + "slug": str(market.slug), + "market_start": market.market_start, + "market_end": market.market_end, + "inferred_outcome": market.outcome, + **decision, + } + ) + return pd.DataFrame(rows) + + +def _download_asset(asset: str, cache_dir: Path) -> tuple[Path, Path]: + markets = hf_hub_download( + repo_id=DATASET_REPO, + repo_type="dataset", + filename=f"{asset}_markets.parquet", + local_dir=cache_dir, + ) + ticks = hf_hub_download( + repo_id=DATASET_REPO, + repo_type="dataset", + filename=f"{asset}_ticks.parquet", + local_dir=cache_dir, + ) + return Path(markets), Path(ticks) + + +def run_audit(candidate_spec: Path, out_dir: Path, cache_dir: Path) -> dict[str, Any]: + out_dir.mkdir(parents=True, exist_ok=True) + cache_dir.mkdir(parents=True, exist_ok=True) + policy_sha256 = verify_candidate_spec(candidate_spec) + ledgers = [] + source_files: dict[str, str] = {} + for asset in ASSETS: + markets_path, ticks_path = _download_asset(asset, cache_dir) + source_files[f"{asset}_markets"] = _sha256_bytes(markets_path.read_bytes()) + source_files[f"{asset}_ticks"] = _sha256_bytes(ticks_path.read_bytes()) + ledgers.append(build_market_ledger(asset, markets_path, ticks_path)) + ledger = pd.concat(ledgers, ignore_index=True).sort_values(["market_start", "asset", "condition_id"]) + eligible = ledger.loc[ledger["hypothetical_fok_fill"]].copy() + + ledger_path = out_dir / "market_ledger.csv.gz" + eligible_path = out_dir / "eligible_trades.csv.gz" + ledger.to_csv(ledger_path, index=False, compression={"method": "gzip", "mtime": 0}) + eligible.to_csv(eligible_path, index=False, compression={"method": "gzip", "mtime": 0}) + + by_asset: dict[str, Any] = {} + for asset, frame in ledger.groupby("asset", sort=True): + fills = frame.loc[frame["hypothetical_fok_fill"]] + by_asset[asset] = { + "markets": int(len(frame)), + "signals": int(frame["signal"].sum()), + "hypothetical_fok_fills": int(frame["hypothetical_fok_fill"].sum()), + "inferred_label_wins": int(fills["won"].fillna(False).sum()), + "inferred_label_pnl_at_five_shares": float(fills["pnl_at_five_shares"].sum()), + "first_market_start": str(frame["market_start"].min()), + "last_market_start": str(frame["market_start"].max()), + "all_condition_ids_sha256": _canonical_id_hash(frame["condition_id"]), + "filled_condition_ids_sha256": _canonical_id_hash(fills["condition_id"]), + } + decision_counts = {str(k): int(v) for k, v in ledger["decision"].value_counts(dropna=False).sort_index().items()} + audit: dict[str, Any] = { + "candidate": "late_favorite_btc_eth_v2_diagnostic", + "candidate_spec_sha256": policy_sha256, + "dataset_repo": DATASET_REPO, + "source_files_sha256": source_files, + "policy": frozen_policy(), + "markets": int(len(ledger)), + "signals": int(ledger["signal"].sum()), + "hypothetical_fok_fills": int(ledger["hypothetical_fok_fill"].sum()), + "all_condition_ids_sha256": _canonical_id_hash(ledger["condition_id"]), + "filled_condition_ids_sha256": _canonical_id_hash(eligible["condition_id"]), + "decision_counts": decision_counts, + "by_asset": by_asset, + "outcomes": "dataset-inferred; not admission evidence", + "official_outcomes_used": 0, + "live_submission": "physically_absent", + "split_reproduction": "not claimed: original deep-EDA split manifest/source is absent", + "future_outcomes_used_for_signal": 0, + } + audit_path = out_dir / "AUDIT.json" + audit_path.write_text(json.dumps(audit, indent=2, sort_keys=True, default=list) + "\n") + + report = f"""# Late-Favorite BTC/ETH Reproducibility Audit\n\n" + report += "## Scope\n\nThis checkpoint applies the frozen 210-second BTC/ETH rule to every market in the public source corpus and binds the resulting market sets by SHA-256. It does **not** claim to reproduce the previously reported final-test split because the original split manifest and exact deep-EDA source were not preserved in the branch.\n\n" + report += f"- Markets: {len(ledger):,}\n- Signals: {int(ledger['signal'].sum()):,}\n- Hypothetical five-share FOK fills: {int(ledger['hypothetical_fok_fill'].sum()):,}\n" + report += f"- All-condition-set SHA-256: `{audit['all_condition_ids_sha256']}`\n- Filled-condition-set SHA-256: `{audit['filled_condition_ids_sha256']}`\n\n" + report += "## Evidence boundary\n\nPnL in this artifact uses the dataset's inferred outcomes only. It is diagnostic and cannot satisfy official-label transfer or prospective admission. No credentials, authenticated request, or order-submission implementation exists in this checkpoint.\n" + (out_dir / "REPORT.md").write_text(report) + + outputs = [audit_path, eligible_path, ledger_path, out_dir / "REPORT.md"] + sums = "".join(f"{_sha256_bytes(path.read_bytes())} {path.name}\n" for path in sorted(outputs)) + (out_dir / "SHA256SUMS").write_text(sums) + return audit + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--candidate-spec", type=Path, required=True) + parser.add_argument("--out-dir", type=Path, required=True) + parser.add_argument("--cache-dir", type=Path, required=True) + args = parser.parse_args() + print(json.dumps(run_audit(args.candidate_spec, args.out_dir, args.cache_dir), indent=2, sort_keys=True, default=list)) + + +if __name__ == "__main__": + main() From aad09abb426a703ce6ee6eb8868a2da0b40ea1ec Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:28:31 -0500 Subject: [PATCH 076/177] test: specify late-favorite transfer behavior --- .../test_late_favorite_transfer.py | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_late_favorite_transfer.py diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_late_favorite_transfer.py b/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_late_favorite_transfer.py new file mode 100644 index 00000000..9f2391a9 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_late_favorite_transfer.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import math + +import pytest + +from late_favorite_transfer import evaluate_market, fee_per_share, frozen_policy + + +def signal(up: float, down: float) -> dict[str, float]: + return {"au": up, "ad": down} + + +def arrival(up: float, down: float, up_size: float = 5.0, down_size: float = 5.0) -> dict[str, float]: + return {"au": up, "ad": down, "sau": up_size, "sad": down_size} + + +def test_frozen_policy_matches_the_reviewed_candidate() -> None: + assert frozen_policy() == { + "assets": ("btc", "eth"), + "entry_second": 210, + "signal_ask_min": 0.85, + "latency_seconds": 1, + "fok_limit": "signal_ask", + "shares": 5, + "cancel_below_signal_by_more_than": 0.01, + "hold": "settlement", + "fee_rate": 0.07, + "live_submission": "disabled", + } + + +def test_fee_uses_reviewed_polymarket_curve() -> None: + assert fee_per_share(0.9) == pytest.approx(0.07 * 0.9 * 0.1) + + +def test_fills_at_frozen_limit_with_sufficient_displayed_depth() -> None: + result = evaluate_market(signal(0.90, 0.12), arrival(0.90, 0.12), outcome="Up") + assert result["decision"] == "hypothetical_fok_fill" + assert result["selected_side"] == "Up" + assert result["hypothetical_fok_fill"] is True + assert result["pnl_per_share"] == pytest.approx(1.0 - 0.90 - fee_per_share(0.90)) + + +def test_cancels_when_price_improves_by_more_than_one_cent() -> None: + result = evaluate_market(signal(0.90, 0.12), arrival(0.88, 0.12), outcome="Up") + assert result["decision"] == "cancel_adverse_move" + assert result["hypothetical_fok_fill"] is False + + +def test_one_cent_improvement_remains_eligible() -> None: + result = evaluate_market(signal(0.90, 0.12), arrival(0.89, 0.12), outcome="Up") + assert result["decision"] == "hypothetical_fok_fill" + + +def test_rejects_arrival_ask_above_frozen_limit() -> None: + result = evaluate_market(signal(0.90, 0.12), arrival(0.91, 0.12), outcome="Up") + assert result["decision"] == "no_fill_ask_above_frozen_limit" + + +def test_rejects_insufficient_displayed_depth() -> None: + result = evaluate_market(signal(0.90, 0.12), arrival(0.90, 0.12, up_size=4.99), outcome="Up") + assert result["decision"] == "no_fill_insufficient_displayed_depth" + + +def test_tied_favorite_fails_closed() -> None: + result = evaluate_market(signal(0.50, 0.50), arrival(0.50, 0.50), outcome="Up") + assert result["decision"] == "no_signal_tied_favorite" + assert result["signal"] is False + + +def test_signal_below_threshold_is_not_traded() -> None: + result = evaluate_market(signal(0.84, 0.18), arrival(0.84, 0.18), outcome="Up") + assert result["decision"] == "no_signal_below_threshold" + + +def test_loss_pnl_uses_executable_ask_and_fee() -> None: + result = evaluate_market(signal(0.90, 0.12), arrival(0.90, 0.12), outcome="Down") + assert result["won"] is False + assert result["pnl_per_share"] == pytest.approx(-0.90 - fee_per_share(0.90)) + assert math.isclose(result["pnl_at_five_shares"], result["pnl_per_share"] * 5) From d0026ecbd30d2b6f13c2ec2bacf0417960704adc Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:29:37 -0500 Subject: [PATCH 077/177] fix: repair late-favorite audit source serialization --- .../late_favorite_transfer.py | 98 +++++++++++-------- 1 file changed, 58 insertions(+), 40 deletions(-) diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py b/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py index a5b6de7e..e37ce4a7 100644 --- a/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py +++ b/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py @@ -1,7 +1,6 @@ from __future__ import annotations import argparse -import gzip import hashlib import json import math @@ -13,8 +12,7 @@ DATASET_REPO = "kachoio/polymarket-5-minute-crypto-up-down-markets" ASSETS = ("btc", "eth") - -_POLICY = { +POLICY = { "assets": ASSETS, "entry_second": 210, "signal_ask_min": 0.85, @@ -29,14 +27,14 @@ def frozen_policy() -> dict[str, Any]: - return dict(_POLICY) + return dict(POLICY) def fee_per_share(price: float) -> float: p = float(price) if not math.isfinite(p) or not 0.0 < p < 1.0: raise ValueError("price must be finite and strictly between zero and one") - return _POLICY["fee_rate"] * p * (1.0 - p) + return POLICY["fee_rate"] * p * (1.0 - p) def _finite(value: Any) -> float | None: @@ -81,7 +79,7 @@ def evaluate_market( selected_side = "Up" if up_ask > down_ask else "Down" signal_ask = up_ask if selected_side == "Up" else down_ask result.update({"selected_side": selected_side, "signal_ask": signal_ask}) - if signal_ask < _POLICY["signal_ask_min"]: + if signal_ask < POLICY["signal_ask_min"]: result["decision"] = "no_signal_below_threshold" return result result["signal"] = True @@ -89,21 +87,23 @@ def evaluate_market( if arrival is None: result["decision"] = "signal_missing_arrival_book" return result - arrival_ask = _finite(arrival.get("au" if selected_side == "Up" else "ad")) - arrival_size = _finite(arrival.get("sau" if selected_side == "Up" else "sad")) + ask_column = "au" if selected_side == "Up" else "ad" + size_column = "sau" if selected_side == "Up" else "sad" + arrival_ask = _finite(arrival.get(ask_column)) + arrival_size = _finite(arrival.get(size_column)) result.update({"arrival_ask": arrival_ask, "arrival_ask_size": arrival_size}) if arrival_ask is None or not 0.0 < arrival_ask < 1.0: result["decision"] = "signal_invalid_arrival_ask" return result - cancel_floor = signal_ask - _POLICY["cancel_below_signal_by_more_than"] + cancel_floor = signal_ask - POLICY["cancel_below_signal_by_more_than"] if arrival_ask < cancel_floor - 1e-12: result["decision"] = "cancel_adverse_move" return result if arrival_ask > signal_ask + 1e-12: result["decision"] = "no_fill_ask_above_frozen_limit" return result - if arrival_size is None or arrival_size < _POLICY["shares"]: + if arrival_size is None or arrival_size < POLICY["shares"]: result["decision"] = "no_fill_insufficient_displayed_depth" return result @@ -119,7 +119,7 @@ def evaluate_market( "won": won, "fee_per_share": fee, "pnl_per_share": pnl_per_share, - "pnl_at_five_shares": pnl_per_share * _POLICY["shares"], + "pnl_at_five_shares": pnl_per_share * POLICY["shares"], } ) return result @@ -131,7 +131,8 @@ def _sha256_bytes(raw: bytes) -> str: def _canonical_id_hash(values: pd.Series) -> str: ids = sorted({str(value) for value in values.dropna()}) - return _sha256_bytes((("\n".join(ids) + "\n") if ids else "").encode()) + canonical = (("\n".join(ids) + "\n") if ids else "").encode() + return _sha256_bytes(canonical) def verify_candidate_spec(path: Path) -> str: @@ -163,24 +164,29 @@ def build_market_ledger(asset: str, markets_path: Path, ticks_path: Path) -> pd. asset = asset.lower() if asset not in ASSETS: raise ValueError(f"unsupported asset: {asset}") - markets = pd.read_parquet( - markets_path, - columns=["condition_id", "event_id", "slug", "market_start", "market_end", "outcome"], - ) + + markets = pd.read_parquet(markets_path) + required_market_columns = {"condition_id", "market_start", "outcome"} + missing_market_columns = sorted(required_market_columns - set(markets.columns)) + if missing_market_columns: + raise RuntimeError(f"market source missing columns: {missing_market_columns}") + for optional in ("event_id", "slug", "market_end"): + if optional not in markets.columns: + markets[optional] = None + markets = markets[["condition_id", "event_id", "slug", "market_start", "market_end", "outcome"]].copy() markets["market_start"] = pd.to_datetime(markets["market_start"], utc=True) markets["market_start_s"] = markets["market_start"].astype("int64") // 1_000_000_000 markets = markets.sort_values(["market_start", "condition_id"]).drop_duplicates("condition_id", keep="last") - ticks = pd.read_parquet( - ticks_path, - columns=["condition_id", "t", "bu", "au", "bd", "ad", "sau", "sad"], - ) + tick_columns = ["condition_id", "t", "bu", "au", "bd", "ad", "sau", "sad"] + ticks = pd.read_parquet(ticks_path, columns=tick_columns) starts = markets.set_index("condition_id")["market_start_s"] ticks["market_start_s"] = ticks["condition_id"].map(starts) ticks = ticks.loc[ticks["market_start_s"].notna()].copy() ticks["elapsed"] = ticks["t"].astype("int64") - ticks["market_start_s"].astype("int64") - needed = {_POLICY["entry_second"], _POLICY["entry_second"] + _POLICY["latency_seconds"]} - ticks = ticks.loc[ticks["elapsed"].isin(needed)].copy() + signal_second = POLICY["entry_second"] + arrival_second = signal_second + POLICY["latency_seconds"] + ticks = ticks.loc[ticks["elapsed"].isin({signal_second, arrival_second})].copy() duplicates = ticks.duplicated(["condition_id", "elapsed"], keep=False) if duplicates.any(): examples = ticks.loc[duplicates, ["condition_id", "elapsed"]].head(10).to_dict("records") @@ -188,8 +194,6 @@ def build_market_ledger(asset: str, markets_path: Path, ticks_path: Path) -> pd. indexed = {(str(row.condition_id), int(row.elapsed)): row._asdict() for row in ticks.itertuples(index=False)} rows: list[dict[str, Any]] = [] - signal_second = _POLICY["entry_second"] - arrival_second = signal_second + _POLICY["latency_seconds"] for market in markets.itertuples(index=False): condition_id = str(market.condition_id) decision = evaluate_market( @@ -201,8 +205,8 @@ def build_market_ledger(asset: str, markets_path: Path, ticks_path: Path) -> pd. { "asset": asset, "condition_id": condition_id, - "event_id": str(market.event_id), - "slug": str(market.slug), + "event_id": None if market.event_id is None else str(market.event_id), + "slug": None if market.slug is None else str(market.slug), "market_start": market.market_start, "market_end": market.market_end, "inferred_outcome": market.outcome, @@ -232,20 +236,21 @@ def run_audit(candidate_spec: Path, out_dir: Path, cache_dir: Path) -> dict[str, out_dir.mkdir(parents=True, exist_ok=True) cache_dir.mkdir(parents=True, exist_ok=True) policy_sha256 = verify_candidate_spec(candidate_spec) - ledgers = [] + ledgers: list[pd.DataFrame] = [] source_files: dict[str, str] = {} for asset in ASSETS: markets_path, ticks_path = _download_asset(asset, cache_dir) source_files[f"{asset}_markets"] = _sha256_bytes(markets_path.read_bytes()) source_files[f"{asset}_ticks"] = _sha256_bytes(ticks_path.read_bytes()) ledgers.append(build_market_ledger(asset, markets_path, ticks_path)) + ledger = pd.concat(ledgers, ignore_index=True).sort_values(["market_start", "asset", "condition_id"]) eligible = ledger.loc[ledger["hypothetical_fok_fill"]].copy() - ledger_path = out_dir / "market_ledger.csv.gz" eligible_path = out_dir / "eligible_trades.csv.gz" - ledger.to_csv(ledger_path, index=False, compression={"method": "gzip", "mtime": 0}) - eligible.to_csv(eligible_path, index=False, compression={"method": "gzip", "mtime": 0}) + compression = {"method": "gzip", "mtime": 0} + ledger.to_csv(ledger_path, index=False, compression=compression) + eligible.to_csv(eligible_path, index=False, compression=compression) by_asset: dict[str, Any] = {} for asset, frame in ledger.groupby("asset", sort=True): @@ -261,19 +266,23 @@ def run_audit(candidate_spec: Path, out_dir: Path, cache_dir: Path) -> dict[str, "all_condition_ids_sha256": _canonical_id_hash(frame["condition_id"]), "filled_condition_ids_sha256": _canonical_id_hash(fills["condition_id"]), } - decision_counts = {str(k): int(v) for k, v in ledger["decision"].value_counts(dropna=False).sort_index().items()} + audit: dict[str, Any] = { "candidate": "late_favorite_btc_eth_v2_diagnostic", "candidate_spec_sha256": policy_sha256, "dataset_repo": DATASET_REPO, "source_files_sha256": source_files, + "source_module_sha256": _sha256_bytes(Path(__file__).read_bytes()), "policy": frozen_policy(), "markets": int(len(ledger)), "signals": int(ledger["signal"].sum()), "hypothetical_fok_fills": int(ledger["hypothetical_fok_fill"].sum()), "all_condition_ids_sha256": _canonical_id_hash(ledger["condition_id"]), "filled_condition_ids_sha256": _canonical_id_hash(eligible["condition_id"]), - "decision_counts": decision_counts, + "decision_counts": { + str(key): int(value) + for key, value in ledger["decision"].value_counts(dropna=False).sort_index().items() + }, "by_asset": by_asset, "outcomes": "dataset-inferred; not admission evidence", "official_outcomes_used": 0, @@ -284,14 +293,22 @@ def run_audit(candidate_spec: Path, out_dir: Path, cache_dir: Path) -> dict[str, audit_path = out_dir / "AUDIT.json" audit_path.write_text(json.dumps(audit, indent=2, sort_keys=True, default=list) + "\n") - report = f"""# Late-Favorite BTC/ETH Reproducibility Audit\n\n" - report += "## Scope\n\nThis checkpoint applies the frozen 210-second BTC/ETH rule to every market in the public source corpus and binds the resulting market sets by SHA-256. It does **not** claim to reproduce the previously reported final-test split because the original split manifest and exact deep-EDA source were not preserved in the branch.\n\n" - report += f"- Markets: {len(ledger):,}\n- Signals: {int(ledger['signal'].sum()):,}\n- Hypothetical five-share FOK fills: {int(ledger['hypothetical_fok_fill'].sum()):,}\n" - report += f"- All-condition-set SHA-256: `{audit['all_condition_ids_sha256']}`\n- Filled-condition-set SHA-256: `{audit['filled_condition_ids_sha256']}`\n\n" - report += "## Evidence boundary\n\nPnL in this artifact uses the dataset's inferred outcomes only. It is diagnostic and cannot satisfy official-label transfer or prospective admission. No credentials, authenticated request, or order-submission implementation exists in this checkpoint.\n" - (out_dir / "REPORT.md").write_text(report) + report = ( + "# Late-Favorite BTC/ETH Reproducibility Audit\n\n" + "## Scope\n\n" + "This checkpoint applies the frozen 210-second BTC/ETH rule to every market in the public source corpus and binds the resulting market sets by SHA-256. It does **not** claim to reproduce the previously reported final-test split because the original split manifest and exact deep-EDA source were not preserved in the branch.\n\n" + f"- Markets: {len(ledger):,}\n" + f"- Signals: {int(ledger['signal'].sum()):,}\n" + f"- Hypothetical five-share FOK fills: {int(ledger['hypothetical_fok_fill'].sum()):,}\n" + f"- All-condition-set SHA-256: `{audit['all_condition_ids_sha256']}`\n" + f"- Filled-condition-set SHA-256: `{audit['filled_condition_ids_sha256']}`\n\n" + "## Evidence boundary\n\n" + "PnL in this artifact uses the dataset's inferred outcomes only. It is diagnostic and cannot satisfy official-label transfer or prospective admission. No credentials, authenticated request, or order-submission implementation exists in this checkpoint.\n" + ) + report_path = out_dir / "REPORT.md" + report_path.write_text(report) - outputs = [audit_path, eligible_path, ledger_path, out_dir / "REPORT.md"] + outputs = [audit_path, eligible_path, ledger_path, report_path] sums = "".join(f"{_sha256_bytes(path.read_bytes())} {path.name}\n" for path in sorted(outputs)) (out_dir / "SHA256SUMS").write_text(sums) return audit @@ -303,7 +320,8 @@ def main() -> None: parser.add_argument("--out-dir", type=Path, required=True) parser.add_argument("--cache-dir", type=Path, required=True) args = parser.parse_args() - print(json.dumps(run_audit(args.candidate_spec, args.out_dir, args.cache_dir), indent=2, sort_keys=True, default=list)) + audit = run_audit(args.candidate_spec, args.out_dir, args.cache_dir) + print(json.dumps(audit, indent=2, sort_keys=True, default=list)) if __name__ == "__main__": From e68f37681fee4a9b2d2a9caa9c0b8c90e5b02bfb Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:30:15 -0500 Subject: [PATCH 078/177] ci: run late-favorite reproducibility audit --- .../workflows/late-favorite-repro-audit.yml | 140 ++++++++++++++---- 1 file changed, 109 insertions(+), 31 deletions(-) diff --git a/.github/workflows/late-favorite-repro-audit.yml b/.github/workflows/late-favorite-repro-audit.yml index 8525e185..1dca7b99 100644 --- a/.github/workflows/late-favorite-repro-audit.yml +++ b/.github/workflows/late-favorite-repro-audit.yml @@ -5,10 +5,14 @@ on: branches: [digital-surface-live-readiness-20260725] paths: - .github/workflows/late-favorite-repro-audit.yml + - ci_payload/digital-surface-live-readiness/late-favorite-repro-source/** + - ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/CANDIDATE_SPEC.json pull_request: branches: [digital-surface-ci-20260724] paths: - .github/workflows/late-favorite-repro-audit.yml + - ci_payload/digital-surface-live-readiness/late-favorite-repro-source/** + - ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/CANDIDATE_SPEC.json workflow_dispatch: permissions: @@ -19,42 +23,116 @@ env: PYTHONUNBUFFERED: '1' jobs: - red-contract: + reproduce-frozen-candidate: runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 90 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: '3.11' - - name: Install test dependency - run: python -m pip install pytest==8.3.5 - - name: Write frozen-policy contract test - shell: bash + + - name: Install pinned analysis stack + run: | + python -m pip install --upgrade pip + python -m pip install \ + numpy==2.2.6 \ + pandas==2.2.3 \ + pyarrow==19.0.1 \ + huggingface_hub==0.33.4 \ + hf_xet==1.1.5 \ + pytest==8.3.5 + + - name: Compile source + run: | + python -m py_compile \ + ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py \ + ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_late_favorite_transfer.py + + - name: Pass frozen-policy behavior tests + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-repro-source + run: | + pytest -q ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_late_favorite_transfer.py + + - name: Verify live submission remains physically absent + run: | + python - <<'PY' + import ast + from pathlib import Path + + path = Path('ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py') + tree = ast.parse(path.read_text()) + imports = { + node.names[0].name + for node in ast.walk(tree) + if isinstance(node, ast.Import) and node.names + } + imported_from = { + node.module + for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) + } + forbidden = {'web3', 'py_clob_client', 'eth_account'} + assert not (forbidden & imports) + assert not (forbidden & imported_from) + text = path.read_text().lower() + for token in ('create_order', 'post_order', 'submit_order', 'private_key', 'api_secret'): + assert token not in text, token + PY + + - name: Build complete BTC ETH reproducibility ledger + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-repro-source + HF_XET_HIGH_PERFORMANCE: '1' run: | - mkdir -p /tmp/late-favorite-tests - cat > /tmp/late-favorite-tests/test_contract.py <<'PY' - import importlib - import pytest - - def test_frozen_policy_contract_exists(): - try: - module = importlib.import_module('late_favorite_transfer') - except ModuleNotFoundError: - pytest.fail('late_favorite_transfer is not implemented') - assert module.frozen_policy() == { - 'assets': ('btc', 'eth'), - 'entry_second': 210, - 'signal_ask_min': 0.85, - 'latency_seconds': 1, - 'fok_limit': 'signal_ask', - 'shares': 5, - 'cancel_below_signal_by_more_than': 0.01, - 'hold': 'settlement', - 'fee_rate': 0.07, - 'live_submission': 'disabled', - } + python ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py \ + --candidate-spec ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/CANDIDATE_SPEC.json \ + --out-dir late-favorite-repro-results \ + --cache-dir .cache/late-favorite-data + + - name: Verify output contract and evidence boundary + run: | + python - <<'PY' + import json + from pathlib import Path + + out = Path('late-favorite-repro-results') + required = {'AUDIT.json', 'REPORT.md', 'SHA256SUMS', 'market_ledger.csv.gz', 'eligible_trades.csv.gz'} + assert required == {path.name for path in out.iterdir() if path.is_file()} + audit = json.loads((out / 'AUDIT.json').read_text()) + assert audit['candidate'] == 'late_favorite_btc_eth_v2_diagnostic' + assert audit['official_outcomes_used'] == 0 + assert audit['future_outcomes_used_for_signal'] == 0 + assert audit['live_submission'] == 'physically_absent' + assert audit['markets'] > 0 + assert audit['signals'] > 0 + assert audit['hypothetical_fok_fills'] > 0 + assert set(audit['by_asset']) == {'btc', 'eth'} + assert len(audit['all_condition_ids_sha256']) == 64 + assert len(audit['filled_condition_ids_sha256']) == 64 + assert audit['split_reproduction'].startswith('not claimed') PY - - name: Verify RED contract - working-directory: /tmp/late-favorite-tests - run: pytest -q + (cd late-favorite-repro-results && sha256sum -c SHA256SUMS) + + - name: Build deterministic checkpoint archive + run: | + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf late-favorite-repro-audit-v1.tar.gz late-favorite-repro-results + sha256sum late-favorite-repro-audit-v1.tar.gz > late-favorite-repro-audit-v1.tar.gz.sha256 + cat late-favorite-repro-results/REPORT.md + cat late-favorite-repro-results/AUDIT.json + + - uses: actions/upload-artifact@v4 + with: + name: late-favorite-repro-audit-v1 + path: | + late-favorite-repro-results/AUDIT.json + late-favorite-repro-results/REPORT.md + late-favorite-repro-results/SHA256SUMS + late-favorite-repro-results/market_ledger.csv.gz + late-favorite-repro-results/eligible_trades.csv.gz + late-favorite-repro-audit-v1.tar.gz + late-favorite-repro-audit-v1.tar.gz.sha256 + compression-level: 0 + retention-days: 30 From 61f5f27c84a05fa72f03a2d9a8cdd214922bdc13 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:37:07 -0500 Subject: [PATCH 079/177] test: require explicit unlabeled-fill accounting --- .../test_late_favorite_transfer.py | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_late_favorite_transfer.py b/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_late_favorite_transfer.py index 9f2391a9..9b333e32 100644 --- a/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_late_favorite_transfer.py +++ b/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_late_favorite_transfer.py @@ -2,9 +2,15 @@ import math +import pandas as pd import pytest -from late_favorite_transfer import evaluate_market, fee_per_share, frozen_policy +from late_favorite_transfer import ( + evaluate_market, + fee_per_share, + frozen_policy, + summarize_asset, +) def signal(up: float, down: float) -> dict[str, float]: @@ -79,3 +85,26 @@ def test_loss_pnl_uses_executable_ask_and_fee() -> None: assert result["won"] is False assert result["pnl_per_share"] == pytest.approx(-0.90 - fee_per_share(0.90)) assert math.isclose(result["pnl_at_five_shares"], result["pnl_per_share"] * 5) + + +def test_asset_summary_exposes_missing_labels_in_denominators() -> None: + frame = pd.DataFrame( + { + "condition_id": ["a", "b", "c"], + "market_start": pd.to_datetime( + ["2026-05-01T00:00:00Z", "2026-05-01T00:05:00Z", "2026-05-01T00:10:00Z"] + ), + "signal": [True, True, True], + "hypothetical_fok_fill": [True, True, True], + "won": [True, False, None], + "pnl_at_five_shares": [0.4, -4.5, None], + "pnl_per_share": [0.08, -0.9, None], + } + ) + summary = summarize_asset(frame) + assert summary["hypothetical_fok_fills"] == 3 + assert summary["labeled_fills"] == 2 + assert summary["unlabeled_fills"] == 1 + assert summary["inferred_label_wins"] == 1 + assert summary["inferred_label_win_rate"] == pytest.approx(0.5) + assert summary["inferred_label_pnl_at_five_shares"] == pytest.approx(-4.1) From 64784ad521237fe1a4e079e0d577d7010f3c7214 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:38:48 -0500 Subject: [PATCH 080/177] fix: add explicit unlabeled-fill accounting patch --- .../unlabeled-accounting.patch | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-repro-source/unlabeled-accounting.patch diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/unlabeled-accounting.patch b/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/unlabeled-accounting.patch new file mode 100644 index 00000000..78cd849b --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/unlabeled-accounting.patch @@ -0,0 +1,76 @@ +--- a/late_favorite_transfer.py ++++ b/late_favorite_transfer.py +@@ -213,6 +213,31 @@ def build_market_ledger(asset: str, markets_path: Path, ticks_path: Path) -> pd. + return pd.DataFrame(rows) + + ++def summarize_asset(frame: pd.DataFrame) -> dict[str, Any]: ++ fills = frame.loc[frame["hypothetical_fok_fill"]].copy() ++ labeled = fills.loc[fills["won"].notna()].copy() ++ win_rate = float(labeled["won"].astype(bool).mean()) if len(labeled) else None ++ mean_pnl = float(labeled["pnl_per_share"].mean()) if len(labeled) else None ++ return { ++ "markets": int(len(frame)), ++ "signals": int(frame["signal"].sum()), ++ "hypothetical_fok_fills": int(len(fills)), ++ "labeled_fills": int(len(labeled)), ++ "unlabeled_fills": int(len(fills) - len(labeled)), ++ "inferred_label_wins": int(labeled["won"].astype(bool).sum()), ++ "inferred_label_win_rate": win_rate, ++ "inferred_label_mean_pnl_per_share": mean_pnl, ++ "inferred_label_pnl_at_five_shares": float(labeled["pnl_at_five_shares"].sum()), ++ "first_market_start": str(frame["market_start"].min()), ++ "last_market_start": str(frame["market_start"].max()), ++ "all_condition_ids_sha256": _canonical_id_hash(frame["condition_id"]), ++ "filled_condition_ids_sha256": _canonical_id_hash(fills["condition_id"]), ++ "labeled_filled_condition_ids_sha256": _canonical_id_hash(labeled["condition_id"]), ++ } ++ ++ + def _download_asset(asset: str, cache_dir: Path) -> tuple[Path, Path]: + markets = hf_hub_download( + repo_id=DATASET_REPO, +@@ -249,20 +274,9 @@ def run_audit(candidate_spec: Path, out_dir: Path, cache_dir: Path) -> dict[str, + ledger.to_csv(ledger_path, index=False, compression=compression) + eligible.to_csv(eligible_path, index=False, compression=compression) + +- by_asset: dict[str, Any] = {} +- for asset, frame in ledger.groupby("asset", sort=True): +- fills = frame.loc[frame["hypothetical_fok_fill"]] +- by_asset[asset] = { +- "markets": int(len(frame)), +- "signals": int(frame["signal"].sum()), +- "hypothetical_fok_fills": int(frame["hypothetical_fok_fill"].sum()), +- "inferred_label_wins": int(fills["won"].fillna(False).sum()), +- "inferred_label_pnl_at_five_shares": float(fills["pnl_at_five_shares"].sum()), +- "first_market_start": str(frame["market_start"].min()), +- "last_market_start": str(frame["market_start"].max()), +- "all_condition_ids_sha256": _canonical_id_hash(frame["condition_id"]), +- "filled_condition_ids_sha256": _canonical_id_hash(fills["condition_id"]), +- } ++ by_asset = {asset: summarize_asset(frame) for asset, frame in ledger.groupby("asset", sort=True)} ++ overall = summarize_asset(ledger) + + audit: dict[str, Any] = { + "candidate": "late_favorite_btc_eth_v2_diagnostic", +@@ -273,6 +287,12 @@ def run_audit(candidate_spec: Path, out_dir: Path, cache_dir: Path) -> dict[str, + "markets": int(len(ledger)), + "signals": int(ledger["signal"].sum()), + "hypothetical_fok_fills": int(ledger["hypothetical_fok_fill"].sum()), ++ "labeled_fills": overall["labeled_fills"], ++ "unlabeled_fills": overall["unlabeled_fills"], ++ "inferred_label_wins": overall["inferred_label_wins"], ++ "inferred_label_win_rate": overall["inferred_label_win_rate"], ++ "inferred_label_mean_pnl_per_share": overall["inferred_label_mean_pnl_per_share"], ++ "inferred_label_pnl_at_five_shares": overall["inferred_label_pnl_at_five_shares"], + "all_condition_ids_sha256": _canonical_id_hash(ledger["condition_id"]), + "filled_condition_ids_sha256": _canonical_id_hash(eligible["condition_id"]), + "decision_counts": { +@@ -295,6 +315,8 @@ def run_audit(candidate_spec: Path, out_dir: Path, cache_dir: Path) -> dict[str, + f"- Markets: {len(ledger):,}\n" + f"- Signals: {int(ledger['signal'].sum()):,}\n" + f"- Hypothetical five-share FOK fills: {int(ledger['hypothetical_fok_fill'].sum()):,}\n" ++ f"- Labeled / unlabeled fills: {overall['labeled_fills']:,} / {overall['unlabeled_fills']:,}\n" ++ f"- Inferred-label win rate: {overall['inferred_label_win_rate']:.4%}\n" + f"- All-condition-set SHA-256: `{audit['all_condition_ids_sha256']}`\n" + f"- Filled-condition-set SHA-256: `{audit['filled_condition_ids_sha256']}`\n\n" From 1191568ef0ccb0435cde455ab71f15d2e34307f9 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:39:30 -0500 Subject: [PATCH 081/177] fix: account for unlabeled late-favorite fills --- .../workflows/late-favorite-repro-audit.yml | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/late-favorite-repro-audit.yml b/.github/workflows/late-favorite-repro-audit.yml index 1dca7b99..93cb9bcb 100644 --- a/.github/workflows/late-favorite-repro-audit.yml +++ b/.github/workflows/late-favorite-repro-audit.yml @@ -43,6 +43,10 @@ jobs: hf_xet==1.1.5 \ pytest==8.3.5 + - name: Apply checksum-visible accounting patch + working-directory: ci_payload/digital-surface-live-readiness/late-favorite-repro-source + run: patch -p1 < unlabeled-accounting.patch + - name: Compile source run: | python -m py_compile \ @@ -108,7 +112,12 @@ jobs: assert audit['markets'] > 0 assert audit['signals'] > 0 assert audit['hypothetical_fok_fills'] > 0 + assert audit['labeled_fills'] + audit['unlabeled_fills'] == audit['hypothetical_fok_fills'] + assert 0.0 <= audit['inferred_label_win_rate'] <= 1.0 assert set(audit['by_asset']) == {'btc', 'eth'} + for asset in ('btc', 'eth'): + row = audit['by_asset'][asset] + assert row['labeled_fills'] + row['unlabeled_fills'] == row['hypothetical_fok_fills'] assert len(audit['all_condition_ids_sha256']) == 64 assert len(audit['filled_condition_ids_sha256']) == 64 assert audit['split_reproduction'].startswith('not claimed') @@ -118,21 +127,21 @@ jobs: - name: Build deterministic checkpoint archive run: | tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ - -czf late-favorite-repro-audit-v1.tar.gz late-favorite-repro-results - sha256sum late-favorite-repro-audit-v1.tar.gz > late-favorite-repro-audit-v1.tar.gz.sha256 + -czf late-favorite-repro-audit-v2.tar.gz late-favorite-repro-results + sha256sum late-favorite-repro-audit-v2.tar.gz > late-favorite-repro-audit-v2.tar.gz.sha256 cat late-favorite-repro-results/REPORT.md cat late-favorite-repro-results/AUDIT.json - uses: actions/upload-artifact@v4 with: - name: late-favorite-repro-audit-v1 + name: late-favorite-repro-audit-v2 path: | late-favorite-repro-results/AUDIT.json late-favorite-repro-results/REPORT.md late-favorite-repro-results/SHA256SUMS late-favorite-repro-results/market_ledger.csv.gz late-favorite-repro-results/eligible_trades.csv.gz - late-favorite-repro-audit-v1.tar.gz - late-favorite-repro-audit-v1.tar.gz.sha256 + late-favorite-repro-audit-v2.tar.gz + late-favorite-repro-audit-v2.tar.gz.sha256 compression-level: 0 retention-days: 30 From b970a5b604779c0b4f15022b37cf28f7144c3f93 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:44:33 -0500 Subject: [PATCH 082/177] fix: integrate explicit unlabeled-fill accounting --- .../late_favorite_transfer.py | 48 +++++++++++++------ 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py b/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py index e37ce4a7..c7be7c86 100644 --- a/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py +++ b/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py @@ -216,6 +216,29 @@ def build_market_ledger(asset: str, markets_path: Path, ticks_path: Path) -> pd. return pd.DataFrame(rows) +def summarize_asset(frame: pd.DataFrame) -> dict[str, Any]: + fills = frame.loc[frame["hypothetical_fok_fill"]].copy() + labeled = fills.loc[fills["won"].notna()].copy() + win_rate = float(labeled["won"].astype(bool).mean()) if len(labeled) else None + mean_pnl = float(labeled["pnl_per_share"].mean()) if len(labeled) else None + return { + "markets": int(len(frame)), + "signals": int(frame["signal"].sum()), + "hypothetical_fok_fills": int(len(fills)), + "labeled_fills": int(len(labeled)), + "unlabeled_fills": int(len(fills) - len(labeled)), + "inferred_label_wins": int(labeled["won"].astype(bool).sum()), + "inferred_label_win_rate": win_rate, + "inferred_label_mean_pnl_per_share": mean_pnl, + "inferred_label_pnl_at_five_shares": float(labeled["pnl_at_five_shares"].sum()), + "first_market_start": str(frame["market_start"].min()), + "last_market_start": str(frame["market_start"].max()), + "all_condition_ids_sha256": _canonical_id_hash(frame["condition_id"]), + "filled_condition_ids_sha256": _canonical_id_hash(fills["condition_id"]), + "labeled_filled_condition_ids_sha256": _canonical_id_hash(labeled["condition_id"]), + } + + def _download_asset(asset: str, cache_dir: Path) -> tuple[Path, Path]: markets = hf_hub_download( repo_id=DATASET_REPO, @@ -252,21 +275,8 @@ def run_audit(candidate_spec: Path, out_dir: Path, cache_dir: Path) -> dict[str, ledger.to_csv(ledger_path, index=False, compression=compression) eligible.to_csv(eligible_path, index=False, compression=compression) - by_asset: dict[str, Any] = {} - for asset, frame in ledger.groupby("asset", sort=True): - fills = frame.loc[frame["hypothetical_fok_fill"]] - by_asset[asset] = { - "markets": int(len(frame)), - "signals": int(frame["signal"].sum()), - "hypothetical_fok_fills": int(frame["hypothetical_fok_fill"].sum()), - "inferred_label_wins": int(fills["won"].fillna(False).sum()), - "inferred_label_pnl_at_five_shares": float(fills["pnl_at_five_shares"].sum()), - "first_market_start": str(frame["market_start"].min()), - "last_market_start": str(frame["market_start"].max()), - "all_condition_ids_sha256": _canonical_id_hash(frame["condition_id"]), - "filled_condition_ids_sha256": _canonical_id_hash(fills["condition_id"]), - } - + by_asset = {asset: summarize_asset(frame) for asset, frame in ledger.groupby("asset", sort=True)} + overall = summarize_asset(ledger) audit: dict[str, Any] = { "candidate": "late_favorite_btc_eth_v2_diagnostic", "candidate_spec_sha256": policy_sha256, @@ -277,6 +287,12 @@ def run_audit(candidate_spec: Path, out_dir: Path, cache_dir: Path) -> dict[str, "markets": int(len(ledger)), "signals": int(ledger["signal"].sum()), "hypothetical_fok_fills": int(ledger["hypothetical_fok_fill"].sum()), + "labeled_fills": overall["labeled_fills"], + "unlabeled_fills": overall["unlabeled_fills"], + "inferred_label_wins": overall["inferred_label_wins"], + "inferred_label_win_rate": overall["inferred_label_win_rate"], + "inferred_label_mean_pnl_per_share": overall["inferred_label_mean_pnl_per_share"], + "inferred_label_pnl_at_five_shares": overall["inferred_label_pnl_at_five_shares"], "all_condition_ids_sha256": _canonical_id_hash(ledger["condition_id"]), "filled_condition_ids_sha256": _canonical_id_hash(eligible["condition_id"]), "decision_counts": { @@ -300,6 +316,8 @@ def run_audit(candidate_spec: Path, out_dir: Path, cache_dir: Path) -> dict[str, f"- Markets: {len(ledger):,}\n" f"- Signals: {int(ledger['signal'].sum()):,}\n" f"- Hypothetical five-share FOK fills: {int(ledger['hypothetical_fok_fill'].sum()):,}\n" + f"- Labeled / unlabeled fills: {overall['labeled_fills']:,} / {overall['unlabeled_fills']:,}\n" + f"- Inferred-label win rate: {overall['inferred_label_win_rate']:.4%}\n" f"- All-condition-set SHA-256: `{audit['all_condition_ids_sha256']}`\n" f"- Filled-condition-set SHA-256: `{audit['filled_condition_ids_sha256']}`\n\n" "## Evidence boundary\n\n" From 6230f9060f463b1c687be8a10201c416fd7eb178 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:45:08 -0500 Subject: [PATCH 083/177] ci: run integrated unlabeled-fill accounting audit --- .github/workflows/late-favorite-repro-audit.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/late-favorite-repro-audit.yml b/.github/workflows/late-favorite-repro-audit.yml index 93cb9bcb..5f48fe99 100644 --- a/.github/workflows/late-favorite-repro-audit.yml +++ b/.github/workflows/late-favorite-repro-audit.yml @@ -43,10 +43,6 @@ jobs: hf_xet==1.1.5 \ pytest==8.3.5 - - name: Apply checksum-visible accounting patch - working-directory: ci_payload/digital-surface-live-readiness/late-favorite-repro-source - run: patch -p1 < unlabeled-accounting.patch - - name: Compile source run: | python -m py_compile \ From 489374244259d366efd8d81c8668e6861913088d Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:46:50 -0500 Subject: [PATCH 084/177] chore: remove superseded accounting patch --- .../unlabeled-accounting.patch | 76 ------------------- 1 file changed, 76 deletions(-) delete mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-repro-source/unlabeled-accounting.patch diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/unlabeled-accounting.patch b/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/unlabeled-accounting.patch deleted file mode 100644 index 78cd849b..00000000 --- a/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/unlabeled-accounting.patch +++ /dev/null @@ -1,76 +0,0 @@ ---- a/late_favorite_transfer.py -+++ b/late_favorite_transfer.py -@@ -213,6 +213,31 @@ def build_market_ledger(asset: str, markets_path: Path, ticks_path: Path) -> pd. - return pd.DataFrame(rows) - - -+def summarize_asset(frame: pd.DataFrame) -> dict[str, Any]: -+ fills = frame.loc[frame["hypothetical_fok_fill"]].copy() -+ labeled = fills.loc[fills["won"].notna()].copy() -+ win_rate = float(labeled["won"].astype(bool).mean()) if len(labeled) else None -+ mean_pnl = float(labeled["pnl_per_share"].mean()) if len(labeled) else None -+ return { -+ "markets": int(len(frame)), -+ "signals": int(frame["signal"].sum()), -+ "hypothetical_fok_fills": int(len(fills)), -+ "labeled_fills": int(len(labeled)), -+ "unlabeled_fills": int(len(fills) - len(labeled)), -+ "inferred_label_wins": int(labeled["won"].astype(bool).sum()), -+ "inferred_label_win_rate": win_rate, -+ "inferred_label_mean_pnl_per_share": mean_pnl, -+ "inferred_label_pnl_at_five_shares": float(labeled["pnl_at_five_shares"].sum()), -+ "first_market_start": str(frame["market_start"].min()), -+ "last_market_start": str(frame["market_start"].max()), -+ "all_condition_ids_sha256": _canonical_id_hash(frame["condition_id"]), -+ "filled_condition_ids_sha256": _canonical_id_hash(fills["condition_id"]), -+ "labeled_filled_condition_ids_sha256": _canonical_id_hash(labeled["condition_id"]), -+ } -+ -+ - def _download_asset(asset: str, cache_dir: Path) -> tuple[Path, Path]: - markets = hf_hub_download( - repo_id=DATASET_REPO, -@@ -249,20 +274,9 @@ def run_audit(candidate_spec: Path, out_dir: Path, cache_dir: Path) -> dict[str, - ledger.to_csv(ledger_path, index=False, compression=compression) - eligible.to_csv(eligible_path, index=False, compression=compression) - -- by_asset: dict[str, Any] = {} -- for asset, frame in ledger.groupby("asset", sort=True): -- fills = frame.loc[frame["hypothetical_fok_fill"]] -- by_asset[asset] = { -- "markets": int(len(frame)), -- "signals": int(frame["signal"].sum()), -- "hypothetical_fok_fills": int(frame["hypothetical_fok_fill"].sum()), -- "inferred_label_wins": int(fills["won"].fillna(False).sum()), -- "inferred_label_pnl_at_five_shares": float(fills["pnl_at_five_shares"].sum()), -- "first_market_start": str(frame["market_start"].min()), -- "last_market_start": str(frame["market_start"].max()), -- "all_condition_ids_sha256": _canonical_id_hash(frame["condition_id"]), -- "filled_condition_ids_sha256": _canonical_id_hash(fills["condition_id"]), -- } -+ by_asset = {asset: summarize_asset(frame) for asset, frame in ledger.groupby("asset", sort=True)} -+ overall = summarize_asset(ledger) - - audit: dict[str, Any] = { - "candidate": "late_favorite_btc_eth_v2_diagnostic", -@@ -273,6 +287,12 @@ def run_audit(candidate_spec: Path, out_dir: Path, cache_dir: Path) -> dict[str, - "markets": int(len(ledger)), - "signals": int(ledger["signal"].sum()), - "hypothetical_fok_fills": int(ledger["hypothetical_fok_fill"].sum()), -+ "labeled_fills": overall["labeled_fills"], -+ "unlabeled_fills": overall["unlabeled_fills"], -+ "inferred_label_wins": overall["inferred_label_wins"], -+ "inferred_label_win_rate": overall["inferred_label_win_rate"], -+ "inferred_label_mean_pnl_per_share": overall["inferred_label_mean_pnl_per_share"], -+ "inferred_label_pnl_at_five_shares": overall["inferred_label_pnl_at_five_shares"], - "all_condition_ids_sha256": _canonical_id_hash(ledger["condition_id"]), - "filled_condition_ids_sha256": _canonical_id_hash(eligible["condition_id"]), - "decision_counts": { -@@ -295,6 +315,8 @@ def run_audit(candidate_spec: Path, out_dir: Path, cache_dir: Path) -> dict[str, - f"- Markets: {len(ledger):,}\n" - f"- Signals: {int(ledger['signal'].sum()):,}\n" - f"- Hypothetical five-share FOK fills: {int(ledger['hypothetical_fok_fill'].sum()):,}\n" -+ f"- Labeled / unlabeled fills: {overall['labeled_fills']:,} / {overall['unlabeled_fills']:,}\n" -+ f"- Inferred-label win rate: {overall['inferred_label_win_rate']:.4%}\n" - f"- All-condition-set SHA-256: `{audit['all_condition_ids_sha256']}`\n" - f"- Filled-condition-set SHA-256: `{audit['filled_condition_ids_sha256']}`\n\n" From 1f4eecb97ed7c9d2fb24367a9b239fcde9eb470e Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:47:16 -0500 Subject: [PATCH 085/177] research: checkpoint late-favorite reproducibility audit v2 --- .../late-favorite-repro-v2/AUDIT.json | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-repro-v2/AUDIT.json diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-repro-v2/AUDIT.json b/ci_payload/digital-surface-live-readiness/late-favorite-repro-v2/AUDIT.json new file mode 100644 index 00000000..ebdb4948 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-repro-v2/AUDIT.json @@ -0,0 +1,68 @@ +{ + "all_condition_ids_sha256": "ed051d81e9d5d4aa57cd598205b34cdd0af804f5fa0ddec45aeca4bb85b919a1", + "artifact_archive_sha256": "0cc4a640d411d3b85130c858def8a84034cf7c293d7c25af6b8a9098382c8afc", + "by_asset": { + "btc": { + "all_condition_ids_sha256": "153b73d0fbf938b562018c9d3d3af7aac1c2a182e2957e0887086c570f8cc50e", + "filled_condition_ids_sha256": "64ba2f09192a5dca87b18dd466b7d7d6b8e48c6f7629272bddeb10a5090a8f88", + "first_market_start": "2026-03-24 22:10:00+00:00", + "hypothetical_fok_fills": 4844, + "inferred_label_mean_pnl_per_share": 0.014374885612004295, + "inferred_label_pnl_at_five_shares": 335.2942069000002, + "inferred_label_win_rate": 0.9596998928188639, + "inferred_label_wins": 4477, + "labeled_filled_condition_ids_sha256": "d070649c7f7cc1614b76301dec4750a379288d709fdb7d6b26f663815bfd3618", + "labeled_fills": 4665, + "last_market_start": "2026-05-18 10:30:00+00:00", + "markets": 15682, + "signals": 6847, + "unlabeled_fills": 179 + }, + "eth": { + "all_condition_ids_sha256": "d051614a07d368c17ae6792eadc4696a52db42cd90f47422a58ca888277ffc75", + "filled_condition_ids_sha256": "8fce860d459b12357547283181e7bb5b570cc4e92cffb403d6b2d9ceca9df5b2", + "first_market_start": "2026-04-05 19:25:00+00:00", + "hypothetical_fok_fills": 4424, + "inferred_label_mean_pnl_per_share": 0.004439617785680423, + "inferred_label_pnl_at_five_shares": 93.63153910000014, + "inferred_label_win_rate": 0.9516358463726885, + "inferred_label_wins": 4014, + "labeled_filled_condition_ids_sha256": "4cb9935e7816ed65d8e5412415ecf522243ed675532aa1ee8d0e8a211029fcef", + "labeled_fills": 4218, + "last_market_start": "2026-05-18 10:30:00+00:00", + "markets": 12258, + "signals": 5848, + "unlabeled_fills": 206 + } + }, + "candidate": "late_favorite_btc_eth_v2_diagnostic", + "candidate_spec_sha256": "e8f09194678ff203b9906b0462c411e327c1c1e1a57fc9e7ce1a1c77c28f7cd3", + "dataset_repo": "kachoio/polymarket-5-minute-crypto-up-down-markets", + "decision_counts": { + "cancel_adverse_move": 527, + "hypothetical_fok_fill": 9268, + "no_fill_ask_above_frozen_limit": 2591, + "no_fill_insufficient_displayed_depth": 238, + "no_signal_below_threshold": 13721, + "no_signal_invalid_signal_book": 1488, + "no_signal_tied_favorite": 36, + "signal_invalid_arrival_ask": 71 + }, + "filled_condition_ids_sha256": "bf276fbce187d9651d56ecae976520d4251140c1d5b0d440696c5d077838a67a", + "future_outcomes_used_for_signal": 0, + "hypothetical_fok_fills": 9268, + "inferred_label_mean_pnl_per_share": 0.00965722719801869, + "inferred_label_pnl_at_five_shares": 428.92574600000034, + "inferred_label_win_rate": 0.9558707643814027, + "inferred_label_wins": 8491, + "labeled_fills": 8883, + "live_submission": "physically_absent", + "markets": 27940, + "official_outcomes_used": 0, + "outcomes": "dataset-inferred; not admission evidence", + "source_commit": "6230f9060f463b1c687be8a10201c416fd7eb178", + "source_module_sha256": "9fee982c6934614ecef52679b49f81826f34f41e0a04fc7e251856ee79b47237", + "split_reproduction": "not claimed: original deep-EDA split manifest/source is absent", + "unlabeled_fills": 385, + "workflow_run_id": 30213239532 +} From 7f2986cdcc8e86557cd5f9517483344e8ff30fd8 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:48:00 -0500 Subject: [PATCH 086/177] research: document late-favorite repro checkpoint v2 --- .../late-favorite-repro-v2/CHECKPOINT.md | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-repro-v2/CHECKPOINT.md diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-repro-v2/CHECKPOINT.md b/ci_payload/digital-surface-live-readiness/late-favorite-repro-v2/CHECKPOINT.md new file mode 100644 index 00000000..156218be --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-repro-v2/CHECKPOINT.md @@ -0,0 +1,75 @@ +# Late-Favorite BTC/ETH Reproducibility Checkpoint v2 + +## Verdict + +The frozen late-favorite rule now has a clean-room, hash-bound full-corpus reproducer and a complete per-market ledger. This is a successful **reproducibility checkpoint**, not strategy admission. + +The exact previously reported final-test result of 2,054 BTC/ETH trades and +$139.72 is **not yet independently reproducible** because the branch does not contain the original deep-EDA source or its exact train/validation/final-test split manifest. No replacement split was tuned to force a match. + +## Clean-room verification + +- Workflow: `Late Favorite Repro Audit` +- Run: `30213239532` +- Source commit: `6230f9060f463b1c687be8a10201c416fd7eb178` +- Result: green +- Behavior tests: 11 passed +- Source compilation: passed +- Live-submission absence check: passed +- Full BTC/ETH ledger build: passed +- Evidence-boundary and checksum verification: passed +- GitHub artifact ZIP digest: `sha256:0cc4a640d411d3b85130c858def8a84034cf7c293d7c25af6b8a9098382c8afc` +- Deterministic tar digest: `703365cd593e8cf38f08a5af4ece1a947c2393ece68a30583b5ba357a3110cdb` + +## Bound market universe + +- Markets: 27,940 +- Signals at second 210: 12,695 +- Hypothetical five-share FOK fills after the frozen one-second latency, limit, depth, and adverse-move rules: 9,268 +- Labeled fills: 8,883 +- Unlabeled fills: 385 +- Inferred-label win rate among labeled fills: 95.5871% +- Diagnostic inferred-label P&L at five shares: +$428.925746 +- Mean diagnostic P&L: +0.9657 cents per labeled share + +Asset detail: + +| Asset | Markets | Signals | Fills | Labeled | Unlabeled | Inferred win rate | Diagnostic P&L at 5 shares | +|---|---:|---:|---:|---:|---:|---:|---:| +| BTC | 15,682 | 6,847 | 4,844 | 4,665 | 179 | 95.9700% | +$335.294207 | +| ETH | 12,258 | 5,848 | 4,424 | 4,218 | 206 | 95.1636% | +$93.631539 | + +## Cryptographic identities + +- Candidate spec SHA-256: `e8f09194678ff203b9906b0462c411e327c1c1e1a57fc9e7ce1a1c77c28f7cd3` +- Source module SHA-256: `9fee982c6934614ecef52679b49f81826f34f41e0a04fc7e251856ee79b47237` +- All-condition-set SHA-256: `ed051d81e9d5d4aa57cd598205b34cdd0af804f5fa0ddec45aeca4bb85b919a1` +- Filled-condition-set SHA-256: `bf276fbce187d9651d56ecae976520d4251140c1d5b0d440696c5d077838a67a` +- BTC filled set: `64ba2f09192a5dca87b18dd466b7d7d6b8e48c6f7629272bddeb10a5090a8f88` +- ETH filled set: `8fce860d459b12357547283181e7bb5b570cc4e92cffb403d6b2d9ceca9df5b2` + +Output file checksums: + +- `AUDIT.json`: `eb05935840395e34601d95954a20a167f16b431cde83beeb463ceba10ebb0c7c` +- `REPORT.md`: `290bf383f91ed01debfe3fe0a910dadcbaef433ff68e17d3753be971344ea9e5` +- `eligible_trades.csv.gz`: `3438e739c24bc4523e36b439c2fad0804d7018f11082787ce490b3abc8b5915a` +- `market_ledger.csv.gz`: `2e46aa667ddae8a0c5ea24d3329b4b0bba084fcb90dab760dca349e4ca154c62` + +## Data-quality finding + +The first green artifact exposed 385 filled rows with no inferred terminal label. The v2 audit now reports labeled and unlabeled denominators separately and tests that they reconcile exactly to total fills. Diagnostic P&L and win rate use only labeled fills. + +Those 385 rows are not assumed winners, losers, or unresolved Polymarket settlements. They must be transferred to authoritative terminal outcomes or explicitly classified as unavailable. + +## Reproducibility limitations + +1. Official outcomes used: **0**. +2. The dataset outcomes remain inferred and cannot count toward admission. +3. Exact deep-EDA split boundaries and the original implementation source were not preserved. +4. This v2 reproducer selects the favorite as the side with the higher signal ask. That is consistent with the frozen `signal_ask` description but is not proven byte-for-byte identical to the missing original deep-EDA implementation. +5. The post-inspection adverse-move cancel remains a diagnostic/prospective rule; it does not inherit untouched holdout status. +6. Top-of-book depth still does not establish live queue position or realized FOK execution. +7. Live submission remains physically absent. + +## Next bounded job + +Transfer the hash-bound filled condition set to official Polymarket terminal outcomes, preserve every lookup response and failure classification, and recompute the same fixed-rule ledger without modifying thresholds, assets, timing, fees, latency, depth, or cancel logic. From aafaf21b8fdce261ba4ff8aba052a96cd398095a Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:51:08 -0500 Subject: [PATCH 087/177] test: define official terminal outcome transfer contract --- .../test_official_transfer.py | 158 ++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/test_official_transfer.py diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/test_official_transfer.py b/ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/test_official_transfer.py new file mode 100644 index 00000000..7712e588 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/test_official_transfer.py @@ -0,0 +1,158 @@ +from __future__ import annotations + +import pytest + +from official_transfer import ( + parse_clob_terminal, + parse_gamma_terminal, + reconcile_terminal_sources, +) + + +CONDITION = "0x" + "ab" * 32 + + +def gamma_payload(**overrides): + payload = { + "conditionId": CONDITION, + "closed": True, + "umaResolutionStatus": "resolved", + "outcomes": '["Up", "Down"]', + "outcomePrices": '["1", "0"]', + } + payload.update(overrides) + return payload + + +def clob_payload(**overrides): + payload = { + "condition_id": CONDITION, + "closed": True, + "tokens": [ + {"outcome": "Up", "price": 1, "winner": True}, + {"outcome": "Down", "price": 0, "winner": False}, + ], + } + payload.update(overrides) + return payload + + +def test_gamma_accepts_json_string_arrays_and_one_hot_terminal_prices() -> None: + result = parse_gamma_terminal(gamma_payload(), expected_condition_id=CONDITION) + assert result["classification"] == "terminal" + assert result["official_outcome"] == "Up" + assert result["closed"] is True + assert result["uma_resolution_status"] == "resolved" + + +def test_gamma_accepts_native_arrays() -> None: + result = parse_gamma_terminal( + gamma_payload(outcomes=["Up", "Down"], outcomePrices=[0, 1]), + expected_condition_id=CONDITION, + ) + assert result["classification"] == "terminal" + assert result["official_outcome"] == "Down" + + +def test_gamma_fails_closed_on_condition_id_mismatch() -> None: + result = parse_gamma_terminal(gamma_payload(conditionId="0x" + "cd" * 32), expected_condition_id=CONDITION) + assert result["classification"] == "mismatched_condition_id" + assert result["official_outcome"] is None + + +def test_gamma_fails_closed_when_market_is_not_closed() -> None: + result = parse_gamma_terminal(gamma_payload(closed=False), expected_condition_id=CONDITION) + assert result["classification"] == "not_closed" + + +def test_gamma_rejects_non_one_hot_prices() -> None: + result = parse_gamma_terminal( + gamma_payload(outcomePrices='["0.999", "0.001"]'), + expected_condition_id=CONDITION, + ) + assert result["classification"] == "ambiguous_terminal_prices" + assert result["official_outcome"] is None + + +def test_gamma_rejects_malformed_array_encoding() -> None: + result = parse_gamma_terminal(gamma_payload(outcomes="not-json"), expected_condition_id=CONDITION) + assert result["classification"] == "malformed_gamma_payload" + + +def test_gamma_rejects_duplicate_outcome_labels() -> None: + result = parse_gamma_terminal( + gamma_payload(outcomes='["Up", "Up"]'), + expected_condition_id=CONDITION, + ) + assert result["classification"] == "malformed_gamma_payload" + + +def test_clob_requires_exactly_one_winner() -> None: + result = parse_clob_terminal(clob_payload(), expected_condition_id=CONDITION) + assert result["classification"] == "terminal" + assert result["official_outcome"] == "Up" + + +def test_clob_rejects_multiple_winners() -> None: + result = parse_clob_terminal( + clob_payload( + tokens=[ + {"outcome": "Up", "winner": True}, + {"outcome": "Down", "winner": True}, + ] + ), + expected_condition_id=CONDITION, + ) + assert result["classification"] == "ambiguous_winner_flags" + + +def test_clob_fails_closed_on_condition_id_mismatch() -> None: + result = parse_clob_terminal( + clob_payload(condition_id="0x" + "ef" * 32), + expected_condition_id=CONDITION, + ) + assert result["classification"] == "mismatched_condition_id" + + +def test_reconciliation_confirms_matching_terminal_sources() -> None: + result = reconcile_terminal_sources( + parse_gamma_terminal(gamma_payload(), expected_condition_id=CONDITION), + parse_clob_terminal(clob_payload(), expected_condition_id=CONDITION), + ) + assert result["classification"] == "terminal_confirmed_both" + assert result["official_outcome"] == "Up" + + +def test_reconciliation_marks_source_disagreement() -> None: + result = reconcile_terminal_sources( + parse_gamma_terminal(gamma_payload(), expected_condition_id=CONDITION), + parse_clob_terminal( + clob_payload( + tokens=[ + {"outcome": "Up", "winner": False}, + {"outcome": "Down", "winner": True}, + ] + ), + expected_condition_id=CONDITION, + ), + ) + assert result["classification"] == "source_disagreement" + assert result["official_outcome"] is None + + +def test_reconciliation_allows_gamma_terminal_when_clob_is_unavailable() -> None: + result = reconcile_terminal_sources( + parse_gamma_terminal(gamma_payload(), expected_condition_id=CONDITION), + {"classification": "unavailable", "official_outcome": None}, + ) + assert result["classification"] == "terminal_gamma_only" + assert result["official_outcome"] == "Up" + + +def test_reconciliation_never_promotes_clob_only_to_primary_official_label() -> None: + result = reconcile_terminal_sources( + {"classification": "unavailable", "official_outcome": None}, + parse_clob_terminal(clob_payload(), expected_condition_id=CONDITION), + ) + assert result["classification"] == "clob_only_unconfirmed" + assert result["official_outcome"] is None From c27401380a23b184073af5af7eb2a29e45205e9d Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:51:19 -0500 Subject: [PATCH 088/177] ci: expose red official outcome transfer contract --- .../late-favorite-official-transfer.yml | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/late-favorite-official-transfer.yml diff --git a/.github/workflows/late-favorite-official-transfer.yml b/.github/workflows/late-favorite-official-transfer.yml new file mode 100644 index 00000000..63544249 --- /dev/null +++ b/.github/workflows/late-favorite-official-transfer.yml @@ -0,0 +1,37 @@ +name: Late Favorite Official Outcome Transfer + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/late-favorite-official-transfer.yml + - ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/** + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/late-favorite-official-transfer.yml + - ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/** + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + red-terminal-contract: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install test dependency + run: python -m pip install pytest==8.3.5 + - name: Verify RED terminal parser contract + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source + run: pytest -q ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/test_official_transfer.py From d1e393b620a1174faba24a2292953509b18b70e0 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:55:30 -0500 Subject: [PATCH 089/177] research: implement official terminal outcome transfer --- .../official_transfer.py | 682 ++++++++++++++++++ 1 file changed, 682 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/official_transfer.py diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/official_transfer.py b/ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/official_transfer.py new file mode 100644 index 00000000..ba1588f1 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/official_transfer.py @@ -0,0 +1,682 @@ +from __future__ import annotations + +import argparse +import gzip +import hashlib +import json +import math +import time +from dataclasses import dataclass, field +from datetime import datetime, timezone +from decimal import Decimal, InvalidOperation +from pathlib import Path +from typing import Any, Iterable, Mapping, Sequence + +import pandas as pd +import requests + +GAMMA_MARKETS_URL = "https://gamma-api.polymarket.com/markets" +GAMMA_MARKET_BY_SLUG_URL = "https://gamma-api.polymarket.com/markets/slug/{slug}" +CLOB_SIMPLIFIED_MARKETS_URL = "https://clob.polymarket.com/simplified-markets" +EXPECTED_FILLED_CONDITION_SET_SHA256 = "bf276fbce187d9651d56ecae976520d4251140c1d5b0d440696c5d077838a67a" +TERMINAL_EPSILON = Decimal("1e-12") +USER_AGENT = "late-favorite-official-transfer/1.0 research-only" + + +def _sha256_bytes(raw: bytes) -> str: + return hashlib.sha256(raw).hexdigest() + + +def _canonical_json_bytes(value: Any) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _normalize_condition_id(value: Any) -> str: + return str(value or "").strip().lower() + + +def _condition_set_sha256(values: Iterable[Any]) -> str: + ids = sorted({_normalize_condition_id(value) for value in values if _normalize_condition_id(value)}) + canonical = (("\n".join(ids) + "\n") if ids else "").encode("utf-8") + return _sha256_bytes(canonical) + + +def _decode_array(value: Any) -> list[Any]: + if isinstance(value, str): + value = json.loads(value) + if not isinstance(value, (list, tuple)): + raise ValueError("value is not an array") + return list(value) + + +def _base_parse_result(source: str, classification: str) -> dict[str, Any]: + return { + "source": source, + "classification": classification, + "official_outcome": None, + "condition_id": None, + "closed": None, + "uma_resolution_status": None, + "payload_sha256": None, + } + + +def parse_gamma_terminal(payload: Any, *, expected_condition_id: str) -> dict[str, Any]: + result = _base_parse_result("gamma", "malformed_gamma_payload") + result["payload_sha256"] = _sha256_bytes(_canonical_json_bytes(payload)) + if not isinstance(payload, Mapping): + return result + + actual = _normalize_condition_id(payload.get("conditionId")) + expected = _normalize_condition_id(expected_condition_id) + result.update( + { + "condition_id": actual or None, + "closed": payload.get("closed"), + "uma_resolution_status": payload.get("umaResolutionStatus"), + } + ) + if actual != expected: + result["classification"] = "mismatched_condition_id" + return result + if payload.get("closed") is not True: + result["classification"] = "not_closed" + return result + + try: + outcomes = [str(value).strip().title() for value in _decode_array(payload.get("outcomes"))] + raw_prices = _decode_array(payload.get("outcomePrices")) + prices = [Decimal(str(value)) for value in raw_prices] + except (ValueError, TypeError, json.JSONDecodeError, InvalidOperation): + return result + + if len(outcomes) != len(prices) or len(outcomes) != 2: + return result + if len(set(outcomes)) != len(outcomes) or set(outcomes) != {"Up", "Down"}: + return result + if any((not price.is_finite()) or price < 0 or price > 1 for price in prices): + return result + + winner_indexes = [index for index, price in enumerate(prices) if abs(price - Decimal(1)) <= TERMINAL_EPSILON] + zero_indexes = [index for index, price in enumerate(prices) if abs(price) <= TERMINAL_EPSILON] + if len(winner_indexes) != 1 or len(zero_indexes) != len(prices) - 1: + result["classification"] = "ambiguous_terminal_prices" + return result + + result["classification"] = "terminal" + result["official_outcome"] = outcomes[winner_indexes[0]] + return result + + +def parse_clob_terminal(payload: Any, *, expected_condition_id: str) -> dict[str, Any]: + result = _base_parse_result("clob", "malformed_clob_payload") + result["payload_sha256"] = _sha256_bytes(_canonical_json_bytes(payload)) + if not isinstance(payload, Mapping): + return result + + actual = _normalize_condition_id(payload.get("condition_id")) + expected = _normalize_condition_id(expected_condition_id) + result.update({"condition_id": actual or None, "closed": payload.get("closed")}) + if actual != expected: + result["classification"] = "mismatched_condition_id" + return result + if payload.get("closed") is not True: + result["classification"] = "not_closed" + return result + + tokens = payload.get("tokens") + if not isinstance(tokens, list) or len(tokens) != 2 or not all(isinstance(token, Mapping) for token in tokens): + return result + outcomes = [str(token.get("outcome") or "").strip().title() for token in tokens] + if len(set(outcomes)) != len(outcomes) or set(outcomes) != {"Up", "Down"}: + return result + winner_indexes = [index for index, token in enumerate(tokens) if token.get("winner") is True] + if len(winner_indexes) != 1: + result["classification"] = "ambiguous_winner_flags" + return result + + result["classification"] = "terminal" + result["official_outcome"] = outcomes[winner_indexes[0]] + return result + + +def reconcile_terminal_sources(gamma: Mapping[str, Any], clob: Mapping[str, Any]) -> dict[str, Any]: + gamma_class = str(gamma.get("classification") or "unavailable") + clob_class = str(clob.get("classification") or "unavailable") + gamma_outcome = gamma.get("official_outcome") + clob_outcome = clob.get("official_outcome") + result = { + "classification": "unavailable", + "official_outcome": None, + "gamma_classification": gamma_class, + "clob_classification": clob_class, + "gamma_outcome": gamma_outcome, + "clob_outcome": clob_outcome, + } + if gamma_class == "terminal": + if clob_class == "terminal": + if gamma_outcome == clob_outcome: + result.update( + {"classification": "terminal_confirmed_both", "official_outcome": gamma_outcome} + ) + else: + result["classification"] = "source_disagreement" + else: + result.update({"classification": "terminal_gamma_only", "official_outcome": gamma_outcome}) + return result + if clob_class == "terminal": + result["classification"] = "clob_only_unconfirmed" + return result + result["classification"] = gamma_class if gamma_class != "unavailable" else clob_class + return result + + +@dataclass +class ResponseArchive: + root: Path + records: list[dict[str, Any]] = field(default_factory=list) + sequence: int = 0 + + def __post_init__(self) -> None: + self.root.mkdir(parents=True, exist_ok=True) + + def save(self, *, source: str, kind: str, response: requests.Response) -> dict[str, Any]: + self.sequence += 1 + relative = Path("raw") / source / f"{self.sequence:06d}_{kind}.body" + path = self.root / relative + path.parent.mkdir(parents=True, exist_ok=True) + raw = bytes(response.content) + path.write_bytes(raw) + record = { + "sequence": self.sequence, + "source": source, + "kind": kind, + "url": response.url, + "status_code": int(response.status_code), + "body_path": str(relative), + "body_sha256": _sha256_bytes(raw), + "content_type": response.headers.get("content-type"), + } + self.records.append(record) + return record + + def save_exception(self, *, source: str, kind: str, url: str, error: Exception) -> dict[str, Any]: + self.sequence += 1 + record = { + "sequence": self.sequence, + "source": source, + "kind": kind, + "url": url, + "status_code": None, + "body_path": None, + "body_sha256": None, + "content_type": None, + "error_type": type(error).__name__, + "error": str(error), + } + self.records.append(record) + return record + + def write_manifest(self) -> Path: + path = self.root / "raw_response_manifest.jsonl.gz" + with gzip.open(path, "wt", encoding="utf-8", mtime=0) as handle: + for record in self.records: + handle.write(json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n") + return path + + +def _session() -> requests.Session: + session = requests.Session() + session.headers.update({"User-Agent": USER_AGENT, "Accept": "application/json"}) + return session + + +def _request_with_retry( + session: requests.Session, + url: str, + *, + params: Any = None, + timeout: float = 30.0, + attempts: int = 5, +) -> requests.Response: + last_error: Exception | None = None + for attempt in range(attempts): + try: + response = session.get(url, params=params, timeout=timeout) + if response.status_code not in {429, 500, 502, 503, 504}: + return response + last_error = RuntimeError(f"retryable HTTP status {response.status_code}") + except requests.RequestException as error: + last_error = error + if attempt + 1 < attempts: + time.sleep(min(8.0, 0.5 * (2**attempt))) + assert last_error is not None + raise last_error + + +def _chunks(values: Sequence[str], size: int) -> Iterable[list[str]]: + for start in range(0, len(values), size): + yield list(values[start : start + size]) + + +def _extract_gamma_payloads(value: Any) -> list[dict[str, Any]]: + if isinstance(value, Mapping): + return [dict(value)] + if isinstance(value, list): + return [dict(item) for item in value if isinstance(item, Mapping)] + return [] + + +def fetch_gamma_payloads( + targets: pd.DataFrame, + *, + archive: ResponseArchive, + session: requests.Session, + batch_size: int, +) -> tuple[dict[str, dict[str, Any]], dict[str, dict[str, Any]]]: + target_ids = sorted({_normalize_condition_id(value) for value in targets["condition_id"]}) + slugs = { + _normalize_condition_id(row.condition_id): str(row.slug) + for row in targets[["condition_id", "slug"]].drop_duplicates("condition_id").itertuples(index=False) + } + payloads: dict[str, dict[str, Any]] = {} + failures: dict[str, dict[str, Any]] = {} + + def absorb(response: requests.Response, requested: set[str], kind: str) -> set[str]: + evidence = archive.save(source="gamma", kind=kind, response=response) + if response.status_code != 200: + return set() + try: + decoded = response.json() + except ValueError: + return set() + matched: set[str] = set() + for payload in _extract_gamma_payloads(decoded): + condition_id = _normalize_condition_id(payload.get("conditionId")) + if condition_id not in requested: + continue + if condition_id in payloads: + failures[condition_id] = { + "classification": "duplicate_gamma_records", + "official_outcome": None, + } + continue + payload = dict(payload) + payload["_response_body_sha256"] = evidence["body_sha256"] + payload["_response_url"] = evidence["url"] + payloads[condition_id] = payload + matched.add(condition_id) + return matched + + for batch_number, batch in enumerate(_chunks(target_ids, batch_size), start=1): + requested = set(batch) + repeated_params: list[tuple[str, str | int]] = [("limit", len(batch)), ("offset", 0)] + repeated_params.extend(("condition_ids", condition_id) for condition_id in batch) + try: + response = _request_with_retry(session, GAMMA_MARKETS_URL, params=repeated_params) + matched = absorb(response, requested, f"batch_{batch_number:04d}_repeated") + except Exception as error: + archive.save_exception( + source="gamma", kind=f"batch_{batch_number:04d}_repeated", url=GAMMA_MARKETS_URL, error=error + ) + matched = set() + + missing = requested - matched - set(payloads) + if missing: + comma_params = { + "limit": len(missing), + "offset": 0, + "condition_ids": ",".join(sorted(missing)), + } + try: + response = _request_with_retry(session, GAMMA_MARKETS_URL, params=comma_params) + absorb(response, missing, f"batch_{batch_number:04d}_comma") + except Exception as error: + archive.save_exception( + source="gamma", kind=f"batch_{batch_number:04d}_comma", url=GAMMA_MARKETS_URL, error=error + ) + + remaining = [condition_id for condition_id in target_ids if condition_id not in payloads] + for index, condition_id in enumerate(remaining, start=1): + slug = slugs.get(condition_id, "").strip() + if not slug or slug.lower() == "nan": + failures[condition_id] = {"classification": "missing_slug", "official_outcome": None} + continue + url = GAMMA_MARKET_BY_SLUG_URL.format(slug=slug) + try: + response = _request_with_retry(session, url) + evidence = archive.save(source="gamma", kind=f"slug_{index:05d}", response=response) + if response.status_code != 200: + failures[condition_id] = { + "classification": f"gamma_http_{response.status_code}", + "official_outcome": None, + } + continue + decoded = response.json() + candidates = _extract_gamma_payloads(decoded) + if len(candidates) != 1: + failures[condition_id] = { + "classification": "gamma_slug_cardinality_error", + "official_outcome": None, + } + continue + payload = candidates[0] + payload["_response_body_sha256"] = evidence["body_sha256"] + payload["_response_url"] = evidence["url"] + payloads[condition_id] = payload + except Exception as error: + archive.save_exception(source="gamma", kind=f"slug_{index:05d}", url=url, error=error) + failures[condition_id] = { + "classification": "gamma_request_exception", + "official_outcome": None, + } + return payloads, failures + + +def fetch_clob_payloads( + target_ids: set[str], + *, + archive: ResponseArchive, + session: requests.Session, + max_pages: int, +) -> tuple[dict[str, dict[str, Any]], dict[str, Any]]: + payloads: dict[str, dict[str, Any]] = {} + cursor: str | None = None + seen_cursors: set[str] = set() + audit: dict[str, Any] = {"pages": 0, "terminated": None, "last_cursor": None} + + for page in range(1, max_pages + 1): + params = None if cursor is None else {"next_cursor": cursor} + try: + response = _request_with_retry(session, CLOB_SIMPLIFIED_MARKETS_URL, params=params) + except Exception as error: + archive.save_exception( + source="clob", kind=f"page_{page:05d}", url=CLOB_SIMPLIFIED_MARKETS_URL, error=error + ) + audit.update({"pages": page - 1, "terminated": "request_exception", "last_cursor": cursor}) + break + evidence = archive.save(source="clob", kind=f"page_{page:05d}", response=response) + if response.status_code != 200: + audit.update( + {"pages": page, "terminated": f"http_{response.status_code}", "last_cursor": cursor} + ) + break + try: + decoded = response.json() + except ValueError: + audit.update({"pages": page, "terminated": "malformed_json", "last_cursor": cursor}) + break + if not isinstance(decoded, Mapping): + audit.update({"pages": page, "terminated": "malformed_payload", "last_cursor": cursor}) + break + data = decoded.get("data") + if not isinstance(data, list): + audit.update({"pages": page, "terminated": "malformed_data", "last_cursor": cursor}) + break + for item in data: + if not isinstance(item, Mapping): + continue + condition_id = _normalize_condition_id(item.get("condition_id")) + if condition_id not in target_ids or condition_id in payloads: + continue + payload = dict(item) + payload["_response_body_sha256"] = evidence["body_sha256"] + payload["_response_url"] = evidence["url"] + payloads[condition_id] = payload + next_cursor = decoded.get("next_cursor") + audit.update({"pages": page, "last_cursor": next_cursor}) + if not next_cursor or next_cursor == "LTE=": + audit["terminated"] = "end_cursor" + break + next_cursor = str(next_cursor) + if next_cursor in seen_cursors: + audit["terminated"] = "repeated_cursor" + break + seen_cursors.add(next_cursor) + cursor = next_cursor + else: + audit["terminated"] = "max_pages" + audit["matched_targets"] = len(payloads) + return payloads, audit + + +def _unavailable(source: str, classification: str = "unavailable") -> dict[str, Any]: + result = _base_parse_result(source, classification) + return result + + +def build_transfer_ledger( + eligible: pd.DataFrame, + gamma_payloads: Mapping[str, Mapping[str, Any]], + gamma_failures: Mapping[str, Mapping[str, Any]], + clob_payloads: Mapping[str, Mapping[str, Any]], +) -> pd.DataFrame: + rows: list[dict[str, Any]] = [] + for trade in eligible.itertuples(index=False): + condition_id = _normalize_condition_id(trade.condition_id) + gamma_payload = gamma_payloads.get(condition_id) + if gamma_payload is not None: + gamma = parse_gamma_terminal(gamma_payload, expected_condition_id=condition_id) + gamma_url = gamma_payload.get("_response_url") + gamma_body_sha256 = gamma_payload.get("_response_body_sha256") + else: + failure = gamma_failures.get(condition_id) or {} + gamma = _unavailable("gamma", str(failure.get("classification") or "unavailable")) + gamma_url = None + gamma_body_sha256 = None + + clob_payload = clob_payloads.get(condition_id) + if clob_payload is not None: + clob = parse_clob_terminal(clob_payload, expected_condition_id=condition_id) + clob_url = clob_payload.get("_response_url") + clob_body_sha256 = clob_payload.get("_response_body_sha256") + else: + clob = _unavailable("clob") + clob_url = None + clob_body_sha256 = None + + reconciled = reconcile_terminal_sources(gamma, clob) + official_outcome = reconciled["official_outcome"] + official_usable = official_outcome in {"Up", "Down"} + official_won = bool(str(trade.selected_side).title() == official_outcome) if official_usable else None + arrival_ask = float(trade.arrival_ask) + fee = float(trade.fee_per_share) if pd.notna(trade.fee_per_share) else 0.07 * arrival_ask * (1.0 - arrival_ask) + official_pnl_per_share = (float(official_won) - arrival_ask - fee) if official_usable else None + inferred = str(trade.inferred_outcome).title() if pd.notna(trade.inferred_outcome) else None + inferred = inferred if inferred in {"Up", "Down"} else None + + rows.append( + { + **trade._asdict(), + "condition_id": condition_id, + "gamma_classification": gamma["classification"], + "gamma_outcome": gamma.get("official_outcome"), + "gamma_payload_sha256": gamma.get("payload_sha256"), + "gamma_response_body_sha256": gamma_body_sha256, + "gamma_url": gamma_url, + "gamma_closed": gamma.get("closed"), + "gamma_uma_resolution_status": gamma.get("uma_resolution_status"), + "clob_classification": clob["classification"], + "clob_outcome": clob.get("official_outcome"), + "clob_payload_sha256": clob.get("payload_sha256"), + "clob_response_body_sha256": clob_body_sha256, + "clob_url": clob_url, + "official_classification": reconciled["classification"], + "official_outcome": official_outcome, + "official_label_usable": official_usable, + "official_won": official_won, + "official_pnl_per_share": official_pnl_per_share, + "official_pnl_at_five_shares": official_pnl_per_share * 5 if official_pnl_per_share is not None else None, + "inferred_label_available": inferred is not None, + "official_vs_inferred_agree": (official_outcome == inferred) if official_usable and inferred else None, + } + ) + return pd.DataFrame(rows) + + +def _counts(series: pd.Series) -> dict[str, int]: + return {str(key): int(value) for key, value in series.value_counts(dropna=False).sort_index().items()} + + +def summarize_transfer(ledger: pd.DataFrame) -> dict[str, Any]: + usable = ledger.loc[ledger["official_label_usable"]].copy() + comparable = usable.loc[usable["inferred_label_available"]].copy() + by_asset: dict[str, Any] = {} + for asset, frame in ledger.groupby("asset", sort=True): + asset_usable = frame.loc[frame["official_label_usable"]] + asset_comparable = asset_usable.loc[asset_usable["inferred_label_available"]] + by_asset[str(asset)] = { + "fills": int(len(frame)), + "official_usable": int(len(asset_usable)), + "official_coverage": float(len(asset_usable) / len(frame)) if len(frame) else None, + "official_wins": int(asset_usable["official_won"].sum()), + "official_win_rate": float(asset_usable["official_won"].mean()) if len(asset_usable) else None, + "official_pnl_at_five_shares": float(asset_usable["official_pnl_at_five_shares"].sum()), + "comparable_to_inferred": int(len(asset_comparable)), + "official_vs_inferred_agreement": ( + float(asset_comparable["official_vs_inferred_agree"].mean()) if len(asset_comparable) else None + ), + } + return { + "fills": int(len(ledger)), + "official_usable": int(len(usable)), + "official_coverage": float(len(usable) / len(ledger)) if len(ledger) else None, + "official_wins": int(usable["official_won"].sum()), + "official_win_rate": float(usable["official_won"].mean()) if len(usable) else None, + "official_mean_pnl_per_share": float(usable["official_pnl_per_share"].mean()) if len(usable) else None, + "official_pnl_at_five_shares": float(usable["official_pnl_at_five_shares"].sum()), + "comparable_to_inferred": int(len(comparable)), + "official_vs_inferred_agreement": ( + float(comparable["official_vs_inferred_agree"].mean()) if len(comparable) else None + ), + "official_vs_inferred_disagreements": int((comparable["official_vs_inferred_agree"] == False).sum()), + "classification_counts": _counts(ledger["official_classification"]), + "gamma_classification_counts": _counts(ledger["gamma_classification"]), + "clob_classification_counts": _counts(ledger["clob_classification"]), + "by_asset": by_asset, + } + + +def run_transfer( + *, + eligible_path: Path, + out_dir: Path, + batch_size: int, + clob_max_pages: int, +) -> dict[str, Any]: + out_dir.mkdir(parents=True, exist_ok=True) + eligible = pd.read_csv(eligible_path) + required = { + "condition_id", + "slug", + "asset", + "selected_side", + "arrival_ask", + "fee_per_share", + "inferred_outcome", + } + missing = sorted(required - set(eligible.columns)) + if missing: + raise RuntimeError(f"eligible ledger missing columns: {missing}") + condition_set_sha256 = _condition_set_sha256(eligible["condition_id"]) + if condition_set_sha256 != EXPECTED_FILLED_CONDITION_SET_SHA256: + raise RuntimeError( + f"filled condition-set SHA-256 mismatch: {condition_set_sha256} != " + f"{EXPECTED_FILLED_CONDITION_SET_SHA256}" + ) + if eligible["condition_id"].duplicated().any(): + raise RuntimeError("eligible ledger contains duplicate condition IDs") + + archive = ResponseArchive(out_dir) + session = _session() + targets = eligible[["condition_id", "slug"]].copy() + gamma_payloads, gamma_failures = fetch_gamma_payloads( + targets, archive=archive, session=session, batch_size=batch_size + ) + clob_payloads, clob_scan = fetch_clob_payloads( + {_normalize_condition_id(value) for value in eligible["condition_id"]}, + archive=archive, + session=session, + max_pages=clob_max_pages, + ) + raw_manifest = archive.write_manifest() + + ledger = build_transfer_ledger(eligible, gamma_payloads, gamma_failures, clob_payloads) + ledger = ledger.sort_values(["market_start", "asset", "condition_id"]) + ledger_path = out_dir / "official_transfer_ledger.csv.gz" + ledger.to_csv(ledger_path, index=False, compression={"method": "gzip", "mtime": 0}) + + summary = summarize_transfer(ledger) + audit = { + "candidate": "late_favorite_btc_eth_v2_diagnostic", + "collected_at_utc": datetime.now(timezone.utc).isoformat(), + "filled_condition_set_sha256": condition_set_sha256, + "expected_filled_condition_set_sha256": EXPECTED_FILLED_CONDITION_SET_SHA256, + "eligible_input_sha256": _sha256_bytes(eligible_path.read_bytes()), + "source_module_sha256": _sha256_bytes(Path(__file__).read_bytes()), + "gamma_endpoint": GAMMA_MARKETS_URL, + "clob_endpoint": CLOB_SIMPLIFIED_MARKETS_URL, + "gamma_payloads": len(gamma_payloads), + "gamma_failures": len(gamma_failures), + "clob_payloads": len(clob_payloads), + "clob_scan": clob_scan, + "raw_responses": len(archive.records), + "raw_response_manifest_sha256": _sha256_bytes(raw_manifest.read_bytes()), + "live_submission": "physically_absent", + "credentials_used": 0, + "thresholds_changed": 0, + **summary, + } + audit_path = out_dir / "AUDIT.json" + audit_path.write_text(json.dumps(audit, indent=2, sort_keys=True) + "\n") + + coverage = audit["official_coverage"] + win_rate = audit["official_win_rate"] + mean_pnl = audit["official_mean_pnl_per_share"] + report = ( + "# Late-Favorite BTC/ETH Official Outcome Transfer\n\n" + "## Result\n\n" + f"- Frozen filled condition set: `{condition_set_sha256}`\n" + f"- Fills: {audit['fills']:,}\n" + f"- Official usable labels: {audit['official_usable']:,} ({coverage:.4%})\n" + f"- Official win rate: {win_rate:.4%}\n" + f"- Official mean P&L/share: {mean_pnl:.6f}\n" + f"- Official P&L at five shares: ${audit['official_pnl_at_five_shares']:.6f}\n" + f"- Comparable inferred labels: {audit['comparable_to_inferred']:,}\n" + f"- Official/inferred disagreements: {audit['official_vs_inferred_disagreements']:,}\n" + f"- Gamma payloads: {audit['gamma_payloads']:,}\n" + f"- CLOB corroborating payloads: {audit['clob_payloads']:,}\n\n" + "## Evidence boundary\n\n" + "Gamma is the primary official terminal source. CLOB winner flags corroborate Gamma where available. CLOB-only outcomes are not promoted. Every HTTP response body is retained with its URL, status code, and SHA-256. This job contains no credentials or order-submission implementation.\n" + ) + report_path = out_dir / "REPORT.md" + report_path.write_text(report) + + top_level_outputs = [audit_path, ledger_path, raw_manifest, report_path] + sums_path = out_dir / "SHA256SUMS" + sums_path.write_text( + "".join(f"{_sha256_bytes(path.read_bytes())} {path.name}\n" for path in sorted(top_level_outputs)) + ) + return audit + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--eligible-trades", type=Path, required=True) + parser.add_argument("--out-dir", type=Path, required=True) + parser.add_argument("--batch-size", type=int, default=25) + parser.add_argument("--clob-max-pages", type=int, default=5000) + args = parser.parse_args() + if args.batch_size < 1 or args.batch_size > 100: + raise SystemExit("batch size must be between 1 and 100") + audit = run_transfer( + eligible_path=args.eligible_trades, + out_dir=args.out_dir, + batch_size=args.batch_size, + clob_max_pages=args.clob_max_pages, + ) + print(json.dumps(audit, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() From 78776a240b37613c1ab0d57aee33e9d2209208ca Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:00:27 -0500 Subject: [PATCH 090/177] fix: harden official transfer evidence serialization --- .../official_transfer.py | 43 ++++++++++++------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/official_transfer.py b/ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/official_transfer.py index ba1588f1..ced72ac2 100644 --- a/ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/official_transfer.py +++ b/ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/official_transfer.py @@ -4,7 +4,6 @@ import gzip import hashlib import json -import math import time from dataclasses import dataclass, field from datetime import datetime, timezone @@ -31,6 +30,12 @@ def _canonical_json_bytes(value: Any) -> bytes: return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") +def _payload_for_hash(value: Any) -> Any: + if isinstance(value, Mapping): + return {key: item for key, item in value.items() if not str(key).startswith("_")} + return value + + def _normalize_condition_id(value: Any) -> str: return str(value or "").strip().lower() @@ -63,7 +68,7 @@ def _base_parse_result(source: str, classification: str) -> dict[str, Any]: def parse_gamma_terminal(payload: Any, *, expected_condition_id: str) -> dict[str, Any]: result = _base_parse_result("gamma", "malformed_gamma_payload") - result["payload_sha256"] = _sha256_bytes(_canonical_json_bytes(payload)) + result["payload_sha256"] = _sha256_bytes(_canonical_json_bytes(_payload_for_hash(payload))) if not isinstance(payload, Mapping): return result @@ -110,7 +115,7 @@ def parse_gamma_terminal(payload: Any, *, expected_condition_id: str) -> dict[st def parse_clob_terminal(payload: Any, *, expected_condition_id: str) -> dict[str, Any]: result = _base_parse_result("clob", "malformed_clob_payload") - result["payload_sha256"] = _sha256_bytes(_canonical_json_bytes(payload)) + result["payload_sha256"] = _sha256_bytes(_canonical_json_bytes(_payload_for_hash(payload))) if not isinstance(payload, Mapping): return result @@ -219,9 +224,13 @@ def save_exception(self, *, source: str, kind: str, url: str, error: Exception) def write_manifest(self) -> Path: path = self.root / "raw_response_manifest.jsonl.gz" - with gzip.open(path, "wt", encoding="utf-8", mtime=0) as handle: - for record in self.records: - handle.write(json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n") + raw = "".join( + json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n" + for record in self.records + ).encode("utf-8") + with path.open("wb") as handle: + with gzip.GzipFile(filename="", mode="wb", fileobj=handle, mtime=0) as compressed: + compressed.write(raw) return path @@ -441,8 +450,7 @@ def fetch_clob_payloads( def _unavailable(source: str, classification: str = "unavailable") -> dict[str, Any]: - result = _base_parse_result(source, classification) - return result + return _base_parse_result(source, classification) def build_transfer_ledger( @@ -557,6 +565,14 @@ def summarize_transfer(ledger: pd.DataFrame) -> dict[str, Any]: } +def _format_percent(value: float | None) -> str: + return "n/a" if value is None else f"{value:.4%}" + + +def _format_decimal(value: float | None, places: int = 6) -> str: + return "n/a" if value is None else f"{value:.{places}f}" + + def run_transfer( *, eligible_path: Path, @@ -630,18 +646,15 @@ def run_transfer( audit_path = out_dir / "AUDIT.json" audit_path.write_text(json.dumps(audit, indent=2, sort_keys=True) + "\n") - coverage = audit["official_coverage"] - win_rate = audit["official_win_rate"] - mean_pnl = audit["official_mean_pnl_per_share"] report = ( "# Late-Favorite BTC/ETH Official Outcome Transfer\n\n" "## Result\n\n" f"- Frozen filled condition set: `{condition_set_sha256}`\n" f"- Fills: {audit['fills']:,}\n" - f"- Official usable labels: {audit['official_usable']:,} ({coverage:.4%})\n" - f"- Official win rate: {win_rate:.4%}\n" - f"- Official mean P&L/share: {mean_pnl:.6f}\n" - f"- Official P&L at five shares: ${audit['official_pnl_at_five_shares']:.6f}\n" + f"- Official usable labels: {audit['official_usable']:,} ({_format_percent(audit['official_coverage'])})\n" + f"- Official win rate: {_format_percent(audit['official_win_rate'])}\n" + f"- Official mean P&L/share: {_format_decimal(audit['official_mean_pnl_per_share'])}\n" + f"- Official P&L at five shares: ${_format_decimal(audit['official_pnl_at_five_shares'])}\n" f"- Comparable inferred labels: {audit['comparable_to_inferred']:,}\n" f"- Official/inferred disagreements: {audit['official_vs_inferred_disagreements']:,}\n" f"- Gamma payloads: {audit['gamma_payloads']:,}\n" From 039c6f249131d5f60a21fac4f7fbbde17c93655f Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:01:21 -0500 Subject: [PATCH 091/177] ci: run hash-bound official outcome transfer --- .../late-favorite-official-transfer.yml | 159 +++++++++++++++++- 1 file changed, 153 insertions(+), 6 deletions(-) diff --git a/.github/workflows/late-favorite-official-transfer.yml b/.github/workflows/late-favorite-official-transfer.yml index 63544249..0f6ecef9 100644 --- a/.github/workflows/late-favorite-official-transfer.yml +++ b/.github/workflows/late-favorite-official-transfer.yml @@ -6,11 +6,15 @@ on: paths: - .github/workflows/late-favorite-official-transfer.yml - ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/** + - ci_payload/digital-surface-live-readiness/late-favorite-repro-source/** + - ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/CANDIDATE_SPEC.json pull_request: branches: [digital-surface-ci-20260724] paths: - .github/workflows/late-favorite-official-transfer.yml - ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/** + - ci_payload/digital-surface-live-readiness/late-favorite-repro-source/** + - ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/CANDIDATE_SPEC.json workflow_dispatch: permissions: @@ -21,17 +25,160 @@ env: PYTHONUNBUFFERED: '1' jobs: - red-terminal-contract: + transfer-official-outcomes: runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 180 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: '3.11' - - name: Install test dependency - run: python -m pip install pytest==8.3.5 - - name: Verify RED terminal parser contract + + - name: Install pinned research stack + run: | + python -m pip install --upgrade pip + python -m pip install \ + numpy==2.2.6 \ + pandas==2.2.3 \ + pyarrow==19.0.1 \ + requests==2.32.4 \ + huggingface_hub==0.33.4 \ + hf_xet==1.1.5 \ + pytest==8.3.5 + + - name: Compile transfer and reproducer sources + run: | + python -m py_compile \ + ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/official_transfer.py \ + ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/test_official_transfer.py \ + ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py + + - name: Pass fail-closed official parser tests + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source + run: | + pytest -q ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/test_official_transfer.py + + - name: Verify live submission and credentials remain physically absent + run: | + python - <<'PY' + import ast + from pathlib import Path + + paths = [ + Path('ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/official_transfer.py'), + Path('ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py'), + ] + forbidden_imports = {'web3', 'py_clob_client', 'eth_account'} + forbidden_tokens = ('create_order', 'post_order', 'submit_order', 'private_key', 'api_secret') + for path in paths: + text = path.read_text() + tree = ast.parse(text) + imports = { + node.names[0].name + for node in ast.walk(tree) + if isinstance(node, ast.Import) and node.names + } + imported_from = { + node.module + for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) + } + assert not (forbidden_imports & imports), (path, imports) + assert not (forbidden_imports & imported_from), (path, imported_from) + lowered = text.lower() + for token in forbidden_tokens: + assert token not in lowered, (path, token) + PY + + - name: Rebuild exact frozen eligible condition set + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-repro-source + HF_XET_HIGH_PERFORMANCE: '1' + run: | + python ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py \ + --candidate-spec ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/CANDIDATE_SPEC.json \ + --out-dir late-favorite-repro-results \ + --cache-dir .cache/late-favorite-data + + - name: Assert immutable filled condition-set identity + run: | + python - <<'PY' + import json + from pathlib import Path + + audit = json.loads(Path('late-favorite-repro-results/AUDIT.json').read_text()) + assert audit['hypothetical_fok_fills'] == 9268 + assert audit['filled_condition_ids_sha256'] == 'bf276fbce187d9651d56ecae976520d4251140c1d5b0d440696c5d077838a67a' + assert audit['live_submission'] == 'physically_absent' + PY + + - name: Transfer official Gamma outcomes and CLOB corroboration env: PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source - run: pytest -q ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/test_official_transfer.py + run: | + python ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/official_transfer.py \ + --eligible-trades late-favorite-repro-results/eligible_trades.csv.gz \ + --out-dir late-favorite-official-transfer-results \ + --batch-size 25 \ + --clob-max-pages 1000 + + - name: Verify transfer evidence and classification contract + run: | + python - <<'PY' + import gzip + import hashlib + import json + from pathlib import Path + + out = Path('late-favorite-official-transfer-results') + required = { + 'AUDIT.json', + 'REPORT.md', + 'SHA256SUMS', + 'official_transfer_ledger.csv.gz', + 'raw_response_manifest.jsonl.gz', + } + assert required <= {path.name for path in out.iterdir() if path.is_file()} + audit = json.loads((out / 'AUDIT.json').read_text()) + assert audit['candidate'] == 'late_favorite_btc_eth_v2_diagnostic' + assert audit['fills'] == 9268 + assert audit['filled_condition_set_sha256'] == 'bf276fbce187d9651d56ecae976520d4251140c1d5b0d440696c5d077838a67a' + assert audit['live_submission'] == 'physically_absent' + assert audit['credentials_used'] == 0 + assert audit['thresholds_changed'] == 0 + assert audit['raw_responses'] > 0 + assert sum(audit['classification_counts'].values()) == audit['fills'] + assert 0 <= audit['official_usable'] <= audit['fills'] + assert audit['gamma_payloads'] + audit['gamma_failures'] <= audit['fills'] + + manifest_path = out / 'raw_response_manifest.jsonl.gz' + with gzip.open(manifest_path, 'rt', encoding='utf-8') as handle: + records = [json.loads(line) for line in handle if line.strip()] + assert len(records) == audit['raw_responses'] + for record in records: + if record.get('body_path') is None: + continue + body = out / record['body_path'] + assert body.is_file(), body + assert hashlib.sha256(body.read_bytes()).hexdigest() == record['body_sha256'] + PY + (cd late-favorite-official-transfer-results && sha256sum -c SHA256SUMS) + + - name: Build evidence archive + run: | + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf late-favorite-official-transfer-v1.tar.gz late-favorite-official-transfer-results + sha256sum late-favorite-official-transfer-v1.tar.gz > late-favorite-official-transfer-v1.tar.gz.sha256 + cat late-favorite-official-transfer-results/REPORT.md + cat late-favorite-official-transfer-results/AUDIT.json + + - uses: actions/upload-artifact@v4 + with: + name: late-favorite-official-transfer-v1 + path: | + late-favorite-official-transfer-results/ + late-favorite-official-transfer-v1.tar.gz + late-favorite-official-transfer-v1.tar.gz.sha256 + compression-level: 0 + retention-days: 30 From 780c4f26c31507bfe72fb8fc509910fce598eef3 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:37:24 -0500 Subject: [PATCH 092/177] ci: publish compact official-transfer evidence --- .../workflows/late-favorite-official-transfer.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/late-favorite-official-transfer.yml b/.github/workflows/late-favorite-official-transfer.yml index 0f6ecef9..cce4c20c 100644 --- a/.github/workflows/late-favorite-official-transfer.yml +++ b/.github/workflows/late-favorite-official-transfer.yml @@ -182,3 +182,17 @@ jobs: late-favorite-official-transfer-v1.tar.gz.sha256 compression-level: 0 retention-days: 30 + + - name: Upload compact decision evidence + uses: actions/upload-artifact@v4 + with: + name: late-favorite-official-transfer-summary-v1 + path: | + late-favorite-official-transfer-results/AUDIT.json + late-favorite-official-transfer-results/REPORT.md + late-favorite-official-transfer-results/SHA256SUMS + late-favorite-official-transfer-results/official_transfer_ledger.csv.gz + late-favorite-official-transfer-results/raw_response_manifest.jsonl.gz + late-favorite-official-transfer-v1.tar.gz.sha256 + compression-level: 0 + retention-days: 30 From eff6c561f66774bb42b535412828f1eb0fa222e3 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:40:55 -0500 Subject: [PATCH 093/177] ci: extract compact official-transfer evidence --- ...ite-official-transfer-compact-existing.yml | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 .github/workflows/late-favorite-official-transfer-compact-existing.yml diff --git a/.github/workflows/late-favorite-official-transfer-compact-existing.yml b/.github/workflows/late-favorite-official-transfer-compact-existing.yml new file mode 100644 index 00000000..b0a3df4e --- /dev/null +++ b/.github/workflows/late-favorite-official-transfer-compact-existing.yml @@ -0,0 +1,86 @@ +name: Late Favorite Compact Existing Official Evidence + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/late-favorite-official-transfer-compact-existing.yml + workflow_dispatch: + +permissions: + actions: read + contents: read + +env: + SOURCE_RUN_ID: '30213818908' + SOURCE_ARTIFACT_DIGEST: '210984541bcd64597ca48d480ad59102427f56b4ac204ab1308ad8f8a4f93ce5' + +jobs: + extract-compact-evidence: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - name: Download the already-green full evidence artifact + uses: actions/download-artifact@v4 + with: + name: late-favorite-official-transfer-v1 + run-id: ${{ env.SOURCE_RUN_ID }} + github-token: ${{ github.token }} + path: source-artifact + + - name: Locate and verify decision evidence + shell: bash + run: | + set -euo pipefail + find source-artifact -maxdepth 4 -type f -printf '%p %s bytes\n' | sort + results="$(find source-artifact -type d -name late-favorite-official-transfer-results -print -quit)" + test -n "$results" + for name in AUDIT.json REPORT.md SHA256SUMS official_transfer_ledger.csv.gz raw_response_manifest.jsonl.gz; do + test -f "$results/$name" + done + (cd "$results" && sha256sum -c SHA256SUMS) + mkdir compact + cp "$results/AUDIT.json" compact/ + cp "$results/REPORT.md" compact/ + cp "$results/SHA256SUMS" compact/ + cp "$results/official_transfer_ledger.csv.gz" compact/ + cp "$results/raw_response_manifest.jsonl.gz" compact/ + printf '%s source-full-artifact.zip\n' "$SOURCE_ARTIFACT_DIGEST" > compact/SOURCE_ARTIFACT_SHA256.txt + printf '%s\n' "$SOURCE_RUN_ID" > compact/SOURCE_RUN_ID.txt + + - name: Verify compact summary contract + run: | + python - <<'PY' + import gzip + import json + from pathlib import Path + + root = Path('compact') + audit = json.loads((root / 'AUDIT.json').read_text()) + assert audit['candidate'] == 'late_favorite_btc_eth_v2_diagnostic' + assert audit['fills'] == 9268 + assert audit['filled_condition_set_sha256'] == 'bf276fbce187d9651d56ecae976520d4251140c1d5b0d440696c5d077838a67a' + assert audit['credentials_used'] == 0 + assert audit['thresholds_changed'] == 0 + assert audit['live_submission'] == 'physically_absent' + assert sum(audit['classification_counts'].values()) == 9268 + with gzip.open(root / 'raw_response_manifest.jsonl.gz', 'rt', encoding='utf-8') as handle: + count = sum(1 for line in handle if line.strip()) + assert count == audit['raw_responses'] + print((root / 'REPORT.md').read_text()) + print(json.dumps(audit, indent=2, sort_keys=True)) + PY + + - name: Hash compact evidence + run: | + (cd compact && sha256sum AUDIT.json REPORT.md SHA256SUMS official_transfer_ledger.csv.gz raw_response_manifest.jsonl.gz SOURCE_ARTIFACT_SHA256.txt SOURCE_RUN_ID.txt > COMPACT_SHA256SUMS) + cat compact/COMPACT_SHA256SUMS + + - uses: actions/upload-artifact@v4 + with: + name: late-favorite-official-transfer-compact-from-run-30213818908 + path: compact/ + compression-level: 0 + retention-days: 30 From 48044601667267053bfefa60991f83c751206a4f Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:41:45 -0500 Subject: [PATCH 094/177] ci: expose compact official evidence on PR --- .../late-favorite-official-transfer-compact-existing.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/late-favorite-official-transfer-compact-existing.yml b/.github/workflows/late-favorite-official-transfer-compact-existing.yml index b0a3df4e..fedcd61c 100644 --- a/.github/workflows/late-favorite-official-transfer-compact-existing.yml +++ b/.github/workflows/late-favorite-official-transfer-compact-existing.yml @@ -5,6 +5,10 @@ on: branches: [digital-surface-live-readiness-20260725] paths: - .github/workflows/late-favorite-official-transfer-compact-existing.yml + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/late-favorite-official-transfer-compact-existing.yml workflow_dispatch: permissions: From cedd659e0b573d8479932c428cd2eaf472c6bf8f Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:54:53 -0500 Subject: [PATCH 095/177] research: reject late-favorite candidate on official outcomes --- .../CHECKPOINT.md | 71 +++++++++++++++ .../DECISION_AUDIT.json | 87 +++++++++++++++++++ .../DISAGREEMENTS.csv | 10 +++ .../ROOT_CAUSE_AUDIT.json | 43 +++++++++ 4 files changed, 211 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/CHECKPOINT.md create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/DECISION_AUDIT.json create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/DISAGREEMENTS.csv create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/ROOT_CAUSE_AUDIT.json diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/CHECKPOINT.md b/ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/CHECKPOINT.md new file mode 100644 index 00000000..2351f2c7 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/CHECKPOINT.md @@ -0,0 +1,71 @@ +# Late-Favorite BTC/ETH Official-Outcome Decision Checkpoint v1 + +## Verdict + +The frozen `late_favorite_btc_eth_v2_diagnostic` candidate is **Rejected / falsified** after transfer to authoritative Polymarket terminal outcomes. + +- Frozen fills: **9,268** +- Official coverage: **100.00%** +- Official win rate: **94.3893%** +- Official mean P&L/share: **-$0.00127314** +- Official P&L at five shares: **-$59.00** +- Additional-cost headroom: **negative**; even zero additional slippage is unprofitable. + +No threshold, asset, timing, latency, fee, depth, FOK, or adverse-move rule was changed during transfer. Real-money submission remains physically absent. + +## Why the diagnostic result reversed + +The earlier inferred-label diagnostic reported +$428.93 on 8,883 labeled fills. Official resolution shows that the nine direct label disagreements account for only **-$15.00** of the correction. The decisive defect was the excluded cohort: + +- Previously unlabeled fills: **385** (4.15% of fills) +- Official win rate in that cohort: **67.53%** +- Official P&L in that cohort: **-$472.92** +- Losses in that cohort: **125**, or **24.04%** of all official losses +- Date span: **2026-03-25T04:20:00+00:00** through **2026-05-18T09:30:00+00:00** + +The public dataset explicitly labels outcomes by inference from the final recorded book and permits null outcomes near edge cases. Those nulls were not missing at random: excluding them removed difficult, reversal-prone markets and materially inflated the apparent edge. + +## Asset decomposition + +| Asset | Fills | Win rate | P&L at 5 shares | Daily 95% bootstrap interval | +|---|---:|---:|---:|---:| +| BTC | 4,844 | 94.6945% | +$64.00 | -$1.72 to +$3.93 | +| ETH | 4,424 | 94.0552% | -$123.00 | -$5.90 to +$0.18 | +| Combined | 9,268 | 94.3893% | -$59.00 | -$5.18 to +$3.00 | + +BTC-only is **not promoted**. BTC was selected for inspection together with ETH, and retaining BTC after seeing official outcomes would be post-hoc. Its daily confidence interval also crosses zero. Any BTC-only continuation must be frozen as a new prospective version with no inherited holdout status. + +## Risk findings + +- 56 official calendar days; 27 positive days (48.21%). +- Daily t-statistic: -0.50; sign-flip p-value: 0.691. +- Nine calendar weeks; five positive. +- BTC/ETH loss correlation in simultaneous windows: 0.522. +- ETH win probability was 97.80% when BTC won, but only 44.57% when BTC lost. +- Additional cost of only 0.25 cents/share worsens total P&L to approximately -$174.85. + +## Evidence identity + +- Filled condition-set SHA-256: `bf276fbce187d9651d56ecae976520d4251140c1d5b0d440696c5d077838a67a` +- Official transfer source run: `30213818908` +- Full official evidence artifact SHA-256: `210984541bcd64597ca48d480ad59102427f56b4ac204ab1308ad8f8a4f93ce5` +- Compact derivative artifact SHA-256: `fb5dfb4621caa8b62ebaefc0b5c15d04619f773c3686cc4f088767ca023416b9` +- Raw-response manifest SHA-256: `80283073bf2fbed3b5f9ff565852c4303c1ed18f1c6df0845a63abf478059655` +- Gamma terminal payloads: 9,268; failures: 0. +- CLOB corroborated 5,615 outcomes; 3,653 were Gamma-only and retained as such. + +## Classification + +- **Candidate:** Rejected / falsified. +- **Cause:** non-random inferred-label censoring plus weak/negative official economics. +- **BTC-only:** post-hoc diagnostic lead only; prospective-only. +- **ETH:** rejected. +- **Live mode:** disabled; no order submitter or credentials. + +## Next valid research + +1. Preserve this rejection as immutable diagnostic evidence. +2. Correct the reproducer so execution fees are recorded even when inferred labels are null; this is an audit-field correction only and does not change fills or the official verdict. +3. Freeze any BTC-only or replacement hypothesis under a new version before observing new data. +4. Run a credential-free, append-only prospective observer with authoritative terminal outcomes and complete no-trade/fill records. +5. Do not use the inspected historical corpus as admission evidence again. diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/DECISION_AUDIT.json b/ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/DECISION_AUDIT.json new file mode 100644 index 00000000..81320a4f --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/DECISION_AUDIT.json @@ -0,0 +1,87 @@ +{ + "additional_cost_stress_at_five_shares": { + "0.0000": -58.99744399999921, + "0.0025": -174.84744399999923, + "0.0050": -290.6974439999992, + "0.0100": -522.3974439999993, + "0.0150": -754.0974439999992 + }, + "break_even_additional_cost_per_share": -0.00127314294346138, + "by_asset": { + "btc": { + "daily_bootstrap_mean_ci95": [-1.7190766216517774, 3.9265995022544726], + "daily_t_statistic": 0.7865088750981918, + "days": 56, + "fills": 4844, + "mean_pnl_per_share": 0.0026423640338563096, + "pnl_at_five_shares": 63.99805690000045, + "positive_day_rate": 0.5357142857142857, + "win_rate": 0.9469446738232865, + "wins": 4587 + }, + "eth": { + "daily_bootstrap_mean_ci95": [-5.903782505852264, 0.18052209448864426], + "daily_t_statistic": -1.7849776737948304, + "days": 44, + "fills": 4424, + "mean_pnl_per_share": -0.005560375266726951, + "pnl_at_five_shares": -122.99550089999961, + "positive_day_rate": 0.4318181818181818, + "win_rate": 0.9405515370705244, + "wins": 4161 + } + }, + "comparable_to_inferred": 8883, + "cross_asset": { + "btc_losses_in_simultaneous_windows": 92, + "eth_losses_in_simultaneous_windows": 96, + "eth_win_given_btc_loss": 0.44565217391304346, + "eth_win_given_btc_win": 0.9780166096726918, + "loss_correlation": 0.5216654451050746, + "simultaneous_official_windows": 2139 + }, + "daily": { + "best_day_pnl": 32.51204815000002, + "bootstrap_mean_ci95": [-5.1750155101785555, 3.0011215983259074], + "days": 56, + "mean_pnl": -1.05352578571427, + "median_pnl": -1.997734124999985, + "positive_day_rate": 0.48214285714285715, + "positive_days": 27, + "sign_flip_p_value_mc": 0.6905366189267621, + "t_statistic": -0.5012307613775938, + "top_two_day_concentration": null, + "worst_day_pnl": -36.45123499999998 + }, + "fills": 9268, + "inferred_comparable_pnl_at_five_shares": 428.92574600000006, + "official_classification_counts": {"terminal_confirmed_both": 5615, "terminal_gamma_only": 3653}, + "official_comparable_pnl_at_five_shares": 413.9257460000008, + "official_coverage": 1.0, + "official_mean_pnl_per_share": -0.00127314294346138, + "official_pnl_at_five_shares": -58.99744399999921, + "official_usable": 9268, + "official_vs_inferred_agreement": 0.9989868287740629, + "official_vs_inferred_disagreements": 9, + "official_win_rate": 0.9438929650410013, + "official_wins": 8748, + "total_pnl_all_unavailable_losses": -58.99744399999921, + "total_pnl_all_unavailable_wins": -58.99744399999921, + "unavailable_fills": 0, + "weekly": { + "blocks": 9, + "pnl_by_week": [ + {"fills": 374, "pnl": 12.639917150000034, "week": "2026-03-23/2026-03-29", "wins": 354}, + {"fills": 624, "pnl": -7.385373599999932, "week": "2026-03-30/2026-04-05", "wins": 588}, + {"fills": 1279, "pnl": 30.049487200000122, "week": "2026-04-06/2026-04-12", "wins": 1216}, + {"fills": 1351, "pnl": -72.57941159999986, "week": "2026-04-13/2026-04-19", "wins": 1262}, + {"fills": 1320, "pnl": -42.79659704999987, "week": "2026-04-20/2026-04-26", "wins": 1234}, + {"fills": 1328, "pnl": 39.82094355000012, "week": "2026-04-27/2026-05-03", "wins": 1264}, + {"fills": 1486, "pnl": -56.12428954999987, "week": "2026-05-04/2026-05-10", "wins": 1399}, + {"fills": 1422, "pnl": 25.568809900000133, "week": "2026-05-11/2026-05-17", "wins": 1349}, + {"fills": 84, "pnl": 11.809070000000007, "week": "2026-05-18/2026-05-24", "wins": 82} + ], + "positive_block_rate": 0.5555555555555556, + "positive_blocks": 5 + } +} diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/DISAGREEMENTS.csv b/ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/DISAGREEMENTS.csv new file mode 100644 index 00000000..440dabd7 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/DISAGREEMENTS.csv @@ -0,0 +1,10 @@ +asset,condition_id,slug,market_start,selected_side,signal_ask,arrival_ask,arrival_ask_size,inferred_outcome,official_outcome,official_classification,official_won,official_pnl_at_five_shares,gamma_url,gamma_response_body_sha256,clob_url,clob_response_body_sha256 +btc,0x877164694b1e24e500bc57f6fddc491a710d592a437a6601dcd0a15397689e01,btc-updown-5m-1774638300,2026-03-27 19:05:00+00:00,Down,0.92,0.92,58.02,Down,Up,terminal_confirmed_both,False,-4.62576,https://gamma-api.polymarket.com/markets/slug/btc-updown-5m-1774638300,982bbcd818737974bcca97318105d5e2322a682280900af7586226d8d8984161,https://clob.polymarket.com/simplified-markets?next_cursor=Njk1MDAw,3a1ced842fbc4ce5c6f103a818dce87510d1d6bd138e2019712595d161f1760b +eth,0xab8463fdee7b57ca4c12d269f6b79dc9a616d7de5c5d652fcd2cb373e458bf9b,eth-updown-5m-1775580300,2026-04-07 16:45:00+00:00,Up,0.99,0.99,791.08,Up,Down,terminal_confirmed_both,False,-4.9534650000000005,https://gamma-api.polymarket.com/markets/slug/eth-updown-5m-1775580300,d5a8a2cf4f4388a47a9e356c9da852fd89ce5e51aebefc1c7781b93552032a3a,https://clob.polymarket.com/simplified-markets?next_cursor=Nzk2MDAw,0df5b94bd5b975865dfcb3eb6d8cfb0621984721b75bb544d87f2cadf58d7a39 +btc,0x9843de3787b17d3e8a56e088055f3be92c67cf9e003ece193129a339c1913039,btc-updown-5m-1775823900,2026-04-10 12:25:00+00:00,Down,0.92,0.91,136.09,Down,Up,terminal_confirmed_both,False,-4.578664999999999,https://gamma-api.polymarket.com/markets/slug/btc-updown-5m-1775823900,0dea7c6750bc31a522ea7ee441e87c7574e20b816d98532482313734c3bff024,https://clob.polymarket.com/simplified-markets?next_cursor=ODIwMDAw,3f53d41926de2b839c3fdd88f964d03fcda30daeac7ea444c1b6295aab5e3d12 +eth,0x2644f6586b1ff3dcc66fc8d902d6ac4e7d89fa7d7184452f222fc97f40a0bcf9,eth-updown-5m-1777034700,2026-04-24 12:45:00+00:00,Up,0.92,0.92,65.63,Up,Down,terminal_confirmed_both,False,-4.62576,https://gamma-api.polymarket.com/markets/slug/eth-updown-5m-1777034700,8410e6a3070397b0a58466a57a5c335cae87f879f7cbd5e44f95d95fc61ebb13,https://clob.polymarket.com/simplified-markets?next_cursor=OTQzMDAw,7c616bc55f639efc3ad5e5c63a6836bbcc3c3c7c2a796d3758e2350dbdf9a8d6 +eth,0x0922e6d8a580df0063dceadc6f9d33480fac5b6121b5933bc4d3609089052f2c,eth-updown-5m-1777642500,2026-05-01 13:35:00+00:00,Down,0.85,0.85,23.83,Up,Down,terminal_gamma_only,True,0.7053750000000002,https://gamma-api.polymarket.com/markets/slug/eth-updown-5m-1777642500,55f6b730aa1687f10d91455dee01df6ea11afd144bc2a48352dad85a77449699,, +eth,0x64a0b8bc82ce31dc6992f56a6f9e0037660835d1fd8af4808a00ed775a4e6ffc,eth-updown-5m-1777659000,2026-05-01 18:10:00+00:00,Up,0.91,0.91,77.16,Down,Up,terminal_gamma_only,True,0.4213350000000003,https://gamma-api.polymarket.com/markets/slug/eth-updown-5m-1777659000,4745638c609e852ffd6a96d7e23fd1a624261a3af39c3a6d1acd83fa54899d30,, +eth,0x4f7d402a972e69b25030ba76a195161cfe851936feb7e7daa79c70ca8546d630,eth-updown-5m-1777684800,2026-05-02 01:20:00+00:00,Up,0.96,0.96,250.0,Down,Up,terminal_gamma_only,True,0.1865600000000001,https://gamma-api.polymarket.com/markets/slug/eth-updown-5m-1777684800,f29109108eee28043e1d7b10063f0d06f5c5932b678f3c872454321a11197ed7,, +btc,0x7cba6e626c5335d2843bdffada0f5d5a5d7a2d6571ca4412d9635151e1ad0c0e,btc-updown-5m-1777857300,2026-05-04 01:15:00+00:00,Up,0.96,0.96,20.0,Up,Down,terminal_gamma_only,False,-4.81344,https://gamma-api.polymarket.com/markets/slug/btc-updown-5m-1777857300,711acbdade53c307071dcfc3af205774a0e93d8d4f94ad74e4e92ab2c3920937,, +eth,0x363c52b17b934dc97ed90043a8c1cfae5923e317409180b92fe79cb8fdceee32,eth-updown-5m-1778319300,2026-05-09 09:35:00+00:00,Up,0.86,0.86,10.0,Up,Down,terminal_gamma_only,False,-4.34214,https://gamma-api.polymarket.com/markets/slug/eth-updown-5m-1778319300,b01d74f0190071fa182cbf0fb461fe9445684a37ddad074bc0302a74d9317f01,, diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/ROOT_CAUSE_AUDIT.json b/ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/ROOT_CAUSE_AUDIT.json new file mode 100644 index 00000000..eef178e3 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/ROOT_CAUSE_AUDIT.json @@ -0,0 +1,43 @@ +{ + "btc_only_boundary": "BTC is positive in this inspected official transfer but its daily confidence interval crosses zero. Selecting BTC-only now is post-hoc and must be treated as a new prospective version, not inherited validation.", + "by_arrival_ask_bucket": [ + {"arrival_ask_bucket": "(0.849, 0.89]", "fills": 142, "official_pnl_at_five_shares": -201.98393, "win_rate": 0.5915492957746479, "wins": 84}, + {"arrival_ask_bucket": "(0.89, 0.92]", "fills": 62, "official_pnl_at_five_shares": -78.545885, "win_rate": 0.6612903225806451, "wins": 41}, + {"arrival_ask_bucket": "(0.92, 0.95]", "fills": 86, "official_pnl_at_five_shares": -71.13024500000002, "win_rate": 0.7790697674418605, "wins": 67}, + {"arrival_ask_bucket": "(0.95, 0.97]", "fills": 40, "official_pnl_at_five_shares": -43.70622499999999, "win_rate": 0.75, "wins": 30}, + {"arrival_ask_bucket": "(0.97, 0.99]", "fills": 50, "official_pnl_at_five_shares": -56.321704999999994, "win_rate": 0.76, "wins": 38}, + {"arrival_ask_bucket": "nan", "fills": 5, "official_pnl_at_five_shares": -21.2352, "win_rate": 0.0, "wins": 0} + ], + "by_asset": { + "btc": {"fills": 179, "first_market_start": "2026-03-25T04:20:00+00:00", "last_market_start": "2026-05-17T20:55:00+00:00", "mean_arrival_ask": 0.9121787709497207, "official_pnl_at_five_shares": -256.29615, "win_rate": 0.6312849162011173, "wins": 113}, + "eth": {"fills": 206, "first_market_start": "2026-04-05T23:45:00+00:00", "last_market_start": "2026-05-18T09:30:00+00:00", "mean_arrival_ask": 0.9188349514563106, "official_pnl_at_five_shares": -216.62704000000002, "win_rate": 0.7135922330097088, "wins": 147} + }, + "by_selected_side": { + "Down": {"fills": 189, "official_pnl_at_five_shares": -208.916355, "win_rate": 0.6984126984126984, "wins": 132}, + "Up": {"fills": 196, "official_pnl_at_five_shares": -264.006835, "win_rate": 0.6530612244897959, "wins": 128} + }, + "candidate": "late_favorite_btc_eth_v2_diagnostic", + "diagnosis": "Non-random label-availability bias: dataset outcomes inferred from final-book state exclude hard edge cases; official resolution restores them and reverses P&L.", + "fills": 9268, + "inferred_comparable_fills": 8883, + "inferred_comparable_pnl_at_five_shares": 428.92574600000006, + "inferred_labeled_mean_arrival_ask": 0.9425450861195542, + "label_disagreements": 9, + "live_submission": "physically_absent", + "official_comparable_pnl_at_five_shares": 413.9257460000008, + "official_coverage": 1.0, + "official_pnl_at_five_shares": -58.99744399999921, + "official_win_rate": 0.9438929650410013, + "pnl_change_from_nine_disagreements": -14.999999999999261, + "previously_unlabeled_fills": 385, + "previously_unlabeled_first_market_start": "2026-03-25T04:20:00+00:00", + "previously_unlabeled_last_market_start": "2026-05-18T09:30:00+00:00", + "previously_unlabeled_mean_arrival_ask": 0.9157402597402597, + "previously_unlabeled_official_losses": 125, + "previously_unlabeled_official_pnl_at_five_shares": -472.92319, + "previously_unlabeled_official_win_rate": 0.6753246753246753, + "previously_unlabeled_official_wins": 260, + "previously_unlabeled_share_of_all_losses": 0.2403846153846154, + "previously_unlabeled_share_of_fills": 0.04154078549848943, + "verdict": "Rejected / falsified on official outcomes" +} From 963e7b3fca288b9b757aa9ae0bd92edde7de2d88 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:56:36 -0500 Subject: [PATCH 096/177] test: require fees on unlabeled hypothetical fills --- .../test_fee_field_regression.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_fee_field_regression.py diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_fee_field_regression.py b/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_fee_field_regression.py new file mode 100644 index 00000000..5838425d --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_fee_field_regression.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import pytest + +from late_favorite_transfer import evaluate_market, fee_per_share + + +def test_unlabeled_hypothetical_fill_still_records_execution_fee() -> None: + result = evaluate_market( + {"au": 0.90, "ad": 0.12}, + {"au": 0.90, "ad": 0.12, "sau": 5.0, "sad": 5.0}, + outcome=None, + ) + + assert result["decision"] == "hypothetical_fok_fill" + assert result["hypothetical_fok_fill"] is True + assert result["fee_per_share"] == pytest.approx(fee_per_share(0.90)) + assert result["won"] is None + assert result["pnl_per_share"] is None + assert result["pnl_at_five_shares"] is None From b517318d43127782eefc7a6faec4b0e3f842a368 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:57:39 -0500 Subject: [PATCH 097/177] ci: isolate late-favorite fee-field regression --- .../late-favorite-fee-field-regression.yml | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/late-favorite-fee-field-regression.yml diff --git a/.github/workflows/late-favorite-fee-field-regression.yml b/.github/workflows/late-favorite-fee-field-regression.yml new file mode 100644 index 00000000..8d9e67b8 --- /dev/null +++ b/.github/workflows/late-favorite-fee-field-regression.yml @@ -0,0 +1,40 @@ +name: Late Favorite Fee-Field Regression + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/late-favorite-fee-field-regression.yml + - ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py + - ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_fee_field_regression.py + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/late-favorite-fee-field-regression.yml + - ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py + - ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_fee_field_regression.py + workflow_dispatch: + +permissions: + contents: read + +jobs: + fee-field-regression: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install minimal pinned stack + run: python -m pip install pandas==2.2.3 huggingface_hub==0.33.4 pytest==8.3.5 + - name: Compile source and regression test + run: | + python -m py_compile \ + ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py \ + ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_fee_field_regression.py + - name: Run fee-field regression + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-repro-source + run: pytest -q ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_fee_field_regression.py From 6bc1353c07533ff85d3fe2cec7bd2160a20c59be Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:59:12 -0500 Subject: [PATCH 098/177] docs: record official late-favorite rejection --- .../digital-surface-live-readiness/STATUS.md | 78 ++++++++++--------- 1 file changed, 42 insertions(+), 36 deletions(-) diff --git a/ci_payload/digital-surface-live-readiness/STATUS.md b/ci_payload/digital-surface-live-readiness/STATUS.md index 834c3d0a..ac3c4388 100644 --- a/ci_payload/digital-surface-live-readiness/STATUS.md +++ b/ci_payload/digital-surface-live-readiness/STATUS.md @@ -5,60 +5,66 @@ - Repository: `Chrisbryan17/hello-world` - Branch: `digital-surface-live-readiness-20260725` - Draft PR: #59 -- Source checkpoint before this status commit: `3f304eba11645a3a19602c8dd334951c619349f2` +- Official late-favorite decision commit: `cedd659e0b573d8479932c428cd2eaf472c6bf8f` - Real-money execution: **physically disabled** -## Verified historical result +## Research verdicts -The checksum-pinned v5 research completed the official-resolution source gate and all four historical transfer folds, but the strategy remained **Rejected**. Those four weeks are now diagnostic and may never be reused as prospective admission evidence. +### Original 5m-vs-15m digital-surface sleeve -## Execution diagnosis +The checksum-pinned v5 research completed the official-resolution source gate and all four historical transfer folds, but the strategy remained **Rejected**. Those four weeks are diagnostic and may never be reused as prospective admission evidence. -Across 224 candidate portfolios / 448 legs: +### Late-favorite BTC/ETH candidate -- insufficient post-arrival tape: 186 legs -- ask above frozen limit: 114 legs -- missing post-arrival book: 110 legs -- filled: 34 legs -- insufficient displayed depth: 4 legs +The frozen `late_favorite_btc_eth_v2_diagnostic` rule was rebuilt over 27,940 markets and transferred across its immutable 9,268-fill condition set to authoritative Polymarket terminal outcomes with 100% coverage. -The original tape-confirmed model produced 8 atomic portfolios. A diagnostic post-latency book-arrival replay produced 62 atomic portfolios, with fold counts `[11, 17, 29, 5]`. This alternative is **not** treated as live truth and requires prospective shadow calibration. +The candidate is **Rejected / falsified**: -## Current source state +- official win rate: 94.3893%; +- official mean P&L/share: -$0.00127314; +- official P&L at five shares: -$59.00; +- BTC: +$64.00, but its daily confidence interval crosses zero; +- ETH: -$123.00; +- BTC-only retention would be post-hoc and receives no inherited validation status. + +The earlier +$428.93 inferred-label diagnostic was biased by 385 null-labeled fills. Official resolution showed that cohort had only 67.53% wins and lost $472.92 at five shares. The nine direct label disagreements changed comparable P&L by only -$15.00. The material defect was non-random label-availability censoring. + +Authoritative decision checkpoint: -The branch contains checksum-pinned, human-readable recovery payloads for: +- `ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/CHECKPOINT.md` +- filled condition-set SHA-256: `bf276fbce187d9651d56ecae976520d4251140c1d5b0d440696c5d077838a67a` +- full official evidence artifact SHA-256: `210984541bcd64597ca48d480ad59102427f56b4ac204ab1308ad8f8a4f93ce5` +- compact derivative artifact SHA-256: `fb5dfb4621caa8b62ebaefc0b5c15d04619f773c3686cc4f088767ca023416b9` -1. deterministic execution-failure attribution; -2. explicit tape-confirmed versus book-arrival replay modes; -3. a shadow-only two-leg FOK preparation gateway using exact `Decimal` tick quantization; -4. fail-closed risk controls for stale feeds/books, daily loss, unresolved orders, orphan exposure, token mismatch, depth, limits, expiry, and ambiguous pair state; -5. an append-only prospective market ledger with canonical SHA-256 chaining and contamination rejection; -6. shadow-canary admission gates requiring at least 500 markets, 100 qualified portfolios, four untouched weekly passes, zero unresolved order states, and at most 5% orphaning; -7. an append-only shadow-report ledger bound to policy SHA-256, source SHA-256, and prospective-ledger head SHA-256. +## Original sleeve execution diagnosis -## Test evidence +Across 224 candidate portfolios / 448 legs: + +- insufficient post-arrival tape: 186 legs; +- ask above frozen limit: 114 legs; +- missing post-arrival book: 110 legs; +- filled: 34 legs; +- insufficient displayed depth: 4 legs. -- Research plus diagnostics and book-arrival source: 32 tests. -- Shadow gateway and prospective ledger checkpoint: 39 tests. -- Shadow canary report checkpoint: 42 tests. -- Shadow report-chain checkpoint: expected **45 tests** in the latest clean-room workflow. -- Focused local safety/prospective/report/report-chain suite: 13 passed. +The original tape-confirmed model produced 8 atomic portfolios. A diagnostic post-latency book-arrival replay produced 62 paired-crossable portfolios, with fold counts `[11, 17, 29, 5]`. This alternative is not live truth. -## Active workflows +## Current source state -- `Digital Surface Book-Arrival Comparison` -- `Digital Surface Shadow Report Chain Checkpoint` +The branch contains checksum-pinned, human-readable recovery payloads for official outcome transfer, deterministic archives, execution diagnostics, public market/book observation, Binance state, causal prospective signals, shadow-only FOK preparation, risk controls, prospective ledgers, report chains, and contamination rejection. -The repository also retains historical workflows; they may queue but are not authoritative for this checkpoint. +A focused regression now requires hypothetical fills to retain their known execution fee even when a diagnostic inferred outcome is null. This is an audit-field correction only; the official transfer already recomputed fees from executable asks and the rejection verdict is unchanged. ## Admission boundary -No real-money order submission implementation exists in the live package. `TRADING_MODE=live` raises `LiveTradingDisabledError` even if credentials and an arm token are supplied. A future result may be classified `Admissible` only from genuinely new prospective markets, but even an admissible report leaves live mode at `disabled_pending_operator_release` until a separate reviewed release and explicit operator action. +No real-money order submission implementation exists in the live package. `TRADING_MODE=live` raises `LiveTradingDisabledError` even if credentials and an arm token are supplied. No historical result in either strategy track is admissible. + +A future candidate may be classified `Admissible` only from a newly frozen policy and genuinely new prospective markets. Even then, live mode remains `disabled_pending_operator_release` until a separately reviewed release and explicit operator action. ## Next work -1. obtain green clean-room artifacts for the 45-test source checkpoint and book-arrival comparison; -2. independently verify outer artifact digests, deterministic TAR sidecars, source hashes, and report/market hash chains; -3. implement the scheduled shadow collector and append-only artifact persistence without credentials; -4. observe at least 500 new markets and four untouched weekly passes; -5. keep real-money execution disabled unless all gates clear and a separately reviewed release is explicitly armed. +1. complete and verify the fee-field regression without changing strategy geometry; +2. preserve both rejected historical tracks as diagnostic evidence only; +3. freeze any BTC-only or replacement hypothesis as a new prospective version before observing new data; +4. run a credential-free append-only observer with authoritative outcomes, complete no-trade/fill records, and measured execution evidence; +5. require at least four untouched weekly prospective blocks and a positive lower confidence bound before reconsidering admission; +6. keep real-money execution physically disabled. From 841121e93e130961fe9d5fa958a77341e61137e9 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:00:30 -0500 Subject: [PATCH 099/177] ci: apply verified late-favorite fee-field fix --- .../apply-late-favorite-fee-field-fix.yml | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 .github/workflows/apply-late-favorite-fee-field-fix.yml diff --git a/.github/workflows/apply-late-favorite-fee-field-fix.yml b/.github/workflows/apply-late-favorite-fee-field-fix.yml new file mode 100644 index 00000000..c55902a5 --- /dev/null +++ b/.github/workflows/apply-late-favorite-fee-field-fix.yml @@ -0,0 +1,81 @@ +name: Apply Late Favorite Fee-Field Fix + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/apply-late-favorite-fee-field-fix.yml + +permissions: + contents: write + +env: + PYTHONUNBUFFERED: '1' + +jobs: + patch-and-verify: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + with: + ref: digital-surface-live-readiness-20260725 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install pinned test stack + run: python -m pip install pandas==2.2.3 huggingface_hub==0.33.4 pytest==8.3.5 + - name: Apply minimal fee-field correction + shell: bash + run: | + python - <<'PY' + from pathlib import Path + + path = Path('ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py') + text = path.read_text() + + anchor = ''' result["hypothetical_fok_fill"] = True + result["decision"] = "hypothetical_fok_fill" + normalized_outcome = str(outcome).title() if outcome is not None else None + '''.replace(' ', '') + replacement = ''' result["hypothetical_fok_fill"] = True + result["decision"] = "hypothetical_fok_fill" + fee = fee_per_share(arrival_ask) + result["fee_per_share"] = fee + normalized_outcome = str(outcome).title() if outcome is not None else None + '''.replace(' ', '') + if text.count(anchor) != 1: + raise RuntimeError(f'fee insertion anchor count={text.count(anchor)}') + text = text.replace(anchor, replacement) + + old_fee_line = ' fee = fee_per_share(arrival_ask)\n' + if text.count(old_fee_line) != 1: + raise RuntimeError(f'old fee line count={text.count(old_fee_line)}') + text = text.replace(old_fee_line, '', 1) + + old_update_line = ' "fee_per_share": fee,\n' + if text.count(old_update_line) != 1: + raise RuntimeError(f'old update fee line count={text.count(old_update_line)}') + text = text.replace(old_update_line, '', 1) + + path.write_text(text) + PY + - name: Compile and verify regressions + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-repro-source + run: | + python -m py_compile ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py + pytest -q \ + ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_late_favorite_transfer.py \ + ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_fee_field_regression.py + - name: Commit verified source correction + shell: bash + run: | + set -euo pipefail + git config user.name 'research-checkpoint-bot' + git config user.email 'research-checkpoint-bot@users.noreply.github.com' + git add ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py + git diff --cached --check + git commit -m 'fix: record fees for unlabeled hypothetical fills' + git push origin HEAD:digital-surface-live-readiness-20260725 From 0dd440509875072f37ce2cad0cdce09d9a5d7dea Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:01:48 -0500 Subject: [PATCH 100/177] ci: expose verified fee-field patch workflow --- .../apply-late-favorite-fee-field-fix.yml | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/.github/workflows/apply-late-favorite-fee-field-fix.yml b/.github/workflows/apply-late-favorite-fee-field-fix.yml index c55902a5..38a4e703 100644 --- a/.github/workflows/apply-late-favorite-fee-field-fix.yml +++ b/.github/workflows/apply-late-favorite-fee-field-fix.yml @@ -5,6 +5,10 @@ on: branches: [digital-surface-live-readiness-20260725] paths: - .github/workflows/apply-late-favorite-fee-field-fix.yml + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/apply-late-favorite-fee-field-fix.yml permissions: contents: write @@ -35,16 +39,18 @@ jobs: path = Path('ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py') text = path.read_text() - anchor = ''' result["hypothetical_fok_fill"] = True - result["decision"] = "hypothetical_fok_fill" - normalized_outcome = str(outcome).title() if outcome is not None else None - '''.replace(' ', '') - replacement = ''' result["hypothetical_fok_fill"] = True - result["decision"] = "hypothetical_fok_fill" - fee = fee_per_share(arrival_ask) - result["fee_per_share"] = fee - normalized_outcome = str(outcome).title() if outcome is not None else None - '''.replace(' ', '') + anchor = ( + ' result["hypothetical_fok_fill"] = True\n' + ' result["decision"] = "hypothetical_fok_fill"\n' + ' normalized_outcome = str(outcome).title() if outcome is not None else None\n' + ) + replacement = ( + ' result["hypothetical_fok_fill"] = True\n' + ' result["decision"] = "hypothetical_fok_fill"\n' + ' fee = fee_per_share(arrival_ask)\n' + ' result["fee_per_share"] = fee\n' + ' normalized_outcome = str(outcome).title() if outcome is not None else None\n' + ) if text.count(anchor) != 1: raise RuntimeError(f'fee insertion anchor count={text.count(anchor)}') text = text.replace(anchor, replacement) @@ -70,6 +76,7 @@ jobs: ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_late_favorite_transfer.py \ ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_fee_field_regression.py - name: Commit verified source correction + if: github.event_name == 'push' shell: bash run: | set -euo pipefail From 43aa55d23fe8ef0469986f700be943ed858543e3 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:07:39 -0500 Subject: [PATCH 101/177] research: classify BTC-only as marginal prospective lead --- .../BTC_ONLY_ASSESSMENT.md | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/BTC_ONLY_ASSESSMENT.md diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/BTC_ONLY_ASSESSMENT.md b/ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/BTC_ONLY_ASSESSMENT.md new file mode 100644 index 00000000..86e6b957 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/BTC_ONLY_ASSESSMENT.md @@ -0,0 +1,64 @@ +# BTC-Only Post-Transfer Diagnostic Assessment + +## Classification + +**Marginal prospective lead only. Not admitted. Not a historical pass.** + +BTC was part of the inspected BTC+ETH candidate. Retaining BTC after observing the official asset split is post-hoc. The statistics below may justify low-cost prospective shadowing under a newly frozen version, but they cannot inherit any validation status from the rejected BTC+ETH rule. + +## Fixed-rule official result + +The entry geometry was not changed: second 210, favorite ask at least $0.85, one-second latency, five-share FOK at the signal ask, cancel when arrival ask is more than $0.01 lower, fee `0.07*p*(1-p)`, settlement hold. + +- Fills: **4,844** +- Official wins: **4,587** +- Official win rate: **94.6945%** +- Mean P&L/share: **+$0.00264236** (**0.2642 cents/share**) +- P&L at five shares: **+$64.00** +- Official days: **56** +- Positive days: **30** (53.57%) +- Daily 95% bootstrap interval for mean P&L: **-$1.73 to +$3.93/day** +- Positive calendar weeks: **6 of 9** +- Maximum cumulative P&L: **+$67.92** +- Maximum dollar drawdown: **-$56.15** +- Approximately **13 additional winner-to-loss flips** erase the observed profit. + +## Cost fragility + +| Additional cost/share | BTC P&L at five shares | +|---:|---:| +| 0.00 cents | +$64.00 | +| 0.10 cents | +$39.78 | +| 0.20 cents | +$15.56 | +| 0.25 cents | +$3.45 | +| 0.30 cents | -$8.66 | +| 0.50 cents | -$57.10 | + +The observed edge is therefore too small to tolerate even modest unmodeled execution friction. Public top-of-book crossability is not proof of queue-position or realized FOK fill probability. + +## Stability diagnostics + +Chronological quartile P&L was `[-$11.31, +$11.47, +$24.16, +$39.67]`, so the result was not produced solely by the earliest segment. Every leave-one-week-out total remained positive. However, weekly outcomes still included three meaningful losing weeks, and the daily confidence interval crosses zero. + +Both selected directions were positive under the unchanged rule: + +- Down favorite: +$55.52 across 2,418 fills. +- Up favorite: +$8.47 across 2,426 fills. + +This split must not be used to introduce a new side filter on the inspected corpus. Price-bucket results were also irregular; no bucket optimization is authorized. + +## Decision + +BTC-only is worth **shadow observation only** because the official point estimate is positive and broadly distributed, but the economic margin is nearly exhausted by 0.25 cents/share of additional cost and statistical uncertainty remains material. + +Any continuation must: + +1. use a new version identifier and policy hash; +2. start after the policy is frozen; +3. record every BTC 5-minute window, including no-signal and no-fill outcomes; +4. preserve signal and one-second-arrival full-depth books; +5. resolve every market from authoritative terminal outcomes; +6. measure actual crossability and displayed depth without assuming queue priority; +7. run at least four untouched weekly blocks; +8. require a positive lower confidence bound after fees and measured execution costs; +9. keep real-money submission physically absent. From 4f943b8e07d62aeea12f58ac36d5e147f182593c Mon Sep 17 00:00:00 2001 From: research-checkpoint-bot Date: Sun, 26 Jul 2026 19:08:39 +0000 Subject: [PATCH 102/177] fix: record fees for unlabeled hypothetical fills --- .../late-favorite-repro-source/late_favorite_transfer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py b/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py index c7be7c86..fb189089 100644 --- a/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py +++ b/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py @@ -109,15 +109,15 @@ def evaluate_market( result["hypothetical_fok_fill"] = True result["decision"] = "hypothetical_fok_fill" + fee = fee_per_share(arrival_ask) + result["fee_per_share"] = fee normalized_outcome = str(outcome).title() if outcome is not None else None if normalized_outcome in {"Up", "Down"}: won = selected_side == normalized_outcome - fee = fee_per_share(arrival_ask) pnl_per_share = float(won) - arrival_ask - fee result.update( { "won": won, - "fee_per_share": fee, "pnl_per_share": pnl_per_share, "pnl_at_five_shares": pnl_per_share * POLICY["shares"], } From bb3a818dc017f18927e6e48b38e1078acb23aaa0 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:11:11 -0500 Subject: [PATCH 103/177] research: freeze BTC-only v3 prospective candidate --- .../CANDIDATE_SPEC.json | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/CANDIDATE_SPEC.json diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/CANDIDATE_SPEC.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/CANDIDATE_SPEC.json new file mode 100644 index 00000000..8e32c009 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/CANDIDATE_SPEC.json @@ -0,0 +1,78 @@ +{ + "arrival": { + "cancel_if": "arrival best ask < signal ask - 0.01", + "fok_limit": "signal_ask", + "latency_seconds": 1, + "no_fill_if": [ + "arrival best ask > signal ask", + "cumulative displayed ask size at prices <= signal ask is less than 5", + "arrival book missing, stale, invalid, or token-mismatched" + ], + "shares": "5" + }, + "contamination_rule": "Only markets whose open time is strictly after the policy freeze commit may count. All previously inspected markets and any observation made before the freeze commit are diagnostic only.", + "economics": { + "additional_slippage_assumption": "0; FOK limit caps price, but prospective reporting must separately measure public-book crossability and hypothetical fill rate", + "fee_per_share": "0.07*p*(1-p)", + "hold": "settlement", + "pnl_per_share": "official_win - executable_arrival_price - fee_per_share" + }, + "name": "late_favorite_btc_only_v3_prospective_shadow", + "observation_contract": { + "append_only_sha256_chain": true, + "arrival_evidence": [ + "request start/end timestamps", + "server/book timestamp", + "full selected-token book", + "raw payload SHA-256", + "cumulative size through FOK limit" + ], + "credentials_used": 0, + "live_submission": "physically_absent", + "record_every_market": true, + "record_no_signal_and_no_fill": true, + "resolution": "authoritative Polymarket terminal outcome with raw response hash", + "signal_evidence": [ + "request start/end timestamps", + "server/book timestamp", + "full Up and Down books", + "raw payload SHA-256" + ], + "unresolved_states_allowed": 0 + }, + "origin": { + "historical_admission_credit": 0, + "official_decision_commit": "cedd659e0b573d8479932c428cd2eaf472c6bf8f", + "official_transfer_filled_condition_set_sha256": "bf276fbce187d9651d56ecae976520d4251140c1d5b0d440696c5d077838a67a", + "selection": "post-hoc BTC subset from rejected late_favorite_btc_eth_v2_diagnostic official transfer" + }, + "prospective_admission_gates": { + "maximum_top_two_day_pnl_concentration": 0.4, + "maximum_unresolved_order_or_outcome_states": 0, + "minimum_hypothetical_fok_fills": 100, + "minimum_observed_markets": 500, + "minimum_untouched_weekly_blocks": 4, + "official_outcome_coverage": 1.0, + "policy_changes_during_run": 0, + "positive_btc_performance": true, + "positive_day_block_bootstrap_lower_bound": true, + "positive_total_pnl_after_fees": true + }, + "release_boundary": "An admissible shadow report cannot enable live trading. Any order submitter requires a separate reviewed release and explicit operator action.", + "signal": { + "entry_second": 210, + "favorite_selection": "higher executable ask between Up and Down; tied or invalid books fail closed", + "required_full_book": true, + "signal_ask_min": "0.85" + }, + "status": "prospective_shadow_only", + "universe": { + "assets": [ + "btc" + ], + "market_duration_seconds": 300, + "maximum_positions_per_asset_per_window": 1, + "maximum_total_positions_per_window": 1 + }, + "version": 3 +} From 0ac9718f66f74a764770555a6cdf48bf659b8721 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:11:46 -0500 Subject: [PATCH 104/177] research: bind BTC-only v3 prospective freeze --- .../FREEZE_MANIFEST.json | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/FREEZE_MANIFEST.json diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/FREEZE_MANIFEST.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/FREEZE_MANIFEST.json new file mode 100644 index 00000000..ab8dba80 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/FREEZE_MANIFEST.json @@ -0,0 +1,18 @@ +{ + "candidate_name": "late_favorite_btc_only_v3_prospective_shadow", + "contamination_boundary": { + "historical_admission_credit": 0, + "rule": "Only BTC 5-minute markets with open_epoch_seconds strictly greater than the freeze timestamp may count toward prospective admission.", + "valid_market_open_after_utc": "2026-07-26T19:11:11Z" + }, + "freeze_commit_sha": "bb3a818dc017f18927e6e48b38e1078acb23aaa0", + "frozen_at_utc": "2026-07-26T19:11:11Z", + "live_submission": "physically_absent", + "official_decision_commit": "cedd659e0b573d8479932c428cd2eaf472c6bf8f", + "policy_path": "ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/CANDIDATE_SPEC.json", + "policy_sha256": "82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba", + "policy_blob_sha": "8e32c00919c77c20dc0d5f0f36bfe04c464d0918", + "release_boundary": "Shadow evidence cannot enable live trading; a separate reviewed and explicitly armed release is required.", + "status": "frozen_for_prospective_shadow_collection", + "version": 1 +} From 38b3a5a0aaddcaf2204bcf5364cf0f13fa510763 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:12:29 -0500 Subject: [PATCH 105/177] chore: remove one-shot fee-field patch workflow --- .../apply-late-favorite-fee-field-fix.yml | 88 ------------------- 1 file changed, 88 deletions(-) delete mode 100644 .github/workflows/apply-late-favorite-fee-field-fix.yml diff --git a/.github/workflows/apply-late-favorite-fee-field-fix.yml b/.github/workflows/apply-late-favorite-fee-field-fix.yml deleted file mode 100644 index 38a4e703..00000000 --- a/.github/workflows/apply-late-favorite-fee-field-fix.yml +++ /dev/null @@ -1,88 +0,0 @@ -name: Apply Late Favorite Fee-Field Fix - -on: - push: - branches: [digital-surface-live-readiness-20260725] - paths: - - .github/workflows/apply-late-favorite-fee-field-fix.yml - pull_request: - branches: [digital-surface-ci-20260724] - paths: - - .github/workflows/apply-late-favorite-fee-field-fix.yml - -permissions: - contents: write - -env: - PYTHONUNBUFFERED: '1' - -jobs: - patch-and-verify: - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - uses: actions/checkout@v4 - with: - ref: digital-surface-live-readiness-20260725 - fetch-depth: 0 - - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Install pinned test stack - run: python -m pip install pandas==2.2.3 huggingface_hub==0.33.4 pytest==8.3.5 - - name: Apply minimal fee-field correction - shell: bash - run: | - python - <<'PY' - from pathlib import Path - - path = Path('ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py') - text = path.read_text() - - anchor = ( - ' result["hypothetical_fok_fill"] = True\n' - ' result["decision"] = "hypothetical_fok_fill"\n' - ' normalized_outcome = str(outcome).title() if outcome is not None else None\n' - ) - replacement = ( - ' result["hypothetical_fok_fill"] = True\n' - ' result["decision"] = "hypothetical_fok_fill"\n' - ' fee = fee_per_share(arrival_ask)\n' - ' result["fee_per_share"] = fee\n' - ' normalized_outcome = str(outcome).title() if outcome is not None else None\n' - ) - if text.count(anchor) != 1: - raise RuntimeError(f'fee insertion anchor count={text.count(anchor)}') - text = text.replace(anchor, replacement) - - old_fee_line = ' fee = fee_per_share(arrival_ask)\n' - if text.count(old_fee_line) != 1: - raise RuntimeError(f'old fee line count={text.count(old_fee_line)}') - text = text.replace(old_fee_line, '', 1) - - old_update_line = ' "fee_per_share": fee,\n' - if text.count(old_update_line) != 1: - raise RuntimeError(f'old update fee line count={text.count(old_update_line)}') - text = text.replace(old_update_line, '', 1) - - path.write_text(text) - PY - - name: Compile and verify regressions - env: - PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-repro-source - run: | - python -m py_compile ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py - pytest -q \ - ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_late_favorite_transfer.py \ - ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_fee_field_regression.py - - name: Commit verified source correction - if: github.event_name == 'push' - shell: bash - run: | - set -euo pipefail - git config user.name 'research-checkpoint-bot' - git config user.email 'research-checkpoint-bot@users.noreply.github.com' - git add ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py - git diff --cached --check - git commit -m 'fix: record fees for unlabeled hypothetical fills' - git push origin HEAD:digital-surface-live-readiness-20260725 From 36de313fa992dda46c073e344b176194e64b0bcf Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:13:56 -0500 Subject: [PATCH 106/177] ci: verify corrected fee-field ordering --- .../late-favorite-fee-field-regression.yml | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/.github/workflows/late-favorite-fee-field-regression.yml b/.github/workflows/late-favorite-fee-field-regression.yml index 8d9e67b8..f5bab4ef 100644 --- a/.github/workflows/late-favorite-fee-field-regression.yml +++ b/.github/workflows/late-favorite-fee-field-regression.yml @@ -6,12 +6,14 @@ on: paths: - .github/workflows/late-favorite-fee-field-regression.yml - ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py + - ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_late_favorite_transfer.py - ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_fee_field_regression.py pull_request: branches: [digital-surface-ci-20260724] paths: - .github/workflows/late-favorite-fee-field-regression.yml - ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py + - ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_late_favorite_transfer.py - ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_fee_field_regression.py workflow_dispatch: @@ -29,12 +31,28 @@ jobs: python-version: '3.11' - name: Install minimal pinned stack run: python -m pip install pandas==2.2.3 huggingface_hub==0.33.4 pytest==8.3.5 - - name: Compile source and regression test + - name: Compile source and regression tests run: | python -m py_compile \ ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py \ + ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_late_favorite_transfer.py \ ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_fee_field_regression.py - - name: Run fee-field regression + - name: Verify fee is recorded before label-dependent PnL + run: | + python - <<'PY' + from pathlib import Path + + text = Path('ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py').read_text() + fee_assignment = ' result["fee_per_share"] = fee\n' + label_branch = ' if normalized_outcome in {"Up", "Down"}:\n' + assert text.count(fee_assignment) == 1 + assert text.count(label_branch) == 1 + assert text.index(fee_assignment) < text.index(label_branch) + PY + - name: Run focused behavior suite env: PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-repro-source - run: pytest -q ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_fee_field_regression.py + run: | + pytest -q \ + ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_late_favorite_transfer.py \ + ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_fee_field_regression.py From 5552f09d0da1df29bb462b2c903b9bf98696544b Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:15:31 -0500 Subject: [PATCH 107/177] test: specify BTC-only v3 prospective observer --- .../tests/test_late_favorite_v3.py | 258 ++++++++++++++++++ 1 file changed, 258 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_late_favorite_v3.py diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_late_favorite_v3.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_late_favorite_v3.py new file mode 100644 index 00000000..94b519a7 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_late_favorite_v3.py @@ -0,0 +1,258 @@ +from __future__ import annotations + +import json +from decimal import Decimal + +import pytest + +from late_favorite_v3 import ( + FreezeViolation, + FrozenPolicy, + LedgerIntegrityError, + LifecycleError, + ProspectiveLifecycleLedger, + evaluate_arrival, + evaluate_signal, + settle_fill, +) + + +def policy() -> FrozenPolicy: + return FrozenPolicy( + policy_sha256="1" * 64, + source_sha256="2" * 64, + valid_market_open_after_epoch_seconds=1_800_000_000, + signal_ask_min=Decimal("0.85"), + entry_second=210, + latency_seconds=1, + shares=Decimal("5"), + adverse_move_cancel=Decimal("0.01"), + fee_rate=Decimal("0.07"), + ) + + +def book(condition_id: str, token_id: str, asks: list[tuple[str, str]]) -> dict[str, object]: + return { + "market": condition_id, + "asset_id": token_id, + "timestamp": "1800000210000", + "hash": f"book-{token_id}", + "bids": [{"price": "0.10", "size": "10"}], + "asks": [{"price": price, "size": size} for price, size in asks], + "min_order_size": "1", + "tick_size": "0.01", + "neg_risk": False, + "last_trade_price": "0.50", + } + + +def test_market_must_open_strictly_after_freeze() -> None: + with pytest.raises(FreezeViolation): + evaluate_signal( + policy(), + condition_id="c1", + market_open_epoch_seconds=1_800_000_000, + up_token_id="up", + down_token_id="down", + up_book=book("c1", "up", [("0.90", "10")]), + down_book=book("c1", "down", [("0.12", "10")]), + observed_ts_ms=1_800_000_210_000, + ) + + +def test_signal_selects_higher_executable_ask_as_favorite() -> None: + result = evaluate_signal( + policy(), + condition_id="c1", + market_open_epoch_seconds=1_800_000_300, + up_token_id="up", + down_token_id="down", + up_book=book("c1", "up", [("0.90", "10")]), + down_book=book("c1", "down", [("0.12", "10")]), + observed_ts_ms=1_800_000_510_000, + ) + assert result["decision"] == "signal" + assert result["selected_side"] == "Up" + assert result["selected_token_id"] == "up" + assert result["signal_ask"] == "0.90" + assert result["signal"] is True + assert len(result["up_book_sha256"]) == 64 + assert len(result["down_book_sha256"]) == 64 + + +def test_tied_or_subthreshold_favorite_fails_closed() -> None: + tied = evaluate_signal( + policy(), + condition_id="c1", + market_open_epoch_seconds=1_800_000_300, + up_token_id="up", + down_token_id="down", + up_book=book("c1", "up", [("0.50", "10")]), + down_book=book("c1", "down", [("0.50", "10")]), + observed_ts_ms=1_800_000_510_000, + ) + assert tied["decision"] == "no_signal_tied_favorite" + + below = evaluate_signal( + policy(), + condition_id="c2", + market_open_epoch_seconds=1_800_000_300, + up_token_id="up2", + down_token_id="down2", + up_book=book("c2", "up2", [("0.84", "10")]), + down_book=book("c2", "down2", [("0.18", "10")]), + observed_ts_ms=1_800_000_510_000, + ) + assert below["decision"] == "no_signal_below_threshold" + + +def test_arrival_cancels_toxic_improvement_and_rejects_price_above_limit() -> None: + signal = evaluate_signal( + policy(), + condition_id="c1", + market_open_epoch_seconds=1_800_000_300, + up_token_id="up", + down_token_id="down", + up_book=book("c1", "up", [("0.90", "10")]), + down_book=book("c1", "down", [("0.12", "10")]), + observed_ts_ms=1_800_000_510_000, + ) + toxic = evaluate_arrival( + policy(), signal, book("c1", "up", [("0.88", "10")]), observed_ts_ms=1_800_000_511_000 + ) + assert toxic["decision"] == "cancel_adverse_move" + + above = evaluate_arrival( + policy(), signal, book("c1", "up", [("0.91", "10")]), observed_ts_ms=1_800_000_511_000 + ) + assert above["decision"] == "no_fill_ask_above_limit" + + +def test_arrival_uses_full_depth_vwap_and_level_fees() -> None: + signal = evaluate_signal( + policy(), + condition_id="c1", + market_open_epoch_seconds=1_800_000_300, + up_token_id="up", + down_token_id="down", + up_book=book("c1", "up", [("0.90", "10")]), + down_book=book("c1", "down", [("0.12", "10")]), + observed_ts_ms=1_800_000_510_000, + ) + fill = evaluate_arrival( + policy(), + signal, + book("c1", "up", [("0.89", "2"), ("0.90", "3"), ("0.91", "100")]), + observed_ts_ms=1_800_000_511_000, + ) + assert fill["decision"] == "hypothetical_fok_fill" + assert fill["filled_shares"] == "5" + assert fill["execution_vwap"] == "0.896" + expected_fee = ( + Decimal("2") * Decimal("0.07") * Decimal("0.89") * Decimal("0.11") + + Decimal("3") * Decimal("0.07") * Decimal("0.90") * Decimal("0.10") + ) / Decimal("5") + assert Decimal(fill["fee_per_share"]) == expected_fee + assert fill["execution_levels"] == [ + {"price": "0.89", "size": "2"}, + {"price": "0.90", "size": "3"}, + ] + + +def test_arrival_requires_exact_latency_and_sufficient_depth() -> None: + signal = evaluate_signal( + policy(), + condition_id="c1", + market_open_epoch_seconds=1_800_000_300, + up_token_id="up", + down_token_id="down", + up_book=book("c1", "up", [("0.90", "10")]), + down_book=book("c1", "down", [("0.12", "10")]), + observed_ts_ms=1_800_000_510_000, + ) + with pytest.raises(LifecycleError): + evaluate_arrival( + policy(), signal, book("c1", "up", [("0.90", "10")]), observed_ts_ms=1_800_000_512_000 + ) + shallow = evaluate_arrival( + policy(), signal, book("c1", "up", [("0.90", "4.99")]), observed_ts_ms=1_800_000_511_000 + ) + assert shallow["decision"] == "no_fill_insufficient_depth" + + +def test_official_settlement_uses_frozen_fill_economics() -> None: + signal = evaluate_signal( + policy(), + condition_id="c1", + market_open_epoch_seconds=1_800_000_300, + up_token_id="up", + down_token_id="down", + up_book=book("c1", "up", [("0.90", "10")]), + down_book=book("c1", "down", [("0.12", "10")]), + observed_ts_ms=1_800_000_510_000, + ) + fill = evaluate_arrival( + policy(), signal, book("c1", "up", [("0.90", "5")]), observed_ts_ms=1_800_000_511_000 + ) + win = settle_fill(fill, official_outcome="Up", resolution_payload_sha256="3" * 64) + loss = settle_fill(fill, official_outcome="Down", resolution_payload_sha256="4" * 64) + assert Decimal(win["pnl_per_share"]) > 0 + assert Decimal(loss["pnl_per_share"]) < Decimal("-0.90") + assert win["official_won"] is True + assert loss["official_won"] is False + + +def test_lifecycle_ledger_is_append_only_and_enforces_order(tmp_path) -> None: + path = tmp_path / "ledger.jsonl" + ledger = ProspectiveLifecycleLedger(path, policy()) + discovered = ledger.append( + condition_id="c1", + market_open_epoch_seconds=1_800_000_300, + event_type="discovered", + observed_ts_ms=1_800_000_301_000, + payload={"slug": "btc-updown-5m-1800000300"}, + ) + assert discovered["previous_hash"] == "0" * 64 + with pytest.raises(LifecycleError): + ledger.append( + condition_id="c2", + market_open_epoch_seconds=1_800_000_300, + event_type="arrival", + observed_ts_ms=1_800_000_511_000, + payload={"decision": "no_fill"}, + ) + ledger.append( + condition_id="c1", + market_open_epoch_seconds=1_800_000_300, + event_type="signal", + observed_ts_ms=1_800_000_510_000, + payload={"decision": "signal"}, + ) + with pytest.raises(LifecycleError): + ledger.append( + condition_id="c1", + market_open_epoch_seconds=1_800_000_300, + event_type="signal", + observed_ts_ms=1_800_000_510_001, + payload={"decision": "signal"}, + ) + reloaded = ProspectiveLifecycleLedger(path, policy()) + assert reloaded.head_hash == ledger.head_hash + assert len(reloaded.records) == 2 + + +def test_lifecycle_ledger_detects_tampering(tmp_path) -> None: + path = tmp_path / "ledger.jsonl" + ledger = ProspectiveLifecycleLedger(path, policy()) + ledger.append( + condition_id="c1", + market_open_epoch_seconds=1_800_000_300, + event_type="discovered", + observed_ts_ms=1_800_000_301_000, + payload={"slug": "btc-updown-5m-1800000300"}, + ) + row = json.loads(path.read_text()) + row["payload"]["slug"] = "tampered" + path.write_text(json.dumps(row) + "\n") + with pytest.raises(LedgerIntegrityError): + ProspectiveLifecycleLedger(path, policy()) From d165f16f8cda6cd4affcc6e5687e9f21a5f57c99 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:15:49 -0500 Subject: [PATCH 108/177] ci: add BTC-only v3 observer contract --- .../btc-only-v3-observer-checkpoint.yml | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .github/workflows/btc-only-v3-observer-checkpoint.yml diff --git a/.github/workflows/btc-only-v3-observer-checkpoint.yml b/.github/workflows/btc-only-v3-observer-checkpoint.yml new file mode 100644 index 00000000..8e10b72b --- /dev/null +++ b/.github/workflows/btc-only-v3-observer-checkpoint.yml @@ -0,0 +1,44 @@ +name: BTC-Only v3 Observer Checkpoint + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/btc-only-v3-observer-checkpoint.yml + - ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/** + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/btc-only-v3-observer-checkpoint.yml + - ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/** + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + observer-contract: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install pinned test stack + run: python -m pip install pytest==8.3.5 + - name: Compile observer source when present + shell: bash + run: | + test -f ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source/late_favorite_v3.py + python -m py_compile \ + ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source/late_favorite_v3.py \ + ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_late_favorite_v3.py + - name: Run observer behavior contract + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source + run: pytest -q ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_late_favorite_v3.py From b243d5a96c5e70bbf9e588439bfa758dfcb6dbb3 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:16:19 -0500 Subject: [PATCH 109/177] ci: expose BTC-only v3 observer contract --- .github/workflows/btc-only-v3-observer-checkpoint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/btc-only-v3-observer-checkpoint.yml b/.github/workflows/btc-only-v3-observer-checkpoint.yml index 8e10b72b..b4d2a4a6 100644 --- a/.github/workflows/btc-only-v3-observer-checkpoint.yml +++ b/.github/workflows/btc-only-v3-observer-checkpoint.yml @@ -31,7 +31,7 @@ jobs: python-version: '3.11' - name: Install pinned test stack run: python -m pip install pytest==8.3.5 - - name: Compile observer source when present + - name: Require and compile observer source shell: bash run: | test -f ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source/late_favorite_v3.py From 7445d56eeef587c7ed9fd3fd67b43592bdfbfe17 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:17:28 -0500 Subject: [PATCH 110/177] ci: isolate BTC-only v3 observer branch --- .github/workflows/btc-only-v3-observer-checkpoint.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/btc-only-v3-observer-checkpoint.yml b/.github/workflows/btc-only-v3-observer-checkpoint.yml index b4d2a4a6..643d7b3c 100644 --- a/.github/workflows/btc-only-v3-observer-checkpoint.yml +++ b/.github/workflows/btc-only-v3-observer-checkpoint.yml @@ -2,12 +2,7 @@ name: BTC-Only v3 Observer Checkpoint on: push: - branches: [digital-surface-live-readiness-20260725] - paths: - - .github/workflows/btc-only-v3-observer-checkpoint.yml - - ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/** - pull_request: - branches: [digital-surface-ci-20260724] + branches: [btc-only-v3-prospective-20260726] paths: - .github/workflows/btc-only-v3-observer-checkpoint.yml - ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/** From f05561901eb7e3b7da9f0916a1ffa0f177f90077 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:17:55 -0500 Subject: [PATCH 111/177] ci: register isolated BTC-only observer workflow --- .github/workflows/btc-only-v3-observer-checkpoint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/btc-only-v3-observer-checkpoint.yml b/.github/workflows/btc-only-v3-observer-checkpoint.yml index 643d7b3c..b31a9015 100644 --- a/.github/workflows/btc-only-v3-observer-checkpoint.yml +++ b/.github/workflows/btc-only-v3-observer-checkpoint.yml @@ -26,7 +26,7 @@ jobs: python-version: '3.11' - name: Install pinned test stack run: python -m pip install pytest==8.3.5 - - name: Require and compile observer source + - name: Require and compile frozen observer source shell: bash run: | test -f ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source/late_favorite_v3.py From fbd5de06d8e6ab7ee93888b76cef0e66e1785d58 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:18:48 -0500 Subject: [PATCH 112/177] ci: expose isolated observer PR checkpoint --- .github/workflows/btc-only-v3-observer-checkpoint.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/btc-only-v3-observer-checkpoint.yml b/.github/workflows/btc-only-v3-observer-checkpoint.yml index b31a9015..93213522 100644 --- a/.github/workflows/btc-only-v3-observer-checkpoint.yml +++ b/.github/workflows/btc-only-v3-observer-checkpoint.yml @@ -6,6 +6,11 @@ on: paths: - .github/workflows/btc-only-v3-observer-checkpoint.yml - ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/** + pull_request: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/btc-only-v3-observer-checkpoint.yml + - ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/** workflow_dispatch: permissions: From c82d8748b7ad2f87b5a5e24a41549fca694cea03 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:19:41 -0500 Subject: [PATCH 113/177] ci: enable isolated BTC-only v3 observer PR --- .github/workflows/btc-only-v3-observer-checkpoint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/btc-only-v3-observer-checkpoint.yml b/.github/workflows/btc-only-v3-observer-checkpoint.yml index b4d2a4a6..72ff032b 100644 --- a/.github/workflows/btc-only-v3-observer-checkpoint.yml +++ b/.github/workflows/btc-only-v3-observer-checkpoint.yml @@ -7,7 +7,7 @@ on: - .github/workflows/btc-only-v3-observer-checkpoint.yml - ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/** pull_request: - branches: [digital-surface-ci-20260724] + branches: [digital-surface-ci-20260724, digital-surface-live-readiness-20260725] paths: - .github/workflows/btc-only-v3-observer-checkpoint.yml - ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/** From a768d008604fff33354ab0ca77746c9b7fd936fc Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:20:07 -0500 Subject: [PATCH 114/177] ci: trigger isolated observer RED checkpoint --- .github/workflows/btc-only-v3-observer-checkpoint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/btc-only-v3-observer-checkpoint.yml b/.github/workflows/btc-only-v3-observer-checkpoint.yml index 93213522..83c45b73 100644 --- a/.github/workflows/btc-only-v3-observer-checkpoint.yml +++ b/.github/workflows/btc-only-v3-observer-checkpoint.yml @@ -31,7 +31,7 @@ jobs: python-version: '3.11' - name: Install pinned test stack run: python -m pip install pytest==8.3.5 - - name: Require and compile frozen observer source + - name: Require observer source before GREEN shell: bash run: | test -f ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source/late_favorite_v3.py From e2518f496f1ca5aeef460f5499ee70be99ebc224 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:20:39 -0500 Subject: [PATCH 115/177] ci: align isolated observer workflow with base --- .github/workflows/btc-only-v3-observer-checkpoint.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/btc-only-v3-observer-checkpoint.yml b/.github/workflows/btc-only-v3-observer-checkpoint.yml index 83c45b73..72ff032b 100644 --- a/.github/workflows/btc-only-v3-observer-checkpoint.yml +++ b/.github/workflows/btc-only-v3-observer-checkpoint.yml @@ -2,12 +2,12 @@ name: BTC-Only v3 Observer Checkpoint on: push: - branches: [btc-only-v3-prospective-20260726] + branches: [digital-surface-live-readiness-20260725] paths: - .github/workflows/btc-only-v3-observer-checkpoint.yml - ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/** pull_request: - branches: [digital-surface-live-readiness-20260725] + branches: [digital-surface-ci-20260724, digital-surface-live-readiness-20260725] paths: - .github/workflows/btc-only-v3-observer-checkpoint.yml - ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/** @@ -31,7 +31,7 @@ jobs: python-version: '3.11' - name: Install pinned test stack run: python -m pip install pytest==8.3.5 - - name: Require observer source before GREEN + - name: Require and compile observer source shell: bash run: | test -f ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source/late_favorite_v3.py From cb34d31f8aa30086a6d3a3cbcf58d5ccc609050f Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:20:51 -0500 Subject: [PATCH 116/177] test: trigger isolated BTC-only observer RED checkpoint --- .../late-favorite-btc-only-v3/RED_CONTRACT_TRIGGER.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/RED_CONTRACT_TRIGGER.txt diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/RED_CONTRACT_TRIGGER.txt b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/RED_CONTRACT_TRIGGER.txt new file mode 100644 index 00000000..0555861e --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/RED_CONTRACT_TRIGGER.txt @@ -0,0 +1 @@ +Observer source intentionally absent at this checkpoint; the isolated contract must fail before implementation. From 7336af3c72b3a4c50baaeb43df086d320ba2178e Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:26:46 -0500 Subject: [PATCH 117/177] research: implement BTC-only v3 observer core --- .../source/late_favorite_v3.py | 485 ++++++++++++++++++ 1 file changed, 485 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source/late_favorite_v3.py diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source/late_favorite_v3.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source/late_favorite_v3.py new file mode 100644 index 00000000..5a233a0c --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source/late_favorite_v3.py @@ -0,0 +1,485 @@ +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from decimal import Decimal, InvalidOperation +from pathlib import Path +from typing import Any, Mapping, Sequence + + +class FreezeViolation(RuntimeError): + pass + + +class LifecycleError(RuntimeError): + pass + + +class LedgerIntegrityError(RuntimeError): + pass + + +def _validate_sha256(name: str, value: str) -> str: + text = str(value).lower() + if len(text) != 64 or any(character not in "0123456789abcdef" for character in text): + raise ValueError(f"{name} must be a 64-character SHA-256 hex digest") + return text + + +def _canonical_json(value: object) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _decimal(value: object, name: str) -> Decimal: + try: + number = Decimal(str(value)) + except (InvalidOperation, ValueError) as exc: + raise ValueError(f"{name} must be a decimal") from exc + if not number.is_finite(): + raise ValueError(f"{name} must be finite") + return number + + +def _decimal_text(value: Decimal) -> str: + return format(value, "f") + + +@dataclass(frozen=True, slots=True) +class FrozenPolicy: + policy_sha256: str + source_sha256: str + valid_market_open_after_epoch_seconds: int + signal_ask_min: Decimal + entry_second: int + latency_seconds: int + shares: Decimal + adverse_move_cancel: Decimal + fee_rate: Decimal + + def __post_init__(self) -> None: + object.__setattr__(self, "policy_sha256", _validate_sha256("policy_sha256", self.policy_sha256)) + object.__setattr__(self, "source_sha256", _validate_sha256("source_sha256", self.source_sha256)) + if int(self.valid_market_open_after_epoch_seconds) < 0: + raise ValueError("valid_market_open_after_epoch_seconds must be non-negative") + if int(self.entry_second) < 0: + raise ValueError("entry_second must be non-negative") + if int(self.latency_seconds) <= 0: + raise ValueError("latency_seconds must be positive") + for name in ("signal_ask_min", "shares", "adverse_move_cancel", "fee_rate"): + value = getattr(self, name) + if not isinstance(value, Decimal) or not value.is_finite() or value <= 0: + raise ValueError(f"{name} must be a positive finite Decimal") + if not Decimal("0") < self.signal_ask_min < Decimal("1"): + raise ValueError("signal_ask_min must be between zero and one") + + +def load_frozen_policy( + candidate_spec_path: str | Path, + freeze_manifest_path: str | Path, + *, + source_sha256: str, +) -> FrozenPolicy: + from datetime import datetime + + spec_path = Path(candidate_spec_path) + manifest_path = Path(freeze_manifest_path) + spec_raw = spec_path.read_bytes() + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + spec = json.loads(spec_raw) + got = hashlib.sha256(spec_raw).hexdigest() + wanted = _validate_sha256("manifest policy_sha256", str(manifest.get("policy_sha256", ""))) + if got != wanted: + raise FreezeViolation(f"candidate policy SHA-256 mismatch: {got} != {wanted}") + if spec.get("name") != manifest.get("candidate_name"): + raise FreezeViolation("candidate name does not match freeze manifest") + if spec.get("status") != "prospective_shadow_only": + raise FreezeViolation("candidate is not prospective_shadow_only") + if spec.get("observation_contract", {}).get("live_submission") != "physically_absent": + raise FreezeViolation("live submission boundary is not physically absent") + freeze_text = str(manifest["contamination_boundary"]["valid_market_open_after_utc"]) + freeze_epoch = int(datetime.fromisoformat(freeze_text.replace("Z", "+00:00")).timestamp()) + return FrozenPolicy( + policy_sha256=got, + source_sha256=source_sha256, + valid_market_open_after_epoch_seconds=freeze_epoch, + signal_ask_min=_decimal(spec["signal"]["signal_ask_min"], "signal_ask_min"), + entry_second=int(spec["signal"]["entry_second"]), + latency_seconds=int(spec["arrival"]["latency_seconds"]), + shares=_decimal(spec["arrival"]["shares"], "shares"), + adverse_move_cancel=Decimal("0.01"), + fee_rate=Decimal("0.07"), + ) + + +def _normalized_book( + payload: Mapping[str, Any], + *, + expected_condition_id: str, + expected_token_id: str, +) -> dict[str, Any]: + condition_id = str(payload.get("market") or "") + token_id = str(payload.get("asset_id") or "") + if condition_id != expected_condition_id: + raise LifecycleError(f"book condition mismatch: {condition_id} != {expected_condition_id}") + if token_id != expected_token_id: + raise LifecycleError(f"book token mismatch: {token_id} != {expected_token_id}") + raw_asks = payload.get("asks", []) + if not isinstance(raw_asks, Sequence) or isinstance(raw_asks, (str, bytes)): + raise LifecycleError("book asks must be a sequence") + asks: list[dict[str, str]] = [] + for index, row in enumerate(raw_asks): + if not isinstance(row, Mapping): + raise LifecycleError(f"ask level {index} is not an object") + price = _decimal(row.get("price"), f"ask[{index}].price") + size = _decimal(row.get("size"), f"ask[{index}].size") + if not Decimal("0") < price < Decimal("1"): + raise LifecycleError("ask price must be strictly between zero and one") + if size < 0: + raise LifecycleError("ask size must be non-negative") + if size > 0: + asks.append({"price": _decimal_text(price), "size": _decimal_text(size)}) + asks.sort(key=lambda row: Decimal(row["price"])) + normalized = { + "market": condition_id, + "asset_id": token_id, + "timestamp": str(payload.get("timestamp") or ""), + "hash": str(payload.get("hash") or ""), + "asks": asks, + "bids": payload.get("bids", []), + "min_order_size": str(payload.get("min_order_size") or "0"), + "tick_size": str(payload.get("tick_size") or "0.01"), + "neg_risk": bool(payload.get("neg_risk", False)), + "last_trade_price": None if payload.get("last_trade_price") in (None, "") else str(payload.get("last_trade_price")), + } + normalized["payload_sha256"] = hashlib.sha256(_canonical_json(dict(payload))).hexdigest() + return normalized + + +def _best_ask(book: Mapping[str, Any]) -> Decimal | None: + asks = book.get("asks", []) + if not asks: + return None + return Decimal(str(asks[0]["price"])) + + +def _assert_post_freeze(policy: FrozenPolicy, market_open_epoch_seconds: int) -> int: + opening = int(market_open_epoch_seconds) + if opening <= int(policy.valid_market_open_after_epoch_seconds): + raise FreezeViolation( + f"market open {opening} is not strictly after freeze {policy.valid_market_open_after_epoch_seconds}" + ) + return opening + + +def evaluate_signal( + policy: FrozenPolicy, + *, + condition_id: str, + market_open_epoch_seconds: int, + up_token_id: str, + down_token_id: str, + up_book: Mapping[str, Any], + down_book: Mapping[str, Any], + observed_ts_ms: int, +) -> dict[str, Any]: + opening = _assert_post_freeze(policy, market_open_epoch_seconds) + expected_ts_ms = (opening + int(policy.entry_second)) * 1000 + observed = int(observed_ts_ms) + if observed != expected_ts_ms: + raise LifecycleError(f"signal observation must be at {expected_ts_ms}, got {observed}") + condition = str(condition_id) + if not condition: + raise ValueError("condition_id must be non-empty") + up = _normalized_book(up_book, expected_condition_id=condition, expected_token_id=str(up_token_id)) + down = _normalized_book(down_book, expected_condition_id=condition, expected_token_id=str(down_token_id)) + up_ask = _best_ask(up) + down_ask = _best_ask(down) + base = { + "event_type": "signal", + "condition_id": condition, + "market_open_epoch_seconds": opening, + "observed_ts_ms": observed, + "policy_sha256": policy.policy_sha256, + "source_sha256": policy.source_sha256, + "up_token_id": str(up_token_id), + "down_token_id": str(down_token_id), + "up_book": up, + "down_book": down, + "up_book_sha256": up["payload_sha256"], + "down_book_sha256": down["payload_sha256"], + "signal": False, + "selected_side": None, + "selected_token_id": None, + "signal_ask": None, + } + if up_ask is None or down_ask is None: + return {**base, "decision": "no_signal_empty_book"} + if up_ask == down_ask: + return {**base, "decision": "no_signal_tied_favorite"} + selected_side = "Up" if up_ask > down_ask else "Down" + selected_token_id = str(up_token_id) if selected_side == "Up" else str(down_token_id) + signal_ask = up_ask if selected_side == "Up" else down_ask + selected = { + **base, + "selected_side": selected_side, + "selected_token_id": selected_token_id, + "signal_ask": _decimal_text(signal_ask), + } + if signal_ask < policy.signal_ask_min: + return {**selected, "decision": "no_signal_below_threshold"} + return {**selected, "decision": "signal", "signal": True} + + +def _fee_per_share(policy: FrozenPolicy, price: Decimal) -> Decimal: + return policy.fee_rate * price * (Decimal("1") - price) + + +def evaluate_arrival( + policy: FrozenPolicy, + signal_record: Mapping[str, Any], + selected_book: Mapping[str, Any], + *, + observed_ts_ms: int, +) -> dict[str, Any]: + if signal_record.get("decision") != "signal" or signal_record.get("signal") is not True: + raise LifecycleError("arrival requires an active signal") + if str(signal_record.get("policy_sha256")) != policy.policy_sha256: + raise LifecycleError("signal policy hash mismatch") + expected = int(signal_record["observed_ts_ms"]) + int(policy.latency_seconds) * 1000 + observed = int(observed_ts_ms) + if observed != expected: + raise LifecycleError(f"arrival observation must be at {expected}, got {observed}") + condition = str(signal_record["condition_id"]) + token_id = str(signal_record["selected_token_id"]) + book = _normalized_book(selected_book, expected_condition_id=condition, expected_token_id=token_id) + best_ask = _best_ask(book) + base = { + "event_type": "arrival", + "condition_id": condition, + "market_open_epoch_seconds": int(signal_record["market_open_epoch_seconds"]), + "observed_ts_ms": observed, + "policy_sha256": policy.policy_sha256, + "source_sha256": policy.source_sha256, + "selected_side": str(signal_record["selected_side"]), + "selected_token_id": token_id, + "signal_ask": str(signal_record["signal_ask"]), + "arrival_book": book, + "arrival_book_sha256": book["payload_sha256"], + "arrival_best_ask": None if best_ask is None else _decimal_text(best_ask), + "hypothetical_fok_fill": False, + "filled_shares": "0", + "execution_levels": [], + "execution_vwap": None, + "fee_per_share": None, + "all_in_cost_per_share": None, + } + if best_ask is None: + return {**base, "decision": "no_fill_empty_arrival_book"} + signal_ask = Decimal(str(signal_record["signal_ask"])) + if best_ask < signal_ask - policy.adverse_move_cancel: + return {**base, "decision": "cancel_adverse_move"} + if best_ask > signal_ask: + return {**base, "decision": "no_fill_ask_above_limit"} + + remaining = policy.shares + notional = Decimal("0") + fee_total = Decimal("0") + execution_levels: list[dict[str, str]] = [] + for row in book["asks"]: + price = Decimal(row["price"]) + if price > signal_ask: + break + available = Decimal(row["size"]) + take = min(remaining, available) + if take <= 0: + continue + execution_levels.append({"price": _decimal_text(price), "size": _decimal_text(take)}) + notional += price * take + fee_total += _fee_per_share(policy, price) * take + remaining -= take + if remaining == 0: + break + if remaining > 0: + return {**base, "decision": "no_fill_insufficient_depth"} + vwap = notional / policy.shares + fee = fee_total / policy.shares + return { + **base, + "decision": "hypothetical_fok_fill", + "hypothetical_fok_fill": True, + "filled_shares": _decimal_text(policy.shares), + "execution_levels": execution_levels, + "execution_vwap": _decimal_text(vwap), + "fee_per_share": _decimal_text(fee), + "all_in_cost_per_share": _decimal_text(vwap + fee), + } + + +def settle_fill( + fill_record: Mapping[str, Any], + *, + official_outcome: str, + resolution_payload_sha256: str, +) -> dict[str, Any]: + if fill_record.get("decision") != "hypothetical_fok_fill": + raise LifecycleError("settlement requires a hypothetical FOK fill") + outcome = str(official_outcome).title() + if outcome not in {"Up", "Down"}: + raise ValueError("official_outcome must be Up or Down") + resolution_sha = _validate_sha256("resolution_payload_sha256", resolution_payload_sha256) + selected_side = str(fill_record["selected_side"]) + won = selected_side == outcome + vwap = Decimal(str(fill_record["execution_vwap"])) + fee = Decimal(str(fill_record["fee_per_share"])) + shares = Decimal(str(fill_record["filled_shares"])) + pnl_per_share = Decimal(int(won)) - vwap - fee + return { + "event_type": "resolution", + "condition_id": str(fill_record["condition_id"]), + "market_open_epoch_seconds": int(fill_record["market_open_epoch_seconds"]), + "policy_sha256": str(fill_record["policy_sha256"]), + "source_sha256": str(fill_record["source_sha256"]), + "selected_side": selected_side, + "official_outcome": outcome, + "official_won": won, + "resolution_payload_sha256": resolution_sha, + "execution_vwap": _decimal_text(vwap), + "fee_per_share": _decimal_text(fee), + "pnl_per_share": _decimal_text(pnl_per_share), + "pnl_total": _decimal_text(pnl_per_share * shares), + } + + +_ALLOWED_EVENT_TYPES = {"discovered", "signal", "arrival", "resolution"} + + +class ProspectiveLifecycleLedger: + def __init__(self, path: str | Path, policy: FrozenPolicy) -> None: + self.path = Path(path) + self.policy = policy + self.records: list[dict[str, Any]] = [] + self.head_hash = "0" * 64 + self._events: dict[str, list[str]] = {} + self._payloads: dict[str, dict[str, Mapping[str, Any]]] = {} + if self.path.exists(): + self._load_and_verify() + + def _validate_order(self, condition_id: str, event_type: str, payload: Mapping[str, Any]) -> None: + sequence = self._events.get(condition_id, []) + if event_type in sequence: + raise LifecycleError(f"duplicate {event_type} event for {condition_id}") + if event_type == "discovered": + if sequence: + raise LifecycleError("discovered must be the first event") + return + if not sequence or sequence[0] != "discovered": + raise LifecycleError(f"{event_type} requires a discovered event") + if event_type == "signal": + if sequence != ["discovered"]: + raise LifecycleError("signal must follow discovered") + return + signal_payload = self._payloads.get(condition_id, {}).get("signal", {}) + if event_type == "arrival": + if sequence != ["discovered", "signal"]: + raise LifecycleError("arrival must follow signal") + if signal_payload.get("decision") != "signal": + raise LifecycleError("arrival is invalid after a no-signal decision") + return + if event_type == "resolution": + if signal_payload.get("decision") == "signal": + if "arrival" not in sequence: + raise LifecycleError("resolution requires arrival after an active signal") + elif sequence != ["discovered", "signal"]: + raise LifecycleError("no-signal resolution must follow signal") + return + raise LifecycleError(f"unsupported event_type: {event_type}") + + def _accept_record(self, record: Mapping[str, Any]) -> None: + condition = str(record["condition_id"]) + event_type = str(record["event_type"]) + payload = record["payload"] + self._validate_order(condition, event_type, payload) + self._events.setdefault(condition, []).append(event_type) + self._payloads.setdefault(condition, {})[event_type] = payload + self.records.append(dict(record)) + self.head_hash = str(record["record_hash"]) + + def _load_and_verify(self) -> None: + previous = "0" * 64 + for line_number, line in enumerate(self.path.read_text(encoding="utf-8").splitlines(), start=1): + if not line.strip(): + continue + try: + record = json.loads(line) + except json.JSONDecodeError as exc: + raise LedgerIntegrityError(f"invalid JSON at line {line_number}") from exc + stored = str(record.get("record_hash", "")) + payload_for_hash = dict(record) + payload_for_hash.pop("record_hash", None) + if payload_for_hash.get("previous_hash") != previous: + raise LedgerIntegrityError(f"hash chain mismatch at line {line_number}") + expected = hashlib.sha256(_canonical_json(payload_for_hash)).hexdigest() + if stored != expected: + raise LedgerIntegrityError(f"record hash mismatch at line {line_number}") + if record.get("policy_sha256") != self.policy.policy_sha256: + raise LedgerIntegrityError(f"policy hash mismatch at line {line_number}") + if record.get("source_sha256") != self.policy.source_sha256: + raise LedgerIntegrityError(f"source hash mismatch at line {line_number}") + try: + _assert_post_freeze(self.policy, int(record["market_open_epoch_seconds"])) + self._accept_record(record) + except (FreezeViolation, LifecycleError, KeyError, TypeError, ValueError) as exc: + raise LedgerIntegrityError(f"invalid lifecycle record at line {line_number}") from exc + previous = stored + + def append( + self, + *, + condition_id: str, + market_open_epoch_seconds: int, + event_type: str, + observed_ts_ms: int, + payload: Mapping[str, Any], + ) -> dict[str, Any]: + condition = str(condition_id) + if not condition: + raise ValueError("condition_id must be non-empty") + opening = _assert_post_freeze(self.policy, market_open_epoch_seconds) + event = str(event_type) + if event not in _ALLOWED_EVENT_TYPES: + raise LifecycleError(f"unsupported event_type: {event}") + observed = int(observed_ts_ms) + if observed < 0: + raise ValueError("observed_ts_ms must be non-negative") + try: + canonical_payload = json.loads(_canonical_json(dict(payload)).decode("utf-8")) + except (TypeError, ValueError) as exc: + raise ValueError("payload must be JSON serializable") from exc + self._validate_order(condition, event, canonical_payload) + payload_sha256 = hashlib.sha256(_canonical_json(canonical_payload)).hexdigest() + event_id = hashlib.sha256( + f"{condition}\0{event}\0{observed}\0{payload_sha256}".encode("utf-8") + ).hexdigest() + record_without_hash = { + "condition_id": condition, + "event_id": event_id, + "event_type": event, + "market_open_epoch_seconds": opening, + "observed_ts_ms": observed, + "payload": canonical_payload, + "payload_sha256": payload_sha256, + "policy_sha256": self.policy.policy_sha256, + "previous_hash": self.head_hash, + "source_sha256": self.policy.source_sha256, + } + record = { + **record_without_hash, + "record_hash": hashlib.sha256(_canonical_json(record_without_hash)).hexdigest(), + } + self.path.parent.mkdir(parents=True, exist_ok=True) + with self.path.open("a", encoding="utf-8", newline="\n") as handle: + handle.write(json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n") + handle.flush() + self._accept_record(record) + return dict(record) From ac6a1a267c958ac8fd327b11b7f72719627bd459 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:35:21 -0500 Subject: [PATCH 118/177] research: freeze BTC-only v3 capture policy --- .../CAPTURE_POLICY.json | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/CAPTURE_POLICY.json diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/CAPTURE_POLICY.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/CAPTURE_POLICY.json new file mode 100644 index 00000000..9b089d3d --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/CAPTURE_POLICY.json @@ -0,0 +1,36 @@ +{ + "book_freshness": { + "max_book_age_at_request_completion_ms": 2000, + "max_book_future_skew_ms": 1000, + "missing_or_unparseable_book_timestamp": "fail_closed" + }, + "event_time": { + "arrival_target_basis": "market_open_not_signal_completion", + "arrival_target_offset_ms": 211000, + "signal_target_offset_ms": 210000 + }, + "name": "late_favorite_btc_only_v3_capture_policy", + "recording": { + "book_timestamp_ms": true, + "full_book": true, + "missed_window_decision": "record_and_fail_closed", + "raw_payload_sha256": true, + "request_completed_ts_ms": true, + "request_started_ts_ms": true, + "target_ts_ms": true + }, + "request_windows": { + "max_request_duration_ms": 1000, + "max_request_start_lateness_ms": 250, + "request_may_start_before_target": false + }, + "status": "frozen_before_prospective_collection", + "transport": { + "arrival_request": "single public CLOB /books request for selected token ID", + "authenticated_requests": 0, + "credentials_used": 0, + "order_submissions": 0, + "signal_request": "single public CLOB /books batch for Up and Down token IDs" + }, + "version": 1 +} From a7ccb908a00faada3c00cf01ffd8969527e658f1 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:35:51 -0500 Subject: [PATCH 119/177] research: bind BTC-only v3 capture freeze --- .../CAPTURE_FREEZE_MANIFEST.json | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/CAPTURE_FREEZE_MANIFEST.json diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/CAPTURE_FREEZE_MANIFEST.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/CAPTURE_FREEZE_MANIFEST.json new file mode 100644 index 00000000..8f1ddc5b --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/CAPTURE_FREEZE_MANIFEST.json @@ -0,0 +1,14 @@ +{ + "candidate_policy_sha256": "82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba", + "capture_policy_blob_sha": "9b089d3d0f82b3d7baa499aed897a07fdbdc1f7d", + "capture_policy_commit_sha": "ac6a1a267c958ac8fd327b11b7f72719627bd459", + "capture_policy_path": "ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/CAPTURE_POLICY.json", + "capture_policy_sha256": "888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc", + "frozen_at_utc": "2026-07-26T19:35:21Z", + "historical_admission_credit": 0, + "live_submission": "physically_absent", + "rule": "Only markets opening strictly after this capture-policy freeze may count toward prospective admission.", + "status": "frozen_before_prospective_collection", + "valid_market_open_after_utc": "2026-07-26T19:35:21Z", + "version": 1 +} From 92f5f9f10f161359227af4ee9d055601f5cc802b Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:36:48 -0500 Subject: [PATCH 120/177] test: specify bounded BTC-only capture semantics --- .../tests/test_capture_policy.py | 183 ++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_capture_policy.py diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_capture_policy.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_capture_policy.py new file mode 100644 index 00000000..35c10ce6 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_capture_policy.py @@ -0,0 +1,183 @@ +from __future__ import annotations + +from decimal import Decimal + +import pytest + +from late_favorite_v3 import ( + CapturePolicy, + CaptureWindowError, + FrozenPolicy, + ProspectiveLifecycleLedger, + evaluate_arrival_capture, + evaluate_signal_capture, +) + + +def trading_policy() -> FrozenPolicy: + return FrozenPolicy( + policy_sha256="1" * 64, + source_sha256="2" * 64, + valid_market_open_after_epoch_seconds=1_800_000_000, + signal_ask_min=Decimal("0.85"), + entry_second=210, + latency_seconds=1, + shares=Decimal("5"), + adverse_move_cancel=Decimal("0.01"), + fee_rate=Decimal("0.07"), + ) + + +def capture_policy() -> CapturePolicy: + return CapturePolicy( + capture_policy_sha256="3" * 64, + valid_market_open_after_epoch_seconds=1_800_000_100, + signal_target_offset_ms=210_000, + arrival_target_offset_ms=211_000, + max_request_start_lateness_ms=250, + max_request_duration_ms=1_000, + max_book_age_at_request_completion_ms=2_000, + max_book_future_skew_ms=1_000, + ) + + +def book(condition_id: str, token_id: str, ask: str, *, timestamp_ms: int) -> dict[str, object]: + return { + "market": condition_id, + "asset_id": token_id, + "timestamp": str(timestamp_ms), + "hash": f"book-{token_id}-{timestamp_ms}", + "bids": [{"price": "0.10", "size": "10"}], + "asks": [{"price": ask, "size": "10"}], + "min_order_size": "1", + "tick_size": "0.01", + "neg_risk": False, + "last_trade_price": "0.50", + } + + +def test_signal_capture_rejects_pre_target_request() -> None: + opening = 1_800_000_300 + target = opening * 1000 + 210_000 + with pytest.raises(CaptureWindowError): + evaluate_signal_capture( + trading_policy(), + capture_policy(), + condition_id="c1", + market_open_epoch_seconds=opening, + up_token_id="up", + down_token_id="down", + up_book=book("c1", "up", "0.90", timestamp_ms=target), + down_book=book("c1", "down", "0.12", timestamp_ms=target), + request_started_ts_ms=target - 1, + request_completed_ts_ms=target + 20, + response_payload_sha256="4" * 64, + ) + + +def test_signal_capture_records_late_and_stale_fail_closed() -> None: + opening = 1_800_000_300 + target = opening * 1000 + 210_000 + late = evaluate_signal_capture( + trading_policy(), + capture_policy(), + condition_id="c1", + market_open_epoch_seconds=opening, + up_token_id="up", + down_token_id="down", + up_book=book("c1", "up", "0.90", timestamp_ms=target), + down_book=book("c1", "down", "0.12", timestamp_ms=target), + request_started_ts_ms=target + 251, + request_completed_ts_ms=target + 300, + response_payload_sha256="4" * 64, + ) + assert late["decision"] == "missed_signal_window" + assert late["signal"] is False + + stale = evaluate_signal_capture( + trading_policy(), + capture_policy(), + condition_id="c1", + market_open_epoch_seconds=opening, + up_token_id="up", + down_token_id="down", + up_book=book("c1", "up", "0.90", timestamp_ms=target - 2_001), + down_book=book("c1", "down", "0.12", timestamp_ms=target), + request_started_ts_ms=target, + request_completed_ts_ms=target + 100, + response_payload_sha256="4" * 64, + ) + assert stale["decision"] == "no_signal_stale_book" + assert stale["signal"] is False + + +def test_valid_signal_capture_preserves_target_and_transport_evidence() -> None: + opening = 1_800_000_300 + target = opening * 1000 + 210_000 + result = evaluate_signal_capture( + trading_policy(), + capture_policy(), + condition_id="c1", + market_open_epoch_seconds=opening, + up_token_id="up", + down_token_id="down", + up_book=book("c1", "up", "0.90", timestamp_ms=target + 10), + down_book=book("c1", "down", "0.12", timestamp_ms=target + 10), + request_started_ts_ms=target + 25, + request_completed_ts_ms=target + 100, + response_payload_sha256="4" * 64, + ) + assert result["decision"] == "signal" + assert result["observed_ts_ms"] == target + assert result["target_ts_ms"] == target + assert result["request_started_ts_ms"] == target + 25 + assert result["request_completed_ts_ms"] == target + 100 + assert result["capture_policy_sha256"] == "3" * 64 + assert result["response_payload_sha256"] == "4" * 64 + + +def test_arrival_target_is_market_open_plus_211_seconds() -> None: + opening = 1_800_000_300 + signal_target = opening * 1000 + 210_000 + arrival_target = opening * 1000 + 211_000 + signal = evaluate_signal_capture( + trading_policy(), + capture_policy(), + condition_id="c1", + market_open_epoch_seconds=opening, + up_token_id="up", + down_token_id="down", + up_book=book("c1", "up", "0.90", timestamp_ms=signal_target), + down_book=book("c1", "down", "0.12", timestamp_ms=signal_target), + request_started_ts_ms=signal_target + 200, + request_completed_ts_ms=signal_target + 900, + response_payload_sha256="4" * 64, + ) + arrival = evaluate_arrival_capture( + trading_policy(), + capture_policy(), + signal, + book("c1", "up", "0.90", timestamp_ms=arrival_target), + request_started_ts_ms=arrival_target, + request_completed_ts_ms=arrival_target + 100, + response_payload_sha256="5" * 64, + ) + assert arrival["decision"] == "hypothetical_fok_fill" + assert arrival["target_ts_ms"] == arrival_target + assert arrival["observed_ts_ms"] == arrival_target + + +def test_lifecycle_record_binds_capture_policy_hash(tmp_path) -> None: + ledger = ProspectiveLifecycleLedger( + tmp_path / "ledger.jsonl", + trading_policy(), + capture_policy=capture_policy(), + ) + row = ledger.append( + condition_id="c1", + market_open_epoch_seconds=1_800_000_300, + event_type="discovered", + observed_ts_ms=1_800_000_301_000, + payload={"slug": "btc-updown-5m-1800000300"}, + ) + assert row["capture_policy_sha256"] == "3" * 64 From 604cc4665afa8d27d472e4e59200c28617ebf026 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:41:22 -0500 Subject: [PATCH 121/177] test: isolate capture-policy API boundary --- .../tests/test_capture_policy.py | 85 ++++++------------- 1 file changed, 28 insertions(+), 57 deletions(-) diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_capture_policy.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_capture_policy.py index 35c10ce6..af061a05 100644 --- a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_capture_policy.py +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_capture_policy.py @@ -4,14 +4,14 @@ import pytest -from late_favorite_v3 import ( +from capture_policy import ( + CaptureBoundLifecycleLedger, CapturePolicy, CaptureWindowError, - FrozenPolicy, - ProspectiveLifecycleLedger, evaluate_arrival_capture, evaluate_signal_capture, ) +from late_favorite_v3 import FrozenPolicy def trading_policy() -> FrozenPolicy: @@ -61,16 +61,11 @@ def test_signal_capture_rejects_pre_target_request() -> None: target = opening * 1000 + 210_000 with pytest.raises(CaptureWindowError): evaluate_signal_capture( - trading_policy(), - capture_policy(), - condition_id="c1", - market_open_epoch_seconds=opening, - up_token_id="up", - down_token_id="down", + trading_policy(), capture_policy(), condition_id="c1", + market_open_epoch_seconds=opening, up_token_id="up", down_token_id="down", up_book=book("c1", "up", "0.90", timestamp_ms=target), down_book=book("c1", "down", "0.12", timestamp_ms=target), - request_started_ts_ms=target - 1, - request_completed_ts_ms=target + 20, + request_started_ts_ms=target - 1, request_completed_ts_ms=target + 20, response_payload_sha256="4" * 64, ) @@ -79,32 +74,21 @@ def test_signal_capture_records_late_and_stale_fail_closed() -> None: opening = 1_800_000_300 target = opening * 1000 + 210_000 late = evaluate_signal_capture( - trading_policy(), - capture_policy(), - condition_id="c1", - market_open_epoch_seconds=opening, - up_token_id="up", - down_token_id="down", + trading_policy(), capture_policy(), condition_id="c1", + market_open_epoch_seconds=opening, up_token_id="up", down_token_id="down", up_book=book("c1", "up", "0.90", timestamp_ms=target), down_book=book("c1", "down", "0.12", timestamp_ms=target), - request_started_ts_ms=target + 251, - request_completed_ts_ms=target + 300, + request_started_ts_ms=target + 251, request_completed_ts_ms=target + 300, response_payload_sha256="4" * 64, ) assert late["decision"] == "missed_signal_window" assert late["signal"] is False - stale = evaluate_signal_capture( - trading_policy(), - capture_policy(), - condition_id="c1", - market_open_epoch_seconds=opening, - up_token_id="up", - down_token_id="down", + trading_policy(), capture_policy(), condition_id="c1", + market_open_epoch_seconds=opening, up_token_id="up", down_token_id="down", up_book=book("c1", "up", "0.90", timestamp_ms=target - 2_001), down_book=book("c1", "down", "0.12", timestamp_ms=target), - request_started_ts_ms=target, - request_completed_ts_ms=target + 100, + request_started_ts_ms=target, request_completed_ts_ms=target + 100, response_payload_sha256="4" * 64, ) assert stale["decision"] == "no_signal_stale_book" @@ -115,16 +99,11 @@ def test_valid_signal_capture_preserves_target_and_transport_evidence() -> None: opening = 1_800_000_300 target = opening * 1000 + 210_000 result = evaluate_signal_capture( - trading_policy(), - capture_policy(), - condition_id="c1", - market_open_epoch_seconds=opening, - up_token_id="up", - down_token_id="down", + trading_policy(), capture_policy(), condition_id="c1", + market_open_epoch_seconds=opening, up_token_id="up", down_token_id="down", up_book=book("c1", "up", "0.90", timestamp_ms=target + 10), down_book=book("c1", "down", "0.12", timestamp_ms=target + 10), - request_started_ts_ms=target + 25, - request_completed_ts_ms=target + 100, + request_started_ts_ms=target + 25, request_completed_ts_ms=target + 100, response_payload_sha256="4" * 64, ) assert result["decision"] == "signal" @@ -141,25 +120,17 @@ def test_arrival_target_is_market_open_plus_211_seconds() -> None: signal_target = opening * 1000 + 210_000 arrival_target = opening * 1000 + 211_000 signal = evaluate_signal_capture( - trading_policy(), - capture_policy(), - condition_id="c1", - market_open_epoch_seconds=opening, - up_token_id="up", - down_token_id="down", + trading_policy(), capture_policy(), condition_id="c1", + market_open_epoch_seconds=opening, up_token_id="up", down_token_id="down", up_book=book("c1", "up", "0.90", timestamp_ms=signal_target), down_book=book("c1", "down", "0.12", timestamp_ms=signal_target), - request_started_ts_ms=signal_target + 200, - request_completed_ts_ms=signal_target + 900, + request_started_ts_ms=signal_target + 200, request_completed_ts_ms=signal_target + 900, response_payload_sha256="4" * 64, ) arrival = evaluate_arrival_capture( - trading_policy(), - capture_policy(), - signal, + trading_policy(), capture_policy(), signal, book("c1", "up", "0.90", timestamp_ms=arrival_target), - request_started_ts_ms=arrival_target, - request_completed_ts_ms=arrival_target + 100, + request_started_ts_ms=arrival_target, request_completed_ts_ms=arrival_target + 100, response_payload_sha256="5" * 64, ) assert arrival["decision"] == "hypothetical_fok_fill" @@ -168,16 +139,16 @@ def test_arrival_target_is_market_open_plus_211_seconds() -> None: def test_lifecycle_record_binds_capture_policy_hash(tmp_path) -> None: - ledger = ProspectiveLifecycleLedger( - tmp_path / "ledger.jsonl", - trading_policy(), - capture_policy=capture_policy(), + ledger = CaptureBoundLifecycleLedger( + tmp_path / "ledger.jsonl", trading_policy(), capture_policy() ) row = ledger.append( - condition_id="c1", - market_open_epoch_seconds=1_800_000_300, - event_type="discovered", - observed_ts_ms=1_800_000_301_000, + condition_id="c1", market_open_epoch_seconds=1_800_000_300, + event_type="discovered", observed_ts_ms=1_800_000_301_000, payload={"slug": "btc-updown-5m-1800000300"}, ) assert row["capture_policy_sha256"] == "3" * 64 + reloaded = CaptureBoundLifecycleLedger( + tmp_path / "ledger.jsonl", trading_policy(), capture_policy() + ) + assert reloaded.head_hash == ledger.head_hash From 93e64f2b7608144fb6a420b5fede8c43d7941216 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:44:02 -0500 Subject: [PATCH 122/177] research: implement BTC-only v3 capture policy --- .../source/capture_policy.py | 372 ++++++++++++++++++ 1 file changed, 372 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source/capture_policy.py diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source/capture_policy.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source/capture_policy.py new file mode 100644 index 00000000..baf4417a --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source/capture_policy.py @@ -0,0 +1,372 @@ +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Any, Mapping, Sequence + +from late_favorite_v3 import ( + FreezeViolation, + FrozenPolicy, + LedgerIntegrityError, + LifecycleError, + ProspectiveLifecycleLedger, + evaluate_arrival, + evaluate_signal, +) + + +class CaptureWindowError(LifecycleError): + pass + + +def _validate_sha256(name: str, value: str) -> str: + text = str(value).lower() + if len(text) != 64 or any(character not in "0123456789abcdef" for character in text): + raise ValueError(f"{name} must be a 64-character SHA-256 hex digest") + return text + + +def _canonical_json(value: object) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +@dataclass(frozen=True, slots=True) +class CapturePolicy: + capture_policy_sha256: str + valid_market_open_after_epoch_seconds: int + signal_target_offset_ms: int + arrival_target_offset_ms: int + max_request_start_lateness_ms: int + max_request_duration_ms: int + max_book_age_at_request_completion_ms: int + max_book_future_skew_ms: int + + def __post_init__(self) -> None: + object.__setattr__( + self, + "capture_policy_sha256", + _validate_sha256("capture_policy_sha256", self.capture_policy_sha256), + ) + for name in ( + "valid_market_open_after_epoch_seconds", + "signal_target_offset_ms", + "arrival_target_offset_ms", + "max_request_start_lateness_ms", + "max_request_duration_ms", + "max_book_age_at_request_completion_ms", + "max_book_future_skew_ms", + ): + if int(getattr(self, name)) < 0: + raise ValueError(f"{name} must be non-negative") + if self.arrival_target_offset_ms <= self.signal_target_offset_ms: + raise ValueError("arrival target must be after signal target") + if self.max_request_duration_ms <= 0: + raise ValueError("max_request_duration_ms must be positive") + + +def load_capture_policy( + capture_policy_path: str | Path, + capture_manifest_path: str | Path, +) -> CapturePolicy: + policy_path = Path(capture_policy_path) + raw = policy_path.read_bytes() + payload = json.loads(raw) + manifest = json.loads(Path(capture_manifest_path).read_text(encoding="utf-8")) + got = hashlib.sha256(raw).hexdigest() + wanted = _validate_sha256( + "capture_policy_sha256", str(manifest.get("capture_policy_sha256", "")) + ) + if got != wanted: + raise FreezeViolation(f"capture policy SHA-256 mismatch: {got} != {wanted}") + if payload.get("name") != "late_favorite_btc_only_v3_capture_policy": + raise FreezeViolation("unexpected capture policy name") + if payload.get("status") != "frozen_before_prospective_collection": + raise FreezeViolation("capture policy is not frozen") + transport = payload.get("transport", {}) + privileged = ("credentials_used", "authenticated_requests", "order_submissions") + if any(int(transport.get(name, -1)) != 0 for name in privileged): + raise FreezeViolation("capture policy permits privileged transport") + freeze_text = str(manifest["valid_market_open_after_utc"]) + freeze_epoch = int(datetime.fromisoformat(freeze_text.replace("Z", "+00:00")).timestamp()) + event_time = payload["event_time"] + windows = payload["request_windows"] + freshness = payload["book_freshness"] + return CapturePolicy( + capture_policy_sha256=got, + valid_market_open_after_epoch_seconds=freeze_epoch, + signal_target_offset_ms=int(event_time["signal_target_offset_ms"]), + arrival_target_offset_ms=int(event_time["arrival_target_offset_ms"]), + max_request_start_lateness_ms=int(windows["max_request_start_lateness_ms"]), + max_request_duration_ms=int(windows["max_request_duration_ms"]), + max_book_age_at_request_completion_ms=int( + freshness["max_book_age_at_request_completion_ms"] + ), + max_book_future_skew_ms=int(freshness["max_book_future_skew_ms"]), + ) + + +def _target_ms( + capture_policy: CapturePolicy, + market_open_epoch_seconds: int, + *, + phase: str, +) -> int: + opening = int(market_open_epoch_seconds) + if opening <= capture_policy.valid_market_open_after_epoch_seconds: + raise FreezeViolation( + f"market open {opening} is not strictly after capture freeze " + f"{capture_policy.valid_market_open_after_epoch_seconds}" + ) + offset = ( + capture_policy.signal_target_offset_ms + if phase == "signal" + else capture_policy.arrival_target_offset_ms + ) + return opening * 1000 + offset + + +def _transport_evidence( + capture_policy: CapturePolicy, + *, + market_open_epoch_seconds: int, + phase: str, + request_started_ts_ms: int, + request_completed_ts_ms: int, + response_payload_sha256: str, +) -> tuple[dict[str, Any], str | None]: + target = _target_ms(capture_policy, market_open_epoch_seconds, phase=phase) + started = int(request_started_ts_ms) + completed = int(request_completed_ts_ms) + response_sha = _validate_sha256("response_payload_sha256", response_payload_sha256) + if started < target: + raise CaptureWindowError( + f"{phase} request started before target: {started} < {target}" + ) + if completed < started: + raise CaptureWindowError("request completion precedes request start") + base = { + "capture_policy_sha256": capture_policy.capture_policy_sha256, + "target_ts_ms": target, + "request_started_ts_ms": started, + "request_completed_ts_ms": completed, + "request_duration_ms": completed - started, + "response_payload_sha256": response_sha, + } + if started - target > capture_policy.max_request_start_lateness_ms: + return base, f"missed_{phase}_window" + if completed - started > capture_policy.max_request_duration_ms: + return base, f"{phase}_request_timeout" + return base, None + + +def _book_timestamp_decision( + capture_policy: CapturePolicy, + books: Sequence[Mapping[str, Any]], + *, + request_completed_ts_ms: int, + phase: str, +) -> tuple[list[int], str | None]: + timestamps: list[int] = [] + for book in books: + try: + timestamp = int(str(book.get("timestamp"))) + except (TypeError, ValueError): + return [], f"no_{phase}_invalid_book_timestamp" + timestamps.append(timestamp) + age = int(request_completed_ts_ms) - timestamp + if age > capture_policy.max_book_age_at_request_completion_ms: + return timestamps, f"no_{phase}_stale_book" + if timestamp - int(request_completed_ts_ms) > capture_policy.max_book_future_skew_ms: + return timestamps, f"no_{phase}_future_book" + return timestamps, None + + +def evaluate_signal_capture( + policy: FrozenPolicy, + capture_policy: CapturePolicy, + *, + condition_id: str, + market_open_epoch_seconds: int, + up_token_id: str, + down_token_id: str, + up_book: Mapping[str, Any], + down_book: Mapping[str, Any], + request_started_ts_ms: int, + request_completed_ts_ms: int, + response_payload_sha256: str, +) -> dict[str, Any]: + transport, transport_decision = _transport_evidence( + capture_policy, + market_open_epoch_seconds=market_open_epoch_seconds, + phase="signal", + request_started_ts_ms=request_started_ts_ms, + request_completed_ts_ms=request_completed_ts_ms, + response_payload_sha256=response_payload_sha256, + ) + fail_base = { + "event_type": "signal", + "condition_id": str(condition_id), + "market_open_epoch_seconds": int(market_open_epoch_seconds), + "observed_ts_ms": transport["target_ts_ms"], + "policy_sha256": policy.policy_sha256, + "source_sha256": policy.source_sha256, + "signal": False, + "selected_side": None, + "selected_token_id": None, + "signal_ask": None, + **transport, + } + if transport_decision is not None: + return {**fail_base, "decision": transport_decision} + timestamps, book_decision = _book_timestamp_decision( + capture_policy, + [up_book, down_book], + request_completed_ts_ms=request_completed_ts_ms, + phase="signal", + ) + if book_decision is not None: + return {**fail_base, "decision": book_decision, "book_timestamps_ms": timestamps} + result = evaluate_signal( + policy, + condition_id=condition_id, + market_open_epoch_seconds=market_open_epoch_seconds, + up_token_id=up_token_id, + down_token_id=down_token_id, + up_book=up_book, + down_book=down_book, + observed_ts_ms=transport["target_ts_ms"], + ) + return {**result, **transport, "book_timestamps_ms": timestamps} + + +def evaluate_arrival_capture( + policy: FrozenPolicy, + capture_policy: CapturePolicy, + signal_record: Mapping[str, Any], + selected_book: Mapping[str, Any], + *, + request_started_ts_ms: int, + request_completed_ts_ms: int, + response_payload_sha256: str, +) -> dict[str, Any]: + opening = int(signal_record["market_open_epoch_seconds"]) + transport, transport_decision = _transport_evidence( + capture_policy, + market_open_epoch_seconds=opening, + phase="arrival", + request_started_ts_ms=request_started_ts_ms, + request_completed_ts_ms=request_completed_ts_ms, + response_payload_sha256=response_payload_sha256, + ) + fail_base = { + "event_type": "arrival", + "condition_id": str(signal_record["condition_id"]), + "market_open_epoch_seconds": opening, + "observed_ts_ms": transport["target_ts_ms"], + "policy_sha256": policy.policy_sha256, + "source_sha256": policy.source_sha256, + "selected_side": signal_record.get("selected_side"), + "selected_token_id": signal_record.get("selected_token_id"), + "signal_ask": signal_record.get("signal_ask"), + "hypothetical_fok_fill": False, + "filled_shares": "0", + "execution_levels": [], + "execution_vwap": None, + "fee_per_share": None, + "all_in_cost_per_share": None, + **transport, + } + if transport_decision is not None: + return {**fail_base, "decision": transport_decision} + timestamps, book_decision = _book_timestamp_decision( + capture_policy, + [selected_book], + request_completed_ts_ms=request_completed_ts_ms, + phase="arrival", + ) + if book_decision is not None: + decision = book_decision.replace("no_arrival_", "no_fill_") + return {**fail_base, "decision": decision, "book_timestamps_ms": timestamps} + result = evaluate_arrival( + policy, + signal_record, + selected_book, + observed_ts_ms=transport["target_ts_ms"], + ) + return {**result, **transport, "book_timestamps_ms": timestamps} + + +class CaptureBoundLifecycleLedger(ProspectiveLifecycleLedger): + def __init__( + self, + path: str | Path, + policy: FrozenPolicy, + capture_policy: CapturePolicy, + ) -> None: + super().__init__(path, policy) + self.capture_policy = capture_policy + for line_number, record in enumerate(self.records, start=1): + if record.get("capture_policy_sha256") != capture_policy.capture_policy_sha256: + raise LedgerIntegrityError( + f"capture policy hash mismatch at line {line_number}" + ) + if int(record["market_open_epoch_seconds"]) <= capture_policy.valid_market_open_after_epoch_seconds: + raise LedgerIntegrityError( + f"capture freeze violation at line {line_number}" + ) + + def append( + self, + *, + condition_id: str, + market_open_epoch_seconds: int, + event_type: str, + observed_ts_ms: int, + payload: Mapping[str, Any], + ) -> dict[str, Any]: + condition = str(condition_id) + if not condition: + raise ValueError("condition_id must be non-empty") + opening = int(market_open_epoch_seconds) + _target_ms(self.capture_policy, opening, phase="signal") + event = str(event_type) + if event not in {"discovered", "signal", "arrival", "resolution"}: + raise LifecycleError(f"unsupported event_type: {event}") + observed = int(observed_ts_ms) + if observed < 0: + raise ValueError("observed_ts_ms must be non-negative") + try: + canonical_payload = json.loads(_canonical_json(dict(payload)).decode("utf-8")) + except (TypeError, ValueError) as exc: + raise ValueError("payload must be JSON serializable") from exc + self._validate_order(condition, event, canonical_payload) + payload_sha256 = hashlib.sha256(_canonical_json(canonical_payload)).hexdigest() + event_id = hashlib.sha256( + f"{condition}\0{event}\0{observed}\0{payload_sha256}".encode("utf-8") + ).hexdigest() + record_without_hash = { + "capture_policy_sha256": self.capture_policy.capture_policy_sha256, + "condition_id": condition, + "event_id": event_id, + "event_type": event, + "market_open_epoch_seconds": opening, + "observed_ts_ms": observed, + "payload": canonical_payload, + "payload_sha256": payload_sha256, + "policy_sha256": self.policy.policy_sha256, + "previous_hash": self.head_hash, + "source_sha256": self.policy.source_sha256, + } + record = { + **record_without_hash, + "record_hash": hashlib.sha256(_canonical_json(record_without_hash)).hexdigest(), + } + self.path.parent.mkdir(parents=True, exist_ok=True) + with self.path.open("a", encoding="utf-8", newline="\n") as handle: + handle.write(json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n") + handle.flush() + self._accept_record(record) + return dict(record) From a921e7d31b8a7f7e27f91b1a4a38d8f48e4105c3 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:45:11 -0500 Subject: [PATCH 123/177] ci: verify BTC-only v3 observer checkpoint --- .../btc-only-v3-observer-checkpoint.yml | 150 +++++++++++++++++- 1 file changed, 143 insertions(+), 7 deletions(-) diff --git a/.github/workflows/btc-only-v3-observer-checkpoint.yml b/.github/workflows/btc-only-v3-observer-checkpoint.yml index 72ff032b..0acf07af 100644 --- a/.github/workflows/btc-only-v3-observer-checkpoint.yml +++ b/.github/workflows/btc-only-v3-observer-checkpoint.yml @@ -23,7 +23,7 @@ env: jobs: observer-contract: runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 30 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -31,14 +31,150 @@ jobs: python-version: '3.11' - name: Install pinned test stack run: python -m pip install pytest==8.3.5 - - name: Require and compile observer source + + - name: Compile observer and capture sources shell: bash run: | - test -f ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source/late_favorite_v3.py + set -euo pipefail + root=ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 python -m py_compile \ - ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source/late_favorite_v3.py \ - ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_late_favorite_v3.py - - name: Run observer behavior contract + "$root/source/late_favorite_v3.py" \ + "$root/source/capture_policy.py" \ + "$root/tests/test_late_favorite_v3.py" \ + "$root/tests/test_capture_policy.py" + + - name: Pass exactly 14 behavior contracts + shell: bash env: PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source - run: pytest -q ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_late_favorite_v3.py + run: | + set -euo pipefail + root=ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 + pytest --collect-only -q "$root/tests" | tee /tmp/collect.txt + grep -Eq '^14 tests collected' /tmp/collect.txt + pytest -q "$root/tests" + + - name: Verify frozen policies and contamination cutoffs + shell: bash + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source + run: | + set -euo pipefail + root=ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 + python - <<'PY' + import hashlib + import json + from pathlib import Path + + from capture_policy import load_capture_policy + from late_favorite_v3 import load_frozen_policy + + root = Path('ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3') + source_paths = [root / 'source/late_favorite_v3.py', root / 'source/capture_policy.py'] + combined = hashlib.sha256() + source_hashes = {} + for path in source_paths: + raw = path.read_bytes() + digest = hashlib.sha256(raw).hexdigest() + source_hashes[str(path.relative_to(root))] = digest + combined.update(path.name.encode() + b'\0' + raw) + combined_source_sha256 = combined.hexdigest() + + trading = load_frozen_policy( + root / 'CANDIDATE_SPEC.json', + root / 'FREEZE_MANIFEST.json', + source_sha256=combined_source_sha256, + ) + capture = load_capture_policy( + root / 'CAPTURE_POLICY.json', + root / 'CAPTURE_FREEZE_MANIFEST.json', + ) + assert trading.policy_sha256 == '82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba' + assert capture.capture_policy_sha256 == '888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc' + assert capture.valid_market_open_after_epoch_seconds > trading.valid_market_open_after_epoch_seconds + assert capture.signal_target_offset_ms == 210_000 + assert capture.arrival_target_offset_ms == 211_000 + assert capture.max_request_start_lateness_ms == 250 + assert capture.max_request_duration_ms == 1_000 + assert capture.max_book_age_at_request_completion_ms == 2_000 + + verification = { + 'candidate': 'late_favorite_btc_only_v3_prospective_shadow', + 'tests_collected': 14, + 'candidate_policy_sha256': trading.policy_sha256, + 'capture_policy_sha256': capture.capture_policy_sha256, + 'candidate_cutoff_epoch_seconds': trading.valid_market_open_after_epoch_seconds, + 'capture_cutoff_epoch_seconds': capture.valid_market_open_after_epoch_seconds, + 'combined_source_sha256': combined_source_sha256, + 'source_files_sha256': source_hashes, + 'credentials_used': 0, + 'authenticated_requests': 0, + 'order_submissions': 0, + 'live_submission': 'physically_absent', + 'historical_admission_credit': 0, + } + (root / 'OBSERVER_VERIFICATION.json').write_text( + json.dumps(verification, indent=2, sort_keys=True) + '\n' + ) + print(json.dumps(verification, indent=2, sort_keys=True)) + PY + + - name: Verify privileged capabilities remain absent + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import ast + from pathlib import Path + + root = Path('ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source') + forbidden_modules = { + 'requests', 'urllib', 'httpx', 'aiohttp', 'web3', + 'py_clob_client', 'eth_account', 'websocket', 'websockets', + } + forbidden_tokens = { + 'create_order', 'post_order', 'submit_order', 'private_key', + 'api_secret', 'authorization', 'sign_transaction', + } + for path in sorted(root.glob('*.py')): + text = path.read_text() + tree = ast.parse(text) + imports = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imports.update(alias.name.split('.')[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + imports.add(node.module.split('.')[0]) + assert not (imports & forbidden_modules), (path, imports & forbidden_modules) + lowered = text.lower() + for token in forbidden_tokens: + assert token not in lowered, (path, token) + PY + + - name: Build deterministic observer checkpoint + shell: bash + run: | + set -euo pipefail + root=ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 + mkdir -p btc-only-v3-observer-checkpoint + cp "$root/CANDIDATE_SPEC.json" btc-only-v3-observer-checkpoint/ + cp "$root/FREEZE_MANIFEST.json" btc-only-v3-observer-checkpoint/ + cp "$root/CAPTURE_POLICY.json" btc-only-v3-observer-checkpoint/ + cp "$root/CAPTURE_FREEZE_MANIFEST.json" btc-only-v3-observer-checkpoint/ + cp "$root/OBSERVER_VERIFICATION.json" btc-only-v3-observer-checkpoint/ + cp "$root/source/late_favorite_v3.py" btc-only-v3-observer-checkpoint/ + cp "$root/source/capture_policy.py" btc-only-v3-observer-checkpoint/ + (cd btc-only-v3-observer-checkpoint && sha256sum * > SHA256SUMS) + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf btc-only-v3-observer-checkpoint-v1.tar.gz btc-only-v3-observer-checkpoint + sha256sum btc-only-v3-observer-checkpoint-v1.tar.gz > btc-only-v3-observer-checkpoint-v1.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: btc-only-v3-observer-checkpoint-v1 + path: | + btc-only-v3-observer-checkpoint/ + btc-only-v3-observer-checkpoint-v1.tar.gz + btc-only-v3-observer-checkpoint-v1.tar.gz.sha256 + compression-level: 0 + retention-days: 30 From 3b6410f0f2628b1cfcb3872208d53ab58460b2d0 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:47:22 -0500 Subject: [PATCH 124/177] ci: cancel obsolete BTC-only v3 checkpoints --- .github/workflows/btc-only-v3-observer-checkpoint.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/btc-only-v3-observer-checkpoint.yml b/.github/workflows/btc-only-v3-observer-checkpoint.yml index 0acf07af..350746e4 100644 --- a/.github/workflows/btc-only-v3-observer-checkpoint.yml +++ b/.github/workflows/btc-only-v3-observer-checkpoint.yml @@ -13,6 +13,10 @@ on: - ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/** workflow_dispatch: +concurrency: + group: btc-only-v3-observer-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + permissions: contents: read From 9147063b859daea413226bbc9f3d7e98458d4a47 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:48:58 -0500 Subject: [PATCH 125/177] test: specify credential-free BTC-only public collector --- .../tests/test_public_collector.py | 175 ++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_public_collector.py diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_public_collector.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_public_collector.py new file mode 100644 index 00000000..15d47791 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_public_collector.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +import json +from decimal import Decimal + +import pytest + +from capture_policy import CaptureBoundLifecycleLedger, CapturePolicy +from late_favorite_v3 import FrozenPolicy +from public_collector import ( + PublicEndpointViolation, + PublicHttpClient, + RawEvidenceStore, + SingleMarketCollector, + next_collectable_open, + parse_btc_five_minute_market, +) + + +class FakeResponse: + def __init__(self, payload: object, status_code: int = 200) -> None: + self.content = json.dumps(payload, separators=(",", ":")).encode() + self.status_code = status_code + + +class FakeRequester: + def __init__(self, responses: list[FakeResponse]) -> None: + self.responses = list(responses) + self.calls: list[dict[str, object]] = [] + + def __call__(self, method: str, url: str, **kwargs: object) -> FakeResponse: + self.calls.append({"method": method, "url": url, **kwargs}) + return self.responses.pop(0) + + +def trading_policy() -> FrozenPolicy: + return FrozenPolicy( + policy_sha256="1" * 64, + source_sha256="2" * 64, + valid_market_open_after_epoch_seconds=1_800_000_000, + signal_ask_min=Decimal("0.85"), + entry_second=210, + latency_seconds=1, + shares=Decimal("5"), + adverse_move_cancel=Decimal("0.01"), + fee_rate=Decimal("0.07"), + ) + + +def capture_policy() -> CapturePolicy: + return CapturePolicy( + capture_policy_sha256="3" * 64, + valid_market_open_after_epoch_seconds=1_800_000_100, + signal_target_offset_ms=210_000, + arrival_target_offset_ms=211_000, + max_request_start_lateness_ms=250, + max_request_duration_ms=1_000, + max_book_age_at_request_completion_ms=2_000, + max_book_future_skew_ms=1_000, + ) + + +def market_payload(opening: int) -> dict[str, object]: + return { + "conditionId": "condition-1", + "slug": f"btc-updown-5m-{opening}", + "active": True, + "closed": False, + "enableOrderBook": True, + "outcomes": json.dumps(["Up", "Down"]), + "clobTokenIds": json.dumps(["token-up", "token-down"]), + } + + +def book(condition: str, token: str, ask: str, timestamp_ms: int) -> dict[str, object]: + return { + "market": condition, + "asset_id": token, + "timestamp": str(timestamp_ms), + "hash": f"hash-{token}-{timestamp_ms}", + "bids": [{"price": "0.10", "size": "10"}], + "asks": [{"price": ask, "size": "10"}], + "min_order_size": "1", + "tick_size": "0.01", + "neg_risk": False, + "last_trade_price": "0.50", + } + + +def test_parse_market_maps_up_and_down_tokens() -> None: + market = parse_btc_five_minute_market(market_payload(1_800_000_300), 1_800_000_300) + assert market.condition_id == "condition-1" + assert market.up_token_id == "token-up" + assert market.down_token_id == "token-down" + + +def test_public_http_client_allows_only_read_only_public_endpoints() -> None: + requester = FakeRequester([FakeResponse({"ok": True})]) + clock = iter([1000, 1025]) + client = PublicHttpClient(requester=requester, clock_ms=lambda: next(clock)) + evidence = client.request_json( + "GET", "https://gamma-api.polymarket.com/markets/slug/test" + ) + assert evidence.status_code == 200 + call = requester.calls[0] + assert "Authorization" not in call["headers"] + assert not any(str(key).startswith("POLY_") for key in call["headers"]) + with pytest.raises(PublicEndpointViolation): + client.request_json("POST", "https://clob.polymarket.com/order", json_body={}) + + +def test_raw_evidence_store_is_hash_chained_and_tamper_evident(tmp_path) -> None: + requester = FakeRequester([FakeResponse({"value": 1})]) + clock = iter([1000, 1010]) + evidence = PublicHttpClient(requester=requester, clock_ms=lambda: next(clock)).request_json( + "GET", "https://gamma-api.polymarket.com/markets/slug/test" + ) + store = RawEvidenceStore(tmp_path) + row = store.append(evidence, purpose="market_discovery") + assert len(row["record_hash"]) == 64 + assert RawEvidenceStore(tmp_path).head_hash == row["record_hash"] + body = tmp_path / row["body_path"] + body.write_bytes(b"tampered") + with pytest.raises(RuntimeError): + RawEvidenceStore(tmp_path) + + +def test_next_collectable_open_uses_current_window_only_before_signal_target() -> None: + opening = 1_800_000_300 + assert next_collectable_open(opening * 1000 + 209_000, 1_800_000_100) == opening + assert next_collectable_open(opening * 1000 + 210_001, 1_800_000_100) == opening + 300 + + +def test_single_market_collector_records_discovery_signal_and_arrival(tmp_path) -> None: + opening = 1_800_000_300 + signal_target = opening * 1000 + 210_000 + arrival_target = opening * 1000 + 211_000 + responses = [ + FakeResponse(market_payload(opening)), + FakeResponse([ + book("condition-1", "token-up", "0.90", signal_target + 10), + book("condition-1", "token-down", "0.12", signal_target + 10), + ]), + FakeResponse([book("condition-1", "token-up", "0.90", arrival_target + 10)]), + ] + times = iter([ + opening * 1000 + 1_000, opening * 1000 + 1_020, + signal_target + 20, signal_target + 100, + arrival_target + 20, arrival_target + 100, + ]) + requester = FakeRequester(responses) + client = PublicHttpClient(requester=requester, clock_ms=lambda: next(times)) + ledger = CaptureBoundLifecycleLedger( + tmp_path / "lifecycle.jsonl", trading_policy(), capture_policy() + ) + store = RawEvidenceStore(tmp_path / "raw") + slept: list[int] = [] + collector = SingleMarketCollector( + client=client, + evidence_store=store, + ledger=ledger, + trading_policy=trading_policy(), + capture_policy=capture_policy(), + sleep_until_ms=slept.append, + ) + result = collector.capture(opening) + assert result["decision"] == "hypothetical_fok_fill" + assert [row["event_type"] for row in ledger.records] == ["discovered", "signal", "arrival"] + assert slept == [signal_target, arrival_target] + assert len(store.records) == 3 + assert requester.calls[1]["method"] == "POST" + assert requester.calls[1]["url"] == "https://clob.polymarket.com/books" + assert requester.calls[1]["json"] == [ + {"token_id": "token-up"}, {"token_id": "token-down"} + ] From 5a8e292a9012f6707e0479ec968f938b721f64d9 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:51:49 -0500 Subject: [PATCH 126/177] research: implement credential-free BTC-only collector --- .../collector/public_collector.py | 365 ++++++++++++++++++ 1 file changed, 365 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector/public_collector.py diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector/public_collector.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector/public_collector.py new file mode 100644 index 00000000..d752399f --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector/public_collector.py @@ -0,0 +1,365 @@ +from __future__ import annotations + +import hashlib +import json +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Mapping, Sequence +from urllib.parse import urlparse + +import requests + +from capture_policy import ( + CaptureBoundLifecycleLedger, + CapturePolicy, + evaluate_arrival_capture, + evaluate_signal_capture, +) +from late_favorite_v3 import FrozenPolicy + +GAMMA_BASE = "https://gamma-api.polymarket.com" +CLOB_BASE = "https://clob.polymarket.com" + + +class PublicEndpointViolation(RuntimeError): + pass + + +class PublicResponseError(RuntimeError): + pass + + +@dataclass(frozen=True, slots=True) +class PublicHttpEvidence: + method: str + url: str + status_code: int + request_started_ts_ms: int + request_completed_ts_ms: int + response_body_sha256: str + request_body_sha256: str | None + body: bytes + payload: object + + +@dataclass(frozen=True, slots=True) +class BtcFiveMinuteMarket: + condition_id: str + slug: str + open_epoch_seconds: int + up_token_id: str + down_token_id: str + + +def _canonical_json(value: object) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _validate_public_endpoint(method: str, url: str) -> None: + parsed = urlparse(url) + if parsed.scheme != "https" or parsed.username or parsed.password: + raise PublicEndpointViolation("only credential-free HTTPS endpoints are allowed") + normalized_method = method.upper() + if parsed.netloc == "gamma-api.polymarket.com": + allowed = normalized_method == "GET" and parsed.path.startswith("/markets/slug/") + elif parsed.netloc == "clob.polymarket.com": + allowed = normalized_method == "POST" and parsed.path == "/books" + else: + allowed = False + if not allowed: + raise PublicEndpointViolation(f"endpoint is outside the frozen public allowlist: {method} {url}") + + +class PublicHttpClient: + def __init__( + self, + *, + requester: Callable[..., Any] = requests.request, + clock_ms: Callable[[], int] | None = None, + timeout_seconds: float = 5.0, + ) -> None: + self.requester = requester + self.clock_ms = clock_ms or (lambda: time.time_ns() // 1_000_000) + self.timeout_seconds = float(timeout_seconds) + if self.timeout_seconds <= 0: + raise ValueError("timeout_seconds must be positive") + + def request_json( + self, + method: str, + url: str, + *, + json_body: object | None = None, + ) -> PublicHttpEvidence: + normalized_method = method.upper() + _validate_public_endpoint(normalized_method, url) + headers = { + "Accept": "application/json", + "User-Agent": "btc-only-v3-prospective-shadow/1", + } + if json_body is not None: + headers["Content-Type"] = "application/json" + started = int(self.clock_ms()) + response = self.requester( + normalized_method, + url, + headers=headers, + timeout=self.timeout_seconds, + json=json_body, + ) + completed = int(self.clock_ms()) + status = int(response.status_code) + body = bytes(response.content) + if status != 200: + raise PublicResponseError(f"public endpoint returned HTTP {status}: {url}") + try: + payload = json.loads(body) + except json.JSONDecodeError as exc: + raise PublicResponseError(f"public endpoint returned invalid JSON: {url}") from exc + request_hash = None if json_body is None else hashlib.sha256(_canonical_json(json_body)).hexdigest() + return PublicHttpEvidence( + method=normalized_method, + url=url, + status_code=status, + request_started_ts_ms=started, + request_completed_ts_ms=completed, + response_body_sha256=hashlib.sha256(body).hexdigest(), + request_body_sha256=request_hash, + body=body, + payload=payload, + ) + + +class RawEvidenceStore: + def __init__(self, root: str | Path) -> None: + self.root = Path(root) + self.manifest_path = self.root / "manifest.jsonl" + self.records: list[dict[str, Any]] = [] + self.head_hash = "0" * 64 + if self.manifest_path.exists(): + self._load_and_verify() + + def _load_and_verify(self) -> None: + previous = "0" * 64 + for line_number, line in enumerate( + self.manifest_path.read_text(encoding="utf-8").splitlines(), start=1 + ): + if not line.strip(): + continue + try: + row = json.loads(line) + except json.JSONDecodeError as exc: + raise RuntimeError(f"invalid evidence manifest JSON at line {line_number}") from exc + stored = str(row.get("record_hash", "")) + without = dict(row) + without.pop("record_hash", None) + if without.get("previous_hash") != previous: + raise RuntimeError(f"evidence hash chain mismatch at line {line_number}") + expected = hashlib.sha256(_canonical_json(without)).hexdigest() + if stored != expected: + raise RuntimeError(f"evidence record hash mismatch at line {line_number}") + body_path = self.root / str(row["body_path"]) + if not body_path.is_file(): + raise RuntimeError(f"missing evidence body at line {line_number}") + if hashlib.sha256(body_path.read_bytes()).hexdigest() != row["response_body_sha256"]: + raise RuntimeError(f"evidence body hash mismatch at line {line_number}") + self.records.append(row) + self.head_hash = stored + previous = stored + + def append(self, evidence: PublicHttpEvidence, *, purpose: str) -> dict[str, Any]: + if not purpose: + raise ValueError("purpose must be non-empty") + body_rel = Path("bodies") / f"{evidence.response_body_sha256}.json" + body_path = self.root / body_rel + body_path.parent.mkdir(parents=True, exist_ok=True) + if body_path.exists() and hashlib.sha256(body_path.read_bytes()).hexdigest() != evidence.response_body_sha256: + raise RuntimeError("existing evidence body conflicts with response hash") + if not body_path.exists(): + body_path.write_bytes(evidence.body) + without = { + "body_path": body_rel.as_posix(), + "method": evidence.method, + "previous_hash": self.head_hash, + "purpose": str(purpose), + "request_body_sha256": evidence.request_body_sha256, + "request_completed_ts_ms": evidence.request_completed_ts_ms, + "request_started_ts_ms": evidence.request_started_ts_ms, + "response_body_sha256": evidence.response_body_sha256, + "status_code": evidence.status_code, + "url": evidence.url, + } + row = {**without, "record_hash": hashlib.sha256(_canonical_json(without)).hexdigest()} + self.root.mkdir(parents=True, exist_ok=True) + with self.manifest_path.open("a", encoding="utf-8", newline="\n") as handle: + handle.write(json.dumps(row, sort_keys=True, separators=(",", ":")) + "\n") + handle.flush() + self.records.append(row) + self.head_hash = row["record_hash"] + return dict(row) + + +def _json_list(value: object, name: str) -> list[str]: + if isinstance(value, str): + try: + value = json.loads(value) + except json.JSONDecodeError as exc: + raise ValueError(f"{name} is not valid JSON") from exc + if not isinstance(value, list) or not all(isinstance(item, str) for item in value): + raise ValueError(f"{name} must be a string list") + return list(value) + + +def parse_btc_five_minute_market( + payload: Mapping[str, Any], expected_open_epoch_seconds: int +) -> BtcFiveMinuteMarket: + opening = int(expected_open_epoch_seconds) + expected_slug = f"btc-updown-5m-{opening}" + if str(payload.get("slug")) != expected_slug: + raise ValueError("Gamma market slug does not match requested BTC window") + if payload.get("active") is not True or payload.get("closed") is not False: + raise ValueError("Gamma market is not active and open") + if payload.get("enableOrderBook") is not True: + raise ValueError("Gamma market is not CLOB-enabled") + condition_id = str(payload.get("conditionId") or "") + if not condition_id: + raise ValueError("Gamma market is missing conditionId") + outcomes = _json_list(payload.get("outcomes"), "outcomes") + token_ids = _json_list(payload.get("clobTokenIds"), "clobTokenIds") + if len(outcomes) != 2 or len(token_ids) != 2 or set(outcomes) != {"Up", "Down"}: + raise ValueError("Gamma market must contain exactly Up and Down outcomes") + mapping = dict(zip(outcomes, token_ids, strict=True)) + if not mapping["Up"] or not mapping["Down"] or mapping["Up"] == mapping["Down"]: + raise ValueError("Gamma market contains invalid token IDs") + return BtcFiveMinuteMarket( + condition_id=condition_id, + slug=expected_slug, + open_epoch_seconds=opening, + up_token_id=mapping["Up"], + down_token_id=mapping["Down"], + ) + + +def _parse_books(payload: object, requested_token_ids: Sequence[str]) -> dict[str, Mapping[str, Any]]: + if not isinstance(payload, list): + raise PublicResponseError("CLOB /books response must be a list") + requested = [str(token) for token in requested_token_ids] + found: dict[str, Mapping[str, Any]] = {} + for row in payload: + if not isinstance(row, Mapping): + raise PublicResponseError("CLOB /books row must be an object") + token = str(row.get("asset_id") or "") + if token in found: + raise PublicResponseError(f"duplicate CLOB book for token {token}") + found[token] = row + if set(found) != set(requested): + raise PublicResponseError("CLOB /books response token set mismatch") + return found + + +def next_collectable_open(now_ts_ms: int, cutoff_epoch_seconds: int) -> int: + now_ms = int(now_ts_ms) + opening = (now_ms // 1000 // 300) * 300 + if now_ms > opening * 1000 + 210_000: + opening += 300 + while opening <= int(cutoff_epoch_seconds): + opening += 300 + return opening + + +class SingleMarketCollector: + def __init__( + self, + *, + client: PublicHttpClient, + evidence_store: RawEvidenceStore, + ledger: CaptureBoundLifecycleLedger, + trading_policy: FrozenPolicy, + capture_policy: CapturePolicy, + sleep_until_ms: Callable[[int], None], + ) -> None: + self.client = client + self.evidence_store = evidence_store + self.ledger = ledger + self.trading_policy = trading_policy + self.capture_policy = capture_policy + self.sleep_until_ms = sleep_until_ms + + def capture(self, market_open_epoch_seconds: int) -> dict[str, Any]: + opening = int(market_open_epoch_seconds) + slug = f"btc-updown-5m-{opening}" + discovery = self.client.request_json("GET", f"{GAMMA_BASE}/markets/slug/{slug}") + discovery_manifest = self.evidence_store.append(discovery, purpose="market_discovery") + if not isinstance(discovery.payload, Mapping): + raise PublicResponseError("Gamma market response must be an object") + market = parse_btc_five_minute_market(discovery.payload, opening) + self.ledger.append( + condition_id=market.condition_id, + market_open_epoch_seconds=opening, + event_type="discovered", + observed_ts_ms=discovery.request_completed_ts_ms, + payload={ + "slug": market.slug, + "up_token_id": market.up_token_id, + "down_token_id": market.down_token_id, + "gamma_response_body_sha256": discovery.response_body_sha256, + "evidence_record_hash": discovery_manifest["record_hash"], + }, + ) + signal_target = opening * 1000 + self.capture_policy.signal_target_offset_ms + self.sleep_until_ms(signal_target) + signal_body = [{"token_id": market.up_token_id}, {"token_id": market.down_token_id}] + signal_http = self.client.request_json( + "POST", f"{CLOB_BASE}/books", json_body=signal_body + ) + self.evidence_store.append(signal_http, purpose="signal_books") + signal_books = _parse_books(signal_http.payload, [market.up_token_id, market.down_token_id]) + signal = evaluate_signal_capture( + self.trading_policy, + self.capture_policy, + condition_id=market.condition_id, + market_open_epoch_seconds=opening, + up_token_id=market.up_token_id, + down_token_id=market.down_token_id, + up_book=signal_books[market.up_token_id], + down_book=signal_books[market.down_token_id], + request_started_ts_ms=signal_http.request_started_ts_ms, + request_completed_ts_ms=signal_http.request_completed_ts_ms, + response_payload_sha256=signal_http.response_body_sha256, + ) + self.ledger.append( + condition_id=market.condition_id, + market_open_epoch_seconds=opening, + event_type="signal", + observed_ts_ms=signal["observed_ts_ms"], + payload=signal, + ) + if signal["decision"] != "signal": + return signal + arrival_target = opening * 1000 + self.capture_policy.arrival_target_offset_ms + self.sleep_until_ms(arrival_target) + selected_token = str(signal["selected_token_id"]) + arrival_body = [{"token_id": selected_token}] + arrival_http = self.client.request_json( + "POST", f"{CLOB_BASE}/books", json_body=arrival_body + ) + self.evidence_store.append(arrival_http, purpose="arrival_book") + arrival_books = _parse_books(arrival_http.payload, [selected_token]) + arrival = evaluate_arrival_capture( + self.trading_policy, + self.capture_policy, + signal, + arrival_books[selected_token], + request_started_ts_ms=arrival_http.request_started_ts_ms, + request_completed_ts_ms=arrival_http.request_completed_ts_ms, + response_payload_sha256=arrival_http.response_body_sha256, + ) + self.ledger.append( + condition_id=market.condition_id, + market_open_epoch_seconds=opening, + event_type="arrival", + observed_ts_ms=arrival["observed_ts_ms"], + payload=arrival, + ) + return arrival From aef3760bed56d71ab3fa771d2d8ef68f030b3ab4 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:53:03 -0500 Subject: [PATCH 127/177] ci: verify credential-free BTC-only collector --- .../btc-only-v3-observer-checkpoint.yml | 63 ++++++++++++++----- 1 file changed, 47 insertions(+), 16 deletions(-) diff --git a/.github/workflows/btc-only-v3-observer-checkpoint.yml b/.github/workflows/btc-only-v3-observer-checkpoint.yml index 350746e4..10fa5a5e 100644 --- a/.github/workflows/btc-only-v3-observer-checkpoint.yml +++ b/.github/workflows/btc-only-v3-observer-checkpoint.yml @@ -34,9 +34,9 @@ jobs: with: python-version: '3.11' - name: Install pinned test stack - run: python -m pip install pytest==8.3.5 + run: python -m pip install requests==2.32.4 pytest==8.3.5 - - name: Compile observer and capture sources + - name: Compile observer, capture, and public collector sources shell: bash run: | set -euo pipefail @@ -44,27 +44,28 @@ jobs: python -m py_compile \ "$root/source/late_favorite_v3.py" \ "$root/source/capture_policy.py" \ + "$root/collector/public_collector.py" \ "$root/tests/test_late_favorite_v3.py" \ - "$root/tests/test_capture_policy.py" + "$root/tests/test_capture_policy.py" \ + "$root/tests/test_public_collector.py" - - name: Pass exactly 14 behavior contracts + - name: Pass exactly 19 behavior contracts shell: bash env: - PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector run: | set -euo pipefail root=ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 pytest --collect-only -q "$root/tests" | tee /tmp/collect.txt - grep -Eq '^14 tests collected' /tmp/collect.txt + grep -Eq '^19 tests collected' /tmp/collect.txt pytest -q "$root/tests" - name: Verify frozen policies and contamination cutoffs shell: bash env: - PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector run: | set -euo pipefail - root=ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 python - <<'PY' import hashlib import json @@ -74,7 +75,11 @@ jobs: from late_favorite_v3 import load_frozen_policy root = Path('ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3') - source_paths = [root / 'source/late_favorite_v3.py', root / 'source/capture_policy.py'] + source_paths = [ + root / 'source/late_favorite_v3.py', + root / 'source/capture_policy.py', + root / 'collector/public_collector.py', + ] combined = hashlib.sha256() source_hashes = {} for path in source_paths: @@ -104,13 +109,17 @@ jobs: verification = { 'candidate': 'late_favorite_btc_only_v3_prospective_shadow', - 'tests_collected': 14, + 'tests_collected': 19, 'candidate_policy_sha256': trading.policy_sha256, 'capture_policy_sha256': capture.capture_policy_sha256, 'candidate_cutoff_epoch_seconds': trading.valid_market_open_after_epoch_seconds, 'capture_cutoff_epoch_seconds': capture.valid_market_open_after_epoch_seconds, 'combined_source_sha256': combined_source_sha256, 'source_files_sha256': source_hashes, + 'public_endpoints': [ + 'GET https://gamma-api.polymarket.com/markets/slug/{slug}', + 'POST https://clob.polymarket.com/books', + ], 'credentials_used': 0, 'authenticated_requests': 0, 'order_submissions': 0, @@ -123,7 +132,7 @@ jobs: print(json.dumps(verification, indent=2, sort_keys=True)) PY - - name: Verify privileged capabilities remain absent + - name: Verify capability boundaries shell: bash run: | set -euo pipefail @@ -131,16 +140,18 @@ jobs: import ast from pathlib import Path - root = Path('ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source') - forbidden_modules = { + root = Path('ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3') + core_forbidden = { 'requests', 'urllib', 'httpx', 'aiohttp', 'web3', 'py_clob_client', 'eth_account', 'websocket', 'websockets', } + privileged = {'web3', 'py_clob_client', 'eth_account', 'websocket', 'websockets'} forbidden_tokens = { 'create_order', 'post_order', 'submit_order', 'private_key', - 'api_secret', 'authorization', 'sign_transaction', + 'api_secret', 'sign_transaction', 'poly_api_key', 'poly_signature', + 'poly_passphrase', 'poly_address', } - for path in sorted(root.glob('*.py')): + for path in sorted((root / 'source').glob('*.py')): text = path.read_text() tree = ast.parse(text) imports = set() @@ -149,10 +160,29 @@ jobs: imports.update(alias.name.split('.')[0] for alias in node.names) elif isinstance(node, ast.ImportFrom) and node.module: imports.add(node.module.split('.')[0]) - assert not (imports & forbidden_modules), (path, imports & forbidden_modules) + assert not (imports & core_forbidden), (path, imports & core_forbidden) lowered = text.lower() for token in forbidden_tokens: assert token not in lowered, (path, token) + + collector = root / 'collector/public_collector.py' + text = collector.read_text() + tree = ast.parse(text) + imports = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imports.update(alias.name.split('.')[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + imports.add(node.module.split('.')[0]) + assert imports & {'requests'} + assert not (imports & privileged), imports & privileged + lowered = text.lower() + for token in forbidden_tokens: + assert token not in lowered, token + assert 'https://gamma-api.polymarket.com' in text + assert 'https://clob.polymarket.com' in text + assert 'parsed.path == "/books"' in text + assert 'parsed.path.startswith("/markets/slug/")' in text PY - name: Build deterministic observer checkpoint @@ -168,6 +198,7 @@ jobs: cp "$root/OBSERVER_VERIFICATION.json" btc-only-v3-observer-checkpoint/ cp "$root/source/late_favorite_v3.py" btc-only-v3-observer-checkpoint/ cp "$root/source/capture_policy.py" btc-only-v3-observer-checkpoint/ + cp "$root/collector/public_collector.py" btc-only-v3-observer-checkpoint/ (cd btc-only-v3-observer-checkpoint && sha256sum * > SHA256SUMS) tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ -czf btc-only-v3-observer-checkpoint-v1.tar.gz btc-only-v3-observer-checkpoint From 189b8aa63549f8d3e6fdade2fdeb24575cdb08f8 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:54:20 -0500 Subject: [PATCH 128/177] test: specify authoritative BTC-only resolution lifecycle --- .../tests/test_resolution.py | 159 ++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_resolution.py diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_resolution.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_resolution.py new file mode 100644 index 00000000..ffb20416 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_resolution.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import json +from decimal import Decimal + +import pytest + +from capture_policy import CaptureBoundLifecycleLedger, CapturePolicy +from late_favorite_v3 import FrozenPolicy +from public_collector import ( + PublicHttpClient, + RawEvidenceStore, + SingleMarketCollector, + SingleMarketResolver, + parse_terminal_outcome, +) + + +class FakeResponse: + def __init__(self, payload: object) -> None: + self.content = json.dumps(payload, separators=(",", ":")).encode() + self.status_code = 200 + + +class FakeRequester: + def __init__(self, responses: list[FakeResponse]) -> None: + self.responses = list(responses) + + def __call__(self, method: str, url: str, **kwargs: object) -> FakeResponse: + return self.responses.pop(0) + + +def trading_policy() -> FrozenPolicy: + return FrozenPolicy( + policy_sha256="1" * 64, + source_sha256="2" * 64, + valid_market_open_after_epoch_seconds=1_800_000_000, + signal_ask_min=Decimal("0.85"), + entry_second=210, + latency_seconds=1, + shares=Decimal("5"), + adverse_move_cancel=Decimal("0.01"), + fee_rate=Decimal("0.07"), + ) + + +def capture_policy() -> CapturePolicy: + return CapturePolicy( + capture_policy_sha256="3" * 64, + valid_market_open_after_epoch_seconds=1_800_000_100, + signal_target_offset_ms=210_000, + arrival_target_offset_ms=211_000, + max_request_start_lateness_ms=250, + max_request_duration_ms=1_000, + max_book_age_at_request_completion_ms=2_000, + max_book_future_skew_ms=1_000, + ) + + +def terminal_payload(opening: int, *, winner: str = "Up") -> dict[str, object]: + prices = ["1", "0"] if winner == "Up" else ["0", "1"] + return { + "conditionId": "condition-1", + "slug": f"btc-updown-5m-{opening}", + "closed": True, + "outcomes": json.dumps(["Up", "Down"]), + "outcomePrices": json.dumps(prices), + } + + +def test_parse_terminal_outcome_requires_closed_binary_terminal_prices() -> None: + opening = 1_800_000_300 + assert parse_terminal_outcome(terminal_payload(opening, winner="Down"), opening, "condition-1") == "Down" + not_closed = terminal_payload(opening) + not_closed["closed"] = False + with pytest.raises(ValueError): + parse_terminal_outcome(not_closed, opening, "condition-1") + ambiguous = terminal_payload(opening) + ambiguous["outcomePrices"] = json.dumps(["0.5", "0.5"]) + with pytest.raises(ValueError): + parse_terminal_outcome(ambiguous, opening, "condition-1") + + +def test_resolver_appends_official_fill_settlement(tmp_path) -> None: + opening = 1_800_000_300 + signal_target = opening * 1000 + 210_000 + arrival_target = opening * 1000 + 211_000 + market = { + "conditionId": "condition-1", "slug": f"btc-updown-5m-{opening}", + "active": True, "closed": False, "enableOrderBook": True, + "outcomes": json.dumps(["Up", "Down"]), + "clobTokenIds": json.dumps(["token-up", "token-down"]), + } + def book(token: str, ask: str, timestamp: int) -> dict[str, object]: + return { + "market": "condition-1", "asset_id": token, "timestamp": str(timestamp), + "hash": f"hash-{token}", "bids": [{"price": "0.10", "size": "10"}], + "asks": [{"price": ask, "size": "10"}], "min_order_size": "1", + "tick_size": "0.01", "neg_risk": False, "last_trade_price": "0.50", + } + responses = [ + FakeResponse(market), + FakeResponse([book("token-up", "0.90", signal_target), book("token-down", "0.12", signal_target)]), + FakeResponse([book("token-up", "0.90", arrival_target)]), + FakeResponse(terminal_payload(opening, winner="Up")), + ] + times = iter([ + opening * 1000 + 1000, opening * 1000 + 1020, + signal_target, signal_target + 100, + arrival_target, arrival_target + 100, + opening * 1000 + 301_000, opening * 1000 + 301_050, + ]) + client = PublicHttpClient(requester=FakeRequester(responses), clock_ms=lambda: next(times)) + ledger = CaptureBoundLifecycleLedger(tmp_path / "ledger.jsonl", trading_policy(), capture_policy()) + store = RawEvidenceStore(tmp_path / "raw") + collector = SingleMarketCollector( + client=client, evidence_store=store, ledger=ledger, + trading_policy=trading_policy(), capture_policy=capture_policy(), + sleep_until_ms=lambda _: None, + ) + assert collector.capture(opening)["decision"] == "hypothetical_fok_fill" + resolver = SingleMarketResolver( + client=client, evidence_store=store, ledger=ledger, + trading_policy=trading_policy(), capture_policy=capture_policy(), + ) + resolution = resolver.resolve("condition-1") + assert resolution["official_outcome"] == "Up" + assert resolution["official_won"] is True + assert Decimal(resolution["pnl_total"]) > 0 + assert ledger.records[-1]["event_type"] == "resolution" + assert len(store.records) == 4 + + +def test_resolver_closes_no_signal_market_without_pnl(tmp_path) -> None: + opening = 1_800_000_300 + ledger = CaptureBoundLifecycleLedger(tmp_path / "ledger.jsonl", trading_policy(), capture_policy()) + ledger.append( + condition_id="condition-1", market_open_epoch_seconds=opening, + event_type="discovered", observed_ts_ms=opening * 1000 + 1000, + payload={"slug": f"btc-updown-5m-{opening}"}, + ) + ledger.append( + condition_id="condition-1", market_open_epoch_seconds=opening, + event_type="signal", observed_ts_ms=opening * 1000 + 210_000, + payload={"decision": "no_signal_below_threshold", "signal": False}, + ) + times = iter([opening * 1000 + 301_000, opening * 1000 + 301_050]) + client = PublicHttpClient( + requester=FakeRequester([FakeResponse(terminal_payload(opening, winner="Down"))]), + clock_ms=lambda: next(times), + ) + resolver = SingleMarketResolver( + client=client, evidence_store=RawEvidenceStore(tmp_path / "raw"), ledger=ledger, + trading_policy=trading_policy(), capture_policy=capture_policy(), + ) + resolution = resolver.resolve("condition-1") + assert resolution["decision"] == "resolved_no_fill" + assert resolution["official_outcome"] == "Down" + assert resolution["pnl_total"] is None From 6b0be316ead2159481a509a69a31fa940adfaef8 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:56:02 -0500 Subject: [PATCH 129/177] research: implement authoritative BTC-only resolution --- .../collector/resolution.py | 137 ++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector/resolution.py diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector/resolution.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector/resolution.py new file mode 100644 index 00000000..4e0a6963 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector/resolution.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import json +from decimal import Decimal, InvalidOperation +from typing import Any, Mapping + +from capture_policy import CaptureBoundLifecycleLedger, CapturePolicy +from late_favorite_v3 import FrozenPolicy, settle_fill +from public_collector import ( + GAMMA_BASE, + PublicHttpClient, + PublicResponseError, + RawEvidenceStore, +) + + +def _json_string_list(value: object, name: str) -> list[str]: + if isinstance(value, str): + try: + value = json.loads(value) + except json.JSONDecodeError as exc: + raise ValueError(f"{name} is not valid JSON") from exc + if not isinstance(value, list) or not all(isinstance(item, str) for item in value): + raise ValueError(f"{name} must be a string list") + return list(value) + + +def parse_terminal_outcome( + payload: Mapping[str, Any], + expected_open_epoch_seconds: int, + expected_condition_id: str, +) -> str: + opening = int(expected_open_epoch_seconds) + if str(payload.get("slug")) != f"btc-updown-5m-{opening}": + raise ValueError("terminal Gamma slug mismatch") + if str(payload.get("conditionId") or "") != str(expected_condition_id): + raise ValueError("terminal Gamma condition mismatch") + if payload.get("closed") is not True: + raise ValueError("Gamma market is not closed") + outcomes = _json_string_list(payload.get("outcomes"), "outcomes") + raw_prices = payload.get("outcomePrices") + if isinstance(raw_prices, str): + try: + raw_prices = json.loads(raw_prices) + except json.JSONDecodeError as exc: + raise ValueError("outcomePrices is not valid JSON") from exc + if not isinstance(raw_prices, list) or len(raw_prices) != len(outcomes): + raise ValueError("outcomePrices must align with outcomes") + try: + prices = [Decimal(str(value)) for value in raw_prices] + except (InvalidOperation, ValueError) as exc: + raise ValueError("outcomePrices contains a non-decimal value") from exc + if set(outcomes) != {"Up", "Down"} or len(outcomes) != 2: + raise ValueError("terminal market must contain Up and Down") + winners = [outcome for outcome, price in zip(outcomes, prices, strict=True) if price == 1] + losers = [outcome for outcome, price in zip(outcomes, prices, strict=True) if price == 0] + if len(winners) != 1 or len(losers) != 1: + raise ValueError("outcomePrices are not terminal binary prices") + return winners[0] + + +class SingleMarketResolver: + def __init__( + self, + *, + client: PublicHttpClient, + evidence_store: RawEvidenceStore, + ledger: CaptureBoundLifecycleLedger, + trading_policy: FrozenPolicy, + capture_policy: CapturePolicy, + ) -> None: + self.client = client + self.evidence_store = evidence_store + self.ledger = ledger + self.trading_policy = trading_policy + self.capture_policy = capture_policy + + def _event_payload(self, condition_id: str, event_type: str) -> Mapping[str, Any] | None: + for record in reversed(self.ledger.records): + if record["condition_id"] == condition_id and record["event_type"] == event_type: + payload = record["payload"] + if not isinstance(payload, Mapping): + raise RuntimeError("lifecycle payload is not an object") + return payload + return None + + def _opening(self, condition_id: str) -> int: + for record in self.ledger.records: + if record["condition_id"] == condition_id: + return int(record["market_open_epoch_seconds"]) + raise KeyError(f"unknown condition_id: {condition_id}") + + def resolve(self, condition_id: str) -> dict[str, Any]: + condition = str(condition_id) + opening = self._opening(condition) + discovered = self._event_payload(condition, "discovered") + signal = self._event_payload(condition, "signal") + arrival = self._event_payload(condition, "arrival") + if discovered is None or signal is None: + raise RuntimeError("resolution requires discovered and signal lifecycle events") + slug = str(discovered.get("slug") or f"btc-updown-5m-{opening}") + evidence = self.client.request_json("GET", f"{GAMMA_BASE}/markets/slug/{slug}") + manifest = self.evidence_store.append(evidence, purpose="official_resolution") + if not isinstance(evidence.payload, Mapping): + raise PublicResponseError("Gamma terminal response must be an object") + outcome = parse_terminal_outcome(evidence.payload, opening, condition) + if arrival is not None and arrival.get("decision") == "hypothetical_fok_fill": + resolution = settle_fill( + arrival, + official_outcome=outcome, + resolution_payload_sha256=evidence.response_body_sha256, + ) + resolution["decision"] = "resolved_fill" + else: + resolution = { + "event_type": "resolution", + "condition_id": condition, + "market_open_epoch_seconds": opening, + "policy_sha256": self.trading_policy.policy_sha256, + "source_sha256": self.trading_policy.source_sha256, + "capture_policy_sha256": self.capture_policy.capture_policy_sha256, + "decision": "resolved_no_fill", + "official_outcome": outcome, + "official_won": None, + "resolution_payload_sha256": evidence.response_body_sha256, + "pnl_per_share": None, + "pnl_total": None, + } + resolution["gamma_evidence_record_hash"] = manifest["record_hash"] + self.ledger.append( + condition_id=condition, + market_open_epoch_seconds=opening, + event_type="resolution", + observed_ts_ms=evidence.request_completed_ts_ms, + payload=resolution, + ) + return resolution From e9ce0ce7b2610dabd4c3135f43d7b07e8c778fe8 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:56:37 -0500 Subject: [PATCH 130/177] test: bind resolution module boundary --- .../late-favorite-btc-only-v3/tests/test_resolution.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_resolution.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_resolution.py index ffb20416..2fd2c815 100644 --- a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_resolution.py +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_resolution.py @@ -7,13 +7,8 @@ from capture_policy import CaptureBoundLifecycleLedger, CapturePolicy from late_favorite_v3 import FrozenPolicy -from public_collector import ( - PublicHttpClient, - RawEvidenceStore, - SingleMarketCollector, - SingleMarketResolver, - parse_terminal_outcome, -) +from public_collector import PublicHttpClient, RawEvidenceStore, SingleMarketCollector +from resolution import SingleMarketResolver, parse_terminal_outcome class FakeResponse: From 87d9c3bd1f6661b0ef21311cf0d7bc9a0ff4b9e9 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:57:37 -0500 Subject: [PATCH 131/177] ci: verify authoritative BTC-only settlement --- .../btc-only-v3-observer-checkpoint.yml | 57 ++++++++++--------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/.github/workflows/btc-only-v3-observer-checkpoint.yml b/.github/workflows/btc-only-v3-observer-checkpoint.yml index 10fa5a5e..aab3f138 100644 --- a/.github/workflows/btc-only-v3-observer-checkpoint.yml +++ b/.github/workflows/btc-only-v3-observer-checkpoint.yml @@ -36,7 +36,7 @@ jobs: - name: Install pinned test stack run: python -m pip install requests==2.32.4 pytest==8.3.5 - - name: Compile observer, capture, and public collector sources + - name: Compile observer, capture, collector, and resolution sources shell: bash run: | set -euo pipefail @@ -45,11 +45,13 @@ jobs: "$root/source/late_favorite_v3.py" \ "$root/source/capture_policy.py" \ "$root/collector/public_collector.py" \ + "$root/collector/resolution.py" \ "$root/tests/test_late_favorite_v3.py" \ "$root/tests/test_capture_policy.py" \ - "$root/tests/test_public_collector.py" + "$root/tests/test_public_collector.py" \ + "$root/tests/test_resolution.py" - - name: Pass exactly 19 behavior contracts + - name: Pass exactly 22 behavior contracts shell: bash env: PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector @@ -57,7 +59,7 @@ jobs: set -euo pipefail root=ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 pytest --collect-only -q "$root/tests" | tee /tmp/collect.txt - grep -Eq '^19 tests collected' /tmp/collect.txt + grep -Eq '^22 tests collected' /tmp/collect.txt pytest -q "$root/tests" - name: Verify frozen policies and contamination cutoffs @@ -79,6 +81,7 @@ jobs: root / 'source/late_favorite_v3.py', root / 'source/capture_policy.py', root / 'collector/public_collector.py', + root / 'collector/resolution.py', ] combined = hashlib.sha256() source_hashes = {} @@ -109,7 +112,7 @@ jobs: verification = { 'candidate': 'late_favorite_btc_only_v3_prospective_shadow', - 'tests_collected': 19, + 'tests_collected': 22, 'candidate_policy_sha256': trading.policy_sha256, 'capture_policy_sha256': capture.capture_policy_sha256, 'candidate_cutoff_epoch_seconds': trading.valid_market_open_after_epoch_seconds, @@ -120,6 +123,7 @@ jobs: 'GET https://gamma-api.polymarket.com/markets/slug/{slug}', 'POST https://clob.polymarket.com/books', ], + 'official_resolution': 'Gamma closed binary terminal outcome', 'credentials_used': 0, 'authenticated_requests': 0, 'order_submissions': 0, @@ -151,38 +155,36 @@ jobs: 'api_secret', 'sign_transaction', 'poly_api_key', 'poly_signature', 'poly_passphrase', 'poly_address', } - for path in sorted((root / 'source').glob('*.py')): - text = path.read_text() - tree = ast.parse(text) + def imports_for(path: Path) -> set[str]: + tree = ast.parse(path.read_text()) imports = set() for node in ast.walk(tree): if isinstance(node, ast.Import): imports.update(alias.name.split('.')[0] for alias in node.names) elif isinstance(node, ast.ImportFrom) and node.module: imports.add(node.module.split('.')[0]) + return imports + + for path in sorted((root / 'source').glob('*.py')): + imports = imports_for(path) assert not (imports & core_forbidden), (path, imports & core_forbidden) - lowered = text.lower() + lowered = path.read_text().lower() for token in forbidden_tokens: assert token not in lowered, (path, token) - collector = root / 'collector/public_collector.py' - text = collector.read_text() - tree = ast.parse(text) - imports = set() - for node in ast.walk(tree): - if isinstance(node, ast.Import): - imports.update(alias.name.split('.')[0] for alias in node.names) - elif isinstance(node, ast.ImportFrom) and node.module: - imports.add(node.module.split('.')[0]) - assert imports & {'requests'} - assert not (imports & privileged), imports & privileged - lowered = text.lower() - for token in forbidden_tokens: - assert token not in lowered, token - assert 'https://gamma-api.polymarket.com' in text - assert 'https://clob.polymarket.com' in text - assert 'parsed.path == "/books"' in text - assert 'parsed.path.startswith("/markets/slug/")' in text + collector_paths = sorted((root / 'collector').glob('*.py')) + all_collector_imports = set().union(*(imports_for(path) for path in collector_paths)) + assert 'requests' in all_collector_imports + assert not (all_collector_imports & privileged), all_collector_imports & privileged + for path in collector_paths: + lowered = path.read_text().lower() + for token in forbidden_tokens: + assert token not in lowered, (path, token) + public_text = (root / 'collector/public_collector.py').read_text() + assert 'https://gamma-api.polymarket.com' in public_text + assert 'https://clob.polymarket.com' in public_text + assert 'parsed.path == "/books"' in public_text + assert 'parsed.path.startswith("/markets/slug/")' in public_text PY - name: Build deterministic observer checkpoint @@ -199,6 +201,7 @@ jobs: cp "$root/source/late_favorite_v3.py" btc-only-v3-observer-checkpoint/ cp "$root/source/capture_policy.py" btc-only-v3-observer-checkpoint/ cp "$root/collector/public_collector.py" btc-only-v3-observer-checkpoint/ + cp "$root/collector/resolution.py" btc-only-v3-observer-checkpoint/ (cd btc-only-v3-observer-checkpoint && sha256sum * > SHA256SUMS) tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ -czf btc-only-v3-observer-checkpoint-v1.tar.gz btc-only-v3-observer-checkpoint From d518bd33acac47116e02da8047cbb427eb0ffcc6 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 15:00:05 -0500 Subject: [PATCH 132/177] research: checkpoint BTC-only v3 prospective observer --- .../late-favorite-btc-only-v3/CHECKPOINT.md | 68 +++++++++++++++++++ .../OBSERVER_VERIFICATION.json | 25 +++++++ .../RED_CONTRACT_TRIGGER.txt | 1 - 3 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/CHECKPOINT.md create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/OBSERVER_VERIFICATION.json delete mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/RED_CONTRACT_TRIGGER.txt diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/CHECKPOINT.md b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/CHECKPOINT.md new file mode 100644 index 00000000..627814c3 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/CHECKPOINT.md @@ -0,0 +1,68 @@ +# BTC-Only v3 Prospective Observer Checkpoint + +## Status + +The frozen `late_favorite_btc_only_v3_prospective_shadow` observer and credential-free public collector are implemented and clean-room verified. This checkpoint is **infrastructure readiness only**. It contains no prospective market observations and grants no admission credit. + +## Frozen boundaries + +- Candidate policy SHA-256: `82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba` +- Candidate cutoff: `2026-07-26T19:11:11Z` +- Capture policy SHA-256: `888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc` +- Effective capture cutoff: `2026-07-26T19:35:21Z` +- Historical admission credit: **0** +- Live submission: **physically absent** + +Only BTC five-minute markets opening strictly after the effective capture cutoff may enter a prospective ledger. + +## Implemented behavior + +1. Signal target remains market open +210 seconds. +2. Arrival target remains market open +211 seconds and is independent of HTTP completion time. +3. Public request starts may be at most 250 ms late; requests longer than 1,000 ms fail closed. +4. Books older than 2,000 ms at request completion fail closed. +5. Signal uses the higher executable Up/Down ask and requires at least $0.85. +6. Arrival applies the frozen adverse-move cancel and FOK price limit. +7. Five-share execution uses cumulative full-depth levels and per-level fee accounting. +8. Every discovery, signal/no-signal, arrival/no-fill, and resolution is append-only and hash chained. +9. Official settlement uses closed binary Gamma terminal prices and raw response SHA-256 evidence. +10. Raw public response bodies are content-addressed and bound to a separate evidence manifest chain. + +## Public capability boundary + +The only allowed network operations are: + +- `GET https://gamma-api.polymarket.com/markets/slug/{slug}` +- `POST https://clob.polymarket.com/books` + +The clean-room capability scan found: + +- credentials used: **0** +- authenticated requests: **0** +- order submissions: **0** +- signing/private-key libraries: **0** + +## Verification evidence + +- Workflow run: `30217953678` +- Artifact ID: `8636344655` +- Artifact ZIP SHA-256: `667d7090ecfe3d174e5821af29a26fde2e18a87a50f7891cdcc49dff740f26af` +- Tests: **22 passed** +- Combined source SHA-256: `f2e086bb5e74f0343a4ea29f46ea2bf8e23062952b456303e204c6b5469b6126` +- Deterministic TAR sidecar: verified +- Internal `SHA256SUMS`: all files verified +- Archive extraction round-trip: byte-for-byte verified + +Source file SHA-256 values are recorded in `OBSERVER_VERIFICATION.json`. + +## Admission boundary + +This checkpoint does not show that BTC-only is profitable. The inspected official historical point estimate was marginal and post-hoc. Admission still requires genuinely new observations, at least four untouched weekly blocks, authoritative outcome coverage, measured public-book fill evidence, acceptable concentration, zero unresolved states, and a positive lower confidence bound after fees and execution costs. + +## Next valid work + +1. Add a one-shot runnable CLI that selects the next eligible BTC window and persists its ledger/evidence directories. +2. Execute the collector only on post-cutoff markets. +3. Add a separate resolver command for closed markets. +4. Package append-only ledger/evidence artifacts after each run. +5. Keep live order submission absent. diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/OBSERVER_VERIFICATION.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/OBSERVER_VERIFICATION.json new file mode 100644 index 00000000..80d7a8bd --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/OBSERVER_VERIFICATION.json @@ -0,0 +1,25 @@ +{ + "authenticated_requests": 0, + "candidate": "late_favorite_btc_only_v3_prospective_shadow", + "candidate_cutoff_epoch_seconds": 1785093071, + "candidate_policy_sha256": "82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba", + "capture_cutoff_epoch_seconds": 1785094521, + "capture_policy_sha256": "888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc", + "combined_source_sha256": "f2e086bb5e74f0343a4ea29f46ea2bf8e23062952b456303e204c6b5469b6126", + "credentials_used": 0, + "historical_admission_credit": 0, + "live_submission": "physically_absent", + "official_resolution": "Gamma closed binary terminal outcome", + "order_submissions": 0, + "public_endpoints": [ + "GET https://gamma-api.polymarket.com/markets/slug/{slug}", + "POST https://clob.polymarket.com/books" + ], + "source_files_sha256": { + "collector/public_collector.py": "85c16a1e9805b89a234139f15140c4f0f39658887419d70c2caf4c238dd5cc6b", + "collector/resolution.py": "bb4a643d4618e78442c11ef21ac15483c048a89e5bc7e900500af661f6a8c91c", + "source/capture_policy.py": "bda91808405d5b9e6ee6a09bd464cecd751c23573584bc2624891f5bf091e955", + "source/late_favorite_v3.py": "190ad8d358229f448b42a4dd8dc0e25532b440ce88a8e897fb9cb3c835cc42ee" + }, + "tests_collected": 22 +} diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/RED_CONTRACT_TRIGGER.txt b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/RED_CONTRACT_TRIGGER.txt deleted file mode 100644 index 0555861e..00000000 --- a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/RED_CONTRACT_TRIGGER.txt +++ /dev/null @@ -1 +0,0 @@ -Observer source intentionally absent at this checkpoint; the isolated contract must fail before implementation. From 4105f9d83f625e7ecaa4bcbdd13ff64a3fea670a Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 15:04:57 -0500 Subject: [PATCH 133/177] test: specify BTC-only v3 one-shot runner --- .../tests/test_observer_cli.py | 152 ++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_observer_cli.py diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_observer_cli.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_observer_cli.py new file mode 100644 index 00000000..f9b75a58 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_observer_cli.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path + +import pytest + +from observer_cli import ( + ObserverRuntime, + build_parser, + collect_next, + compute_combined_source_sha256, + resolve_condition, + runtime_status, +) + + +@dataclass +class FakeCapturePolicy: + valid_market_open_after_epoch_seconds: int = 1_800_000_100 + + +class FakeCollector: + def __init__(self) -> None: + self.openings: list[int] = [] + + def capture(self, opening: int) -> dict[str, object]: + self.openings.append(opening) + return { + "condition_id": f"condition-{opening}", + "market_open_epoch_seconds": opening, + "decision": "hypothetical_fok_fill", + } + + +class FakeResolver: + def __init__(self) -> None: + self.conditions: list[str] = [] + + def resolve(self, condition_id: str) -> dict[str, object]: + self.conditions.append(condition_id) + return { + "condition_id": condition_id, + "decision": "resolved_fill", + "official_outcome": "Up", + "pnl_total": "0.45", + } + + +class FakeLedger: + def __init__(self) -> None: + self.head_hash = "a" * 64 + self.records = [ + { + "condition_id": "resolved", + "event_type": "resolution", + "record_hash": "1" * 64, + }, + { + "condition_id": "unresolved", + "event_type": "signal", + "record_hash": "2" * 64, + }, + ] + + +class FakeEvidence: + head_hash = "b" * 64 + records = [{"record_hash": "3" * 64}] + + +def runtime(tmp_path: Path, *, now_ms: int, sleeps: list[int]) -> ObserverRuntime: + return ObserverRuntime( + project_root=tmp_path, + state_dir=tmp_path / "state", + capture_policy=FakeCapturePolicy(), + ledger=FakeLedger(), + evidence_store=FakeEvidence(), + collector=FakeCollector(), + resolver=FakeResolver(), + clock_ms=lambda: now_ms, + sleep_until_ms=sleeps.append, + source_sha256="c" * 64, + ) + + +def test_combined_source_hash_binds_all_runtime_modules(tmp_path: Path) -> None: + root = tmp_path / "candidate" + (root / "source").mkdir(parents=True) + (root / "collector").mkdir(parents=True) + files = { + root / "source/late_favorite_v3.py": b"core\n", + root / "source/capture_policy.py": b"capture\n", + root / "collector/public_collector.py": b"collector\n", + root / "collector/resolution.py": b"resolution\n", + root / "runner/observer_cli.py": b"runner\n", + } + (root / "runner").mkdir(parents=True) + for path, content in files.items(): + path.write_bytes(content) + first = compute_combined_source_sha256(root) + assert len(first) == 64 + files[root / "runner/observer_cli.py"] = b"runner changed\n" + (root / "runner/observer_cli.py").write_bytes(files[root / "runner/observer_cli.py"]) + assert compute_combined_source_sha256(root) != first + + +def test_collect_next_selects_future_window_and_persists_summary(tmp_path: Path) -> None: + opening = 1_800_000_300 + now_ms = opening * 1000 + 210_001 + sleeps: list[int] = [] + rt = runtime(tmp_path, now_ms=now_ms, sleeps=sleeps) + summary = collect_next(rt, now_ts_ms=now_ms) + selected = opening + 300 + assert rt.collector.openings == [selected] + assert sleeps == [selected * 1000 + 1_000] + assert summary["condition_id"] == f"condition-{selected}" + assert summary["lifecycle_head_sha256"] == "a" * 64 + assert summary["raw_evidence_head_sha256"] == "b" * 64 + saved = json.loads((rt.state_dir / "summaries" / f"collect-{selected}.json").read_text()) + assert saved == summary + with pytest.raises(FileExistsError): + collect_next(rt, now_ts_ms=now_ms) + + +def test_resolve_condition_persists_official_summary(tmp_path: Path) -> None: + rt = runtime(tmp_path, now_ms=1_800_000_000_000, sleeps=[]) + summary = resolve_condition(rt, "condition-1") + assert rt.resolver.conditions == ["condition-1"] + assert summary["official_outcome"] == "Up" + path = rt.state_dir / "summaries" / "resolve-condition-1.json" + assert json.loads(path.read_text()) == summary + + +def test_runtime_status_lists_unresolved_conditions(tmp_path: Path) -> None: + status = runtime_status(runtime(tmp_path, now_ms=0, sleeps=[])) + assert status["conditions_seen"] == 2 + assert status["conditions_resolved"] == 1 + assert status["unresolved_condition_ids"] == ["unresolved"] + assert status["lifecycle_head_sha256"] == "a" * 64 + + +def test_cli_exposes_only_collect_resolve_and_status() -> None: + parser = build_parser() + assert parser.parse_args(["collect-next"]).command == "collect-next" + assert parser.parse_args(["resolve", "--condition-id", "c1"]).condition_id == "c1" + assert parser.parse_args(["status"]).command == "status" + with pytest.raises(SystemExit): + parser.parse_args(["live"]) + with pytest.raises(SystemExit): + parser.parse_args(["submit-order"]) From 62b21d5e375e766c1607dfa498c1142d7cc63cfd Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 15:07:48 -0500 Subject: [PATCH 134/177] research: implement BTC-only v3 one-shot runner --- .../runner/observer_cli.py | 257 ++++++++++++++++++ 1 file changed, 257 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/observer_cli.py diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/observer_cli.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/observer_cli.py new file mode 100644 index 00000000..0687631f --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/observer_cli.py @@ -0,0 +1,257 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable + +from capture_policy import CaptureBoundLifecycleLedger, load_capture_policy +from late_favorite_v3 import load_frozen_policy +from public_collector import ( + PublicHttpClient, + RawEvidenceStore, + SingleMarketCollector, + next_collectable_open, +) +from resolution import SingleMarketResolver + +_RUNTIME_SOURCE_PATHS = ( + "source/late_favorite_v3.py", + "source/capture_policy.py", + "collector/public_collector.py", + "collector/resolution.py", + "runner/observer_cli.py", +) + + +@dataclass(slots=True) +class ObserverRuntime: + project_root: Path + state_dir: Path + capture_policy: Any + ledger: Any + evidence_store: Any + collector: Any + resolver: Any + clock_ms: Callable[[], int] + sleep_until_ms: Callable[[int], None] + source_sha256: str + trading_policy: Any = None + client: Any = None + + +def compute_combined_source_sha256(project_root: str | Path) -> str: + root = Path(project_root) + digest = hashlib.sha256() + for relative in _RUNTIME_SOURCE_PATHS: + path = root / relative + raw = path.read_bytes() + digest.update(relative.encode("utf-8") + b"\0" + raw) + return digest.hexdigest() + + +def _real_clock_ms() -> int: + return time.time_ns() // 1_000_000 + + +def make_sleep_until_ms( + clock_ms: Callable[[], int], + sleeper: Callable[[float], None] = time.sleep, +) -> Callable[[int], None]: + def sleep_until(target_ts_ms: int) -> None: + target = int(target_ts_ms) + while True: + remaining_ms = target - int(clock_ms()) + if remaining_ms <= 0: + return + sleeper(min(remaining_ms / 1000.0, 0.25)) + + return sleep_until + + +def build_runtime( + project_root: str | Path, + state_dir: str | Path, + *, + clock_ms: Callable[[], int] | None = None, + sleep_until_ms: Callable[[int], None] | None = None, + requester: Callable[..., Any] | None = None, +) -> ObserverRuntime: + root = Path(project_root).resolve() + state = Path(state_dir).resolve() + source_sha256 = compute_combined_source_sha256(root) + trading_policy = load_frozen_policy( + root / "CANDIDATE_SPEC.json", + root / "FREEZE_MANIFEST.json", + source_sha256=source_sha256, + ) + capture_policy = load_capture_policy( + root / "CAPTURE_POLICY.json", + root / "CAPTURE_FREEZE_MANIFEST.json", + ) + clock = clock_ms or _real_clock_ms + sleep_until = sleep_until_ms or make_sleep_until_ms(clock) + ledger = CaptureBoundLifecycleLedger( + state / "lifecycle.jsonl", + trading_policy, + capture_policy, + ) + evidence = RawEvidenceStore(state / "raw_evidence") + client_kwargs: dict[str, Any] = {"clock_ms": clock} + if requester is not None: + client_kwargs["requester"] = requester + client = PublicHttpClient(**client_kwargs) + collector = SingleMarketCollector( + client=client, + evidence_store=evidence, + ledger=ledger, + trading_policy=trading_policy, + capture_policy=capture_policy, + sleep_until_ms=sleep_until, + ) + resolver = SingleMarketResolver( + client=client, + evidence_store=evidence, + ledger=ledger, + trading_policy=trading_policy, + capture_policy=capture_policy, + ) + return ObserverRuntime( + project_root=root, + state_dir=state, + capture_policy=capture_policy, + ledger=ledger, + evidence_store=evidence, + collector=collector, + resolver=resolver, + clock_ms=clock, + sleep_until_ms=sleep_until, + source_sha256=source_sha256, + trading_policy=trading_policy, + client=client, + ) + + +def _summary_path(runtime: ObserverRuntime, name: str) -> Path: + safe = re.sub(r"[^A-Za-z0-9._-]+", "_", str(name)).strip("._") + if not safe: + raise ValueError("summary name is empty after sanitization") + return runtime.state_dir / "summaries" / f"{safe}.json" + + +def _write_summary(runtime: ObserverRuntime, name: str, summary: dict[str, Any]) -> dict[str, Any]: + path = _summary_path(runtime, name) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("x", encoding="utf-8", newline="\n") as handle: + json.dump(summary, handle, indent=2, sort_keys=True) + handle.write("\n") + return summary + + +def _evidence_head(runtime: ObserverRuntime) -> str: + return str(getattr(runtime.evidence_store, "head_hash", "0" * 64)) + + +def collect_next(runtime: ObserverRuntime, *, now_ts_ms: int | None = None) -> dict[str, Any]: + now = int(runtime.clock_ms() if now_ts_ms is None else now_ts_ms) + opening = next_collectable_open( + now, + int(runtime.capture_policy.valid_market_open_after_epoch_seconds), + ) + discovery_target = opening * 1000 + 1_000 + if now < discovery_target: + runtime.sleep_until_ms(discovery_target) + result = dict(runtime.collector.capture(opening)) + summary = { + **result, + "command": "collect-next", + "market_open_epoch_seconds": opening, + "source_sha256": runtime.source_sha256, + "lifecycle_head_sha256": str(runtime.ledger.head_hash), + "raw_evidence_head_sha256": _evidence_head(runtime), + "lifecycle_records": len(getattr(runtime.ledger, "records", [])), + "raw_evidence_records": len(getattr(runtime.evidence_store, "records", [])), + } + return _write_summary(runtime, f"collect-{opening}", summary) + + +def resolve_condition(runtime: ObserverRuntime, condition_id: str) -> dict[str, Any]: + condition = str(condition_id).strip() + if not condition: + raise ValueError("condition_id must be non-empty") + result = dict(runtime.resolver.resolve(condition)) + summary = { + **result, + "command": "resolve", + "source_sha256": runtime.source_sha256, + "lifecycle_head_sha256": str(runtime.ledger.head_hash), + "raw_evidence_head_sha256": _evidence_head(runtime), + "lifecycle_records": len(getattr(runtime.ledger, "records", [])), + "raw_evidence_records": len(getattr(runtime.evidence_store, "records", [])), + } + return _write_summary(runtime, f"resolve-{condition}", summary) + + +def runtime_status(runtime: ObserverRuntime) -> dict[str, Any]: + condition_ids: set[str] = set() + resolved: set[str] = set() + event_counts: dict[str, int] = {} + for record in getattr(runtime.ledger, "records", []): + condition = str(record.get("condition_id", "")) + event_type = str(record.get("event_type", "")) + if condition: + condition_ids.add(condition) + if event_type == "resolution": + resolved.add(condition) + event_counts[event_type] = event_counts.get(event_type, 0) + 1 + return { + "command": "status", + "conditions_seen": len(condition_ids), + "conditions_resolved": len(resolved), + "unresolved_condition_ids": sorted(condition_ids - resolved), + "event_counts": dict(sorted(event_counts.items())), + "source_sha256": runtime.source_sha256, + "lifecycle_head_sha256": str(runtime.ledger.head_hash), + "raw_evidence_head_sha256": _evidence_head(runtime), + "lifecycle_records": len(getattr(runtime.ledger, "records", [])), + "raw_evidence_records": len(getattr(runtime.evidence_store, "records", [])), + } + + +def build_parser() -> argparse.ArgumentParser: + default_root = Path(__file__).resolve().parents[1] + parser = argparse.ArgumentParser( + description="Credential-free BTC-only v3 prospective shadow runner" + ) + parser.add_argument("--project-root", type=Path, default=default_root) + parser.add_argument("--state-dir", type=Path, default=Path(".prospective/btc-only-v3")) + commands = parser.add_subparsers(dest="command", required=True) + collect = commands.add_parser("collect-next", help="collect the next eligible BTC window") + collect.add_argument("--now-ms", type=int) + resolve = commands.add_parser("resolve", help="resolve one collected condition") + resolve.add_argument("--condition-id", required=True) + commands.add_parser("status", help="show ledger and unresolved-condition status") + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + runtime = build_runtime(args.project_root, args.state_dir) + if args.command == "collect-next": + output = collect_next(runtime, now_ts_ms=args.now_ms) + elif args.command == "resolve": + output = resolve_condition(runtime, args.condition_id) + elif args.command == "status": + output = runtime_status(runtime) + else: # pragma: no cover + raise RuntimeError(f"unsupported command: {args.command}") + print(json.dumps(output, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 03941cdbc763366cdb4d4f07f123d5c1e99413e8 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 15:09:23 -0500 Subject: [PATCH 135/177] test: isolate BTC-only runner contracts --- .../{ => runner}/tests/test_observer_cli.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/{ => runner}/tests/test_observer_cli.py (100%) diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_observer_cli.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/tests/test_observer_cli.py similarity index 100% rename from ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_observer_cli.py rename to ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/tests/test_observer_cli.py From 695ac1159db800424284353d4197e3ffc111b7a1 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 15:10:19 -0500 Subject: [PATCH 136/177] ci: add BTC-only v3 runner checkpoint --- .../btc-only-v3-runner-checkpoint.yml | 174 ++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 .github/workflows/btc-only-v3-runner-checkpoint.yml diff --git a/.github/workflows/btc-only-v3-runner-checkpoint.yml b/.github/workflows/btc-only-v3-runner-checkpoint.yml new file mode 100644 index 00000000..5af6cef2 --- /dev/null +++ b/.github/workflows/btc-only-v3-runner-checkpoint.yml @@ -0,0 +1,174 @@ +name: BTC-Only v3 Runner Checkpoint + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/btc-only-v3-runner-checkpoint.yml + - ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/** + pull_request: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/btc-only-v3-runner-checkpoint.yml + - ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/** + workflow_dispatch: + +concurrency: + group: btc-only-v3-runner-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + runner-contract: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install pinned test stack + run: python -m pip install requests==2.32.4 pytest==8.3.5 + + - name: Compile runner source and tests + shell: bash + run: | + set -euo pipefail + root=ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 + python -m py_compile \ + "$root/runner/observer_cli.py" \ + "$root/runner/tests/test_observer_cli.py" + + - name: Pass exactly 5 runner contracts + shell: bash + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner + run: | + set -euo pipefail + root=ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 + pytest --collect-only -q "$root/runner/tests" | tee /tmp/collect.txt + grep -Eq '^5 tests collected' /tmp/collect.txt + pytest -q "$root/runner/tests" + + - name: Verify runtime loads frozen policies without network + shell: bash + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner + run: | + set -euo pipefail + python - <<'PY' + import hashlib + import json + import tempfile + from pathlib import Path + + from observer_cli import build_runtime, compute_combined_source_sha256, runtime_status + + root = Path('ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3') + relative_paths = ( + 'source/late_favorite_v3.py', + 'source/capture_policy.py', + 'collector/public_collector.py', + 'collector/resolution.py', + 'runner/observer_cli.py', + ) + independent = hashlib.sha256() + source_hashes = {} + for relative in relative_paths: + raw = (root / relative).read_bytes() + source_hashes[relative] = hashlib.sha256(raw).hexdigest() + independent.update(relative.encode('utf-8') + b'\0' + raw) + source_sha256 = compute_combined_source_sha256(root) + assert source_sha256 == independent.hexdigest() + with tempfile.TemporaryDirectory() as directory: + runtime = build_runtime(root, directory, clock_ms=lambda: 1_900_000_000_000) + status = runtime_status(runtime) + assert status['conditions_seen'] == 0 + assert status['conditions_resolved'] == 0 + assert status['source_sha256'] == source_sha256 + assert status['lifecycle_head_sha256'] == '0' * 64 + assert status['raw_evidence_head_sha256'] == '0' * 64 + verification = { + 'candidate': 'late_favorite_btc_only_v3_prospective_shadow', + 'tests_collected': 5, + 'runner_commands': ['collect-next', 'resolve', 'status'], + 'combined_runtime_source_sha256': source_sha256, + 'source_files_sha256': source_hashes, + 'credentials_used': 0, + 'authenticated_requests': 0, + 'order_submissions': 0, + 'live_submission': 'physically_absent', + 'historical_admission_credit': 0, + } + out = root / 'runner/RUNNER_VERIFICATION.json' + out.write_text(json.dumps(verification, indent=2, sort_keys=True) + '\n') + print(json.dumps(verification, indent=2, sort_keys=True)) + PY + + - name: Verify runner has no privileged capability + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import ast + from pathlib import Path + + path = Path('ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/observer_cli.py') + text = path.read_text() + tree = ast.parse(text) + imports = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imports.update(alias.name.split('.')[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + imports.add(node.module.split('.')[0]) + forbidden_modules = { + 'requests', 'urllib', 'httpx', 'aiohttp', 'web3', + 'py_clob_client', 'eth_account', 'websocket', 'websockets', + } + assert not (imports & forbidden_modules), imports & forbidden_modules + lowered = text.lower() + for token in ( + 'create_order', 'post_order', 'submit_order', 'private_key', + 'api_secret', 'sign_transaction', 'poly_api_key', 'poly_signature', + 'poly_passphrase', 'poly_address', + ): + assert token not in lowered, token + assert 'collect-next' in text + assert 'resolve' in text + assert 'status' in text + assert 'submit-order' not in text + PY + + - name: Build deterministic runner checkpoint + shell: bash + run: | + set -euo pipefail + root=ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 + mkdir -p btc-only-v3-runner-checkpoint + cp "$root/CANDIDATE_SPEC.json" btc-only-v3-runner-checkpoint/ + cp "$root/FREEZE_MANIFEST.json" btc-only-v3-runner-checkpoint/ + cp "$root/CAPTURE_POLICY.json" btc-only-v3-runner-checkpoint/ + cp "$root/CAPTURE_FREEZE_MANIFEST.json" btc-only-v3-runner-checkpoint/ + cp "$root/runner/observer_cli.py" btc-only-v3-runner-checkpoint/ + cp "$root/runner/RUNNER_VERIFICATION.json" btc-only-v3-runner-checkpoint/ + (cd btc-only-v3-runner-checkpoint && sha256sum * > SHA256SUMS) + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf btc-only-v3-runner-checkpoint-v1.tar.gz btc-only-v3-runner-checkpoint + sha256sum btc-only-v3-runner-checkpoint-v1.tar.gz > btc-only-v3-runner-checkpoint-v1.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: btc-only-v3-runner-checkpoint-v1 + path: | + btc-only-v3-runner-checkpoint/ + btc-only-v3-runner-checkpoint-v1.tar.gz + btc-only-v3-runner-checkpoint-v1.tar.gz.sha256 + compression-level: 0 + retention-days: 30 From 4f997d70293f3b9aa9db5331657e76b05809eb7b Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 15:14:42 -0500 Subject: [PATCH 137/177] docs: checkpoint BTC-only v3 runner readiness --- .../digital-surface-live-readiness/STATUS.md | 97 +++++++++++++------ .../runner/CHECKPOINT.md | 53 ++++++++++ .../runner/RUNNER_VERIFICATION.json | 22 +++++ 3 files changed, 143 insertions(+), 29 deletions(-) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/CHECKPOINT.md create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/RUNNER_VERIFICATION.json diff --git a/ci_payload/digital-surface-live-readiness/STATUS.md b/ci_payload/digital-surface-live-readiness/STATUS.md index ac3c4388..bdcf1b00 100644 --- a/ci_payload/digital-surface-live-readiness/STATUS.md +++ b/ci_payload/digital-surface-live-readiness/STATUS.md @@ -6,65 +6,104 @@ - Branch: `digital-surface-live-readiness-20260725` - Draft PR: #59 - Official late-favorite decision commit: `cedd659e0b573d8479932c428cd2eaf472c6bf8f` +- BTC-only v3 observer merge: `e88eada8589119baf755f0ab9f40d37160ff125a` +- BTC-only v3 runner merge: `9e794700600ba8a9173040711d09b7feb4817ff9` - Real-money execution: **physically disabled** -## Research verdicts +## Historical research verdicts ### Original 5m-vs-15m digital-surface sleeve -The checksum-pinned v5 research completed the official-resolution source gate and all four historical transfer folds, but the strategy remained **Rejected**. Those four weeks are diagnostic and may never be reused as prospective admission evidence. +The official-resolution source gate and all four historical transfer folds completed, but the frozen sleeve remained **Rejected**. Those folds are diagnostic only and may never count toward prospective admission. ### Late-favorite BTC/ETH candidate -The frozen `late_favorite_btc_eth_v2_diagnostic` rule was rebuilt over 27,940 markets and transferred across its immutable 9,268-fill condition set to authoritative Polymarket terminal outcomes with 100% coverage. - -The candidate is **Rejected / falsified**: +The immutable 9,268-fill set was transferred to authoritative Polymarket terminal outcomes with 100% coverage. The candidate is **Rejected / falsified**: - official win rate: 94.3893%; - official mean P&L/share: -$0.00127314; - official P&L at five shares: -$59.00; - BTC: +$64.00, but its daily confidence interval crosses zero; -- ETH: -$123.00; -- BTC-only retention would be post-hoc and receives no inherited validation status. +- ETH: -$123.00. -The earlier +$428.93 inferred-label diagnostic was biased by 385 null-labeled fills. Official resolution showed that cohort had only 67.53% wins and lost $472.92 at five shares. The nine direct label disagreements changed comparable P&L by only -$15.00. The material defect was non-random label-availability censoring. +The earlier +$428.93 inferred-label result was biased by 385 null-labeled fills. Official resolution showed that excluded cohort had only 67.53% wins and lost $472.92. Nine direct label disagreements changed comparable P&L by only -$15.00; the material defect was non-random label-availability censoring. -Authoritative decision checkpoint: +Authoritative decision evidence: - `ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/CHECKPOINT.md` - filled condition-set SHA-256: `bf276fbce187d9651d56ecae976520d4251140c1d5b0d440696c5d077838a67a` -- full official evidence artifact SHA-256: `210984541bcd64597ca48d480ad59102427f56b4ac204ab1308ad8f8a4f93ce5` +- full official evidence artifact SHA-256: `210984541bcd64597ca48d480ad59102427f56b4ac204ab1308adf8a4f93ce5` - compact derivative artifact SHA-256: `fb5dfb4621caa8b62ebaefc0b5c15d04619f773c3686cc4f088767ca023416b9` -## Original sleeve execution diagnosis +## BTC-only v3 prospective shadow track + +BTC-only is a **marginal post-hoc lead**, not a historical pass. It has zero inherited validation or admission credit. + +### Frozen candidate and capture boundaries + +- candidate: `late_favorite_btc_only_v3_prospective_shadow` +- candidate policy SHA-256: `82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba` +- candidate freeze: `2026-07-26T19:11:11Z` +- capture policy SHA-256: `888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc` +- effective capture cutoff: `2026-07-26T19:35:21Z` +- only markets opening strictly after the effective capture cutoff may count. + +### Observer, collector, and resolver + +The merged observer implements: + +1. market-open-anchored signal at +210 seconds and arrival at +211 seconds; +2. frozen request-start, duration, book-age, and future-skew limits; +3. Decimal full-depth five-share FOK VWAP and per-level fee accounting; +4. adverse-move, price-limit, depth, stale, timeout, token, and lifecycle fail-closed decisions; +5. append-only candidate/capture/source-bound lifecycle records; +6. content-addressed raw public evidence and a separate manifest hash chain; +7. public Gamma market-by-slug discovery and terminal resolution; +8. public CLOB `POST /books` signal and arrival snapshots; +9. official fill P&L and terminal no-fill closure. + +Observer verification: + +- 22/22 behavior contracts passed; +- workflow run: `30217953678`; +- artifact ID: `8636344655`; +- artifact ZIP SHA-256: `667d7090ecfe3d174e5821af29a26fde2e18a87a50f7891cdcc49dff740f26af`; +- combined source SHA-256: `f2e086bb5e74f0343a4ea29f46ea2bf8e23062952b456303e204c6b5469b6126`. -Across 224 candidate portfolios / 448 legs: +### One-shot runner -- insufficient post-arrival tape: 186 legs; -- ask above frozen limit: 114 legs; -- missing post-arrival book: 110 legs; -- filled: 34 legs; -- insufficient displayed depth: 4 legs. +The merged runner exposes only: -The original tape-confirmed model produced 8 atomic portfolios. A diagnostic post-latency book-arrival replay produced 62 paired-crossable portfolios, with fold counts `[11, 17, 29, 5]`. This alternative is not live truth. +- `collect-next`; +- `resolve --condition-id`; +- `status`. -## Current source state +Runner verification: -The branch contains checksum-pinned, human-readable recovery payloads for official outcome transfer, deterministic archives, execution diagnostics, public market/book observation, Binance state, causal prospective signals, shadow-only FOK preparation, risk controls, prospective ledgers, report chains, and contamination rejection. +- 5/5 runner contracts passed; +- workflow run: `30218421705`; +- artifact ID: `8636471934`; +- artifact ZIP SHA-256: `762294c4141c25d6dd0a1f6678e2f90795bfbecf909deaf130d3001358ca7b88`; +- combined runtime source SHA-256: `b398e7b9893170cb47ea17923448a87a10b017249a950ca670f9b09081b0d845`. -A focused regression now requires hypothetical fills to retain their known execution fee even when a diagnostic inferred outcome is null. This is an audit-field correction only; the official transfer already recomputed fees from executable asks and the rejection verdict is unchanged. +The known-fee field is now recorded for hypothetical fills even when diagnostic inferred outcomes are null. This audit correction does not change the official rejection result. -## Admission boundary +## Safety and admission boundary -No real-money order submission implementation exists in the live package. `TRADING_MODE=live` raises `LiveTradingDisabledError` even if credentials and an arm token are supplied. No historical result in either strategy track is admissible. +- credentials used: **0**; +- authenticated requests: **0**; +- order submissions: **0**; +- live submission: **physically absent**; +- historical admission credit: **0**; +- prospective markets collected so far: **0**. -A future candidate may be classified `Admissible` only from a newly frozen policy and genuinely new prospective markets. Even then, live mode remains `disabled_pending_operator_release` until a separately reviewed release and explicit operator action. +No historical result is admissible. Even a future admissible shadow report cannot enable live trading; any order submitter requires a separate reviewed release and explicit operator action. ## Next work -1. complete and verify the fee-field regression without changing strategy geometry; -2. preserve both rejected historical tracks as diagnostic evidence only; -3. freeze any BTC-only or replacement hypothesis as a new prospective version before observing new data; -4. run a credential-free append-only observer with authoritative outcomes, complete no-trade/fill records, and measured execution evidence; -5. require at least four untouched weekly prospective blocks and a positive lower confidence bound before reconsidering admission; +1. execute one credential-free `collect-next` run on a genuinely new post-cutoff BTC window; +2. preserve its append-only lifecycle, raw evidence, summaries, policy hashes, source hash, and artifact digest; +3. resolve the condition from authoritative terminal outcomes after closure; +4. repeat without policy changes for at least four untouched weekly blocks; +5. require measured fill evidence, acceptable concentration, zero unresolved states, and a positive lower confidence bound after fees and execution costs; 6. keep real-money execution physically disabled. diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/CHECKPOINT.md b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/CHECKPOINT.md new file mode 100644 index 00000000..3443a79d --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/CHECKPOINT.md @@ -0,0 +1,53 @@ +# BTC-Only v3 One-Shot Runner Checkpoint + +## Status + +The credential-free one-shot runner for `late_favorite_btc_only_v3_prospective_shadow` is implemented, clean-room verified, independently artifact-checked, and merged into the durable live-readiness branch. + +This checkpoint adds orchestration only. It does not change the frozen candidate, capture policy, execution model, or admission gates. + +## Commands + +- `collect-next`: choose the next eligible post-cutoff BTC five-minute window, persist discovery/signal/arrival evidence, and write a non-overwriting summary. +- `resolve --condition-id`: fetch authoritative terminal resolution and append official fill P&L or no-fill closure. +- `status`: report lifecycle/evidence heads, event counts, and unresolved condition IDs. + +There is no live, trading, signing, or order-submission command. + +## Persistence + +The runner loads the two frozen policies, binds all five runtime modules into the lifecycle source hash, and persists: + +- `lifecycle.jsonl` — append-only candidate/capture/source-bound lifecycle chain; +- `raw_evidence/manifest.jsonl` and content-addressed response bodies; +- exclusive `summaries/collect-.json` files; +- exclusive `summaries/resolve-.json` files. + +Duplicate summaries fail closed rather than overwriting evidence. + +## Verification + +- Runner PR: #67 +- Merge commit: `9e794700600ba8a9173040711d09b7feb4817ff9` +- Workflow run: `30218421705` +- Artifact ID: `8636471934` +- Artifact ZIP SHA-256: `762294c4141c25d6dd0a1f6678e2f90795bfbecf909deaf130d3001358ca7b88` +- Tests: **5 passed** +- Combined runtime source SHA-256: `b398e7b9893170cb47ea17923448a87a10b017249a950ca670f9b09081b0d845` +- Internal `SHA256SUMS`: all files verified +- Deterministic TAR sidecar: verified +- Archive extraction round-trip: byte-for-byte verified + +The unchanged observer core also passed its independent 22-contract checkpoint on the runner PR. + +## Safety boundary + +- credentials used: **0** +- authenticated requests: **0** +- order submissions: **0** +- live submission: **physically absent** +- historical admission credit: **0** + +## Next valid work + +Run `collect-next` in a one-shot GitHub workflow, upload the resulting append-only state artifact, then resolve that condition after official closure. Only post-capture-freeze markets may count. diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/RUNNER_VERIFICATION.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/RUNNER_VERIFICATION.json new file mode 100644 index 00000000..ded85564 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/RUNNER_VERIFICATION.json @@ -0,0 +1,22 @@ +{ + "authenticated_requests": 0, + "candidate": "late_favorite_btc_only_v3_prospective_shadow", + "combined_runtime_source_sha256": "b398e7b9893170cb47ea17923448a87a10b017249a950ca670f9b09081b0d845", + "credentials_used": 0, + "historical_admission_credit": 0, + "live_submission": "physically_absent", + "order_submissions": 0, + "runner_commands": [ + "collect-next", + "resolve", + "status" + ], + "source_files_sha256": { + "collector/public_collector.py": "85c16a1e9805b89a234139f15140c4f0f39658887419d70c2caf4c238dd5cc6b", + "collector/resolution.py": "bb4a643d4618e78442c11ef21ac15483c048a89e5bc7e900500af661f6a8c91c", + "runner/observer_cli.py": "07025cd07f79787c69d6ff6cd189f01c846e5b8c824a727cea63fe1e288125e7", + "source/capture_policy.py": "bda91808405d5b9e6ee6a09bd464cecd751c23573584bc2624891f5bf091e955", + "source/late_favorite_v3.py": "190ad8d358229f448b42a4dd8dc0e25532b440ce88a8e897fb9cb3c835cc42ee" + }, + "tests_collected": 5 +} From c9609cf1d8437079e9cb56d18700150f884aac45 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 15:16:29 -0500 Subject: [PATCH 138/177] research: launch first BTC-only v3 prospective observation --- .../btc-only-v3-prospective-one-shot.yml | 237 ++++++++++++++++++ 1 file changed, 237 insertions(+) create mode 100644 .github/workflows/btc-only-v3-prospective-one-shot.yml diff --git a/.github/workflows/btc-only-v3-prospective-one-shot.yml b/.github/workflows/btc-only-v3-prospective-one-shot.yml new file mode 100644 index 00000000..cf2f47de --- /dev/null +++ b/.github/workflows/btc-only-v3-prospective-one-shot.yml @@ -0,0 +1,237 @@ +name: BTC-Only v3 Prospective One-Shot + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/btc-only-v3-prospective-one-shot.yml + workflow_dispatch: + +concurrency: + group: btc-only-v3-prospective-one-shot + cancel-in-progress: false + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + CANDIDATE_ROOT: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 + STATE_DIR: btc-only-v3-prospective-state + +jobs: + collect-and-resolve-one-market: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + ref: digital-surface-live-readiness-20260725 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install pinned public client + run: python -m pip install requests==2.32.4 + + - name: Verify frozen source and command boundary + shell: bash + run: | + set -euo pipefail + python -m py_compile \ + "$CANDIDATE_ROOT/source/late_favorite_v3.py" \ + "$CANDIDATE_ROOT/source/capture_policy.py" \ + "$CANDIDATE_ROOT/collector/public_collector.py" \ + "$CANDIDATE_ROOT/collector/resolution.py" \ + "$CANDIDATE_ROOT/runner/observer_cli.py" + PYTHONPATH="$CANDIDATE_ROOT/source:$CANDIDATE_ROOT/collector:$CANDIDATE_ROOT/runner" \ + python "$CANDIDATE_ROOT/runner/observer_cli.py" \ + --project-root "$CANDIDATE_ROOT" \ + --state-dir "$STATE_DIR" \ + status | tee initial-status.json + python - <<'PY' + import json + from pathlib import Path + status = json.loads(Path('initial-status.json').read_text()) + assert status['conditions_seen'] == 0 + assert status['conditions_resolved'] == 0 + assert status['lifecycle_head_sha256'] == '0' * 64 + assert status['raw_evidence_head_sha256'] == '0' * 64 + PY + + - name: Collect next genuinely new BTC window + shell: bash + run: | + set -euo pipefail + PYTHONPATH="$CANDIDATE_ROOT/source:$CANDIDATE_ROOT/collector:$CANDIDATE_ROOT/runner" \ + python "$CANDIDATE_ROOT/runner/observer_cli.py" \ + --project-root "$CANDIDATE_ROOT" \ + --state-dir "$STATE_DIR" \ + collect-next | tee collect-output.json + python - <<'PY' + import json + from pathlib import Path + output = json.loads(Path('collect-output.json').read_text()) + condition = str(output['condition_id']) + opening = int(output['market_open_epoch_seconds']) + assert opening > 1785094521 + assert output['command'] == 'collect-next' + assert len(output['lifecycle_head_sha256']) == 64 + assert len(output['raw_evidence_head_sha256']) == 64 + with Path('/tmp/prospective.env').open('w') as handle: + handle.write(f'CONDITION_ID={condition}\n') + handle.write(f'MARKET_OPEN_EPOCH_SECONDS={opening}\n') + handle.write(f'COLLECT_DECISION={output["decision"]}\n') + print(json.dumps(output, indent=2, sort_keys=True)) + PY + cat /tmp/prospective.env >> "$GITHUB_ENV" + + - name: Wait for terminal resolution window + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import os + import time + opening = int(os.environ['MARKET_OPEN_EPOCH_SECONDS']) + target_ms = (opening + 305) * 1000 + while True: + remaining = target_ms - time.time_ns() // 1_000_000 + if remaining <= 0: + break + time.sleep(min(remaining / 1000.0, 1.0)) + PY + + - name: Resolve from authoritative terminal outcome + shell: bash + run: | + set -euo pipefail + success=0 + for attempt in $(seq 1 24); do + echo "resolution attempt $attempt" + if PYTHONPATH="$CANDIDATE_ROOT/source:$CANDIDATE_ROOT/collector:$CANDIDATE_ROOT/runner" \ + python "$CANDIDATE_ROOT/runner/observer_cli.py" \ + --project-root "$CANDIDATE_ROOT" \ + --state-dir "$STATE_DIR" \ + resolve --condition-id "$CONDITION_ID" \ + > resolve-output.json; then + cat resolve-output.json + success=1 + break + fi + sleep 15 + done + test "$success" -eq 1 + + - name: Verify complete prospective lifecycle and raw evidence + shell: bash + run: | + set -euo pipefail + PYTHONPATH="$CANDIDATE_ROOT/source:$CANDIDATE_ROOT/collector:$CANDIDATE_ROOT/runner" \ + python "$CANDIDATE_ROOT/runner/observer_cli.py" \ + --project-root "$CANDIDATE_ROOT" \ + --state-dir "$STATE_DIR" \ + status | tee final-status.json + python - <<'PY' + import hashlib + import json + import os + from pathlib import Path + + state = Path(os.environ['STATE_DIR']) + collect = json.loads(Path('collect-output.json').read_text()) + resolution = json.loads(Path('resolve-output.json').read_text()) + status = json.loads(Path('final-status.json').read_text()) + condition = os.environ['CONDITION_ID'] + opening = int(os.environ['MARKET_OPEN_EPOCH_SECONDS']) + + assert status['conditions_seen'] == 1 + assert status['conditions_resolved'] == 1 + assert status['unresolved_condition_ids'] == [] + assert resolution['condition_id'] == condition + assert resolution['command'] == 'resolve' + assert resolution['official_outcome'] in {'Up', 'Down'} + assert resolution['decision'] in {'resolved_fill', 'resolved_no_fill'} + assert opening > 1785094521 + + lifecycle_rows = [ + json.loads(line) + for line in (state / 'lifecycle.jsonl').read_text().splitlines() + if line.strip() + ] + assert lifecycle_rows[0]['event_type'] == 'discovered' + assert lifecycle_rows[-1]['event_type'] == 'resolution' + assert all(row['condition_id'] == condition for row in lifecycle_rows) + assert all(row['policy_sha256'] == '82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba' for row in lifecycle_rows) + assert all(row['capture_policy_sha256'] == '888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc' for row in lifecycle_rows) + assert all(int(row['market_open_epoch_seconds']) == opening for row in lifecycle_rows) + + manifest_rows = [ + json.loads(line) + for line in (state / 'raw_evidence/manifest.jsonl').read_text().splitlines() + if line.strip() + ] + assert len(manifest_rows) >= 3 + allowed = { + ('GET', 'gamma-api.polymarket.com'), + ('POST', 'clob.polymarket.com'), + } + from urllib.parse import urlparse + for row in manifest_rows: + parsed = urlparse(row['url']) + assert (row['method'], parsed.netloc) in allowed + body = state / 'raw_evidence' / row['body_path'] + assert hashlib.sha256(body.read_bytes()).hexdigest() == row['response_body_sha256'] + + audit = { + 'candidate': 'late_favorite_btc_only_v3_prospective_shadow', + 'condition_id': condition, + 'market_open_epoch_seconds': opening, + 'collect_decision': collect['decision'], + 'resolution_decision': resolution['decision'], + 'official_outcome': resolution['official_outcome'], + 'official_won': resolution.get('official_won'), + 'pnl_total': resolution.get('pnl_total'), + 'lifecycle_records': len(lifecycle_rows), + 'raw_evidence_records': len(manifest_rows), + 'lifecycle_head_sha256': status['lifecycle_head_sha256'], + 'raw_evidence_head_sha256': status['raw_evidence_head_sha256'], + 'source_sha256': status['source_sha256'], + 'candidate_policy_sha256': '82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba', + 'capture_policy_sha256': '888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc', + 'credentials_used': 0, + 'authenticated_requests': 0, + 'order_submissions': 0, + 'live_submission': 'physically_absent', + 'historical_admission_credit': 0, + 'prospective_markets': 1, + } + Path('PROSPECTIVE_RUN_AUDIT.json').write_text( + json.dumps(audit, indent=2, sort_keys=True) + '\n' + ) + print(json.dumps(audit, indent=2, sort_keys=True)) + PY + + - name: Build deterministic prospective evidence archive + shell: bash + run: | + set -euo pipefail + cp collect-output.json "$STATE_DIR/" + cp resolve-output.json "$STATE_DIR/" + cp final-status.json "$STATE_DIR/" + cp PROSPECTIVE_RUN_AUDIT.json "$STATE_DIR/" + (cd "$STATE_DIR" && find . -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum > SHA256SUMS) + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf btc-only-v3-prospective-one-shot-v1.tar.gz "$STATE_DIR" + sha256sum btc-only-v3-prospective-one-shot-v1.tar.gz > btc-only-v3-prospective-one-shot-v1.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: btc-only-v3-prospective-one-shot-${{ github.run_id }} + path: | + ${{ env.STATE_DIR }}/ + btc-only-v3-prospective-one-shot-v1.tar.gz + btc-only-v3-prospective-one-shot-v1.tar.gz.sha256 + compression-level: 0 + retention-days: 90 From be153371a6632e8beeaac27546f606b23c1a5155 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 15:18:31 -0500 Subject: [PATCH 139/177] ci: isolate first BTC-only v3 prospective run --- .../btc-only-v3-prospective-run-1.yml | 211 ++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 .github/workflows/btc-only-v3-prospective-run-1.yml diff --git a/.github/workflows/btc-only-v3-prospective-run-1.yml b/.github/workflows/btc-only-v3-prospective-run-1.yml new file mode 100644 index 00000000..aaa8186a --- /dev/null +++ b/.github/workflows/btc-only-v3-prospective-run-1.yml @@ -0,0 +1,211 @@ +name: BTC-Only v3 Prospective Run 1 + +on: + pull_request: + branches: [digital-surface-live-readiness-20260725] + paths: + - ci_payload/prospective-run-triggers/btc-v3-run-1.txt + workflow_dispatch: + +concurrency: + group: btc-only-v3-prospective-run-1 + cancel-in-progress: false + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + CANDIDATE_ROOT: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 + STATE_DIR: btc-only-v3-prospective-state + +jobs: + collect-and-resolve-one-market: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + ref: digital-surface-live-readiness-20260725 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install pinned public client + run: python -m pip install requests==2.32.4 + + - name: Verify empty state and frozen runner + shell: bash + run: | + set -euo pipefail + python -m py_compile \ + "$CANDIDATE_ROOT/source/late_favorite_v3.py" \ + "$CANDIDATE_ROOT/source/capture_policy.py" \ + "$CANDIDATE_ROOT/collector/public_collector.py" \ + "$CANDIDATE_ROOT/collector/resolution.py" \ + "$CANDIDATE_ROOT/runner/observer_cli.py" + PYTHONPATH="$CANDIDATE_ROOT/source:$CANDIDATE_ROOT/collector:$CANDIDATE_ROOT/runner" \ + python "$CANDIDATE_ROOT/runner/observer_cli.py" \ + --project-root "$CANDIDATE_ROOT" --state-dir "$STATE_DIR" status \ + | tee initial-status.json + python - <<'PY' + import json + from pathlib import Path + status = json.loads(Path('initial-status.json').read_text()) + assert status['conditions_seen'] == 0 + assert status['conditions_resolved'] == 0 + assert status['lifecycle_head_sha256'] == '0' * 64 + assert status['raw_evidence_head_sha256'] == '0' * 64 + PY + + - name: Collect next post-cutoff BTC window + shell: bash + run: | + set -euo pipefail + PYTHONPATH="$CANDIDATE_ROOT/source:$CANDIDATE_ROOT/collector:$CANDIDATE_ROOT/runner" \ + python "$CANDIDATE_ROOT/runner/observer_cli.py" \ + --project-root "$CANDIDATE_ROOT" --state-dir "$STATE_DIR" collect-next \ + | tee collect-output.json + python - <<'PY' + import json + from pathlib import Path + output = json.loads(Path('collect-output.json').read_text()) + condition = str(output['condition_id']) + opening = int(output['market_open_epoch_seconds']) + assert opening > 1785094521 + assert output['command'] == 'collect-next' + with Path('/tmp/prospective.env').open('w') as handle: + handle.write(f'CONDITION_ID={condition}\n') + handle.write(f'MARKET_OPEN_EPOCH_SECONDS={opening}\n') + handle.write(f'COLLECT_DECISION={output["decision"]}\n') + print(json.dumps(output, indent=2, sort_keys=True)) + PY + cat /tmp/prospective.env >> "$GITHUB_ENV" + + - name: Wait for official close + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import os + import time + target_ms = (int(os.environ['MARKET_OPEN_EPOCH_SECONDS']) + 305) * 1000 + while True: + remaining = target_ms - time.time_ns() // 1_000_000 + if remaining <= 0: + break + time.sleep(min(remaining / 1000.0, 1.0)) + PY + + - name: Resolve authoritative terminal outcome + shell: bash + run: | + set -euo pipefail + success=0 + for attempt in $(seq 1 24); do + echo "resolution attempt $attempt" + if PYTHONPATH="$CANDIDATE_ROOT/source:$CANDIDATE_ROOT/collector:$CANDIDATE_ROOT/runner" \ + python "$CANDIDATE_ROOT/runner/observer_cli.py" \ + --project-root "$CANDIDATE_ROOT" --state-dir "$STATE_DIR" \ + resolve --condition-id "$CONDITION_ID" > resolve-output.json; then + cat resolve-output.json + success=1 + break + fi + sleep 15 + done + test "$success" -eq 1 + + - name: Verify complete append-only evidence + shell: bash + run: | + set -euo pipefail + PYTHONPATH="$CANDIDATE_ROOT/source:$CANDIDATE_ROOT/collector:$CANDIDATE_ROOT/runner" \ + python "$CANDIDATE_ROOT/runner/observer_cli.py" \ + --project-root "$CANDIDATE_ROOT" --state-dir "$STATE_DIR" status \ + | tee final-status.json + python - <<'PY' + import hashlib + import json + import os + from pathlib import Path + from urllib.parse import urlparse + + state = Path(os.environ['STATE_DIR']) + collect = json.loads(Path('collect-output.json').read_text()) + resolution = json.loads(Path('resolve-output.json').read_text()) + status = json.loads(Path('final-status.json').read_text()) + condition = os.environ['CONDITION_ID'] + opening = int(os.environ['MARKET_OPEN_EPOCH_SECONDS']) + assert status['conditions_seen'] == 1 + assert status['conditions_resolved'] == 1 + assert status['unresolved_condition_ids'] == [] + assert resolution['condition_id'] == condition + assert resolution['official_outcome'] in {'Up', 'Down'} + assert resolution['decision'] in {'resolved_fill', 'resolved_no_fill'} + assert opening > 1785094521 + + lifecycle_rows = [json.loads(line) for line in (state / 'lifecycle.jsonl').read_text().splitlines() if line.strip()] + assert lifecycle_rows[0]['event_type'] == 'discovered' + assert lifecycle_rows[-1]['event_type'] == 'resolution' + assert all(row['condition_id'] == condition for row in lifecycle_rows) + assert all(row['policy_sha256'] == '82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba' for row in lifecycle_rows) + assert all(row['capture_policy_sha256'] == '888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc' for row in lifecycle_rows) + + manifest_rows = [json.loads(line) for line in (state / 'raw_evidence/manifest.jsonl').read_text().splitlines() if line.strip()] + assert len(manifest_rows) >= 3 + for row in manifest_rows: + parsed = urlparse(row['url']) + assert (row['method'], parsed.netloc) in { + ('GET', 'gamma-api.polymarket.com'), + ('POST', 'clob.polymarket.com'), + } + body = state / 'raw_evidence' / row['body_path'] + assert hashlib.sha256(body.read_bytes()).hexdigest() == row['response_body_sha256'] + + audit = { + 'candidate': 'late_favorite_btc_only_v3_prospective_shadow', + 'condition_id': condition, + 'market_open_epoch_seconds': opening, + 'collect_decision': collect['decision'], + 'resolution_decision': resolution['decision'], + 'official_outcome': resolution['official_outcome'], + 'official_won': resolution.get('official_won'), + 'pnl_total': resolution.get('pnl_total'), + 'lifecycle_records': len(lifecycle_rows), + 'raw_evidence_records': len(manifest_rows), + 'lifecycle_head_sha256': status['lifecycle_head_sha256'], + 'raw_evidence_head_sha256': status['raw_evidence_head_sha256'], + 'source_sha256': status['source_sha256'], + 'candidate_policy_sha256': '82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba', + 'capture_policy_sha256': '888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc', + 'credentials_used': 0, + 'authenticated_requests': 0, + 'order_submissions': 0, + 'live_submission': 'physically_absent', + 'historical_admission_credit': 0, + 'prospective_markets': 1, + } + Path('PROSPECTIVE_RUN_AUDIT.json').write_text(json.dumps(audit, indent=2, sort_keys=True) + '\n') + print(json.dumps(audit, indent=2, sort_keys=True)) + PY + + - name: Build deterministic prospective artifact + shell: bash + run: | + set -euo pipefail + cp collect-output.json resolve-output.json final-status.json PROSPECTIVE_RUN_AUDIT.json "$STATE_DIR/" + (cd "$STATE_DIR" && find . -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum > SHA256SUMS) + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf btc-only-v3-prospective-run-1.tar.gz "$STATE_DIR" + sha256sum btc-only-v3-prospective-run-1.tar.gz > btc-only-v3-prospective-run-1.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: btc-only-v3-prospective-run-1-${{ github.run_id }} + path: | + ${{ env.STATE_DIR }}/ + btc-only-v3-prospective-run-1.tar.gz + btc-only-v3-prospective-run-1.tar.gz.sha256 + compression-level: 0 + retention-days: 90 From 9d3d1f9a078d8505b51af4ec981208738a1d970c Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 15:26:07 -0500 Subject: [PATCH 140/177] feat: add one-shot BTC-only observer CLI --- .../collector/cli.py | 176 ++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector/cli.py diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector/cli.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector/cli.py new file mode 100644 index 00000000..00461e0b --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector/cli.py @@ -0,0 +1,176 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import time +from pathlib import Path +from typing import Callable, Sequence + +from capture_policy import CaptureBoundLifecycleLedger, load_capture_policy +from late_favorite_v3 import load_frozen_policy +from public_collector import PublicHttpClient, RawEvidenceStore, SingleMarketCollector, next_collectable_open +from resolution import SingleMarketResolver + + +def _combined_source_sha256(root: Path) -> str: + paths = [ + root / "source/late_favorite_v3.py", + root / "source/capture_policy.py", + root / "collector/public_collector.py", + root / "collector/resolution.py", + ] + digest = hashlib.sha256() + for path in paths: + raw = path.read_bytes() + digest.update(path.name.encode("utf-8") + b"\0" + raw) + return digest.hexdigest() + + +def _load_policies(root: Path): + source_sha256 = _combined_source_sha256(root) + trading = load_frozen_policy( + root / "CANDIDATE_SPEC.json", + root / "FREEZE_MANIFEST.json", + source_sha256=source_sha256, + ) + capture = load_capture_policy( + root / "CAPTURE_POLICY.json", + root / "CAPTURE_FREEZE_MANIFEST.json", + ) + return trading, capture + + +def sleep_until_ms(target_ts_ms: int) -> None: + target = int(target_ts_ms) + while True: + remaining = target - time.time_ns() // 1_000_000 + if remaining <= 0: + return + time.sleep(min(remaining / 1000.0, 0.25)) + + +def _write_json_atomic(path: Path, payload: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + os.replace(temporary, path) + + +def _runtime( + *, + root: Path, + output_dir: Path, + client: PublicHttpClient | None, +): + trading, capture = _load_policies(root) + resolved_client = client or PublicHttpClient() + ledger = CaptureBoundLifecycleLedger(output_dir / "lifecycle.jsonl", trading, capture) + evidence = RawEvidenceStore(output_dir / "raw") + return trading, capture, resolved_client, ledger, evidence + + +def capture_once( + *, + root: str | Path, + output_dir: str | Path, + market_open_epoch_seconds: int | None = None, + now_ms: Callable[[], int] | None = None, + client: PublicHttpClient | None = None, + sleep_until_ms: Callable[[int], None] = sleep_until_ms, +) -> dict[str, object]: + root_path = Path(root).resolve() + output_path = Path(output_dir).resolve() + output_path.mkdir(parents=True, exist_ok=True) + trading, capture, resolved_client, ledger, evidence = _runtime( + root=root_path, + output_dir=output_path, + client=client, + ) + if market_open_epoch_seconds is None: + clock = now_ms or (lambda: time.time_ns() // 1_000_000) + market_open_epoch_seconds = next_collectable_open( + int(clock()), capture.valid_market_open_after_epoch_seconds + ) + opening = int(market_open_epoch_seconds) + collector = SingleMarketCollector( + client=resolved_client, + evidence_store=evidence, + ledger=ledger, + trading_policy=trading, + capture_policy=capture, + sleep_until_ms=sleep_until_ms, + ) + result = collector.capture(opening) + _write_json_atomic(output_path / "last_capture.json", result) + return dict(result) + + +def resolve_once( + *, + root: str | Path, + output_dir: str | Path, + condition_id: str, + client: PublicHttpClient | None = None, +) -> dict[str, object]: + root_path = Path(root).resolve() + output_path = Path(output_dir).resolve() + trading, capture, resolved_client, ledger, evidence = _runtime( + root=root_path, + output_dir=output_path, + client=client, + ) + resolver = SingleMarketResolver( + client=resolved_client, + evidence_store=evidence, + ledger=ledger, + trading_policy=trading, + capture_policy=capture, + ) + result = resolver.resolve(str(condition_id)) + _write_json_atomic(output_path / "last_resolution.json", result) + return dict(result) + + +def build_parser() -> argparse.ArgumentParser: + default_root = Path(__file__).resolve().parents[1] + parser = argparse.ArgumentParser( + description="Credential-free BTC-only v3 prospective observer" + ) + parser.add_argument("--root", type=Path, default=default_root) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--timeout-seconds", type=float, default=5.0) + subparsers = parser.add_subparsers(dest="command", required=True) + + capture_parser = subparsers.add_parser("capture", help="Capture one eligible BTC 5-minute market") + capture_parser.add_argument("--market-open", type=int) + + resolve_parser = subparsers.add_parser("resolve", help="Resolve one previously captured market") + resolve_parser.add_argument("--condition-id", required=True) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + client = PublicHttpClient(timeout_seconds=args.timeout_seconds) + if args.command == "capture": + result = capture_once( + root=args.root, + output_dir=args.output_dir, + market_open_epoch_seconds=args.market_open, + client=client, + ) + else: + result = resolve_once( + root=args.root, + output_dir=args.output_dir, + condition_id=args.condition_id, + client=client, + ) + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From aeab3792120b434e3ab09d34aef74202fc72557f Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 15:26:41 -0500 Subject: [PATCH 141/177] test: cover one-shot BTC-only CLI lifecycle --- .../tests/test_cli.py | 140 ++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_cli.py diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_cli.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_cli.py new file mode 100644 index 00000000..965ac2d7 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_cli.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from cli import capture_once, resolve_once +from public_collector import PublicHttpClient + + +class FakeResponse: + def __init__(self, payload: object) -> None: + self.content = json.dumps(payload, separators=(",", ":")).encode() + self.status_code = 200 + + +class FakeRequester: + def __init__(self, responses: list[FakeResponse]) -> None: + self.responses = list(responses) + self.calls: list[dict[str, object]] = [] + + def __call__(self, method: str, url: str, **kwargs: object) -> FakeResponse: + self.calls.append({"method": method, "url": url, **kwargs}) + return self.responses.pop(0) + + +def market_payload(opening: int) -> dict[str, object]: + return { + "conditionId": "condition-1", + "slug": f"btc-updown-5m-{opening}", + "active": True, + "closed": False, + "enableOrderBook": True, + "outcomes": json.dumps(["Up", "Down"]), + "clobTokenIds": json.dumps(["token-up", "token-down"]), + } + + +def terminal_payload(opening: int) -> dict[str, object]: + return { + "conditionId": "condition-1", + "slug": f"btc-updown-5m-{opening}", + "closed": True, + "outcomes": json.dumps(["Up", "Down"]), + "outcomePrices": json.dumps(["1", "0"]), + } + + +def book(token: str, ask: str, timestamp_ms: int) -> dict[str, object]: + return { + "market": "condition-1", + "asset_id": token, + "timestamp": str(timestamp_ms), + "hash": f"hash-{token}-{timestamp_ms}", + "bids": [{"price": "0.10", "size": "10"}], + "asks": [{"price": ask, "size": "10"}], + "min_order_size": "1", + "tick_size": "0.01", + "neg_risk": False, + "last_trade_price": "0.50", + } + + +def test_capture_and_resolve_persist_complete_lifecycle(tmp_path: Path) -> None: + opening = 1_800_000_300 + signal_target = opening * 1000 + 210_000 + arrival_target = opening * 1000 + 211_000 + requester = FakeRequester([ + FakeResponse(market_payload(opening)), + FakeResponse([ + book("token-up", "0.90", signal_target), + book("token-down", "0.12", signal_target), + ]), + FakeResponse([book("token-up", "0.90", arrival_target)]), + FakeResponse(terminal_payload(opening)), + ]) + times = iter([ + opening * 1000 + 1_000, opening * 1000 + 1_020, + signal_target, signal_target + 100, + arrival_target, arrival_target + 100, + opening * 1000 + 301_000, opening * 1000 + 301_050, + ]) + client = PublicHttpClient(requester=requester, clock_ms=lambda: next(times)) + slept: list[int] = [] + root = Path(__file__).parents[1] + + capture = capture_once( + root=root, + output_dir=tmp_path, + market_open_epoch_seconds=opening, + client=client, + sleep_until_ms=slept.append, + ) + assert capture["decision"] == "hypothetical_fok_fill" + assert capture["condition_id"] == "condition-1" + assert slept == [signal_target, arrival_target] + assert (tmp_path / "lifecycle.jsonl").is_file() + assert (tmp_path / "raw" / "manifest.jsonl").is_file() + assert json.loads((tmp_path / "last_capture.json").read_text())["decision"] == "hypothetical_fok_fill" + + resolution = resolve_once( + root=root, + output_dir=tmp_path, + condition_id="condition-1", + client=client, + ) + assert resolution["decision"] == "resolved_fill" + assert resolution["official_outcome"] == "Up" + assert resolution["official_won"] is True + assert json.loads((tmp_path / "last_resolution.json").read_text())["decision"] == "resolved_fill" + rows = [json.loads(line) for line in (tmp_path / "lifecycle.jsonl").read_text().splitlines()] + assert [row["event_type"] for row in rows] == ["discovered", "signal", "arrival", "resolution"] + + +def test_capture_selects_next_eligible_window_when_opening_is_omitted(tmp_path: Path) -> None: + now_ms = 1_800_000_300_000 + 210_001 + opening = 1_800_000_600 + signal_target = opening * 1000 + 210_000 + requester = FakeRequester([ + FakeResponse(market_payload(opening)), + FakeResponse([ + book("token-up", "0.84", signal_target), + book("token-down", "0.18", signal_target), + ]), + ]) + times = iter([ + opening * 1000 + 1_000, opening * 1000 + 1_020, + signal_target, signal_target + 100, + ]) + client = PublicHttpClient(requester=requester, clock_ms=lambda: next(times)) + root = Path(__file__).parents[1] + result = capture_once( + root=root, + output_dir=tmp_path, + market_open_epoch_seconds=None, + now_ms=lambda: now_ms, + client=client, + sleep_until_ms=lambda _: None, + ) + assert result["market_open_epoch_seconds"] == opening + assert result["decision"] == "no_signal_below_threshold" From 8cddf60dfbabecd65cd51ea00a77fe6539766d34 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 15:27:44 -0500 Subject: [PATCH 142/177] ci: verify one-shot BTC-only observer CLI --- .../btc-only-v3-observer-checkpoint.yml | 36 ++++++++++++------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/.github/workflows/btc-only-v3-observer-checkpoint.yml b/.github/workflows/btc-only-v3-observer-checkpoint.yml index aab3f138..89ff8b1d 100644 --- a/.github/workflows/btc-only-v3-observer-checkpoint.yml +++ b/.github/workflows/btc-only-v3-observer-checkpoint.yml @@ -36,7 +36,7 @@ jobs: - name: Install pinned test stack run: python -m pip install requests==2.32.4 pytest==8.3.5 - - name: Compile observer, capture, collector, and resolution sources + - name: Compile observer, collector, resolution, and CLI sources shell: bash run: | set -euo pipefail @@ -46,12 +46,14 @@ jobs: "$root/source/capture_policy.py" \ "$root/collector/public_collector.py" \ "$root/collector/resolution.py" \ + "$root/collector/cli.py" \ "$root/tests/test_late_favorite_v3.py" \ "$root/tests/test_capture_policy.py" \ "$root/tests/test_public_collector.py" \ - "$root/tests/test_resolution.py" + "$root/tests/test_resolution.py" \ + "$root/tests/test_cli.py" - - name: Pass exactly 22 behavior contracts + - name: Pass exactly 24 behavior contracts shell: bash env: PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector @@ -59,7 +61,7 @@ jobs: set -euo pipefail root=ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 pytest --collect-only -q "$root/tests" | tee /tmp/collect.txt - grep -Eq '^22 tests collected' /tmp/collect.txt + grep -Eq '^24 tests collected' /tmp/collect.txt pytest -q "$root/tests" - name: Verify frozen policies and contamination cutoffs @@ -77,7 +79,7 @@ jobs: from late_favorite_v3 import load_frozen_policy root = Path('ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3') - source_paths = [ + frozen_source_paths = [ root / 'source/late_favorite_v3.py', root / 'source/capture_policy.py', root / 'collector/public_collector.py', @@ -85,12 +87,15 @@ jobs: ] combined = hashlib.sha256() source_hashes = {} - for path in source_paths: + for path in frozen_source_paths: raw = path.read_bytes() digest = hashlib.sha256(raw).hexdigest() source_hashes[str(path.relative_to(root))] = digest combined.update(path.name.encode() + b'\0' + raw) combined_source_sha256 = combined.hexdigest() + cli_path = root / 'collector/cli.py' + cli_sha256 = hashlib.sha256(cli_path.read_bytes()).hexdigest() + source_hashes[str(cli_path.relative_to(root))] = cli_sha256 trading = load_frozen_policy( root / 'CANDIDATE_SPEC.json', @@ -112,18 +117,20 @@ jobs: verification = { 'candidate': 'late_favorite_btc_only_v3_prospective_shadow', - 'tests_collected': 22, + 'tests_collected': 24, 'candidate_policy_sha256': trading.policy_sha256, 'capture_policy_sha256': capture.capture_policy_sha256, 'candidate_cutoff_epoch_seconds': trading.valid_market_open_after_epoch_seconds, 'capture_cutoff_epoch_seconds': capture.valid_market_open_after_epoch_seconds, - 'combined_source_sha256': combined_source_sha256, + 'combined_frozen_source_sha256': combined_source_sha256, + 'cli_sha256': cli_sha256, 'source_files_sha256': source_hashes, 'public_endpoints': [ 'GET https://gamma-api.polymarket.com/markets/slug/{slug}', 'POST https://clob.polymarket.com/books', ], 'official_resolution': 'Gamma closed binary terminal outcome', + 'commands': ['capture', 'resolve'], 'credentials_used': 0, 'authenticated_requests': 0, 'order_submissions': 0, @@ -185,6 +192,8 @@ jobs: assert 'https://clob.polymarket.com' in public_text assert 'parsed.path == "/books"' in public_text assert 'parsed.path.startswith("/markets/slug/")' in public_text + cli_text = (root / 'collector/cli.py').read_text() + assert 'capture' in cli_text and 'resolve' in cli_text PY - name: Build deterministic observer checkpoint @@ -202,17 +211,18 @@ jobs: cp "$root/source/capture_policy.py" btc-only-v3-observer-checkpoint/ cp "$root/collector/public_collector.py" btc-only-v3-observer-checkpoint/ cp "$root/collector/resolution.py" btc-only-v3-observer-checkpoint/ + cp "$root/collector/cli.py" btc-only-v3-observer-checkpoint/ (cd btc-only-v3-observer-checkpoint && sha256sum * > SHA256SUMS) tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ - -czf btc-only-v3-observer-checkpoint-v1.tar.gz btc-only-v3-observer-checkpoint - sha256sum btc-only-v3-observer-checkpoint-v1.tar.gz > btc-only-v3-observer-checkpoint-v1.tar.gz.sha256 + -czf btc-only-v3-observer-checkpoint-v2.tar.gz btc-only-v3-observer-checkpoint + sha256sum btc-only-v3-observer-checkpoint-v2.tar.gz > btc-only-v3-observer-checkpoint-v2.tar.gz.sha256 - uses: actions/upload-artifact@v4 with: - name: btc-only-v3-observer-checkpoint-v1 + name: btc-only-v3-observer-checkpoint-v2 path: | btc-only-v3-observer-checkpoint/ - btc-only-v3-observer-checkpoint-v1.tar.gz - btc-only-v3-observer-checkpoint-v1.tar.gz.sha256 + btc-only-v3-observer-checkpoint-v2.tar.gz + btc-only-v3-observer-checkpoint-v2.tar.gz.sha256 compression-level: 0 retention-days: 30 From d0b13c9832172de73e9adf912aeed33e603a17f8 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 15:31:10 -0500 Subject: [PATCH 143/177] ci: launch first BTC-only prospective observation --- .../btc-only-v3-prospective-once.yml | 198 ++++++++++++++++++ 1 file changed, 198 insertions(+) create mode 100644 .github/workflows/btc-only-v3-prospective-once.yml diff --git a/.github/workflows/btc-only-v3-prospective-once.yml b/.github/workflows/btc-only-v3-prospective-once.yml new file mode 100644 index 00000000..49b2e038 --- /dev/null +++ b/.github/workflows/btc-only-v3-prospective-once.yml @@ -0,0 +1,198 @@ +name: BTC-Only v3 Prospective Observation Once + +on: + push: + branches: [btc-only-v3-prospective-run-20260726] + paths: + - .github/workflows/btc-only-v3-prospective-once.yml + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: btc-only-v3-prospective-once-20260726 + cancel-in-progress: false + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + observe-one-market: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install pinned public client + run: python -m pip install requests==2.32.4 + + - name: Select a genuinely prospective BTC window + id: select + shell: bash + run: | + set -euo pipefail + python - <<'PY' >> "$GITHUB_OUTPUT" + import time + now = time.time_ns() // 1_000_000_000 + opening = (now // 300) * 300 + if now > opening + 205: + opening += 300 + print(f"opening={opening}") + print(f"signal_target_ms={opening * 1000 + 210000}") + print(f"arrival_target_ms={opening * 1000 + 211000}") + print(f"resolution_target_ms={opening * 1000 + 305000}") + PY + + - name: Wait until market discovery is valid + env: + OPENING: ${{ steps.select.outputs.opening }} + run: | + python - <<'PY' + import os + import time + target = int(os.environ['OPENING']) * 1000 + 5000 + while True: + remaining = target - time.time_ns() // 1_000_000 + if remaining <= 0: + break + time.sleep(min(remaining / 1000, 0.5)) + PY + + - name: Capture discovery, signal, and arrival + env: + OPENING: ${{ steps.select.outputs.opening }} + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector + shell: bash + run: | + set -euo pipefail + root=ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 + python "$root/collector/cli.py" \ + --root "$root" \ + --output-dir prospective-data \ + --timeout-seconds 5 \ + capture --market-open "$OPENING" | tee capture-output.json + cat prospective-data/last_capture.json + + - name: Wait until terminal resolution window + env: + RESOLUTION_TARGET_MS: ${{ steps.select.outputs.resolution_target_ms }} + run: | + python - <<'PY' + import os + import time + target = int(os.environ['RESOLUTION_TARGET_MS']) + while True: + remaining = target - time.time_ns() // 1_000_000 + if remaining <= 0: + break + time.sleep(min(remaining / 1000, 0.5)) + PY + + - name: Resolve from authoritative Gamma terminal prices + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector + shell: bash + run: | + set -euo pipefail + root=ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 + condition_id="$(python - <<'PY' + import json + print(json.load(open('prospective-data/last_capture.json'))['condition_id']) + PY + )" + success=0 + for attempt in $(seq 1 24); do + if python "$root/collector/cli.py" \ + --root "$root" \ + --output-dir prospective-data \ + --timeout-seconds 5 \ + resolve --condition-id "$condition_id" | tee resolution-output.json; then + success=1 + break + fi + sleep 10 + done + test "$success" -eq 1 + cat prospective-data/last_resolution.json + + - name: Verify append-only prospective evidence + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector + EXPECTED_OPENING: ${{ steps.select.outputs.opening }} + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import hashlib + import json + import os + from pathlib import Path + + from capture_policy import CaptureBoundLifecycleLedger, load_capture_policy + from cli import _combined_source_sha256 + from late_favorite_v3 import load_frozen_policy + from public_collector import RawEvidenceStore + + root = Path('ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3') + out = Path('prospective-data') + source_sha = _combined_source_sha256(root) + trading = load_frozen_policy(root / 'CANDIDATE_SPEC.json', root / 'FREEZE_MANIFEST.json', source_sha256=source_sha) + capture = load_capture_policy(root / 'CAPTURE_POLICY.json', root / 'CAPTURE_FREEZE_MANIFEST.json') + ledger = CaptureBoundLifecycleLedger(out / 'lifecycle.jsonl', trading, capture) + evidence = RawEvidenceStore(out / 'raw') + assert len(ledger.records) == 4, len(ledger.records) + assert [row['event_type'] for row in ledger.records] == ['discovered', 'signal', 'arrival', 'resolution'] + opening = int(os.environ['EXPECTED_OPENING']) + assert opening > capture.valid_market_open_after_epoch_seconds + assert {int(row['market_open_epoch_seconds']) for row in ledger.records} == {opening} + assert len(evidence.records) >= 4 + resolution = json.loads((out / 'last_resolution.json').read_text()) + assert resolution['official_outcome'] in {'Up', 'Down'} + assert resolution['decision'] in {'resolved_fill', 'resolved_no_fill'} + run = { + 'candidate': 'late_favorite_btc_only_v3_prospective_shadow', + 'github_sha': os.environ.get('GITHUB_SHA'), + 'market_open_epoch_seconds': opening, + 'condition_id': ledger.records[0]['condition_id'], + 'capture_decision': json.loads((out / 'last_capture.json').read_text())['decision'], + 'resolution': resolution, + 'lifecycle_records': len(ledger.records), + 'lifecycle_head_sha256': ledger.head_hash, + 'raw_evidence_records': len(evidence.records), + 'raw_evidence_head_sha256': evidence.head_hash, + 'candidate_policy_sha256': trading.policy_sha256, + 'capture_policy_sha256': capture.capture_policy_sha256, + 'combined_source_sha256': source_sha, + 'credentials_used': 0, + 'authenticated_requests': 0, + 'order_submissions': 0, + 'historical_admission_credit': 0, + 'live_submission': 'physically_absent', + } + (out / 'PROSPECTIVE_RUN.json').write_text(json.dumps(run, indent=2, sort_keys=True) + '\n') + print(json.dumps(run, indent=2, sort_keys=True)) + PY + + - name: Build deterministic evidence archive + shell: bash + run: | + set -euo pipefail + find prospective-data -type f -print0 | sort -z | xargs -0 sha256sum > prospective-data/SHA256SUMS + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf btc-only-v3-prospective-run-v1.tar.gz prospective-data + sha256sum btc-only-v3-prospective-run-v1.tar.gz > btc-only-v3-prospective-run-v1.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: btc-only-v3-prospective-run-v1 + path: | + prospective-data/ + btc-only-v3-prospective-run-v1.tar.gz + btc-only-v3-prospective-run-v1.tar.gz.sha256 + compression-level: 0 + retention-days: 30 From 67ae0cb5d62e5be08b89c5a6c9c0e24c59782006 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 15:32:18 -0500 Subject: [PATCH 144/177] ci: expose first prospective BTC observation run --- .github/workflows/btc-only-v3-prospective-once.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/btc-only-v3-prospective-once.yml b/.github/workflows/btc-only-v3-prospective-once.yml index 49b2e038..06e4a7ed 100644 --- a/.github/workflows/btc-only-v3-prospective-once.yml +++ b/.github/workflows/btc-only-v3-prospective-once.yml @@ -17,6 +17,7 @@ concurrency: env: PYTHONHASHSEED: '17' PYTHONUNBUFFERED: '1' + RUN_LABEL: first-post-cutoff-observation jobs: observe-one-market: @@ -128,7 +129,6 @@ jobs: run: | set -euo pipefail python - <<'PY' - import hashlib import json import os from pathlib import Path @@ -156,6 +156,7 @@ jobs: assert resolution['decision'] in {'resolved_fill', 'resolved_no_fill'} run = { 'candidate': 'late_favorite_btc_only_v3_prospective_shadow', + 'run_label': os.environ.get('RUN_LABEL'), 'github_sha': os.environ.get('GITHUB_SHA'), 'market_open_epoch_seconds': opening, 'condition_id': ledger.records[0]['condition_id'], From db8352ae1d9e8a640618ee137e2e9873a1a9cfb7 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 15:35:02 -0500 Subject: [PATCH 145/177] ci: enforce exactly one prospective observation run --- .github/workflows/btc-only-v3-prospective-once.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/btc-only-v3-prospective-once.yml b/.github/workflows/btc-only-v3-prospective-once.yml index 06e4a7ed..5111423e 100644 --- a/.github/workflows/btc-only-v3-prospective-once.yml +++ b/.github/workflows/btc-only-v3-prospective-once.yml @@ -12,7 +12,7 @@ permissions: concurrency: group: btc-only-v3-prospective-once-20260726 - cancel-in-progress: false + cancel-in-progress: true env: PYTHONHASHSEED: '17' From 2d75675be57bb852aab7ed1720f35b7ca55af4ce Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 15:36:58 -0500 Subject: [PATCH 146/177] ci: fix prospective no-signal and checksum verification --- .../workflows/btc-only-v3-prospective-once.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/btc-only-v3-prospective-once.yml b/.github/workflows/btc-only-v3-prospective-once.yml index 5111423e..c99c2547 100644 --- a/.github/workflows/btc-only-v3-prospective-once.yml +++ b/.github/workflows/btc-only-v3-prospective-once.yml @@ -145,12 +145,17 @@ jobs: capture = load_capture_policy(root / 'CAPTURE_POLICY.json', root / 'CAPTURE_FREEZE_MANIFEST.json') ledger = CaptureBoundLifecycleLedger(out / 'lifecycle.jsonl', trading, capture) evidence = RawEvidenceStore(out / 'raw') - assert len(ledger.records) == 4, len(ledger.records) - assert [row['event_type'] for row in ledger.records] == ['discovered', 'signal', 'arrival', 'resolution'] + event_types = [row['event_type'] for row in ledger.records] + assert event_types in ( + ['discovered', 'signal', 'resolution'], + ['discovered', 'signal', 'arrival', 'resolution'], + ), event_types opening = int(os.environ['EXPECTED_OPENING']) assert opening > capture.valid_market_open_after_epoch_seconds assert {int(row['market_open_epoch_seconds']) for row in ledger.records} == {opening} - assert len(evidence.records) >= 4 + minimum_evidence_records = 3 if len(event_types) == 3 else 4 + assert len(evidence.records) >= minimum_evidence_records + capture_result = json.loads((out / 'last_capture.json').read_text()) resolution = json.loads((out / 'last_resolution.json').read_text()) assert resolution['official_outcome'] in {'Up', 'Down'} assert resolution['decision'] in {'resolved_fill', 'resolved_no_fill'} @@ -160,8 +165,9 @@ jobs: 'github_sha': os.environ.get('GITHUB_SHA'), 'market_open_epoch_seconds': opening, 'condition_id': ledger.records[0]['condition_id'], - 'capture_decision': json.loads((out / 'last_capture.json').read_text())['decision'], + 'capture_decision': capture_result['decision'], 'resolution': resolution, + 'lifecycle_event_types': event_types, 'lifecycle_records': len(ledger.records), 'lifecycle_head_sha256': ledger.head_hash, 'raw_evidence_records': len(evidence.records), @@ -183,7 +189,8 @@ jobs: shell: bash run: | set -euo pipefail - find prospective-data -type f -print0 | sort -z | xargs -0 sha256sum > prospective-data/SHA256SUMS + find prospective-data -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum > prospective-data/SHA256SUMS + sha256sum -c prospective-data/SHA256SUMS tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ -czf btc-only-v3-prospective-run-v1.tar.gz prospective-data sha256sum btc-only-v3-prospective-run-v1.tar.gz > btc-only-v3-prospective-run-v1.tar.gz.sha256 From bb0790151521e890d11118133bb0eb81dfca9280 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 15:46:44 -0500 Subject: [PATCH 147/177] ci: preserve capture evidence across delayed resolution --- .../btc-only-v3-prospective-once.yml | 66 ++++++++++++++----- 1 file changed, 49 insertions(+), 17 deletions(-) diff --git a/.github/workflows/btc-only-v3-prospective-once.yml b/.github/workflows/btc-only-v3-prospective-once.yml index c99c2547..6421a0fb 100644 --- a/.github/workflows/btc-only-v3-prospective-once.yml +++ b/.github/workflows/btc-only-v3-prospective-once.yml @@ -22,7 +22,7 @@ env: jobs: observe-one-market: runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 35 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -72,6 +72,7 @@ jobs: run: | set -euo pipefail root=ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 + mkdir -p prospective-data python "$root/collector/cli.py" \ --root "$root" \ --output-dir prospective-data \ @@ -94,37 +95,48 @@ jobs: time.sleep(min(remaining / 1000, 0.5)) PY - - name: Resolve from authoritative Gamma terminal prices + - name: Watch authoritative Gamma terminal prices + id: resolution env: PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector shell: bash run: | - set -euo pipefail + set -uo pipefail root=ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 condition_id="$(python - <<'PY' import json print(json.load(open('prospective-data/last_capture.json'))['condition_id']) PY )" - success=0 - for attempt in $(seq 1 24); do + resolved=0 + attempts=0 + for attempt in $(seq 1 120); do + attempts="$attempt" if python "$root/collector/cli.py" \ --root "$root" \ --output-dir prospective-data \ --timeout-seconds 5 \ resolve --condition-id "$condition_id" | tee resolution-output.json; then - success=1 + resolved=1 break fi sleep 10 done - test "$success" -eq 1 - cat prospective-data/last_resolution.json + echo "resolved=$resolved" >> "$GITHUB_OUTPUT" + echo "attempts=$attempts" >> "$GITHUB_OUTPUT" + if test "$resolved" -eq 1; then + cat prospective-data/last_resolution.json + else + echo "Gamma remained non-terminal after $attempts attempts; preserving unresolved evidence." + fi + exit 0 - name: Verify append-only prospective evidence env: PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector EXPECTED_OPENING: ${{ steps.select.outputs.opening }} + RESOLVED: ${{ steps.resolution.outputs.resolved }} + RESOLUTION_ATTEMPTS: ${{ steps.resolution.outputs.attempts }} shell: bash run: | set -euo pipefail @@ -146,19 +158,29 @@ jobs: ledger = CaptureBoundLifecycleLedger(out / 'lifecycle.jsonl', trading, capture) evidence = RawEvidenceStore(out / 'raw') event_types = [row['event_type'] for row in ledger.records] - assert event_types in ( - ['discovered', 'signal', 'resolution'], - ['discovered', 'signal', 'arrival', 'resolution'], - ), event_types + allowed = { + ('discovered', 'signal'), + ('discovered', 'signal', 'arrival'), + ('discovered', 'signal', 'resolution'), + ('discovered', 'signal', 'arrival', 'resolution'), + } + assert tuple(event_types) in allowed, event_types opening = int(os.environ['EXPECTED_OPENING']) assert opening > capture.valid_market_open_after_epoch_seconds assert {int(row['market_open_epoch_seconds']) for row in ledger.records} == {opening} - minimum_evidence_records = 3 if len(event_types) == 3 else 4 - assert len(evidence.records) >= minimum_evidence_records + assert len(evidence.records) >= 2 capture_result = json.loads((out / 'last_capture.json').read_text()) - resolution = json.loads((out / 'last_resolution.json').read_text()) - assert resolution['official_outcome'] in {'Up', 'Down'} - assert resolution['decision'] in {'resolved_fill', 'resolved_no_fill'} + resolution_path = out / 'last_resolution.json' + resolution = json.loads(resolution_path.read_text()) if resolution_path.exists() else None + resolved = os.environ.get('RESOLVED') == '1' + if resolved: + assert event_types[-1] == 'resolution' + assert resolution is not None + assert resolution['official_outcome'] in {'Up', 'Down'} + assert resolution['decision'] in {'resolved_fill', 'resolved_no_fill'} + else: + assert event_types[-1] != 'resolution' + assert resolution is None run = { 'candidate': 'late_favorite_btc_only_v3_prospective_shadow', 'run_label': os.environ.get('RUN_LABEL'), @@ -166,6 +188,8 @@ jobs: 'market_open_epoch_seconds': opening, 'condition_id': ledger.records[0]['condition_id'], 'capture_decision': capture_result['decision'], + 'resolution_status': 'terminal' if resolved else 'unresolved', + 'resolution_attempts': int(os.environ.get('RESOLUTION_ATTEMPTS') or 0), 'resolution': resolution, 'lifecycle_event_types': event_types, 'lifecycle_records': len(ledger.records), @@ -204,3 +228,11 @@ jobs: btc-only-v3-prospective-run-v1.tar.gz.sha256 compression-level: 0 retention-days: 30 + + - name: Require terminal resolution for admission evidence + run: | + python - <<'PY' + import json + run = json.load(open('prospective-data/PROSPECTIVE_RUN.json')) + assert run['resolution_status'] == 'terminal', run['resolution_status'] + PY From 08f832b5bb1958bbc0b8601b4c457ba74ff87dcf Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 15:57:12 -0500 Subject: [PATCH 148/177] test: require cache-busted Gamma resolution requests --- .../tests/test_resolution_cache_bust.py | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_resolution_cache_bust.py diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_resolution_cache_bust.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_resolution_cache_bust.py new file mode 100644 index 00000000..728770f7 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_resolution_cache_bust.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from types import SimpleNamespace + +from resolution import SingleMarketResolver + + +class FakeClient: + def __init__(self) -> None: + self.urls: list[str] = [] + + def request_json(self, method: str, url: str): + self.urls.append(url) + return SimpleNamespace( + payload={ + "conditionId": "c1", + "slug": "btc-updown-5m-1800000300", + "closed": True, + "outcomes": '["Up", "Down"]', + "outcomePrices": '["1", "0"]', + }, + response_body_sha256="4" * 64, + request_completed_ts_ms=1_800_000_600_000, + ) + + +class FakeEvidenceStore: + def __init__(self) -> None: + self.records = [object()] * 7 + + def append(self, evidence, *, purpose: str): + return {"record_hash": "5" * 64} + + +class FakeLedger: + def __init__(self) -> None: + self.records = [ + { + "condition_id": "c1", + "market_open_epoch_seconds": 1_800_000_300, + "event_type": "discovered", + "payload": {"slug": "btc-updown-5m-1800000300"}, + }, + { + "condition_id": "c1", + "market_open_epoch_seconds": 1_800_000_300, + "event_type": "signal", + "payload": {"decision": "no_signal_below_threshold", "signal": False}, + }, + ] + self.appended: list[dict[str, object]] = [] + + def append(self, **kwargs) -> None: + self.appended.append(kwargs) + + +def test_resolution_request_cache_busts_gamma_slug_url() -> None: + client = FakeClient() + resolver = SingleMarketResolver( + client=client, + evidence_store=FakeEvidenceStore(), + ledger=FakeLedger(), + trading_policy=SimpleNamespace(policy_sha256="1" * 64, source_sha256="2" * 64), + capture_policy=SimpleNamespace(capture_policy_sha256="3" * 64), + ) + + resolver.resolve("c1") + + assert client.urls == [ + "https://gamma-api.polymarket.com/markets/slug/" + "btc-updown-5m-1800000300?cache_bust=7" + ] From b4d2340b199dce2330a0e9f16764babdb06450bc Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 15:57:49 -0500 Subject: [PATCH 149/177] fix: cache-bust Gamma terminal resolution requests --- .../late-favorite-btc-only-v3/collector/resolution.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector/resolution.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector/resolution.py index 4e0a6963..9e5ee980 100644 --- a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector/resolution.py +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector/resolution.py @@ -99,7 +99,10 @@ def resolve(self, condition_id: str) -> dict[str, Any]: if discovered is None or signal is None: raise RuntimeError("resolution requires discovered and signal lifecycle events") slug = str(discovered.get("slug") or f"btc-updown-5m-{opening}") - evidence = self.client.request_json("GET", f"{GAMMA_BASE}/markets/slug/{slug}") + cache_bust = len(self.evidence_store.records) + evidence = self.client.request_json( + "GET", f"{GAMMA_BASE}/markets/slug/{slug}?cache_bust={cache_bust}" + ) manifest = self.evidence_store.append(evidence, purpose="official_resolution") if not isinstance(evidence.payload, Mapping): raise PublicResponseError("Gamma terminal response must be an object") From 257e3b5f1bfa4118a4375fbbfc4693152a296c55 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 15:59:11 -0500 Subject: [PATCH 150/177] ci: run cache-busted prospective resolver --- .github/workflows/btc-only-v3-prospective-once.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/btc-only-v3-prospective-once.yml b/.github/workflows/btc-only-v3-prospective-once.yml index 6421a0fb..b6f95008 100644 --- a/.github/workflows/btc-only-v3-prospective-once.yml +++ b/.github/workflows/btc-only-v3-prospective-once.yml @@ -29,8 +29,16 @@ jobs: with: python-version: '3.11' - - name: Install pinned public client - run: python -m pip install requests==2.32.4 + - name: Install pinned public and test clients + run: python -m pip install requests==2.32.4 pytest==8.3.5 + + - name: Verify cache-busted terminal resolver + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector + run: | + root=ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 + python -m py_compile "$root/collector/resolution.py" "$root/tests/test_resolution_cache_bust.py" + pytest -q "$root/tests/test_resolution_cache_bust.py" - name: Select a genuinely prospective BTC window id: select From 4143d11cd2c674bd983b961acd6b5f1a6c5b201e Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 16:14:01 -0500 Subject: [PATCH 151/177] ci: verify cache-busted resolver in observer checkpoint --- .../btc-only-v3-observer-checkpoint.yml | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/workflows/btc-only-v3-observer-checkpoint.yml b/.github/workflows/btc-only-v3-observer-checkpoint.yml index 89ff8b1d..58f39a17 100644 --- a/.github/workflows/btc-only-v3-observer-checkpoint.yml +++ b/.github/workflows/btc-only-v3-observer-checkpoint.yml @@ -51,9 +51,10 @@ jobs: "$root/tests/test_capture_policy.py" \ "$root/tests/test_public_collector.py" \ "$root/tests/test_resolution.py" \ + "$root/tests/test_resolution_cache_bust.py" \ "$root/tests/test_cli.py" - - name: Pass exactly 24 behavior contracts + - name: Pass exactly 25 behavior contracts shell: bash env: PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector @@ -61,7 +62,7 @@ jobs: set -euo pipefail root=ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 pytest --collect-only -q "$root/tests" | tee /tmp/collect.txt - grep -Eq '^24 tests collected' /tmp/collect.txt + grep -Eq '^25 tests collected' /tmp/collect.txt pytest -q "$root/tests" - name: Verify frozen policies and contamination cutoffs @@ -117,7 +118,7 @@ jobs: verification = { 'candidate': 'late_favorite_btc_only_v3_prospective_shadow', - 'tests_collected': 24, + 'tests_collected': 25, 'candidate_policy_sha256': trading.policy_sha256, 'capture_policy_sha256': capture.capture_policy_sha256, 'candidate_cutoff_epoch_seconds': trading.valid_market_open_after_epoch_seconds, @@ -126,7 +127,7 @@ jobs: 'cli_sha256': cli_sha256, 'source_files_sha256': source_hashes, 'public_endpoints': [ - 'GET https://gamma-api.polymarket.com/markets/slug/{slug}', + 'GET https://gamma-api.polymarket.com/markets/slug/{slug}?cache_bust={attempt}', 'POST https://clob.polymarket.com/books', ], 'official_resolution': 'Gamma closed binary terminal outcome', @@ -192,6 +193,8 @@ jobs: assert 'https://clob.polymarket.com' in public_text assert 'parsed.path == "/books"' in public_text assert 'parsed.path.startswith("/markets/slug/")' in public_text + resolution_text = (root / 'collector/resolution.py').read_text() + assert '?cache_bust=' in resolution_text cli_text = (root / 'collector/cli.py').read_text() assert 'capture' in cli_text and 'resolve' in cli_text PY @@ -214,15 +217,15 @@ jobs: cp "$root/collector/cli.py" btc-only-v3-observer-checkpoint/ (cd btc-only-v3-observer-checkpoint && sha256sum * > SHA256SUMS) tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ - -czf btc-only-v3-observer-checkpoint-v2.tar.gz btc-only-v3-observer-checkpoint - sha256sum btc-only-v3-observer-checkpoint-v2.tar.gz > btc-only-v3-observer-checkpoint-v2.tar.gz.sha256 + -czf btc-only-v3-observer-checkpoint-v3.tar.gz btc-only-v3-observer-checkpoint + sha256sum btc-only-v3-observer-checkpoint-v3.tar.gz > btc-only-v3-observer-checkpoint-v3.tar.gz.sha256 - uses: actions/upload-artifact@v4 with: - name: btc-only-v3-observer-checkpoint-v2 + name: btc-only-v3-observer-checkpoint-v3 path: | btc-only-v3-observer-checkpoint/ - btc-only-v3-observer-checkpoint-v2.tar.gz - btc-only-v3-observer-checkpoint-v2.tar.gz.sha256 + btc-only-v3-observer-checkpoint-v3.tar.gz + btc-only-v3-observer-checkpoint-v3.tar.gz.sha256 compression-level: 0 retention-days: 30 From 7bcd54afa5b14dac4d7baa62bd24fb837edc29c2 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 16:14:32 -0500 Subject: [PATCH 152/177] research: freeze first BTC-only prospective checkpoint --- .../prospective-run-v1/CHECKPOINT.md | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-run-v1/CHECKPOINT.md diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-run-v1/CHECKPOINT.md b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-run-v1/CHECKPOINT.md new file mode 100644 index 00000000..cd9c78c4 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-run-v1/CHECKPOINT.md @@ -0,0 +1,80 @@ +# BTC-Only v3 Prospective Run v1 + +## Classification + +**Valid prospective observation; not strategy admission.** + +This is the first complete post-freeze market collected by `late_favorite_btc_only_v3_prospective_shadow` using the frozen candidate and capture policies. It counts as one observed prospective market and zero hypothetical FOK fills. It does not satisfy any weekly, fill-count, confidence-bound, concentration, or release gate. + +## Market and decision + +- Window: **2026-07-26 21:00–21:05 UTC** / **4:00–4:05 PM Jamaica** +- Slug: `btc-updown-5m-1785099600` +- Condition ID: `0x4c02c25039452fa49c84a0a8d10f59390ac9b26b8b11f52a58ecfcec7962f13c` +- Frozen signal time: market open +210 seconds +- Frozen arrival time: market open +211 seconds +- Selected side: **Up** +- Signal ask: **$0.98** +- Arrival best ask: **$0.99** +- Decision: **`no_fill_ask_above_limit`** +- Official outcome: **Up** +- Hypothetical fill: **No** +- P&L: **Not applicable; no fill** + +The observer did not chase the one-cent adverse price move. Although Up ultimately won, the arrival ask exceeded the frozen 98-cent FOK limit, so no position and no P&L were recorded. + +## Timing and market-data evidence + +- Signal request: started exactly at target; completed in **155 ms**. +- Signal books: timestamped **80 ms** after target and **75 ms** before request completion. +- Arrival request: started exactly at target; completed in **152 ms**. +- Arrival book: timestamped **69 ms** after target and **83 ms** before request completion. +- Full Up/Down signal books and selected-token arrival book were preserved and hashed. + +## Authoritative resolution + +Gamma returned 20 non-terminal payloads followed by one terminal payload. Every poll used a unique deterministic cache-busting query. The terminal response reported: + +- `closed: true` +- outcomes: `["Up", "Down"]` +- terminal prices: `["1", "0"]` +- `updatedAt`: `2026-07-26T21:06:32.32566Z` + +A stale-CDN defect was discovered during the preceding discarded attempt: the unqualified Gamma slug URL could continue returning a pre-close body after terminalization. Resolution requests now append `?cache_bust=`. This changes only transport freshness; it does not alter the candidate, timing, entry, FOK, fee, or outcome rules. + +## Integrity verification + +- GitHub Actions run: `30220121542` +- Artifact ID: `8637040765` +- Artifact ZIP SHA-256: `8d00c88d6046806a2abc07f43da8b2f0ab08449890ee8a7f9e1f7071b92c4e21` +- Workflow head SHA: `257e3b5f1bfa4118a4375fbbfc4693152a296c55` +- Runtime source SHA-256: `ce477bf97589997899c91ad22af9025de521520f8ee7aa46a21b50a313958bd7` +- Lifecycle records: **4**, chain verified +- Lifecycle head: `a2c123f6ba2413d7dddee24ef20d75b096503e0d31a7b1051d8118cfd9911a1d` +- Raw evidence records: **24**, chain and bodies verified +- Raw evidence head: `baa3454eab1f1d8417a0b494a34a0a5ab87a7b7a1ab0e96e058400fafab8d6e0` +- Internal `SHA256SUMS`: passed +- Deterministic TAR sidecar: passed +- ZIP/TAR directory round-trip: byte-for-byte passed +- Credentials used: **0** +- Authenticated Polymarket requests: **0** +- Order submissions: **0** +- Historical admission credit: **0** +- Live submission: **physically absent** + +## Admission ledger after this run + +- Prospective markets observed: **1 / 500 minimum** +- Hypothetical FOK fills: **0 / 100 minimum** +- Official outcome coverage: **1 / 1** +- Complete untouched weekly blocks: **0 / 4 minimum** +- Realized prospective P&L: **$0.00**, because no fill occurred +- Candidate status: **prospective shadow only** + +## Next valid work + +1. Merge the cache-busting resolver correction after the 25-test clean-room checkpoint passes. +2. Continue collecting every eligible BTC five-minute market, including no-signal and no-fill decisions. +3. Preserve authoritative outcomes and complete raw/lifecycle chains for every market. +4. Do not inspect or tune thresholds during the prospective block. +5. Keep live order submission physically absent. From 01efab54c369ff7e3d2f5116c0bccac03e73c0e6 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 16:14:52 -0500 Subject: [PATCH 153/177] research: add prospective run summary --- .../prospective-run-v1/PROSPECTIVE_RUN.json | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-run-v1/PROSPECTIVE_RUN.json diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-run-v1/PROSPECTIVE_RUN.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-run-v1/PROSPECTIVE_RUN.json new file mode 100644 index 00000000..2e1ca3e9 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-run-v1/PROSPECTIVE_RUN.json @@ -0,0 +1,43 @@ +{ + "authenticated_requests": 0, + "candidate": "late_favorite_btc_only_v3_prospective_shadow", + "candidate_policy_sha256": "82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba", + "capture_decision": "no_fill_ask_above_limit", + "capture_policy_sha256": "888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc", + "combined_source_sha256": "ce477bf97589997899c91ad22af9025de521520f8ee7aa46a21b50a313958bd7", + "condition_id": "0x4c02c25039452fa49c84a0a8d10f59390ac9b26b8b11f52a58ecfcec7962f13c", + "credentials_used": 0, + "github_sha": "257e3b5f1bfa4118a4375fbbfc4693152a296c55", + "historical_admission_credit": 0, + "lifecycle_event_types": [ + "discovered", + "signal", + "arrival", + "resolution" + ], + "lifecycle_head_sha256": "a2c123f6ba2413d7dddee24ef20d75b096503e0d31a7b1051d8118cfd9911a1d", + "lifecycle_records": 4, + "live_submission": "physically_absent", + "market_open_epoch_seconds": 1785099600, + "order_submissions": 0, + "raw_evidence_head_sha256": "baa3454eab1f1d8417a0b494a34a0a5ab87a7b7a1ab0e96e058400fafab8d6e0", + "raw_evidence_records": 24, + "resolution": { + "capture_policy_sha256": "888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc", + "condition_id": "0x4c02c25039452fa49c84a0a8d10f59390ac9b26b8b11f52a58ecfcec7962f13c", + "decision": "resolved_no_fill", + "event_type": "resolution", + "gamma_evidence_record_hash": "baa3454eab1f1d8417a0b494a34a0a5ab87a7b7a1ab0e96e058400fafab8d6e0", + "market_open_epoch_seconds": 1785099600, + "official_outcome": "Up", + "official_won": null, + "pnl_per_share": null, + "pnl_total": null, + "policy_sha256": "82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba", + "resolution_payload_sha256": "aa55f228e06bcdbcfb71679b5d417e7571bd8e97bd35ef78fbaa3498e984df14", + "source_sha256": "ce477bf97589997899c91ad22af9025de521520f8ee7aa46a21b50a313958bd7" + }, + "resolution_attempts": 21, + "resolution_status": "terminal", + "run_label": "first-post-cutoff-observation" +} From 6183bc8f17dfab1404791ecf4896be3426c1adae Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 16:15:09 -0500 Subject: [PATCH 154/177] research: add prospective capture summary --- .../prospective-run-v1/CAPTURE_SUMMARY.json | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-run-v1/CAPTURE_SUMMARY.json diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-run-v1/CAPTURE_SUMMARY.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-run-v1/CAPTURE_SUMMARY.json new file mode 100644 index 00000000..2ffbd7ac --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-run-v1/CAPTURE_SUMMARY.json @@ -0,0 +1,37 @@ +{ + "arrival": { + "arrival_best_ask": "0.99", + "arrival_book_sha256": "b58474a8562bb06af33e7dce84be1b27af50c482ac945c2a0226f0c61836996d", + "book_timestamp_ms": 1785099811069, + "decision": "no_fill_ask_above_limit", + "hypothetical_fok_fill": false, + "request_completed_ts_ms": 1785099811152, + "request_duration_ms": 152, + "request_started_ts_ms": 1785099811000, + "signal_ask": "0.98" + }, + "condition_id": "0x4c02c25039452fa49c84a0a8d10f59390ac9b26b8b11f52a58ecfcec7962f13c", + "market_open_epoch_seconds": 1785099600, + "resolution": { + "decision": "resolved_no_fill", + "official_outcome": "Up", + "official_won": null, + "pnl_per_share": null, + "pnl_total": null, + "resolution_payload_sha256": "aa55f228e06bcdbcfb71679b5d417e7571bd8e97bd35ef78fbaa3498e984df14" + }, + "signal": { + "book_timestamps_ms": [ + 1785099810080, + 1785099810080 + ], + "decision": "signal", + "down_book_sha256": "02fe0740029381cbab1dcb7c6c697b7790f3a8e451ba5fb017adee0bce6438b2", + "request_completed_ts_ms": 1785099810155, + "request_duration_ms": 155, + "request_started_ts_ms": 1785099810000, + "selected_side": "Up", + "signal_ask": "0.98", + "up_book_sha256": "da0b50980004035a111c91241adf5cd0381eec5a5345a561e86396039c5b3f42" + } +} From c2945850528d0be0ec0eab3797e54d938a7d43b9 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 16:15:31 -0500 Subject: [PATCH 155/177] research: add prospective verification summary --- .../prospective-run-v1/VERIFICATION.json | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-run-v1/VERIFICATION.json diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-run-v1/VERIFICATION.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-run-v1/VERIFICATION.json new file mode 100644 index 00000000..18a13b40 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-run-v1/VERIFICATION.json @@ -0,0 +1,54 @@ +{ + "artifact": { + "artifact_id": 8637040765, + "artifact_name": "btc-only-v3-prospective-run-v1", + "artifact_zip_sha256": "8d00c88d6046806a2abc07f43da8b2f0ab08449890ee8a7f9e1f7071b92c4e21", + "deterministic_tar_round_trip": "passed", + "internal_sha256sums": "passed", + "tar_sidecar": "passed" + }, + "candidate_policy_sha256": "82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba", + "capture_policy_sha256": "888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc", + "integrity": { + "lifecycle_chain": "passed", + "lifecycle_event_types": [ + "discovered", + "signal", + "arrival", + "resolution" + ], + "lifecycle_head_sha256": "a2c123f6ba2413d7dddee24ef20d75b096503e0d31a7b1051d8118cfd9911a1d", + "lifecycle_records": 4, + "raw_body_hashes": "passed", + "raw_evidence_chain": "passed", + "raw_evidence_head_sha256": "baa3454eab1f1d8417a0b494a34a0a5ab87a7b7a1ab0e96e058400fafab8d6e0", + "raw_evidence_records": 24 + }, + "resolution_poll_audit": { + "cache_bust_first": 3, + "cache_bust_last": 23, + "cache_bust_sequence_complete": true, + "nonterminal_polls": 20, + "resolution_polls": 21, + "terminal_closed": true, + "terminal_outcome_prices": [ + "1", + "0" + ], + "terminal_payload_sha256": "aa55f228e06bcdbcfb71679b5d417e7571bd8e97bd35ef78fbaa3498e984df14", + "terminal_updated_at": "2026-07-26T21:06:32.32566Z" + }, + "run": { + "github_actions_run_id": 30220121542, + "github_head_sha": "257e3b5f1bfa4118a4375fbbfc4693152a296c55", + "runtime_source_sha256": "ce477bf97589997899c91ad22af9025de521520f8ee7aa46a21b50a313958bd7" + }, + "safety": { + "authenticated_requests": 0, + "credentials_used": 0, + "historical_admission_credit": 0, + "live_submission": "physically_absent", + "order_submissions": 0 + }, + "status": "valid_prospective_observation_not_admission" +} From c04e0c64b6b748ebaf5489b228103d4f01e77df7 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 16:18:16 -0500 Subject: [PATCH 156/177] docs: record first BTC-only prospective observation --- .../digital-surface-live-readiness/STATUS.md | 67 ++++++++++--------- 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/ci_payload/digital-surface-live-readiness/STATUS.md b/ci_payload/digital-surface-live-readiness/STATUS.md index bdcf1b00..d81012af 100644 --- a/ci_payload/digital-surface-live-readiness/STATUS.md +++ b/ci_payload/digital-surface-live-readiness/STATUS.md @@ -6,8 +6,7 @@ - Branch: `digital-surface-live-readiness-20260725` - Draft PR: #59 - Official late-favorite decision commit: `cedd659e0b573d8479932c428cd2eaf472c6bf8f` -- BTC-only v3 observer merge: `e88eada8589119baf755f0ab9f40d37160ff125a` -- BTC-only v3 runner merge: `9e794700600ba8a9173040711d09b7feb4817ff9` +- First BTC-only prospective checkpoint merge: `f76097ef513a2e70881e7624cd8d9a60d12a642b` - Real-money execution: **physically disabled** ## Historical research verdicts @@ -32,7 +31,7 @@ Authoritative decision evidence: - `ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/CHECKPOINT.md` - filled condition-set SHA-256: `bf276fbce187d9651d56ecae976520d4251140c1d5b0d440696c5d077838a67a` -- full official evidence artifact SHA-256: `210984541bcd64597ca48d480ad59102427f56b4ac204ab1308adf8a4f93ce5` +- full official evidence artifact SHA-256: `210984541bcd64597ca48d480ad59102427f56b4ac204ab1308ad8f8a4f93ce5` - compact derivative artifact SHA-256: `fb5dfb4621caa8b62ebaefc0b5c15d04619f773c3686cc4f088767ca023416b9` ## BTC-only v3 prospective shadow track @@ -48,7 +47,7 @@ BTC-only is a **marginal post-hoc lead**, not a historical pass. It has zero inh - effective capture cutoff: `2026-07-26T19:35:21Z` - only markets opening strictly after the effective capture cutoff may count. -### Observer, collector, and resolver +### Observer, collector, resolver, and CLI The merged observer implements: @@ -58,35 +57,41 @@ The merged observer implements: 4. adverse-move, price-limit, depth, stale, timeout, token, and lifecycle fail-closed decisions; 5. append-only candidate/capture/source-bound lifecycle records; 6. content-addressed raw public evidence and a separate manifest hash chain; -7. public Gamma market-by-slug discovery and terminal resolution; +7. public Gamma discovery and cache-busted terminal resolution; 8. public CLOB `POST /books` signal and arrival snapshots; -9. official fill P&L and terminal no-fill closure. +9. official fill P&L and terminal no-fill closure; +10. one-shot `capture` and `resolve` command boundaries. -Observer verification: +Current observer verification: -- 22/22 behavior contracts passed; -- workflow run: `30217953678`; -- artifact ID: `8636344655`; -- artifact ZIP SHA-256: `667d7090ecfe3d174e5821af29a26fde2e18a87a50f7891cdcc49dff740f26af`; -- combined source SHA-256: `f2e086bb5e74f0343a4ea29f46ea2bf8e23062952b456303e204c6b5469b6126`. +- 25/25 behavior contracts passed; +- workflow run: `30220717065`; +- artifact ID: `8637116192`; +- artifact ZIP SHA-256: `172ef7213e7860a8bee00db5e0a79d66495d31199cde85d39dcfef201f14de86`; +- combined frozen source SHA-256: `ce477bf97589997899c91ad22af9025de521520f8ee7aa46a21b50a313958bd7`; +- resolver source SHA-256: `90c417be653f5f0fdf22dfc7d4e20168869594a021fc3229d87b7f203bc9dc77`. -### One-shot runner +### First complete prospective observation -The merged runner exposes only: +Checkpoint: -- `collect-next`; -- `resolve --condition-id`; -- `status`. +- `ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-run-v1/CHECKPOINT.md` +- workflow run: `30220121542`; +- evidence artifact ID: `8637040765`; +- evidence ZIP SHA-256: `8d00c88d6046806a2abc07f43da8b2f0ab08449890ee8a7f9e1f7071b92c4e21`. -Runner verification: +Result: -- 5/5 runner contracts passed; -- workflow run: `30218421705`; -- artifact ID: `8636471934`; -- artifact ZIP SHA-256: `762294c4141c25d6dd0a1f6678e2f90795bfbecf909deaf130d3001358ca7b88`; -- combined runtime source SHA-256: `b398e7b9893170cb47ea17923448a87a10b017249a950ca670f9b09081b0d845`. +- market: `btc-updown-5m-1785099600`, 2026-07-26 21:00–21:05 UTC; +- selected side: Up; +- signal ask: $0.98; +- arrival best ask: $0.99; +- frozen decision: `no_fill_ask_above_limit`; +- official outcome: Up; +- hypothetical fill: no; +- P&L: not applicable. -The known-fee field is now recorded for hypothetical fills even when diagnostic inferred outcomes are null. This audit correction does not change the official rejection result. +The complete lifecycle and raw-evidence chains were independently verified. Twenty non-terminal Gamma payloads were followed by one terminal `[1,0]` payload. The one-cent adverse price move was correctly rejected rather than chased. ## Safety and admission boundary @@ -95,15 +100,17 @@ The known-fee field is now recorded for hypothetical fills even when diagnostic - order submissions: **0**; - live submission: **physically absent**; - historical admission credit: **0**; -- prospective markets collected so far: **0**. +- prospective markets observed: **1 / 500 minimum**; +- hypothetical FOK fills: **0 / 100 minimum**; +- complete untouched weekly blocks: **0 / 4 minimum**. -No historical result is admissible. Even a future admissible shadow report cannot enable live trading; any order submitter requires a separate reviewed release and explicit operator action. +No historical result is admissible. This single no-fill observation is valid prospective evidence but cannot support admission. Even a future admissible shadow report cannot enable live trading; any order submitter requires a separate reviewed release and explicit operator action. ## Next work -1. execute one credential-free `collect-next` run on a genuinely new post-cutoff BTC window; -2. preserve its append-only lifecycle, raw evidence, summaries, policy hashes, source hash, and artifact digest; -3. resolve the condition from authoritative terminal outcomes after closure; -4. repeat without policy changes for at least four untouched weekly blocks; +1. continue credential-free collection of every eligible BTC five-minute market, including no-signal and no-fill cases; +2. preserve append-only lifecycle and raw evidence with authoritative terminal outcomes; +3. make no threshold or policy changes during the prospective block; +4. accumulate at least 500 observed markets, 100 hypothetical FOK fills, and four untouched weekly blocks; 5. require measured fill evidence, acceptable concentration, zero unresolved states, and a positive lower confidence bound after fees and execution costs; 6. keep real-money execution physically disabled. From 230774b4ac0f1ef51f6c90cb484e24f45eb4548f Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 16:24:08 -0500 Subject: [PATCH 157/177] test: specify bounded BTC-only prospective blocks --- .../runner/tests/test_observer_block.py | 236 ++++++++++++++++++ 1 file changed, 236 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/tests/test_observer_block.py diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/tests/test_observer_block.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/tests/test_observer_block.py new file mode 100644 index 00000000..3f2988e0 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/tests/test_observer_block.py @@ -0,0 +1,236 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path + +import pytest + +from observer_cli import ( + MAX_BLOCK_MARKETS, + ObserverRuntime, + build_parser, + collect_block, +) + + +@dataclass +class FakeCapturePolicy: + valid_market_open_after_epoch_seconds: int = 1_800_000_100 + + +@dataclass +class FakeLedger: + operations: list[str] + head_hash: str = "a" * 64 + records: list[dict[str, object]] = field(default_factory=list) + + +@dataclass +class FakeEvidence: + head_hash: str = "b" * 64 + records: list[dict[str, object]] = field(default_factory=list) + + +class FakeCollector: + def __init__(self, operations: list[str], ledger: FakeLedger, evidence: FakeEvidence) -> None: + self.operations = operations + self.ledger = ledger + self.evidence = evidence + self.openings: list[int] = [] + + def capture(self, opening: int) -> dict[str, object]: + self.openings.append(opening) + self.operations.append(f"capture:{opening}") + condition = f"condition-{opening}" + decision = ( + "hypothetical_fok_fill" + if len(self.openings) == 1 + else "no_signal_below_threshold" + if len(self.openings) == 2 + else "no_fill_ask_above_limit" + ) + self.ledger.records.append( + { + "condition_id": condition, + "event_type": "arrival" if decision != "no_signal_below_threshold" else "signal", + } + ) + self.evidence.records.append({"opening": opening}) + return { + "condition_id": condition, + "market_open_epoch_seconds": opening, + "decision": decision, + "hypothetical_fok_fill": decision == "hypothetical_fok_fill", + } + + +class FakeResolver: + def __init__( + self, + operations: list[str], + ledger: FakeLedger, + evidence: FakeEvidence, + *, + failures_before_success: int = 0, + never_resolve: bool = False, + ) -> None: + self.operations = operations + self.ledger = ledger + self.evidence = evidence + self.failures_before_success = failures_before_success + self.never_resolve = never_resolve + self.attempts: dict[str, int] = {} + + def resolve(self, condition_id: str) -> dict[str, object]: + self.operations.append(f"resolve:{condition_id}") + attempt = self.attempts.get(condition_id, 0) + 1 + self.attempts[condition_id] = attempt + self.evidence.records.append({"condition_id": condition_id, "attempt": attempt}) + if self.never_resolve or attempt <= self.failures_before_success: + raise ValueError("Gamma market is not closed") + self.ledger.records.append( + {"condition_id": condition_id, "event_type": "resolution"} + ) + return { + "condition_id": condition_id, + "decision": "resolved_fill" if condition_id.endswith("300") else "resolved_no_fill", + "official_outcome": "Up", + "pnl_total": "0.45" if condition_id.endswith("300") else None, + } + + +def runtime( + tmp_path: Path, + *, + now_ms: int, + failures_before_success: int = 0, + never_resolve: bool = False, +) -> tuple[ObserverRuntime, list[str], list[int]]: + operations: list[str] = [] + sleeps: list[int] = [] + ledger = FakeLedger(operations) + evidence = FakeEvidence() + collector = FakeCollector(operations, ledger, evidence) + resolver = FakeResolver( + operations, + ledger, + evidence, + failures_before_success=failures_before_success, + never_resolve=never_resolve, + ) + return ( + ObserverRuntime( + project_root=tmp_path, + state_dir=tmp_path / "state", + capture_policy=FakeCapturePolicy(), + ledger=ledger, + evidence_store=evidence, + collector=collector, + resolver=resolver, + clock_ms=lambda: now_ms, + sleep_until_ms=sleeps.append, + source_sha256="c" * 64, + ), + operations, + sleeps, + ) + + +def test_collect_block_captures_consecutive_windows_before_any_resolution(tmp_path: Path) -> None: + first_opening = 1_800_000_300 + now_ms = first_opening * 1000 + 500 + rt, operations, sleeps = runtime(tmp_path, now_ms=now_ms, failures_before_success=1) + + summary = collect_block( + rt, + markets=3, + now_ts_ms=now_ms, + max_resolution_attempts=3, + retry_delay_ms=10_000, + ) + + openings = [first_opening, first_opening + 300, first_opening + 600] + assert rt.collector.openings == openings + first_resolve_index = next( + index for index, operation in enumerate(operations) if operation.startswith("resolve:") + ) + assert all(operation.startswith("capture:") for operation in operations[:first_resolve_index]) + assert operations[:3] == [f"capture:{opening}" for opening in openings] + assert sleeps[:3] == [opening * 1000 + 1_000 for opening in openings] + assert summary["status"] == "complete" + assert summary["markets_requested"] == 3 + assert summary["markets_collected"] == 3 + assert summary["conditions_resolved"] == 3 + assert summary["unresolved_condition_ids"] == [] + assert summary["hypothetical_fok_fills"] == 1 + assert summary["capture_decision_counts"] == { + "hypothetical_fok_fill": 1, + "no_fill_ask_above_limit": 1, + "no_signal_below_threshold": 1, + } + assert summary["resolution_attempts_by_condition"] == { + f"condition-{opening}": 2 for opening in openings + } + saved = json.loads( + ( + rt.state_dir + / "summaries" + / f"block-{openings[0]}-{openings[-1]}.json" + ).read_text() + ) + assert saved == summary + + +def test_collect_block_records_unresolved_conditions_after_bounded_attempts(tmp_path: Path) -> None: + opening = 1_800_000_300 + now_ms = opening * 1000 + 500 + rt, operations, sleeps = runtime(tmp_path, now_ms=now_ms, never_resolve=True) + + summary = collect_block( + rt, + markets=1, + now_ts_ms=now_ms, + max_resolution_attempts=2, + retry_delay_ms=5_000, + ) + + condition = f"condition-{opening}" + assert summary["status"] == "unresolved" + assert summary["conditions_resolved"] == 0 + assert summary["unresolved_condition_ids"] == [condition] + assert summary["resolution_attempts_by_condition"] == {condition: 2} + assert operations == [ + f"capture:{opening}", + f"resolve:{condition}", + f"resolve:{condition}", + ] + assert sleeps == [opening * 1000 + 1_000, now_ms + 5_000] + + +@pytest.mark.parametrize("markets", [0, -1, MAX_BLOCK_MARKETS + 1]) +def test_collect_block_rejects_unbounded_market_counts(tmp_path: Path, markets: int) -> None: + rt, _, _ = runtime(tmp_path, now_ms=1_800_000_300_500) + with pytest.raises(ValueError): + collect_block(rt, markets=markets) + + +def test_cli_exposes_bounded_collect_block_without_live_commands() -> None: + parser = build_parser() + args = parser.parse_args( + [ + "collect-block", + "--markets", + "3", + "--max-resolution-attempts", + "120", + "--retry-delay-ms", + "10000", + ] + ) + assert args.command == "collect-block" + assert args.markets == 3 + assert args.max_resolution_attempts == 120 + assert args.retry_delay_ms == 10_000 + with pytest.raises(SystemExit): + parser.parse_args(["submit-block"]) From a2ba091c1977b9098131c8d51173112d3e448eae Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 16:26:29 -0500 Subject: [PATCH 158/177] feat: add bounded capture-first prospective blocks --- .../runner/observer_cli.py | 169 ++++++++++++++++-- 1 file changed, 157 insertions(+), 12 deletions(-) diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/observer_cli.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/observer_cli.py index 0687631f..fdc7040c 100644 --- a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/observer_cli.py +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/observer_cli.py @@ -6,6 +6,7 @@ import re import time from dataclasses import dataclass +from decimal import Decimal, InvalidOperation from pathlib import Path from typing import Any, Callable @@ -27,6 +28,10 @@ "runner/observer_cli.py", ) +MAX_BLOCK_MARKETS = 12 +DEFAULT_RESOLUTION_ATTEMPTS = 120 +DEFAULT_RETRY_DELAY_MS = 10_000 + @dataclass(slots=True) class ObserverRuntime: @@ -156,27 +161,32 @@ def _evidence_head(runtime: ObserverRuntime) -> str: return str(getattr(runtime.evidence_store, "head_hash", "0" * 64)) -def collect_next(runtime: ObserverRuntime, *, now_ts_ms: int | None = None) -> dict[str, Any]: - now = int(runtime.clock_ms() if now_ts_ms is None else now_ts_ms) - opening = next_collectable_open( - now, - int(runtime.capture_policy.valid_market_open_after_epoch_seconds), - ) - discovery_target = opening * 1000 + 1_000 - if now < discovery_target: +def _collect_opening(runtime: ObserverRuntime, opening: int) -> dict[str, Any]: + selected_opening = int(opening) + discovery_target = selected_opening * 1000 + 1_000 + if int(runtime.clock_ms()) < discovery_target: runtime.sleep_until_ms(discovery_target) - result = dict(runtime.collector.capture(opening)) + result = dict(runtime.collector.capture(selected_opening)) summary = { **result, "command": "collect-next", - "market_open_epoch_seconds": opening, + "market_open_epoch_seconds": selected_opening, "source_sha256": runtime.source_sha256, "lifecycle_head_sha256": str(runtime.ledger.head_hash), "raw_evidence_head_sha256": _evidence_head(runtime), "lifecycle_records": len(getattr(runtime.ledger, "records", [])), "raw_evidence_records": len(getattr(runtime.evidence_store, "records", [])), } - return _write_summary(runtime, f"collect-{opening}", summary) + return _write_summary(runtime, f"collect-{selected_opening}", summary) + + +def collect_next(runtime: ObserverRuntime, *, now_ts_ms: int | None = None) -> dict[str, Any]: + now = int(runtime.clock_ms() if now_ts_ms is None else now_ts_ms) + opening = next_collectable_open( + now, + int(runtime.capture_policy.valid_market_open_after_epoch_seconds), + ) + return _collect_opening(runtime, opening) def resolve_condition(runtime: ObserverRuntime, condition_id: str) -> dict[str, Any]: @@ -196,6 +206,121 @@ def resolve_condition(runtime: ObserverRuntime, condition_id: str) -> dict[str, return _write_summary(runtime, f"resolve-{condition}", summary) +def _positive_int(name: str, value: int) -> int: + parsed = int(value) + if parsed <= 0: + raise ValueError(f"{name} must be positive") + return parsed + + +def _pnl_total(resolution: dict[str, Any]) -> Decimal: + value = resolution.get("pnl_total") + if value is None: + return Decimal("0") + try: + return Decimal(str(value)) + except (InvalidOperation, ValueError) as exc: + raise ValueError(f"invalid pnl_total in resolution: {value!r}") from exc + + +def collect_block( + runtime: ObserverRuntime, + *, + markets: int, + now_ts_ms: int | None = None, + max_resolution_attempts: int = DEFAULT_RESOLUTION_ATTEMPTS, + retry_delay_ms: int = DEFAULT_RETRY_DELAY_MS, +) -> dict[str, Any]: + market_count = _positive_int("markets", markets) + if market_count > MAX_BLOCK_MARKETS: + raise ValueError(f"markets must be <= {MAX_BLOCK_MARKETS}") + attempt_limit = _positive_int("max_resolution_attempts", max_resolution_attempts) + retry_delay = _positive_int("retry_delay_ms", retry_delay_ms) + + now = int(runtime.clock_ms() if now_ts_ms is None else now_ts_ms) + first_opening = next_collectable_open( + now, + int(runtime.capture_policy.valid_market_open_after_epoch_seconds), + ) + openings = [first_opening + 300 * index for index in range(market_count)] + + # Capture every requested consecutive window before making any resolution + # request. This keeps terminal latency from causing a missed +210s signal. + captures = [_collect_opening(runtime, opening) for opening in openings] + condition_ids = [str(capture["condition_id"]) for capture in captures] + + resolutions: list[dict[str, Any]] = [] + unresolved: list[str] = [] + attempts_by_condition: dict[str, int] = {} + last_errors: dict[str, dict[str, str]] = {} + + for condition in condition_ids: + resolved: dict[str, Any] | None = None + for attempt in range(1, attempt_limit + 1): + attempts_by_condition[condition] = attempt + try: + resolved = resolve_condition(runtime, condition) + break + except Exception as exc: # fail closed and preserve the block evidence + last_errors[condition] = { + "type": type(exc).__name__, + "message": str(exc), + } + if attempt < attempt_limit: + runtime.sleep_until_ms(int(runtime.clock_ms()) + retry_delay) + if resolved is None: + unresolved.append(condition) + else: + resolutions.append(resolved) + last_errors.pop(condition, None) + + decision_counts: dict[str, int] = {} + hypothetical_fok_fills = 0 + for capture in captures: + decision = str(capture.get("decision", "unknown")) + decision_counts[decision] = decision_counts.get(decision, 0) + 1 + if capture.get("hypothetical_fok_fill") is True or decision == "hypothetical_fok_fill": + hypothetical_fok_fills += 1 + + total_pnl = sum((_pnl_total(resolution) for resolution in resolutions), Decimal("0")) + summary = { + "command": "collect-block", + "status": "complete" if not unresolved else "unresolved", + "markets_requested": market_count, + "markets_collected": len(captures), + "first_market_open_epoch_seconds": openings[0], + "last_market_open_epoch_seconds": openings[-1], + "market_open_epoch_seconds": openings, + "condition_ids": condition_ids, + "capture_decision_counts": dict(sorted(decision_counts.items())), + "hypothetical_fok_fills": hypothetical_fok_fills, + "conditions_resolved": len(resolutions), + "unresolved_condition_ids": unresolved, + "resolution_attempts_by_condition": attempts_by_condition, + "last_resolution_errors": last_errors, + "resolution_decision_counts": { + decision: sum(1 for row in resolutions if str(row.get("decision")) == decision) + for decision in sorted({str(row.get("decision")) for row in resolutions}) + }, + "prospective_pnl_total": format(total_pnl, "f"), + "source_sha256": runtime.source_sha256, + "lifecycle_head_sha256": str(runtime.ledger.head_hash), + "raw_evidence_head_sha256": _evidence_head(runtime), + "lifecycle_records": len(getattr(runtime.ledger, "records", [])), + "raw_evidence_records": len(getattr(runtime.evidence_store, "records", [])), + "credentials_used": 0, + "authenticated_requests": 0, + "order_submissions": 0, + "live_submission": "physically_absent", + "policy_changes": 0, + } + return _write_summary( + runtime, + f"block-{openings[0]}-{openings[-1]}", + summary, + ) + + def runtime_status(runtime: ObserverRuntime) -> dict[str, Any]: condition_ids: set[str] = set() resolved: set[str] = set() @@ -232,6 +357,18 @@ def build_parser() -> argparse.ArgumentParser: commands = parser.add_subparsers(dest="command", required=True) collect = commands.add_parser("collect-next", help="collect the next eligible BTC window") collect.add_argument("--now-ms", type=int) + block = commands.add_parser( + "collect-block", + help="capture consecutive BTC windows, then resolve the captured queue", + ) + block.add_argument("--markets", type=int, required=True) + block.add_argument("--now-ms", type=int) + block.add_argument( + "--max-resolution-attempts", + type=int, + default=DEFAULT_RESOLUTION_ATTEMPTS, + ) + block.add_argument("--retry-delay-ms", type=int, default=DEFAULT_RETRY_DELAY_MS) resolve = commands.add_parser("resolve", help="resolve one collected condition") resolve.add_argument("--condition-id", required=True) commands.add_parser("status", help="show ledger and unresolved-condition status") @@ -243,6 +380,14 @@ def main(argv: list[str] | None = None) -> int: runtime = build_runtime(args.project_root, args.state_dir) if args.command == "collect-next": output = collect_next(runtime, now_ts_ms=args.now_ms) + elif args.command == "collect-block": + output = collect_block( + runtime, + markets=args.markets, + now_ts_ms=args.now_ms, + max_resolution_attempts=args.max_resolution_attempts, + retry_delay_ms=args.retry_delay_ms, + ) elif args.command == "resolve": output = resolve_condition(runtime, args.condition_id) elif args.command == "status": @@ -250,7 +395,7 @@ def main(argv: list[str] | None = None) -> int: else: # pragma: no cover raise RuntimeError(f"unsupported command: {args.command}") print(json.dumps(output, indent=2, sort_keys=True)) - return 0 + return 0 if output.get("status") != "unresolved" else 2 if __name__ == "__main__": From 606f39cc96881bf87090028288ba418d75a6b83a Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 16:27:10 -0500 Subject: [PATCH 159/177] ci: verify bounded BTC-only block runner --- .../btc-only-v3-runner-checkpoint.yml | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/.github/workflows/btc-only-v3-runner-checkpoint.yml b/.github/workflows/btc-only-v3-runner-checkpoint.yml index 5af6cef2..d424a37b 100644 --- a/.github/workflows/btc-only-v3-runner-checkpoint.yml +++ b/.github/workflows/btc-only-v3-runner-checkpoint.yml @@ -43,9 +43,10 @@ jobs: root=ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 python -m py_compile \ "$root/runner/observer_cli.py" \ - "$root/runner/tests/test_observer_cli.py" + "$root/runner/tests/test_observer_cli.py" \ + "$root/runner/tests/test_observer_block.py" - - name: Pass exactly 5 runner contracts + - name: Pass exactly 11 runner contracts shell: bash env: PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner @@ -53,7 +54,7 @@ jobs: set -euo pipefail root=ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 pytest --collect-only -q "$root/runner/tests" | tee /tmp/collect.txt - grep -Eq '^5 tests collected' /tmp/collect.txt + grep -Eq '^11 tests collected' /tmp/collect.txt pytest -q "$root/runner/tests" - name: Verify runtime loads frozen policies without network @@ -68,7 +69,12 @@ jobs: import tempfile from pathlib import Path - from observer_cli import build_runtime, compute_combined_source_sha256, runtime_status + from observer_cli import ( + MAX_BLOCK_MARKETS, + build_runtime, + compute_combined_source_sha256, + runtime_status, + ) root = Path('ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3') relative_paths = ( @@ -86,6 +92,7 @@ jobs: independent.update(relative.encode('utf-8') + b'\0' + raw) source_sha256 = compute_combined_source_sha256(root) assert source_sha256 == independent.hexdigest() + assert MAX_BLOCK_MARKETS == 12 with tempfile.TemporaryDirectory() as directory: runtime = build_runtime(root, directory, clock_ms=lambda: 1_900_000_000_000) status = runtime_status(runtime) @@ -96,8 +103,10 @@ jobs: assert status['raw_evidence_head_sha256'] == '0' * 64 verification = { 'candidate': 'late_favorite_btc_only_v3_prospective_shadow', - 'tests_collected': 5, - 'runner_commands': ['collect-next', 'resolve', 'status'], + 'tests_collected': 11, + 'runner_commands': ['collect-next', 'collect-block', 'resolve', 'status'], + 'maximum_block_markets': MAX_BLOCK_MARKETS, + 'block_order': 'capture_all_then_resolve_queue', 'combined_runtime_source_sha256': source_sha256, 'source_files_sha256': source_hashes, 'credentials_used': 0, @@ -141,6 +150,7 @@ jobs: ): assert token not in lowered, token assert 'collect-next' in text + assert 'collect-block' in text assert 'resolve' in text assert 'status' in text assert 'submit-order' not in text @@ -160,15 +170,15 @@ jobs: cp "$root/runner/RUNNER_VERIFICATION.json" btc-only-v3-runner-checkpoint/ (cd btc-only-v3-runner-checkpoint && sha256sum * > SHA256SUMS) tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ - -czf btc-only-v3-runner-checkpoint-v1.tar.gz btc-only-v3-runner-checkpoint - sha256sum btc-only-v3-runner-checkpoint-v1.tar.gz > btc-only-v3-runner-checkpoint-v1.tar.gz.sha256 + -czf btc-only-v3-runner-checkpoint-v2.tar.gz btc-only-v3-runner-checkpoint + sha256sum btc-only-v3-runner-checkpoint-v2.tar.gz > btc-only-v3-runner-checkpoint-v2.tar.gz.sha256 - uses: actions/upload-artifact@v4 with: - name: btc-only-v3-runner-checkpoint-v1 + name: btc-only-v3-runner-checkpoint-v2 path: | btc-only-v3-runner-checkpoint/ - btc-only-v3-runner-checkpoint-v1.tar.gz - btc-only-v3-runner-checkpoint-v1.tar.gz.sha256 + btc-only-v3-runner-checkpoint-v2.tar.gz + btc-only-v3-runner-checkpoint-v2.tar.gz.sha256 compression-level: 0 retention-days: 30 From 0a563da70129d227dca7ef1419fd2bef2963c89d Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 16:52:37 -0500 Subject: [PATCH 160/177] research: freeze first three-market BTC prospective block --- .../prospective-block-v1/CHECKPOINT.md | 115 ++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v1/CHECKPOINT.md diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v1/CHECKPOINT.md b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v1/CHECKPOINT.md new file mode 100644 index 00000000..a59c0ec9 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v1/CHECKPOINT.md @@ -0,0 +1,115 @@ +# BTC-Only v3 Prospective Block v1 + +## Classification + +**Valid prospective block evidence; not strategy admission.** + +This checkpoint covers three consecutive post-freeze BTC five-minute markets collected under the unchanged `late_favorite_btc_only_v3_prospective_shadow` candidate and capture policies. Every market was captured before any resolution request was made. All three conditions later received authoritative Gamma terminal outcomes. + +The block produced **zero hypothetical FOK fills** and **$0.00 prospective P&L**. It increases the cumulative prospective market count from 1 to 4, but does not satisfy the minimum market, fill, weekly-block, confidence-bound, concentration, or release gates. + +## Block identity + +- Windows: **2026-07-26 21:30–21:45 UTC** / **4:30–4:45 PM Jamaica** +- Consecutive market openings: `1785101400`, `1785101700`, `1785102000` +- GitHub Actions run: `30221188931` +- Workflow head SHA: `be327b99530d9a5b1142e0c9896ae01716d4b070` +- Artifact ID: `8637433570` +- Artifact ZIP SHA-256: `222d8261ea92dfdfcd3fbb9ccb12dbf6948d03f6bed1e7558f3d9e4bdc6a5580` +- Runtime source SHA-256: `81a13d2aeb3df3c2088b8aa99272b10e217a392eabc839a01a8f2b71f570ecbb` +- Candidate policy SHA-256: `82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba` +- Capture policy SHA-256: `888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc` + +## Market results + +| Jamaica window | Selected side | Signal ask | Arrival | Frozen decision | Official outcome | P&L | +|---|---:|---:|---:|---|---:|---:| +| 4:30–4:35 PM | Up | $0.66 | Not requested | `no_signal_below_threshold` | Up | $0.00 | +| 4:35–4:40 PM | Down | $0.76 | Not requested | `no_signal_below_threshold` | Down | $0.00 | +| 4:40–4:45 PM | Up | $0.95 | $0.98 best ask | `no_fill_ask_above_limit` | Down | $0.00 | + +The third market is decision-useful. Up qualified at a 95-cent signal ask, but one second later its best ask was 98 cents—above the frozen 95-cent FOK limit. The observer refused to chase the move. The market later resolved Down, so the no-fill rule avoided a losing position. + +## Capture timing + +### Market 1 — `btc-updown-5m-1785101400` + +- Condition: `0x5452c4c60c9d3ed96739f7a46a58533244f2221ec504c352a808c7e3e90b379b` +- Signal request start: exactly +210,000 ms +- Signal request duration: 323 ms +- Signal book timestamp: +210,256 ms +- Decision: below 85-cent threshold + +### Market 2 — `btc-updown-5m-1785101700` + +- Condition: `0x7bd9fe5bfea03a72553b721d0824512a3e9dee6a0fec44992135d5cf08b3e352` +- Signal request start: +210,001 ms +- Signal request duration: 152 ms +- Signal book timestamp: +210,083 ms +- Decision: below 85-cent threshold + +### Market 3 — `btc-updown-5m-1785102000` + +- Condition: `0x1dcecaa2f4ee994fd4ab1b9f309bb18a56f24a5f9c84b82c290a7489d9cfd27d` +- Signal request start: exactly +210,000 ms +- Signal request duration: 170 ms +- Signal book timestamp: +210,112 ms +- Arrival request start: exactly +211,000 ms +- Arrival request duration: 265 ms +- Arrival book timestamp: +211,206 ms +- Decision: arrival ask above frozen limit + +All request starts, durations, and book timestamps remained within the frozen capture-policy bounds. + +## Authoritative outcomes + +| Market | Resolution polls | Official outcome | Terminal prices | Gamma `updatedAt` | Terminal payload SHA-256 | +|---|---:|---:|---:|---|---| +| `1785101400` | 1 | Up | `[1, 0]` | `2026-07-26T21:37:05.248916Z` | `52737e9eb6279c33689c1044cf9074dad2dd98511b6fcc8e48439f5dea81e8ae` | +| `1785101700` | 5 | Down | `[0, 1]` | `2026-07-26T21:43:14.819851Z` | `a6fe3e627f6ac977e727d5fceb9fa47fd1b6ac9d2e1fc98a61aea9eccfa5aa86` | +| `1785102000` | 31 | Down | `[0, 1]` | `2026-07-26T21:46:24.754458Z` | `8693fbc94d74fbd218e18c3a8d65961efd2e997223509135b3b97395af6e89ca` | + +Resolution polling began only after all three captures completed. Every poll used the cache-busted public Gamma endpoint and was preserved in the raw-evidence chain. + +## Integrity verification + +- Internal `SHA256SUMS`: **passed** +- Deterministic TAR sidecar: **passed** +- ZIP/TAR directory round-trip: **byte-for-byte passed** +- Lifecycle records: **10** + - discovered: 3 + - signal: 3 + - arrival: 1 + - resolution: 3 +- Lifecycle chain: **passed** +- Lifecycle head: `1f6f21cf25c1acd11ea5b06e4bd3dec617d69d84c8b25fb5471be22fc9430c11` +- Raw evidence records: **44** + - market discovery: 3 + - signal books: 3 + - arrival books: 1 + - official resolution payloads: 37 +- Raw evidence chain and body hashes: **passed** +- Raw evidence head: `9b43483f4a310d1ad84a3b6f5a029dc772f2181cb06dce08db476aab00dc2cf0` + +## Admission ledger after this block + +- Prospective markets observed: **4 / 500 minimum** +- Official outcomes available: **4 / 4** +- Hypothetical FOK fills: **0 / 100 minimum** +- Prospective P&L: **$0.00** +- Complete untouched weekly blocks: **0 / 4 minimum** +- Credentials used: **0** +- Authenticated requests: **0** +- Order submissions: **0** +- Historical admission credit: **0** +- Live submission: **physically absent** + +The first one-shot observation and this block used different orchestration-source hashes, while the frozen candidate and capture policies remained identical. No weekly block receives completion credit yet. + +## Next valid work + +1. Continue capture-first prospective blocks without modifying candidate or capture thresholds. +2. Preserve every no-signal, no-fill, fill, resolution, and transport failure. +3. Accumulate at least 500 observed markets, 100 hypothetical fills, and four untouched weekly blocks. +4. Require zero unresolved states, acceptable concentration, measured execution evidence, and a positive lower confidence bound after fees and costs. +5. Keep real-money order submission physically absent. From f9d12d94729b9874fed8455331df4561d8976da3 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 16:52:56 -0500 Subject: [PATCH 161/177] research: add three-market block summary --- .../prospective-block-v1/BLOCK_SUMMARY.json | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v1/BLOCK_SUMMARY.json diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v1/BLOCK_SUMMARY.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v1/BLOCK_SUMMARY.json new file mode 100644 index 00000000..4916e62c --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v1/BLOCK_SUMMARY.json @@ -0,0 +1,45 @@ +{ + "authenticated_requests": 0, + "capture_decision_counts": { + "no_fill_ask_above_limit": 1, + "no_signal_below_threshold": 2 + }, + "command": "collect-block", + "condition_ids": [ + "0x5452c4c60c9d3ed96739f7a46a58533244f2221ec504c352a808c7e3e90b379b", + "0x7bd9fe5bfea03a72553b721d0824512a3e9dee6a0fec44992135d5cf08b3e352", + "0x1dcecaa2f4ee994fd4ab1b9f309bb18a56f24a5f9c84b82c290a7489d9cfd27d" + ], + "conditions_resolved": 3, + "credentials_used": 0, + "first_market_open_epoch_seconds": 1785101400, + "hypothetical_fok_fills": 0, + "last_market_open_epoch_seconds": 1785102000, + "last_resolution_errors": {}, + "lifecycle_head_sha256": "1f6f21cf25c1acd11ea5b06e4bd3dec617d69d84c8b25fb5471be22fc9430c11", + "lifecycle_records": 10, + "live_submission": "physically_absent", + "market_open_epoch_seconds": [ + 1785101400, + 1785101700, + 1785102000 + ], + "markets_collected": 3, + "markets_requested": 3, + "order_submissions": 0, + "policy_changes": 0, + "prospective_pnl_total": "0", + "raw_evidence_head_sha256": "9b43483f4a310d1ad84a3b6f5a029dc772f2181cb06dce08db476aab00dc2cf0", + "raw_evidence_records": 44, + "resolution_attempts_by_condition": { + "0x1dcecaa2f4ee994fd4ab1b9f309bb18a56f24a5f9c84b82c290a7489d9cfd27d": 31, + "0x5452c4c60c9d3ed96739f7a46a58533244f2221ec504c352a808c7e3e90b379b": 1, + "0x7bd9fe5bfea03a72553b721d0824512a3e9dee6a0fec44992135d5cf08b3e352": 5 + }, + "resolution_decision_counts": { + "resolved_no_fill": 3 + }, + "source_sha256": "81a13d2aeb3df3c2088b8aa99272b10e217a392eabc839a01a8f2b71f570ecbb", + "status": "complete", + "unresolved_condition_ids": [] +} From 30e453923a5de34ec544a888d6f611a46e8deba5 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 16:53:17 -0500 Subject: [PATCH 162/177] research: add block market decisions --- .../prospective-block-v1/MARKETS.json | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v1/MARKETS.json diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v1/MARKETS.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v1/MARKETS.json new file mode 100644 index 00000000..9a8a0406 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v1/MARKETS.json @@ -0,0 +1,80 @@ +[ + { + "arrival": null, + "condition_id": "0x5452c4c60c9d3ed96739f7a46a58533244f2221ec504c352a808c7e3e90b379b", + "decision": "no_signal_below_threshold", + "market_open_epoch_seconds": 1785101400, + "official_outcome": "Up", + "pnl_total": null, + "resolution_payload_sha256": "52737e9eb6279c33689c1044cf9074dad2dd98511b6fcc8e48439f5dea81e8ae", + "resolution_polls": 1, + "selected_side": "Up", + "signal": { + "book_timestamp_ms": 1785101610256, + "request_completed_ts_ms": 1785101610323, + "request_duration_ms": 323, + "request_started_ts_ms": 1785101610000, + "signal_ask": "0.66" + }, + "slug": "btc-updown-5m-1785101400", + "terminal_outcome_prices": [ + "1", + "0" + ], + "terminal_updated_at": "2026-07-26T21:37:05.248916Z" + }, + { + "arrival": null, + "condition_id": "0x7bd9fe5bfea03a72553b721d0824512a3e9dee6a0fec44992135d5cf08b3e352", + "decision": "no_signal_below_threshold", + "market_open_epoch_seconds": 1785101700, + "official_outcome": "Down", + "pnl_total": null, + "resolution_payload_sha256": "a6fe3e627f6ac977e727d5fceb9fa47fd1b6ac9d2e1fc98a61aea9eccfa5aa86", + "resolution_polls": 5, + "selected_side": "Down", + "signal": { + "book_timestamp_ms": 1785101910083, + "request_completed_ts_ms": 1785101910153, + "request_duration_ms": 152, + "request_started_ts_ms": 1785101910001, + "signal_ask": "0.76" + }, + "slug": "btc-updown-5m-1785101700", + "terminal_outcome_prices": [ + "0", + "1" + ], + "terminal_updated_at": "2026-07-26T21:43:14.819851Z" + }, + { + "arrival": { + "best_ask": "0.98", + "book_timestamp_ms": 1785102211206, + "request_completed_ts_ms": 1785102211265, + "request_duration_ms": 265, + "request_started_ts_ms": 1785102211000 + }, + "condition_id": "0x1dcecaa2f4ee994fd4ab1b9f309bb18a56f24a5f9c84b82c290a7489d9cfd27d", + "decision": "no_fill_ask_above_limit", + "market_open_epoch_seconds": 1785102000, + "official_outcome": "Down", + "pnl_total": null, + "resolution_payload_sha256": "8693fbc94d74fbd218e18c3a8d65961efd2e997223509135b3b97395af6e89ca", + "resolution_polls": 31, + "selected_side": "Up", + "signal": { + "book_timestamp_ms": 1785102210112, + "request_completed_ts_ms": 1785102210170, + "request_duration_ms": 170, + "request_started_ts_ms": 1785102210000, + "signal_ask": "0.95" + }, + "slug": "btc-updown-5m-1785102000", + "terminal_outcome_prices": [ + "0", + "1" + ], + "terminal_updated_at": "2026-07-26T21:46:24.754458Z" + } +] From 2e90a276ce5686bee9449fb81b02a6e7bc0e185f Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 16:53:37 -0500 Subject: [PATCH 163/177] research: add block verification evidence --- .../prospective-block-v1/VERIFICATION.json | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v1/VERIFICATION.json diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v1/VERIFICATION.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v1/VERIFICATION.json new file mode 100644 index 00000000..291f22da --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v1/VERIFICATION.json @@ -0,0 +1,47 @@ +{ + "artifact": { + "artifact_id": 8637433570, + "artifact_name": "btc-only-v3-prospective-block-v1", + "artifact_zip_sha256": "222d8261ea92dfdfcd3fbb9ccb12dbf6948d03f6bed1e7558f3d9e4bdc6a5580", + "deterministic_tar_round_trip": "passed", + "internal_sha256sums": "passed", + "tar_sidecar": "passed" + }, + "candidate_policy_sha256": "82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba", + "capture_policy_sha256": "888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc", + "integrity": { + "lifecycle_chain": "passed", + "lifecycle_event_counts": { + "arrival": 1, + "discovered": 3, + "resolution": 3, + "signal": 3 + }, + "lifecycle_head_sha256": "1f6f21cf25c1acd11ea5b06e4bd3dec617d69d84c8b25fb5471be22fc9430c11", + "lifecycle_records": 10, + "raw_body_hashes": "passed", + "raw_evidence_chain": "passed", + "raw_evidence_head_sha256": "9b43483f4a310d1ad84a3b6f5a029dc772f2181cb06dce08db476aab00dc2cf0", + "raw_evidence_purpose_counts": { + "arrival_book": 1, + "market_discovery": 3, + "official_resolution": 37, + "signal_books": 3 + }, + "raw_evidence_records": 44 + }, + "run": { + "github_actions_run_id": 30221188931, + "github_head_sha": "be327b99530d9a5b1142e0c9896ae01716d4b070", + "runtime_source_sha256": "81a13d2aeb3df3c2088b8aa99272b10e217a392eabc839a01a8f2b71f570ecbb" + }, + "safety": { + "authenticated_requests": 0, + "credentials_used": 0, + "historical_admission_credit": 0, + "live_submission": "physically_absent", + "order_submissions": 0, + "policy_changes": 0 + }, + "status": "valid_complete_prospective_block_not_admission" +} From a7b38c6e18643f2f4a6cd0db6f7fae609cf79ea7 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 16:53:55 -0500 Subject: [PATCH 164/177] research: update BTC-only prospective admission ledger --- .../ADMISSION_LEDGER.json | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/ADMISSION_LEDGER.json diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/ADMISSION_LEDGER.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/ADMISSION_LEDGER.json new file mode 100644 index 00000000..78223b49 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/ADMISSION_LEDGER.json @@ -0,0 +1,46 @@ +{ + "candidate": "late_favorite_btc_only_v3_prospective_shadow", + "candidate_policy_sha256": "82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba", + "capture_policy_sha256": "888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc", + "current": { + "complete_untouched_weekly_blocks": 0, + "hypothetical_fok_fills": 0, + "official_outcome_coverage": { + "denominator": 4, + "numerator": 4 + }, + "prospective_markets_observed": 4, + "prospective_pnl_total": "0.00", + "unresolved_states": 0 + }, + "gates": { + "maximum_top_two_day_pnl_concentration": 0.4, + "maximum_unresolved_states": 0, + "minimum_complete_untouched_weekly_blocks": 4, + "minimum_hypothetical_fok_fills": 100, + "minimum_prospective_markets_observed": 500, + "official_outcome_coverage": 1.0, + "positive_day_block_bootstrap_lower_bound": true, + "positive_total_pnl_after_fees": true + }, + "observations": [ + { + "artifact_zip_sha256": "8d00c88d6046806a2abc07f43da8b2f0ab08449890ee8a7f9e1f7071b92c4e21", + "checkpoint": "prospective-run-v1", + "hypothetical_fok_fills": 0, + "markets": 1, + "pnl_total": "0.00", + "runtime_source_sha256": "ce477bf97589997899c91ad22af9025de521520f8ee7aa46a21b50a313958bd7" + }, + { + "artifact_zip_sha256": "222d8261ea92dfdfcd3fbb9ccb12dbf6948d03f6bed1e7558f3d9e4bdc6a5580", + "checkpoint": "prospective-block-v1", + "hypothetical_fok_fills": 0, + "markets": 3, + "pnl_total": "0.00", + "runtime_source_sha256": "81a13d2aeb3df3c2088b8aa99272b10e217a392eabc839a01a8f2b71f570ecbb" + } + ], + "status": "prospective_shadow_only_not_admitted", + "version": 1 +} From 7823ac1bce91644e40114eed17199536e099d68d Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 16:54:47 -0500 Subject: [PATCH 165/177] docs: record first complete BTC prospective block --- .../digital-surface-live-readiness/STATUS.md | 92 +++++++++++-------- 1 file changed, 54 insertions(+), 38 deletions(-) diff --git a/ci_payload/digital-surface-live-readiness/STATUS.md b/ci_payload/digital-surface-live-readiness/STATUS.md index d81012af..9614f75c 100644 --- a/ci_payload/digital-surface-live-readiness/STATUS.md +++ b/ci_payload/digital-surface-live-readiness/STATUS.md @@ -7,6 +7,7 @@ - Draft PR: #59 - Official late-favorite decision commit: `cedd659e0b573d8479932c428cd2eaf472c6bf8f` - First BTC-only prospective checkpoint merge: `f76097ef513a2e70881e7624cd8d9a60d12a642b` +- Bounded block-runner merge: `e0b4917c2c7895568d7acb0ca53d4a6ccfcfdf1c` - Real-money execution: **physically disabled** ## Historical research verdicts @@ -47,51 +48,65 @@ BTC-only is a **marginal post-hoc lead**, not a historical pass. It has zero inh - effective capture cutoff: `2026-07-26T19:35:21Z` - only markets opening strictly after the effective capture cutoff may count. -### Observer, collector, resolver, and CLI +### Observer and runner verification -The merged observer implements: +The observer implements market-open-anchored +210s/+211s capture, frozen timing/freshness gates, Decimal full-depth five-share FOK evaluation, fail-closed decision states, append-only lifecycle and raw-evidence chains, public Gamma/CLOB collection, cache-busted terminal resolution, and official fill/no-fill closure. -1. market-open-anchored signal at +210 seconds and arrival at +211 seconds; -2. frozen request-start, duration, book-age, and future-skew limits; -3. Decimal full-depth five-share FOK VWAP and per-level fee accounting; -4. adverse-move, price-limit, depth, stale, timeout, token, and lifecycle fail-closed decisions; -5. append-only candidate/capture/source-bound lifecycle records; -6. content-addressed raw public evidence and a separate manifest hash chain; -7. public Gamma discovery and cache-busted terminal resolution; -8. public CLOB `POST /books` signal and arrival snapshots; -9. official fill P&L and terminal no-fill closure; -10. one-shot `capture` and `resolve` command boundaries. - -Current observer verification: +Observer checkpoint: - 25/25 behavior contracts passed; - workflow run: `30220717065`; - artifact ID: `8637116192`; - artifact ZIP SHA-256: `172ef7213e7860a8bee00db5e0a79d66495d31199cde85d39dcfef201f14de86`; -- combined frozen source SHA-256: `ce477bf97589997899c91ad22af9025de521520f8ee7aa46a21b50a313958bd7`; -- resolver source SHA-256: `90c417be653f5f0fdf22dfc7d4e20168869594a021fc3229d87b7f203bc9dc77`. +- combined frozen source SHA-256: `ce477bf97589997899c91ad22af9025de521520f8ee7aa46a21b50a313958bd7`. -### First complete prospective observation +The bounded runner adds `collect-block`: 1–12 consecutive windows are captured before any resolution request, then the condition queue is resolved with bounded cache-busted retries. Unresolved state is preserved in a durable summary. -Checkpoint: +Runner checkpoint: -- `ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-run-v1/CHECKPOINT.md` -- workflow run: `30220121542`; -- evidence artifact ID: `8637040765`; -- evidence ZIP SHA-256: `8d00c88d6046806a2abc07f43da8b2f0ab08449890ee8a7f9e1f7071b92c4e21`. +- 11/11 contracts passed; +- workflow run: `30221103255`; +- artifact ID: `8637227551`; +- artifact ZIP SHA-256: `fbc509ffa17046b885f25ef13b982f08c4f706bf4d65a73a179c66f0f1fb8f3b`; +- runner SHA-256: `48b45fb823b6fd9f6edf469d34a2c3a85c767c0bf4a456b919127dfeb39dcb8b`; +- combined runtime source SHA-256: `81a13d2aeb3df3c2088b8aa99272b10e217a392eabc839a01a8f2b71f570ecbb`. -Result: +### Prospective observation v1 +- checkpoint: `ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-run-v1/CHECKPOINT.md` - market: `btc-updown-5m-1785099600`, 2026-07-26 21:00–21:05 UTC; -- selected side: Up; -- signal ask: $0.98; -- arrival best ask: $0.99; -- frozen decision: `no_fill_ask_above_limit`; +- Up signal at $0.98; arrival ask $0.99; +- decision: `no_fill_ask_above_limit`; - official outcome: Up; -- hypothetical fill: no; -- P&L: not applicable. +- fill/P&L: none; +- artifact ZIP SHA-256: `8d00c88d6046806a2abc07f43da8b2f0ab08449890ee8a7f9e1f7071b92c4e21`. + +### Prospective block v1 + +Checkpoint: + +- `ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v1/CHECKPOINT.md` +- workflow run: `30221188931`; +- artifact ID: `8637433570`; +- artifact ZIP SHA-256: `222d8261ea92dfdfcd3fbb9ccb12dbf6948d03f6bed1e7558f3d9e4bdc6a5580`. + +Three consecutive markets were collected from 21:30–21:45 UTC: + +1. Up at $0.66: below threshold; official Up. +2. Down at $0.76: below threshold; official Down. +3. Up at $0.95: arrival ask $0.98, so no fill above the frozen limit; official Down. + +Block result: + +- markets: 3; +- official outcomes: 3/3; +- hypothetical fills: 0; +- P&L: $0.00; +- unresolved conditions: 0; +- lifecycle records: 10, chain verified; +- raw evidence records: 44, chain and bodies verified. -The complete lifecycle and raw-evidence chains were independently verified. Twenty non-terminal Gamma payloads were followed by one terminal `[1,0]` payload. The one-cent adverse price move was correctly rejected rather than chased. +The third decision avoided a loss: the observer refused to chase Up from 95 cents to 98 cents, and the market resolved Down. ## Safety and admission boundary @@ -100,17 +115,18 @@ The complete lifecycle and raw-evidence chains were independently verified. Twen - order submissions: **0**; - live submission: **physically absent**; - historical admission credit: **0**; -- prospective markets observed: **1 / 500 minimum**; +- prospective markets observed: **4 / 500 minimum**; +- official outcome coverage: **4 / 4**; - hypothetical FOK fills: **0 / 100 minimum**; +- prospective P&L: **$0.00**; - complete untouched weekly blocks: **0 / 4 minimum**. -No historical result is admissible. This single no-fill observation is valid prospective evidence but cannot support admission. Even a future admissible shadow report cannot enable live trading; any order submitter requires a separate reviewed release and explicit operator action. +No historical result is admissible. These four markets are valid prospective evidence but cannot support admission. Even a future admissible shadow report cannot enable live trading; any order submitter requires a separate reviewed release and explicit operator action. ## Next work -1. continue credential-free collection of every eligible BTC five-minute market, including no-signal and no-fill cases; -2. preserve append-only lifecycle and raw evidence with authoritative terminal outcomes; -3. make no threshold or policy changes during the prospective block; -4. accumulate at least 500 observed markets, 100 hypothetical FOK fills, and four untouched weekly blocks; -5. require measured fill evidence, acceptable concentration, zero unresolved states, and a positive lower confidence bound after fees and execution costs; -6. keep real-money execution physically disabled. +1. continue bounded capture-first blocks without modifying candidate or capture thresholds; +2. preserve every no-signal, no-fill, fill, resolution, and transport-failure record; +3. accumulate at least 500 observed markets, 100 hypothetical fills, and four untouched weekly blocks; +4. require measured execution evidence, acceptable concentration, zero unresolved states, and a positive lower confidence bound after fees and costs; +5. keep real-money execution physically disabled. From 625d1c1963cdc076adfc8bec1f9ea90819738a3b Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 19:04:25 -0500 Subject: [PATCH 166/177] ci: add first BTC-only v3 prospective observation --- .../btc-only-v3-first-prospective.yml | 201 ++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 .github/workflows/btc-only-v3-first-prospective.yml diff --git a/.github/workflows/btc-only-v3-first-prospective.yml b/.github/workflows/btc-only-v3-first-prospective.yml new file mode 100644 index 00000000..7dbb49e2 --- /dev/null +++ b/.github/workflows/btc-only-v3-first-prospective.yml @@ -0,0 +1,201 @@ +name: BTC-Only v3 First Prospective Observation + +on: + push: + branches: [btc-only-v3-first-prospective-20260727] + paths: + - ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/FIRST_PROSPECTIVE_TRIGGER.txt + workflow_dispatch: + +concurrency: + group: btc-only-v3-first-prospective + cancel-in-progress: false + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + ROOT: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 + OUT: btc-only-v3-first-prospective + +jobs: + capture-and-resolve: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install pinned public client + run: python -m pip install requests==2.32.4 + + - name: Verify observer source and frozen policy bytes + shell: bash + run: | + set -euo pipefail + export PYTHONPATH="$ROOT/source:$ROOT/collector" + python - <<'PY' + import hashlib + from pathlib import Path + from capture_policy import load_capture_policy + from late_favorite_v3 import load_frozen_policy + + root = Path('ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3') + source_paths = [ + root / 'source/late_favorite_v3.py', + root / 'source/capture_policy.py', + root / 'collector/public_collector.py', + root / 'collector/resolution.py', + ] + combined = hashlib.sha256() + for path in source_paths: + raw = path.read_bytes() + combined.update(path.name.encode() + b'\0' + raw) + source_sha = combined.hexdigest() + trading = load_frozen_policy(root / 'CANDIDATE_SPEC.json', root / 'FREEZE_MANIFEST.json', source_sha256=source_sha) + capture = load_capture_policy(root / 'CAPTURE_POLICY.json', root / 'CAPTURE_FREEZE_MANIFEST.json') + assert trading.policy_sha256 == '82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba' + assert capture.capture_policy_sha256 == '888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc' + assert source_sha == 'f2e086bb5e74f0343a4ea29f46ea2bf8e23062952b456303e204c6b5469b6126' + PY + + - name: Capture next eligible post-cutoff BTC market + shell: bash + run: | + set -euo pipefail + export PYTHONPATH="$ROOT/source:$ROOT/collector" + python "$ROOT/collector/cli.py" --output-dir "$OUT" capture | tee capture.stdout.json + + - name: Derive immutable market identity + id: market + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import json + import os + from pathlib import Path + + payload = json.loads(Path('btc-only-v3-first-prospective/last_capture.json').read_text()) + condition = str(payload['condition_id']) + opening = int(payload['market_open_epoch_seconds']) + assert opening > 1785094521 + assert condition + with open(os.environ['GITHUB_OUTPUT'], 'a', encoding='utf-8') as handle: + handle.write(f'condition_id={condition}\n') + handle.write(f'opening={opening}\n') + print(json.dumps({'condition_id': condition, 'opening': opening, 'capture_decision': payload['decision']}, indent=2)) + PY + + - name: Wait for terminal resolution window + shell: bash + env: + OPENING: ${{ steps.market.outputs.opening }} + run: | + set -euo pipefail + target=$((OPENING + 330)) + now=$(date +%s) + delay=$((target - now)) + if [ "$delay" -gt 0 ]; then + sleep "$delay" + fi + + - name: Resolve from authoritative Gamma terminal prices + shell: bash + env: + CONDITION_ID: ${{ steps.market.outputs.condition_id }} + run: | + set -euo pipefail + export PYTHONPATH="$ROOT/source:$ROOT/collector" + success=0 + for attempt in $(seq 1 40); do + if python "$ROOT/collector/cli.py" --output-dir "$OUT" resolve --condition-id "$CONDITION_ID" | tee resolve.stdout.json; then + success=1 + break + fi + echo "resolution not terminal on attempt $attempt" + sleep 15 + done + test "$success" -eq 1 + + - name: Verify complete append-only evidence + shell: bash + run: | + set -euo pipefail + export PYTHONPATH="$ROOT/source:$ROOT/collector" + python - <<'PY' + import hashlib + import json + from pathlib import Path + from capture_policy import CaptureBoundLifecycleLedger, load_capture_policy + from late_favorite_v3 import load_frozen_policy + from public_collector import RawEvidenceStore + + root = Path('ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3') + out = Path('btc-only-v3-first-prospective') + source_paths = [root / 'source/late_favorite_v3.py', root / 'source/capture_policy.py', root / 'collector/public_collector.py', root / 'collector/resolution.py'] + combined = hashlib.sha256() + for path in source_paths: + raw = path.read_bytes() + combined.update(path.name.encode() + b'\0' + raw) + trading = load_frozen_policy(root / 'CANDIDATE_SPEC.json', root / 'FREEZE_MANIFEST.json', source_sha256=combined.hexdigest()) + capture = load_capture_policy(root / 'CAPTURE_POLICY.json', root / 'CAPTURE_FREEZE_MANIFEST.json') + ledger = CaptureBoundLifecycleLedger(out / 'lifecycle.jsonl', trading, capture) + evidence = RawEvidenceStore(out / 'raw') + assert ledger.records[-1]['event_type'] == 'resolution' + conditions = {row['condition_id'] for row in ledger.records} + assert len(conditions) == 1 + assert all(int(row['market_open_epoch_seconds']) > capture.valid_market_open_after_epoch_seconds for row in ledger.records) + allowed = { + ('GET', 'gamma-api.polymarket.com'), + ('POST', 'clob.polymarket.com'), + } + from urllib.parse import urlparse + for row in evidence.records: + parsed = urlparse(row['url']) + assert (row['method'], parsed.netloc) in allowed + assert parsed.path.startswith('/markets/slug/') or parsed.path == '/books' + resolution = json.loads((out / 'last_resolution.json').read_text()) + summary = { + 'condition_id': next(iter(conditions)), + 'market_open_epoch_seconds': ledger.records[0]['market_open_epoch_seconds'], + 'capture_decision': json.loads((out / 'last_capture.json').read_text())['decision'], + 'resolution_decision': resolution['decision'], + 'official_outcome': resolution['official_outcome'], + 'official_won': resolution.get('official_won'), + 'pnl_total': resolution.get('pnl_total'), + 'lifecycle_records': len(ledger.records), + 'raw_responses': len(evidence.records), + 'lifecycle_head_sha256': ledger.head_hash, + 'evidence_head_sha256': evidence.head_hash, + 'candidate_policy_sha256': trading.policy_sha256, + 'capture_policy_sha256': capture.capture_policy_sha256, + 'historical_admission_credit': 0, + 'live_submission': 'physically_absent', + } + (out / 'PROSPECTIVE_SUMMARY.json').write_text(json.dumps(summary, indent=2, sort_keys=True) + '\n') + print(json.dumps(summary, indent=2, sort_keys=True)) + PY + + - name: Build deterministic prospective artifact + shell: bash + run: | + set -euo pipefail + (cd "$OUT" && find . -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum > SHA256SUMS) + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf btc-only-v3-first-prospective-v1.tar.gz "$OUT" + sha256sum btc-only-v3-first-prospective-v1.tar.gz > btc-only-v3-first-prospective-v1.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: btc-only-v3-first-prospective-v1 + path: | + btc-only-v3-first-prospective/ + btc-only-v3-first-prospective-v1.tar.gz + btc-only-v3-first-prospective-v1.tar.gz.sha256 + compression-level: 0 + retention-days: 30 From e49bd963e6db34da69c3673850c2b1f86bd9a07b Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 19:06:14 -0500 Subject: [PATCH 167/177] ci: expose first prospective run on draft PR --- .github/workflows/btc-only-v3-first-prospective.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/btc-only-v3-first-prospective.yml b/.github/workflows/btc-only-v3-first-prospective.yml index 7dbb49e2..feb001bd 100644 --- a/.github/workflows/btc-only-v3-first-prospective.yml +++ b/.github/workflows/btc-only-v3-first-prospective.yml @@ -5,6 +5,10 @@ on: branches: [btc-only-v3-first-prospective-20260727] paths: - ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/FIRST_PROSPECTIVE_TRIGGER.txt + pull_request: + branches: [digital-surface-live-readiness-20260725] + paths: + - ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/FIRST_PROSPECTIVE_TRIGGER.txt workflow_dispatch: concurrency: From 81a7a4d846f47f38a356808fa906aeffe3c7639f Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 19:10:55 -0500 Subject: [PATCH 168/177] ci: add BTC-only prospective block v2 run --- .../btc-only-v3-prospective-block-v2.yml | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 .github/workflows/btc-only-v3-prospective-block-v2.yml diff --git a/.github/workflows/btc-only-v3-prospective-block-v2.yml b/.github/workflows/btc-only-v3-prospective-block-v2.yml new file mode 100644 index 00000000..0458ed66 --- /dev/null +++ b/.github/workflows/btc-only-v3-prospective-block-v2.yml @@ -0,0 +1,146 @@ +name: BTC-Only v3 Prospective Block v2 + +on: + pull_request: + branches: [digital-surface-live-readiness-20260725] + paths: + - ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/BLOCK_V2_TRIGGER.txt + workflow_dispatch: + +concurrency: + group: btc-only-v3-prospective-block-v2 + cancel-in-progress: false + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + ROOT: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 + STATE: btc-only-v3-prospective-block-v2 + +jobs: + capture-resolve-verify: + runs-on: ubuntu-latest + timeout-minutes: 35 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install pinned public and test clients + run: python -m pip install requests==2.32.4 pytest==8.3.5 + + - name: Verify committed runner and frozen boundaries + shell: bash + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner + run: | + set -euo pipefail + python -m py_compile "$ROOT/runner/observer_cli.py" "$ROOT/runner/tests/test_observer_block.py" + pytest -q "$ROOT/runner/tests/test_observer_block.py" + python - <<'PY' + from pathlib import Path + from capture_policy import load_capture_policy + from late_favorite_v3 import load_frozen_policy + from observer_cli import compute_combined_source_sha256 + + root = Path('ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3') + source_sha = compute_combined_source_sha256(root) + trading = load_frozen_policy(root / 'CANDIDATE_SPEC.json', root / 'FREEZE_MANIFEST.json', source_sha256=source_sha) + capture = load_capture_policy(root / 'CAPTURE_POLICY.json', root / 'CAPTURE_FREEZE_MANIFEST.json') + assert trading.policy_sha256 == '82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba' + assert capture.capture_policy_sha256 == '888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc' + assert capture.valid_market_open_after_epoch_seconds == 1785094521 + print(source_sha) + PY + + - name: Capture three consecutive markets, then resolve the queue + shell: bash + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner + run: | + set -euo pipefail + python "$ROOT/runner/observer_cli.py" \ + --project-root "$ROOT" \ + --state-dir "$STATE" \ + collect-block \ + --markets 3 \ + --max-resolution-attempts 120 \ + --retry-delay-ms 10000 | tee block-v2-output.json + + - name: Verify complete block evidence and write summary + shell: bash + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner + run: | + set -euo pipefail + python - <<'PY' + import json + from pathlib import Path + from capture_policy import CaptureBoundLifecycleLedger, load_capture_policy + from late_favorite_v3 import load_frozen_policy + from observer_cli import compute_combined_source_sha256 + from public_collector import RawEvidenceStore + + root = Path('ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3') + state = Path('btc-only-v3-prospective-block-v2') + source_sha = compute_combined_source_sha256(root) + trading = load_frozen_policy(root / 'CANDIDATE_SPEC.json', root / 'FREEZE_MANIFEST.json', source_sha256=source_sha) + capture = load_capture_policy(root / 'CAPTURE_POLICY.json', root / 'CAPTURE_FREEZE_MANIFEST.json') + ledger = CaptureBoundLifecycleLedger(state / 'lifecycle.jsonl', trading, capture) + evidence = RawEvidenceStore(state / 'raw_evidence') + block_paths = sorted((state / 'summaries').glob('block-*.json')) + assert len(block_paths) == 1, block_paths + block = json.loads(block_paths[0].read_text()) + assert block['status'] == 'complete' + assert block['markets_requested'] == 3 + assert block['markets_collected'] == 3 + assert block['conditions_resolved'] == 3 + assert block['unresolved_condition_ids'] == [] + assert len(block['condition_ids']) == 3 + assert len(set(block['condition_ids'])) == 3 + assert all(int(row['market_open_epoch_seconds']) > capture.valid_market_open_after_epoch_seconds for row in ledger.records) + assert sum(1 for row in ledger.records if row['event_type'] == 'discovered') == 3 + assert sum(1 for row in ledger.records if row['event_type'] == 'signal') == 3 + assert sum(1 for row in ledger.records if row['event_type'] == 'resolution') == 3 + summary = { + **block, + 'candidate_policy_sha256': trading.policy_sha256, + 'capture_policy_sha256': capture.capture_policy_sha256, + 'runtime_source_sha256': source_sha, + 'lifecycle_head_sha256': ledger.head_hash, + 'raw_evidence_head_sha256': evidence.head_hash, + 'lifecycle_records': len(ledger.records), + 'raw_evidence_records': len(evidence.records), + 'historical_admission_credit': 0, + 'credentials_used': 0, + 'authenticated_requests': 0, + 'order_submissions': 0, + 'live_submission': 'physically_absent', + } + (state / 'BLOCK_V2_SUMMARY.json').write_text(json.dumps(summary, indent=2, sort_keys=True) + '\n') + print(json.dumps(summary, indent=2, sort_keys=True)) + PY + + - name: Build deterministic block artifact + shell: bash + run: | + set -euo pipefail + (cd "$STATE" && find . -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum > SHA256SUMS) + (cd "$STATE" && sha256sum -c SHA256SUMS) + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf btc-only-v3-prospective-block-v2.tar.gz "$STATE" + sha256sum btc-only-v3-prospective-block-v2.tar.gz > btc-only-v3-prospective-block-v2.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: btc-only-v3-prospective-block-v2 + path: | + btc-only-v3-prospective-block-v2/ + btc-only-v3-prospective-block-v2.tar.gz + btc-only-v3-prospective-block-v2.tar.gz.sha256 + compression-level: 0 + retention-days: 30 From af1d6dfc7d9d301534050e0e3de9c38b9d6cf7f5 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 19:38:55 -0500 Subject: [PATCH 169/177] research: freeze BTC-only prospective block v2 --- .../digital-surface-live-readiness/STATUS.md | 41 +++--- .../ADMISSION_LEDGER.json | 20 ++- .../prospective-block-v2/BLOCK_SUMMARY.json | 50 +++++++ .../prospective-block-v2/CHECKPOINT.md | 130 ++++++++++++++++++ .../prospective-block-v2/MARKETS.json | 117 ++++++++++++++++ .../prospective-block-v2/VERIFICATION.json | 31 +++++ 6 files changed, 365 insertions(+), 24 deletions(-) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v2/BLOCK_SUMMARY.json create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v2/CHECKPOINT.md create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v2/MARKETS.json create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v2/VERIFICATION.json diff --git a/ci_payload/digital-surface-live-readiness/STATUS.md b/ci_payload/digital-surface-live-readiness/STATUS.md index 9614f75c..6a2cb5af 100644 --- a/ci_payload/digital-surface-live-readiness/STATUS.md +++ b/ci_payload/digital-surface-live-readiness/STATUS.md @@ -83,30 +83,34 @@ Runner checkpoint: ### Prospective block v1 -Checkpoint: - -- `ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v1/CHECKPOINT.md` +- checkpoint: `ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v1/CHECKPOINT.md` - workflow run: `30221188931`; - artifact ID: `8637433570`; -- artifact ZIP SHA-256: `222d8261ea92dfdfcd3fbb9ccb12dbf6948d03f6bed1e7558f3d9e4bdc6a5580`. - -Three consecutive markets were collected from 21:30–21:45 UTC: +- artifact ZIP SHA-256: `222d8261ea92dfdfcd3fbb9ccb12dbf6948d03f6bed1e7558f3d9e4bdc6a5580`; +- markets: 3; +- official outcomes: 3/3; +- hypothetical fills: 0; +- P&L: $0.00; +- unresolved conditions: 0. -1. Up at $0.66: below threshold; official Up. -2. Down at $0.76: below threshold; official Down. -3. Up at $0.95: arrival ask $0.98, so no fill above the frozen limit; official Down. +The block contained two below-threshold no-signals and one no-fill above the frozen FOK limit. The no-fill avoided a loss when the market later resolved against the selected favorite. -Block result: +### Prospective block v2 +- checkpoint: `ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v2/CHECKPOINT.md` +- workflow run: `30226869672`; +- artifact ID: `8638983600`; +- artifact ZIP SHA-256: `69ee1bb111f8aec0b52f982f305136a5e835ca2b2e50f4b66a129d3370b5ac58`; +- deterministic TAR SHA-256: `4fc0e0b68e7261899e133e1ba9af05939a72303d26609f73391967b5251c73cc`; - markets: 3; - official outcomes: 3/3; -- hypothetical fills: 0; -- P&L: $0.00; +- hypothetical fills: 1; +- P&L: **+$0.233375**; - unresolved conditions: 0; - lifecycle records: 10, chain verified; - raw evidence records: 44, chain and bodies verified. -The third decision avoided a loss: the observer refused to chase Up from 95 cents to 98 cents, and the market resolved Down. +The filled market selected Down at a 95-cent signal ask, executed five shares at 95 cents one second later, paid $0.003325/share in fees, officially resolved Down, and earned $0.046675/share or $0.233375 total. ## Safety and admission boundary @@ -115,13 +119,14 @@ The third decision avoided a loss: the observer refused to chase Up from 95 cent - order submissions: **0**; - live submission: **physically absent**; - historical admission credit: **0**; -- prospective markets observed: **4 / 500 minimum**; -- official outcome coverage: **4 / 4**; -- hypothetical FOK fills: **0 / 100 minimum**; -- prospective P&L: **$0.00**; +- prospective markets observed: **7 / 500 minimum**; +- official outcome coverage: **7 / 7**; +- hypothetical FOK fills: **1 / 100 minimum**; +- prospective P&L: **+$0.233375**; +- unresolved states: **0**; - complete untouched weekly blocks: **0 / 4 minimum**. -No historical result is admissible. These four markets are valid prospective evidence but cannot support admission. Even a future admissible shadow report cannot enable live trading; any order submitter requires a separate reviewed release and explicit operator action. +One winning fill is not statistical validation. These seven markets are valid prospective evidence but cannot support admission, a confidence bound, a concentration pass, or live release. Even a future admissible shadow report cannot enable trading; any order submitter requires a separate reviewed release and explicit operator action. ## Next work diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/ADMISSION_LEDGER.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/ADMISSION_LEDGER.json index 78223b49..1afaf48f 100644 --- a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/ADMISSION_LEDGER.json +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/ADMISSION_LEDGER.json @@ -4,13 +4,13 @@ "capture_policy_sha256": "888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc", "current": { "complete_untouched_weekly_blocks": 0, - "hypothetical_fok_fills": 0, + "hypothetical_fok_fills": 1, "official_outcome_coverage": { - "denominator": 4, - "numerator": 4 + "denominator": 7, + "numerator": 7 }, - "prospective_markets_observed": 4, - "prospective_pnl_total": "0.00", + "prospective_markets_observed": 7, + "prospective_pnl_total": "0.233375", "unresolved_states": 0 }, "gates": { @@ -39,8 +39,16 @@ "markets": 3, "pnl_total": "0.00", "runtime_source_sha256": "81a13d2aeb3df3c2088b8aa99272b10e217a392eabc839a01a8f2b71f570ecbb" + }, + { + "artifact_zip_sha256": "69ee1bb111f8aec0b52f982f305136a5e835ca2b2e50f4b66a129d3370b5ac58", + "checkpoint": "prospective-block-v2", + "hypothetical_fok_fills": 1, + "markets": 3, + "pnl_total": "0.233375", + "runtime_source_sha256": "81a13d2aeb3df3c2088b8aa99272b10e217a392eabc839a01a8f2b71f570ecbb" } ], "status": "prospective_shadow_only_not_admitted", - "version": 1 + "version": 2 } diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v2/BLOCK_SUMMARY.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v2/BLOCK_SUMMARY.json new file mode 100644 index 00000000..8b06c017 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v2/BLOCK_SUMMARY.json @@ -0,0 +1,50 @@ +{ + "authenticated_requests": 0, + "candidate_policy_sha256": "82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba", + "capture_decision_counts": { + "hypothetical_fok_fill": 1, + "no_signal_below_threshold": 2 + }, + "capture_policy_sha256": "888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc", + "command": "collect-block", + "condition_ids": [ + "0x32d793352dbae3fd435a200c1cbcb9cea3dfb6c03030c891ae21e562ee7db11b", + "0x24846fbe7d9babc0f24d8a470322b84e0bef0f689b4a1f4e69e72ff2c2fe3233", + "0xc5a4c25812fc718b99cb34694ae389abeacf4d03d158720be4a8a53358e0cef2" + ], + "conditions_resolved": 3, + "credentials_used": 0, + "first_market_open_epoch_seconds": 1785111000, + "historical_admission_credit": 0, + "hypothetical_fok_fills": 1, + "last_market_open_epoch_seconds": 1785111600, + "last_resolution_errors": {}, + "lifecycle_head_sha256": "a04aac52de49a03fca237b888cbb284788a6dd5a53560147fc6b561489c9c87b", + "lifecycle_records": 10, + "live_submission": "physically_absent", + "market_open_epoch_seconds": [ + 1785111000, + 1785111300, + 1785111600 + ], + "markets_collected": 3, + "markets_requested": 3, + "order_submissions": 0, + "policy_changes": 0, + "prospective_pnl_total": "0.233375", + "raw_evidence_head_sha256": "b44fc443b8c80537ae9f40243034194d45633e34f3d46694f0b3a895b9928606", + "raw_evidence_records": 44, + "resolution_attempts_by_condition": { + "0x24846fbe7d9babc0f24d8a470322b84e0bef0f689b4a1f4e69e72ff2c2fe3233": 7, + "0x32d793352dbae3fd435a200c1cbcb9cea3dfb6c03030c891ae21e562ee7db11b": 1, + "0xc5a4c25812fc718b99cb34694ae389abeacf4d03d158720be4a8a53358e0cef2": 29 + }, + "resolution_decision_counts": { + "resolved_fill": 1, + "resolved_no_fill": 2 + }, + "runtime_source_sha256": "81a13d2aeb3df3c2088b8aa99272b10e217a392eabc839a01a8f2b71f570ecbb", + "source_sha256": "81a13d2aeb3df3c2088b8aa99272b10e217a392eabc839a01a8f2b71f570ecbb", + "status": "complete", + "unresolved_condition_ids": [] +} diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v2/CHECKPOINT.md b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v2/CHECKPOINT.md new file mode 100644 index 00000000..07475e0b --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v2/CHECKPOINT.md @@ -0,0 +1,130 @@ +# BTC-Only v3 Prospective Block v2 + +## Classification + +**Valid prospective block evidence; not strategy admission.** + +This checkpoint covers three consecutive post-freeze BTC five-minute markets collected under the unchanged `late_favorite_btc_only_v3_prospective_shadow` candidate and capture policies. All three markets were captured before the runner made any resolution request. Every condition later received an authoritative Gamma terminal outcome. + +The block produced the first genuine prospective hypothetical FOK fill: **one winning five-share Down fill and +$0.233375 P&L**. This is a useful observation, but one fill is nowhere near sufficient for admission. + +## Block identity + +- Windows: **2026-07-27 00:10–00:25 UTC** / **2026-07-26 7:10–7:25 PM Jamaica** +- Consecutive openings: `1785111000`, `1785111300`, `1785111600` +- GitHub Actions run: `30226869672` +- Workflow head SHA: `afc14b93c69bed593f4d51ce92fcbdb900d4a145` +- Artifact ID: `8638983600` +- Artifact ZIP SHA-256: `69ee1bb111f8aec0b52f982f305136a5e835ca2b2e50f4b66a129d3370b5ac58` +- Deterministic TAR SHA-256: `4fc0e0b68e7261899e133e1ba9af05939a72303d26609f73391967b5251c73cc` +- Runtime source SHA-256: `81a13d2aeb3df3c2088b8aa99272b10e217a392eabc839a01a8f2b71f570ecbb` +- Candidate policy SHA-256: `82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba` +- Capture policy SHA-256: `888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc` + +## Market results + +| Jamaica window | Selected side | Signal ask | Arrival | Frozen decision | Official outcome | P&L | +|---|---:|---:|---:|---|---:|---:| +| 7:10–7:15 PM | Up | $0.64 | Not requested | `no_signal_below_threshold` | Up | $0.00 | +| 7:15–7:20 PM | Down | $0.95 | $0.95; five shares | `hypothetical_fok_fill` | Down | **+$0.233375** | +| 7:20–7:25 PM | Up | $0.66 | Not requested | `no_signal_below_threshold` | Up | $0.00 | + +## First prospective fill + +The middle market, `btc-updown-5m-1785111300`, qualified with Down as the favorite at a 95-cent executable ask. + +One second later: + +- arrival best ask: **$0.95**; +- displayed executable size at or below the frozen limit: at least **5 shares**; +- execution: **5 shares at $0.95**; +- execution VWAP: **$0.95**; +- taker fee/share: **$0.003325**; +- all-in cost/share: **$0.953325**. + +The market officially resolved Down. Therefore: + +- official win: **yes**; +- P&L/share: **$0.046675**; +- P&L at five shares: **+$0.233375**. + +No threshold or execution rule was changed before or during this market. + +## Capture timing + +### Market 1 — `btc-updown-5m-1785111000` + +- Signal request start: +210,001 ms +- Signal request duration: 161 ms +- Signal book timestamps: +210,102 ms +- Decision: below the frozen 85-cent threshold + +### Market 2 — `btc-updown-5m-1785111300` + +- Signal request start: exactly +210,000 ms +- Signal request duration: 122 ms +- Signal book timestamps: +210,070 ms and +210,067 ms +- Arrival request start: exactly +211,000 ms +- Arrival request duration: 143 ms +- Arrival book timestamp: +211,095 ms +- Decision: hypothetical FOK fill + +### Market 3 — `btc-updown-5m-1785111600` + +- Signal request start: exactly +210,000 ms +- Signal request duration: 129 ms +- Signal book timestamps: +210,080 ms +- Decision: below the frozen 85-cent threshold + +Every request start, request duration, and book timestamp remained within the frozen capture-policy bounds. + +## Authoritative outcomes + +| Market opening | Resolution polls | Official outcome | Terminal prices | Gamma `updatedAt` | Terminal payload SHA-256 | +|---:|---:|---:|---|---|---| +| `1785111000` | 1 | Up | `[1, 0]` | `2026-07-27T00:21:14.688273Z` | `c504beb6519ee8965f84c8a3122c701659672e7fa0ce0882689681325d77130b` | +| `1785111300` | 7 | Down | `[0, 1]` | `2026-07-27T00:23:15.362879Z` | `2863a051c35dd91192df4fc4088313937b328949e2bd0ac909597f2fe722d482` | +| `1785111600` | 29 | Up | `[1, 0]` | `2026-07-27T00:28:59.884201Z` | `a2ad563e7d0f24894088ca5c14206739b8f894041591462ff085b54136ff4ece` | + +Resolution began only after all three capture windows completed. Every cache-busted public Gamma payload was preserved in the raw-evidence chain. + +## Integrity verification + +- Internal `SHA256SUMS`: **passed** +- Deterministic TAR sidecar: **passed** +- ZIP/TAR directory round-trip: **byte-for-byte passed** +- Lifecycle records: **10** + - discovered: 3 + - signal: 3 + - arrival: 1 + - resolution: 3 +- Lifecycle chain: **passed** +- Lifecycle head: `a04aac52de49a03fca237b888cbb284788a6dd5a53560147fc6b561489c9c87b` +- Raw evidence records: **44** +- Raw evidence chain and body hashes: **passed** +- Raw evidence head: `b44fc443b8c80537ae9f40243034194d45633e34f3d46694f0b3a895b9928606` +- Unresolved conditions: **0** +- Credentials used: **0** +- Authenticated requests: **0** +- Order submissions: **0** +- Historical admission credit: **0** +- Live submission: **physically absent** + +## Cumulative admission ledger + +- Prospective markets observed: **7 / 500 minimum** +- Official outcomes available: **7 / 7** +- Hypothetical FOK fills: **1 / 100 minimum** +- Prospective P&L: **+$0.233375** +- Unresolved states: **0** +- Complete untouched weekly blocks: **0 / 4 minimum** + +The positive result is a single observation. It cannot support a confidence bound, concentration assessment, weekly-block pass, or strategy admission. + +## Next valid work + +1. Continue bounded capture-first blocks without modifying candidate or capture thresholds. +2. Preserve every no-signal, no-fill, fill, resolution, and transport-failure record. +3. Accumulate at least 500 observed markets, 100 hypothetical fills, and four untouched weekly blocks. +4. Require zero unresolved states, measured execution evidence, acceptable concentration, and a positive lower confidence bound after fees and costs. +5. Keep real-money order submission physically absent. diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v2/MARKETS.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v2/MARKETS.json new file mode 100644 index 00000000..470cfca8 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v2/MARKETS.json @@ -0,0 +1,117 @@ +[ + { + "arrival": null, + "condition_id": "0x32d793352dbae3fd435a200c1cbcb9cea3dfb6c03030c891ae21e562ee7db11b", + "decision": "no_signal_below_threshold", + "jamaica_window": "2026-07-26 07:10–07:15 PM Jamaica", + "market_open_epoch_seconds": 1785111000, + "official_outcome": "Up", + "official_won": null, + "pnl_per_share": null, + "pnl_total": null, + "resolution_payload_sha256": "c504beb6519ee8965f84c8a3122c701659672e7fa0ce0882689681325d77130b", + "resolution_polls": 1, + "selected_side": "Up", + "signal": { + "book_timestamps_ms": [ + 1785111210102, + 1785111210102 + ], + "request_completed_ts_ms": 1785111210162, + "request_duration_ms": 161, + "request_started_ts_ms": 1785111210001, + "target_ts_ms": 1785111210000 + }, + "signal_ask": "0.64", + "slug": "btc-updown-5m-1785111000", + "terminal_outcome_prices": [ + "1", + "0" + ], + "terminal_updated_at": "2026-07-27T00:21:14.688273Z", + "utc_window": "2026-07-27 00:10–00:15 UTC" + }, + { + "arrival": { + "all_in_cost_per_share": "0.953325", + "best_ask": "0.95", + "book_timestamps_ms": [ + 1785111511095 + ], + "execution_levels": [ + { + "price": "0.95", + "size": "5" + } + ], + "execution_vwap": "0.95", + "fee_per_share": "0.003325", + "filled_shares": "5", + "request_completed_ts_ms": 1785111511143, + "request_duration_ms": 143, + "request_started_ts_ms": 1785111511000, + "target_ts_ms": 1785111511000 + }, + "condition_id": "0x24846fbe7d9babc0f24d8a470322b84e0bef0f689b4a1f4e69e72ff2c2fe3233", + "decision": "hypothetical_fok_fill", + "jamaica_window": "2026-07-26 07:15–07:20 PM Jamaica", + "market_open_epoch_seconds": 1785111300, + "official_outcome": "Down", + "official_won": true, + "pnl_per_share": "0.046675", + "pnl_total": "0.233375", + "resolution_payload_sha256": "2863a051c35dd91192df4fc4088313937b328949e2bd0ac909597f2fe722d482", + "resolution_polls": 7, + "selected_side": "Down", + "signal": { + "book_timestamps_ms": [ + 1785111510070, + 1785111510067 + ], + "request_completed_ts_ms": 1785111510122, + "request_duration_ms": 122, + "request_started_ts_ms": 1785111510000, + "target_ts_ms": 1785111510000 + }, + "signal_ask": "0.95", + "slug": "btc-updown-5m-1785111300", + "terminal_outcome_prices": [ + "0", + "1" + ], + "terminal_updated_at": "2026-07-27T00:23:15.362879Z", + "utc_window": "2026-07-27 00:15–00:20 UTC" + }, + { + "arrival": null, + "condition_id": "0xc5a4c25812fc718b99cb34694ae389abeacf4d03d158720be4a8a53358e0cef2", + "decision": "no_signal_below_threshold", + "jamaica_window": "2026-07-26 07:20–07:25 PM Jamaica", + "market_open_epoch_seconds": 1785111600, + "official_outcome": "Up", + "official_won": null, + "pnl_per_share": null, + "pnl_total": null, + "resolution_payload_sha256": "a2ad563e7d0f24894088ca5c14206739b8f894041591462ff085b54136ff4ece", + "resolution_polls": 29, + "selected_side": "Up", + "signal": { + "book_timestamps_ms": [ + 1785111810080, + 1785111810080 + ], + "request_completed_ts_ms": 1785111810129, + "request_duration_ms": 129, + "request_started_ts_ms": 1785111810000, + "target_ts_ms": 1785111810000 + }, + "signal_ask": "0.66", + "slug": "btc-updown-5m-1785111600", + "terminal_outcome_prices": [ + "1", + "0" + ], + "terminal_updated_at": "2026-07-27T00:28:59.884201Z", + "utc_window": "2026-07-27 00:20–00:25 UTC" + } +] diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v2/VERIFICATION.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v2/VERIFICATION.json new file mode 100644 index 00000000..e0daf68b --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v2/VERIFICATION.json @@ -0,0 +1,31 @@ +{ + "artifact_id": 8638983600, + "artifact_zip_sha256": "69ee1bb111f8aec0b52f982f305136a5e835ca2b2e50f4b66a129d3370b5ac58", + "authenticated_requests": 0, + "candidate_policy_sha256": "82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba", + "capture_policy_sha256": "888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc", + "credentials_used": 0, + "deterministic_tar_sha256": "4fc0e0b68e7261899e133e1ba9af05939a72303d26609f73391967b5251c73cc", + "historical_admission_credit": 0, + "hypothetical_fok_fills": 1, + "internal_sha256sums_verified": true, + "lifecycle_chain_verified": true, + "lifecycle_head_sha256": "a04aac52de49a03fca237b888cbb284788a6dd5a53560147fc6b561489c9c87b", + "lifecycle_records": 10, + "live_submission": "physically_absent", + "markets": 3, + "official_outcomes": { + "denominator": 3, + "numerator": 3 + }, + "order_submissions": 0, + "prospective_pnl_total": "0.233375", + "raw_evidence_chain_and_bodies_verified": true, + "raw_evidence_head_sha256": "b44fc443b8c80537ae9f40243034194d45633e34f3d46694f0b3a895b9928606", + "raw_evidence_records": 44, + "runtime_source_sha256": "81a13d2aeb3df3c2088b8aa99272b10e217a392eabc839a01a8f2b71f570ecbb", + "tar_round_trip_verified": true, + "unresolved_states": 0, + "workflow_head_sha": "afc14b93c69bed593f4d51ce92fcbdb900d4a145", + "workflow_run_id": 30226869672 +} From ff4c24c6b522427f16968e4bbd2db62db05dce35 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 19:42:03 -0500 Subject: [PATCH 170/177] ci: add BTC-only prospective block v3 run --- .../btc-only-v3-prospective-block-v3.yml | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 .github/workflows/btc-only-v3-prospective-block-v3.yml diff --git a/.github/workflows/btc-only-v3-prospective-block-v3.yml b/.github/workflows/btc-only-v3-prospective-block-v3.yml new file mode 100644 index 00000000..a674ca20 --- /dev/null +++ b/.github/workflows/btc-only-v3-prospective-block-v3.yml @@ -0,0 +1,146 @@ +name: BTC-Only v3 Prospective Block v3 + +on: + pull_request: + branches: [digital-surface-live-readiness-20260725] + paths: + - ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/BLOCK_V3_TRIGGER.txt + workflow_dispatch: + +concurrency: + group: btc-only-v3-prospective-block-v3 + cancel-in-progress: false + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + ROOT: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 + STATE: btc-only-v3-prospective-block-v3 + +jobs: + capture-resolve-verify: + runs-on: ubuntu-latest + timeout-minutes: 35 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install pinned public and test clients + run: python -m pip install requests==2.32.4 pytest==8.3.5 + + - name: Verify committed runner and frozen boundaries + shell: bash + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner + run: | + set -euo pipefail + python -m py_compile "$ROOT/runner/observer_cli.py" "$ROOT/runner/tests/test_observer_block.py" + pytest -q "$ROOT/runner/tests/test_observer_block.py" + python - <<'PY' + from pathlib import Path + from capture_policy import load_capture_policy + from late_favorite_v3 import load_frozen_policy + from observer_cli import compute_combined_source_sha256 + + root = Path('ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3') + source_sha = compute_combined_source_sha256(root) + trading = load_frozen_policy(root / 'CANDIDATE_SPEC.json', root / 'FREEZE_MANIFEST.json', source_sha256=source_sha) + capture = load_capture_policy(root / 'CAPTURE_POLICY.json', root / 'CAPTURE_FREEZE_MANIFEST.json') + assert trading.policy_sha256 == '82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba' + assert capture.capture_policy_sha256 == '888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc' + assert capture.valid_market_open_after_epoch_seconds == 1785094521 + print(source_sha) + PY + + - name: Capture three consecutive markets, then resolve the queue + shell: bash + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner + run: | + set -euo pipefail + python "$ROOT/runner/observer_cli.py" \ + --project-root "$ROOT" \ + --state-dir "$STATE" \ + collect-block \ + --markets 3 \ + --max-resolution-attempts 120 \ + --retry-delay-ms 10000 | tee block-v3-output.json + + - name: Verify complete block evidence and write summary + shell: bash + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner + run: | + set -euo pipefail + python - <<'PY' + import json + from pathlib import Path + from capture_policy import CaptureBoundLifecycleLedger, load_capture_policy + from late_favorite_v3 import load_frozen_policy + from observer_cli import compute_combined_source_sha256 + from public_collector import RawEvidenceStore + + root = Path('ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3') + state = Path('btc-only-v3-prospective-block-v3') + source_sha = compute_combined_source_sha256(root) + trading = load_frozen_policy(root / 'CANDIDATE_SPEC.json', root / 'FREEZE_MANIFEST.json', source_sha256=source_sha) + capture = load_capture_policy(root / 'CAPTURE_POLICY.json', root / 'CAPTURE_FREEZE_MANIFEST.json') + ledger = CaptureBoundLifecycleLedger(state / 'lifecycle.jsonl', trading, capture) + evidence = RawEvidenceStore(state / 'raw_evidence') + block_paths = sorted((state / 'summaries').glob('block-*.json')) + assert len(block_paths) == 1, block_paths + block = json.loads(block_paths[0].read_text()) + assert block['status'] == 'complete' + assert block['markets_requested'] == 3 + assert block['markets_collected'] == 3 + assert block['conditions_resolved'] == 3 + assert block['unresolved_condition_ids'] == [] + assert len(block['condition_ids']) == 3 + assert len(set(block['condition_ids'])) == 3 + assert all(int(row['market_open_epoch_seconds']) > capture.valid_market_open_after_epoch_seconds for row in ledger.records) + assert sum(1 for row in ledger.records if row['event_type'] == 'discovered') == 3 + assert sum(1 for row in ledger.records if row['event_type'] == 'signal') == 3 + assert sum(1 for row in ledger.records if row['event_type'] == 'resolution') == 3 + summary = { + **block, + 'candidate_policy_sha256': trading.policy_sha256, + 'capture_policy_sha256': capture.capture_policy_sha256, + 'runtime_source_sha256': source_sha, + 'lifecycle_head_sha256': ledger.head_hash, + 'raw_evidence_head_sha256': evidence.head_hash, + 'lifecycle_records': len(ledger.records), + 'raw_evidence_records': len(evidence.records), + 'historical_admission_credit': 0, + 'credentials_used': 0, + 'authenticated_requests': 0, + 'order_submissions': 0, + 'live_submission': 'physically_absent', + } + (state / 'BLOCK_V3_SUMMARY.json').write_text(json.dumps(summary, indent=2, sort_keys=True) + '\n') + print(json.dumps(summary, indent=2, sort_keys=True)) + PY + + - name: Build deterministic block artifact + shell: bash + run: | + set -euo pipefail + (cd "$STATE" && find . -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum > SHA256SUMS) + (cd "$STATE" && sha256sum -c SHA256SUMS) + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf btc-only-v3-prospective-block-v3.tar.gz "$STATE" + sha256sum btc-only-v3-prospective-block-v3.tar.gz > btc-only-v3-prospective-block-v3.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: btc-only-v3-prospective-block-v3 + path: | + btc-only-v3-prospective-block-v3/ + btc-only-v3-prospective-block-v3.tar.gz + btc-only-v3-prospective-block-v3.tar.gz.sha256 + compression-level: 0 + retention-days: 30 From bff0083acd96962cfa901acdebb8fa184ab9865c Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:04:42 -0500 Subject: [PATCH 171/177] research: add BTC-only block v3 checkpoint --- .../prospective-block-v3/CHECKPOINT.md | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v3/CHECKPOINT.md diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v3/CHECKPOINT.md b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v3/CHECKPOINT.md new file mode 100644 index 00000000..870b626b --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v3/CHECKPOINT.md @@ -0,0 +1,121 @@ +# BTC-Only v3 Prospective Block v3 + +## Classification + +**Valid prospective block evidence; not strategy admission.** + +This checkpoint covers three consecutive post-freeze BTC five-minute markets collected under the unchanged candidate and capture policies. The runner captured all three windows before sending any resolution request, then resolved every condition from authoritative cache-busted Gamma terminal responses. + +The block produced one winning hypothetical FOK fill: **five Down shares at $0.87, +$0.610415**. Cumulatively, the prospective track now has two winning fills, but two observations remain far below every admission threshold. + +## Block identity + +- Windows: **2026-07-27 00:40–00:55 UTC** / **2026-07-26 7:40–7:55 PM Jamaica** +- Consecutive openings: `1785112800`, `1785113100`, `1785113400` +- GitHub Actions run: `30228071397` +- Workflow head SHA: `e5463d3ecb2b150de857ac73bf31e08d3317cd2f` +- Artifact ID: `8639355447` +- Artifact ZIP SHA-256: `b30e9e10de839070ad1ae88f98c952ff2d8d4378ab828bed1533cff0e588cc0e` +- Deterministic TAR SHA-256: `5af345750399cde55f782140c8ba0427b74d39c113ce864e7a045c3f246cc6ee` +- Runtime source SHA-256: `81a13d2aeb3df3c2088b8aa99272b10e217a392eabc839a01a8f2b71f570ecbb` +- Candidate policy SHA-256: `82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba` +- Capture policy SHA-256: `888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc` + +## Market results + +| Jamaica window | Selected side | Signal ask | Arrival | Frozen decision | Official outcome | P&L | +|---|---:|---:|---:|---|---:|---:| +| 7:40–7:45 PM | Up | $0.64 | Not requested | `no_signal_below_threshold` | Up | $0.00 | +| 7:45–7:50 PM | Down | $0.60 | Not requested | `no_signal_below_threshold` | Up | $0.00 | +| 7:50–7:55 PM | Down | $0.88 | $0.87; five shares | `hypothetical_fok_fill` | Down | **+$0.610415** | + +## Filled-market economics + +The final market, `btc-updown-5m-1785113400`, selected Down at an 88-cent signal ask. One second later, the best ask improved to 87 cents—exactly one cent below the signal ask, which remains eligible under the frozen adverse-move rule. + +- signal ask: **$0.88** +- arrival best ask: **$0.87** +- execution: **5 shares at $0.87** +- execution VWAP: **$0.87** +- fee/share: **$0.007917** +- all-in cost/share: **$0.877917** +- official outcome: **Down** +- P&L/share: **$0.122083** +- five-share P&L: **+$0.610415** + +No policy, threshold, size, fee, or execution rule changed. + +## Capture timing + +### Market 1 — `btc-updown-5m-1785112800` + +- Signal request start: exactly +210,000 ms +- Signal request duration: 183 ms +- Signal book timestamps: +210,110 ms +- Decision: below the frozen 85-cent threshold + +### Market 2 — `btc-updown-5m-1785113100` + +- Signal request start: exactly +210,000 ms +- Signal request duration: 187 ms +- Signal book timestamps: +210,110 ms and +210,109 ms +- Decision: below the frozen 85-cent threshold + +### Market 3 — `btc-updown-5m-1785113400` + +- Signal request start: +210,001 ms +- Signal request duration: 198 ms +- Signal book timestamps: +210,122 ms +- Arrival request start: exactly +211,000 ms +- Arrival request duration: 163 ms +- Arrival book timestamp: +211,088 ms +- Decision: hypothetical FOK fill + +Every request and book timestamp remained inside the frozen capture-policy limits. + +## Authoritative outcomes + +| Market opening | Resolution polls | Official outcome | Terminal prices | Gamma `updatedAt` | Terminal payload SHA-256 | +|---:|---:|---:|---|---|---| +| `1785112800` | 1 | Up | `[1, 0]` | `2026-07-27T00:51:14.942925Z` | `51fd814498aae2f743d5b6354eba77461066c333042c8a6614b263df0a85c8aa` | +| `1785113100` | 1 | Up | `[1, 0]` | `2026-07-27T00:51:35.707758Z` | `dbdcd84b4eb322624a45ae22543ed369be91021958779f9a52ca16c45e2d3423` | +| `1785113400` | 35 | Down | `[0, 1]` | `2026-07-27T00:56:26.240184Z` | `c37484c919bf833195bad9c90ecf508eafeda0cfc7af7560f6aff21fb465992a` | + +## Integrity verification + +- Internal `SHA256SUMS`: **passed** +- Deterministic TAR sidecar: **passed** +- ZIP/TAR directory round-trip: **byte-for-byte passed** +- Lifecycle records: **10** + - discovered: 3 + - signal: 3 + - arrival: 1 + - resolution: 3 +- Lifecycle chain: **passed** +- Lifecycle head: `fa080692b88dbfea6233fb6d2dc0f510008c7ff3c10342aec0b476afdb2f7579` +- Raw evidence records: **44** +- Raw evidence chain and body hashes: **passed** +- Raw evidence head: `5b24cf59091c974219c7d38abe776743b25fe445b41ecd00aa8812d47977ff26` +- Unresolved conditions: **0** +- Credentials/authenticated requests/order submissions: **0 / 0 / 0** +- Historical admission credit: **0** +- Live submission: **physically absent** + +## Cumulative admission ledger + +- Prospective markets observed: **10 / 500 minimum** +- Official outcomes available: **10 / 10** +- Hypothetical FOK fills: **2 / 100 minimum** +- Prospective P&L: **+$0.843790** +- Unresolved states: **0** +- Complete untouched weekly blocks: **0 / 4 minimum** + +Both fills won, but a two-trade record has no meaningful statistical power and cannot satisfy confidence, concentration, weekly-block, or admission gates. + +## Next valid work + +1. Continue bounded capture-first blocks without modifying candidate or capture thresholds. +2. Preserve every no-signal, no-fill, fill, resolution, and transport-failure record. +3. Accumulate at least 500 markets, 100 hypothetical fills, and four untouched weekly blocks. +4. Require zero unresolved states, measured execution evidence, acceptable concentration, and a positive lower confidence bound after fees and costs. +5. Keep real-money order submission physically absent. From 37c3f0f2a1a53a24d60251d16c3aa9cee46b0e86 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:05:10 -0500 Subject: [PATCH 172/177] research: add BTC-only block v3 summary --- .../prospective-block-v3/BLOCK_SUMMARY.json | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v3/BLOCK_SUMMARY.json diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v3/BLOCK_SUMMARY.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v3/BLOCK_SUMMARY.json new file mode 100644 index 00000000..ab4c9766 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v3/BLOCK_SUMMARY.json @@ -0,0 +1,50 @@ +{ + "authenticated_requests": 0, + "candidate_policy_sha256": "82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba", + "capture_decision_counts": { + "hypothetical_fok_fill": 1, + "no_signal_below_threshold": 2 + }, + "capture_policy_sha256": "888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc", + "command": "collect-block", + "condition_ids": [ + "0x9e511bc8afffe4491ac7d943976763e9edcc6acac5c7fdf2de6ddccf9729536d", + "0xe7ac80be371db56b3a8c1a682427f2cc3b887f4bdccb7b9b0f2f4933b4147f8e", + "0xf739b004a67e777b0ef3de33bfa417782e70c81595bd707ad7fcdf26a40eaaa8" + ], + "conditions_resolved": 3, + "credentials_used": 0, + "first_market_open_epoch_seconds": 1785112800, + "historical_admission_credit": 0, + "hypothetical_fok_fills": 1, + "last_market_open_epoch_seconds": 1785113400, + "last_resolution_errors": {}, + "lifecycle_head_sha256": "fa080692b88dbfea6233fb6d2dc0f510008c7ff3c10342aec0b476afdb2f7579", + "lifecycle_records": 10, + "live_submission": "physically_absent", + "market_open_epoch_seconds": [ + 1785112800, + 1785113100, + 1785113400 + ], + "markets_collected": 3, + "markets_requested": 3, + "order_submissions": 0, + "policy_changes": 0, + "prospective_pnl_total": "0.610415", + "raw_evidence_head_sha256": "5b24cf59091c974219c7d38abe776743b25fe445b41ecd00aa8812d47977ff26", + "raw_evidence_records": 44, + "resolution_attempts_by_condition": { + "0x9e511bc8afffe4491ac7d943976763e9edcc6acac5c7fdf2de6ddccf9729536d": 1, + "0xe7ac80be371db56b3a8c1a682427f2cc3b887f4bdccb7b9b0f2f4933b4147f8e": 1, + "0xf739b004a67e777b0ef3de33bfa417782e70c81595bd707ad7fcdf26a40eaaa8": 35 + }, + "resolution_decision_counts": { + "resolved_fill": 1, + "resolved_no_fill": 2 + }, + "runtime_source_sha256": "81a13d2aeb3df3c2088b8aa99272b10e217a392eabc839a01a8f2b71f570ecbb", + "source_sha256": "81a13d2aeb3df3c2088b8aa99272b10e217a392eabc839a01a8f2b71f570ecbb", + "status": "complete", + "unresolved_condition_ids": [] +} From 6ae0d8a9a9c6456559b52f70a6841dc338200032 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:05:44 -0500 Subject: [PATCH 173/177] research: add BTC-only block v3 markets --- .../prospective-block-v3/MARKETS.json | 117 ++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v3/MARKETS.json diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v3/MARKETS.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v3/MARKETS.json new file mode 100644 index 00000000..73a1ec79 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v3/MARKETS.json @@ -0,0 +1,117 @@ +[ + { + "arrival": null, + "condition_id": "0x9e511bc8afffe4491ac7d943976763e9edcc6acac5c7fdf2de6ddccf9729536d", + "decision": "no_signal_below_threshold", + "jamaica_window": "2026-07-26 07:40–07:45 PM Jamaica", + "market_open_epoch_seconds": 1785112800, + "official_outcome": "Up", + "official_won": null, + "pnl_per_share": null, + "pnl_total": null, + "resolution_payload_sha256": "51fd814498aae2f743d5b6354eba77461066c333042c8a6614b263df0a85c8aa", + "resolution_polls": 1, + "selected_side": "Up", + "signal": { + "book_timestamps_ms": [ + 1785113010110, + 1785113010110 + ], + "request_completed_ts_ms": 1785113010183, + "request_duration_ms": 183, + "request_started_ts_ms": 1785113010000, + "target_ts_ms": 1785113010000 + }, + "signal_ask": "0.64", + "slug": "btc-updown-5m-1785112800", + "terminal_outcome_prices": [ + "1", + "0" + ], + "terminal_updated_at": "2026-07-27T00:51:14.942925Z", + "utc_window": "2026-07-27 00:40–00:45 UTC" + }, + { + "arrival": null, + "condition_id": "0xe7ac80be371db56b3a8c1a682427f2cc3b887f4bdccb7b9b0f2f4933b4147f8e", + "decision": "no_signal_below_threshold", + "jamaica_window": "2026-07-26 07:45–07:50 PM Jamaica", + "market_open_epoch_seconds": 1785113100, + "official_outcome": "Up", + "official_won": null, + "pnl_per_share": null, + "pnl_total": null, + "resolution_payload_sha256": "dbdcd84b4eb322624a45ae22543ed369be91021958779f9a52ca16c45e2d3423", + "resolution_polls": 1, + "selected_side": "Down", + "signal": { + "book_timestamps_ms": [ + 1785113310110, + 1785113310109 + ], + "request_completed_ts_ms": 1785113310187, + "request_duration_ms": 187, + "request_started_ts_ms": 1785113310000, + "target_ts_ms": 1785113310000 + }, + "signal_ask": "0.6", + "slug": "btc-updown-5m-1785113100", + "terminal_outcome_prices": [ + "1", + "0" + ], + "terminal_updated_at": "2026-07-27T00:51:35.707758Z", + "utc_window": "2026-07-27 00:45–00:50 UTC" + }, + { + "arrival": { + "all_in_cost_per_share": "0.877917", + "best_ask": "0.87", + "book_timestamps_ms": [ + 1785113611088 + ], + "execution_levels": [ + { + "price": "0.87", + "size": "5" + } + ], + "execution_vwap": "0.87", + "fee_per_share": "0.007917", + "filled_shares": "5", + "request_completed_ts_ms": 1785113611163, + "request_duration_ms": 163, + "request_started_ts_ms": 1785113611000, + "target_ts_ms": 1785113611000 + }, + "condition_id": "0xf739b004a67e777b0ef3de33bfa417782e70c81595bd707ad7fcdf26a40eaaa8", + "decision": "hypothetical_fok_fill", + "jamaica_window": "2026-07-26 07:50–07:55 PM Jamaica", + "market_open_epoch_seconds": 1785113400, + "official_outcome": "Down", + "official_won": true, + "pnl_per_share": "0.122083", + "pnl_total": "0.610415", + "resolution_payload_sha256": "c37484c919bf833195bad9c90ecf508eafeda0cfc7af7560f6aff21fb465992a", + "resolution_polls": 35, + "selected_side": "Down", + "signal": { + "book_timestamps_ms": [ + 1785113610122, + 1785113610122 + ], + "request_completed_ts_ms": 1785113610199, + "request_duration_ms": 198, + "request_started_ts_ms": 1785113610001, + "target_ts_ms": 1785113610000 + }, + "signal_ask": "0.88", + "slug": "btc-updown-5m-1785113400", + "terminal_outcome_prices": [ + "0", + "1" + ], + "terminal_updated_at": "2026-07-27T00:56:26.240184Z", + "utc_window": "2026-07-27 00:50–00:55 UTC" + } +] From cdee0ebef22a877b6f2b64600ac49bebc44430cf Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:06:08 -0500 Subject: [PATCH 174/177] research: add BTC-only block v3 verification --- .../prospective-block-v3/VERIFICATION.json | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v3/VERIFICATION.json diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v3/VERIFICATION.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v3/VERIFICATION.json new file mode 100644 index 00000000..f9cc82cd --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v3/VERIFICATION.json @@ -0,0 +1,31 @@ +{ + "artifact_id": 8639355447, + "artifact_zip_sha256": "b30e9e10de839070ad1ae88f98c952ff2d8d4378ab828bed1533cff0e588cc0e", + "authenticated_requests": 0, + "candidate_policy_sha256": "82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba", + "capture_policy_sha256": "888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc", + "credentials_used": 0, + "deterministic_tar_sha256": "5af345750399cde55f782140c8ba0427b74d39c113ce864e7a045c3f246cc6ee", + "historical_admission_credit": 0, + "hypothetical_fok_fills": 1, + "internal_sha256sums_verified": true, + "lifecycle_chain_verified": true, + "lifecycle_head_sha256": "fa080692b88dbfea6233fb6d2dc0f510008c7ff3c10342aec0b476afdb2f7579", + "lifecycle_records": 10, + "live_submission": "physically_absent", + "markets": 3, + "official_outcomes": { + "denominator": 3, + "numerator": 3 + }, + "order_submissions": 0, + "prospective_pnl_total": "0.610415", + "raw_evidence_chain_and_bodies_verified": true, + "raw_evidence_head_sha256": "5b24cf59091c974219c7d38abe776743b25fe445b41ecd00aa8812d47977ff26", + "raw_evidence_records": 44, + "runtime_source_sha256": "81a13d2aeb3df3c2088b8aa99272b10e217a392eabc839a01a8f2b71f570ecbb", + "tar_round_trip_verified": true, + "unresolved_states": 0, + "workflow_head_sha": "e5463d3ecb2b150de857ac73bf31e08d3317cd2f", + "workflow_run_id": 30228071397 +} From ec45478a679f4323bcfb6cbe2af72df9b550206f Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:06:36 -0500 Subject: [PATCH 175/177] research: update BTC-only admission ledger through block v3 --- .../ADMISSION_LEDGER.json | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/ADMISSION_LEDGER.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/ADMISSION_LEDGER.json index 1afaf48f..15dee8c1 100644 --- a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/ADMISSION_LEDGER.json +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/ADMISSION_LEDGER.json @@ -4,13 +4,13 @@ "capture_policy_sha256": "888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc", "current": { "complete_untouched_weekly_blocks": 0, - "hypothetical_fok_fills": 1, + "hypothetical_fok_fills": 2, "official_outcome_coverage": { - "denominator": 7, - "numerator": 7 + "denominator": 10, + "numerator": 10 }, - "prospective_markets_observed": 7, - "prospective_pnl_total": "0.233375", + "prospective_markets_observed": 10, + "prospective_pnl_total": "0.843790", "unresolved_states": 0 }, "gates": { @@ -47,8 +47,16 @@ "markets": 3, "pnl_total": "0.233375", "runtime_source_sha256": "81a13d2aeb3df3c2088b8aa99272b10e217a392eabc839a01a8f2b71f570ecbb" + }, + { + "artifact_zip_sha256": "b30e9e10de839070ad1ae88f98c952ff2d8d4378ab828bed1533cff0e588cc0e", + "checkpoint": "prospective-block-v3", + "hypothetical_fok_fills": 1, + "markets": 3, + "pnl_total": "0.610415", + "runtime_source_sha256": "81a13d2aeb3df3c2088b8aa99272b10e217a392eabc839a01a8f2b71f570ecbb" } ], "status": "prospective_shadow_only_not_admitted", - "version": 2 + "version": 3 } From d5fc75054f24714c6db3219f49fc47e48c634cf6 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:07:37 -0500 Subject: [PATCH 176/177] docs: record BTC-only prospective block v3 --- .../digital-surface-live-readiness/STATUS.md | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/ci_payload/digital-surface-live-readiness/STATUS.md b/ci_payload/digital-surface-live-readiness/STATUS.md index 6a2cb5af..9accd32e 100644 --- a/ci_payload/digital-surface-live-readiness/STATUS.md +++ b/ci_payload/digital-surface-live-readiness/STATUS.md @@ -112,6 +112,23 @@ The block contained two below-threshold no-signals and one no-fill above the fro The filled market selected Down at a 95-cent signal ask, executed five shares at 95 cents one second later, paid $0.003325/share in fees, officially resolved Down, and earned $0.046675/share or $0.233375 total. +### Prospective block v3 + +- checkpoint: `ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v3/CHECKPOINT.md` +- workflow run: `30228071397`; +- artifact ID: `8639355447`; +- artifact ZIP SHA-256: `b30e9e10de839070ad1ae88f98c952ff2d8d4378ab828bed1533cff0e588cc0e`; +- deterministic TAR SHA-256: `5af345750399cde55f782140c8ba0427b74d39c113ce864e7a045c3f246cc6ee`; +- markets: 3; +- official outcomes: 3/3; +- hypothetical fills: 1; +- P&L: **+$0.610415**; +- unresolved conditions: 0; +- lifecycle records: 10, chain verified; +- raw evidence records: 44, chain and bodies verified. + +The filled market selected Down at an 88-cent signal ask. The arrival best ask improved to 87 cents, remained eligible under the frozen one-cent adverse-move rule, executed five shares at 87 cents, officially resolved Down, and earned $0.122083/share or $0.610415 total. + ## Safety and admission boundary - credentials used: **0**; @@ -119,14 +136,14 @@ The filled market selected Down at a 95-cent signal ask, executed five shares at - order submissions: **0**; - live submission: **physically absent**; - historical admission credit: **0**; -- prospective markets observed: **7 / 500 minimum**; -- official outcome coverage: **7 / 7**; -- hypothetical FOK fills: **1 / 100 minimum**; -- prospective P&L: **+$0.233375**; +- prospective markets observed: **10 / 500 minimum**; +- official outcome coverage: **10 / 10**; +- hypothetical FOK fills: **2 / 100 minimum**; +- prospective P&L: **+$0.843790**; - unresolved states: **0**; - complete untouched weekly blocks: **0 / 4 minimum**. -One winning fill is not statistical validation. These seven markets are valid prospective evidence but cannot support admission, a confidence bound, a concentration pass, or live release. Even a future admissible shadow report cannot enable trading; any order submitter requires a separate reviewed release and explicit operator action. +Two winning fills are not statistical validation. These ten markets are valid prospective evidence but cannot support admission, a confidence bound, a concentration pass, or live release. Even a future admissible shadow report cannot enable trading; any order submitter requires a separate reviewed release and explicit operator action. ## Next work From 2aefe929648faa675f1ebeb5c42c0f4fafde5ee4 Mon Sep 17 00:00:00 2001 From: Chrisbryan17 <138815930+Chrisbryan17@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:09:39 -0500 Subject: [PATCH 177/177] ci: add BTC-only prospective block v4 run --- .../btc-only-v3-prospective-block-v4.yml | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 .github/workflows/btc-only-v3-prospective-block-v4.yml diff --git a/.github/workflows/btc-only-v3-prospective-block-v4.yml b/.github/workflows/btc-only-v3-prospective-block-v4.yml new file mode 100644 index 00000000..e22de287 --- /dev/null +++ b/.github/workflows/btc-only-v3-prospective-block-v4.yml @@ -0,0 +1,146 @@ +name: BTC-Only v3 Prospective Block v4 + +on: + pull_request: + branches: [digital-surface-live-readiness-20260725] + paths: + - ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/BLOCK_V4_TRIGGER.txt + workflow_dispatch: + +concurrency: + group: btc-only-v3-prospective-block-v4 + cancel-in-progress: false + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + ROOT: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 + STATE: btc-only-v3-prospective-block-v4 + +jobs: + capture-resolve-verify: + runs-on: ubuntu-latest + timeout-minutes: 35 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install pinned public and test clients + run: python -m pip install requests==2.32.4 pytest==8.3.5 + + - name: Verify committed runner and frozen boundaries + shell: bash + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner + run: | + set -euo pipefail + python -m py_compile "$ROOT/runner/observer_cli.py" "$ROOT/runner/tests/test_observer_block.py" + pytest -q "$ROOT/runner/tests/test_observer_block.py" + python - <<'PY' + from pathlib import Path + from capture_policy import load_capture_policy + from late_favorite_v3 import load_frozen_policy + from observer_cli import compute_combined_source_sha256 + + root = Path('ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3') + source_sha = compute_combined_source_sha256(root) + trading = load_frozen_policy(root / 'CANDIDATE_SPEC.json', root / 'FREEZE_MANIFEST.json', source_sha256=source_sha) + capture = load_capture_policy(root / 'CAPTURE_POLICY.json', root / 'CAPTURE_FREEZE_MANIFEST.json') + assert trading.policy_sha256 == '82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba' + assert capture.capture_policy_sha256 == '888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc' + assert capture.valid_market_open_after_epoch_seconds == 1785094521 + print(source_sha) + PY + + - name: Capture three consecutive markets, then resolve the queue + shell: bash + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner + run: | + set -euo pipefail + python "$ROOT/runner/observer_cli.py" \ + --project-root "$ROOT" \ + --state-dir "$STATE" \ + collect-block \ + --markets 3 \ + --max-resolution-attempts 120 \ + --retry-delay-ms 10000 | tee block-v4-output.json + + - name: Verify complete block evidence and write summary + shell: bash + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner + run: | + set -euo pipefail + python - <<'PY' + import json + from pathlib import Path + from capture_policy import CaptureBoundLifecycleLedger, load_capture_policy + from late_favorite_v3 import load_frozen_policy + from observer_cli import compute_combined_source_sha256 + from public_collector import RawEvidenceStore + + root = Path('ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3') + state = Path('btc-only-v3-prospective-block-v4') + source_sha = compute_combined_source_sha256(root) + trading = load_frozen_policy(root / 'CANDIDATE_SPEC.json', root / 'FREEZE_MANIFEST.json', source_sha256=source_sha) + capture = load_capture_policy(root / 'CAPTURE_POLICY.json', root / 'CAPTURE_FREEZE_MANIFEST.json') + ledger = CaptureBoundLifecycleLedger(state / 'lifecycle.jsonl', trading, capture) + evidence = RawEvidenceStore(state / 'raw_evidence') + block_paths = sorted((state / 'summaries').glob('block-*.json')) + assert len(block_paths) == 1, block_paths + block = json.loads(block_paths[0].read_text()) + assert block['status'] == 'complete' + assert block['markets_requested'] == 3 + assert block['markets_collected'] == 3 + assert block['conditions_resolved'] == 3 + assert block['unresolved_condition_ids'] == [] + assert len(block['condition_ids']) == 3 + assert len(set(block['condition_ids'])) == 3 + assert all(int(row['market_open_epoch_seconds']) > capture.valid_market_open_after_epoch_seconds for row in ledger.records) + assert sum(1 for row in ledger.records if row['event_type'] == 'discovered') == 3 + assert sum(1 for row in ledger.records if row['event_type'] == 'signal') == 3 + assert sum(1 for row in ledger.records if row['event_type'] == 'resolution') == 3 + summary = { + **block, + 'candidate_policy_sha256': trading.policy_sha256, + 'capture_policy_sha256': capture.capture_policy_sha256, + 'runtime_source_sha256': source_sha, + 'lifecycle_head_sha256': ledger.head_hash, + 'raw_evidence_head_sha256': evidence.head_hash, + 'lifecycle_records': len(ledger.records), + 'raw_evidence_records': len(evidence.records), + 'historical_admission_credit': 0, + 'credentials_used': 0, + 'authenticated_requests': 0, + 'order_submissions': 0, + 'live_submission': 'physically_absent', + } + (state / 'BLOCK_V4_SUMMARY.json').write_text(json.dumps(summary, indent=2, sort_keys=True) + '\n') + print(json.dumps(summary, indent=2, sort_keys=True)) + PY + + - name: Build deterministic block artifact + shell: bash + run: | + set -euo pipefail + (cd "$STATE" && find . -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum > SHA256SUMS) + (cd "$STATE" && sha256sum -c SHA256SUMS) + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf btc-only-v3-prospective-block-v4.tar.gz "$STATE" + sha256sum btc-only-v3-prospective-block-v4.tar.gz > btc-only-v3-prospective-block-v4.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: btc-only-v3-prospective-block-v4 + path: | + btc-only-v3-prospective-block-v4/ + btc-only-v3-prospective-block-v4.tar.gz + btc-only-v3-prospective-block-v4.tar.gz.sha256 + compression-level: 0 + retention-days: 30