diff --git a/.github/workflows/btc-only-v3-first-prospective.yml b/.github/workflows/btc-only-v3-first-prospective.yml new file mode 100644 index 00000000..feb001bd --- /dev/null +++ b/.github/workflows/btc-only-v3-first-prospective.yml @@ -0,0 +1,205 @@ +name: BTC-Only v3 First Prospective Observation + +on: + push: + branches: [btc-only-v3-first-prospective-20260727] + paths: + - ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/FIRST_PROSPECTIVE_TRIGGER.txt + pull_request: + branches: [digital-surface-live-readiness-20260725] + paths: + - ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/FIRST_PROSPECTIVE_TRIGGER.txt + workflow_dispatch: + +concurrency: + group: btc-only-v3-first-prospective + cancel-in-progress: false + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + ROOT: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 + OUT: btc-only-v3-first-prospective + +jobs: + capture-and-resolve: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install pinned public client + run: python -m pip install requests==2.32.4 + + - name: Verify observer source and frozen policy bytes + shell: bash + run: | + set -euo pipefail + export PYTHONPATH="$ROOT/source:$ROOT/collector" + python - <<'PY' + import hashlib + from pathlib import Path + from capture_policy import load_capture_policy + from late_favorite_v3 import load_frozen_policy + + root = Path('ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3') + source_paths = [ + root / 'source/late_favorite_v3.py', + root / 'source/capture_policy.py', + root / 'collector/public_collector.py', + root / 'collector/resolution.py', + ] + combined = hashlib.sha256() + for path in source_paths: + raw = path.read_bytes() + combined.update(path.name.encode() + b'\0' + raw) + source_sha = combined.hexdigest() + trading = load_frozen_policy(root / 'CANDIDATE_SPEC.json', root / 'FREEZE_MANIFEST.json', source_sha256=source_sha) + capture = load_capture_policy(root / 'CAPTURE_POLICY.json', root / 'CAPTURE_FREEZE_MANIFEST.json') + assert trading.policy_sha256 == '82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba' + assert capture.capture_policy_sha256 == '888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc' + assert source_sha == 'f2e086bb5e74f0343a4ea29f46ea2bf8e23062952b456303e204c6b5469b6126' + PY + + - name: Capture next eligible post-cutoff BTC market + shell: bash + run: | + set -euo pipefail + export PYTHONPATH="$ROOT/source:$ROOT/collector" + python "$ROOT/collector/cli.py" --output-dir "$OUT" capture | tee capture.stdout.json + + - name: Derive immutable market identity + id: market + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import json + import os + from pathlib import Path + + payload = json.loads(Path('btc-only-v3-first-prospective/last_capture.json').read_text()) + condition = str(payload['condition_id']) + opening = int(payload['market_open_epoch_seconds']) + assert opening > 1785094521 + assert condition + with open(os.environ['GITHUB_OUTPUT'], 'a', encoding='utf-8') as handle: + handle.write(f'condition_id={condition}\n') + handle.write(f'opening={opening}\n') + print(json.dumps({'condition_id': condition, 'opening': opening, 'capture_decision': payload['decision']}, indent=2)) + PY + + - name: Wait for terminal resolution window + shell: bash + env: + OPENING: ${{ steps.market.outputs.opening }} + run: | + set -euo pipefail + target=$((OPENING + 330)) + now=$(date +%s) + delay=$((target - now)) + if [ "$delay" -gt 0 ]; then + sleep "$delay" + fi + + - name: Resolve from authoritative Gamma terminal prices + shell: bash + env: + CONDITION_ID: ${{ steps.market.outputs.condition_id }} + run: | + set -euo pipefail + export PYTHONPATH="$ROOT/source:$ROOT/collector" + success=0 + for attempt in $(seq 1 40); do + if python "$ROOT/collector/cli.py" --output-dir "$OUT" resolve --condition-id "$CONDITION_ID" | tee resolve.stdout.json; then + success=1 + break + fi + echo "resolution not terminal on attempt $attempt" + sleep 15 + done + test "$success" -eq 1 + + - name: Verify complete append-only evidence + shell: bash + run: | + set -euo pipefail + export PYTHONPATH="$ROOT/source:$ROOT/collector" + python - <<'PY' + import hashlib + import json + from pathlib import Path + from capture_policy import CaptureBoundLifecycleLedger, load_capture_policy + from late_favorite_v3 import load_frozen_policy + from public_collector import RawEvidenceStore + + root = Path('ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3') + out = Path('btc-only-v3-first-prospective') + source_paths = [root / 'source/late_favorite_v3.py', root / 'source/capture_policy.py', root / 'collector/public_collector.py', root / 'collector/resolution.py'] + combined = hashlib.sha256() + for path in source_paths: + raw = path.read_bytes() + combined.update(path.name.encode() + b'\0' + raw) + trading = load_frozen_policy(root / 'CANDIDATE_SPEC.json', root / 'FREEZE_MANIFEST.json', source_sha256=combined.hexdigest()) + capture = load_capture_policy(root / 'CAPTURE_POLICY.json', root / 'CAPTURE_FREEZE_MANIFEST.json') + ledger = CaptureBoundLifecycleLedger(out / 'lifecycle.jsonl', trading, capture) + evidence = RawEvidenceStore(out / 'raw') + assert ledger.records[-1]['event_type'] == 'resolution' + conditions = {row['condition_id'] for row in ledger.records} + assert len(conditions) == 1 + assert all(int(row['market_open_epoch_seconds']) > capture.valid_market_open_after_epoch_seconds for row in ledger.records) + allowed = { + ('GET', 'gamma-api.polymarket.com'), + ('POST', 'clob.polymarket.com'), + } + from urllib.parse import urlparse + for row in evidence.records: + parsed = urlparse(row['url']) + assert (row['method'], parsed.netloc) in allowed + assert parsed.path.startswith('/markets/slug/') or parsed.path == '/books' + resolution = json.loads((out / 'last_resolution.json').read_text()) + summary = { + 'condition_id': next(iter(conditions)), + 'market_open_epoch_seconds': ledger.records[0]['market_open_epoch_seconds'], + 'capture_decision': json.loads((out / 'last_capture.json').read_text())['decision'], + 'resolution_decision': resolution['decision'], + 'official_outcome': resolution['official_outcome'], + 'official_won': resolution.get('official_won'), + 'pnl_total': resolution.get('pnl_total'), + 'lifecycle_records': len(ledger.records), + 'raw_responses': len(evidence.records), + 'lifecycle_head_sha256': ledger.head_hash, + 'evidence_head_sha256': evidence.head_hash, + 'candidate_policy_sha256': trading.policy_sha256, + 'capture_policy_sha256': capture.capture_policy_sha256, + 'historical_admission_credit': 0, + 'live_submission': 'physically_absent', + } + (out / 'PROSPECTIVE_SUMMARY.json').write_text(json.dumps(summary, indent=2, sort_keys=True) + '\n') + print(json.dumps(summary, indent=2, sort_keys=True)) + PY + + - name: Build deterministic prospective artifact + shell: bash + run: | + set -euo pipefail + (cd "$OUT" && find . -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum > SHA256SUMS) + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf btc-only-v3-first-prospective-v1.tar.gz "$OUT" + sha256sum btc-only-v3-first-prospective-v1.tar.gz > btc-only-v3-first-prospective-v1.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: btc-only-v3-first-prospective-v1 + path: | + btc-only-v3-first-prospective/ + btc-only-v3-first-prospective-v1.tar.gz + btc-only-v3-first-prospective-v1.tar.gz.sha256 + compression-level: 0 + retention-days: 30 diff --git a/.github/workflows/btc-only-v3-observer-checkpoint.yml b/.github/workflows/btc-only-v3-observer-checkpoint.yml new file mode 100644 index 00000000..58f39a17 --- /dev/null +++ b/.github/workflows/btc-only-v3-observer-checkpoint.yml @@ -0,0 +1,231 @@ +name: BTC-Only v3 Observer Checkpoint + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/btc-only-v3-observer-checkpoint.yml + - ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/** + pull_request: + branches: [digital-surface-ci-20260724, digital-surface-live-readiness-20260725] + paths: + - .github/workflows/btc-only-v3-observer-checkpoint.yml + - ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/** + workflow_dispatch: + +concurrency: + group: btc-only-v3-observer-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + observer-contract: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install pinned test stack + run: python -m pip install requests==2.32.4 pytest==8.3.5 + + - name: Compile observer, collector, resolution, and CLI sources + shell: bash + run: | + set -euo pipefail + root=ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 + python -m py_compile \ + "$root/source/late_favorite_v3.py" \ + "$root/source/capture_policy.py" \ + "$root/collector/public_collector.py" \ + "$root/collector/resolution.py" \ + "$root/collector/cli.py" \ + "$root/tests/test_late_favorite_v3.py" \ + "$root/tests/test_capture_policy.py" \ + "$root/tests/test_public_collector.py" \ + "$root/tests/test_resolution.py" \ + "$root/tests/test_resolution_cache_bust.py" \ + "$root/tests/test_cli.py" + + - name: Pass exactly 25 behavior contracts + shell: bash + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector + run: | + set -euo pipefail + root=ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 + pytest --collect-only -q "$root/tests" | tee /tmp/collect.txt + grep -Eq '^25 tests collected' /tmp/collect.txt + pytest -q "$root/tests" + + - name: Verify frozen policies and contamination cutoffs + shell: bash + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector + run: | + set -euo pipefail + python - <<'PY' + import hashlib + import json + from pathlib import Path + + from capture_policy import load_capture_policy + from late_favorite_v3 import load_frozen_policy + + root = Path('ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3') + frozen_source_paths = [ + root / 'source/late_favorite_v3.py', + root / 'source/capture_policy.py', + root / 'collector/public_collector.py', + root / 'collector/resolution.py', + ] + combined = hashlib.sha256() + source_hashes = {} + for path in frozen_source_paths: + raw = path.read_bytes() + digest = hashlib.sha256(raw).hexdigest() + source_hashes[str(path.relative_to(root))] = digest + combined.update(path.name.encode() + b'\0' + raw) + combined_source_sha256 = combined.hexdigest() + cli_path = root / 'collector/cli.py' + cli_sha256 = hashlib.sha256(cli_path.read_bytes()).hexdigest() + source_hashes[str(cli_path.relative_to(root))] = cli_sha256 + + trading = load_frozen_policy( + root / 'CANDIDATE_SPEC.json', + root / 'FREEZE_MANIFEST.json', + source_sha256=combined_source_sha256, + ) + capture = load_capture_policy( + root / 'CAPTURE_POLICY.json', + root / 'CAPTURE_FREEZE_MANIFEST.json', + ) + assert trading.policy_sha256 == '82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba' + assert capture.capture_policy_sha256 == '888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc' + assert capture.valid_market_open_after_epoch_seconds > trading.valid_market_open_after_epoch_seconds + assert capture.signal_target_offset_ms == 210_000 + assert capture.arrival_target_offset_ms == 211_000 + assert capture.max_request_start_lateness_ms == 250 + assert capture.max_request_duration_ms == 1_000 + assert capture.max_book_age_at_request_completion_ms == 2_000 + + verification = { + 'candidate': 'late_favorite_btc_only_v3_prospective_shadow', + 'tests_collected': 25, + 'candidate_policy_sha256': trading.policy_sha256, + 'capture_policy_sha256': capture.capture_policy_sha256, + 'candidate_cutoff_epoch_seconds': trading.valid_market_open_after_epoch_seconds, + 'capture_cutoff_epoch_seconds': capture.valid_market_open_after_epoch_seconds, + 'combined_frozen_source_sha256': combined_source_sha256, + 'cli_sha256': cli_sha256, + 'source_files_sha256': source_hashes, + 'public_endpoints': [ + 'GET https://gamma-api.polymarket.com/markets/slug/{slug}?cache_bust={attempt}', + 'POST https://clob.polymarket.com/books', + ], + 'official_resolution': 'Gamma closed binary terminal outcome', + 'commands': ['capture', 'resolve'], + 'credentials_used': 0, + 'authenticated_requests': 0, + 'order_submissions': 0, + 'live_submission': 'physically_absent', + 'historical_admission_credit': 0, + } + (root / 'OBSERVER_VERIFICATION.json').write_text( + json.dumps(verification, indent=2, sort_keys=True) + '\n' + ) + print(json.dumps(verification, indent=2, sort_keys=True)) + PY + + - name: Verify capability boundaries + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import ast + from pathlib import Path + + root = Path('ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3') + core_forbidden = { + 'requests', 'urllib', 'httpx', 'aiohttp', 'web3', + 'py_clob_client', 'eth_account', 'websocket', 'websockets', + } + privileged = {'web3', 'py_clob_client', 'eth_account', 'websocket', 'websockets'} + forbidden_tokens = { + 'create_order', 'post_order', 'submit_order', 'private_key', + 'api_secret', 'sign_transaction', 'poly_api_key', 'poly_signature', + 'poly_passphrase', 'poly_address', + } + def imports_for(path: Path) -> set[str]: + tree = ast.parse(path.read_text()) + imports = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imports.update(alias.name.split('.')[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + imports.add(node.module.split('.')[0]) + return imports + + for path in sorted((root / 'source').glob('*.py')): + imports = imports_for(path) + assert not (imports & core_forbidden), (path, imports & core_forbidden) + lowered = path.read_text().lower() + for token in forbidden_tokens: + assert token not in lowered, (path, token) + + collector_paths = sorted((root / 'collector').glob('*.py')) + all_collector_imports = set().union(*(imports_for(path) for path in collector_paths)) + assert 'requests' in all_collector_imports + assert not (all_collector_imports & privileged), all_collector_imports & privileged + for path in collector_paths: + lowered = path.read_text().lower() + for token in forbidden_tokens: + assert token not in lowered, (path, token) + public_text = (root / 'collector/public_collector.py').read_text() + assert 'https://gamma-api.polymarket.com' in public_text + assert 'https://clob.polymarket.com' in public_text + assert 'parsed.path == "/books"' in public_text + assert 'parsed.path.startswith("/markets/slug/")' in public_text + resolution_text = (root / 'collector/resolution.py').read_text() + assert '?cache_bust=' in resolution_text + cli_text = (root / 'collector/cli.py').read_text() + assert 'capture' in cli_text and 'resolve' in cli_text + PY + + - name: Build deterministic observer checkpoint + shell: bash + run: | + set -euo pipefail + root=ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 + mkdir -p btc-only-v3-observer-checkpoint + cp "$root/CANDIDATE_SPEC.json" btc-only-v3-observer-checkpoint/ + cp "$root/FREEZE_MANIFEST.json" btc-only-v3-observer-checkpoint/ + cp "$root/CAPTURE_POLICY.json" btc-only-v3-observer-checkpoint/ + cp "$root/CAPTURE_FREEZE_MANIFEST.json" btc-only-v3-observer-checkpoint/ + cp "$root/OBSERVER_VERIFICATION.json" btc-only-v3-observer-checkpoint/ + cp "$root/source/late_favorite_v3.py" btc-only-v3-observer-checkpoint/ + cp "$root/source/capture_policy.py" btc-only-v3-observer-checkpoint/ + cp "$root/collector/public_collector.py" btc-only-v3-observer-checkpoint/ + cp "$root/collector/resolution.py" btc-only-v3-observer-checkpoint/ + cp "$root/collector/cli.py" btc-only-v3-observer-checkpoint/ + (cd btc-only-v3-observer-checkpoint && sha256sum * > SHA256SUMS) + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf btc-only-v3-observer-checkpoint-v3.tar.gz btc-only-v3-observer-checkpoint + sha256sum btc-only-v3-observer-checkpoint-v3.tar.gz > btc-only-v3-observer-checkpoint-v3.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: btc-only-v3-observer-checkpoint-v3 + path: | + btc-only-v3-observer-checkpoint/ + btc-only-v3-observer-checkpoint-v3.tar.gz + btc-only-v3-observer-checkpoint-v3.tar.gz.sha256 + compression-level: 0 + retention-days: 30 diff --git a/.github/workflows/btc-only-v3-prospective-block-v2.yml b/.github/workflows/btc-only-v3-prospective-block-v2.yml new file mode 100644 index 00000000..0458ed66 --- /dev/null +++ b/.github/workflows/btc-only-v3-prospective-block-v2.yml @@ -0,0 +1,146 @@ +name: BTC-Only v3 Prospective Block v2 + +on: + pull_request: + branches: [digital-surface-live-readiness-20260725] + paths: + - ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/BLOCK_V2_TRIGGER.txt + workflow_dispatch: + +concurrency: + group: btc-only-v3-prospective-block-v2 + cancel-in-progress: false + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + ROOT: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 + STATE: btc-only-v3-prospective-block-v2 + +jobs: + capture-resolve-verify: + runs-on: ubuntu-latest + timeout-minutes: 35 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install pinned public and test clients + run: python -m pip install requests==2.32.4 pytest==8.3.5 + + - name: Verify committed runner and frozen boundaries + shell: bash + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner + run: | + set -euo pipefail + python -m py_compile "$ROOT/runner/observer_cli.py" "$ROOT/runner/tests/test_observer_block.py" + pytest -q "$ROOT/runner/tests/test_observer_block.py" + python - <<'PY' + from pathlib import Path + from capture_policy import load_capture_policy + from late_favorite_v3 import load_frozen_policy + from observer_cli import compute_combined_source_sha256 + + root = Path('ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3') + source_sha = compute_combined_source_sha256(root) + trading = load_frozen_policy(root / 'CANDIDATE_SPEC.json', root / 'FREEZE_MANIFEST.json', source_sha256=source_sha) + capture = load_capture_policy(root / 'CAPTURE_POLICY.json', root / 'CAPTURE_FREEZE_MANIFEST.json') + assert trading.policy_sha256 == '82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba' + assert capture.capture_policy_sha256 == '888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc' + assert capture.valid_market_open_after_epoch_seconds == 1785094521 + print(source_sha) + PY + + - name: Capture three consecutive markets, then resolve the queue + shell: bash + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner + run: | + set -euo pipefail + python "$ROOT/runner/observer_cli.py" \ + --project-root "$ROOT" \ + --state-dir "$STATE" \ + collect-block \ + --markets 3 \ + --max-resolution-attempts 120 \ + --retry-delay-ms 10000 | tee block-v2-output.json + + - name: Verify complete block evidence and write summary + shell: bash + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner + run: | + set -euo pipefail + python - <<'PY' + import json + from pathlib import Path + from capture_policy import CaptureBoundLifecycleLedger, load_capture_policy + from late_favorite_v3 import load_frozen_policy + from observer_cli import compute_combined_source_sha256 + from public_collector import RawEvidenceStore + + root = Path('ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3') + state = Path('btc-only-v3-prospective-block-v2') + source_sha = compute_combined_source_sha256(root) + trading = load_frozen_policy(root / 'CANDIDATE_SPEC.json', root / 'FREEZE_MANIFEST.json', source_sha256=source_sha) + capture = load_capture_policy(root / 'CAPTURE_POLICY.json', root / 'CAPTURE_FREEZE_MANIFEST.json') + ledger = CaptureBoundLifecycleLedger(state / 'lifecycle.jsonl', trading, capture) + evidence = RawEvidenceStore(state / 'raw_evidence') + block_paths = sorted((state / 'summaries').glob('block-*.json')) + assert len(block_paths) == 1, block_paths + block = json.loads(block_paths[0].read_text()) + assert block['status'] == 'complete' + assert block['markets_requested'] == 3 + assert block['markets_collected'] == 3 + assert block['conditions_resolved'] == 3 + assert block['unresolved_condition_ids'] == [] + assert len(block['condition_ids']) == 3 + assert len(set(block['condition_ids'])) == 3 + assert all(int(row['market_open_epoch_seconds']) > capture.valid_market_open_after_epoch_seconds for row in ledger.records) + assert sum(1 for row in ledger.records if row['event_type'] == 'discovered') == 3 + assert sum(1 for row in ledger.records if row['event_type'] == 'signal') == 3 + assert sum(1 for row in ledger.records if row['event_type'] == 'resolution') == 3 + summary = { + **block, + 'candidate_policy_sha256': trading.policy_sha256, + 'capture_policy_sha256': capture.capture_policy_sha256, + 'runtime_source_sha256': source_sha, + 'lifecycle_head_sha256': ledger.head_hash, + 'raw_evidence_head_sha256': evidence.head_hash, + 'lifecycle_records': len(ledger.records), + 'raw_evidence_records': len(evidence.records), + 'historical_admission_credit': 0, + 'credentials_used': 0, + 'authenticated_requests': 0, + 'order_submissions': 0, + 'live_submission': 'physically_absent', + } + (state / 'BLOCK_V2_SUMMARY.json').write_text(json.dumps(summary, indent=2, sort_keys=True) + '\n') + print(json.dumps(summary, indent=2, sort_keys=True)) + PY + + - name: Build deterministic block artifact + shell: bash + run: | + set -euo pipefail + (cd "$STATE" && find . -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum > SHA256SUMS) + (cd "$STATE" && sha256sum -c SHA256SUMS) + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf btc-only-v3-prospective-block-v2.tar.gz "$STATE" + sha256sum btc-only-v3-prospective-block-v2.tar.gz > btc-only-v3-prospective-block-v2.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: btc-only-v3-prospective-block-v2 + path: | + btc-only-v3-prospective-block-v2/ + btc-only-v3-prospective-block-v2.tar.gz + btc-only-v3-prospective-block-v2.tar.gz.sha256 + compression-level: 0 + retention-days: 30 diff --git a/.github/workflows/btc-only-v3-prospective-block-v3.yml b/.github/workflows/btc-only-v3-prospective-block-v3.yml new file mode 100644 index 00000000..a674ca20 --- /dev/null +++ b/.github/workflows/btc-only-v3-prospective-block-v3.yml @@ -0,0 +1,146 @@ +name: BTC-Only v3 Prospective Block v3 + +on: + pull_request: + branches: [digital-surface-live-readiness-20260725] + paths: + - ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/BLOCK_V3_TRIGGER.txt + workflow_dispatch: + +concurrency: + group: btc-only-v3-prospective-block-v3 + cancel-in-progress: false + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + ROOT: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 + STATE: btc-only-v3-prospective-block-v3 + +jobs: + capture-resolve-verify: + runs-on: ubuntu-latest + timeout-minutes: 35 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install pinned public and test clients + run: python -m pip install requests==2.32.4 pytest==8.3.5 + + - name: Verify committed runner and frozen boundaries + shell: bash + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner + run: | + set -euo pipefail + python -m py_compile "$ROOT/runner/observer_cli.py" "$ROOT/runner/tests/test_observer_block.py" + pytest -q "$ROOT/runner/tests/test_observer_block.py" + python - <<'PY' + from pathlib import Path + from capture_policy import load_capture_policy + from late_favorite_v3 import load_frozen_policy + from observer_cli import compute_combined_source_sha256 + + root = Path('ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3') + source_sha = compute_combined_source_sha256(root) + trading = load_frozen_policy(root / 'CANDIDATE_SPEC.json', root / 'FREEZE_MANIFEST.json', source_sha256=source_sha) + capture = load_capture_policy(root / 'CAPTURE_POLICY.json', root / 'CAPTURE_FREEZE_MANIFEST.json') + assert trading.policy_sha256 == '82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba' + assert capture.capture_policy_sha256 == '888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc' + assert capture.valid_market_open_after_epoch_seconds == 1785094521 + print(source_sha) + PY + + - name: Capture three consecutive markets, then resolve the queue + shell: bash + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner + run: | + set -euo pipefail + python "$ROOT/runner/observer_cli.py" \ + --project-root "$ROOT" \ + --state-dir "$STATE" \ + collect-block \ + --markets 3 \ + --max-resolution-attempts 120 \ + --retry-delay-ms 10000 | tee block-v3-output.json + + - name: Verify complete block evidence and write summary + shell: bash + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner + run: | + set -euo pipefail + python - <<'PY' + import json + from pathlib import Path + from capture_policy import CaptureBoundLifecycleLedger, load_capture_policy + from late_favorite_v3 import load_frozen_policy + from observer_cli import compute_combined_source_sha256 + from public_collector import RawEvidenceStore + + root = Path('ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3') + state = Path('btc-only-v3-prospective-block-v3') + source_sha = compute_combined_source_sha256(root) + trading = load_frozen_policy(root / 'CANDIDATE_SPEC.json', root / 'FREEZE_MANIFEST.json', source_sha256=source_sha) + capture = load_capture_policy(root / 'CAPTURE_POLICY.json', root / 'CAPTURE_FREEZE_MANIFEST.json') + ledger = CaptureBoundLifecycleLedger(state / 'lifecycle.jsonl', trading, capture) + evidence = RawEvidenceStore(state / 'raw_evidence') + block_paths = sorted((state / 'summaries').glob('block-*.json')) + assert len(block_paths) == 1, block_paths + block = json.loads(block_paths[0].read_text()) + assert block['status'] == 'complete' + assert block['markets_requested'] == 3 + assert block['markets_collected'] == 3 + assert block['conditions_resolved'] == 3 + assert block['unresolved_condition_ids'] == [] + assert len(block['condition_ids']) == 3 + assert len(set(block['condition_ids'])) == 3 + assert all(int(row['market_open_epoch_seconds']) > capture.valid_market_open_after_epoch_seconds for row in ledger.records) + assert sum(1 for row in ledger.records if row['event_type'] == 'discovered') == 3 + assert sum(1 for row in ledger.records if row['event_type'] == 'signal') == 3 + assert sum(1 for row in ledger.records if row['event_type'] == 'resolution') == 3 + summary = { + **block, + 'candidate_policy_sha256': trading.policy_sha256, + 'capture_policy_sha256': capture.capture_policy_sha256, + 'runtime_source_sha256': source_sha, + 'lifecycle_head_sha256': ledger.head_hash, + 'raw_evidence_head_sha256': evidence.head_hash, + 'lifecycle_records': len(ledger.records), + 'raw_evidence_records': len(evidence.records), + 'historical_admission_credit': 0, + 'credentials_used': 0, + 'authenticated_requests': 0, + 'order_submissions': 0, + 'live_submission': 'physically_absent', + } + (state / 'BLOCK_V3_SUMMARY.json').write_text(json.dumps(summary, indent=2, sort_keys=True) + '\n') + print(json.dumps(summary, indent=2, sort_keys=True)) + PY + + - name: Build deterministic block artifact + shell: bash + run: | + set -euo pipefail + (cd "$STATE" && find . -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum > SHA256SUMS) + (cd "$STATE" && sha256sum -c SHA256SUMS) + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf btc-only-v3-prospective-block-v3.tar.gz "$STATE" + sha256sum btc-only-v3-prospective-block-v3.tar.gz > btc-only-v3-prospective-block-v3.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: btc-only-v3-prospective-block-v3 + path: | + btc-only-v3-prospective-block-v3/ + btc-only-v3-prospective-block-v3.tar.gz + btc-only-v3-prospective-block-v3.tar.gz.sha256 + compression-level: 0 + retention-days: 30 diff --git a/.github/workflows/btc-only-v3-prospective-block-v4.yml b/.github/workflows/btc-only-v3-prospective-block-v4.yml new file mode 100644 index 00000000..e22de287 --- /dev/null +++ b/.github/workflows/btc-only-v3-prospective-block-v4.yml @@ -0,0 +1,146 @@ +name: BTC-Only v3 Prospective Block v4 + +on: + pull_request: + branches: [digital-surface-live-readiness-20260725] + paths: + - ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/BLOCK_V4_TRIGGER.txt + workflow_dispatch: + +concurrency: + group: btc-only-v3-prospective-block-v4 + cancel-in-progress: false + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + ROOT: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 + STATE: btc-only-v3-prospective-block-v4 + +jobs: + capture-resolve-verify: + runs-on: ubuntu-latest + timeout-minutes: 35 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install pinned public and test clients + run: python -m pip install requests==2.32.4 pytest==8.3.5 + + - name: Verify committed runner and frozen boundaries + shell: bash + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner + run: | + set -euo pipefail + python -m py_compile "$ROOT/runner/observer_cli.py" "$ROOT/runner/tests/test_observer_block.py" + pytest -q "$ROOT/runner/tests/test_observer_block.py" + python - <<'PY' + from pathlib import Path + from capture_policy import load_capture_policy + from late_favorite_v3 import load_frozen_policy + from observer_cli import compute_combined_source_sha256 + + root = Path('ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3') + source_sha = compute_combined_source_sha256(root) + trading = load_frozen_policy(root / 'CANDIDATE_SPEC.json', root / 'FREEZE_MANIFEST.json', source_sha256=source_sha) + capture = load_capture_policy(root / 'CAPTURE_POLICY.json', root / 'CAPTURE_FREEZE_MANIFEST.json') + assert trading.policy_sha256 == '82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba' + assert capture.capture_policy_sha256 == '888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc' + assert capture.valid_market_open_after_epoch_seconds == 1785094521 + print(source_sha) + PY + + - name: Capture three consecutive markets, then resolve the queue + shell: bash + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner + run: | + set -euo pipefail + python "$ROOT/runner/observer_cli.py" \ + --project-root "$ROOT" \ + --state-dir "$STATE" \ + collect-block \ + --markets 3 \ + --max-resolution-attempts 120 \ + --retry-delay-ms 10000 | tee block-v4-output.json + + - name: Verify complete block evidence and write summary + shell: bash + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner + run: | + set -euo pipefail + python - <<'PY' + import json + from pathlib import Path + from capture_policy import CaptureBoundLifecycleLedger, load_capture_policy + from late_favorite_v3 import load_frozen_policy + from observer_cli import compute_combined_source_sha256 + from public_collector import RawEvidenceStore + + root = Path('ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3') + state = Path('btc-only-v3-prospective-block-v4') + source_sha = compute_combined_source_sha256(root) + trading = load_frozen_policy(root / 'CANDIDATE_SPEC.json', root / 'FREEZE_MANIFEST.json', source_sha256=source_sha) + capture = load_capture_policy(root / 'CAPTURE_POLICY.json', root / 'CAPTURE_FREEZE_MANIFEST.json') + ledger = CaptureBoundLifecycleLedger(state / 'lifecycle.jsonl', trading, capture) + evidence = RawEvidenceStore(state / 'raw_evidence') + block_paths = sorted((state / 'summaries').glob('block-*.json')) + assert len(block_paths) == 1, block_paths + block = json.loads(block_paths[0].read_text()) + assert block['status'] == 'complete' + assert block['markets_requested'] == 3 + assert block['markets_collected'] == 3 + assert block['conditions_resolved'] == 3 + assert block['unresolved_condition_ids'] == [] + assert len(block['condition_ids']) == 3 + assert len(set(block['condition_ids'])) == 3 + assert all(int(row['market_open_epoch_seconds']) > capture.valid_market_open_after_epoch_seconds for row in ledger.records) + assert sum(1 for row in ledger.records if row['event_type'] == 'discovered') == 3 + assert sum(1 for row in ledger.records if row['event_type'] == 'signal') == 3 + assert sum(1 for row in ledger.records if row['event_type'] == 'resolution') == 3 + summary = { + **block, + 'candidate_policy_sha256': trading.policy_sha256, + 'capture_policy_sha256': capture.capture_policy_sha256, + 'runtime_source_sha256': source_sha, + 'lifecycle_head_sha256': ledger.head_hash, + 'raw_evidence_head_sha256': evidence.head_hash, + 'lifecycle_records': len(ledger.records), + 'raw_evidence_records': len(evidence.records), + 'historical_admission_credit': 0, + 'credentials_used': 0, + 'authenticated_requests': 0, + 'order_submissions': 0, + 'live_submission': 'physically_absent', + } + (state / 'BLOCK_V4_SUMMARY.json').write_text(json.dumps(summary, indent=2, sort_keys=True) + '\n') + print(json.dumps(summary, indent=2, sort_keys=True)) + PY + + - name: Build deterministic block artifact + shell: bash + run: | + set -euo pipefail + (cd "$STATE" && find . -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum > SHA256SUMS) + (cd "$STATE" && sha256sum -c SHA256SUMS) + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf btc-only-v3-prospective-block-v4.tar.gz "$STATE" + sha256sum btc-only-v3-prospective-block-v4.tar.gz > btc-only-v3-prospective-block-v4.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: btc-only-v3-prospective-block-v4 + path: | + btc-only-v3-prospective-block-v4/ + btc-only-v3-prospective-block-v4.tar.gz + btc-only-v3-prospective-block-v4.tar.gz.sha256 + compression-level: 0 + retention-days: 30 diff --git a/.github/workflows/btc-only-v3-prospective-once.yml b/.github/workflows/btc-only-v3-prospective-once.yml new file mode 100644 index 00000000..b6f95008 --- /dev/null +++ b/.github/workflows/btc-only-v3-prospective-once.yml @@ -0,0 +1,246 @@ +name: BTC-Only v3 Prospective Observation Once + +on: + push: + branches: [btc-only-v3-prospective-run-20260726] + paths: + - .github/workflows/btc-only-v3-prospective-once.yml + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: btc-only-v3-prospective-once-20260726 + cancel-in-progress: true + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + RUN_LABEL: first-post-cutoff-observation + +jobs: + observe-one-market: + runs-on: ubuntu-latest + timeout-minutes: 35 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install pinned public and test clients + run: python -m pip install requests==2.32.4 pytest==8.3.5 + + - name: Verify cache-busted terminal resolver + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector + run: | + root=ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 + python -m py_compile "$root/collector/resolution.py" "$root/tests/test_resolution_cache_bust.py" + pytest -q "$root/tests/test_resolution_cache_bust.py" + + - name: Select a genuinely prospective BTC window + id: select + shell: bash + run: | + set -euo pipefail + python - <<'PY' >> "$GITHUB_OUTPUT" + import time + now = time.time_ns() // 1_000_000_000 + opening = (now // 300) * 300 + if now > opening + 205: + opening += 300 + print(f"opening={opening}") + print(f"signal_target_ms={opening * 1000 + 210000}") + print(f"arrival_target_ms={opening * 1000 + 211000}") + print(f"resolution_target_ms={opening * 1000 + 305000}") + PY + + - name: Wait until market discovery is valid + env: + OPENING: ${{ steps.select.outputs.opening }} + run: | + python - <<'PY' + import os + import time + target = int(os.environ['OPENING']) * 1000 + 5000 + while True: + remaining = target - time.time_ns() // 1_000_000 + if remaining <= 0: + break + time.sleep(min(remaining / 1000, 0.5)) + PY + + - name: Capture discovery, signal, and arrival + env: + OPENING: ${{ steps.select.outputs.opening }} + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector + shell: bash + run: | + set -euo pipefail + root=ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 + mkdir -p prospective-data + python "$root/collector/cli.py" \ + --root "$root" \ + --output-dir prospective-data \ + --timeout-seconds 5 \ + capture --market-open "$OPENING" | tee capture-output.json + cat prospective-data/last_capture.json + + - name: Wait until terminal resolution window + env: + RESOLUTION_TARGET_MS: ${{ steps.select.outputs.resolution_target_ms }} + run: | + python - <<'PY' + import os + import time + target = int(os.environ['RESOLUTION_TARGET_MS']) + while True: + remaining = target - time.time_ns() // 1_000_000 + if remaining <= 0: + break + time.sleep(min(remaining / 1000, 0.5)) + PY + + - name: Watch authoritative Gamma terminal prices + id: resolution + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector + shell: bash + run: | + set -uo pipefail + root=ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 + condition_id="$(python - <<'PY' + import json + print(json.load(open('prospective-data/last_capture.json'))['condition_id']) + PY + )" + resolved=0 + attempts=0 + for attempt in $(seq 1 120); do + attempts="$attempt" + if python "$root/collector/cli.py" \ + --root "$root" \ + --output-dir prospective-data \ + --timeout-seconds 5 \ + resolve --condition-id "$condition_id" | tee resolution-output.json; then + resolved=1 + break + fi + sleep 10 + done + echo "resolved=$resolved" >> "$GITHUB_OUTPUT" + echo "attempts=$attempts" >> "$GITHUB_OUTPUT" + if test "$resolved" -eq 1; then + cat prospective-data/last_resolution.json + else + echo "Gamma remained non-terminal after $attempts attempts; preserving unresolved evidence." + fi + exit 0 + + - name: Verify append-only prospective evidence + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector + EXPECTED_OPENING: ${{ steps.select.outputs.opening }} + RESOLVED: ${{ steps.resolution.outputs.resolved }} + RESOLUTION_ATTEMPTS: ${{ steps.resolution.outputs.attempts }} + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import json + import os + from pathlib import Path + + from capture_policy import CaptureBoundLifecycleLedger, load_capture_policy + from cli import _combined_source_sha256 + from late_favorite_v3 import load_frozen_policy + from public_collector import RawEvidenceStore + + root = Path('ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3') + out = Path('prospective-data') + source_sha = _combined_source_sha256(root) + trading = load_frozen_policy(root / 'CANDIDATE_SPEC.json', root / 'FREEZE_MANIFEST.json', source_sha256=source_sha) + capture = load_capture_policy(root / 'CAPTURE_POLICY.json', root / 'CAPTURE_FREEZE_MANIFEST.json') + ledger = CaptureBoundLifecycleLedger(out / 'lifecycle.jsonl', trading, capture) + evidence = RawEvidenceStore(out / 'raw') + event_types = [row['event_type'] for row in ledger.records] + allowed = { + ('discovered', 'signal'), + ('discovered', 'signal', 'arrival'), + ('discovered', 'signal', 'resolution'), + ('discovered', 'signal', 'arrival', 'resolution'), + } + assert tuple(event_types) in allowed, event_types + opening = int(os.environ['EXPECTED_OPENING']) + assert opening > capture.valid_market_open_after_epoch_seconds + assert {int(row['market_open_epoch_seconds']) for row in ledger.records} == {opening} + assert len(evidence.records) >= 2 + capture_result = json.loads((out / 'last_capture.json').read_text()) + resolution_path = out / 'last_resolution.json' + resolution = json.loads(resolution_path.read_text()) if resolution_path.exists() else None + resolved = os.environ.get('RESOLVED') == '1' + if resolved: + assert event_types[-1] == 'resolution' + assert resolution is not None + assert resolution['official_outcome'] in {'Up', 'Down'} + assert resolution['decision'] in {'resolved_fill', 'resolved_no_fill'} + else: + assert event_types[-1] != 'resolution' + assert resolution is None + run = { + 'candidate': 'late_favorite_btc_only_v3_prospective_shadow', + 'run_label': os.environ.get('RUN_LABEL'), + 'github_sha': os.environ.get('GITHUB_SHA'), + 'market_open_epoch_seconds': opening, + 'condition_id': ledger.records[0]['condition_id'], + 'capture_decision': capture_result['decision'], + 'resolution_status': 'terminal' if resolved else 'unresolved', + 'resolution_attempts': int(os.environ.get('RESOLUTION_ATTEMPTS') or 0), + 'resolution': resolution, + 'lifecycle_event_types': event_types, + 'lifecycle_records': len(ledger.records), + 'lifecycle_head_sha256': ledger.head_hash, + 'raw_evidence_records': len(evidence.records), + 'raw_evidence_head_sha256': evidence.head_hash, + 'candidate_policy_sha256': trading.policy_sha256, + 'capture_policy_sha256': capture.capture_policy_sha256, + 'combined_source_sha256': source_sha, + 'credentials_used': 0, + 'authenticated_requests': 0, + 'order_submissions': 0, + 'historical_admission_credit': 0, + 'live_submission': 'physically_absent', + } + (out / 'PROSPECTIVE_RUN.json').write_text(json.dumps(run, indent=2, sort_keys=True) + '\n') + print(json.dumps(run, indent=2, sort_keys=True)) + PY + + - name: Build deterministic evidence archive + shell: bash + run: | + set -euo pipefail + find prospective-data -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum > prospective-data/SHA256SUMS + sha256sum -c prospective-data/SHA256SUMS + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf btc-only-v3-prospective-run-v1.tar.gz prospective-data + sha256sum btc-only-v3-prospective-run-v1.tar.gz > btc-only-v3-prospective-run-v1.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: btc-only-v3-prospective-run-v1 + path: | + prospective-data/ + btc-only-v3-prospective-run-v1.tar.gz + btc-only-v3-prospective-run-v1.tar.gz.sha256 + compression-level: 0 + retention-days: 30 + + - name: Require terminal resolution for admission evidence + run: | + python - <<'PY' + import json + run = json.load(open('prospective-data/PROSPECTIVE_RUN.json')) + assert run['resolution_status'] == 'terminal', run['resolution_status'] + PY diff --git a/.github/workflows/btc-only-v3-prospective-one-shot.yml b/.github/workflows/btc-only-v3-prospective-one-shot.yml new file mode 100644 index 00000000..cf2f47de --- /dev/null +++ b/.github/workflows/btc-only-v3-prospective-one-shot.yml @@ -0,0 +1,237 @@ +name: BTC-Only v3 Prospective One-Shot + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/btc-only-v3-prospective-one-shot.yml + workflow_dispatch: + +concurrency: + group: btc-only-v3-prospective-one-shot + cancel-in-progress: false + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + CANDIDATE_ROOT: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 + STATE_DIR: btc-only-v3-prospective-state + +jobs: + collect-and-resolve-one-market: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + ref: digital-surface-live-readiness-20260725 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install pinned public client + run: python -m pip install requests==2.32.4 + + - name: Verify frozen source and command boundary + shell: bash + run: | + set -euo pipefail + python -m py_compile \ + "$CANDIDATE_ROOT/source/late_favorite_v3.py" \ + "$CANDIDATE_ROOT/source/capture_policy.py" \ + "$CANDIDATE_ROOT/collector/public_collector.py" \ + "$CANDIDATE_ROOT/collector/resolution.py" \ + "$CANDIDATE_ROOT/runner/observer_cli.py" + PYTHONPATH="$CANDIDATE_ROOT/source:$CANDIDATE_ROOT/collector:$CANDIDATE_ROOT/runner" \ + python "$CANDIDATE_ROOT/runner/observer_cli.py" \ + --project-root "$CANDIDATE_ROOT" \ + --state-dir "$STATE_DIR" \ + status | tee initial-status.json + python - <<'PY' + import json + from pathlib import Path + status = json.loads(Path('initial-status.json').read_text()) + assert status['conditions_seen'] == 0 + assert status['conditions_resolved'] == 0 + assert status['lifecycle_head_sha256'] == '0' * 64 + assert status['raw_evidence_head_sha256'] == '0' * 64 + PY + + - name: Collect next genuinely new BTC window + shell: bash + run: | + set -euo pipefail + PYTHONPATH="$CANDIDATE_ROOT/source:$CANDIDATE_ROOT/collector:$CANDIDATE_ROOT/runner" \ + python "$CANDIDATE_ROOT/runner/observer_cli.py" \ + --project-root "$CANDIDATE_ROOT" \ + --state-dir "$STATE_DIR" \ + collect-next | tee collect-output.json + python - <<'PY' + import json + from pathlib import Path + output = json.loads(Path('collect-output.json').read_text()) + condition = str(output['condition_id']) + opening = int(output['market_open_epoch_seconds']) + assert opening > 1785094521 + assert output['command'] == 'collect-next' + assert len(output['lifecycle_head_sha256']) == 64 + assert len(output['raw_evidence_head_sha256']) == 64 + with Path('/tmp/prospective.env').open('w') as handle: + handle.write(f'CONDITION_ID={condition}\n') + handle.write(f'MARKET_OPEN_EPOCH_SECONDS={opening}\n') + handle.write(f'COLLECT_DECISION={output["decision"]}\n') + print(json.dumps(output, indent=2, sort_keys=True)) + PY + cat /tmp/prospective.env >> "$GITHUB_ENV" + + - name: Wait for terminal resolution window + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import os + import time + opening = int(os.environ['MARKET_OPEN_EPOCH_SECONDS']) + target_ms = (opening + 305) * 1000 + while True: + remaining = target_ms - time.time_ns() // 1_000_000 + if remaining <= 0: + break + time.sleep(min(remaining / 1000.0, 1.0)) + PY + + - name: Resolve from authoritative terminal outcome + shell: bash + run: | + set -euo pipefail + success=0 + for attempt in $(seq 1 24); do + echo "resolution attempt $attempt" + if PYTHONPATH="$CANDIDATE_ROOT/source:$CANDIDATE_ROOT/collector:$CANDIDATE_ROOT/runner" \ + python "$CANDIDATE_ROOT/runner/observer_cli.py" \ + --project-root "$CANDIDATE_ROOT" \ + --state-dir "$STATE_DIR" \ + resolve --condition-id "$CONDITION_ID" \ + > resolve-output.json; then + cat resolve-output.json + success=1 + break + fi + sleep 15 + done + test "$success" -eq 1 + + - name: Verify complete prospective lifecycle and raw evidence + shell: bash + run: | + set -euo pipefail + PYTHONPATH="$CANDIDATE_ROOT/source:$CANDIDATE_ROOT/collector:$CANDIDATE_ROOT/runner" \ + python "$CANDIDATE_ROOT/runner/observer_cli.py" \ + --project-root "$CANDIDATE_ROOT" \ + --state-dir "$STATE_DIR" \ + status | tee final-status.json + python - <<'PY' + import hashlib + import json + import os + from pathlib import Path + + state = Path(os.environ['STATE_DIR']) + collect = json.loads(Path('collect-output.json').read_text()) + resolution = json.loads(Path('resolve-output.json').read_text()) + status = json.loads(Path('final-status.json').read_text()) + condition = os.environ['CONDITION_ID'] + opening = int(os.environ['MARKET_OPEN_EPOCH_SECONDS']) + + assert status['conditions_seen'] == 1 + assert status['conditions_resolved'] == 1 + assert status['unresolved_condition_ids'] == [] + assert resolution['condition_id'] == condition + assert resolution['command'] == 'resolve' + assert resolution['official_outcome'] in {'Up', 'Down'} + assert resolution['decision'] in {'resolved_fill', 'resolved_no_fill'} + assert opening > 1785094521 + + lifecycle_rows = [ + json.loads(line) + for line in (state / 'lifecycle.jsonl').read_text().splitlines() + if line.strip() + ] + assert lifecycle_rows[0]['event_type'] == 'discovered' + assert lifecycle_rows[-1]['event_type'] == 'resolution' + assert all(row['condition_id'] == condition for row in lifecycle_rows) + assert all(row['policy_sha256'] == '82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba' for row in lifecycle_rows) + assert all(row['capture_policy_sha256'] == '888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc' for row in lifecycle_rows) + assert all(int(row['market_open_epoch_seconds']) == opening for row in lifecycle_rows) + + manifest_rows = [ + json.loads(line) + for line in (state / 'raw_evidence/manifest.jsonl').read_text().splitlines() + if line.strip() + ] + assert len(manifest_rows) >= 3 + allowed = { + ('GET', 'gamma-api.polymarket.com'), + ('POST', 'clob.polymarket.com'), + } + from urllib.parse import urlparse + for row in manifest_rows: + parsed = urlparse(row['url']) + assert (row['method'], parsed.netloc) in allowed + body = state / 'raw_evidence' / row['body_path'] + assert hashlib.sha256(body.read_bytes()).hexdigest() == row['response_body_sha256'] + + audit = { + 'candidate': 'late_favorite_btc_only_v3_prospective_shadow', + 'condition_id': condition, + 'market_open_epoch_seconds': opening, + 'collect_decision': collect['decision'], + 'resolution_decision': resolution['decision'], + 'official_outcome': resolution['official_outcome'], + 'official_won': resolution.get('official_won'), + 'pnl_total': resolution.get('pnl_total'), + 'lifecycle_records': len(lifecycle_rows), + 'raw_evidence_records': len(manifest_rows), + 'lifecycle_head_sha256': status['lifecycle_head_sha256'], + 'raw_evidence_head_sha256': status['raw_evidence_head_sha256'], + 'source_sha256': status['source_sha256'], + 'candidate_policy_sha256': '82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba', + 'capture_policy_sha256': '888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc', + 'credentials_used': 0, + 'authenticated_requests': 0, + 'order_submissions': 0, + 'live_submission': 'physically_absent', + 'historical_admission_credit': 0, + 'prospective_markets': 1, + } + Path('PROSPECTIVE_RUN_AUDIT.json').write_text( + json.dumps(audit, indent=2, sort_keys=True) + '\n' + ) + print(json.dumps(audit, indent=2, sort_keys=True)) + PY + + - name: Build deterministic prospective evidence archive + shell: bash + run: | + set -euo pipefail + cp collect-output.json "$STATE_DIR/" + cp resolve-output.json "$STATE_DIR/" + cp final-status.json "$STATE_DIR/" + cp PROSPECTIVE_RUN_AUDIT.json "$STATE_DIR/" + (cd "$STATE_DIR" && find . -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum > SHA256SUMS) + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf btc-only-v3-prospective-one-shot-v1.tar.gz "$STATE_DIR" + sha256sum btc-only-v3-prospective-one-shot-v1.tar.gz > btc-only-v3-prospective-one-shot-v1.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: btc-only-v3-prospective-one-shot-${{ github.run_id }} + path: | + ${{ env.STATE_DIR }}/ + btc-only-v3-prospective-one-shot-v1.tar.gz + btc-only-v3-prospective-one-shot-v1.tar.gz.sha256 + compression-level: 0 + retention-days: 90 diff --git a/.github/workflows/btc-only-v3-prospective-run-1.yml b/.github/workflows/btc-only-v3-prospective-run-1.yml new file mode 100644 index 00000000..aaa8186a --- /dev/null +++ b/.github/workflows/btc-only-v3-prospective-run-1.yml @@ -0,0 +1,211 @@ +name: BTC-Only v3 Prospective Run 1 + +on: + pull_request: + branches: [digital-surface-live-readiness-20260725] + paths: + - ci_payload/prospective-run-triggers/btc-v3-run-1.txt + workflow_dispatch: + +concurrency: + group: btc-only-v3-prospective-run-1 + cancel-in-progress: false + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + CANDIDATE_ROOT: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 + STATE_DIR: btc-only-v3-prospective-state + +jobs: + collect-and-resolve-one-market: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + ref: digital-surface-live-readiness-20260725 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install pinned public client + run: python -m pip install requests==2.32.4 + + - name: Verify empty state and frozen runner + shell: bash + run: | + set -euo pipefail + python -m py_compile \ + "$CANDIDATE_ROOT/source/late_favorite_v3.py" \ + "$CANDIDATE_ROOT/source/capture_policy.py" \ + "$CANDIDATE_ROOT/collector/public_collector.py" \ + "$CANDIDATE_ROOT/collector/resolution.py" \ + "$CANDIDATE_ROOT/runner/observer_cli.py" + PYTHONPATH="$CANDIDATE_ROOT/source:$CANDIDATE_ROOT/collector:$CANDIDATE_ROOT/runner" \ + python "$CANDIDATE_ROOT/runner/observer_cli.py" \ + --project-root "$CANDIDATE_ROOT" --state-dir "$STATE_DIR" status \ + | tee initial-status.json + python - <<'PY' + import json + from pathlib import Path + status = json.loads(Path('initial-status.json').read_text()) + assert status['conditions_seen'] == 0 + assert status['conditions_resolved'] == 0 + assert status['lifecycle_head_sha256'] == '0' * 64 + assert status['raw_evidence_head_sha256'] == '0' * 64 + PY + + - name: Collect next post-cutoff BTC window + shell: bash + run: | + set -euo pipefail + PYTHONPATH="$CANDIDATE_ROOT/source:$CANDIDATE_ROOT/collector:$CANDIDATE_ROOT/runner" \ + python "$CANDIDATE_ROOT/runner/observer_cli.py" \ + --project-root "$CANDIDATE_ROOT" --state-dir "$STATE_DIR" collect-next \ + | tee collect-output.json + python - <<'PY' + import json + from pathlib import Path + output = json.loads(Path('collect-output.json').read_text()) + condition = str(output['condition_id']) + opening = int(output['market_open_epoch_seconds']) + assert opening > 1785094521 + assert output['command'] == 'collect-next' + with Path('/tmp/prospective.env').open('w') as handle: + handle.write(f'CONDITION_ID={condition}\n') + handle.write(f'MARKET_OPEN_EPOCH_SECONDS={opening}\n') + handle.write(f'COLLECT_DECISION={output["decision"]}\n') + print(json.dumps(output, indent=2, sort_keys=True)) + PY + cat /tmp/prospective.env >> "$GITHUB_ENV" + + - name: Wait for official close + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import os + import time + target_ms = (int(os.environ['MARKET_OPEN_EPOCH_SECONDS']) + 305) * 1000 + while True: + remaining = target_ms - time.time_ns() // 1_000_000 + if remaining <= 0: + break + time.sleep(min(remaining / 1000.0, 1.0)) + PY + + - name: Resolve authoritative terminal outcome + shell: bash + run: | + set -euo pipefail + success=0 + for attempt in $(seq 1 24); do + echo "resolution attempt $attempt" + if PYTHONPATH="$CANDIDATE_ROOT/source:$CANDIDATE_ROOT/collector:$CANDIDATE_ROOT/runner" \ + python "$CANDIDATE_ROOT/runner/observer_cli.py" \ + --project-root "$CANDIDATE_ROOT" --state-dir "$STATE_DIR" \ + resolve --condition-id "$CONDITION_ID" > resolve-output.json; then + cat resolve-output.json + success=1 + break + fi + sleep 15 + done + test "$success" -eq 1 + + - name: Verify complete append-only evidence + shell: bash + run: | + set -euo pipefail + PYTHONPATH="$CANDIDATE_ROOT/source:$CANDIDATE_ROOT/collector:$CANDIDATE_ROOT/runner" \ + python "$CANDIDATE_ROOT/runner/observer_cli.py" \ + --project-root "$CANDIDATE_ROOT" --state-dir "$STATE_DIR" status \ + | tee final-status.json + python - <<'PY' + import hashlib + import json + import os + from pathlib import Path + from urllib.parse import urlparse + + state = Path(os.environ['STATE_DIR']) + collect = json.loads(Path('collect-output.json').read_text()) + resolution = json.loads(Path('resolve-output.json').read_text()) + status = json.loads(Path('final-status.json').read_text()) + condition = os.environ['CONDITION_ID'] + opening = int(os.environ['MARKET_OPEN_EPOCH_SECONDS']) + assert status['conditions_seen'] == 1 + assert status['conditions_resolved'] == 1 + assert status['unresolved_condition_ids'] == [] + assert resolution['condition_id'] == condition + assert resolution['official_outcome'] in {'Up', 'Down'} + assert resolution['decision'] in {'resolved_fill', 'resolved_no_fill'} + assert opening > 1785094521 + + lifecycle_rows = [json.loads(line) for line in (state / 'lifecycle.jsonl').read_text().splitlines() if line.strip()] + assert lifecycle_rows[0]['event_type'] == 'discovered' + assert lifecycle_rows[-1]['event_type'] == 'resolution' + assert all(row['condition_id'] == condition for row in lifecycle_rows) + assert all(row['policy_sha256'] == '82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba' for row in lifecycle_rows) + assert all(row['capture_policy_sha256'] == '888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc' for row in lifecycle_rows) + + manifest_rows = [json.loads(line) for line in (state / 'raw_evidence/manifest.jsonl').read_text().splitlines() if line.strip()] + assert len(manifest_rows) >= 3 + for row in manifest_rows: + parsed = urlparse(row['url']) + assert (row['method'], parsed.netloc) in { + ('GET', 'gamma-api.polymarket.com'), + ('POST', 'clob.polymarket.com'), + } + body = state / 'raw_evidence' / row['body_path'] + assert hashlib.sha256(body.read_bytes()).hexdigest() == row['response_body_sha256'] + + audit = { + 'candidate': 'late_favorite_btc_only_v3_prospective_shadow', + 'condition_id': condition, + 'market_open_epoch_seconds': opening, + 'collect_decision': collect['decision'], + 'resolution_decision': resolution['decision'], + 'official_outcome': resolution['official_outcome'], + 'official_won': resolution.get('official_won'), + 'pnl_total': resolution.get('pnl_total'), + 'lifecycle_records': len(lifecycle_rows), + 'raw_evidence_records': len(manifest_rows), + 'lifecycle_head_sha256': status['lifecycle_head_sha256'], + 'raw_evidence_head_sha256': status['raw_evidence_head_sha256'], + 'source_sha256': status['source_sha256'], + 'candidate_policy_sha256': '82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba', + 'capture_policy_sha256': '888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc', + 'credentials_used': 0, + 'authenticated_requests': 0, + 'order_submissions': 0, + 'live_submission': 'physically_absent', + 'historical_admission_credit': 0, + 'prospective_markets': 1, + } + Path('PROSPECTIVE_RUN_AUDIT.json').write_text(json.dumps(audit, indent=2, sort_keys=True) + '\n') + print(json.dumps(audit, indent=2, sort_keys=True)) + PY + + - name: Build deterministic prospective artifact + shell: bash + run: | + set -euo pipefail + cp collect-output.json resolve-output.json final-status.json PROSPECTIVE_RUN_AUDIT.json "$STATE_DIR/" + (cd "$STATE_DIR" && find . -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum > SHA256SUMS) + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf btc-only-v3-prospective-run-1.tar.gz "$STATE_DIR" + sha256sum btc-only-v3-prospective-run-1.tar.gz > btc-only-v3-prospective-run-1.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: btc-only-v3-prospective-run-1-${{ github.run_id }} + path: | + ${{ env.STATE_DIR }}/ + btc-only-v3-prospective-run-1.tar.gz + btc-only-v3-prospective-run-1.tar.gz.sha256 + compression-level: 0 + retention-days: 90 diff --git a/.github/workflows/btc-only-v3-runner-checkpoint.yml b/.github/workflows/btc-only-v3-runner-checkpoint.yml new file mode 100644 index 00000000..d424a37b --- /dev/null +++ b/.github/workflows/btc-only-v3-runner-checkpoint.yml @@ -0,0 +1,184 @@ +name: BTC-Only v3 Runner Checkpoint + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/btc-only-v3-runner-checkpoint.yml + - ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/** + pull_request: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/btc-only-v3-runner-checkpoint.yml + - ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/** + workflow_dispatch: + +concurrency: + group: btc-only-v3-runner-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + runner-contract: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install pinned test stack + run: python -m pip install requests==2.32.4 pytest==8.3.5 + + - name: Compile runner source and tests + shell: bash + run: | + set -euo pipefail + root=ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 + python -m py_compile \ + "$root/runner/observer_cli.py" \ + "$root/runner/tests/test_observer_cli.py" \ + "$root/runner/tests/test_observer_block.py" + + - name: Pass exactly 11 runner contracts + shell: bash + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner + run: | + set -euo pipefail + root=ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 + pytest --collect-only -q "$root/runner/tests" | tee /tmp/collect.txt + grep -Eq '^11 tests collected' /tmp/collect.txt + pytest -q "$root/runner/tests" + + - name: Verify runtime loads frozen policies without network + shell: bash + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector:ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner + run: | + set -euo pipefail + python - <<'PY' + import hashlib + import json + import tempfile + from pathlib import Path + + from observer_cli import ( + MAX_BLOCK_MARKETS, + build_runtime, + compute_combined_source_sha256, + runtime_status, + ) + + root = Path('ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3') + relative_paths = ( + 'source/late_favorite_v3.py', + 'source/capture_policy.py', + 'collector/public_collector.py', + 'collector/resolution.py', + 'runner/observer_cli.py', + ) + independent = hashlib.sha256() + source_hashes = {} + for relative in relative_paths: + raw = (root / relative).read_bytes() + source_hashes[relative] = hashlib.sha256(raw).hexdigest() + independent.update(relative.encode('utf-8') + b'\0' + raw) + source_sha256 = compute_combined_source_sha256(root) + assert source_sha256 == independent.hexdigest() + assert MAX_BLOCK_MARKETS == 12 + with tempfile.TemporaryDirectory() as directory: + runtime = build_runtime(root, directory, clock_ms=lambda: 1_900_000_000_000) + status = runtime_status(runtime) + assert status['conditions_seen'] == 0 + assert status['conditions_resolved'] == 0 + assert status['source_sha256'] == source_sha256 + assert status['lifecycle_head_sha256'] == '0' * 64 + assert status['raw_evidence_head_sha256'] == '0' * 64 + verification = { + 'candidate': 'late_favorite_btc_only_v3_prospective_shadow', + 'tests_collected': 11, + 'runner_commands': ['collect-next', 'collect-block', 'resolve', 'status'], + 'maximum_block_markets': MAX_BLOCK_MARKETS, + 'block_order': 'capture_all_then_resolve_queue', + 'combined_runtime_source_sha256': source_sha256, + 'source_files_sha256': source_hashes, + 'credentials_used': 0, + 'authenticated_requests': 0, + 'order_submissions': 0, + 'live_submission': 'physically_absent', + 'historical_admission_credit': 0, + } + out = root / 'runner/RUNNER_VERIFICATION.json' + out.write_text(json.dumps(verification, indent=2, sort_keys=True) + '\n') + print(json.dumps(verification, indent=2, sort_keys=True)) + PY + + - name: Verify runner has no privileged capability + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import ast + from pathlib import Path + + path = Path('ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/observer_cli.py') + text = path.read_text() + tree = ast.parse(text) + imports = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imports.update(alias.name.split('.')[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + imports.add(node.module.split('.')[0]) + forbidden_modules = { + 'requests', 'urllib', 'httpx', 'aiohttp', 'web3', + 'py_clob_client', 'eth_account', 'websocket', 'websockets', + } + assert not (imports & forbidden_modules), imports & forbidden_modules + lowered = text.lower() + for token in ( + 'create_order', 'post_order', 'submit_order', 'private_key', + 'api_secret', 'sign_transaction', 'poly_api_key', 'poly_signature', + 'poly_passphrase', 'poly_address', + ): + assert token not in lowered, token + assert 'collect-next' in text + assert 'collect-block' in text + assert 'resolve' in text + assert 'status' in text + assert 'submit-order' not in text + PY + + - name: Build deterministic runner checkpoint + shell: bash + run: | + set -euo pipefail + root=ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3 + mkdir -p btc-only-v3-runner-checkpoint + cp "$root/CANDIDATE_SPEC.json" btc-only-v3-runner-checkpoint/ + cp "$root/FREEZE_MANIFEST.json" btc-only-v3-runner-checkpoint/ + cp "$root/CAPTURE_POLICY.json" btc-only-v3-runner-checkpoint/ + cp "$root/CAPTURE_FREEZE_MANIFEST.json" btc-only-v3-runner-checkpoint/ + cp "$root/runner/observer_cli.py" btc-only-v3-runner-checkpoint/ + cp "$root/runner/RUNNER_VERIFICATION.json" btc-only-v3-runner-checkpoint/ + (cd btc-only-v3-runner-checkpoint && sha256sum * > SHA256SUMS) + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf btc-only-v3-runner-checkpoint-v2.tar.gz btc-only-v3-runner-checkpoint + sha256sum btc-only-v3-runner-checkpoint-v2.tar.gz > btc-only-v3-runner-checkpoint-v2.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: btc-only-v3-runner-checkpoint-v2 + path: | + btc-only-v3-runner-checkpoint/ + btc-only-v3-runner-checkpoint-v2.tar.gz + btc-only-v3-runner-checkpoint-v2.tar.gz.sha256 + compression-level: 0 + retention-days: 30 diff --git a/.github/workflows/digital-surface-binance-market-state-checkpoint.yml b/.github/workflows/digital-surface-binance-market-state-checkpoint.yml new file mode 100644 index 00000000..598cd73a --- /dev/null +++ b/.github/workflows/digital-surface-binance-market-state-checkpoint.yml @@ -0,0 +1,189 @@ +name: Digital Surface Binance Market-State Checkpoint + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/digital-surface-binance-market-state-checkpoint.yml + - ci_payload/digital-surface-live-readiness/market-state-source/** + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-binance-market-state-checkpoint.yml + - ci_payload/digital-surface-live-readiness/market-state-source/** + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + verify-binance-market-state: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct checksum-pinned source + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source && tar -xzf base.tar.gz -C source + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + for patch_file in timestamp-unit-fix.patch full-universe-gate-fix.patch resolved-universe-fix.patch asset-merge-fix.patch; do + patch -d source -p1 < "ci_payload/digital-surface-official-resolution-patch/${patch_file}" + done + echo 'f71f3a653ca58b4bf98609fc29d8ab16f9d031c9ed14906ebe5159f2b0de8eab ci_payload/digital-surface-live-readiness/execution-diagnostics.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/execution-diagnostics.patch + echo '8606cc925aa3b3a6417abb1fba9a11df916cc2b281880978fb6229aa25875be9 ci_payload/digital-surface-live-readiness/book-arrival-model.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/book-arrival-model.patch + cp -a ci_payload/digital-surface-live-readiness/shadow-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/prospective-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/observer-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/simulation-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/market-state-source/. source/ + + cat > /tmp/market-state-source.sha256 <<'EOF' + 157594f006173023f5ee88ee2b1b72f5af3eba431288a448aab86a57bdd1949e source/research/digital_surface/binance_public.py + c50d2f58c964e38e2e8a5f30c561cda1ea85c6dbbfc6fa169cca161607874262 source/research/digital_surface/spot_ledger.py + 80e91c2d2214260d9bbf6eb7bf0d736d74581de98ed8ac16bcd2f20ab35d6913 source/tests/digital_surface/test_binance_public.py + f5eeb0c0f5a746f1556c7f4cfe801251327d5b7275edc2af1760a64fd0f9775f source/tests/digital_surface/test_spot_ledger.py + EOF + sed -i 's/^ //' /tmp/market-state-source.sha256 + sha256sum -c /tmp/market-state-source.sha256 + cp /tmp/market-state-source.sha256 source/BINANCE_MARKET_STATE_SOURCE_SHA256.txt + + - name: Install pinned test stack + run: | + python -m pip install --upgrade pip + python -m pip install numpy==2.2.6 pandas==2.2.3 scipy==1.15.3 scikit-learn==1.6.1 pyarrow==19.0.1 requests==2.32.3 pytest==8.3.5 tabulate==0.9.0 + + - name: Compile and pass exactly 71 tests + working-directory: source + shell: bash + run: | + set -euo pipefail + python -m compileall -q research/digital_surface tests/digital_surface + PYTHONPATH=. pytest --collect-only -q tests/digital_surface | tee /tmp/collect.txt + grep -Eq '^71 tests collected' /tmp/collect.txt + PYTHONPATH=. pytest tests/digital_surface -q + + - name: Verify public-only Binance state and tamper-evident ledger + working-directory: source + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import ast + import hashlib + import json + import tempfile + from decimal import Decimal + from pathlib import Path + + from research.digital_surface.binance_public import ( + BINANCE_MARKET_DATA_BASE, + BTCMarketState, + realized_log_volatility, + ) + from research.digital_surface.spot_ledger import SpotStateLedger + + assert BINANCE_MARKET_DATA_BASE == 'https://data-api.binance.vision' + tree = ast.parse(Path('research/digital_surface/binance_public.py').read_text()) + request_methods = { + node.func.attr for node in ast.walk(tree) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) + and getattr(node.func.value, 'id', None) == 'requests' + } + assert request_methods == {'get'}, request_methods + text = Path('research/digital_surface/binance_public.py').read_text() + assert 'api/v3/order' not in text + assert 'apiKey' not in text and 'signature' not in text + + closes = [Decimal('100') * (Decimal('1.0001') ** index) for index in range(121)] + assert realized_log_volatility(closes, returns=30) >= 0 + assert realized_log_volatility(closes, returns=120) >= 0 + + state = BTCMarketState( + symbol='BTCUSDT', + server_time_ms=2_000, + observed_ts_ms=2_100, + spot=Decimal('100.50'), + vol_30s=0.001, + vol_120s=0.002, + strikes={1_800_000: Decimal('99.00')}, + closed_one_second_bars=121, + raw_response_sha256={ + 'server_time': 'a' * 64, + 'one_second_klines': 'b' * 64, + 'strike:1800000': 'c' * 64, + }, + ) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'btc_state.jsonl' + ledger = SpotStateLedger(path) + record = ledger.append( + state, + policy_sha256='1' * 64, + source_sha256='2' * 64, + prospective_head_sha256='3' * 64, + ) + assert record['state']['closed_one_second_bars'] == 121 + assert record['state']['strikes'] == {'1800000': '99.00'} + assert SpotStateLedger(path).head_hash == record['record_hash'] + + verified = {} + for line in Path('BINANCE_MARKET_STATE_SOURCE_SHA256.txt').read_text().splitlines(): + wanted, name = line.split(maxsplit=1) + got = hashlib.sha256(Path(name).read_bytes()).hexdigest() + assert got == wanted, (name, got, wanted) + verified[name] = got + result = { + 'tests_collected': 71, + 'binance_market_data_base': BINANCE_MARKET_DATA_BASE, + 'symbol': 'BTCUSDT', + 'authenticated_requests': 0, + 'order_submissions': 0, + 'transport_methods': ['GET'], + 'closed_one_second_bars_required': 121, + 'exact_minute_boundary_strikes_required': True, + 'spot_state_hash_chain_verified': True, + 'live_mode': 'physically_disabled', + 'source_files_verified': verified, + } + Path('BINANCE_MARKET_STATE_VERIFICATION.json').write_text(json.dumps(result, indent=2, sort_keys=True)) + print(json.dumps(result, indent=2)) + PY + + - name: Build deterministic source checkpoint + working-directory: source + shell: bash + run: | + set -euo pipefail + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-binance-market-state-source-v1.tar.gz \ + research tests BINANCE_MARKET_STATE_SOURCE_SHA256.txt BINANCE_MARKET_STATE_VERIFICATION.json + sha256sum digital-surface-binance-market-state-source-v1.tar.gz > digital-surface-binance-market-state-source-v1.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: digital-surface-binance-market-state-checkpoint-v1 + path: | + source/BINANCE_MARKET_STATE_SOURCE_SHA256.txt + source/BINANCE_MARKET_STATE_VERIFICATION.json + source/digital-surface-binance-market-state-source-v1.tar.gz + source/digital-surface-binance-market-state-source-v1.tar.gz.sha256 + retention-days: 30 diff --git a/.github/workflows/digital-surface-binance-state-observer-checkpoint.yml b/.github/workflows/digital-surface-binance-state-observer-checkpoint.yml new file mode 100644 index 00000000..caae6297 --- /dev/null +++ b/.github/workflows/digital-surface-binance-state-observer-checkpoint.yml @@ -0,0 +1,192 @@ +name: Digital Surface Binance State Observer Checkpoint + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/digital-surface-binance-state-observer-checkpoint.yml + - ci_payload/digital-surface-live-readiness/market-state-observer-source/** + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-binance-state-observer-checkpoint.yml + - ci_payload/digital-surface-live-readiness/market-state-observer-source/** + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + verify-binance-state-observer: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct checksum-pinned source + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source && tar -xzf base.tar.gz -C source + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + for patch_file in timestamp-unit-fix.patch full-universe-gate-fix.patch resolved-universe-fix.patch asset-merge-fix.patch; do + patch -d source -p1 < "ci_payload/digital-surface-official-resolution-patch/${patch_file}" + done + echo 'f71f3a653ca58b4bf98609fc29d8ab16f9d031c9ed14906ebe5159f2b0de8eab ci_payload/digital-surface-live-readiness/execution-diagnostics.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/execution-diagnostics.patch + echo '8606cc925aa3b3a6417abb1fba9a11df916cc2b281880978fb6229aa25875be9 ci_payload/digital-surface-live-readiness/book-arrival-model.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/book-arrival-model.patch + cp -a ci_payload/digital-surface-live-readiness/shadow-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/prospective-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/observer-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/simulation-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/market-state-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/market-state-observer-source/. source/ + + cat > /tmp/state-observer.sha256 <<'EOF' + 157594f006173023f5ee88ee2b1b72f5af3eba431288a448aab86a57bdd1949e source/research/digital_surface/binance_public.py + c50d2f58c964e38e2e8a5f30c561cda1ea85c6dbbfc6fa169cca161607874262 source/research/digital_surface/spot_ledger.py + ead39cd8014cccb0d3036c9d37f292606121a4650bff7429fed873e8417eee10 source/research/digital_surface/public_state_observer.py + 80e91c2d2214260d9bbf6eb7bf0d736d74581de98ed8ac16bcd2f20ab35d6913 source/tests/digital_surface/test_binance_public.py + f5eeb0c0f5a746f1556c7f4cfe801251327d5b7275edc2af1760a64fd0f9775f source/tests/digital_surface/test_spot_ledger.py + 9c1eed5c878ba4b351b3bff5528468c49a8b2b8994ac7a9030477e3a97bbaf80 source/tests/digital_surface/test_public_state_observer.py + EOF + sed -i 's/^ //' /tmp/state-observer.sha256 + sha256sum -c /tmp/state-observer.sha256 + cp /tmp/state-observer.sha256 source/BINANCE_STATE_OBSERVER_SOURCE_SHA256.txt + + - name: Install pinned test stack + run: | + python -m pip install --upgrade pip + python -m pip install numpy==2.2.6 pandas==2.2.3 scipy==1.15.3 scikit-learn==1.6.1 pyarrow==19.0.1 requests==2.32.3 pytest==8.3.5 tabulate==0.9.0 + + - name: Compile and pass exactly 73 tests + working-directory: source + shell: bash + run: | + set -euo pipefail + python -m compileall -q research/digital_surface tests/digital_surface + PYTHONPATH=. pytest --collect-only -q tests/digital_surface | tee /tmp/collect.txt + grep -Eq '^73 tests collected' /tmp/collect.txt + PYTHONPATH=. pytest tests/digital_surface -q + + - name: Verify prospective boundary selection and state chaining + working-directory: source + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import hashlib + import json + import tempfile + from decimal import Decimal + from pathlib import Path + + from research.digital_surface.binance_public import BTCMarketState + from research.digital_surface.market_discovery import GammaMarketRecord + from research.digital_surface.public_state_observer import observe_public_btc_state + from research.digital_surface.spot_ledger import SpotStateLedger + + markets = [ + GammaMarketRecord( + condition_id='registered-5', slug='btc-updown-5m-1800', question='Bitcoin Up or Down', + yes_token_id='yes-5', no_token_id='no-5', duration_seconds=300, + open_epoch_seconds=1800, end_date='', tick_size=Decimal('0.01'), + ), + GammaMarketRecord( + condition_id='registered-15', slug='btc-updown-15m-1800', question='Bitcoin Up or Down', + yes_token_id='yes-15', no_token_id='no-15', duration_seconds=900, + open_epoch_seconds=1800, end_date='', tick_size=Decimal('0.01'), + ), + GammaMarketRecord( + condition_id='unregistered', slug='btc-updown-5m-2100', question='Bitcoin Up or Down', + yes_token_id='yes-u', no_token_id='no-u', duration_seconds=300, + open_epoch_seconds=2100, end_date='', tick_size=Decimal('0.01'), + ), + ] + calls = [] + def collect(boundaries, *, observed_ts_ms): + calls.append(list(boundaries)) + return BTCMarketState( + symbol='BTCUSDT', server_time_ms=5_000, observed_ts_ms=observed_ts_ms, + spot=Decimal('100.50'), vol_30s=0.001, vol_120s=0.002, + strikes={1_800_000: Decimal('99.00')}, closed_one_second_bars=121, + raw_response_sha256={ + 'server_time': 'a'*64, + 'one_second_klines': 'b'*64, + 'strike:1800000': 'c'*64, + }, + ) + with tempfile.TemporaryDirectory() as directory: + ledger = SpotStateLedger(Path(directory) / 'btc_state.jsonl') + summary = observe_public_btc_state( + markets, + prospective_market_ids={'registered-5', 'registered-15'}, + prospective_head_sha256='3'*64, + ledger=ledger, + observed_ts_ms=5_100, + policy_sha256='1'*64, + source_sha256='2'*64, + collect=collect, + ) + assert calls == [[1_800_000]] + assert summary['markets_registered'] == 2 + assert summary['markets_skipped_unregistered'] == 1 + assert summary['strike_boundaries'] == [1_800_000] + assert summary['authenticated_requests'] == 0 + assert summary['order_submissions'] == 0 + assert SpotStateLedger(ledger.path).head_hash == summary['spot_state_head_sha256'] + + verified = {} + for line in Path('BINANCE_STATE_OBSERVER_SOURCE_SHA256.txt').read_text().splitlines(): + wanted, name = line.split(maxsplit=1) + got = hashlib.sha256(Path(name).read_bytes()).hexdigest() + assert got == wanted, (name, got, wanted) + verified[name] = got + result = { + 'tests_collected': 73, + 'registered_market_boundaries_only': True, + 'deduplicated_strike_boundaries': True, + 'prospective_head_binding': True, + 'authenticated_requests': 0, + 'order_submissions': 0, + 'live_mode': 'physically_disabled', + 'source_files_verified': verified, + } + Path('BINANCE_STATE_OBSERVER_VERIFICATION.json').write_text(json.dumps(result, indent=2, sort_keys=True)) + print(json.dumps(result, indent=2)) + PY + + - name: Build deterministic source checkpoint + working-directory: source + shell: bash + run: | + set -euo pipefail + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-binance-state-observer-source-v1.tar.gz \ + research tests BINANCE_STATE_OBSERVER_SOURCE_SHA256.txt BINANCE_STATE_OBSERVER_VERIFICATION.json + sha256sum digital-surface-binance-state-observer-source-v1.tar.gz > digital-surface-binance-state-observer-source-v1.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: digital-surface-binance-state-observer-checkpoint-v1 + path: | + source/BINANCE_STATE_OBSERVER_SOURCE_SHA256.txt + source/BINANCE_STATE_OBSERVER_VERIFICATION.json + source/digital-surface-binance-state-observer-source-v1.tar.gz + source/digital-surface-binance-state-observer-source-v1.tar.gz.sha256 + retention-days: 30 diff --git a/.github/workflows/digital-surface-book-arrival-comparison.yml b/.github/workflows/digital-surface-book-arrival-comparison.yml new file mode 100644 index 00000000..90530309 --- /dev/null +++ b/.github/workflows/digital-surface-book-arrival-comparison.yml @@ -0,0 +1,170 @@ +name: Digital Surface Book-Arrival Comparison + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/digital-surface-book-arrival-comparison.yml + - ci_payload/digital-surface-live-readiness/book-arrival-model.patch + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-book-arrival-comparison.yml + - ci_payload/digital-surface-live-readiness/book-arrival-model.patch + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + compare-fill-models: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct checksum-pinned source + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source && tar -xzf base.tar.gz -C source + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + for patch_file in timestamp-unit-fix.patch full-universe-gate-fix.patch resolved-universe-fix.patch asset-merge-fix.patch; do + patch -d source -p1 < "ci_payload/digital-surface-official-resolution-patch/${patch_file}" + done + echo 'f71f3a653ca58b4bf98609fc29d8ab16f9d031c9ed14906ebe5159f2b0de8eab ci_payload/digital-surface-live-readiness/execution-diagnostics.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/execution-diagnostics.patch + echo '8606cc925aa3b3a6417abb1fba9a11df916cc2b281880978fb6229aa25875be9 ci_payload/digital-surface-live-readiness/book-arrival-model.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/book-arrival-model.patch + cat > /tmp/source.sha256 <<'EOF' + 0e858f150d858f9b9ea2e6a4509d74eed7c27d9e864ac6e2bba7d89756102599 source/research/digital_surface/diagnostics.py + 15171104e206f58587ceeff169bf6f9fdf014bfacee4c271217f2f4bc68effd8 source/research/digital_surface/execution.py + ae0dd4e5943ae1c3fd1ea6baf741fe182cba4a3e36d47ecde73c9fb630aa57f4 source/research/digital_surface/run_research.py + b5daad831aadfb67c9fd51182d0ff23add6a560e14ffcb90bca4d395cf730da6 source/tests/digital_surface/test_diagnostics.py + c2d7a06e349ea26ebb1ba540d47974fa7741b1ab85e65d02b3d0a6948a9c26fa source/tests/digital_surface/test_execution.py + 5a1825a337daba295b32fd2a942f6a5a7e14a6061cd5eb33f68a4b241551b6a5 source/tests/digital_surface/test_integration.py + EOF + sed -i 's/^ //' /tmp/source.sha256 + sha256sum -c /tmp/source.sha256 + cp /tmp/source.sha256 source/BOOK_ARRIVAL_COMPARISON_SOURCE_SHA256.txt + + - uses: actions/cache@v4 + with: + path: source/.cache/digital-surface + key: digital-surface-official-resolution-obadiaha-11793901-kinzik-eb4e9fc7-v5 + + - name: Install pinned stack + run: | + python -m pip install --upgrade pip + python -m pip install numpy==2.2.6 pandas==2.2.3 scipy==1.15.3 scikit-learn==1.6.1 pyarrow==19.0.1 requests==2.32.3 pytest==8.3.5 tabulate==0.9.0 + + - name: Pass exactly 32 tests + working-directory: source + shell: bash + run: | + set -euo pipefail + python -m compileall -q research/digital_surface tests/digital_surface + PYTHONPATH=. pytest --collect-only -q tests/digital_surface | tee /tmp/collect.txt + grep -Eq '^32 tests collected' /tmp/collect.txt + PYTHONPATH=. pytest tests/digital_surface -q + + - name: Replay both fill models + working-directory: source + shell: bash + run: | + set -euo pipefail + PYTHONPATH=. python -m research.digital_surface.run_research \ + --cache-dir .cache/digital-surface \ + --output-dir artifacts/digital-surface-book-arrival-comparison \ + 2>&1 | tee digital-surface-book-arrival-comparison.log + + - name: Verify comparison + working-directory: source + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import hashlib, json + from pathlib import Path + import pandas as pd + root = Path('artifacts/digital-surface-book-arrival-comparison') + assert hashlib.sha256((root/'fold_results.csv').read_bytes()).hexdigest() == 'c4e51189ee942929c9764012501da9580de2fa8b4f983c38cee593707df5c0d7' + assert hashlib.sha256((root/'admission.json').read_bytes()).hexdigest() == '9912874614b9558eefc48e1a69507b6bad14a4e13004134d7f785f239907371a' + tape = pd.read_parquet(root/'execution_ledger.parquet') + book = pd.read_parquet(root/'book_execution_ledger.parquet') + folds = pd.read_csv(root/'book_fold_results.csv').sort_values('fold_id') + expected = [11, 17, 29, 5] + assert folds['atomic_count'].astype(int).tolist() == expected, folds + assert int(tape['atomic'].astype(bool).sum()) == 8 + assert int(book['atomic'].astype(bool).sum()) == 62 + assert set(tape['fill_model']) == {'tape_confirmed'} + assert set(book['fill_model']) == {'book_arrival'} + admission = json.loads((root/'admission.json').read_text()) + assert admission['classification'] == 'Rejected' + internal = json.loads((root/'SHA256SUMS.json').read_text()) + for name, wanted in internal.items(): + got = hashlib.sha256((root/name).read_bytes()).hexdigest() + assert got == wanted, (name, got, wanted) + result = { + 'admission_model': 'tape_confirmed', + 'admission_classification': admission['classification'], + 'tape_confirmed_atomic': 8, + 'book_arrival_atomic': 62, + 'book_arrival_atomic_by_fold': expected, + 'book_arrival_fold_results': folds.to_dict('records'), + 'book_arrival_is_diagnostic_only': True, + 'prospective_shadow_calibration_required': True, + 'live_mode': 'disabled', + 'internal_checksums_verified': len(internal), + } + (root/'BOOK_ARRIVAL_COMPARISON_VERIFICATION.json').write_text(json.dumps(result, indent=2, sort_keys=True)) + print(json.dumps(result, indent=2)) + PY + + - name: Build deterministic archive + if: always() + working-directory: source + shell: bash + run: | + set -euo pipefail + if test -d artifacts/digital-surface-book-arrival-comparison; then + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-book-arrival-comparison-v1.tar.gz \ + -C artifacts digital-surface-book-arrival-comparison + sha256sum digital-surface-book-arrival-comparison-v1.tar.gz > digital-surface-book-arrival-comparison-v1.tar.gz.sha256 + fi + + - name: Show comparison + if: always() + working-directory: source + run: | + cat artifacts/digital-surface-book-arrival-comparison/BOOK_ARRIVAL_COMPARISON_VERIFICATION.json || true + cat artifacts/digital-surface-book-arrival-comparison/book_fold_results.csv || true + tail -200 digital-surface-book-arrival-comparison.log || true + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: digital-surface-book-arrival-comparison-v1 + path: | + source/artifacts/digital-surface-book-arrival-comparison/ + source/digital-surface-book-arrival-comparison.log + source/digital-surface-book-arrival-comparison-v1.tar.gz + source/digital-surface-book-arrival-comparison-v1.tar.gz.sha256 + source/BOOK_ARRIVAL_COMPARISON_SOURCE_SHA256.txt + if-no-files-found: warn + retention-days: 30 diff --git a/.github/workflows/digital-surface-execution-diagnostics-verify-v2.yml b/.github/workflows/digital-surface-execution-diagnostics-verify-v2.yml new file mode 100644 index 00000000..f7ff966f --- /dev/null +++ b/.github/workflows/digital-surface-execution-diagnostics-verify-v2.yml @@ -0,0 +1,127 @@ +name: Verify Digital Surface Execution Diagnostics v2 + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/digital-surface-execution-diagnostics-verify-v2.yml + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-execution-diagnostics-verify-v2.yml + workflow_dispatch: + +permissions: + contents: read + actions: read + +env: + BASELINE_ARTIFACT_ID: '8615107471' + DIAGNOSTIC_ARTIFACT_ID: '8617727219' + BASELINE_ZIP_SHA256: '479a25e235841d94077b3d2df070c8ed46bb736c7ebd8f9f1bde8016198a9c39' + DIAGNOSTIC_ZIP_SHA256: '80a442cded15e26703c401e197a53db28ae59b61efa6dcb808a0d65ead961293' + +jobs: + verify-artifacts: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - run: python -m pip install pandas==2.2.3 pyarrow==19.0.1 numpy==2.2.6 + + - name: Download immutable artifacts + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + for pair in "baseline:${BASELINE_ARTIFACT_ID}" "diagnostic:${DIAGNOSTIC_ARTIFACT_ID}"; do + name=${pair%%:*}; id=${pair##*:} + curl --fail --location --silent --show-error \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H 'Accept: application/vnd.github+json' \ + -H 'X-GitHub-Api-Version: 2022-11-28' \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/artifacts/${id}/zip" \ + -o "${name}.zip" + mkdir "${name}" && unzip -q "${name}.zip" -d "${name}" + done + echo "${BASELINE_ZIP_SHA256} baseline.zip" | sha256sum -c - + echo "${DIAGNOSTIC_ZIP_SHA256} diagnostic.zip" | sha256sum -c - + + - name: Verify semantic invariance and diagnostics + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import csv, hashlib, json + from collections import Counter + from pathlib import Path + import numpy as np + import pandas as pd + + baseline = Path('baseline/artifacts/digital-surface-arbitrage') + current = Path('diagnostic/artifacts/digital-surface-execution-diagnostics') + left = pd.read_parquet(baseline / 'candidate_ledger.parquet') + right = pd.read_parquet(current / 'candidate_ledger.parquet') + assert left.shape == right.shape + assert left.columns.tolist() == right.columns.tolist() + + numeric_report = {} + for column in left.columns: + if pd.api.types.is_numeric_dtype(left[column]): + a = left[column].to_numpy(dtype=float) + b = right[column].to_numpy(dtype=float) + finite = np.isfinite(a) & np.isfinite(b) + max_abs = float(np.max(np.abs(a[finite] - b[finite]))) if finite.any() else 0.0 + numeric_report[column] = max_abs + assert np.allclose(a, b, rtol=0.0, atol=1e-12, equal_nan=True), (column, max_abs) + else: + assert left[column].fillna('').astype(str).equals(right[column].fillna('').astype(str)), column + + exact_files = ['execution_ledger.parquet', 'fold_results.csv', 'admission.json'] + for name in exact_files: + assert (baseline / name).read_bytes() == (current / name).read_bytes(), name + + rows = list(csv.DictReader((current / 'execution_failure_attribution.csv').open())) + reasons = Counter(row['reason'] for row in rows) + expected = Counter({ + 'insufficient_post_arrival_tape': 186, + 'ask_above_limit': 114, + 'missing_post_arrival_book': 110, + 'filled': 34, + 'insufficient_displayed_depth': 4, + }) + assert len(rows) == 448 + assert reasons == expected, (reasons, expected) + + internal = json.loads((current / 'SHA256SUMS.json').read_text()) + for name, wanted in internal.items(): + got = hashlib.sha256((current / name).read_bytes()).hexdigest() + assert got == wanted, (name, got, wanted) + + sidecar = Path('diagnostic/digital-surface-execution-diagnostics-v1.tar.gz.sha256').read_text().split()[0] + tar_sha = hashlib.sha256(Path('diagnostic/digital-surface-execution-diagnostics-v1.tar.gz').read_bytes()).hexdigest() + assert tar_sha == sidecar + + result = { + 'candidate_semantics_equal_at_1e-12': True, + 'maximum_numeric_difference': max(numeric_report.values(), default=0.0), + 'numeric_column_max_abs_differences': numeric_report, + 'exact_downstream_files': exact_files, + 'reason_counts': dict(expected), + 'internal_checksums_verified': len(internal), + 'diagnostic_tar_sha256': tar_sha, + 'live_mode': 'disabled', + } + Path('DIAGNOSTIC_ARTIFACT_VERIFICATION_V2.json').write_text(json.dumps(result, indent=2, sort_keys=True)) + print(json.dumps(result, indent=2)) + PY + + - uses: actions/upload-artifact@v4 + with: + name: digital-surface-execution-diagnostic-verification-v2 + path: DIAGNOSTIC_ARTIFACT_VERIFICATION_V2.json + retention-days: 30 diff --git a/.github/workflows/digital-surface-execution-diagnostics-verify.yml b/.github/workflows/digital-surface-execution-diagnostics-verify.yml new file mode 100644 index 00000000..b2e6de3c --- /dev/null +++ b/.github/workflows/digital-surface-execution-diagnostics-verify.yml @@ -0,0 +1,143 @@ +name: Verify Digital Surface Execution Diagnostics + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/digital-surface-execution-diagnostics-verify.yml + - ci_payload/digital-surface-live-readiness/DIAGNOSTIC_FINDINGS.md + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-execution-diagnostics-verify.yml + - ci_payload/digital-surface-live-readiness/DIAGNOSTIC_FINDINGS.md + workflow_dispatch: + +permissions: + contents: read + actions: read + +env: + BASELINE_ARTIFACT_ID: '8615107471' + DIAGNOSTIC_ARTIFACT_ID: '8617727219' + BASELINE_ZIP_SHA256: '479a25e235841d94077b3d2df070c8ed46bb736c7ebd8f9f1bde8016198a9c39' + DIAGNOSTIC_ZIP_SHA256: '80a442cded15e26703c401e197a53db28ae59b61efa6dcb808a0d65ead961293' + +jobs: + verify-artifacts: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install pinned readers + run: python -m pip install pandas==2.2.3 pyarrow==19.0.1 + + - name: Download immutable baseline and diagnostic artifacts + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + for pair in "baseline:${BASELINE_ARTIFACT_ID}" "diagnostic:${DIAGNOSTIC_ARTIFACT_ID}"; do + name=${pair%%:*} + id=${pair##*:} + curl --fail --location --silent --show-error \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H 'Accept: application/vnd.github+json' \ + -H 'X-GitHub-Api-Version: 2022-11-28' \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/artifacts/${id}/zip" \ + -o "${name}.zip" + mkdir "${name}" + unzip -q "${name}.zip" -d "${name}" + done + echo "${BASELINE_ZIP_SHA256} baseline.zip" | sha256sum -c - + echo "${DIAGNOSTIC_ZIP_SHA256} diagnostic.zip" | sha256sum -c - + + - name: Verify semantic identity and execution attribution + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import csv + import hashlib + import json + from collections import Counter + from pathlib import Path + + import pandas as pd + + baseline = Path('baseline/artifacts/digital-surface-arbitrage') + current = Path('diagnostic/artifacts/digital-surface-execution-diagnostics') + + baseline_candidates = pd.read_parquet(baseline / 'candidate_ledger.parquet') + current_candidates = pd.read_parquet(current / 'candidate_ledger.parquet') + pd.testing.assert_frame_equal( + baseline_candidates, + current_candidates, + check_exact=True, + check_dtype=True, + check_like=False, + ) + + exact_files = { + 'execution_ledger.parquet': 'cf6733acd95cf3e474724258c88bd4fa2e8987c203e3f5a81f6c9ab268601ae0', + 'fold_results.csv': 'c4e51189ee942929c9764012501da9580de2fa8b4f983c38cee593707df5c0d7', + 'admission.json': '9912874614b9558eefc48e1a69507b6bad14a4e13004134d7f785f239907371a', + } + for name, expected in exact_files.items(): + actual = hashlib.sha256((current / name).read_bytes()).hexdigest() + assert actual == expected, (name, actual, expected) + assert (current / name).read_bytes() == (baseline / name).read_bytes(), name + + rows = list(csv.DictReader((current / 'execution_failure_attribution.csv').open())) + candidate_ids = {str(value) for value in current_candidates['candidate_id']} + assert len(rows) == 2 * len(candidate_ids) == 448 + assert Counter(row['candidate_id'] for row in rows) == Counter({value: 2 for value in candidate_ids}) + + reasons = Counter(row['reason'] for row in rows) + expected_reasons = { + 'insufficient_post_arrival_tape': 186, + 'ask_above_limit': 114, + 'missing_post_arrival_book': 110, + 'filled': 34, + 'insufficient_displayed_depth': 4, + } + assert dict(reasons) == expected_reasons, (reasons, expected_reasons) + + summary = json.loads((current / 'EXECUTION_DIAGNOSTICS.json').read_text()) + assert summary['complete_attribution'] is True + assert summary['candidates_attributed'] == 224 + assert summary['legs_attributed'] == 448 + + internal = json.loads((current / 'SHA256SUMS.json').read_text()) + for name, expected in internal.items(): + actual = hashlib.sha256((current / name).read_bytes()).hexdigest() + assert actual == expected, (name, actual, expected) + + sidecar = Path('diagnostic/digital-surface-execution-diagnostics-v1.tar.gz.sha256').read_text().split()[0] + tar_digest = hashlib.sha256(Path('diagnostic/digital-surface-execution-diagnostics-v1.tar.gz').read_bytes()).hexdigest() + assert tar_digest == sidecar, (tar_digest, sidecar) + + result = { + 'baseline_candidates_semantically_identical': True, + 'exact_downstream_outputs_identical': sorted(exact_files), + 'candidates_attributed': 224, + 'legs_attributed': 448, + 'reason_counts': expected_reasons, + 'internal_checksums_verified': len(internal), + 'diagnostic_tar_sha256': tar_digest, + 'live_mode': 'disabled', + } + Path('DIAGNOSTIC_ARTIFACT_VERIFICATION.json').write_text(json.dumps(result, indent=2, sort_keys=True)) + print(json.dumps(result, indent=2)) + PY + + - uses: actions/upload-artifact@v4 + with: + name: digital-surface-execution-diagnostic-verification + path: DIAGNOSTIC_ARTIFACT_VERIFICATION.json + retention-days: 30 diff --git a/.github/workflows/digital-surface-execution-diagnostics.yml b/.github/workflows/digital-surface-execution-diagnostics.yml new file mode 100644 index 00000000..f92499bd --- /dev/null +++ b/.github/workflows/digital-surface-execution-diagnostics.yml @@ -0,0 +1,197 @@ +name: Digital Surface Execution Diagnostics + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/digital-surface-execution-diagnostics.yml + - ci_payload/digital-surface-live-readiness/** + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-execution-diagnostics.yml + - ci_payload/digital-surface-live-readiness/** + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + diagnose-execution: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct checksum-pinned v5 source + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source + tar -xzf base.tar.gz -C source + + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + patch -d source -p1 < ci_payload/digital-surface-official-resolution-patch/timestamp-unit-fix.patch + patch -d source -p1 < ci_payload/digital-surface-official-resolution-patch/full-universe-gate-fix.patch + patch -d source -p1 < ci_payload/digital-surface-official-resolution-patch/resolved-universe-fix.patch + patch -d source -p1 < ci_payload/digital-surface-official-resolution-patch/asset-merge-fix.patch + + echo 'f71f3a653ca58b4bf98609fc29d8ab16f9d031c9ed14906ebe5159f2b0de8eab ci_payload/digital-surface-live-readiness/execution-diagnostics.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/execution-diagnostics.patch + + cat > /tmp/diagnostics-source.sha256 <<'EOF' + 0e858f150d858f9b9ea2e6a4509d74eed7c27d9e864ac6e2bba7d89756102599 source/research/digital_surface/diagnostics.py + 5343101ab3f3a097f86b3edc3c3e8a4ce340feeb3eb1c5d810ec98dbd4cf9f72 source/research/digital_surface/run_research.py + b5daad831aadfb67c9fd51182d0ff23add6a560e14ffcb90bca4d395cf730da6 source/tests/digital_surface/test_diagnostics.py + c25a78f5d3a34977e1bbf21a6a3c883bcd1ec543cde5ae94104e292f4266244a source/tests/digital_surface/test_integration.py + EOF + sed -i 's/^ //' /tmp/diagnostics-source.sha256 + sha256sum -c /tmp/diagnostics-source.sha256 + cp /tmp/diagnostics-source.sha256 source/EXECUTION_DIAGNOSTICS_SOURCE_SHA256.txt + + - name: Cache immutable source payloads + uses: actions/cache@v4 + with: + path: source/.cache/digital-surface + key: digital-surface-official-resolution-obadiaha-11793901-kinzik-eb4e9fc7-v5 + + - name: Install pinned research stack + run: | + python -m pip install --upgrade pip + python -m pip install numpy==2.2.6 pandas==2.2.3 scipy==1.15.3 scikit-learn==1.6.1 pyarrow==19.0.1 requests==2.32.3 pytest==8.3.5 tabulate==0.9.0 + + - name: Compile and pass exactly 30 tests + working-directory: source + shell: bash + run: | + set -euo pipefail + python -m compileall -q research/digital_surface tests/digital_surface + PYTHONPATH=. pytest --collect-only -q tests/digital_surface | tee /tmp/collect.txt + grep -Eq '^30 tests collected' /tmp/collect.txt + PYTHONPATH=. pytest tests/digital_surface -q + + - name: Replay immutable corpus with attribution only + working-directory: source + shell: bash + run: | + set -euo pipefail + PYTHONPATH=. python -m research.digital_surface.run_research \ + --cache-dir .cache/digital-surface \ + --output-dir artifacts/digital-surface-execution-diagnostics \ + 2>&1 | tee digital-surface-execution-diagnostics.log + + - name: Verify complete attribution and unchanged strategy outputs + working-directory: source + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import hashlib + import json + from pathlib import Path + import pandas as pd + + root = Path('artifacts/digital-surface-execution-diagnostics') + candidates = pd.read_parquet(root / 'candidate_ledger.parquet') + executions = pd.read_parquet(root / 'execution_ledger.parquet') + attribution = pd.read_csv(root / 'execution_failure_attribution.csv') + summary = json.loads((root / 'EXECUTION_DIAGNOSTICS.json').read_text()) + + assert len(candidates) == len(executions), (len(candidates), len(executions)) + assert len(attribution) == 2 * len(candidates), (len(attribution), len(candidates)) + assert attribution['candidate_id'].value_counts().eq(2).all() + assert set(attribution['candidate_id']) == set(candidates['candidate_id'].astype(str)) + allowed = { + 'missing_post_arrival_book', + 'ask_above_limit', + 'insufficient_displayed_depth', + 'insufficient_post_arrival_tape', + 'filled', + } + assert set(attribution['reason']).issubset(allowed), set(attribution['reason']) + assert summary['complete_attribution'] is True, summary + assert summary['candidates_attributed'] == len(candidates), summary + assert summary['legs_attributed'] == len(attribution), summary + + unchanged = { + 'candidate_ledger.parquet': 'e9eecb4f9f2ba56d1ccec915d44ccf4eb73dde50495729e69cf79658176c434a', + 'execution_ledger.parquet': 'cf6733acd95cf3e474724258c88bd4fa2e8987c203e3f5a81f6c9ab268601ae0', + 'fold_results.csv': 'c4e51189ee942929c9764012501da9580de2fa8b4f983c38cee593707df5c0d7', + 'admission.json': '9912874614b9558eefc48e1a69507b6bad14a4e13004134d7f785f239907371a', + } + for name, want in unchanged.items(): + got = hashlib.sha256((root / name).read_bytes()).hexdigest() + assert got == want, (name, got, want) + + folds = pd.read_csv(root / 'fold_results.csv') + admission = json.loads((root / 'admission.json').read_text()) + assert len(folds) == 4 + assert admission['classification'] == 'Rejected' + + reason_totals = attribution.groupby(['leg', 'reason']).size().rename('count').reset_index() + reason_totals.to_csv(root / 'execution_reason_totals.csv', index=False) + diagnostic = { + 'classification_unchanged': admission['classification'], + 'candidate_count': int(len(candidates)), + 'leg_count': int(len(attribution)), + 'atomic_count': int(executions['atomic'].astype(bool).sum()), + 'reason_totals': reason_totals.to_dict('records'), + } + (root / 'EXECUTION_DIAGNOSTIC_VERIFICATION.json').write_text( + json.dumps(diagnostic, indent=2, sort_keys=True) + ) + print(json.dumps(diagnostic, indent=2)) + + expected = json.loads((root / 'SHA256SUMS.json').read_text()) + for name, want in expected.items(): + got = hashlib.sha256((root / name).read_bytes()).hexdigest() + assert got == want, (name, got, want) + PY + + - name: Build deterministic diagnostics archive + if: always() + working-directory: source + shell: bash + run: | + set -euo pipefail + if test -d artifacts/digital-surface-execution-diagnostics; then + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-execution-diagnostics-v1.tar.gz \ + -C artifacts digital-surface-execution-diagnostics + sha256sum digital-surface-execution-diagnostics-v1.tar.gz > digital-surface-execution-diagnostics-v1.tar.gz.sha256 + fi + + - name: Show attribution summary + if: always() + working-directory: source + run: | + cat artifacts/digital-surface-execution-diagnostics/EXECUTION_DIAGNOSTIC_VERIFICATION.json || true + cat artifacts/digital-surface-execution-diagnostics/EXECUTION_DIAGNOSTICS.json || true + tail -200 digital-surface-execution-diagnostics.log || true + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: digital-surface-execution-diagnostics-v1 + path: | + source/artifacts/digital-surface-execution-diagnostics/ + source/digital-surface-execution-diagnostics.log + source/digital-surface-execution-diagnostics-v1.tar.gz + source/digital-surface-execution-diagnostics-v1.tar.gz.sha256 + source/EXECUTION_DIAGNOSTICS_SOURCE_SHA256.txt + if-no-files-found: warn + retention-days: 30 diff --git a/.github/workflows/digital-surface-freeze-prospective-policy.yml b/.github/workflows/digital-surface-freeze-prospective-policy.yml new file mode 100644 index 00000000..e513628d --- /dev/null +++ b/.github/workflows/digital-surface-freeze-prospective-policy.yml @@ -0,0 +1,205 @@ +name: Digital Surface Freeze Prospective Policy + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/digital-surface-freeze-prospective-policy.yml + - ci_payload/digital-surface-live-readiness/prospective-signal-source/** + workflow_dispatch: + +permissions: + contents: read + actions: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + V5_ARTIFACT_ID: '8615107471' + V5_ARTIFACT_ZIP_SHA256: '479a25e235841d94077b3d2df070c8ed46bb736c7ebd8f9f1bde8016198a9c39' + +jobs: + freeze-policy: + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct checksum-pinned v5 source + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source && tar -xzf base.tar.gz -C source + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + for patch_file in timestamp-unit-fix.patch full-universe-gate-fix.patch resolved-universe-fix.patch asset-merge-fix.patch; do + patch -d source -p1 < "ci_payload/digital-surface-official-resolution-patch/${patch_file}" + done + cp -a ci_payload/digital-surface-live-readiness/prospective-signal-source/. source/ + + - name: Install pinned research stack + run: | + python -m pip install --upgrade pip + python -m pip install numpy==2.2.6 pandas==2.2.3 scipy==1.15.3 scikit-learn==1.6.1 pyarrow==19.0.1 requests==2.32.3 pytest==8.3.5 tabulate==0.9.0 + + - name: Download and verify v5 research artifact + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + curl --fail --location --silent --show-error \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H 'Accept: application/vnd.github+json' \ + -H 'X-GitHub-Api-Version: 2022-11-28' \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/artifacts/${V5_ARTIFACT_ID}/zip" \ + -o v5.zip + echo "${V5_ARTIFACT_ZIP_SHA256} v5.zip" | sha256sum -c - + mkdir v5 && unzip -q v5.zip -d v5 + causal=$(find v5 -type f -name causal_panel.parquet -print -quit) + choices=$(find v5 -type f -name validation_choices.json -print -quit) + checksums=$(find v5 -type f -name SHA256SUMS.json -print -quit) + test -n "${causal}" && test -n "${choices}" && test -n "${checksums}" + cp "${causal}" source/causal_panel.parquet + cp "${choices}" source/validation_choices.json + cp "${checksums}" source/V5_SHA256SUMS.json + python - <<'PY' + import hashlib, json + from pathlib import Path + expected = json.loads(Path('source/V5_SHA256SUMS.json').read_text()) + for name in ('causal_panel.parquet', 'validation_choices.json'): + got = hashlib.sha256(Path('source', name).read_bytes()).hexdigest() + assert got == expected[name], (name, got, expected[name]) + PY + + - name: Fit and freeze deterministic policy + working-directory: source + shell: bash + run: | + set -euo pipefail + PYTHONPATH=. python - <<'PY' + import hashlib + import json + from pathlib import Path + + import numpy as np + import pandas as pd + + from research.digital_surface.prospective_signal import load_frozen_surface_policy + from research.digital_surface.run_research import _fit_surface + + causal_path = Path('causal_panel.parquet') + choices_path = Path('validation_choices.json') + state = pd.read_parquet(causal_path) + choices = json.loads(choices_path.read_text()) + assert len(choices) == 4, len(choices) + final_choice = max(choices, key=lambda row: int(row['fold_id'])) + assert int(final_choice['fold_id']) == 4 + assert final_choice['validation_weeks'] == ['03-2026-03-16', '04-2026-03-23'], final_choice + assert final_choice['test_week'] == '05-2026-03-30', final_choice + config = dict(final_choice['selected_config']) + assert config['config_id'] == 'e0.0200-c1.2500-tau3-60-cad5-slip0.010-u0.00', config + + model_a = _fit_surface(state) + model_b = _fit_surface(state) + coeff_a = np.asarray(model_a.coefficients, dtype=float) + coeff_b = np.asarray(model_b.coefficients, dtype=float) + assert np.array_equal(coeff_a, coeff_b), (coeff_a, coeff_b) + assert coeff_a.shape == (4,) + assert np.isfinite(coeff_a).all() + assert coeff_a[1] >= 0.05 + + historical_weeks = sorted(state['week'].astype(str).unique().tolist()) + payload = { + 'version': 1, + 'model': { + 'coefficients': [float(value) for value in coeff_a], + 'training_rows': int(model_a.training_rows), + 'training_contracts': int(model_a.training_contracts), + 'fit_scope': 'all_historical_pre_prospective', + 'historical_weeks': historical_weeks, + }, + 'candidate': { + 'min_edge': float(config['min_edge']), + 'max_total_cost': float(config['max_total_cost']), + 'min_tau_s': float(config['min_tau_s']), + 'max_tau_s': float(config['max_tau_s']), + 'decision_cadence_s': int(config['decision_cadence_s']), + 'pair_gap_ms': int(config['pair_gap_ms']), + 'slippage': float(config['slippage']), + 'uncertainty_penalty': float(config['uncertainty_penalty']), + }, + 'selection': 'first_trigger_per_expiry', + 'flow_assumption': 'zero_without_public_causal_trade_flow', + 'provenance': { + 'v5_artifact_id': 8615107471, + 'v5_artifact_zip_sha256': '479a25e235841d94077b3d2df070c8ed46bb736c7ebd8f9f1bde8016198a9c39', + 'causal_panel_sha256': hashlib.sha256(causal_path.read_bytes()).hexdigest(), + 'validation_choices_sha256': hashlib.sha256(choices_path.read_bytes()).hexdigest(), + 'configuration_selected_in_fold': 4, + 'configuration_validation_weeks': final_choice['validation_weeks'], + 'configuration_test_week_excluded_from_selection': final_choice['test_week'], + 'policy_frozen_before_prospective_observation': True, + 'historical_folds_are_diagnostic_only': True, + }, + } + policy_path = Path('frozen_prospective_surface_policy_v1.json') + policy_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + '\n') + policy_sha = hashlib.sha256(policy_path.read_bytes()).hexdigest() + policy = load_frozen_surface_policy(policy_path) + assert policy.policy_sha256 == policy_sha + assert policy.config.config_id == config['config_id'] + assert policy.training_rows == int(model_a.training_rows) + assert policy.training_contracts == int(model_a.training_contracts) + + verification = { + 'policy_sha256': policy_sha, + 'coefficients': payload['model']['coefficients'], + 'training_rows': policy.training_rows, + 'training_contracts': policy.training_contracts, + 'candidate_config_id': policy.config.config_id, + 'configuration_validation_weeks': final_choice['validation_weeks'], + 'configuration_test_week_excluded_from_selection': final_choice['test_week'], + 'historical_weeks': historical_weeks, + 'deterministic_refit_exact': True, + 'future_prospective_rows_used': 0, + 'live_mode': 'physically_disabled', + } + Path('FROZEN_POLICY_VERIFICATION.json').write_text(json.dumps(verification, indent=2, sort_keys=True)) + Path('frozen_prospective_surface_policy_v1.json.sha256').write_text( + policy_sha + ' frozen_prospective_surface_policy_v1.json\n' + ) + print(json.dumps(verification, indent=2)) + PY + + - name: Build deterministic policy artifact + working-directory: source + shell: bash + run: | + set -euo pipefail + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-frozen-prospective-policy-v1.tar.gz \ + frozen_prospective_surface_policy_v1.json \ + frozen_prospective_surface_policy_v1.json.sha256 \ + FROZEN_POLICY_VERIFICATION.json + sha256sum digital-surface-frozen-prospective-policy-v1.tar.gz > digital-surface-frozen-prospective-policy-v1.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: digital-surface-frozen-prospective-policy-v1 + path: | + source/frozen_prospective_surface_policy_v1.json + source/frozen_prospective_surface_policy_v1.json.sha256 + source/FROZEN_POLICY_VERIFICATION.json + source/digital-surface-frozen-prospective-policy-v1.tar.gz + source/digital-surface-frozen-prospective-policy-v1.tar.gz.sha256 + retention-days: 90 diff --git a/.github/workflows/digital-surface-official-resolution-ci-v2.yml b/.github/workflows/digital-surface-official-resolution-ci-v2.yml new file mode 100644 index 00000000..c823fa6c --- /dev/null +++ b/.github/workflows/digital-surface-official-resolution-ci-v2.yml @@ -0,0 +1,173 @@ +name: Digital Surface Official Resolution CI v2 + +on: + push: + branches: [digital-surface-official-resolution-ci-20260724] + paths: + - .github/workflows/digital-surface-official-resolution-ci-v2.yml + - ci_payload/digital-surface-official-resolution-patch/** + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-official-resolution-ci-v2.yml + - ci_payload/digital-surface-official-resolution-patch/** + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + verify-and-research: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct checksum-pinned source + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source + tar -xzf base.tar.gz -C source + + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + patch -d source -p1 < ci_payload/digital-surface-official-resolution-patch/timestamp-unit-fix.patch + + cat > /tmp/patch.sha256 <<'EOF' + 896bef750c9f1510252b695ef10517fa6502419cfce0d66dd9403d8a2696e903 source/research/digital_surface/__init__.py + 36f34ff7ee5c2f82e89b4d8674ce4371aea27ad1a63287ba537a6939d997d728 source/research/digital_surface/obadiaha_official_resolution_snapshot.json + 6fba14ab7e7882333ea3c2c7c5e4da6da1e8c81f015018469efd43ba25c08cde source/research/digital_surface/official_resolution.py + 797d10ac1fa1a181de42cdcd4f75c717755cebc1a0acf197727874c6cb253d40 source/tests/digital_surface/test_data_adapter.py + EOF + sed -i 's/^ //' /tmp/patch.sha256 + sha256sum -c /tmp/patch.sha256 + cp /tmp/patch.sha256 source/OFFICIAL_RESOLUTION_PATCH_SHA256.txt + + - name: Cache immutable source payloads + uses: actions/cache@v4 + with: + path: source/.cache/digital-surface + key: digital-surface-official-resolution-obadiaha-11793901-kinzik-eb4e9fc7-v2 + + - name: Install pinned research stack + run: | + python -m pip install --upgrade pip + python -m pip install numpy==2.2.6 pandas==2.2.3 scipy==1.15.3 scikit-learn==1.6.1 pyarrow==19.0.1 requests==2.32.3 pytest==8.3.5 tabulate==0.9.0 + + - name: Compile and pass exactly 29 tests + working-directory: source + shell: bash + run: | + set -euo pipefail + python -m compileall -q research/digital_surface tests/digital_surface + PYTHONPATH=. pytest --collect-only -q tests/digital_surface | tee /tmp/collect.txt + grep -Eq '^29 tests collected' /tmp/collect.txt + PYTHONPATH=. pytest tests/digital_surface -q + + - name: Run immutable two-validation four-transfer protocol + id: research + working-directory: source + shell: bash + run: | + set -euo pipefail + PYTHONPATH=. python -m research.digital_surface.run_research \ + --cache-dir .cache/digital-surface \ + --output-dir artifacts/digital-surface-arbitrage \ + 2>&1 | tee digital-surface-arbitrage.log + + - name: Verify source gate, four folds, and checksums + working-directory: source + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import hashlib, json + from pathlib import Path + import pandas as pd + + root=Path('artifacts/digital-surface-arbitrage') + audit=json.loads((root/'SOURCE_AUDIT.json').read_text()) + validation=audit['validation'] + assert validation['official_resolution_coverage'] == 1.0, validation + assert validation['official_resolution_agreement'] >= 0.99, validation + assert validation['official_resolution_rows'] == 5228, validation + assert 0.0 <= validation['binance_direction_agreement_diagnostic'] <= 1.0, validation + + folds=pd.read_csv(root/'fold_results.csv') + assert len(folds) == 4, folds + assert folds['fold_id'].nunique() == 4, folds + assert folds['test_week'].nunique() == 4, folds + assert folds['test_week'].astype(str).str.startswith(('02-','03-','04-','05-')).all(), folds + + admission=json.loads((root/'admission.json').read_text()) + assert admission['classification'] in {'Rejected','Exploratory','Admissible'}, admission + expected=json.loads((root/'SHA256SUMS.json').read_text()) + for name,want in expected.items(): + got=hashlib.sha256((root/name).read_bytes()).hexdigest() + assert got == want, (name,got,want) + + verification={ + 'classification': admission['classification'], + 'source_coverage': validation['official_resolution_coverage'], + 'source_agreement': validation['official_resolution_agreement'], + 'required_agreement': 0.99, + 'binance_direction_agreement_diagnostic': validation['binance_direction_agreement_diagnostic'], + 'folds_completed': len(folds), + 'test_weeks': folds['test_week'].astype(str).tolist(), + 'checksums_verified': len(expected), + } + (root/'CI_VERIFICATION.json').write_text(json.dumps(verification,indent=2,sort_keys=True)) + print(json.dumps(verification,indent=2)) + PY + + - name: Build deterministic source and result archives + if: always() + working-directory: source + shell: bash + run: | + set -euo pipefail + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-official-resolution-source-v2.tar.gz \ + .github research tests OFFICIAL_RESOLUTION_PATCH_SHA256.txt + sha256sum digital-surface-official-resolution-source-v2.tar.gz > digital-surface-official-resolution-source-v2.tar.gz.sha256 + if test -d artifacts/digital-surface-arbitrage; then + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-official-resolution-results-v2.tar.gz -C artifacts digital-surface-arbitrage + sha256sum digital-surface-official-resolution-results-v2.tar.gz > digital-surface-official-resolution-results-v2.tar.gz.sha256 + fi + + - name: Show final decision + if: always() + working-directory: source + run: | + cat artifacts/digital-surface-arbitrage/CI_VERIFICATION.json || true + cat artifacts/digital-surface-arbitrage/admission.json || true + tail -200 digital-surface-arbitrage.log || true + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: digital-surface-official-resolution-four-fold-v2 + path: | + source/artifacts/digital-surface-arbitrage/ + source/digital-surface-arbitrage.log + source/digital-surface-official-resolution-source-v2.tar.gz + source/digital-surface-official-resolution-source-v2.tar.gz.sha256 + source/digital-surface-official-resolution-results-v2.tar.gz + source/digital-surface-official-resolution-results-v2.tar.gz.sha256 + if-no-files-found: warn + retention-days: 30 diff --git a/.github/workflows/digital-surface-official-resolution-ci-v3.yml b/.github/workflows/digital-surface-official-resolution-ci-v3.yml new file mode 100644 index 00000000..0911d789 --- /dev/null +++ b/.github/workflows/digital-surface-official-resolution-ci-v3.yml @@ -0,0 +1,174 @@ +name: Digital Surface Official Resolution CI v3 + +on: + push: + branches: [digital-surface-official-resolution-ci-20260724] + paths: + - .github/workflows/digital-surface-official-resolution-ci-v3.yml + - ci_payload/digital-surface-official-resolution-patch/** + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-official-resolution-ci-v3.yml + - ci_payload/digital-surface-official-resolution-patch/** + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + verify-and-research: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct checksum-pinned source + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source + tar -xzf base.tar.gz -C source + + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + patch -d source -p1 < ci_payload/digital-surface-official-resolution-patch/timestamp-unit-fix.patch + patch -d source -p1 < ci_payload/digital-surface-official-resolution-patch/full-universe-gate-fix.patch + + cat > /tmp/patch.sha256 <<'EOF' + 896bef750c9f1510252b695ef10517fa6502419cfce0d66dd9403d8a2696e903 source/research/digital_surface/__init__.py + 36f34ff7ee5c2f82e89b4d8674ce4371aea27ad1a63287ba537a6939d997d728 source/research/digital_surface/obadiaha_official_resolution_snapshot.json + a6813634fd8732f4f2a1a9ee8b59c8f8f906656e3cd84feac8cade761e1b49aa source/research/digital_surface/official_resolution.py + ffda7620f668bf0a1d1315a56230418f516ff476cb1f8f686360104a27e8e9d1 source/tests/digital_surface/test_data_adapter.py + EOF + sed -i 's/^ //' /tmp/patch.sha256 + sha256sum -c /tmp/patch.sha256 + cp /tmp/patch.sha256 source/OFFICIAL_RESOLUTION_PATCH_SHA256.txt + + - name: Cache immutable source payloads + uses: actions/cache@v4 + with: + path: source/.cache/digital-surface + key: digital-surface-official-resolution-obadiaha-11793901-kinzik-eb4e9fc7-v3 + + - name: Install pinned research stack + run: | + python -m pip install --upgrade pip + python -m pip install numpy==2.2.6 pandas==2.2.3 scipy==1.15.3 scikit-learn==1.6.1 pyarrow==19.0.1 requests==2.32.3 pytest==8.3.5 tabulate==0.9.0 + + - name: Compile and pass exactly 29 tests + working-directory: source + shell: bash + run: | + set -euo pipefail + python -m compileall -q research/digital_surface tests/digital_surface + PYTHONPATH=. pytest --collect-only -q tests/digital_surface | tee /tmp/collect.txt + grep -Eq '^29 tests collected' /tmp/collect.txt + PYTHONPATH=. pytest tests/digital_surface -q + + - name: Run immutable two-validation four-transfer protocol + id: research + working-directory: source + shell: bash + run: | + set -euo pipefail + PYTHONPATH=. python -m research.digital_surface.run_research \ + --cache-dir .cache/digital-surface \ + --output-dir artifacts/digital-surface-arbitrage \ + 2>&1 | tee digital-surface-arbitrage.log + + - name: Verify source gate, four folds, and checksums + working-directory: source + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import hashlib, json + from pathlib import Path + import pandas as pd + + root=Path('artifacts/digital-surface-arbitrage') + audit=json.loads((root/'SOURCE_AUDIT.json').read_text()) + validation=audit['validation'] + assert validation['official_resolution_coverage'] == 1.0, validation + assert validation['official_resolution_agreement'] >= 0.99, validation + assert validation['official_resolution_rows'] == 5228, validation + assert 0.0 <= validation['binance_direction_agreement_diagnostic'] <= 1.0, validation + + folds=pd.read_csv(root/'fold_results.csv') + assert len(folds) == 4, folds + assert folds['fold_id'].nunique() == 4, folds + assert folds['test_week'].nunique() == 4, folds + assert folds['test_week'].astype(str).str.startswith(('02-','03-','04-','05-')).all(), folds + + admission=json.loads((root/'admission.json').read_text()) + assert admission['classification'] in {'Rejected','Exploratory','Admissible'}, admission + expected=json.loads((root/'SHA256SUMS.json').read_text()) + for name,want in expected.items(): + got=hashlib.sha256((root/name).read_bytes()).hexdigest() + assert got == want, (name,got,want) + + verification={ + 'classification': admission['classification'], + 'source_coverage': validation['official_resolution_coverage'], + 'source_agreement': validation['official_resolution_agreement'], + 'required_agreement': 0.99, + 'binance_direction_agreement_diagnostic': validation['binance_direction_agreement_diagnostic'], + 'folds_completed': len(folds), + 'test_weeks': folds['test_week'].astype(str).tolist(), + 'checksums_verified': len(expected), + } + (root/'CI_VERIFICATION.json').write_text(json.dumps(verification,indent=2,sort_keys=True)) + print(json.dumps(verification,indent=2)) + PY + + - name: Build deterministic source and result archives + if: always() + working-directory: source + shell: bash + run: | + set -euo pipefail + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-official-resolution-source-v3.tar.gz \ + .github research tests OFFICIAL_RESOLUTION_PATCH_SHA256.txt + sha256sum digital-surface-official-resolution-source-v3.tar.gz > digital-surface-official-resolution-source-v3.tar.gz.sha256 + if test -d artifacts/digital-surface-arbitrage; then + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-official-resolution-results-v3.tar.gz -C artifacts digital-surface-arbitrage + sha256sum digital-surface-official-resolution-results-v3.tar.gz > digital-surface-official-resolution-results-v3.tar.gz.sha256 + fi + + - name: Show final decision + if: always() + working-directory: source + run: | + cat artifacts/digital-surface-arbitrage/CI_VERIFICATION.json || true + cat artifacts/digital-surface-arbitrage/admission.json || true + tail -200 digital-surface-arbitrage.log || true + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: digital-surface-official-resolution-four-fold-v3 + path: | + source/artifacts/digital-surface-arbitrage/ + source/digital-surface-arbitrage.log + source/digital-surface-official-resolution-source-v3.tar.gz + source/digital-surface-official-resolution-source-v3.tar.gz.sha256 + source/digital-surface-official-resolution-results-v3.tar.gz + source/digital-surface-official-resolution-results-v3.tar.gz.sha256 + if-no-files-found: warn + retention-days: 30 diff --git a/.github/workflows/digital-surface-official-resolution-ci-v4.yml b/.github/workflows/digital-surface-official-resolution-ci-v4.yml new file mode 100644 index 00000000..a993838b --- /dev/null +++ b/.github/workflows/digital-surface-official-resolution-ci-v4.yml @@ -0,0 +1,183 @@ +name: Digital Surface Official Resolution CI v4 + +on: + push: + branches: [digital-surface-official-resolution-ci-20260724] + paths: + - .github/workflows/digital-surface-official-resolution-ci-v4.yml + - ci_payload/digital-surface-official-resolution-patch/** + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-official-resolution-ci-v4.yml + - ci_payload/digital-surface-official-resolution-patch/** + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + verify-and-research: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct checksum-pinned source + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source + tar -xzf base.tar.gz -C source + + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + patch -d source -p1 < ci_payload/digital-surface-official-resolution-patch/timestamp-unit-fix.patch + patch -d source -p1 < ci_payload/digital-surface-official-resolution-patch/full-universe-gate-fix.patch + patch -d source -p1 < ci_payload/digital-surface-official-resolution-patch/resolved-universe-fix.patch + + cat > /tmp/patch.sha256 <<'EOF' + 896bef750c9f1510252b695ef10517fa6502419cfce0d66dd9403d8a2696e903 source/research/digital_surface/__init__.py + 36f34ff7ee5c2f82e89b4d8674ce4371aea27ad1a63287ba537a6939d997d728 source/research/digital_surface/obadiaha_official_resolution_snapshot.json + bb80dece443b4629f90502bf7bb9f881e8e85cd363f806a5de3192b55243d2da source/research/digital_surface/official_resolution.py + 6a074f39f601762458fb59610e226a9dcade4b91956afae4e10cdcc4a1d7f752 source/tests/digital_surface/test_data_adapter.py + EOF + sed -i 's/^ //' /tmp/patch.sha256 + sha256sum -c /tmp/patch.sha256 + cp /tmp/patch.sha256 source/OFFICIAL_RESOLUTION_PATCH_SHA256.txt + + - name: Cache immutable source payloads + uses: actions/cache@v4 + with: + path: source/.cache/digital-surface + key: digital-surface-official-resolution-obadiaha-11793901-kinzik-eb4e9fc7-v4 + + - name: Install pinned research stack + run: | + python -m pip install --upgrade pip + python -m pip install numpy==2.2.6 pandas==2.2.3 scipy==1.15.3 scikit-learn==1.6.1 pyarrow==19.0.1 requests==2.32.3 pytest==8.3.5 tabulate==0.9.0 + + - name: Compile and pass exactly 29 tests + working-directory: source + shell: bash + run: | + set -euo pipefail + python -m compileall -q research/digital_surface tests/digital_surface + PYTHONPATH=. pytest --collect-only -q tests/digital_surface | tee /tmp/collect.txt + grep -Eq '^29 tests collected' /tmp/collect.txt + PYTHONPATH=. pytest tests/digital_surface -q + + - name: Run immutable two-validation four-transfer protocol + id: research + working-directory: source + shell: bash + run: | + set -euo pipefail + PYTHONPATH=. python -m research.digital_surface.run_research \ + --cache-dir .cache/digital-surface \ + --output-dir artifacts/digital-surface-arbitrage \ + 2>&1 | tee digital-surface-arbitrage.log + + - name: Verify source gate, four folds, and checksums + working-directory: source + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import hashlib, json + from pathlib import Path + import pandas as pd + + root=Path('artifacts/digital-surface-arbitrage') + audit=json.loads((root/'SOURCE_AUDIT.json').read_text()) + validation=audit['validation'] + assert validation['official_resolution_coverage'] == 1.0, validation + assert validation['official_resolution_agreement'] >= 0.99, validation + assert validation['official_resolution_rows'] == 5228, validation + assert validation['official_resolution_market_contracts'] == 5233, validation + assert validation['official_resolution_source_contracts'] == 5228, validation + assert validation['official_resolution_unresolved_market_contracts'] == 5, validation + assert validation['official_resolution_panel_contracts'] == 5169, validation + assert 0.0 <= validation['binance_direction_agreement_diagnostic'] <= 1.0, validation + + folds=pd.read_csv(root/'fold_results.csv') + assert len(folds) == 4, folds + assert folds['fold_id'].nunique() == 4, folds + assert folds['test_week'].nunique() == 4, folds + assert folds['test_week'].astype(str).str.startswith(('02-','03-','04-','05-')).all(), folds + + admission=json.loads((root/'admission.json').read_text()) + assert admission['classification'] in {'Rejected','Exploratory','Admissible'}, admission + expected=json.loads((root/'SHA256SUMS.json').read_text()) + for name,want in expected.items(): + got=hashlib.sha256((root/name).read_bytes()).hexdigest() + assert got == want, (name,got,want) + + verification={ + 'classification': admission['classification'], + 'source_coverage': validation['official_resolution_coverage'], + 'source_agreement': validation['official_resolution_agreement'], + 'raw_market_contracts': validation['official_resolution_market_contracts'], + 'resolved_source_contracts': validation['official_resolution_source_contracts'], + 'unresolved_market_contracts': validation['official_resolution_unresolved_market_contracts'], + 'causal_panel_contracts': validation['official_resolution_panel_contracts'], + 'required_agreement': 0.99, + 'binance_direction_agreement_diagnostic': validation['binance_direction_agreement_diagnostic'], + 'folds_completed': len(folds), + 'test_weeks': folds['test_week'].astype(str).tolist(), + 'checksums_verified': len(expected), + } + (root/'CI_VERIFICATION.json').write_text(json.dumps(verification,indent=2,sort_keys=True)) + print(json.dumps(verification,indent=2)) + PY + + - name: Build deterministic source and result archives + if: always() + working-directory: source + shell: bash + run: | + set -euo pipefail + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-official-resolution-source-v4.tar.gz \ + .github research tests OFFICIAL_RESOLUTION_PATCH_SHA256.txt + sha256sum digital-surface-official-resolution-source-v4.tar.gz > digital-surface-official-resolution-source-v4.tar.gz.sha256 + if test -d artifacts/digital-surface-arbitrage; then + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-official-resolution-results-v4.tar.gz -C artifacts digital-surface-arbitrage + sha256sum digital-surface-official-resolution-results-v4.tar.gz > digital-surface-official-resolution-results-v4.tar.gz.sha256 + fi + + - name: Show final decision + if: always() + working-directory: source + run: | + cat artifacts/digital-surface-arbitrage/CI_VERIFICATION.json || true + cat artifacts/digital-surface-arbitrage/admission.json || true + tail -200 digital-surface-arbitrage.log || true + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: digital-surface-official-resolution-four-fold-v4 + path: | + source/artifacts/digital-surface-arbitrage/ + source/digital-surface-arbitrage.log + source/digital-surface-official-resolution-source-v4.tar.gz + source/digital-surface-official-resolution-source-v4.tar.gz.sha256 + source/digital-surface-official-resolution-results-v4.tar.gz + source/digital-surface-official-resolution-results-v4.tar.gz.sha256 + if-no-files-found: warn + retention-days: 30 diff --git a/.github/workflows/digital-surface-official-resolution-ci-v5.yml b/.github/workflows/digital-surface-official-resolution-ci-v5.yml new file mode 100644 index 00000000..22dca48a --- /dev/null +++ b/.github/workflows/digital-surface-official-resolution-ci-v5.yml @@ -0,0 +1,186 @@ +name: Digital Surface Official Resolution CI v5 + +on: + push: + branches: [digital-surface-official-resolution-ci-20260724] + paths: + - .github/workflows/digital-surface-official-resolution-ci-v5.yml + - ci_payload/digital-surface-official-resolution-patch/** + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-official-resolution-ci-v5.yml + - ci_payload/digital-surface-official-resolution-patch/** + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + verify-and-research: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct checksum-pinned source + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source + tar -xzf base.tar.gz -C source + + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + patch -d source -p1 < ci_payload/digital-surface-official-resolution-patch/timestamp-unit-fix.patch + patch -d source -p1 < ci_payload/digital-surface-official-resolution-patch/full-universe-gate-fix.patch + patch -d source -p1 < ci_payload/digital-surface-official-resolution-patch/resolved-universe-fix.patch + patch -d source -p1 < ci_payload/digital-surface-official-resolution-patch/asset-merge-fix.patch + + cat > /tmp/patch.sha256 <<'EOF' + 896bef750c9f1510252b695ef10517fa6502419cfce0d66dd9403d8a2696e903 source/research/digital_surface/__init__.py + 36f34ff7ee5c2f82e89b4d8674ce4371aea27ad1a63287ba537a6939d997d728 source/research/digital_surface/obadiaha_official_resolution_snapshot.json + bb80dece443b4629f90502bf7bb9f881e8e85cd363f806a5de3192b55243d2da source/research/digital_surface/official_resolution.py + 6a074f39f601762458fb59610e226a9dcade4b91956afae4e10cdcc4a1d7f752 source/tests/digital_surface/test_data_adapter.py + 4aae126313ff8ece97b8c9611be95a6528dae8c6934c5f29036b24f341dc8c40 source/research/digital_surface/causal.py + bcf5c32614bae2a7e3249ebd626cab5d9d2c2fc5999e3d90b4e8430e55d8a915 source/tests/digital_surface/test_causal.py + EOF + sed -i 's/^ //' /tmp/patch.sha256 + sha256sum -c /tmp/patch.sha256 + cp /tmp/patch.sha256 source/OFFICIAL_RESOLUTION_PATCH_SHA256.txt + + - name: Cache immutable source payloads + uses: actions/cache@v4 + with: + path: source/.cache/digital-surface + key: digital-surface-official-resolution-obadiaha-11793901-kinzik-eb4e9fc7-v5 + + - name: Install pinned research stack + run: | + python -m pip install --upgrade pip + python -m pip install numpy==2.2.6 pandas==2.2.3 scipy==1.15.3 scikit-learn==1.6.1 pyarrow==19.0.1 requests==2.32.3 pytest==8.3.5 tabulate==0.9.0 + + - name: Compile and pass exactly 29 tests + working-directory: source + shell: bash + run: | + set -euo pipefail + python -m compileall -q research/digital_surface tests/digital_surface + PYTHONPATH=. pytest --collect-only -q tests/digital_surface | tee /tmp/collect.txt + grep -Eq '^29 tests collected' /tmp/collect.txt + PYTHONPATH=. pytest tests/digital_surface -q + + - name: Run immutable two-validation four-transfer protocol + id: research + working-directory: source + shell: bash + run: | + set -euo pipefail + PYTHONPATH=. python -m research.digital_surface.run_research \ + --cache-dir .cache/digital-surface \ + --output-dir artifacts/digital-surface-arbitrage \ + 2>&1 | tee digital-surface-arbitrage.log + + - name: Verify source gate, four folds, and checksums + working-directory: source + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import hashlib, json + from pathlib import Path + import pandas as pd + + root=Path('artifacts/digital-surface-arbitrage') + audit=json.loads((root/'SOURCE_AUDIT.json').read_text()) + validation=audit['validation'] + assert validation['official_resolution_coverage'] == 1.0, validation + assert validation['official_resolution_agreement'] >= 0.99, validation + assert validation['official_resolution_rows'] == 5228, validation + assert validation['official_resolution_market_contracts'] == 5233, validation + assert validation['official_resolution_source_contracts'] == 5228, validation + assert validation['official_resolution_unresolved_market_contracts'] == 5, validation + assert validation['official_resolution_panel_contracts'] == 5169, validation + assert 0.0 <= validation['binance_direction_agreement_diagnostic'] <= 1.0, validation + + folds=pd.read_csv(root/'fold_results.csv') + assert len(folds) == 4, folds + assert folds['fold_id'].nunique() == 4, folds + assert folds['test_week'].nunique() == 4, folds + assert folds['test_week'].astype(str).str.startswith(('02-','03-','04-','05-')).all(), folds + + admission=json.loads((root/'admission.json').read_text()) + assert admission['classification'] in {'Rejected','Exploratory','Admissible'}, admission + expected=json.loads((root/'SHA256SUMS.json').read_text()) + for name,want in expected.items(): + got=hashlib.sha256((root/name).read_bytes()).hexdigest() + assert got == want, (name,got,want) + + verification={ + 'classification': admission['classification'], + 'source_coverage': validation['official_resolution_coverage'], + 'source_agreement': validation['official_resolution_agreement'], + 'raw_market_contracts': validation['official_resolution_market_contracts'], + 'resolved_source_contracts': validation['official_resolution_source_contracts'], + 'unresolved_market_contracts': validation['official_resolution_unresolved_market_contracts'], + 'causal_panel_contracts': validation['official_resolution_panel_contracts'], + 'required_agreement': 0.99, + 'binance_direction_agreement_diagnostic': validation['binance_direction_agreement_diagnostic'], + 'folds_completed': len(folds), + 'test_weeks': folds['test_week'].astype(str).tolist(), + 'checksums_verified': len(expected), + } + (root/'CI_VERIFICATION.json').write_text(json.dumps(verification,indent=2,sort_keys=True)) + print(json.dumps(verification,indent=2)) + PY + + - name: Build deterministic source and result archives + if: always() + working-directory: source + shell: bash + run: | + set -euo pipefail + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-official-resolution-source-v5.tar.gz \ + .github research tests OFFICIAL_RESOLUTION_PATCH_SHA256.txt + sha256sum digital-surface-official-resolution-source-v5.tar.gz > digital-surface-official-resolution-source-v5.tar.gz.sha256 + if test -d artifacts/digital-surface-arbitrage; then + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-official-resolution-results-v5.tar.gz -C artifacts digital-surface-arbitrage + sha256sum digital-surface-official-resolution-results-v5.tar.gz > digital-surface-official-resolution-results-v5.tar.gz.sha256 + fi + + - name: Show final decision + if: always() + working-directory: source + run: | + cat artifacts/digital-surface-arbitrage/CI_VERIFICATION.json || true + cat artifacts/digital-surface-arbitrage/admission.json || true + tail -200 digital-surface-arbitrage.log || true + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: digital-surface-official-resolution-four-fold-v5 + path: | + source/artifacts/digital-surface-arbitrage/ + source/digital-surface-arbitrage.log + source/digital-surface-official-resolution-source-v5.tar.gz + source/digital-surface-official-resolution-source-v5.tar.gz.sha256 + source/digital-surface-official-resolution-results-v5.tar.gz + source/digital-surface-official-resolution-results-v5.tar.gz.sha256 + if-no-files-found: warn + retention-days: 30 diff --git a/.github/workflows/digital-surface-official-resolution-ci.yml b/.github/workflows/digital-surface-official-resolution-ci.yml new file mode 100644 index 00000000..97c3d706 --- /dev/null +++ b/.github/workflows/digital-surface-official-resolution-ci.yml @@ -0,0 +1,229 @@ +name: Digital Surface Official Resolution CI + +on: + push: + branches: [digital-surface-official-resolution-ci-20260724] + paths: + - .github/workflows/digital-surface-official-resolution-ci.yml + - ci_payload/digital-surface-official-resolution-patch/** + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-official-resolution-ci.yml + - ci_payload/digital-surface-official-resolution-patch/** + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + verify-and-research: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct exact verified source checkpoint + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source + tar -xzf base.tar.gz -C source + + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + + python - <<'PY' + import hashlib + import json + from pathlib import Path + + expected = { + '.github/workflows/digital-surface-arbitrage.yml': '79a5b17999aca8bf9c2a41dff88db418ee3ed06de0c817523f692b68e9e7ff60', + 'research/digital_surface/__init__.py': '896bef750c9f1510252b695ef10517fa6502419cfce0d66dd9403d8a2696e903', + 'research/digital_surface/candidates.py': 'a3deb48ed87d826c0f28e1e03b38796e372246aa0140c70dcafb41f2fe067a74', + 'research/digital_surface/causal.py': 'e631e36fa2f8b516cdd7efd1fde1528c1e2a5d33b1e580495a9f5f88c6a6b14e', + 'research/digital_surface/data.py': '439e56c236f95352c62a707b35b2747429d067711c3c6c9ff6955e6018f2f939', + 'research/digital_surface/execution.py': 'cef6cbe145e9ced229e6216b880ab58503207032fb415c3ec217224d288a64ca', + 'research/digital_surface/fees.py': '9ba2b5bac6518f04fe88a2f96e54a9e7d4f6f708e28426cf095e860736d5e694', + 'research/digital_surface/obadiaha_official_resolution_snapshot.json': '36f34ff7ee5c2f82e89b4d8674ce4371aea27ad1a63287ba537a6939d997d728', + 'research/digital_surface/obadiaha_source_manifest.json': 'cd6941ea34f6f19477e99be1628f83d7a24a2922a645538ebfe5cae9d3e2c289', + 'research/digital_surface/official_resolution.py': 'e6b0fcb92027b4c16abd8ea73be487c7572bad8e98054d5dc0060ab9c51763c6', + 'research/digital_surface/portfolio.py': '10187c74eb8f23a4e88ba6e9ab26d5dc91e5588f2f5098999bc3a759ec2fa71c', + 'research/digital_surface/registry.py': '55f3e13db599fc2ffffed8a3bdbc776e75b4978ab7011e25dc99b8bd15fe51e3', + 'research/digital_surface/run_research.py': '83a36e1a90934e42ce5a7d3c5373e97ac47b902717c69526856ec8ae94e658aa', + 'research/digital_surface/surface.py': '6226c7f19b33e97e6d1f20790ec4c3c723bddda2f0aa7e93035ab55607b95920', + 'research/digital_surface/types.py': 'a72db216c81fbec009342b1ee703f51dbba3b5abeab8865be2632191bf835910', + 'research/digital_surface/validation.py': 'e9cada082c5e7c680ce3ac6f48dbac45367c492fe8e7a24cebeeaddc86fff39c', + 'tests/digital_surface/test_candidates.py': '41e35dabe0919db4ae20b3a6d029710f9eac1c8ca3cb06d23e3cc503082751eb', + 'tests/digital_surface/test_causal.py': '5abbf228f78bb03d0382da18f1a218b9d642a3de9bfac174fdffe2c12659870f', + 'tests/digital_surface/test_data_adapter.py': 'da246d7bd26065f26231da7c842881866f807cf31efe9e36e577709ea49e2c7c', + 'tests/digital_surface/test_execution.py': '34d366b80fee5f0844c7118a3ac1eff56b2e7ce5c00018d965d8c46947644715', + 'tests/digital_surface/test_fees_and_types.py': 'a5107698a54d0d14e28a767b2add438f5c70521f0429e49e3fc7beb9163ef19f', + 'tests/digital_surface/test_integration.py': 'cf2bfa0eca57810bb83569f9519ada694b541b5bafb3e609802787a3445dbc9d', + 'tests/digital_surface/test_portfolio.py': '3b22a7064c1ad96dfdc728349d64e6cbde9ee2524c233fe3a69b670ba9fd5020', + 'tests/digital_surface/test_registry.py': 'e99114ab5986be6870605526407d15dc80ca835c150af446bc84554be42781bd', + 'tests/digital_surface/test_surface.py': 'c06b2160572f171bf294bffefd2707a5f8a8bade3099df16973824c41cde6aa4', + 'tests/digital_surface/test_validation.py': '7a4ef5a43e4b2999e3eabc53eb930088f0f514f2adefdba6a5a7207b952bea3b', + } + root = Path('source') + for name, want in expected.items(): + got = hashlib.sha256((root / name).read_bytes()).hexdigest() + assert got == want, (name, got, want) + (root / 'SOURCE_CHECKPOINT_SHA256.json').write_text( + json.dumps(expected, indent=2, sort_keys=True), encoding='utf-8' + ) + print(f'verified {len(expected)} exact source files') + PY + + - name: Cache immutable source payloads + uses: actions/cache@v4 + with: + path: source/.cache/digital-surface + key: digital-surface-official-resolution-obadiaha-11793901-kinzik-eb4e9fc7-v1 + + - name: Install pinned research stack + run: | + python -m pip install --upgrade pip + python -m pip install \ + numpy==2.2.6 \ + pandas==2.2.3 \ + scipy==1.15.3 \ + scikit-learn==1.6.1 \ + pyarrow==19.0.1 \ + requests==2.32.3 \ + pytest==8.3.5 \ + tabulate==0.9.0 + + - name: Compile and pass exactly 29 tests + working-directory: source + shell: bash + run: | + set -euo pipefail + python -m compileall -q research/digital_surface tests/digital_surface + PYTHONPATH=. pytest --collect-only -q tests/digital_surface | tee /tmp/collect.txt + grep -Eq '^29 tests collected' /tmp/collect.txt + PYTHONPATH=. pytest tests/digital_surface -q + + - name: Run immutable two-validation four-transfer protocol + id: research + working-directory: source + continue-on-error: true + shell: bash + run: | + set -o pipefail + PYTHONPATH=. python -m research.digital_surface.run_research \ + --cache-dir .cache/digital-surface \ + --output-dir artifacts/digital-surface-arbitrage \ + 2>&1 | tee digital-surface-arbitrage.log + + - name: Verify source admission, four untouched folds, and result checksums + working-directory: source + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import hashlib + import json + from pathlib import Path + import pandas as pd + + root = Path('artifacts/digital-surface-arbitrage') + source = json.loads((root / 'SOURCE_AUDIT.json').read_text()) + validation = source['validation'] + assert validation['official_resolution_coverage'] == 1.0, validation + assert validation['official_resolution_agreement'] >= 0.99, validation + assert 0.0 <= validation['binance_direction_agreement_diagnostic'] <= 1.0, validation + assert validation['official_resolution_rows'] == 5228, validation + + folds = pd.read_csv(root / 'fold_results.csv') + assert len(folds) == 4, folds + assert folds['fold_id'].nunique() == 4, folds + assert folds['test_week'].nunique() == 4, folds + assert all(str(value).startswith(('02-', '03-', '04-', '05-')) for value in folds['test_week']), folds + + admission = json.loads((root / 'admission.json').read_text()) + assert admission['classification'] in {'Rejected', 'Exploratory', 'Admissible'}, admission + expected = json.loads((root / 'SHA256SUMS.json').read_text()) + for name, want in expected.items(): + got = hashlib.sha256((root / name).read_bytes()).hexdigest() + assert got == want, (name, got, want) + + verification = { + 'source_gate': { + 'coverage': validation['official_resolution_coverage'], + 'agreement': validation['official_resolution_agreement'], + 'required_agreement': 0.99, + 'binance_direction_agreement_diagnostic': validation['binance_direction_agreement_diagnostic'], + }, + 'folds_completed': int(len(folds)), + 'test_weeks': folds['test_week'].astype(str).tolist(), + 'classification': admission['classification'], + 'result_checksums_verified': len(expected), + } + (root / 'CI_VERIFICATION.json').write_text( + json.dumps(verification, indent=2, sort_keys=True), encoding='utf-8' + ) + print(json.dumps(verification, indent=2)) + PY + + - name: Build deterministic source and result archives + if: always() + working-directory: source + shell: bash + run: | + set -euo pipefail + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-official-resolution-source.tar.gz \ + .github research tests SOURCE_CHECKPOINT_SHA256.json + sha256sum digital-surface-official-resolution-source.tar.gz > digital-surface-official-resolution-source.tar.gz.sha256 + if test -d artifacts/digital-surface-arbitrage; then + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-official-resolution-results.tar.gz \ + -C artifacts digital-surface-arbitrage + sha256sum digital-surface-official-resolution-results.tar.gz > digital-surface-official-resolution-results.tar.gz.sha256 + fi + + - name: Show final decision + if: always() + working-directory: source + run: | + cat artifacts/digital-surface-arbitrage/CI_VERIFICATION.json || true + cat artifacts/digital-surface-arbitrage/admission.json || true + tail -200 digital-surface-arbitrage.log || true + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: digital-surface-official-resolution-four-fold + path: | + source/artifacts/digital-surface-arbitrage/ + source/digital-surface-arbitrage.log + source/digital-surface-official-resolution-source.tar.gz + source/digital-surface-official-resolution-source.tar.gz.sha256 + source/digital-surface-official-resolution-results.tar.gz + source/digital-surface-official-resolution-results.tar.gz.sha256 + if-no-files-found: warn + retention-days: 30 + + - name: Enforce completed research process + if: always() + shell: bash + run: | + test '${{ steps.research.outcome }}' = 'success' + test -f source/artifacts/digital-surface-arbitrage/CI_VERIFICATION.json diff --git a/.github/workflows/digital-surface-prospective-signal-checkpoint.yml b/.github/workflows/digital-surface-prospective-signal-checkpoint.yml new file mode 100644 index 00000000..6af80e5f --- /dev/null +++ b/.github/workflows/digital-surface-prospective-signal-checkpoint.yml @@ -0,0 +1,181 @@ +name: Digital Surface Prospective Signal Checkpoint + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/digital-surface-prospective-signal-checkpoint.yml + - ci_payload/digital-surface-live-readiness/prospective-signal-source/** + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-prospective-signal-checkpoint.yml + - ci_payload/digital-surface-live-readiness/prospective-signal-source/** + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + verify-prospective-signal: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct checksum-pinned source + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source && tar -xzf base.tar.gz -C source + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + for patch_file in timestamp-unit-fix.patch full-universe-gate-fix.patch resolved-universe-fix.patch asset-merge-fix.patch; do + patch -d source -p1 < "ci_payload/digital-surface-official-resolution-patch/${patch_file}" + done + echo 'f71f3a653ca58b4bf98609fc29d8ab16f9d031c9ed14906ebe5159f2b0de8eab ci_payload/digital-surface-live-readiness/execution-diagnostics.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/execution-diagnostics.patch + echo '8606cc925aa3b3a6417abb1fba9a11df916cc2b281880978fb6229aa25875be9 ci_payload/digital-surface-live-readiness/book-arrival-model.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/book-arrival-model.patch + cp -a ci_payload/digital-surface-live-readiness/shadow-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/prospective-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/observer-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/simulation-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/market-state-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/market-state-observer-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/prospective-signal-source/. source/ + + cat > /tmp/prospective-signal.sha256 <<'EOF' + c147efb911b0f6f049902fdf037f5aef8f044bc944a5aac9e298f93283c38996 source/research/digital_surface/prospective_signal.py + e2c21e61bd450343bc2271db83cd9b27cd316e5c50d3f9ac1128f3f38be998ed source/tests/digital_surface/test_prospective_signal.py + EOF + sed -i 's/^ //' /tmp/prospective-signal.sha256 + sha256sum -c /tmp/prospective-signal.sha256 + cp /tmp/prospective-signal.sha256 source/PROSPECTIVE_SIGNAL_SOURCE_SHA256.txt + + - name: Install pinned test stack + run: | + python -m pip install --upgrade pip + python -m pip install numpy==2.2.6 pandas==2.2.3 scipy==1.15.3 scikit-learn==1.6.1 pyarrow==19.0.1 requests==2.32.3 pytest==8.3.5 tabulate==0.9.0 + + - name: Compile and pass exactly 76 tests + working-directory: source + shell: bash + run: | + set -euo pipefail + python -m compileall -q research/digital_surface tests/digital_surface + PYTHONPATH=. pytest --collect-only -q tests/digital_surface | tee /tmp/collect.txt + grep -Eq '^76 tests collected' /tmp/collect.txt + PYTHONPATH=. pytest tests/digital_surface -q + + - name: Verify frozen causal signal semantics + working-directory: source + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import hashlib + import json + from decimal import Decimal + from pathlib import Path + + from research.digital_surface.binance_public import BTCMarketState + from research.digital_surface.market_discovery import GammaMarketRecord + from research.digital_surface.prospective_signal import FrozenSurfacePolicy, generate_prospective_candidates + from research.digital_surface.public_books import parse_public_order_book + + payload = { + 'version': 1, + 'model': {'coefficients': [0.0, 1.0, 0.0, 0.0], 'training_rows': 1000, 'training_contracts': 500}, + 'candidate': { + 'min_edge': 0.02, 'max_total_cost': 1.25, 'min_tau_s': 3.0, 'max_tau_s': 60.0, + 'decision_cadence_s': 5, 'pair_gap_ms': 2000, 'slippage': 0.01, 'uncertainty_penalty': 0.0, + }, + 'selection': 'first_trigger_per_expiry', + 'flow_assumption': 'zero_without_public_causal_trade_flow', + } + policy = FrozenSurfacePolicy.from_payload(payload, policy_sha256='1'*64) + markets = [ + GammaMarketRecord('five', 'btc-updown-5m-1000', 'Bitcoin Up or Down', 'yes-five', 'no-five', 300, 1000, '', Decimal('0.01')), + GammaMarketRecord('fifteen', 'btc-updown-15m-400', 'Bitcoin Up or Down', 'yes-fifteen', 'no-fifteen', 900, 400, '', Decimal('0.01')), + ] + def book(token, condition): + return parse_public_order_book({ + 'market': condition, 'asset_id': token, 'timestamp': '1250000', 'hash': 'h-'+token, + 'bids': [{'price': '0.40', 'size': '10'}], 'asks': [{'price': '0.45', 'size': '10'}], + 'min_order_size': '1', 'tick_size': '0.01', 'neg_risk': False, + }) + books = {token: book(token, market.condition_id) for market in markets for token in (market.yes_token_id, market.no_token_id)} + state = BTCMarketState( + symbol='BTCUSDT', server_time_ms=1_250_000, observed_ts_ms=1_250_000, + spot=Decimal('100'), vol_30s=0.001, vol_120s=0.0015, + strikes={1_000_000: Decimal('99'), 400_000: Decimal('101')}, closed_one_second_bars=121, + raw_response_sha256={'server_time': 'a'*64, 'one_second_klines': 'b'*64, 'strike:1000000': 'c'*64, 'strike:400000': 'd'*64}, + ) + candidates, summary = generate_prospective_candidates( + markets, books, state, book_observed_ts_ms=1_250_000, policy=policy, + prospective_head_sha256='2'*64, book_ledger_head_sha256='3'*64, spot_state_head_sha256='4'*64, + ) + assert len(candidates) == 1 + row = candidates.iloc[0] + assert row['low_condition_id'] == 'five' + assert row['high_condition_id'] == 'fifteen' + assert row['low_yes_token_id'] == 'yes-five' + assert row['high_no_token_id'] == 'no-fifteen' + assert summary['future_outcomes_used'] == 0 + assert bool(row['transactionally_atomic']) is False + assert row['fill_model'] == 'public_book_arrival_shadow' + + verified = {} + for line in Path('PROSPECTIVE_SIGNAL_SOURCE_SHA256.txt').read_text().splitlines(): + wanted, name = line.split(maxsplit=1) + got = hashlib.sha256(Path(name).read_bytes()).hexdigest() + assert got == wanted + verified[name] = got + result = { + 'tests_collected': 76, + 'same_expiry_pairing': True, + 'lower_strike_yes_higher_strike_no': True, + 'first_trigger_selection': True, + 'future_outcomes_used': 0, + 'transactionally_atomic': False, + 'order_submissions': 0, + 'live_mode': 'physically_disabled', + 'source_files_verified': verified, + } + Path('PROSPECTIVE_SIGNAL_VERIFICATION.json').write_text(json.dumps(result, indent=2, sort_keys=True)) + print(json.dumps(result, indent=2)) + PY + + - name: Build deterministic source checkpoint + working-directory: source + shell: bash + run: | + set -euo pipefail + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-prospective-signal-source-v1.tar.gz \ + research tests PROSPECTIVE_SIGNAL_SOURCE_SHA256.txt PROSPECTIVE_SIGNAL_VERIFICATION.json + sha256sum digital-surface-prospective-signal-source-v1.tar.gz > digital-surface-prospective-signal-source-v1.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: digital-surface-prospective-signal-checkpoint-v1 + path: | + source/PROSPECTIVE_SIGNAL_SOURCE_SHA256.txt + source/PROSPECTIVE_SIGNAL_VERIFICATION.json + source/digital-surface-prospective-signal-source-v1.tar.gz + source/digital-surface-prospective-signal-source-v1.tar.gz.sha256 + retention-days: 30 diff --git a/.github/workflows/digital-surface-public-book-observer-checkpoint.yml b/.github/workflows/digital-surface-public-book-observer-checkpoint.yml new file mode 100644 index 00000000..9d81a59a --- /dev/null +++ b/.github/workflows/digital-surface-public-book-observer-checkpoint.yml @@ -0,0 +1,212 @@ +name: Digital Surface Public Book Observer Checkpoint + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/digital-surface-public-book-observer-checkpoint.yml + - ci_payload/digital-surface-live-readiness/observer-source/** + - ci_payload/digital-surface-live-readiness/prospective-source/** + - ci_payload/digital-surface-live-readiness/shadow-source/** + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-public-book-observer-checkpoint.yml + - ci_payload/digital-surface-live-readiness/observer-source/** + - ci_payload/digital-surface-live-readiness/prospective-source/** + - ci_payload/digital-surface-live-readiness/shadow-source/** + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + verify-public-book-observer: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct checksum-pinned research source + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source && tar -xzf base.tar.gz -C source + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + for patch_file in timestamp-unit-fix.patch full-universe-gate-fix.patch resolved-universe-fix.patch asset-merge-fix.patch; do + patch -d source -p1 < "ci_payload/digital-surface-official-resolution-patch/${patch_file}" + done + echo 'f71f3a653ca58b4bf98609fc29d8ab16f9d031c9ed14906ebe5159f2b0de8eab ci_payload/digital-surface-live-readiness/execution-diagnostics.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/execution-diagnostics.patch + echo '8606cc925aa3b3a6417abb1fba9a11df916cc2b281880978fb6229aa25875be9 ci_payload/digital-surface-live-readiness/book-arrival-model.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/book-arrival-model.patch + cp -a ci_payload/digital-surface-live-readiness/shadow-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/prospective-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/observer-source/. source/ + + cat > /tmp/public-book-source.sha256 <<'EOF' + d96e722e57b24b2dc37bdf772972133ceeb1fcbf3629973f3abae701b4943051 source/research/digital_surface/book_ledger.py + f3f1e2417305cd0ceee4b6971e461e488960f04f0e0b80ad292733a2a4b248af source/research/digital_surface/diagnostic_filter.py + efd186a4ca00c2e920bc87b8d137ac46db52f2aa36385a275dc99e70b5618439 source/research/digital_surface/market_discovery.py + 3fbb96286445b60b0b7deeb5e1f9ee45f53bfd5efdcc7a727b50617b19463224 source/research/digital_surface/prospective.py + 088bab6729e5a1beb22baa853483433fd23180d3f8597035e0d3a4d036a8aec3 source/research/digital_surface/public_book_observer.py + 1ee608819769dad77c19f343427fbe604cacb3afc95b693aa5153b4ea22a166e source/research/digital_surface/public_books.py + 73c5aaa7a4ca10baf85886eb28c6135d975eafaaab19e5193819d1ccbf82f2be source/research/digital_surface/public_observer.py + ea56803a606ff33b7231b373141df3d22c159cedc1063603fef3dfb6255c60c9 source/research/digital_surface/shadow_observation_policy_v1.json + 7220101c9f0d3684d241843f2647fef81d45725a178d27e23afb16068218272f source/tests/digital_surface/test_book_ledger.py + 774575a18473899bfd2304f93be3786c60c0288fd28aa98d94a40aa043c74858 source/tests/digital_surface/test_diagnostic_filter.py + adf17ffb26a23332194eaa67c07201e713f2edf81927365cb54b21a348b5a7bd source/tests/digital_surface/test_market_discovery.py + 6efbf535ddfb50ca4b88209ecfa72c8e208a9f4049b0960781805cb888449b53 source/tests/digital_surface/test_prospective.py + 53b3d577327ab9b5fee56c1905a585f9caf2ef0d2f91c6e100ce83371da6f494 source/tests/digital_surface/test_public_book_observer.py + cf1f1c4cde4b0ec9db7766ada185640c4fbfc20bbc55bb7869b61b9f528df2b8 source/tests/digital_surface/test_public_books.py + 87e32c6fc215d8ffe8f587357acd1d3efc12b1c78021f2f01740ebc06b9a0554 source/tests/digital_surface/test_public_observer.py + EOF + sed -i 's/^ //' /tmp/public-book-source.sha256 + sha256sum -c /tmp/public-book-source.sha256 + cp /tmp/public-book-source.sha256 source/PUBLIC_BOOK_OBSERVER_SOURCE_SHA256.txt + + - name: Install pinned test stack + run: | + python -m pip install --upgrade pip + python -m pip install numpy==2.2.6 pandas==2.2.3 scipy==1.15.3 scikit-learn==1.6.1 pyarrow==19.0.1 requests==2.32.3 pytest==8.3.5 tabulate==0.9.0 + + - name: Compile and pass exactly 60 tests + working-directory: source + shell: bash + run: | + set -euo pipefail + python -m compileall -q research/digital_surface tests/digital_surface + PYTHONPATH=. pytest --collect-only -q tests/digital_surface | tee /tmp/collect.txt + grep -Eq '^60 tests collected' /tmp/collect.txt + PYTHONPATH=. pytest tests/digital_surface -q + + - name: Verify public-only surfaces and chained evidence + working-directory: source + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import ast + import hashlib + import json + import tempfile + from decimal import Decimal + from pathlib import Path + + from research.digital_surface.book_ledger import PublicBookLedger + from research.digital_surface.diagnostic_filter import DiagnosticBloomFilter, build_diagnostic_bloom + from research.digital_surface.live import LiveTradingDisabledError, load_config + from research.digital_surface.market_discovery import GammaMarketRecord + from research.digital_surface.prospective import ProspectiveLedger + from research.digital_surface.public_book_observer import observe_public_books + from research.digital_surface.public_books import CLOB_BOOKS_URL, parse_public_order_book + from research.digital_surface.public_observer import GAMMA_MARKETS_URL, observe_markets + + try: + load_config({'TRADING_MODE': 'live', 'LIVE_ARM_TOKEN': 'present'}) + except LiveTradingDisabledError: + pass + else: + raise AssertionError('live mode unexpectedly armed') + + assert GAMMA_MARKETS_URL == 'https://gamma-api.polymarket.com/markets' + assert CLOB_BOOKS_URL == 'https://clob.polymarket.com/books' + forbidden_literals = { + 'https://clob.polymarket.com/order', + 'wss://ws-subscriptions-clob.polymarket.com/ws/user', + } + for path in Path('research/digital_surface').glob('public_*.py'): + text = path.read_text() + assert not any(value in text for value in forbidden_literals), path + + bloom = DiagnosticBloomFilter.from_payload( + build_diagnostic_bloom(['historical'], source_sha256='a'*64, m_bits=1024, k_hashes=4) + ) + with tempfile.TemporaryDirectory() as directory: + directory = Path(directory) + market_ledger = ProspectiveLedger(directory/'markets.jsonl', diagnostic_market_ids=bloom) + market = GammaMarketRecord( + condition_id='fresh', slug='btc-updown-5m-1999999800', question='Bitcoin Up or Down', + yes_token_id='yes-fresh', no_token_id='no-fresh', duration_seconds=300, + open_epoch_seconds=1_999_999_800, end_date='2033-05-18T03:35:00Z', + tick_size=Decimal('0.01'), + ) + market_summary = observe_markets( + [market], ledger=market_ledger, observed_ts_ms=1, + policy_sha256='1'*64, source_sha256='2'*64, + ) + books = { + token: parse_public_order_book({ + 'market': 'fresh', 'asset_id': token, 'timestamp': '1', 'hash': 'h-'+token, + 'bids': [{'price': '0.40', 'size': '10'}], + 'asks': [{'price': '0.50', 'size': '10'}], + 'min_order_size': '1', 'tick_size': '0.01', 'neg_risk': False, + }) + for token in ('yes-fresh', 'no-fresh') + } + book_ledger = PublicBookLedger(directory/'books.jsonl') + book_summary = observe_public_books( + [market], books, ledger=book_ledger, + prospective_market_ids=market_ledger.market_ids, + prospective_head_sha256=market_ledger.head_hash, + observed_ts_ms=2, policy_sha256='1'*64, source_sha256='2'*64, + ) + assert market_summary['appended'] == 1 + assert book_summary['books_appended'] == 2 + assert book_summary['prospective_head_sha256'] == market_ledger.head_hash + assert PublicBookLedger(book_ledger.path).head_hash == book_ledger.head_hash + + verified = {} + for line in Path('PUBLIC_BOOK_OBSERVER_SOURCE_SHA256.txt').read_text().splitlines(): + wanted, name = line.split(maxsplit=1) + got = hashlib.sha256(Path(name).read_bytes()).hexdigest() + assert got == wanted, (name, got, wanted) + verified[name] = got + result = { + 'tests_collected': 60, + 'public_market_endpoint': GAMMA_MARKETS_URL, + 'public_books_endpoint': CLOB_BOOKS_URL, + 'authenticated_requests': 0, + 'order_submissions': 0, + 'market_hash_chain_verified': True, + 'book_hash_chain_verified': True, + 'live_mode': 'physically_disabled', + 'source_files_verified': verified, + } + Path('PUBLIC_BOOK_OBSERVER_VERIFICATION.json').write_text(json.dumps(result, indent=2, sort_keys=True)) + print(json.dumps(result, indent=2)) + PY + + - name: Build deterministic source checkpoint + working-directory: source + shell: bash + run: | + set -euo pipefail + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-public-book-observer-source-v1.tar.gz \ + research tests PUBLIC_BOOK_OBSERVER_SOURCE_SHA256.txt PUBLIC_BOOK_OBSERVER_VERIFICATION.json + sha256sum digital-surface-public-book-observer-source-v1.tar.gz > digital-surface-public-book-observer-source-v1.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: digital-surface-public-book-observer-checkpoint-v1 + path: | + source/PUBLIC_BOOK_OBSERVER_SOURCE_SHA256.txt + source/PUBLIC_BOOK_OBSERVER_VERIFICATION.json + source/digital-surface-public-book-observer-source-v1.tar.gz + source/digital-surface-public-book-observer-source-v1.tar.gz.sha256 + retention-days: 30 diff --git a/.github/workflows/digital-surface-public-btc-state-observer.yml b/.github/workflows/digital-surface-public-btc-state-observer.yml new file mode 100644 index 00000000..d8a7b559 --- /dev/null +++ b/.github/workflows/digital-surface-public-btc-state-observer.yml @@ -0,0 +1,197 @@ +name: Digital Surface Public BTC State Observer + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/digital-surface-public-btc-state-observer.yml + - ci_payload/digital-surface-live-readiness/market-state-source/** + - ci_payload/digital-surface-live-readiness/market-state-observer-source/** + workflow_dispatch: + schedule: + - cron: '2-59/5 * * * *' + +permissions: + contents: read + actions: read + +concurrency: + group: digital-surface-public-observer + cancel-in-progress: false + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + V5_ARTIFACT_ID: '8615107471' + V5_ARTIFACT_ZIP_SHA256: '479a25e235841d94077b3d2df070c8ed46bb736c7ebd8f9f1bde8016198a9c39' + CONTRACT_INVENTORY_SHA256: 'cea26cf5221e81b90ba311317f31e97b54e4110f0f1c716fcb14f06d9f1651f3' + FILTER_CACHE_KEY: digital-surface-diagnostic-filter-v1-cea26cf5221e81b90ba311317f31e97b54e4110f0f1c716fcb14f06d9f1651f3 + +jobs: + observe-public-btc-state: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct observation-only state source + shell: bash + run: | + set -euo pipefail + mkdir -p source/research/digital_surface + touch source/research/__init__.py source/research/digital_surface/__init__.py + cp -a ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/. source/research/digital_surface/ + cp -a ci_payload/digital-surface-live-readiness/prospective-source/research/digital_surface/. source/research/digital_surface/ + cp -a ci_payload/digital-surface-live-readiness/market-state-source/research/digital_surface/. source/research/digital_surface/ + cp -a ci_payload/digital-surface-live-readiness/market-state-observer-source/research/digital_surface/. source/research/digital_surface/ + cat > source/PUBLIC_BTC_STATE_RUNTIME_SOURCE_SHA256.txt <<'EOF' + 157594f006173023f5ee88ee2b1b72f5af3eba431288a448aab86a57bdd1949e research/digital_surface/binance_public.py + f3f1e2417305cd0ceee4b6971e461e488960f04f0e0b80ad292733a2a4b248af research/digital_surface/diagnostic_filter.py + efd186a4ca00c2e920bc87b8d137ac46db52f2aa36385a275dc99e70b5618439 research/digital_surface/market_discovery.py + 3fbb96286445b60b0b7deeb5e1f9ee45f53bfd5efdcc7a727b50617b19463224 research/digital_surface/prospective.py + 73c5aaa7a4ca10baf85886eb28c6135d975eafaaab19e5193819d1ccbf82f2be research/digital_surface/public_observer.py + ead39cd8014cccb0d3036c9d37f292606121a4650bff7429fed873e8417eee10 research/digital_surface/public_state_observer.py + ea56803a606ff33b7231b373141df3d22c159cedc1063603fef3dfb6255c60c9 research/digital_surface/shadow_observation_policy_v1.json + c50d2f58c964e38e2e8a5f30c561cda1ea85c6dbbfc6fa169cca161607874262 research/digital_surface/spot_ledger.py + EOF + sed -i 's/^ //' source/PUBLIC_BTC_STATE_RUNTIME_SOURCE_SHA256.txt + (cd source && sha256sum -c PUBLIC_BTC_STATE_RUNTIME_SOURCE_SHA256.txt) + + - name: Install public-data dependencies + run: | + python -m pip install --upgrade pip + python -m pip install pandas==2.2.3 requests==2.32.3 + + - name: Restore diagnostic filter + id: filter-cache + uses: actions/cache@v4 + with: + path: observer-filter + key: ${{ env.FILTER_CACHE_KEY }} + + - name: Generate diagnostic filter from immutable v5 inventory + if: steps.filter-cache.outputs.cache-hit != 'true' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + curl --fail --location --silent --show-error \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H 'Accept: application/vnd.github+json' \ + -H 'X-GitHub-Api-Version: 2022-11-28' \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/artifacts/${V5_ARTIFACT_ID}/zip" \ + -o v5.zip + echo "${V5_ARTIFACT_ZIP_SHA256} v5.zip" | sha256sum -c - + mkdir v5 observer-filter && unzip -q v5.zip -d v5 + inventory=$(find v5 -type f -name contract_inventory.csv -print -quit) + test -n "${inventory}" + cp "${inventory}" observer-filter/contract_inventory.csv + echo "${CONTRACT_INVENTORY_SHA256} observer-filter/contract_inventory.csv" | sha256sum -c - + PYTHONPATH=source python - <<'PY' + import hashlib + import json + from pathlib import Path + import pandas as pd + from research.digital_surface.diagnostic_filter import build_diagnostic_bloom + + path = Path('observer-filter/contract_inventory.csv') + ids = sorted(set(pd.read_csv(path)['condition_id'].astype(str))) + assert len(ids) == 12_465 + payload = build_diagnostic_bloom(ids, source_sha256=hashlib.sha256(path.read_bytes()).hexdigest()) + assert payload['bitset_sha256'] == '5fd0bc41ab1ae15ddaf9ca33354d803ab74c112d19136240f00d77156186d113' + Path('observer-filter/diagnostic_market_filter.json').write_text(json.dumps(payload, indent=2, sort_keys=True) + '\n') + PY + + - name: Restore append-only observation state + uses: actions/cache/restore@v4 + with: + path: observer-state + key: digital-surface-public-observer-state-${{ github.run_id }} + restore-keys: | + digital-surface-public-observer-state- + + - name: Observe public BTC state + shell: bash + run: | + set -euo pipefail + mkdir -p observer-state observer-output + touch observer-state/prospective_markets.jsonl observer-state/btc_state.jsonl + PYTHONPATH=source python -m research.digital_surface.public_state_observer \ + --diagnostic-filter observer-filter/diagnostic_market_filter.json \ + --prospective-ledger observer-state/prospective_markets.jsonl \ + --spot-ledger observer-state/btc_state.jsonl \ + --policy source/research/digital_surface/shadow_observation_policy_v1.json \ + --source-manifest source/PUBLIC_BTC_STATE_RUNTIME_SOURCE_SHA256.txt \ + --summary observer-output/btc_state_summary.json \ + --page-size 100 \ + --max-pages 20 + + - name: Verify state chain and observation-only boundary + shell: bash + run: | + set -euo pipefail + PYTHONPATH=source python - <<'PY' + import ast + import hashlib + import json + from pathlib import Path + from research.digital_surface.diagnostic_filter import DiagnosticBloomFilter + from research.digital_surface.prospective import ProspectiveLedger + from research.digital_surface.spot_ledger import SpotStateLedger + + summary = json.loads(Path('observer-output/btc_state_summary.json').read_text()) + assert summary['trading_mode'] == 'observation_only' + assert summary['authenticated_requests'] == 0 + assert summary['order_submissions'] == 0 + assert summary['binance_market_data_base'] == 'https://data-api.binance.vision' + + bloom = DiagnosticBloomFilter.from_path('observer-filter/diagnostic_market_filter.json') + prospective = ProspectiveLedger('observer-state/prospective_markets.jsonl', diagnostic_market_ids=bloom) + states = SpotStateLedger('observer-state/btc_state.jsonl') + assert states.head_hash == summary['spot_state_head_sha256'] + assert len(states.rows) == summary['spot_state_observations_total'] + assert summary['prospective_head_sha256'] == prospective.head_hash + assert set(states.rows[-1]['state']['strikes']) == {str(value) for value in summary['strike_boundaries']} + + tree = ast.parse(Path('source/research/digital_surface/binance_public.py').read_text()) + methods = { + node.func.attr for node in ast.walk(tree) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) + and getattr(node.func.value, 'id', None) == 'requests' + } + assert methods == {'get'}, methods + + verification = { + 'workflow_run_id': '${{ github.run_id }}', + 'workflow_run_attempt': '${{ github.run_attempt }}', + 'prospective_head_sha256': prospective.head_hash, + 'spot_state_head_sha256': states.head_hash, + 'spot_state_sha256': hashlib.sha256(Path('observer-state/btc_state.jsonl').read_bytes()).hexdigest(), + 'spot_state_observations_total': len(states.rows), + 'summary': summary, + 'transport_methods': ['GET'], + 'authenticated_requests': 0, + 'order_submissions': 0, + 'trading_mode': 'observation_only', + } + Path('observer-output/BTC_STATE_VERIFICATION.json').write_text(json.dumps(verification, indent=2, sort_keys=True)) + print(json.dumps(verification, indent=2)) + PY + cp observer-state/btc_state.jsonl observer-output/btc_state.jsonl + sha256sum observer-output/btc_state.jsonl observer-output/btc_state_summary.json observer-output/BTC_STATE_VERIFICATION.json > observer-output/BTC_STATE_SHA256SUMS.txt + + - name: Save combined append-only observation state + uses: actions/cache/save@v4 + with: + path: observer-state + key: digital-surface-public-observer-state-${{ github.run_id }} + + - uses: actions/upload-artifact@v4 + with: + name: digital-surface-public-btc-state-${{ github.run_id }} + path: observer-output/ + retention-days: 90 diff --git a/.github/workflows/digital-surface-public-observer-checkpoint.yml b/.github/workflows/digital-surface-public-observer-checkpoint.yml new file mode 100644 index 00000000..47abed6c --- /dev/null +++ b/.github/workflows/digital-surface-public-observer-checkpoint.yml @@ -0,0 +1,271 @@ +name: Digital Surface Public Observer Checkpoint + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/digital-surface-public-observer-checkpoint.yml + - ci_payload/digital-surface-live-readiness/observer-source/** + - ci_payload/digital-surface-live-readiness/prospective-source/** + - ci_payload/digital-surface-live-readiness/shadow-source/** + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-public-observer-checkpoint.yml + - ci_payload/digital-surface-live-readiness/observer-source/** + - ci_payload/digital-surface-live-readiness/prospective-source/** + - ci_payload/digital-surface-live-readiness/shadow-source/** + workflow_dispatch: + +permissions: + contents: read + actions: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + V5_ARTIFACT_ID: '8615107471' + V5_ARTIFACT_ZIP_SHA256: '479a25e235841d94077b3d2df070c8ed46bb736c7ebd8f9f1bde8016198a9c39' + CONTRACT_INVENTORY_SHA256: 'cea26cf5221e81b90ba311317f31e97b54e4110f0f1c716fcb14f06d9f1651f3' + EXPECTED_BLOOM_BITSET_SHA256: '5fd0bc41ab1ae15ddaf9ca33354d803ab74c112d19136240f00d77156186d113' + +jobs: + verify-public-observer: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct checksum-pinned source + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source && tar -xzf base.tar.gz -C source + + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + for patch_file in timestamp-unit-fix.patch full-universe-gate-fix.patch resolved-universe-fix.patch asset-merge-fix.patch; do + patch -d source -p1 < "ci_payload/digital-surface-official-resolution-patch/${patch_file}" + done + echo 'f71f3a653ca58b4bf98609fc29d8ab16f9d031c9ed14906ebe5159f2b0de8eab ci_payload/digital-surface-live-readiness/execution-diagnostics.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/execution-diagnostics.patch + echo '8606cc925aa3b3a6417abb1fba9a11df916cc2b281880978fb6229aa25875be9 ci_payload/digital-surface-live-readiness/book-arrival-model.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/book-arrival-model.patch + + cp -a ci_payload/digital-surface-live-readiness/shadow-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/prospective-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/observer-source/. source/ + + cat > /tmp/observer-source.sha256 <<'EOF' + 6aa02760d4d038098a6a69a99d9349bb5c6fef8d4593fa4c4cf0574971828fb2 source/research/digital_surface/live/__init__.py + 9c5c9bcbd39e1262ac468024f5a96fb0667c0c95fc2879d846f7ca2066c38b5f source/research/digital_surface/live/config.py + 9e6dff73a17227f907dce4bf56cfd54921f4eb677280b1774d1a73c979b9065a source/research/digital_surface/live/gateway.py + 3be74a910e787a3f9e0f3548691e63ed93d78eeb3328fd9538e3a0e643b57d1c source/research/digital_surface/live/report.py + b0ca12139437c004ec88e41902796cdd540f7e45c1b27c3aba96e3695d2267a1 source/research/digital_surface/live/report_chain.py + f9197d38fd8dcfcc073d83028ac498fd9ff07619fea2b2b6a90de9ab470a86e2 source/research/digital_surface/live/risk.py + 50f32b2e400767702073b97ba4fd77a7e8c26f5fd4c7d6f7d9aafe282d6258f0 source/research/digital_surface/live/types.py + 10d833f89194246ea2fe5dec8bb2fc4388c8509a4d8276c7fcdc0a2b1a13e76c source/tests/digital_surface/live/test_report.py + a09860af2f96e950884e1a2d37d0dfe59c41d869f3cb183ef839bbec2896db4a source/tests/digital_surface/live/test_report_chain.py + be1f59a4ae742f062b419ad7f7fb974eadf9bd076cb2ecdabcd0063eedf1df6d source/tests/digital_surface/live/test_shadow_gateway.py + f3f1e2417305cd0ceee4b6971e461e488960f04f0e0b80ad292733a2a4b248af source/research/digital_surface/diagnostic_filter.py + efd186a4ca00c2e920bc87b8d137ac46db52f2aa36385a275dc99e70b5618439 source/research/digital_surface/market_discovery.py + 3fbb96286445b60b0b7deeb5e1f9ee45f53bfd5efdcc7a727b50617b19463224 source/research/digital_surface/prospective.py + 73c5aaa7a4ca10baf85886eb28c6135d975eafaaab19e5193819d1ccbf82f2be source/research/digital_surface/public_observer.py + ea56803a606ff33b7231b373141df3d22c159cedc1063603fef3dfb6255c60c9 source/research/digital_surface/shadow_observation_policy_v1.json + 774575a18473899bfd2304f93be3786c60c0288fd28aa98d94a40aa043c74858 source/tests/digital_surface/test_diagnostic_filter.py + adf17ffb26a23332194eaa67c07201e713f2edf81927365cb54b21a348b5a7bd source/tests/digital_surface/test_market_discovery.py + 6efbf535ddfb50ca4b88209ecfa72c8e208a9f4049b0960781805cb888449b53 source/tests/digital_surface/test_prospective.py + 87e32c6fc215d8ffe8f587357acd1d3efc12b1c78021f2f01740ebc06b9a0554 source/tests/digital_surface/test_public_observer.py + EOF + sed -i 's/^ //' /tmp/observer-source.sha256 + sha256sum -c /tmp/observer-source.sha256 + cp /tmp/observer-source.sha256 source/PUBLIC_OBSERVER_SOURCE_SHA256.txt + + - name: Install pinned test stack + run: | + python -m pip install --upgrade pip + python -m pip install numpy==2.2.6 pandas==2.2.3 scipy==1.15.3 scikit-learn==1.6.1 pyarrow==19.0.1 requests==2.32.3 pytest==8.3.5 tabulate==0.9.0 + + - name: Download immutable v5 inventory + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + curl --fail --location --silent --show-error \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H 'Accept: application/vnd.github+json' \ + -H 'X-GitHub-Api-Version: 2022-11-28' \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/artifacts/${V5_ARTIFACT_ID}/zip" \ + -o v5.zip + echo "${V5_ARTIFACT_ZIP_SHA256} v5.zip" | sha256sum -c - + mkdir v5 && unzip -q v5.zip -d v5 + inventory=$(find v5 -type f -name contract_inventory.csv -print -quit) + test -n "${inventory}" + cp "${inventory}" source/contract_inventory.csv + echo "${CONTRACT_INVENTORY_SHA256} source/contract_inventory.csv" | sha256sum -c - + + - name: Regenerate and verify diagnostic Bloom filter + working-directory: source + shell: bash + run: | + set -euo pipefail + PYTHONPATH=. python - <<'PY' + import hashlib + import json + from pathlib import Path + import pandas as pd + from research.digital_surface.diagnostic_filter import DiagnosticBloomFilter, build_diagnostic_bloom + + path = Path('contract_inventory.csv') + frame = pd.read_csv(path) + assert 'condition_id' in frame.columns, frame.columns.tolist() + ids = sorted(set(frame['condition_id'].astype(str))) + assert len(ids) == 12_465, len(ids) + source_sha = hashlib.sha256(path.read_bytes()).hexdigest() + payload = build_diagnostic_bloom(ids, source_sha256=source_sha) + assert payload['item_count'] == 12_465 + assert payload['m_bits'] == 262_144 + assert payload['k_hashes'] == 7 + assert payload['bitset_sha256'] == '5fd0bc41ab1ae15ddaf9ca33354d803ab74c112d19136240f00d77156186d113' + Path('diagnostic_market_filter.json').write_text(json.dumps(payload, indent=2, sort_keys=True) + '\n') + loaded = DiagnosticBloomFilter.from_path('diagnostic_market_filter.json') + assert loaded.item_count == 12_465 + assert loaded.source_sha256 == source_sha + assert all(value in loaded for value in ids) + canonical_ids = ('\n'.join(ids) + '\n').encode() + ids_sha = hashlib.sha256(canonical_ids).hexdigest() + assert ids_sha == '4605a921c49ac3a73766f96f8f2cfd13b2436c7925f17b673b4b8f1c210a56e5' + Path('diagnostic_market_ids.sha256').write_text(ids_sha + '\n') + PY + + - name: Compile and pass exactly 53 tests + working-directory: source + shell: bash + run: | + set -euo pipefail + python -m compileall -q research/digital_surface tests/digital_surface + PYTHONPATH=. pytest --collect-only -q tests/digital_surface | tee /tmp/collect.txt + grep -Eq '^53 tests collected' /tmp/collect.txt + PYTHONPATH=. pytest tests/digital_surface -q + + - name: Prove observation-only boundary and hash chains + working-directory: source + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import ast + import hashlib + import json + import tempfile + from pathlib import Path + + from research.digital_surface.diagnostic_filter import DiagnosticBloomFilter + from research.digital_surface.live import LiveTradingDisabledError, load_config + from research.digital_surface.market_discovery import GammaMarketRecord + from research.digital_surface.prospective import ProspectiveLedger + from research.digital_surface.public_observer import GAMMA_MARKETS_URL, observe_markets + + try: + load_config({'TRADING_MODE': 'live', 'LIVE_ARM_TOKEN': 'present'}) + except LiveTradingDisabledError: + pass + else: + raise AssertionError('live mode unexpectedly armed') + + policy = json.loads(Path('research/digital_surface/shadow_observation_policy_v1.json').read_text()) + assert policy['mode'] == 'observation_only' + assert policy['authenticated_requests'] is False + assert policy['order_submission'] is False + assert GAMMA_MARKETS_URL == 'https://gamma-api.polymarket.com/markets' + + observer_tree = ast.parse(Path('research/digital_surface/public_observer.py').read_text()) + call_names = { + node.func.attr for node in ast.walk(observer_tree) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) + } + assert not ({'post', 'put', 'patch', 'delete'} & call_names), call_names + + bloom = DiagnosticBloomFilter.from_path('diagnostic_market_filter.json') + with tempfile.TemporaryDirectory() as directory: + directory = Path(directory) + ledger = ProspectiveLedger(directory/'ledger.jsonl', diagnostic_market_ids=bloom) + market = GammaMarketRecord( + condition_id='prospective-smoke-test', + slug='btc-updown-5m-1999999800', + question='Bitcoin Up or Down', + yes_token_id='yes-prospective', + no_token_id='no-prospective', + duration_seconds=300, + open_epoch_seconds=1_999_999_800, + end_date='2033-05-18T03:35:00Z', + tick_size=__import__('decimal').Decimal('0.01'), + ) + summary = observe_markets( + [market], ledger=ledger, observed_ts_ms=1, + policy_sha256='1'*64, source_sha256='2'*64, + ) + assert summary['appended'] == 1 + assert summary['markets_total_in_ledger'] == 1 + reloaded = ProspectiveLedger(ledger.path, diagnostic_market_ids=bloom) + assert reloaded.head_hash == summary['ledger_head_sha256'] + row = json.loads(ledger.path.read_text()) + assert row['metadata']['yes_token_id'] == 'yes-prospective' + assert row['metadata']['tick_size'] == '0.01' + + verified = {} + for line in Path('PUBLIC_OBSERVER_SOURCE_SHA256.txt').read_text().splitlines(): + wanted, name = line.split(maxsplit=1) + got = hashlib.sha256(Path(name).read_bytes()).hexdigest() + assert got == wanted, (name, got, wanted) + verified[name] = got + result = { + 'tests_collected': 53, + 'historical_condition_ids': 12_465, + 'historical_filter_bitset_sha256': bloom.bitset.hex() and hashlib.sha256(bloom.bitset).hexdigest(), + 'public_market_endpoint': GAMMA_MARKETS_URL, + 'authenticated_requests': 0, + 'order_submissions': 0, + 'trading_mode': 'observation_only', + 'live_mode': 'physically_disabled', + 'prospective_hash_chain_verified': True, + 'source_files_verified': verified, + } + assert result['historical_filter_bitset_sha256'] == '5fd0bc41ab1ae15ddaf9ca33354d803ab74c112d19136240f00d77156186d113' + Path('PUBLIC_OBSERVER_VERIFICATION.json').write_text(json.dumps(result, indent=2, sort_keys=True)) + print(json.dumps(result, indent=2)) + PY + + - name: Build deterministic source checkpoint + working-directory: source + shell: bash + run: | + set -euo pipefail + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-public-observer-source-v1.tar.gz \ + research tests PUBLIC_OBSERVER_SOURCE_SHA256.txt PUBLIC_OBSERVER_VERIFICATION.json \ + diagnostic_market_filter.json diagnostic_market_ids.sha256 + sha256sum digital-surface-public-observer-source-v1.tar.gz > digital-surface-public-observer-source-v1.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: digital-surface-public-observer-checkpoint-v1 + path: | + source/PUBLIC_OBSERVER_SOURCE_SHA256.txt + source/PUBLIC_OBSERVER_VERIFICATION.json + source/diagnostic_market_filter.json + source/diagnostic_market_ids.sha256 + source/digital-surface-public-observer-source-v1.tar.gz + source/digital-surface-public-observer-source-v1.tar.gz.sha256 + retention-days: 30 diff --git a/.github/workflows/digital-surface-public-observer.yml b/.github/workflows/digital-surface-public-observer.yml new file mode 100644 index 00000000..b417065e --- /dev/null +++ b/.github/workflows/digital-surface-public-observer.yml @@ -0,0 +1,225 @@ +name: Digital Surface Public Observer + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/digital-surface-public-observer.yml + - ci_payload/digital-surface-live-readiness/observer-source/** + - ci_payload/digital-surface-live-readiness/prospective-source/** + workflow_dispatch: + schedule: + - cron: '*/5 * * * *' + +permissions: + contents: read + actions: read + +concurrency: + group: digital-surface-public-observer + cancel-in-progress: false + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + V5_ARTIFACT_ID: '8615107471' + V5_ARTIFACT_ZIP_SHA256: '479a25e235841d94077b3d2df070c8ed46bb736c7ebd8f9f1bde8016198a9c39' + CONTRACT_INVENTORY_SHA256: 'cea26cf5221e81b90ba311317f31e97b54e4110f0f1c716fcb14f06d9f1651f3' + FILTER_CACHE_KEY: digital-surface-diagnostic-filter-v1-cea26cf5221e81b90ba311317f31e97b54e4110f0f1c716fcb14f06d9f1651f3 + +jobs: + observe-public-markets: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct observation-only source + shell: bash + run: | + set -euo pipefail + mkdir -p source/research/digital_surface + touch source/research/__init__.py source/research/digital_surface/__init__.py + cp -a ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/. source/research/digital_surface/ + cp -a ci_payload/digital-surface-live-readiness/prospective-source/research/digital_surface/. source/research/digital_surface/ + cat > source/PUBLIC_OBSERVER_RUNTIME_SOURCE_SHA256.txt <<'EOF' + d96e722e57b24b2dc37bdf772972133ceeb1fcbf3629973f3abae701b4943051 research/digital_surface/book_ledger.py + f3f1e2417305cd0ceee4b6971e461e488960f04f0e0b80ad292733a2a4b248af research/digital_surface/diagnostic_filter.py + efd186a4ca00c2e920bc87b8d137ac46db52f2aa36385a275dc99e70b5618439 research/digital_surface/market_discovery.py + 3fbb96286445b60b0b7deeb5e1f9ee45f53bfd5efdcc7a727b50617b19463224 research/digital_surface/prospective.py + 088bab6729e5a1beb22baa853483433fd23180d3f8597035e0d3a4d036a8aec3 research/digital_surface/public_book_observer.py + 1ee608819769dad77c19f343427fbe604cacb3afc95b693aa5153b4ea22a166e research/digital_surface/public_books.py + 73c5aaa7a4ca10baf85886eb28c6135d975eafaaab19e5193819d1ccbf82f2be research/digital_surface/public_observer.py + ea56803a606ff33b7231b373141df3d22c159cedc1063603fef3dfb6255c60c9 research/digital_surface/shadow_observation_policy_v1.json + EOF + sed -i 's/^ //' source/PUBLIC_OBSERVER_RUNTIME_SOURCE_SHA256.txt + (cd source && sha256sum -c PUBLIC_OBSERVER_RUNTIME_SOURCE_SHA256.txt) + + - name: Install observation-only dependencies + run: | + python -m pip install --upgrade pip + python -m pip install pandas==2.2.3 requests==2.32.3 + + - name: Restore diagnostic filter + id: filter-cache + uses: actions/cache@v4 + with: + path: observer-filter + key: ${{ env.FILTER_CACHE_KEY }} + + - name: Generate diagnostic filter from immutable v5 inventory + if: steps.filter-cache.outputs.cache-hit != 'true' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + curl --fail --location --silent --show-error \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H 'Accept: application/vnd.github+json' \ + -H 'X-GitHub-Api-Version: 2022-11-28' \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/artifacts/${V5_ARTIFACT_ID}/zip" \ + -o v5.zip + echo "${V5_ARTIFACT_ZIP_SHA256} v5.zip" | sha256sum -c - + mkdir v5 observer-filter && unzip -q v5.zip -d v5 + inventory=$(find v5 -type f -name contract_inventory.csv -print -quit) + test -n "${inventory}" + cp "${inventory}" observer-filter/contract_inventory.csv + echo "${CONTRACT_INVENTORY_SHA256} observer-filter/contract_inventory.csv" | sha256sum -c - + PYTHONPATH=source python - <<'PY' + import hashlib + import json + from pathlib import Path + import pandas as pd + from research.digital_surface.diagnostic_filter import build_diagnostic_bloom + + path = Path('observer-filter/contract_inventory.csv') + frame = pd.read_csv(path) + ids = sorted(set(frame['condition_id'].astype(str))) + assert len(ids) == 12_465 + source_sha = hashlib.sha256(path.read_bytes()).hexdigest() + payload = build_diagnostic_bloom(ids, source_sha256=source_sha) + assert payload['bitset_sha256'] == '5fd0bc41ab1ae15ddaf9ca33354d803ab74c112d19136240f00d77156186d113' + Path('observer-filter/diagnostic_market_filter.json').write_text( + json.dumps(payload, indent=2, sort_keys=True) + '\n' + ) + PY + + - name: Verify restored diagnostic filter + shell: bash + run: | + set -euo pipefail + PYTHONPATH=source python - <<'PY' + import hashlib + from research.digital_surface.diagnostic_filter import DiagnosticBloomFilter + bloom = DiagnosticBloomFilter.from_path('observer-filter/diagnostic_market_filter.json') + assert bloom.item_count == 12_465 + assert bloom.m_bits == 262_144 + assert bloom.k_hashes == 7 + assert bloom.source_sha256 == 'cea26cf5221e81b90ba311317f31e97b54e4110f0f1c716fcb14f06d9f1651f3' + assert hashlib.sha256(bloom.bitset).hexdigest() == '5fd0bc41ab1ae15ddaf9ca33354d803ab74c112d19136240f00d77156186d113' + PY + + - name: Restore append-only observation state + uses: actions/cache/restore@v4 + with: + path: observer-state + key: digital-surface-public-observer-state-${{ github.run_id }} + restore-keys: | + digital-surface-public-observer-state- + + - name: Observe public markets and orderbooks + shell: bash + run: | + set -euo pipefail + mkdir -p observer-state observer-output + touch observer-state/prospective_markets.jsonl observer-state/public_books.jsonl + PYTHONPATH=source python -m research.digital_surface.public_observer \ + --diagnostic-filter observer-filter/diagnostic_market_filter.json \ + --ledger observer-state/prospective_markets.jsonl \ + --policy source/research/digital_surface/shadow_observation_policy_v1.json \ + --source-manifest source/PUBLIC_OBSERVER_RUNTIME_SOURCE_SHA256.txt \ + --summary observer-output/observer_summary.json \ + --page-size 100 \ + --max-pages 20 + PYTHONPATH=source python -m research.digital_surface.public_book_observer \ + --diagnostic-filter observer-filter/diagnostic_market_filter.json \ + --prospective-ledger observer-state/prospective_markets.jsonl \ + --book-ledger observer-state/public_books.jsonl \ + --policy source/research/digital_surface/shadow_observation_policy_v1.json \ + --source-manifest source/PUBLIC_OBSERVER_RUNTIME_SOURCE_SHA256.txt \ + --summary observer-output/book_observer_summary.json \ + --page-size 100 \ + --max-pages 20 + + - name: Verify observation-only artifacts + shell: bash + run: | + set -euo pipefail + PYTHONPATH=source python - <<'PY' + import hashlib + import json + from pathlib import Path + from research.digital_surface.book_ledger import PublicBookLedger + from research.digital_surface.diagnostic_filter import DiagnosticBloomFilter + from research.digital_surface.prospective import ProspectiveLedger + + market_summary = json.loads(Path('observer-output/observer_summary.json').read_text()) + book_summary = json.loads(Path('observer-output/book_observer_summary.json').read_text()) + for summary in (market_summary, book_summary): + assert summary['trading_mode'] == 'observation_only' + assert summary['authenticated_requests'] == 0 + assert summary['order_submissions'] == 0 + + bloom = DiagnosticBloomFilter.from_path('observer-filter/diagnostic_market_filter.json') + markets = ProspectiveLedger('observer-state/prospective_markets.jsonl', diagnostic_market_ids=bloom) + books = PublicBookLedger('observer-state/public_books.jsonl') + assert markets.head_hash == market_summary['ledger_head_sha256'] + assert len(markets.market_ids) == market_summary['markets_total_in_ledger'] + assert books.head_hash == book_summary['book_ledger_head_sha256'] + assert len(books.rows) == book_summary['book_observations_total'] + assert book_summary['prospective_head_sha256'] == markets.head_hash + assert book_summary['public_books_endpoint'] == 'https://clob.polymarket.com/books' + + verification = { + 'workflow_run_id': '${{ github.run_id }}', + 'workflow_run_attempt': '${{ github.run_attempt }}', + 'market_ledger_head_sha256': markets.head_hash, + 'market_ledger_sha256': hashlib.sha256(Path('observer-state/prospective_markets.jsonl').read_bytes()).hexdigest(), + 'book_ledger_head_sha256': books.head_hash, + 'book_ledger_sha256': hashlib.sha256(Path('observer-state/public_books.jsonl').read_bytes()).hexdigest(), + 'markets_total_in_ledger': len(markets.market_ids), + 'book_observations_total': len(books.rows), + 'market_summary': market_summary, + 'book_summary': book_summary, + 'authenticated_requests': 0, + 'order_submissions': 0, + 'trading_mode': 'observation_only', + } + Path('observer-output/OBSERVER_VERIFICATION.json').write_text(json.dumps(verification, indent=2, sort_keys=True)) + print(json.dumps(verification, indent=2)) + PY + cp observer-state/prospective_markets.jsonl observer-output/prospective_markets.jsonl + cp observer-state/public_books.jsonl observer-output/public_books.jsonl + sha256sum \ + observer-output/prospective_markets.jsonl \ + observer-output/public_books.jsonl \ + observer-output/observer_summary.json \ + observer-output/book_observer_summary.json \ + observer-output/OBSERVER_VERIFICATION.json \ + > observer-output/SHA256SUMS.txt + + - name: Save append-only observation state + uses: actions/cache/save@v4 + with: + path: observer-state + key: digital-surface-public-observer-state-${{ github.run_id }} + + - uses: actions/upload-artifact@v4 + with: + name: digital-surface-public-observer-${{ github.run_id }} + path: observer-output/ + retention-days: 90 diff --git a/.github/workflows/digital-surface-shadow-fill-simulation-checkpoint.yml b/.github/workflows/digital-surface-shadow-fill-simulation-checkpoint.yml new file mode 100644 index 00000000..b3c0f9ec --- /dev/null +++ b/.github/workflows/digital-surface-shadow-fill-simulation-checkpoint.yml @@ -0,0 +1,169 @@ +name: Digital Surface Shadow Fill Simulation Checkpoint + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/digital-surface-shadow-fill-simulation-checkpoint.yml + - ci_payload/digital-surface-live-readiness/simulation-source/** + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-shadow-fill-simulation-checkpoint.yml + - ci_payload/digital-surface-live-readiness/simulation-source/** + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + verify-shadow-fill-simulation: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct checksum-pinned source + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source && tar -xzf base.tar.gz -C source + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + for patch_file in timestamp-unit-fix.patch full-universe-gate-fix.patch resolved-universe-fix.patch asset-merge-fix.patch; do + patch -d source -p1 < "ci_payload/digital-surface-official-resolution-patch/${patch_file}" + done + echo 'f71f3a653ca58b4bf98609fc29d8ab16f9d031c9ed14906ebe5159f2b0de8eab ci_payload/digital-surface-live-readiness/execution-diagnostics.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/execution-diagnostics.patch + echo '8606cc925aa3b3a6417abb1fba9a11df916cc2b281880978fb6229aa25875be9 ci_payload/digital-surface-live-readiness/book-arrival-model.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/book-arrival-model.patch + cp -a ci_payload/digital-surface-live-readiness/shadow-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/prospective-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/observer-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/simulation-source/. source/ + + cat > /tmp/simulation-source.sha256 <<'EOF' + 0808805ad3dcbb1c74ef437dddd33fc71b0800b4fd8ecf5daef5967de08bafe5 source/research/digital_surface/shadow_fill.py + d2fe18723b9b7780d3db82cef22f4d45c70c6e1f38e5fd5bcbaf72907eb6fb61 source/tests/digital_surface/test_shadow_fill.py + EOF + sed -i 's/^ //' /tmp/simulation-source.sha256 + sha256sum -c /tmp/simulation-source.sha256 + cp /tmp/simulation-source.sha256 source/SHADOW_FILL_SIMULATION_SOURCE_SHA256.txt + + - name: Install pinned test stack + run: | + python -m pip install --upgrade pip + python -m pip install numpy==2.2.6 pandas==2.2.3 scipy==1.15.3 scikit-learn==1.6.1 pyarrow==19.0.1 requests==2.32.3 pytest==8.3.5 tabulate==0.9.0 + + - name: Compile and pass exactly 64 tests + working-directory: source + shell: bash + run: | + set -euo pipefail + python -m compileall -q research/digital_surface tests/digital_surface + PYTHONPATH=. pytest --collect-only -q tests/digital_surface | tee /tmp/collect.txt + grep -Eq '^64 tests collected' /tmp/collect.txt + PYTHONPATH=. pytest tests/digital_surface -q + + - name: Verify non-atomic paired FOK semantics + working-directory: source + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import hashlib + import json + from decimal import Decimal + from pathlib import Path + + from research.digital_surface.public_books import parse_public_order_book + from research.digital_surface.shadow_fill import estimate_paired_fok + + def book(token, condition, asks): + return parse_public_order_book({ + 'market': condition, + 'asset_id': token, + 'timestamp': '1', + 'hash': 'h-' + token, + 'bids': [], + 'asks': [{'price': price, 'size': size} for price, size in asks], + 'min_order_size': '1', + 'tick_size': '0.01', + 'neg_risk': False, + }) + + paired = estimate_paired_fok( + book('yes-low', 'low', [('0.50', '2'), ('0.51', '3')]), + book('no-high', 'high', [('0.48', '5')]), + shares=Decimal('5'), + low_max_price=Decimal('0.51'), + high_max_price=Decimal('0.49'), + ) + assert paired.paired_crossable is True + assert paired.transactionally_atomic is False + assert paired.orphanable_leg is None + assert paired.low.average_price == Decimal('0.506') + assert paired.high.average_price == Decimal('0.48') + + orphanable = estimate_paired_fok( + book('yes-low', 'low', [('0.50', '5')]), + book('no-high', 'high', [('0.60', '5')]), + shares=Decimal('5'), + low_max_price=Decimal('0.51'), + high_max_price=Decimal('0.49'), + ) + assert orphanable.paired_crossable is False + assert orphanable.orphanable_leg == 'low_yes' + assert orphanable.transactionally_atomic is False + + verified = {} + for line in Path('SHADOW_FILL_SIMULATION_SOURCE_SHA256.txt').read_text().splitlines(): + wanted, name = line.split(maxsplit=1) + got = hashlib.sha256(Path(name).read_bytes()).hexdigest() + assert got == wanted + verified[name] = got + result = { + 'tests_collected': 64, + 'paired_crossable_is_not_atomic': True, + 'partial_fok_counted_as_unfilled': True, + 'causal_digest_binding_required': True, + 'order_submissions': 0, + 'live_mode': 'physically_disabled', + 'source_files_verified': verified, + } + Path('SHADOW_FILL_SIMULATION_VERIFICATION.json').write_text(json.dumps(result, indent=2, sort_keys=True)) + print(json.dumps(result, indent=2)) + PY + + - name: Build deterministic source checkpoint + working-directory: source + shell: bash + run: | + set -euo pipefail + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-shadow-fill-simulation-source-v1.tar.gz \ + research tests SHADOW_FILL_SIMULATION_SOURCE_SHA256.txt SHADOW_FILL_SIMULATION_VERIFICATION.json + sha256sum digital-surface-shadow-fill-simulation-source-v1.tar.gz > digital-surface-shadow-fill-simulation-source-v1.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: digital-surface-shadow-fill-simulation-checkpoint-v1 + path: | + source/SHADOW_FILL_SIMULATION_SOURCE_SHA256.txt + source/SHADOW_FILL_SIMULATION_VERIFICATION.json + source/digital-surface-shadow-fill-simulation-source-v1.tar.gz + source/digital-surface-shadow-fill-simulation-source-v1.tar.gz.sha256 + retention-days: 30 diff --git a/.github/workflows/digital-surface-shadow-prospective-checkpoint.yml b/.github/workflows/digital-surface-shadow-prospective-checkpoint.yml new file mode 100644 index 00000000..dc670aaf --- /dev/null +++ b/.github/workflows/digital-surface-shadow-prospective-checkpoint.yml @@ -0,0 +1,178 @@ +name: Digital Surface Shadow and Prospective Checkpoint + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/digital-surface-shadow-prospective-checkpoint.yml + - ci_payload/digital-surface-live-readiness/shadow-source/** + - ci_payload/digital-surface-live-readiness/prospective-source/** + - ci_payload/digital-surface-live-readiness/book-arrival-model.patch + - ci_payload/digital-surface-live-readiness/execution-diagnostics.patch + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-shadow-prospective-checkpoint.yml + - ci_payload/digital-surface-live-readiness/shadow-source/** + - ci_payload/digital-surface-live-readiness/prospective-source/** + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + verify-shadow-and-prospective: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct checksum-pinned source + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source && tar -xzf base.tar.gz -C source + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + for patch_file in timestamp-unit-fix.patch full-universe-gate-fix.patch resolved-universe-fix.patch asset-merge-fix.patch; do + patch -d source -p1 < "ci_payload/digital-surface-official-resolution-patch/${patch_file}" + done + echo 'f71f3a653ca58b4bf98609fc29d8ab16f9d031c9ed14906ebe5159f2b0de8eab ci_payload/digital-surface-live-readiness/execution-diagnostics.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/execution-diagnostics.patch + echo '8606cc925aa3b3a6417abb1fba9a11df916cc2b281880978fb6229aa25875be9 ci_payload/digital-surface-live-readiness/book-arrival-model.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/book-arrival-model.patch + cp -a ci_payload/digital-surface-live-readiness/shadow-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/prospective-source/. source/ + + cat > /tmp/checkpoint.sha256 <<'EOF' + ff4025e5eee69cdad8e0bbbee4e60b16455da2710da61a733c4151732e4a2cce source/research/digital_surface/live/__init__.py + 9c5c9bcbd39e1262ac468024f5a96fb0667c0c95fc2879d846f7ca2066c38b5f source/research/digital_surface/live/config.py + 9e6dff73a17227f907dce4bf56cfd54921f4eb677280b1774d1a73c979b9065a source/research/digital_surface/live/gateway.py + f9197d38fd8dcfcc073d83028ac498fd9ff07619fea2b2b6a90de9ab470a86e2 source/research/digital_surface/live/risk.py + 50f32b2e400767702073b97ba4fd77a7e8c26f5fd4c7d6f7d9aafe282d6258f0 source/research/digital_surface/live/types.py + be1f59a4ae742f062b419ad7f7fb974eadf9bd076cb2ecdabcd0063eedf1df6d source/tests/digital_surface/live/test_shadow_gateway.py + f219222d7b89c5ab4ef0619da9163a77b79ba7f2a9fe611c4072746771e2fa41 source/research/digital_surface/prospective.py + 9ed734554e0296e94212cac3911466ac230241efdd4ab911e9adc74e41ef05b2 source/tests/digital_surface/test_prospective.py + EOF + sed -i 's/^ //' /tmp/checkpoint.sha256 + sha256sum -c /tmp/checkpoint.sha256 + cp /tmp/checkpoint.sha256 source/SHADOW_PROSPECTIVE_SOURCE_SHA256.txt + + - name: Install pinned test stack + run: | + python -m pip install --upgrade pip + python -m pip install numpy==2.2.6 pandas==2.2.3 scipy==1.15.3 scikit-learn==1.6.1 pyarrow==19.0.1 requests==2.32.3 pytest==8.3.5 tabulate==0.9.0 + + - name: Compile and pass exactly 39 tests + working-directory: source + shell: bash + run: | + set -euo pipefail + python -m compileall -q research/digital_surface tests/digital_surface + PYTHONPATH=. pytest --collect-only -q tests/digital_surface | tee /tmp/collect.txt + grep -Eq '^39 tests collected' /tmp/collect.txt + PYTHONPATH=. pytest tests/digital_surface -q + + - name: Prove live submission is absent and prospective chain is valid + working-directory: source + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import ast + import hashlib + import json + import tempfile + from pathlib import Path + + from research.digital_surface.live import LiveTradingDisabledError, load_config + from research.digital_surface.prospective import ProspectiveLedger + + try: + load_config({ + 'TRADING_MODE': 'live', + 'POLYMARKET_API_KEY': 'present', + 'POLYMARKET_PRIVATE_KEY': 'present', + 'LIVE_ARM_TOKEN': 'present', + }) + except LiveTradingDisabledError: + pass + else: + raise AssertionError('live mode unexpectedly armed') + + forbidden_import_roots = {'requests', 'httpx', 'aiohttp', 'websocket', 'websockets', 'socket'} + live_root = Path('research/digital_surface/live') + seen_imports = set() + for path in live_root.glob('*.py'): + tree = ast.parse(path.read_text(), filename=str(path)) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + seen_imports.update(alias.name.split('.')[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + seen_imports.add(node.module.split('.')[0]) + assert not (seen_imports & forbidden_import_roots), seen_imports & forbidden_import_roots + + with tempfile.TemporaryDirectory() as directory: + ledger = ProspectiveLedger(Path(directory) / 'ledger.jsonl', diagnostic_market_ids={'old'}) + first = ledger.append_observation( + market_id='fresh-1', first_seen_ts_ms=1, + policy_sha256='1' * 64, source_sha256='2' * 64, + ) + second = ledger.append_observation( + market_id='fresh-2', first_seen_ts_ms=2, + policy_sha256='1' * 64, source_sha256='2' * 64, + ) + assert first['previous_hash'] == '0' * 64 + assert second['previous_hash'] == first['record_hash'] + reloaded = ProspectiveLedger(ledger.path, diagnostic_market_ids={'old'}) + assert reloaded.head_hash == second['record_hash'] + + files = {} + for line in Path('SHADOW_PROSPECTIVE_SOURCE_SHA256.txt').read_text().splitlines(): + wanted, name = line.split(maxsplit=1) + got = hashlib.sha256(Path(name).read_bytes()).hexdigest() + assert got == wanted + files[name] = got + result = { + 'tests_collected': 39, + 'live_mode': 'physically_disabled', + 'network_capable_imports': [], + 'prospective_hash_chain_verified': True, + 'source_files_verified': files, + } + Path('SHADOW_PROSPECTIVE_VERIFICATION.json').write_text(json.dumps(result, indent=2, sort_keys=True)) + print(json.dumps(result, indent=2)) + PY + + - name: Build deterministic source checkpoint + working-directory: source + shell: bash + run: | + set -euo pipefail + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-shadow-prospective-source-v1.tar.gz \ + research tests SHADOW_PROSPECTIVE_SOURCE_SHA256.txt SHADOW_PROSPECTIVE_VERIFICATION.json + sha256sum digital-surface-shadow-prospective-source-v1.tar.gz > digital-surface-shadow-prospective-source-v1.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: digital-surface-shadow-prospective-checkpoint-v1 + path: | + source/SHADOW_PROSPECTIVE_SOURCE_SHA256.txt + source/SHADOW_PROSPECTIVE_VERIFICATION.json + source/digital-surface-shadow-prospective-source-v1.tar.gz + source/digital-surface-shadow-prospective-source-v1.tar.gz.sha256 + retention-days: 30 diff --git a/.github/workflows/digital-surface-shadow-prospective-report-checkpoint.yml b/.github/workflows/digital-surface-shadow-prospective-report-checkpoint.yml new file mode 100644 index 00000000..67fb697f --- /dev/null +++ b/.github/workflows/digital-surface-shadow-prospective-report-checkpoint.yml @@ -0,0 +1,199 @@ +name: Digital Surface Shadow Prospective Report Checkpoint + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/digital-surface-shadow-prospective-report-checkpoint.yml + - ci_payload/digital-surface-live-readiness/shadow-source/** + - ci_payload/digital-surface-live-readiness/prospective-source/** + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-shadow-prospective-report-checkpoint.yml + - ci_payload/digital-surface-live-readiness/shadow-source/** + - ci_payload/digital-surface-live-readiness/prospective-source/** + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + verify-shadow-prospective-report: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct checksum-pinned source + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source && tar -xzf base.tar.gz -C source + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + for patch_file in timestamp-unit-fix.patch full-universe-gate-fix.patch resolved-universe-fix.patch asset-merge-fix.patch; do + patch -d source -p1 < "ci_payload/digital-surface-official-resolution-patch/${patch_file}" + done + echo 'f71f3a653ca58b4bf98609fc29d8ab16f9d031c9ed14906ebe5159f2b0de8eab ci_payload/digital-surface-live-readiness/execution-diagnostics.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/execution-diagnostics.patch + echo '8606cc925aa3b3a6417abb1fba9a11df916cc2b281880978fb6229aa25875be9 ci_payload/digital-surface-live-readiness/book-arrival-model.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/book-arrival-model.patch + cp -a ci_payload/digital-surface-live-readiness/shadow-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/prospective-source/. source/ + + cat > /tmp/checkpoint.sha256 <<'EOF' + af2a1e50e84f13064424f662bf02847df92ca63abff1692fbfea866ac71c3816 source/research/digital_surface/live/__init__.py + 9c5c9bcbd39e1262ac468024f5a96fb0667c0c95fc2879d846f7ca2066c38b5f source/research/digital_surface/live/config.py + 9e6dff73a17227f907dce4bf56cfd54921f4eb677280b1774d1a73c979b9065a source/research/digital_surface/live/gateway.py + 3be74a910e787a3f9e0f3548691e63ed93d78eeb3328fd9538e3a0e643b57d1c source/research/digital_surface/live/report.py + f9197d38fd8dcfcc073d83028ac498fd9ff07619fea2b2b6a90de9ab470a86e2 source/research/digital_surface/live/risk.py + 50f32b2e400767702073b97ba4fd77a7e8c26f5fd4c7d6f7d9aafe282d6258f0 source/research/digital_surface/live/types.py + 10d833f89194246ea2fe5dec8bb2fc4388c8509a4d8276c7fcdc0a2b1a13e76c source/tests/digital_surface/live/test_report.py + be1f59a4ae742f062b419ad7f7fb974eadf9bd076cb2ecdabcd0063eedf1df6d source/tests/digital_surface/live/test_shadow_gateway.py + f219222d7b89c5ab4ef0619da9163a77b79ba7f2a9fe611c4072746771e2fa41 source/research/digital_surface/prospective.py + 9ed734554e0296e94212cac3911466ac230241efdd4ab911e9adc74e41ef05b2 source/tests/digital_surface/test_prospective.py + EOF + sed -i 's/^ //' /tmp/checkpoint.sha256 + sha256sum -c /tmp/checkpoint.sha256 + cp /tmp/checkpoint.sha256 source/SHADOW_PROSPECTIVE_REPORT_SOURCE_SHA256.txt + + - name: Install pinned test stack + run: | + python -m pip install --upgrade pip + python -m pip install numpy==2.2.6 pandas==2.2.3 scipy==1.15.3 scikit-learn==1.6.1 pyarrow==19.0.1 requests==2.32.3 pytest==8.3.5 tabulate==0.9.0 + + - name: Compile and pass exactly 42 tests + working-directory: source + shell: bash + run: | + set -euo pipefail + python -m compileall -q research/digital_surface tests/digital_surface + PYTHONPATH=. pytest --collect-only -q tests/digital_surface | tee /tmp/collect.txt + grep -Eq '^42 tests collected' /tmp/collect.txt + PYTHONPATH=. pytest tests/digital_surface -q + + - name: Prove fail-closed execution and canary gates + working-directory: source + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import ast + import hashlib + import json + import tempfile + from pathlib import Path + + from research.digital_surface.live import ( + LiveTradingDisabledError, + ShadowCanaryMetrics, + evaluate_shadow_admission, + load_config, + ) + from research.digital_surface.prospective import ProspectiveLedger + + try: + load_config({'TRADING_MODE': 'live', 'LIVE_ARM_TOKEN': 'present'}) + except LiveTradingDisabledError: + pass + else: + raise AssertionError('live mode unexpectedly armed') + + forbidden_import_roots = {'requests', 'httpx', 'aiohttp', 'websocket', 'websockets', 'socket'} + seen_imports = set() + for path in Path('research/digital_surface/live').glob('*.py'): + tree = ast.parse(path.read_text(), filename=str(path)) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + seen_imports.update(alias.name.split('.')[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + seen_imports.add(node.module.split('.')[0]) + assert not (seen_imports & forbidden_import_roots), seen_imports & forbidden_import_roots + + insufficient = evaluate_shadow_admission(ShadowCanaryMetrics( + markets_observed=499, + qualified_portfolios=99, + simulated_atomic=95, + orphaned=5, + unresolved_order_states=0, + feed_outages=0, + weekly_passes=3, + )) + assert insufficient['classification'] == 'Rejected' + passing = evaluate_shadow_admission(ShadowCanaryMetrics( + markets_observed=650, + qualified_portfolios=120, + simulated_atomic=116, + orphaned=4, + unresolved_order_states=0, + feed_outages=1, + weekly_passes=4, + )) + assert passing['classification'] == 'Admissible' + assert passing['live_mode'] == 'disabled_pending_operator_release' + + with tempfile.TemporaryDirectory() as directory: + ledger = ProspectiveLedger(Path(directory) / 'ledger.jsonl', diagnostic_market_ids={'old'}) + first = ledger.append_observation( + market_id='fresh-1', first_seen_ts_ms=1, + policy_sha256='1' * 64, source_sha256='2' * 64, + ) + second = ledger.append_observation( + market_id='fresh-2', first_seen_ts_ms=2, + policy_sha256='1' * 64, source_sha256='2' * 64, + ) + assert second['previous_hash'] == first['record_hash'] + assert ProspectiveLedger(ledger.path, diagnostic_market_ids={'old'}).head_hash == second['record_hash'] + + files = {} + for line in Path('SHADOW_PROSPECTIVE_REPORT_SOURCE_SHA256.txt').read_text().splitlines(): + wanted, name = line.split(maxsplit=1) + got = hashlib.sha256(Path(name).read_bytes()).hexdigest() + assert got == wanted + files[name] = got + result = { + 'tests_collected': 42, + 'live_mode': 'physically_disabled', + 'network_capable_imports': [], + 'prospective_hash_chain_verified': True, + 'shadow_canary_gates_verified': True, + 'admissible_report_still_requires_separate_operator_release': True, + 'source_files_verified': files, + } + Path('SHADOW_PROSPECTIVE_REPORT_VERIFICATION.json').write_text(json.dumps(result, indent=2, sort_keys=True)) + print(json.dumps(result, indent=2)) + PY + + - name: Build deterministic source checkpoint + working-directory: source + shell: bash + run: | + set -euo pipefail + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-shadow-prospective-report-source-v1.tar.gz \ + research tests SHADOW_PROSPECTIVE_REPORT_SOURCE_SHA256.txt SHADOW_PROSPECTIVE_REPORT_VERIFICATION.json + sha256sum digital-surface-shadow-prospective-report-source-v1.tar.gz > digital-surface-shadow-prospective-report-source-v1.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: digital-surface-shadow-prospective-report-checkpoint-v1 + path: | + source/SHADOW_PROSPECTIVE_REPORT_SOURCE_SHA256.txt + source/SHADOW_PROSPECTIVE_REPORT_VERIFICATION.json + source/digital-surface-shadow-prospective-report-source-v1.tar.gz + source/digital-surface-shadow-prospective-report-source-v1.tar.gz.sha256 + retention-days: 30 diff --git a/.github/workflows/digital-surface-shadow-report-chain-checkpoint.yml b/.github/workflows/digital-surface-shadow-report-chain-checkpoint.yml new file mode 100644 index 00000000..6a3d4c3a --- /dev/null +++ b/.github/workflows/digital-surface-shadow-report-chain-checkpoint.yml @@ -0,0 +1,194 @@ +name: Digital Surface Shadow Report Chain Checkpoint + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/digital-surface-shadow-report-chain-checkpoint.yml + - ci_payload/digital-surface-live-readiness/shadow-source/** + - ci_payload/digital-surface-live-readiness/prospective-source/** + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/digital-surface-shadow-report-chain-checkpoint.yml + - ci_payload/digital-surface-live-readiness/shadow-source/** + - ci_payload/digital-surface-live-readiness/prospective-source/** + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + verify-shadow-report-chain: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Reconstruct checksum-pinned source + shell: bash + run: | + set -euo pipefail + cat ci_payload/digital-surface/chunk-* > base.b64 + base64 --decode base.b64 > base.tar.gz + echo 'e2a181ac3966e7bb1f786c4fab854294ae96449dc19c7a51bb83814728ec1330 base.tar.gz' | sha256sum -c - + mkdir source && tar -xzf base.tar.gz -C source + cat ci_payload/digital-surface-final-recovered/chunk-* > recovered.b64 + base64 --decode recovered.b64 > recovered.tar.gz + echo 'ac3cf5f39d71facc9a1e4bc39a9ab05c64ae5ae5137889b74cab64bb540f9a15 recovered.tar.gz' | sha256sum -c - + tar -xzf recovered.tar.gz -C source + cp -a ci_payload/digital-surface-official-resolution-patch/. source/ + for patch_file in timestamp-unit-fix.patch full-universe-gate-fix.patch resolved-universe-fix.patch asset-merge-fix.patch; do + patch -d source -p1 < "ci_payload/digital-surface-official-resolution-patch/${patch_file}" + done + echo 'f71f3a653ca58b4bf98609fc29d8ab16f9d031c9ed14906ebe5159f2b0de8eab ci_payload/digital-surface-live-readiness/execution-diagnostics.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/execution-diagnostics.patch + echo '8606cc925aa3b3a6417abb1fba9a11df916cc2b281880978fb6229aa25875be9 ci_payload/digital-surface-live-readiness/book-arrival-model.patch' | sha256sum -c - + patch -d source -p1 < ci_payload/digital-surface-live-readiness/book-arrival-model.patch + cp -a ci_payload/digital-surface-live-readiness/shadow-source/. source/ + cp -a ci_payload/digital-surface-live-readiness/prospective-source/. source/ + + cat > /tmp/checkpoint.sha256 <<'EOF' + 6aa02760d4d038098a6a69a99d9349bb5c6fef8d4593fa4c4cf0574971828fb2 source/research/digital_surface/live/__init__.py + 9c5c9bcbd39e1262ac468024f5a96fb0667c0c95fc2879d846f7ca2066c38b5f source/research/digital_surface/live/config.py + 9e6dff73a17227f907dce4bf56cfd54921f4eb677280b1774d1a73c979b9065a source/research/digital_surface/live/gateway.py + 3be74a910e787a3f9e0f3548691e63ed93d78eeb3328fd9538e3a0e643b57d1c source/research/digital_surface/live/report.py + b0ca12139437c004ec88e41902796cdd540f7e45c1b27c3aba96e3695d2267a1 source/research/digital_surface/live/report_chain.py + f9197d38fd8dcfcc073d83028ac498fd9ff07619fea2b2b6a90de9ab470a86e2 source/research/digital_surface/live/risk.py + 50f32b2e400767702073b97ba4fd77a7e8c26f5fd4c7d6f7d9aafe282d6258f0 source/research/digital_surface/live/types.py + 10d833f89194246ea2fe5dec8bb2fc4388c8509a4d8276c7fcdc0a2b1a13e76c source/tests/digital_surface/live/test_report.py + a09860af2f96e950884e1a2d37d0dfe59c41d869f3cb183ef839bbec2896db4a source/tests/digital_surface/live/test_report_chain.py + be1f59a4ae742f062b419ad7f7fb974eadf9bd076cb2ecdabcd0063eedf1df6d source/tests/digital_surface/live/test_shadow_gateway.py + f219222d7b89c5ab4ef0619da9163a77b79ba7f2a9fe611c4072746771e2fa41 source/research/digital_surface/prospective.py + 9ed734554e0296e94212cac3911466ac230241efdd4ab911e9adc74e41ef05b2 source/tests/digital_surface/test_prospective.py + EOF + sed -i 's/^ //' /tmp/checkpoint.sha256 + sha256sum -c /tmp/checkpoint.sha256 + cp /tmp/checkpoint.sha256 source/SHADOW_REPORT_CHAIN_SOURCE_SHA256.txt + + - name: Install pinned test stack + run: | + python -m pip install --upgrade pip + python -m pip install numpy==2.2.6 pandas==2.2.3 scipy==1.15.3 scikit-learn==1.6.1 pyarrow==19.0.1 requests==2.32.3 pytest==8.3.5 tabulate==0.9.0 + + - name: Compile and pass exactly 45 tests + working-directory: source + shell: bash + run: | + set -euo pipefail + python -m compileall -q research/digital_surface tests/digital_surface + PYTHONPATH=. pytest --collect-only -q tests/digital_surface | tee /tmp/collect.txt + grep -Eq '^45 tests collected' /tmp/collect.txt + PYTHONPATH=. pytest tests/digital_surface -q + + - name: Verify chained reports and physical live disable + working-directory: source + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import ast + import hashlib + import json + import tempfile + from pathlib import Path + + from research.digital_surface.live import ( + LiveTradingDisabledError, + ShadowCanaryMetrics, + ShadowReportLedger, + load_config, + ) + from research.digital_surface.prospective import ProspectiveLedger + + try: + load_config({'TRADING_MODE': 'live', 'LIVE_ARM_TOKEN': 'present'}) + except LiveTradingDisabledError: + pass + else: + raise AssertionError('live mode unexpectedly armed') + + forbidden = {'requests', 'httpx', 'aiohttp', 'websocket', 'websockets', 'socket'} + imports = set() + for path in Path('research/digital_surface/live').glob('*.py'): + tree = ast.parse(path.read_text(), filename=str(path)) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imports.update(alias.name.split('.')[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + imports.add(node.module.split('.')[0]) + assert not imports.intersection(forbidden), imports.intersection(forbidden) + + with tempfile.TemporaryDirectory() as directory: + directory = Path(directory) + prospective = ProspectiveLedger(directory/'prospective.jsonl', diagnostic_market_ids={'old'}) + observed = prospective.append_observation( + market_id='fresh', first_seen_ts_ms=1, + policy_sha256='1'*64, source_sha256='2'*64, + ) + reports = ShadowReportLedger(directory/'reports.jsonl') + report = reports.append( + generated_ts_ms=2, + metrics=ShadowCanaryMetrics( + markets_observed=650, + qualified_portfolios=120, + simulated_atomic=116, + orphaned=4, + unresolved_order_states=0, + feed_outages=1, + weekly_passes=4, + ), + policy_sha256='1'*64, + source_sha256='2'*64, + prospective_head_sha256=observed['record_hash'], + ) + assert report['decision']['classification'] == 'Admissible' + assert report['decision']['live_mode'] == 'disabled_pending_operator_release' + assert ShadowReportLedger(reports.path).head_hash == report['report_hash'] + + files = {} + for line in Path('SHADOW_REPORT_CHAIN_SOURCE_SHA256.txt').read_text().splitlines(): + wanted, name = line.split(maxsplit=1) + got = hashlib.sha256(Path(name).read_bytes()).hexdigest() + assert got == wanted + files[name] = got + result = { + 'tests_collected': 45, + 'live_mode': 'physically_disabled', + 'network_capable_imports': [], + 'prospective_hash_chain_verified': True, + 'shadow_report_hash_chain_verified': True, + 'report_bound_to_policy_source_and_prospective_head': True, + 'admissible_report_still_requires_separate_operator_release': True, + 'source_files_verified': files, + } + Path('SHADOW_REPORT_CHAIN_VERIFICATION.json').write_text(json.dumps(result, indent=2, sort_keys=True)) + print(json.dumps(result, indent=2)) + PY + + - name: Build deterministic source checkpoint + working-directory: source + shell: bash + run: | + set -euo pipefail + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf digital-surface-shadow-report-chain-source-v1.tar.gz \ + research tests SHADOW_REPORT_CHAIN_SOURCE_SHA256.txt SHADOW_REPORT_CHAIN_VERIFICATION.json + sha256sum digital-surface-shadow-report-chain-source-v1.tar.gz > digital-surface-shadow-report-chain-source-v1.tar.gz.sha256 + + - uses: actions/upload-artifact@v4 + with: + name: digital-surface-shadow-report-chain-checkpoint-v1 + path: | + source/SHADOW_REPORT_CHAIN_SOURCE_SHA256.txt + source/SHADOW_REPORT_CHAIN_VERIFICATION.json + source/digital-surface-shadow-report-chain-source-v1.tar.gz + source/digital-surface-shadow-report-chain-source-v1.tar.gz.sha256 + retention-days: 30 diff --git a/.github/workflows/late-favorite-fee-field-regression.yml b/.github/workflows/late-favorite-fee-field-regression.yml new file mode 100644 index 00000000..f5bab4ef --- /dev/null +++ b/.github/workflows/late-favorite-fee-field-regression.yml @@ -0,0 +1,58 @@ +name: Late Favorite Fee-Field Regression + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/late-favorite-fee-field-regression.yml + - ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py + - ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_late_favorite_transfer.py + - ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_fee_field_regression.py + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/late-favorite-fee-field-regression.yml + - ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py + - ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_late_favorite_transfer.py + - ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_fee_field_regression.py + workflow_dispatch: + +permissions: + contents: read + +jobs: + fee-field-regression: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install minimal pinned stack + run: python -m pip install pandas==2.2.3 huggingface_hub==0.33.4 pytest==8.3.5 + - name: Compile source and regression tests + run: | + python -m py_compile \ + ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py \ + ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_late_favorite_transfer.py \ + ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_fee_field_regression.py + - name: Verify fee is recorded before label-dependent PnL + run: | + python - <<'PY' + from pathlib import Path + + text = Path('ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py').read_text() + fee_assignment = ' result["fee_per_share"] = fee\n' + label_branch = ' if normalized_outcome in {"Up", "Down"}:\n' + assert text.count(fee_assignment) == 1 + assert text.count(label_branch) == 1 + assert text.index(fee_assignment) < text.index(label_branch) + PY + - name: Run focused behavior suite + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-repro-source + run: | + pytest -q \ + ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_late_favorite_transfer.py \ + ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_fee_field_regression.py diff --git a/.github/workflows/late-favorite-official-transfer-compact-existing.yml b/.github/workflows/late-favorite-official-transfer-compact-existing.yml new file mode 100644 index 00000000..fedcd61c --- /dev/null +++ b/.github/workflows/late-favorite-official-transfer-compact-existing.yml @@ -0,0 +1,90 @@ +name: Late Favorite Compact Existing Official Evidence + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/late-favorite-official-transfer-compact-existing.yml + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/late-favorite-official-transfer-compact-existing.yml + workflow_dispatch: + +permissions: + actions: read + contents: read + +env: + SOURCE_RUN_ID: '30213818908' + SOURCE_ARTIFACT_DIGEST: '210984541bcd64597ca48d480ad59102427f56b4ac204ab1308ad8f8a4f93ce5' + +jobs: + extract-compact-evidence: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - name: Download the already-green full evidence artifact + uses: actions/download-artifact@v4 + with: + name: late-favorite-official-transfer-v1 + run-id: ${{ env.SOURCE_RUN_ID }} + github-token: ${{ github.token }} + path: source-artifact + + - name: Locate and verify decision evidence + shell: bash + run: | + set -euo pipefail + find source-artifact -maxdepth 4 -type f -printf '%p %s bytes\n' | sort + results="$(find source-artifact -type d -name late-favorite-official-transfer-results -print -quit)" + test -n "$results" + for name in AUDIT.json REPORT.md SHA256SUMS official_transfer_ledger.csv.gz raw_response_manifest.jsonl.gz; do + test -f "$results/$name" + done + (cd "$results" && sha256sum -c SHA256SUMS) + mkdir compact + cp "$results/AUDIT.json" compact/ + cp "$results/REPORT.md" compact/ + cp "$results/SHA256SUMS" compact/ + cp "$results/official_transfer_ledger.csv.gz" compact/ + cp "$results/raw_response_manifest.jsonl.gz" compact/ + printf '%s source-full-artifact.zip\n' "$SOURCE_ARTIFACT_DIGEST" > compact/SOURCE_ARTIFACT_SHA256.txt + printf '%s\n' "$SOURCE_RUN_ID" > compact/SOURCE_RUN_ID.txt + + - name: Verify compact summary contract + run: | + python - <<'PY' + import gzip + import json + from pathlib import Path + + root = Path('compact') + audit = json.loads((root / 'AUDIT.json').read_text()) + assert audit['candidate'] == 'late_favorite_btc_eth_v2_diagnostic' + assert audit['fills'] == 9268 + assert audit['filled_condition_set_sha256'] == 'bf276fbce187d9651d56ecae976520d4251140c1d5b0d440696c5d077838a67a' + assert audit['credentials_used'] == 0 + assert audit['thresholds_changed'] == 0 + assert audit['live_submission'] == 'physically_absent' + assert sum(audit['classification_counts'].values()) == 9268 + with gzip.open(root / 'raw_response_manifest.jsonl.gz', 'rt', encoding='utf-8') as handle: + count = sum(1 for line in handle if line.strip()) + assert count == audit['raw_responses'] + print((root / 'REPORT.md').read_text()) + print(json.dumps(audit, indent=2, sort_keys=True)) + PY + + - name: Hash compact evidence + run: | + (cd compact && sha256sum AUDIT.json REPORT.md SHA256SUMS official_transfer_ledger.csv.gz raw_response_manifest.jsonl.gz SOURCE_ARTIFACT_SHA256.txt SOURCE_RUN_ID.txt > COMPACT_SHA256SUMS) + cat compact/COMPACT_SHA256SUMS + + - uses: actions/upload-artifact@v4 + with: + name: late-favorite-official-transfer-compact-from-run-30213818908 + path: compact/ + compression-level: 0 + retention-days: 30 diff --git a/.github/workflows/late-favorite-official-transfer.yml b/.github/workflows/late-favorite-official-transfer.yml new file mode 100644 index 00000000..cce4c20c --- /dev/null +++ b/.github/workflows/late-favorite-official-transfer.yml @@ -0,0 +1,198 @@ +name: Late Favorite Official Outcome Transfer + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/late-favorite-official-transfer.yml + - ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/** + - ci_payload/digital-surface-live-readiness/late-favorite-repro-source/** + - ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/CANDIDATE_SPEC.json + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/late-favorite-official-transfer.yml + - ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/** + - ci_payload/digital-surface-live-readiness/late-favorite-repro-source/** + - ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/CANDIDATE_SPEC.json + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + transfer-official-outcomes: + runs-on: ubuntu-latest + timeout-minutes: 180 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install pinned research stack + run: | + python -m pip install --upgrade pip + python -m pip install \ + numpy==2.2.6 \ + pandas==2.2.3 \ + pyarrow==19.0.1 \ + requests==2.32.4 \ + huggingface_hub==0.33.4 \ + hf_xet==1.1.5 \ + pytest==8.3.5 + + - name: Compile transfer and reproducer sources + run: | + python -m py_compile \ + ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/official_transfer.py \ + ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/test_official_transfer.py \ + ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py + + - name: Pass fail-closed official parser tests + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source + run: | + pytest -q ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/test_official_transfer.py + + - name: Verify live submission and credentials remain physically absent + run: | + python - <<'PY' + import ast + from pathlib import Path + + paths = [ + Path('ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/official_transfer.py'), + Path('ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py'), + ] + forbidden_imports = {'web3', 'py_clob_client', 'eth_account'} + forbidden_tokens = ('create_order', 'post_order', 'submit_order', 'private_key', 'api_secret') + for path in paths: + text = path.read_text() + tree = ast.parse(text) + imports = { + node.names[0].name + for node in ast.walk(tree) + if isinstance(node, ast.Import) and node.names + } + imported_from = { + node.module + for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) + } + assert not (forbidden_imports & imports), (path, imports) + assert not (forbidden_imports & imported_from), (path, imported_from) + lowered = text.lower() + for token in forbidden_tokens: + assert token not in lowered, (path, token) + PY + + - name: Rebuild exact frozen eligible condition set + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-repro-source + HF_XET_HIGH_PERFORMANCE: '1' + run: | + python ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py \ + --candidate-spec ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/CANDIDATE_SPEC.json \ + --out-dir late-favorite-repro-results \ + --cache-dir .cache/late-favorite-data + + - name: Assert immutable filled condition-set identity + run: | + python - <<'PY' + import json + from pathlib import Path + + audit = json.loads(Path('late-favorite-repro-results/AUDIT.json').read_text()) + assert audit['hypothetical_fok_fills'] == 9268 + assert audit['filled_condition_ids_sha256'] == 'bf276fbce187d9651d56ecae976520d4251140c1d5b0d440696c5d077838a67a' + assert audit['live_submission'] == 'physically_absent' + PY + + - name: Transfer official Gamma outcomes and CLOB corroboration + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source + run: | + python ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/official_transfer.py \ + --eligible-trades late-favorite-repro-results/eligible_trades.csv.gz \ + --out-dir late-favorite-official-transfer-results \ + --batch-size 25 \ + --clob-max-pages 1000 + + - name: Verify transfer evidence and classification contract + run: | + python - <<'PY' + import gzip + import hashlib + import json + from pathlib import Path + + out = Path('late-favorite-official-transfer-results') + required = { + 'AUDIT.json', + 'REPORT.md', + 'SHA256SUMS', + 'official_transfer_ledger.csv.gz', + 'raw_response_manifest.jsonl.gz', + } + assert required <= {path.name for path in out.iterdir() if path.is_file()} + audit = json.loads((out / 'AUDIT.json').read_text()) + assert audit['candidate'] == 'late_favorite_btc_eth_v2_diagnostic' + assert audit['fills'] == 9268 + assert audit['filled_condition_set_sha256'] == 'bf276fbce187d9651d56ecae976520d4251140c1d5b0d440696c5d077838a67a' + assert audit['live_submission'] == 'physically_absent' + assert audit['credentials_used'] == 0 + assert audit['thresholds_changed'] == 0 + assert audit['raw_responses'] > 0 + assert sum(audit['classification_counts'].values()) == audit['fills'] + assert 0 <= audit['official_usable'] <= audit['fills'] + assert audit['gamma_payloads'] + audit['gamma_failures'] <= audit['fills'] + + manifest_path = out / 'raw_response_manifest.jsonl.gz' + with gzip.open(manifest_path, 'rt', encoding='utf-8') as handle: + records = [json.loads(line) for line in handle if line.strip()] + assert len(records) == audit['raw_responses'] + for record in records: + if record.get('body_path') is None: + continue + body = out / record['body_path'] + assert body.is_file(), body + assert hashlib.sha256(body.read_bytes()).hexdigest() == record['body_sha256'] + PY + (cd late-favorite-official-transfer-results && sha256sum -c SHA256SUMS) + + - name: Build evidence archive + run: | + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf late-favorite-official-transfer-v1.tar.gz late-favorite-official-transfer-results + sha256sum late-favorite-official-transfer-v1.tar.gz > late-favorite-official-transfer-v1.tar.gz.sha256 + cat late-favorite-official-transfer-results/REPORT.md + cat late-favorite-official-transfer-results/AUDIT.json + + - uses: actions/upload-artifact@v4 + with: + name: late-favorite-official-transfer-v1 + path: | + late-favorite-official-transfer-results/ + late-favorite-official-transfer-v1.tar.gz + late-favorite-official-transfer-v1.tar.gz.sha256 + compression-level: 0 + retention-days: 30 + + - name: Upload compact decision evidence + uses: actions/upload-artifact@v4 + with: + name: late-favorite-official-transfer-summary-v1 + path: | + late-favorite-official-transfer-results/AUDIT.json + late-favorite-official-transfer-results/REPORT.md + late-favorite-official-transfer-results/SHA256SUMS + late-favorite-official-transfer-results/official_transfer_ledger.csv.gz + late-favorite-official-transfer-results/raw_response_manifest.jsonl.gz + late-favorite-official-transfer-v1.tar.gz.sha256 + compression-level: 0 + retention-days: 30 diff --git a/.github/workflows/late-favorite-repro-audit.yml b/.github/workflows/late-favorite-repro-audit.yml new file mode 100644 index 00000000..5f48fe99 --- /dev/null +++ b/.github/workflows/late-favorite-repro-audit.yml @@ -0,0 +1,143 @@ +name: Late Favorite Repro Audit + +on: + push: + branches: [digital-surface-live-readiness-20260725] + paths: + - .github/workflows/late-favorite-repro-audit.yml + - ci_payload/digital-surface-live-readiness/late-favorite-repro-source/** + - ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/CANDIDATE_SPEC.json + pull_request: + branches: [digital-surface-ci-20260724] + paths: + - .github/workflows/late-favorite-repro-audit.yml + - ci_payload/digital-surface-live-readiness/late-favorite-repro-source/** + - ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/CANDIDATE_SPEC.json + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONUNBUFFERED: '1' + +jobs: + reproduce-frozen-candidate: + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install pinned analysis stack + run: | + python -m pip install --upgrade pip + python -m pip install \ + numpy==2.2.6 \ + pandas==2.2.3 \ + pyarrow==19.0.1 \ + huggingface_hub==0.33.4 \ + hf_xet==1.1.5 \ + pytest==8.3.5 + + - name: Compile source + run: | + python -m py_compile \ + ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py \ + ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_late_favorite_transfer.py + + - name: Pass frozen-policy behavior tests + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-repro-source + run: | + pytest -q ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_late_favorite_transfer.py + + - name: Verify live submission remains physically absent + run: | + python - <<'PY' + import ast + from pathlib import Path + + path = Path('ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py') + tree = ast.parse(path.read_text()) + imports = { + node.names[0].name + for node in ast.walk(tree) + if isinstance(node, ast.Import) and node.names + } + imported_from = { + node.module + for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) + } + forbidden = {'web3', 'py_clob_client', 'eth_account'} + assert not (forbidden & imports) + assert not (forbidden & imported_from) + text = path.read_text().lower() + for token in ('create_order', 'post_order', 'submit_order', 'private_key', 'api_secret'): + assert token not in text, token + PY + + - name: Build complete BTC ETH reproducibility ledger + env: + PYTHONPATH: ci_payload/digital-surface-live-readiness/late-favorite-repro-source + HF_XET_HIGH_PERFORMANCE: '1' + run: | + python ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py \ + --candidate-spec ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/CANDIDATE_SPEC.json \ + --out-dir late-favorite-repro-results \ + --cache-dir .cache/late-favorite-data + + - name: Verify output contract and evidence boundary + run: | + python - <<'PY' + import json + from pathlib import Path + + out = Path('late-favorite-repro-results') + required = {'AUDIT.json', 'REPORT.md', 'SHA256SUMS', 'market_ledger.csv.gz', 'eligible_trades.csv.gz'} + assert required == {path.name for path in out.iterdir() if path.is_file()} + audit = json.loads((out / 'AUDIT.json').read_text()) + assert audit['candidate'] == 'late_favorite_btc_eth_v2_diagnostic' + assert audit['official_outcomes_used'] == 0 + assert audit['future_outcomes_used_for_signal'] == 0 + assert audit['live_submission'] == 'physically_absent' + assert audit['markets'] > 0 + assert audit['signals'] > 0 + assert audit['hypothetical_fok_fills'] > 0 + assert audit['labeled_fills'] + audit['unlabeled_fills'] == audit['hypothetical_fok_fills'] + assert 0.0 <= audit['inferred_label_win_rate'] <= 1.0 + assert set(audit['by_asset']) == {'btc', 'eth'} + for asset in ('btc', 'eth'): + row = audit['by_asset'][asset] + assert row['labeled_fills'] + row['unlabeled_fills'] == row['hypothetical_fok_fills'] + assert len(audit['all_condition_ids_sha256']) == 64 + assert len(audit['filled_condition_ids_sha256']) == 64 + assert audit['split_reproduction'].startswith('not claimed') + PY + (cd late-favorite-repro-results && sha256sum -c SHA256SUMS) + + - name: Build deterministic checkpoint archive + run: | + tar --sort=name --mtime='UTC 1970-01-01' --owner=0 --group=0 --numeric-owner \ + -czf late-favorite-repro-audit-v2.tar.gz late-favorite-repro-results + sha256sum late-favorite-repro-audit-v2.tar.gz > late-favorite-repro-audit-v2.tar.gz.sha256 + cat late-favorite-repro-results/REPORT.md + cat late-favorite-repro-results/AUDIT.json + + - uses: actions/upload-artifact@v4 + with: + name: late-favorite-repro-audit-v2 + path: | + late-favorite-repro-results/AUDIT.json + late-favorite-repro-results/REPORT.md + late-favorite-repro-results/SHA256SUMS + late-favorite-repro-results/market_ledger.csv.gz + late-favorite-repro-results/eligible_trades.csv.gz + late-favorite-repro-audit-v2.tar.gz + late-favorite-repro-audit-v2.tar.gz.sha256 + compression-level: 0 + retention-days: 30 diff --git a/ci_payload/digital-surface-live-readiness/CHECKPOINT_SHA.txt b/ci_payload/digital-surface-live-readiness/CHECKPOINT_SHA.txt new file mode 100644 index 00000000..4d181aaf --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/CHECKPOINT_SHA.txt @@ -0,0 +1 @@ +6827eaf6f0ad7b1a70f0dbe1738b4ab50d065cab diff --git a/ci_payload/digital-surface-live-readiness/DEEP_EDA_FRESH_CHAT_HANDOFF_2026-07-26.md b/ci_payload/digital-surface-live-readiness/DEEP_EDA_FRESH_CHAT_HANDOFF_2026-07-26.md new file mode 100644 index 00000000..901d31dc --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/DEEP_EDA_FRESH_CHAT_HANDOFF_2026-07-26.md @@ -0,0 +1,249 @@ +# Polymarket 5-Minute Crypto HFT — Fresh-Chat Handoff + +Created 2026-07-26 after the prior chat reached its length limit and the next message crashed. + +## Paste into the fresh chat + +Continue the Polymarket 5-minute crypto high-frequency research project from this handoff. + +Start by restoring and verifying the newest remote state: + +- Active research repo: `Chrisbryan17/hello-world` +- Active branch: `digital-surface-live-readiness-20260725` +- Draft PR: `#59` +- Last verified research head before this handoff: `f8750201dc421b1c0e543712d8f207ef76705eaf` +- Latest verified commit message: `research: freeze diagnostic BTC ETH favorite candidate` +- Do not assume the branch has not advanced. Inspect PR #59 and branch history first. +- Read immediately: + - `ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/REPORT.md` + - `ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/CANDIDATE_SPEC.json` + - `ci_payload/digital-surface-live-readiness/STATUS.md` + +The strategic instruction is explicit: stop centering work on harness construction. Perform deep, microstructure-first EDA like a top Jane Street quant. Focus on event-time alignment, lead-lag discovery, conditional edge decay, fillability, regime clustering, strict walk-forward falsification, executable prices, and multiple-testing controls. Back up every material result to GitHub. Do not manufacture a passing strategy by leaking holdouts. + +The initial deep EDA is already complete. Do not redo it from scratch. It found a leading diagnostic candidate: + +- name: `late_favorite_btc_eth_v2_diagnostic` +- assets: BTC and ETH +- entry: 210 seconds into the 5-minute window +- favorite signal ask: at least $0.85 +- latency: 1 second +- FOK limit: signal ask +- size: 5 shares +- fee: `0.07*p*(1-p)` per share +- hold: settlement +- maximum one position per asset per window and two total positions per window +- diagnostic cancel: cancel when arrival ask is more than $0.01 below signal ask +- live submission: disabled +- status: profitable diagnostic candidate requiring new official-label prospective validation + +Next valid work: + +1. verify and preserve the latest branch/artifacts; +2. audit the deep EDA result for reproducibility and contamination; +3. transfer the frozen BTC/ETH candidate to official Polymarket terminal outcomes; +4. run an append-only prospective observer recording every eligible and ineligible BTC/ETH window, signal/no-signal, signal ask, 1-second arrival book, full depth, hypothetical FOK result, adverse-move cancel, official outcome, fees, slippage, and per-window exposure; +5. run at least four untouched weekly prospective blocks; +6. continue deeper EDA in parallel, but version every new hypothesis discovered using inspected data; +7. keep live order submission physically absent. + +For every meaningful pass: tests -> source/config/results/audit files -> atomic commit -> push -> fetch/read-back -> SHA/blob verification -> next experiment. + +## Distinguish the two strategy tracks + +### A. Original 5m-vs-15m digital-surface sleeve + +After official-resolution correction and four untouched historical folds, this sleeve was honestly **Rejected**. Those weeks are diagnostic and may never count as prospective evidence. + +Execution diagnosis across 224 portfolios / 448 legs: + +- insufficient post-arrival tape: 186 +- ask above frozen limit: 114 +- missing post-arrival book: 110 +- filled: 34 +- insufficient displayed depth: 4 + +Tape-confirmed replay produced 8 paired fills. Diagnostic book-arrival replay estimated 62 paired-crossable portfolios with fold counts `[11,17,29,5]`. This is not transactionally atomic and not live truth. + +### B. Deep late-favorite candidate + +The broader EDA corpus contained 89,233 markets and approximately 26.77M one-second observations across BTC, ETH, SOL, XRP, DOGE, BNB, and HYPE. + +The strongest candidate was not BTC-to-alt lead-lag. It was late-window settlement favorite, with BTC+ETH the statistically credible core. + +## Deep EDA evidence + +All-seven strict walk-forward: + +- train: 38 days, 18,493 trades, +$1,088.33 at five shares, daily t=5.91 +- validation: 14 days, 6,819 trades, +$395.42, daily t=5.35, every day positive +- final chronological test: 12 days, 6,769 trades, +$249.11, 24.67 trades/hour, daily t=1.67 +- 210s / 85c ranked #1 of 982 eligible configurations by validation daily t +- tested latency 0-3 seconds remained positive in train and validation + +BTC+ETH core: + +- 2,054 final-test trades +- +$139.72 at five shares +- 95.67% win rate +- 1.3604 cents mean P&L/share after fee +- daily t=3.85 +- exact one-sided day-level sign-flip p=0.00244 +- 95% day-bootstrap mean interval: +$5.84 to +$17.13/day at five shares +- BTC and ETH survive BH control at 5% across seven assets + +Adverse selection: + +- favorite ask falling by >=$0.02 during the 1-second latency produced 288 trades, 84.38% win rate, and -$48.44 +- $0.01 improvement remained positive +- therefore the prospective rule is cancel when `execution_ask < signal_ask - 0.01` +- this rule was found after final-test inspection and has no untouched status + +Cross-asset risk: + +- 38.66% of losses occurred in windows with at least two simultaneous losses +- BTC/ETH loss correlation: 0.559 +- when BTC won, ETH won 98.39% of simultaneous windows +- when BTC lost, ETH won only 38.46% +- all-seven top-two-day P&L concentration: 66.62% +- BTC+ETH concentration: 38.02% +- cap by five-minute window, not only by asset + +Capacity/cost: + +- BTC+ETH break-even additional cost: 1.3604 cents/share +- all-seven break-even additional cost: 0.7360 cents/share +- at 100 shares, 56.13% of BTC/ETH test signals showed enough top-of-book size and stayed positive, but tail risk scaled sharply +- 250+ shares deteriorated +- top-of-book size is not a live capacity guarantee + +## Non-negotiable uncertainty + +1. Dataset outcomes were inferred from the final recorded book, not authoritative settlement. +2. Only 28 falsely labeled winners erase BTC+ETH final-test profit. +3. Official terminal-outcome validation is mandatory. +4. BTC/ETH selection and the adverse-move cancel were learned after final-test inspection. +5. Top-of-book depth does not model queue position. +6. Two separately submitted FOK orders are not transactionally atomic. +7. Previously inspected markets cannot count prospectively. +8. Policy, source, ledger, and reports must be hash-bound. +9. Real-money execution remains absent. + +## Existing infrastructure to reuse + +PR #59 already contains checksum-pinned recovery material for: + +- official outcome/source correction; +- slug/timestamp/universe/asset merge fixes; +- deterministic archives; +- leg-level execution attribution; +- tape-confirmed and book-arrival replay; +- Decimal tick quantization; +- shadow-only pair preparation; +- stale data, loss, orphan, unresolved-order, token, depth, limit, expiry, and ambiguous-state kill switches; +- append-only prospective market, book, BTC-state, and report chains; +- historical contamination rejection; +- public Gamma market discovery; +- public CLOB book collection; +- public Binance spot/strike/volatility collection; +- causal prospective signal adapter; +- live startup physically disabled. + +## Raw inputs from prior chat + +- `polymarket-5m-btc(1).zip` +- `polymarket-5m-eth.zip` +- `polymarket-5m-sol(1).zip` +- `polymarket-5m-xrp.zip` +- `polymarket-5m-doge.zip` +- `polymarket-5m-bnb.zip` +- `polymarket-5m-hype.zip` +- `Branch · BTC Lead-Lag Strategy.txt` +- `Transcript.txt` + +The new filesystem is temporary. Re-upload raw ZIPs only when needed; durable state belongs on GitHub. + +## Required deep EDA continuation + +Continue all of these, using executable bid/ask economics: + +1. common-factor-residual cross-asset lead-lag at 1,2,3,5,10,15,30 seconds; +2. same-market complement arbitrage with both-leg fees/depth/latency; +3. microstructure markouts from spread, imbalance, depth curvature, update intensity, replenishment, and time-to-expiry; +4. regime clustering by volatility, cross-asset correlation, liquidity, time-of-day, favorite price, expiry, and crash state; +5. purged walk-forward, embargo, block bootstrap, sign-flip, BH/FWER, White Reality Check/SPA, PBO, deflated Sharpe, leave-day/regime-out, and concentration diagnostics. + +Any candidate discovered using an inspected final test becomes a new prospective version and cannot inherit old holdout status. + +## Admission standard + +The BTC/ETH candidate needs: + +- authoritative outcomes; +- four untouched weekly blocks; +- positive P&L after measured fees/slippage; +- positive lower confidence bound; +- no extreme day concentration; +- stable BTC/ETH performance; +- acceptable common-factor crash exposure; +- measured fill/capacity; +- zero unresolved states; +- frozen policy throughout; +- complete append-only verification. + +Even then, live requires a separate reviewed and explicitly armed release. + +## Recovery commands + +```bash +git clone https://github.com/Chrisbryan17/hello-world.git +cd hello-world +git fetch --all --prune +git checkout digital-surface-live-readiness-20260725 +git pull --ff-only +git rev-parse HEAD +git status --short --branch +git log --oneline --decorate -30 + +sed -n '1,240p' ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/REPORT.md +cat ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/CANDIDATE_SPEC.json +sed -n '1,220p' ci_payload/digital-surface-live-readiness/STATUS.md +find ci_payload/digital-surface-live-readiness -maxdepth 4 -type f | sort +``` + +Original authoritative provenance when needed: + +```bash +git clone https://github.com/Chrisbryan17/trading-tools.git ../trading-tools +cd ../trading-tools +git fetch --all --prune +git checkout digital-surface-arbitrage-implementation-20260723 +git log --oneline --decorate -30 +``` + +## What not to do + +- do not rebuild generic harnesses; +- do not tune known historical folds until they pass; +- do not use midpoint fills; +- do not ignore fees; +- do not fit a fine lookup table to inspected price buckets; +- do not call correlated BTC/ETH independent diversification; +- do not increase size from top-of-book alone; +- do not count diagnostic markets prospectively; +- do not add credentials or an order submitter; +- do not claim profitability before official-label transfer and prospective evidence; +- do not start the next experiment before committing, pushing, reading back, and verifying the current one. + +## Definition of success for the next chat + +- restore the actual latest PR #59 head; +- confirm it is newer than or equal to `f875020…`; +- avoid repeating initial harness work; +- reproduce/audit the deep-favorite result; +- begin official-label transfer or genuine prospective collection; +- produce a decision-useful new EDA result or falsification; +- push source, tests, configs, results, and audit metadata; +- keep live execution disabled. + +The goal is not to force a pass. The goal is to find a genuine executable edge or falsify this candidate quickly without fooling ourselves. diff --git a/ci_payload/digital-surface-live-readiness/DIAGNOSTIC_FINDINGS.md b/ci_payload/digital-surface-live-readiness/DIAGNOSTIC_FINDINGS.md new file mode 100644 index 00000000..90d893d9 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/DIAGNOSTIC_FINDINGS.md @@ -0,0 +1,45 @@ +# Execution Diagnostic Findings + +Source artifact: GitHub Actions artifact `digital-surface-execution-diagnostics-v1`, artifact ID `8617727219`, outer SHA-256 `80a442cded15e26703c401e197a53db28ae59b61efa6dcb808a0d65ead961293`. + +The immutable replay completed successfully and attributed exactly two legs for each of 224 candidates (448 legs total). + +## Terminal leg reasons + +| Reason | Legs | Share | +|---|---:|---:| +| Insufficient post-arrival tape | 186 | 41.52% | +| Ask above frozen limit | 114 | 25.45% | +| Missing post-arrival book | 110 | 24.55% | +| Filled | 34 | 7.59% | +| Insufficient displayed depth | 4 | 0.89% | + +Only eight candidates filled both legs under the existing tape-confirmed replay. + +## Size counterfactual + +Reducing requested size does not solve the bottleneck: + +| Shares | Tape-confirmed paired fills | +|---:|---:| +| 5 | 8 | +| 3 | 8 | +| 2 | 9 | +| 1 | 9 | +| 0.5 | 9 | +| 0.1 | 9 | + +At five shares, 62 candidates have both post-latency arrival books, both asks within their frozen limits, and at least five displayed shares on both legs. The tape-confirmation requirement reduces those 62 book-executable candidates to eight. + +## Root-cause conclusion + +The fixed five-share size is not the principal execution bottleneck. The dominant blocker is the historical replay's requirement that unrelated post-arrival taker-buy tape independently demonstrate sufficient volume after the arrival snapshot. A real FOK order would itself consume displayed executable depth; its fill does not require another trader to print the same side afterward. + +This does not prove that all 62 historical book-executable candidates would have filled live. It establishes two bounds that must remain separate: + +1. **Tape-confirmed lower bound:** eight paired fills. +2. **Post-latency book-executable estimate:** 62 paired fills at five shares, before prospective calibration. + +The next experiment must therefore implement a book-arrival FOK estimator with explicit uncertainty/haircuts, retain the tape-confirmed result as a conservative lower bound, and calibrate the estimator against prospective shadow orders. The four already observed weeks remain diagnostic and cannot become admissible holdouts. + +Real-money execution remains disabled. diff --git a/ci_payload/digital-surface-live-readiness/FINAL_PRE_PR_HEAD.txt b/ci_payload/digital-surface-live-readiness/FINAL_PRE_PR_HEAD.txt new file mode 100644 index 00000000..f00f3a90 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/FINAL_PRE_PR_HEAD.txt @@ -0,0 +1 @@ +9504e172d970dd41977278de44d71e9206797b80 diff --git a/ci_payload/digital-surface-live-readiness/LIVE_DISABLED b/ci_payload/digital-surface-live-readiness/LIVE_DISABLED new file mode 100644 index 00000000..66ccc69a --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/LIVE_DISABLED @@ -0,0 +1 @@ +shadow-only diff --git a/ci_payload/digital-surface-live-readiness/NOTE.txt b/ci_payload/digital-surface-live-readiness/NOTE.txt new file mode 100644 index 00000000..0bb3c796 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/NOTE.txt @@ -0,0 +1 @@ +The next operation is opening the draft pull request and observing the clean-room diagnostics workflow. diff --git a/ci_payload/digital-surface-live-readiness/PR_READY b/ci_payload/digital-surface-live-readiness/PR_READY new file mode 100644 index 00000000..f3d43775 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/PR_READY @@ -0,0 +1 @@ +draft diff --git a/ci_payload/digital-surface-live-readiness/README.md b/ci_payload/digital-surface-live-readiness/README.md new file mode 100644 index 00000000..16ccb80f --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/README.md @@ -0,0 +1,8 @@ +# Digital Surface Live Readiness + +This checkpoint extends the checksum-pinned v5 research source with diagnostics only. + +- The four observed Kinzik transfer weeks are diagnostic data and may not be reused as admissible holdouts. +- `execution-diagnostics.patch` adds deterministic one-reason attribution for both legs of every candidate. +- The workflow asserts that candidate, execution, fold, and admission outputs remain byte-identical to v5. +- Real-money execution remains disabled. Future live components must default to shadow mode and pass prospective gates. diff --git a/ci_payload/digital-surface-live-readiness/STATUS.md b/ci_payload/digital-surface-live-readiness/STATUS.md new file mode 100644 index 00000000..9accd32e --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/STATUS.md @@ -0,0 +1,154 @@ +# Digital Surface Live-Readiness Status + +## Durable checkpoint + +- Repository: `Chrisbryan17/hello-world` +- Branch: `digital-surface-live-readiness-20260725` +- Draft PR: #59 +- Official late-favorite decision commit: `cedd659e0b573d8479932c428cd2eaf472c6bf8f` +- First BTC-only prospective checkpoint merge: `f76097ef513a2e70881e7624cd8d9a60d12a642b` +- Bounded block-runner merge: `e0b4917c2c7895568d7acb0ca53d4a6ccfcfdf1c` +- Real-money execution: **physically disabled** + +## Historical research verdicts + +### Original 5m-vs-15m digital-surface sleeve + +The official-resolution source gate and all four historical transfer folds completed, but the frozen sleeve remained **Rejected**. Those folds are diagnostic only and may never count toward prospective admission. + +### Late-favorite BTC/ETH candidate + +The immutable 9,268-fill set was transferred to authoritative Polymarket terminal outcomes with 100% coverage. The candidate is **Rejected / falsified**: + +- official win rate: 94.3893%; +- official mean P&L/share: -$0.00127314; +- official P&L at five shares: -$59.00; +- BTC: +$64.00, but its daily confidence interval crosses zero; +- ETH: -$123.00. + +The earlier +$428.93 inferred-label result was biased by 385 null-labeled fills. Official resolution showed that excluded cohort had only 67.53% wins and lost $472.92. Nine direct label disagreements changed comparable P&L by only -$15.00; the material defect was non-random label-availability censoring. + +Authoritative decision evidence: + +- `ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/CHECKPOINT.md` +- filled condition-set SHA-256: `bf276fbce187d9651d56ecae976520d4251140c1d5b0d440696c5d077838a67a` +- full official evidence artifact SHA-256: `210984541bcd64597ca48d480ad59102427f56b4ac204ab1308ad8f8a4f93ce5` +- compact derivative artifact SHA-256: `fb5dfb4621caa8b62ebaefc0b5c15d04619f773c3686cc4f088767ca023416b9` + +## BTC-only v3 prospective shadow track + +BTC-only is a **marginal post-hoc lead**, not a historical pass. It has zero inherited validation or admission credit. + +### Frozen candidate and capture boundaries + +- candidate: `late_favorite_btc_only_v3_prospective_shadow` +- candidate policy SHA-256: `82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba` +- candidate freeze: `2026-07-26T19:11:11Z` +- capture policy SHA-256: `888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc` +- effective capture cutoff: `2026-07-26T19:35:21Z` +- only markets opening strictly after the effective capture cutoff may count. + +### Observer and runner verification + +The observer implements market-open-anchored +210s/+211s capture, frozen timing/freshness gates, Decimal full-depth five-share FOK evaluation, fail-closed decision states, append-only lifecycle and raw-evidence chains, public Gamma/CLOB collection, cache-busted terminal resolution, and official fill/no-fill closure. + +Observer checkpoint: + +- 25/25 behavior contracts passed; +- workflow run: `30220717065`; +- artifact ID: `8637116192`; +- artifact ZIP SHA-256: `172ef7213e7860a8bee00db5e0a79d66495d31199cde85d39dcfef201f14de86`; +- combined frozen source SHA-256: `ce477bf97589997899c91ad22af9025de521520f8ee7aa46a21b50a313958bd7`. + +The bounded runner adds `collect-block`: 1–12 consecutive windows are captured before any resolution request, then the condition queue is resolved with bounded cache-busted retries. Unresolved state is preserved in a durable summary. + +Runner checkpoint: + +- 11/11 contracts passed; +- workflow run: `30221103255`; +- artifact ID: `8637227551`; +- artifact ZIP SHA-256: `fbc509ffa17046b885f25ef13b982f08c4f706bf4d65a73a179c66f0f1fb8f3b`; +- runner SHA-256: `48b45fb823b6fd9f6edf469d34a2c3a85c767c0bf4a456b919127dfeb39dcb8b`; +- combined runtime source SHA-256: `81a13d2aeb3df3c2088b8aa99272b10e217a392eabc839a01a8f2b71f570ecbb`. + +### Prospective observation v1 + +- checkpoint: `ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-run-v1/CHECKPOINT.md` +- market: `btc-updown-5m-1785099600`, 2026-07-26 21:00–21:05 UTC; +- Up signal at $0.98; arrival ask $0.99; +- decision: `no_fill_ask_above_limit`; +- official outcome: Up; +- fill/P&L: none; +- artifact ZIP SHA-256: `8d00c88d6046806a2abc07f43da8b2f0ab08449890ee8a7f9e1f7071b92c4e21`. + +### Prospective block v1 + +- checkpoint: `ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v1/CHECKPOINT.md` +- workflow run: `30221188931`; +- artifact ID: `8637433570`; +- artifact ZIP SHA-256: `222d8261ea92dfdfcd3fbb9ccb12dbf6948d03f6bed1e7558f3d9e4bdc6a5580`; +- markets: 3; +- official outcomes: 3/3; +- hypothetical fills: 0; +- P&L: $0.00; +- unresolved conditions: 0. + +The block contained two below-threshold no-signals and one no-fill above the frozen FOK limit. The no-fill avoided a loss when the market later resolved against the selected favorite. + +### Prospective block v2 + +- checkpoint: `ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v2/CHECKPOINT.md` +- workflow run: `30226869672`; +- artifact ID: `8638983600`; +- artifact ZIP SHA-256: `69ee1bb111f8aec0b52f982f305136a5e835ca2b2e50f4b66a129d3370b5ac58`; +- deterministic TAR SHA-256: `4fc0e0b68e7261899e133e1ba9af05939a72303d26609f73391967b5251c73cc`; +- markets: 3; +- official outcomes: 3/3; +- hypothetical fills: 1; +- P&L: **+$0.233375**; +- unresolved conditions: 0; +- lifecycle records: 10, chain verified; +- raw evidence records: 44, chain and bodies verified. + +The filled market selected Down at a 95-cent signal ask, executed five shares at 95 cents one second later, paid $0.003325/share in fees, officially resolved Down, and earned $0.046675/share or $0.233375 total. + +### Prospective block v3 + +- checkpoint: `ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v3/CHECKPOINT.md` +- workflow run: `30228071397`; +- artifact ID: `8639355447`; +- artifact ZIP SHA-256: `b30e9e10de839070ad1ae88f98c952ff2d8d4378ab828bed1533cff0e588cc0e`; +- deterministic TAR SHA-256: `5af345750399cde55f782140c8ba0427b74d39c113ce864e7a045c3f246cc6ee`; +- markets: 3; +- official outcomes: 3/3; +- hypothetical fills: 1; +- P&L: **+$0.610415**; +- unresolved conditions: 0; +- lifecycle records: 10, chain verified; +- raw evidence records: 44, chain and bodies verified. + +The filled market selected Down at an 88-cent signal ask. The arrival best ask improved to 87 cents, remained eligible under the frozen one-cent adverse-move rule, executed five shares at 87 cents, officially resolved Down, and earned $0.122083/share or $0.610415 total. + +## Safety and admission boundary + +- credentials used: **0**; +- authenticated requests: **0**; +- order submissions: **0**; +- live submission: **physically absent**; +- historical admission credit: **0**; +- prospective markets observed: **10 / 500 minimum**; +- official outcome coverage: **10 / 10**; +- hypothetical FOK fills: **2 / 100 minimum**; +- prospective P&L: **+$0.843790**; +- unresolved states: **0**; +- complete untouched weekly blocks: **0 / 4 minimum**. + +Two winning fills are not statistical validation. These ten markets are valid prospective evidence but cannot support admission, a confidence bound, a concentration pass, or live release. Even a future admissible shadow report cannot enable trading; any order submitter requires a separate reviewed release and explicit operator action. + +## Next work + +1. continue bounded capture-first blocks without modifying candidate or capture thresholds; +2. preserve every no-signal, no-fill, fill, resolution, and transport-failure record; +3. accumulate at least 500 observed markets, 100 hypothetical fills, and four untouched weekly blocks; +4. require measured execution evidence, acceptable concentration, zero unresolved states, and a positive lower confidence bound after fees and costs; +5. keep real-money execution physically disabled. diff --git a/ci_payload/digital-surface-live-readiness/STOP_CREATING_FILES.txt b/ci_payload/digital-surface-live-readiness/STOP_CREATING_FILES.txt new file mode 100644 index 00000000..7e97dae8 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/STOP_CREATING_FILES.txt @@ -0,0 +1 @@ +Use create_pull_request next. diff --git a/ci_payload/digital-surface-live-readiness/book-arrival-model.patch b/ci_payload/digital-surface-live-readiness/book-arrival-model.patch new file mode 100644 index 00000000..07c68579 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/book-arrival-model.patch @@ -0,0 +1,264 @@ +diff --git a/research/digital_surface/execution.py b/research/digital_surface/execution.py +--- a/research/digital_surface/execution.py ++++ b/research/digital_surface/execution.py +@@ -25,8 +25,12 @@ + tape_full: bool + + @property ++ def book_executable(self) -> bool: ++ return self.book_crossable and self.book_full ++ ++ @property + def filled(self) -> bool: +- return self.book_crossable and self.book_full and self.tape_full ++ return self.book_executable and self.tape_full + + + def _prepare_books(books: pd.DataFrame, condition_ids: set[str] | None = None) -> dict[str, pd.DataFrame]: +@@ -146,6 +150,7 @@ + max_book_gap_ms: int = 4_000, + shares: float = 5.0, + adverse_slippage: float = 0.0, ++ fill_model: str = "tape_confirmed", + ) -> pd.DataFrame: + required = { + "candidate_id", +@@ -165,6 +170,8 @@ + raise ValueError(f"candidates missing required columns: {missing}") + if shares <= 0: + raise ValueError("shares must be positive") ++ if fill_model not in {"tape_confirmed", "book_arrival"}: ++ raise ValueError("fill_model must be 'tape_confirmed' or 'book_arrival'") + condition_ids = set(candidates["low_condition_id"].astype(str)) | set(candidates["high_condition_id"].astype(str)) + token_ids = set(candidates["low_yes_token_id"].astype(str)) | set(candidates["high_no_token_id"].astype(str)) + book_groups = _prepare_books(books, condition_ids) +@@ -198,15 +205,17 @@ + tape_window_ms=tape_window_ms, + max_book_gap_ms=max_book_gap_ms, + ) +- atomic = low.filled and high.filled ++ low_filled = low.filled if fill_model == "tape_confirmed" else low.book_executable ++ high_filled = high.filled if fill_model == "tape_confirmed" else high.book_executable ++ atomic = low_filled and high_filled + orphan_leg: str | None = None +- if low.filled != high.filled: +- orphan_leg = "low_yes" if low.filled else "high_no" ++ if low_filled != high_filled: ++ orphan_leg = "low_yes" if low_filled else "high_no" + +- low_px = low.arrival_ask if low.filled else math.nan +- high_px = high.arrival_ask if high.filled else math.nan +- low_cost = all_in_cost(min(0.99, low_px + adverse_slippage)) if low.filled else 0.0 +- high_cost = all_in_cost(min(0.99, high_px + adverse_slippage)) if high.filled else 0.0 ++ low_px = low.arrival_ask if low_filled else math.nan ++ high_px = high.arrival_ask if high_filled else math.nan ++ low_cost = all_in_cost(min(0.99, low_px + adverse_slippage)) if low_filled else 0.0 ++ high_cost = all_in_cost(min(0.99, high_px + adverse_slippage)) if high_filled else 0.0 + low_payoff = 1.0 if bool(candidate.low_resolved_yes) else 0.0 + high_no_payoff = 0.0 if bool(candidate.high_resolved_yes) else 1.0 + if atomic: +@@ -229,10 +238,15 @@ + "arrival_ts_ms": arrival, + "close_ts": int(candidate.close_ts), + "shares": float(shares), ++ "fill_model": fill_model, + "atomic": bool(atomic), + "orphan_leg": orphan_leg, +- "low_yes_filled": bool(low.filled), +- "high_no_filled": bool(high.filled), ++ "low_yes_filled": bool(low_filled), ++ "high_no_filled": bool(high_filled), ++ "low_yes_book_executable": bool(low.book_executable), ++ "high_no_book_executable": bool(high.book_executable), ++ "low_yes_tape_confirmed": bool(low.filled), ++ "high_no_tape_confirmed": bool(high.filled), + "low_yes_book_ts_ms": low.book_ts_ms, + "high_no_book_ts_ms": high.book_ts_ms, + "low_yes_arrival_ask": low.arrival_ask, +diff --git a/research/digital_surface/run_research.py b/research/digital_surface/run_research.py +--- a/research/digital_surface/run_research.py ++++ b/research/digital_surface/run_research.py +@@ -109,6 +109,7 @@ + week: str, + hours: float, + adverse_slippage: float = 0.0, ++ fill_model: str = "tape_confirmed", + ) -> tuple[pd.DataFrame, pd.DataFrame, dict[str, Any]]: + if candidates.empty: + summary = { +@@ -121,6 +122,7 @@ + "quality_ratio": 0.0, + "max_drawdown": 0.0, + "adverse_slippage": adverse_slippage, ++ "fill_model": fill_model, + } + return pd.DataFrame(), pd.DataFrame(), summary + fills = replay_atomic_fok( +@@ -132,6 +134,7 @@ + max_book_gap_ms=4_000, + shares=BASE_SHARES, + adverse_slippage=adverse_slippage, ++ fill_model=fill_model, + ) + eligible = fills[fills["atomic"] | fills["orphan_leg"].notna()].copy() + if eligible.empty: +@@ -157,6 +160,7 @@ + "quality_ratio": float(report["quality_ratio"]), + "max_drawdown": float(report["max_drawdown"]), + "adverse_slippage": float(adverse_slippage), ++ "fill_model": fill_model, + } + return fills, ledger, summary + +@@ -361,6 +365,8 @@ + benchmark_rows: list[dict[str, Any]] = [] + all_candidates: list[pd.DataFrame] = [] + all_fills: list[pd.DataFrame] = [] ++ all_book_fills: list[pd.DataFrame] = [] ++ book_fold_rows: list[dict[str, Any]] = [] + all_benchmark_fills: list[pd.DataFrame] = [] + coherence_rows: list[dict[str, Any]] = [] + calibration_rows: list[dict[str, Any]] = [] +@@ -410,6 +416,30 @@ + two_fills, _, two = _execute_pair_week( + candidates, books, trades, week=fold.test_week, hours=_hours(test_state), adverse_slippage=0.02 + ) ++ book_fills, _, book_summary = _execute_pair_week( ++ candidates, ++ books, ++ trades, ++ week=fold.test_week, ++ hours=_hours(test_state), ++ adverse_slippage=0.0, ++ fill_model="book_arrival", ++ ) ++ book_fold_rows.append( ++ { ++ "fold_id": fold.fold_id, ++ "test_week": fold.test_week, ++ "config_id": selected_id, ++ "candidate_count": book_summary["candidate_count"], ++ "atomic_count": book_summary["atomic_count"], ++ "atomic_per_hour": book_summary["atomic_per_hour"], ++ "orphan_rate": book_summary["orphan_rate"], ++ "terminal_growth": book_summary["terminal_growth"], ++ "quality_ratio": book_summary["quality_ratio"], ++ "max_drawdown": book_summary["max_drawdown"], ++ "fill_model": "book_arrival", ++ } ++ ) + fold_rows.append( + { + "fold_id": fold.fold_id, +@@ -437,6 +467,12 @@ + tagged_fills["test_week"] = fold.test_week + all_fills.append(taged_fills) + ++ if not book_fills.empty: ++ tagged_book_fills = book_fills.copy() ++ tagged_book_fills["fold_id"] = fold.fold_id ++ tagged_book_fills["test_week"] = fold.test_week ++ all_book_fills.append(tagged_book_fills) ++ + benchmark_fills, benchmark = _evaluate_benchmark_week(test_state, books, trades, fold.test_week) + benchmark["fold_id"] = fold.fold_id + benchmark_rows.append(benchmark) +@@ -450,6 +486,8 @@ + validation_frame = pd.DataFrame(validation_rows) + candidate_ledger = pd.concat(all_candidates, ignore_index=True) if all_candidates else pd.DataFrame() + fill_ledger = pd.concat(all_fills, ignore_index=True) if all_fills else pd.DataFrame() ++ book_fill_ledger = pd.concat(all_book_fills, ignore_index=True) if all_book_fills else pd.DataFrame() ++ book_fold_results = pd.DataFrame(book_fold_rows) + benchmark_fill_ledger = pd.concat(all_benchmark_fills, ignore_index=True) if all_benchmark_fills else pd.DataFrame() + + atomic_pnl = fill_ledger.loc[fill_ledger.get("atomic", False).astype(bool), "pnl"] if not fill_ledger.empty else pd.Series(dtype=float) +@@ -488,12 +526,15 @@ + admission["permutation_ruin_rate"] = 0.0 + + fold_results.to_csv(output_dir / "fold_results.csv", index=False) ++ book_fold_results.to_csv(output_dir / "book_fold_results.csv", index=False) + benchmark_results.to_csv(output_dir / "benchmark_results.csv", index=False) + validation_frame.to_csv(output_dir / "validation_grid.csv", index=False) + if not candidate_ledger.empty: + _write_frame(candidate_ledger, output_dir / "candidate_ledger.parquet") + if not fill_ledger.empty: + _write_frame(fill_ledger, output_dir / "execution_ledger.parquet") ++ if not book_fill_ledger.empty: ++ _write_frame(book_fill_ledger, output_dir / "book_execution_ledger.parquet") + if not benchmark_fill_ledger.empty: + _write_frame(benchmark_fill_ledger, output_dir / "benchmark_execution_ledger.parquet") + if not candidate_ledger.empty: +@@ -531,7 +572,14 @@ + "weeks": weeks, + "folds": [fold.__dict__ if hasattr(fold, "__dict__") else {"fold_id": fold.fold_id, "validation_weeks": fold.validation_weeks, "test_week": fold.test_week, "fit_weeks": fold.fit_weeks} for fold in folds], + "configs": [config.__dict__ if hasattr(config, "__dict__") else {field: getattr(config, field) for field in config.__dataclass_fields__} for config in configs], +- "execution": {"latency_ms": 1000, "tape_window_ms": 1500, "shares": BASE_SHARES, "base_slippage": 0.01}, ++ "execution": { ++ "latency_ms": 1000, ++ "tape_window_ms": 1500, ++ "shares": BASE_SHARES, ++ "base_slippage": 0.01, ++ "admission_fill_model": "tape_confirmed", ++ "diagnostic_fill_model": "book_arrival", ++ }, + "risk": {"per_position_max_loss_fraction": 0.01, "concurrent_max_loss_fraction": 0.20}, + "environment": {"python": sys.version, "platform": platform.platform(), "pythonhashseed": os.environ.get("PYTHONHASHSEED")}, + "admission": admission, +diff --git a/tests/digital_surface/test_execution.py b/tests/digital_surface/test_execution.py +--- a/tests/digital_surface/test_execution.py ++++ b/tests/digital_surface/test_execution.py +@@ -59,3 +59,36 @@ + bad = replay_single_leg_fok(signals, books(), tape(prearrival_only=True), shares=5) + assert good.iloc[0].filled and good.iloc[0].pnl > 0 + assert not bad.iloc[0].filled ++ ++ ++def test_book_arrival_model_uses_post_latency_executable_depth_without_future_tape(): ++ conservative = replay_atomic_fok( ++ candidate(), books(), tape(prearrival_only=True), shares=5, latency_ms=1000 ++ ) ++ estimated = replay_atomic_fok( ++ candidate(), ++ books(), ++ tape(prearrival_only=True), ++ shares=5, ++ latency_ms=1000, ++ fill_model="book_arrival", ++ ) ++ assert not conservative.iloc[0].atomic ++ assert estimated.iloc[0].atomic ++ assert estimated.iloc[0].fill_model == "book_arrival" ++ assert estimated.iloc[0].low_yes_book_executable ++ assert estimated.iloc[0].high_no_book_executable ++ assert not estimated.iloc[0].low_yes_tape_confirmed ++ assert not estimated.iloc[0].high_no_tape_confirmed ++ ++ ++def test_book_arrival_model_still_rejects_missing_or_uncrossable_books(): ++ missing_high = books().loc[lambda frame: frame["condition_id"] != "high"] ++ missing = replay_atomic_fok( ++ candidate(), missing_high, tape(), shares=5, latency_ms=1000, fill_model="book_arrival" ++ ) ++ expensive = replay_atomic_fok( ++ candidate(), books(high_ask=.60), tape(), shares=5, latency_ms=1000, fill_model="book_arrival" ++ ) ++ assert not missing.iloc[0].atomic ++ assert not expensive.iloc[0].atomic +diff --git a/tests/digital_surface/test_integration.py b/tests/digital_surface/test_integration.py +--- a/tests/digital_surface/test_integration.py ++++ b/tests/digital_surface/test_integration.py +@@ -65,9 +65,14 @@ + "REPORT.md", "MANIFEST.json", "SHA256SUMS.json", "fold_results.csv", + "validation_grid.csv", "admission.json", "INVALIDATIONS.md", + "execution_failure_attribution.csv", "EXECUTION_DIAGNOSTICS.json", ++ "book_fold_results.csv", + ]: + assert (output / name).exists(), name ++ assert (output / "book_execution_ledger.parquet").exists() or (output / "book_execution_ledger.csv").exists() + folds = pd.read_csv(output / "fold_results.csv") + assert len(folds) == 4 + assert folds["atomic_count"].sum() == 4 ++ book_folds = pd.read_csv(output / "book_fold_results.csv") ++ assert len(book_folds) == 4 ++ assert book_folds["atomic_count"].sum() == 4 + assert "pre-arrival" in (output / "INVALIDATIONS.md").read_text().lower() diff --git a/ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/CANDIDATE_SPEC.json b/ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/CANDIDATE_SPEC.json new file mode 100644 index 00000000..f29caac6 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/CANDIDATE_SPEC.json @@ -0,0 +1,23 @@ +{ + "arrival_adverse_move_cancel": { + "max_price_improvement": 0.01, + "rule": "cancel if execution_ask < signal_ask - 0.01" + }, + "assets": [ + "btc", + "eth" + ], + "entry_second": 210, + "fee_per_share": "0.07*p*(1-p)", + "fok_limit": "signal_ask", + "hold": "settlement", + "latency_seconds": 1, + "live_submission": "disabled", + "max_positions_per_asset_per_window": 1, + "max_total_positions_per_window": 2, + "name": "late_favorite_btc_eth_v2_diagnostic", + "shares": 5, + "signal_ask_min": 0.85, + "source_artifact": "polymarket-5m-favorite-walkforward-v1", + "status": "diagnostic; prospective validation required" +} diff --git a/ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/REPORT.md b/ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/REPORT.md new file mode 100644 index 00000000..d511a202 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/deep-favorite-eda-v2/REPORT.md @@ -0,0 +1,94 @@ +# Polymarket 5-Minute Crypto — Deep Favorite EDA Checkpoint + +## Decision-useful answer + +The strongest high-frequency candidate in the 89,233-market / 26.77M-tick corpus is not BTC-to-alt lead-lag. It is a **late-window settlement-favorite strategy** at 210 seconds with a signal ask of at least 85 cents, one-second execution latency, five-share FOK sizing, and taker fees included. + +The all-seven-asset version was positive on the final chronological test segment, but the edge was diluted by weak assets and concentrated in a few days. The statistically credible core is **BTC + ETH**. This is a research candidate for official-label transfer and prospective shadowing, not permission for live trading. + +## Evidence spine + +### Original strict walk-forward result + +- Train: 38 active days, 18,493 trades, +$1,088.33 at five shares, daily t = 5.91. +- Validation: 14 active days, 6,819 trades, +$395.42, daily t = 5.35, every validation day positive. +- Final chronological test: 12 active days, 6,769 trades, +$249.11, 24.67 trades/hour, daily t = 1.67. +- The selected 210s / 85c configuration ranked #1 of 982 eligible configurations by validation daily t. +- Every tested latency from 0 to 3 seconds remained positive in train and validation. + +### BTC + ETH alpha spine + +- 2,054 final-test trades. +- +$139.72 at five shares. +- 95.67% win rate. +- 1.3604 cents mean P&L per share after the modeled fee. +- Daily t = 3.85. +- Exact one-sided day-level sign-flip p = 0.00244. +- 95% day-bootstrap mean interval: +$5.84 to +$17.13 per day at five shares. +- BTC and ETH individually survive Benjamini-Hochberg control at 5% across seven assets. + +### Adverse-selection finding + +A lower execution ask is not always price improvement. When the favorite ask fell by at least 2 cents during the one-second latency window, the final test produced: + +- 288 trades; +- 84.38% win rate; +- -$48.44 at five shares; +- negative daily t. + +A one-cent improvement was positive, while two cents or more was toxic. This supports a causal **cancel if arrival ask is more than 1 cent below the signal ask** rule for the next prospective candidate. Because this rule was diagnosed after inspecting the final test, it must be validated only on new observations. + +### Cross-asset loss clustering + +- 38.66% of losses occurred in windows with at least two simultaneous asset losses. +- BTC/ETH loss correlation was 0.559. +- When BTC won, ETH won 98.39% of simultaneous windows; when BTC lost, ETH won only 38.46%. +- The all-seven portfolio derived 66.62% of total P&L from its two best days. +- BTC + ETH reduced top-two-day concentration to 38.02%. + +This is a common-factor crash problem, not independent diversification. Portfolio exposure must be capped by five-minute window, not only by asset. + +### Execution-price calibration + +BTC/ETH edge is concentrated in the 85c-93c range. Diagnostic one-cent buckets showed especially strong empirical edge at 86c, 88c, 91c and 92c. The 95c and 98c buckets were flat-to-negative, while 99c remained slightly positive. This irregularity argues against fitting a fine price lookup table from the inspected test; retain the broad 85c rule until prospective evidence accumulates. + +### Capacity and cost sensitivity + +- BTC/ETH break-even additional cost: 1.3604 cents/share. +- All-seven break-even additional cost: 0.7360 cents/share. +- At 100 shares, 56.13% of BTC/ETH test signals displayed enough top-of-book size and remained positive, but daily tail risk scaled sharply. +- At 250+ shares, statistical quality deteriorated; top-of-book size is selection-biased and is not a reliable live capacity guarantee. +- The all-seven strategy turned negative at 1 cent extra slippage; the BTC/ETH core has more cost headroom but still requires measured prospective fill slippage. + +## Frozen prospective hypothesis + +```json +{ + "name": "late_favorite_btc_eth_v2_diagnostic", + "assets": ["btc", "eth"], + "entry_second": 210, + "signal_ask_min": 0.85, + "latency_seconds": 1, + "fok_limit": "signal ask", + "shares": 5, + "arrival_adverse_move_cancel": "cancel when execution ask < signal ask - 0.01", + "maximum_positions_per_asset_per_window": 1, + "maximum_total_positions_per_window": 2, + "hold": "settlement", + "fee_per_share": "0.07 * p * (1-p)", + "live_submission": "disabled", + "status": "diagnostic candidate; requires new official-label prospective validation" +} +``` + +## Non-negotiable uncertainty + +1. Dataset outcomes are inferred from the final recorded book, not authoritative settlement. +2. Only 28 false winning labels would erase the BTC/ETH final-test profit; official-resolution validation is mandatory. +3. Ask-side capacity is top-of-book only; queue position and full depth were not recorded. +4. The BTC/ETH universe and adverse-move cancel were identified after inspecting the final test. They cannot inherit the original untouched-test status. +5. Real-money order submission remains absent. The next valid evidence is an append-only prospective shadow ledger. + +## Next experiment + +Run the frozen BTC/ETH candidate prospectively with official terminal outcomes and public-book evidence. Report every five-minute window, including no-trade decisions, arrival price movement, displayed size, hypothetical FOK fill, settlement, and per-window portfolio exposure. Admission requires at least four untouched weekly blocks and enough trades to establish a positive lower confidence bound after measured fees and slippage. diff --git a/ci_payload/digital-surface-live-readiness/execution-diagnostics.patch b/ci_payload/digital-surface-live-readiness/execution-diagnostics.patch new file mode 100644 index 00000000..bac5be7c --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/execution-diagnostics.patch @@ -0,0 +1,269 @@ +diff --git a/research/digital_surface/run_research.py b/research/digital_surface/run_research.py +index cbe6068..7d5d5cd 100644 +--- a/research/digital_surface/run_research.py ++++ b/research/digital_surface/run_research.py +@@ -14,6 +14,7 @@ import pandas as pd + + from .candidates import CandidateConfig, generate_cross_horizon_candidates, select_one_candidate_per_expiry + from .data import SOURCE_SPEC, build_research_panel, sha256_file ++from .diagnostics import attribute_execution_failures + from .execution import replay_atomic_fok, replay_single_leg_fok + from .fees import all_in_cost + from .portfolio import replay_bankroll +@@ -495,6 +496,29 @@ def run(cache_dir: Path, output_dir: Path) -> dict[str, Any]: + _write_frame(fill_ledger, output_dir / "execution_ledger.parquet") + if not benchmark_fill_ledger.empty: + _write_frame(benchmark_fill_ledger, output_dir / "benchmark_execution_ledger.parquet") ++ if not candidate_ledger.empty: ++ attribution, diagnostic_summary = attribute_execution_failures(candidate_ledger, fill_ledger) ++ attribution.to_csv(output_dir / "execution_failure_attribution.csv", index=False) ++ (output_dir / "EXECUTION_DIAGNOSTICS.json").write_text( ++ json.dumps(diagnostic_summary, indent=2, default=_json_default), ++ encoding="utf-8", ++ ) ++ else: ++ pd.DataFrame(columns=["candidate_id", "leg", "reason"]).to_csv( ++ output_dir / "execution_failure_attribution.csv", index=False ++ ) ++ (output_dir / "EXECUTION_DIAGNOSTICS.json").write_text( ++ json.dumps( ++ { ++ "candidates_attributed": 0, ++ "legs_attributed": 0, ++ "complete_attribution": True, ++ "reason_counts": [], ++ }, ++ indent=2, ++ ), ++ encoding="utf-8", ++ ) + pd.DataFrame(coherence_rows).to_csv(output_dir / "coherence.csv", index=False) + pd.DataFrame(calibration_rows).to_csv(output_dir / "calibration.csv", index=False) + (output_dir / "validation_choices.json").write_text(json.dumps(validation_choices, indent=2, default=_json_default), encoding="utf-8") +diff --git a/research/digital_surface/diagnostics.py b/research/digital_surface/diagnostics.py +new file mode 100644 +index 0000000..c80fd53 +--- /dev/null ++++ b/research/digital_surface/diagnostics.py +@@ -0,0 +1,135 @@ ++from __future__ import annotations ++ ++from typing import Any ++ ++import numpy as np ++import pandas as pd ++ ++ ++_CANDIDATE_REQUIRED = {"candidate_id"} ++_EXECUTION_REQUIRED = { ++ "candidate_id", ++ "shares", ++ "atomic", ++ "low_yes_book_ts_ms", ++ "high_no_book_ts_ms", ++ "low_yes_arrival_ask", ++ "high_no_arrival_ask", ++ "low_yes_arrival_size", ++ "high_no_arrival_size", ++ "low_yes_tape_size", ++ "high_no_tape_size", ++ "low_yes_limit", ++ "high_no_limit", ++} ++ ++ ++def _require_columns(frame: pd.DataFrame, required: set[str], name: str) -> None: ++ missing = sorted(required - set(frame.columns)) ++ if missing: ++ raise ValueError(f"{name} missing required columns: {missing}") ++ ++ ++def _reason_for_leg(row: pd.Series, prefix: str) -> str: ++ book_ts = row[f"{prefix}_book_ts_ms"] ++ ask = float(row[f"{prefix}_arrival_ask"]) if pd.notna(row[f"{prefix}_arrival_ask"]) else np.nan ++ limit_px = float(row[f"{prefix}_limit"]) ++ displayed = float(row[f"{prefix}_arrival_size"]) if pd.notna(row[f"{prefix}_arrival_size"]) else 0.0 ++ tape = float(row[f"{prefix}_tape_size"]) if pd.notna(row[f"{prefix}_tape_size"]) else 0.0 ++ shares = float(row["shares"]) ++ ++ if pd.isna(book_ts): ++ return "missing_post_arrival_book" ++ if not np.isfinite(ask) or ask > limit_px + 1e-12: ++ return "ask_above_limit" ++ if not np.isfinite(displayed) or displayed < shares: ++ return "insufficient_displayed_depth" ++ if not np.isfinite(tape) or tape < shares: ++ return "insufficient_post_arrival_tape" ++ return "filled" ++ ++ ++def attribute_execution_failures( ++ candidates: pd.DataFrame, ++ executions: pd.DataFrame, ++) -> tuple[pd.DataFrame, dict[str, Any]]: ++ """Assign exactly one terminal execution reason to each candidate leg.""" ++ _require_columns(candidates, _CANDIDATE_REQUIRED, "candidates") ++ _require_columns(executions, _EXECUTION_REQUIRED, "executions") ++ if candidates["candidate_id"].astype(str).duplicated().any(): ++ raise ValueError("candidates contain duplicate candidate_id values") ++ if executions["candidate_id"].astype(str).duplicated().any(): ++ raise ValueError("executions contain duplicate candidate_id values") ++ ++ candidate_frame = candidates.copy() ++ execution_frame = executions.copy() ++ candidate_frame["candidate_id"] = candidate_frame["candidate_id"].astype(str) ++ execution_frame["candidate_id"] = execution_frame["candidate_id"].astype(str) ++ merged = candidate_frame.merge( ++ execution_frame, ++ on="candidate_id", ++ how="left", ++ validate="one_to_one", ++ suffixes=("", "_execution"), ++ indicator=True, ++ ) ++ if not merged["_merge"].eq("both").all(): ++ missing = merged.loc[merged["_merge"] != "both", "candidate_id"].head(10).tolist() ++ raise ValueError(f"missing execution rows for candidates: {missing}") ++ merged = merged.drop(columns="_merge") ++ ++ rows: list[dict[str, Any]] = [] ++ passthrough = [ ++ column ++ for column in ( ++ "fold_id", ++ "test_week", ++ "decision_ts_ms", ++ "close_ts", ++ "tau_s", ++ "expected_cost", ++ "theoretical_edge", ++ "config_id", ++ "shares", ++ "atomic", ++ "orphan_leg", ++ ) ++ if column in merged.columns ++ ] ++ for _, row in merged.iterrows(): ++ base = {column: row[column] for column in passthrough} ++ for leg, prefix in (("low_yes", "low_yes"), ("high_no", "high_no")): ++ reason = _reason_for_leg(row, prefix) ++ rows.append( ++ { ++ "candidate_id": str(row["candidate_id"]), ++ "leg": leg, ++ "reason": reason, ++ "book_ts_ms": row[f"{prefix}_book_ts_ms"], ++ "arrival_ask": row[f"{prefix}_arrival_ask"], ++ "limit_px": row[f"{prefix}_limit"], ++ "displayed_size": row[f"{prefix}_arrival_size"], ++ "post_arrival_tape_size": row[f"{prefix}_tape_size"], ++ **base, ++ } ++ ) ++ ledger = pd.DataFrame(rows) ++ expected_legs = 2 * len(candidates) ++ complete = bool(len(ledger) == expected_legs and ledger["reason"].notna().all()) ++ if not complete: ++ raise AssertionError("execution attribution is incomplete") ++ ++ reason_counts = ( ++ ledger.groupby([column for column in ("test_week", "leg", "reason") if column in ledger.columns], dropna=False) ++ .size() ++ .rename("count") ++ .reset_index() ++ .to_dict("records") ++ ) ++ summary: dict[str, Any] = { ++ "candidates_attributed": int(len(candidates)), ++ "legs_attributed": int(len(ledger)), ++ "complete_attribution": complete, ++ "reason_counts": reason_counts, ++ } ++ return ledger.sort_values(["candidate_id", "leg"], kind="mergesort").reset_index(drop=True), summary +diff --git a/tests/digital_surface/test_diagnostics.py b/tests/digital_surface/test_diagnostics.py +new file mode 100644 +index 0000000..ad441ac +--- /dev/null ++++ b/tests/digital_surface/test_diagnostics.py +@@ -0,0 +1,68 @@ ++import numpy as np ++import pandas as pd ++ ++from research.digital_surface.diagnostics import attribute_execution_failures ++ ++ ++def test_attributes_each_leg_to_one_terminal_reason(): ++ candidates = pd.DataFrame([ ++ { ++ "candidate_id": "a", ++ "tau_s": 30.0, ++ "expected_cost": 1.05, ++ "theoretical_edge": 0.03, ++ "fold_id": 1, ++ "test_week": "w1", ++ }, ++ { ++ "candidate_id": "b", ++ "tau_s": 12.0, ++ "expected_cost": 1.02, ++ "theoretical_edge": 0.04, ++ "fold_id": 1, ++ "test_week": "w1", ++ }, ++ ]) ++ executions = pd.DataFrame([ ++ { ++ "candidate_id": "a", ++ "shares": 5.0, ++ "atomic": False, ++ "orphan_leg": None, ++ "low_yes_book_ts_ms": np.nan, ++ "high_no_book_ts_ms": 2, ++ "low_yes_arrival_ask": np.nan, ++ "high_no_arrival_ask": 0.40, ++ "low_yes_arrival_size": 0.0, ++ "high_no_arrival_size": 10.0, ++ "low_yes_tape_size": 0.0, ++ "high_no_tape_size": 2.0, ++ "low_yes_limit": 0.50, ++ "high_no_limit": 0.50, ++ }, ++ { ++ "candidate_id": "b", ++ "shares": 5.0, ++ "atomic": True, ++ "orphan_leg": None, ++ "low_yes_book_ts_ms": 2, ++ "high_no_book_ts_ms": 2, ++ "low_yes_arrival_ask": 0.30, ++ "high_no_arrival_ask": 0.40, ++ "low_yes_arrival_size": 10.0, ++ "high_no_arrival_size": 10.0, ++ "low_yes_tape_size": 8.0, ++ "high_no_tape_size": 7.0, ++ "low_yes_limit": 0.50, ++ "high_no_limit": 0.50, ++ }, ++ ]) ++ ++ ledger, summary = attribute_execution_failures(candidates, executions) ++ indexed = ledger.set_index(["candidate_id", "leg"]) ++ assert indexed.loc[("a", "low_yes"), "reason"] == "missing_post_arrival_book" ++ assert indexed.loc[("a", "high_no"), "reason"] == "insufficient_post_arrival_tape" ++ assert ledger[ledger["candidate_id"] == "b"]["reason"].tolist() == ["filled", "filled"] ++ assert summary["candidates_attributed"] == 2 ++ assert summary["legs_attributed"] == 4 ++ assert summary["complete_attribution"] is True +diff --git a/tests/digital_surface/test_integration.py b/tests/digital_surface/test_integration.py +index f972c36..a7c2788 100644 +--- a/tests/digital_surface/test_integration.py ++++ b/tests/digital_surface/test_integration.py +@@ -64,6 +64,7 @@ def test_end_to_end_runner_writes_auditable_artifacts(tmp_path, monkeypatch): + for name in [ + "REPORT.md", "MANIFEST.json", "SHA256SUMS.json", "fold_results.csv", + "validation_grid.csv", "admission.json", "INVALIDATIONS.md", ++ "execution_failure_attribution.csv", "EXECUTION_DIAGNOSTICS.json", + ]: + assert (output / name).exists(), name + folds = pd.read_csv(output / "fold_results.csv") diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/ADMISSION_LEDGER.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/ADMISSION_LEDGER.json new file mode 100644 index 00000000..15dee8c1 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/ADMISSION_LEDGER.json @@ -0,0 +1,62 @@ +{ + "candidate": "late_favorite_btc_only_v3_prospective_shadow", + "candidate_policy_sha256": "82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba", + "capture_policy_sha256": "888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc", + "current": { + "complete_untouched_weekly_blocks": 0, + "hypothetical_fok_fills": 2, + "official_outcome_coverage": { + "denominator": 10, + "numerator": 10 + }, + "prospective_markets_observed": 10, + "prospective_pnl_total": "0.843790", + "unresolved_states": 0 + }, + "gates": { + "maximum_top_two_day_pnl_concentration": 0.4, + "maximum_unresolved_states": 0, + "minimum_complete_untouched_weekly_blocks": 4, + "minimum_hypothetical_fok_fills": 100, + "minimum_prospective_markets_observed": 500, + "official_outcome_coverage": 1.0, + "positive_day_block_bootstrap_lower_bound": true, + "positive_total_pnl_after_fees": true + }, + "observations": [ + { + "artifact_zip_sha256": "8d00c88d6046806a2abc07f43da8b2f0ab08449890ee8a7f9e1f7071b92c4e21", + "checkpoint": "prospective-run-v1", + "hypothetical_fok_fills": 0, + "markets": 1, + "pnl_total": "0.00", + "runtime_source_sha256": "ce477bf97589997899c91ad22af9025de521520f8ee7aa46a21b50a313958bd7" + }, + { + "artifact_zip_sha256": "222d8261ea92dfdfcd3fbb9ccb12dbf6948d03f6bed1e7558f3d9e4bdc6a5580", + "checkpoint": "prospective-block-v1", + "hypothetical_fok_fills": 0, + "markets": 3, + "pnl_total": "0.00", + "runtime_source_sha256": "81a13d2aeb3df3c2088b8aa99272b10e217a392eabc839a01a8f2b71f570ecbb" + }, + { + "artifact_zip_sha256": "69ee1bb111f8aec0b52f982f305136a5e835ca2b2e50f4b66a129d3370b5ac58", + "checkpoint": "prospective-block-v2", + "hypothetical_fok_fills": 1, + "markets": 3, + "pnl_total": "0.233375", + "runtime_source_sha256": "81a13d2aeb3df3c2088b8aa99272b10e217a392eabc839a01a8f2b71f570ecbb" + }, + { + "artifact_zip_sha256": "b30e9e10de839070ad1ae88f98c952ff2d8d4378ab828bed1533cff0e588cc0e", + "checkpoint": "prospective-block-v3", + "hypothetical_fok_fills": 1, + "markets": 3, + "pnl_total": "0.610415", + "runtime_source_sha256": "81a13d2aeb3df3c2088b8aa99272b10e217a392eabc839a01a8f2b71f570ecbb" + } + ], + "status": "prospective_shadow_only_not_admitted", + "version": 3 +} diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/CANDIDATE_SPEC.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/CANDIDATE_SPEC.json new file mode 100644 index 00000000..8e32c009 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/CANDIDATE_SPEC.json @@ -0,0 +1,78 @@ +{ + "arrival": { + "cancel_if": "arrival best ask < signal ask - 0.01", + "fok_limit": "signal_ask", + "latency_seconds": 1, + "no_fill_if": [ + "arrival best ask > signal ask", + "cumulative displayed ask size at prices <= signal ask is less than 5", + "arrival book missing, stale, invalid, or token-mismatched" + ], + "shares": "5" + }, + "contamination_rule": "Only markets whose open time is strictly after the policy freeze commit may count. All previously inspected markets and any observation made before the freeze commit are diagnostic only.", + "economics": { + "additional_slippage_assumption": "0; FOK limit caps price, but prospective reporting must separately measure public-book crossability and hypothetical fill rate", + "fee_per_share": "0.07*p*(1-p)", + "hold": "settlement", + "pnl_per_share": "official_win - executable_arrival_price - fee_per_share" + }, + "name": "late_favorite_btc_only_v3_prospective_shadow", + "observation_contract": { + "append_only_sha256_chain": true, + "arrival_evidence": [ + "request start/end timestamps", + "server/book timestamp", + "full selected-token book", + "raw payload SHA-256", + "cumulative size through FOK limit" + ], + "credentials_used": 0, + "live_submission": "physically_absent", + "record_every_market": true, + "record_no_signal_and_no_fill": true, + "resolution": "authoritative Polymarket terminal outcome with raw response hash", + "signal_evidence": [ + "request start/end timestamps", + "server/book timestamp", + "full Up and Down books", + "raw payload SHA-256" + ], + "unresolved_states_allowed": 0 + }, + "origin": { + "historical_admission_credit": 0, + "official_decision_commit": "cedd659e0b573d8479932c428cd2eaf472c6bf8f", + "official_transfer_filled_condition_set_sha256": "bf276fbce187d9651d56ecae976520d4251140c1d5b0d440696c5d077838a67a", + "selection": "post-hoc BTC subset from rejected late_favorite_btc_eth_v2_diagnostic official transfer" + }, + "prospective_admission_gates": { + "maximum_top_two_day_pnl_concentration": 0.4, + "maximum_unresolved_order_or_outcome_states": 0, + "minimum_hypothetical_fok_fills": 100, + "minimum_observed_markets": 500, + "minimum_untouched_weekly_blocks": 4, + "official_outcome_coverage": 1.0, + "policy_changes_during_run": 0, + "positive_btc_performance": true, + "positive_day_block_bootstrap_lower_bound": true, + "positive_total_pnl_after_fees": true + }, + "release_boundary": "An admissible shadow report cannot enable live trading. Any order submitter requires a separate reviewed release and explicit operator action.", + "signal": { + "entry_second": 210, + "favorite_selection": "higher executable ask between Up and Down; tied or invalid books fail closed", + "required_full_book": true, + "signal_ask_min": "0.85" + }, + "status": "prospective_shadow_only", + "universe": { + "assets": [ + "btc" + ], + "market_duration_seconds": 300, + "maximum_positions_per_asset_per_window": 1, + "maximum_total_positions_per_window": 1 + }, + "version": 3 +} diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/CAPTURE_FREEZE_MANIFEST.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/CAPTURE_FREEZE_MANIFEST.json new file mode 100644 index 00000000..8f1ddc5b --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/CAPTURE_FREEZE_MANIFEST.json @@ -0,0 +1,14 @@ +{ + "candidate_policy_sha256": "82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba", + "capture_policy_blob_sha": "9b089d3d0f82b3d7baa499aed897a07fdbdc1f7d", + "capture_policy_commit_sha": "ac6a1a267c958ac8fd327b11b7f72719627bd459", + "capture_policy_path": "ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/CAPTURE_POLICY.json", + "capture_policy_sha256": "888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc", + "frozen_at_utc": "2026-07-26T19:35:21Z", + "historical_admission_credit": 0, + "live_submission": "physically_absent", + "rule": "Only markets opening strictly after this capture-policy freeze may count toward prospective admission.", + "status": "frozen_before_prospective_collection", + "valid_market_open_after_utc": "2026-07-26T19:35:21Z", + "version": 1 +} diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/CAPTURE_POLICY.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/CAPTURE_POLICY.json new file mode 100644 index 00000000..9b089d3d --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/CAPTURE_POLICY.json @@ -0,0 +1,36 @@ +{ + "book_freshness": { + "max_book_age_at_request_completion_ms": 2000, + "max_book_future_skew_ms": 1000, + "missing_or_unparseable_book_timestamp": "fail_closed" + }, + "event_time": { + "arrival_target_basis": "market_open_not_signal_completion", + "arrival_target_offset_ms": 211000, + "signal_target_offset_ms": 210000 + }, + "name": "late_favorite_btc_only_v3_capture_policy", + "recording": { + "book_timestamp_ms": true, + "full_book": true, + "missed_window_decision": "record_and_fail_closed", + "raw_payload_sha256": true, + "request_completed_ts_ms": true, + "request_started_ts_ms": true, + "target_ts_ms": true + }, + "request_windows": { + "max_request_duration_ms": 1000, + "max_request_start_lateness_ms": 250, + "request_may_start_before_target": false + }, + "status": "frozen_before_prospective_collection", + "transport": { + "arrival_request": "single public CLOB /books request for selected token ID", + "authenticated_requests": 0, + "credentials_used": 0, + "order_submissions": 0, + "signal_request": "single public CLOB /books batch for Up and Down token IDs" + }, + "version": 1 +} diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/CHECKPOINT.md b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/CHECKPOINT.md new file mode 100644 index 00000000..627814c3 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/CHECKPOINT.md @@ -0,0 +1,68 @@ +# BTC-Only v3 Prospective Observer Checkpoint + +## Status + +The frozen `late_favorite_btc_only_v3_prospective_shadow` observer and credential-free public collector are implemented and clean-room verified. This checkpoint is **infrastructure readiness only**. It contains no prospective market observations and grants no admission credit. + +## Frozen boundaries + +- Candidate policy SHA-256: `82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba` +- Candidate cutoff: `2026-07-26T19:11:11Z` +- Capture policy SHA-256: `888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc` +- Effective capture cutoff: `2026-07-26T19:35:21Z` +- Historical admission credit: **0** +- Live submission: **physically absent** + +Only BTC five-minute markets opening strictly after the effective capture cutoff may enter a prospective ledger. + +## Implemented behavior + +1. Signal target remains market open +210 seconds. +2. Arrival target remains market open +211 seconds and is independent of HTTP completion time. +3. Public request starts may be at most 250 ms late; requests longer than 1,000 ms fail closed. +4. Books older than 2,000 ms at request completion fail closed. +5. Signal uses the higher executable Up/Down ask and requires at least $0.85. +6. Arrival applies the frozen adverse-move cancel and FOK price limit. +7. Five-share execution uses cumulative full-depth levels and per-level fee accounting. +8. Every discovery, signal/no-signal, arrival/no-fill, and resolution is append-only and hash chained. +9. Official settlement uses closed binary Gamma terminal prices and raw response SHA-256 evidence. +10. Raw public response bodies are content-addressed and bound to a separate evidence manifest chain. + +## Public capability boundary + +The only allowed network operations are: + +- `GET https://gamma-api.polymarket.com/markets/slug/{slug}` +- `POST https://clob.polymarket.com/books` + +The clean-room capability scan found: + +- credentials used: **0** +- authenticated requests: **0** +- order submissions: **0** +- signing/private-key libraries: **0** + +## Verification evidence + +- Workflow run: `30217953678` +- Artifact ID: `8636344655` +- Artifact ZIP SHA-256: `667d7090ecfe3d174e5821af29a26fde2e18a87a50f7891cdcc49dff740f26af` +- Tests: **22 passed** +- Combined source SHA-256: `f2e086bb5e74f0343a4ea29f46ea2bf8e23062952b456303e204c6b5469b6126` +- Deterministic TAR sidecar: verified +- Internal `SHA256SUMS`: all files verified +- Archive extraction round-trip: byte-for-byte verified + +Source file SHA-256 values are recorded in `OBSERVER_VERIFICATION.json`. + +## Admission boundary + +This checkpoint does not show that BTC-only is profitable. The inspected official historical point estimate was marginal and post-hoc. Admission still requires genuinely new observations, at least four untouched weekly blocks, authoritative outcome coverage, measured public-book fill evidence, acceptable concentration, zero unresolved states, and a positive lower confidence bound after fees and execution costs. + +## Next valid work + +1. Add a one-shot runnable CLI that selects the next eligible BTC window and persists its ledger/evidence directories. +2. Execute the collector only on post-cutoff markets. +3. Add a separate resolver command for closed markets. +4. Package append-only ledger/evidence artifacts after each run. +5. Keep live order submission absent. diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/FREEZE_MANIFEST.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/FREEZE_MANIFEST.json new file mode 100644 index 00000000..ab8dba80 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/FREEZE_MANIFEST.json @@ -0,0 +1,18 @@ +{ + "candidate_name": "late_favorite_btc_only_v3_prospective_shadow", + "contamination_boundary": { + "historical_admission_credit": 0, + "rule": "Only BTC 5-minute markets with open_epoch_seconds strictly greater than the freeze timestamp may count toward prospective admission.", + "valid_market_open_after_utc": "2026-07-26T19:11:11Z" + }, + "freeze_commit_sha": "bb3a818dc017f18927e6e48b38e1078acb23aaa0", + "frozen_at_utc": "2026-07-26T19:11:11Z", + "live_submission": "physically_absent", + "official_decision_commit": "cedd659e0b573d8479932c428cd2eaf472c6bf8f", + "policy_path": "ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/CANDIDATE_SPEC.json", + "policy_sha256": "82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba", + "policy_blob_sha": "8e32c00919c77c20dc0d5f0f36bfe04c464d0918", + "release_boundary": "Shadow evidence cannot enable live trading; a separate reviewed and explicitly armed release is required.", + "status": "frozen_for_prospective_shadow_collection", + "version": 1 +} diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/OBSERVER_VERIFICATION.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/OBSERVER_VERIFICATION.json new file mode 100644 index 00000000..80d7a8bd --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/OBSERVER_VERIFICATION.json @@ -0,0 +1,25 @@ +{ + "authenticated_requests": 0, + "candidate": "late_favorite_btc_only_v3_prospective_shadow", + "candidate_cutoff_epoch_seconds": 1785093071, + "candidate_policy_sha256": "82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba", + "capture_cutoff_epoch_seconds": 1785094521, + "capture_policy_sha256": "888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc", + "combined_source_sha256": "f2e086bb5e74f0343a4ea29f46ea2bf8e23062952b456303e204c6b5469b6126", + "credentials_used": 0, + "historical_admission_credit": 0, + "live_submission": "physically_absent", + "official_resolution": "Gamma closed binary terminal outcome", + "order_submissions": 0, + "public_endpoints": [ + "GET https://gamma-api.polymarket.com/markets/slug/{slug}", + "POST https://clob.polymarket.com/books" + ], + "source_files_sha256": { + "collector/public_collector.py": "85c16a1e9805b89a234139f15140c4f0f39658887419d70c2caf4c238dd5cc6b", + "collector/resolution.py": "bb4a643d4618e78442c11ef21ac15483c048a89e5bc7e900500af661f6a8c91c", + "source/capture_policy.py": "bda91808405d5b9e6ee6a09bd464cecd751c23573584bc2624891f5bf091e955", + "source/late_favorite_v3.py": "190ad8d358229f448b42a4dd8dc0e25532b440ce88a8e897fb9cb3c835cc42ee" + }, + "tests_collected": 22 +} diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector/cli.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector/cli.py new file mode 100644 index 00000000..00461e0b --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector/cli.py @@ -0,0 +1,176 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import time +from pathlib import Path +from typing import Callable, Sequence + +from capture_policy import CaptureBoundLifecycleLedger, load_capture_policy +from late_favorite_v3 import load_frozen_policy +from public_collector import PublicHttpClient, RawEvidenceStore, SingleMarketCollector, next_collectable_open +from resolution import SingleMarketResolver + + +def _combined_source_sha256(root: Path) -> str: + paths = [ + root / "source/late_favorite_v3.py", + root / "source/capture_policy.py", + root / "collector/public_collector.py", + root / "collector/resolution.py", + ] + digest = hashlib.sha256() + for path in paths: + raw = path.read_bytes() + digest.update(path.name.encode("utf-8") + b"\0" + raw) + return digest.hexdigest() + + +def _load_policies(root: Path): + source_sha256 = _combined_source_sha256(root) + trading = load_frozen_policy( + root / "CANDIDATE_SPEC.json", + root / "FREEZE_MANIFEST.json", + source_sha256=source_sha256, + ) + capture = load_capture_policy( + root / "CAPTURE_POLICY.json", + root / "CAPTURE_FREEZE_MANIFEST.json", + ) + return trading, capture + + +def sleep_until_ms(target_ts_ms: int) -> None: + target = int(target_ts_ms) + while True: + remaining = target - time.time_ns() // 1_000_000 + if remaining <= 0: + return + time.sleep(min(remaining / 1000.0, 0.25)) + + +def _write_json_atomic(path: Path, payload: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + os.replace(temporary, path) + + +def _runtime( + *, + root: Path, + output_dir: Path, + client: PublicHttpClient | None, +): + trading, capture = _load_policies(root) + resolved_client = client or PublicHttpClient() + ledger = CaptureBoundLifecycleLedger(output_dir / "lifecycle.jsonl", trading, capture) + evidence = RawEvidenceStore(output_dir / "raw") + return trading, capture, resolved_client, ledger, evidence + + +def capture_once( + *, + root: str | Path, + output_dir: str | Path, + market_open_epoch_seconds: int | None = None, + now_ms: Callable[[], int] | None = None, + client: PublicHttpClient | None = None, + sleep_until_ms: Callable[[int], None] = sleep_until_ms, +) -> dict[str, object]: + root_path = Path(root).resolve() + output_path = Path(output_dir).resolve() + output_path.mkdir(parents=True, exist_ok=True) + trading, capture, resolved_client, ledger, evidence = _runtime( + root=root_path, + output_dir=output_path, + client=client, + ) + if market_open_epoch_seconds is None: + clock = now_ms or (lambda: time.time_ns() // 1_000_000) + market_open_epoch_seconds = next_collectable_open( + int(clock()), capture.valid_market_open_after_epoch_seconds + ) + opening = int(market_open_epoch_seconds) + collector = SingleMarketCollector( + client=resolved_client, + evidence_store=evidence, + ledger=ledger, + trading_policy=trading, + capture_policy=capture, + sleep_until_ms=sleep_until_ms, + ) + result = collector.capture(opening) + _write_json_atomic(output_path / "last_capture.json", result) + return dict(result) + + +def resolve_once( + *, + root: str | Path, + output_dir: str | Path, + condition_id: str, + client: PublicHttpClient | None = None, +) -> dict[str, object]: + root_path = Path(root).resolve() + output_path = Path(output_dir).resolve() + trading, capture, resolved_client, ledger, evidence = _runtime( + root=root_path, + output_dir=output_path, + client=client, + ) + resolver = SingleMarketResolver( + client=resolved_client, + evidence_store=evidence, + ledger=ledger, + trading_policy=trading, + capture_policy=capture, + ) + result = resolver.resolve(str(condition_id)) + _write_json_atomic(output_path / "last_resolution.json", result) + return dict(result) + + +def build_parser() -> argparse.ArgumentParser: + default_root = Path(__file__).resolve().parents[1] + parser = argparse.ArgumentParser( + description="Credential-free BTC-only v3 prospective observer" + ) + parser.add_argument("--root", type=Path, default=default_root) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--timeout-seconds", type=float, default=5.0) + subparsers = parser.add_subparsers(dest="command", required=True) + + capture_parser = subparsers.add_parser("capture", help="Capture one eligible BTC 5-minute market") + capture_parser.add_argument("--market-open", type=int) + + resolve_parser = subparsers.add_parser("resolve", help="Resolve one previously captured market") + resolve_parser.add_argument("--condition-id", required=True) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + client = PublicHttpClient(timeout_seconds=args.timeout_seconds) + if args.command == "capture": + result = capture_once( + root=args.root, + output_dir=args.output_dir, + market_open_epoch_seconds=args.market_open, + client=client, + ) + else: + result = resolve_once( + root=args.root, + output_dir=args.output_dir, + condition_id=args.condition_id, + client=client, + ) + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector/public_collector.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector/public_collector.py new file mode 100644 index 00000000..d752399f --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector/public_collector.py @@ -0,0 +1,365 @@ +from __future__ import annotations + +import hashlib +import json +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Mapping, Sequence +from urllib.parse import urlparse + +import requests + +from capture_policy import ( + CaptureBoundLifecycleLedger, + CapturePolicy, + evaluate_arrival_capture, + evaluate_signal_capture, +) +from late_favorite_v3 import FrozenPolicy + +GAMMA_BASE = "https://gamma-api.polymarket.com" +CLOB_BASE = "https://clob.polymarket.com" + + +class PublicEndpointViolation(RuntimeError): + pass + + +class PublicResponseError(RuntimeError): + pass + + +@dataclass(frozen=True, slots=True) +class PublicHttpEvidence: + method: str + url: str + status_code: int + request_started_ts_ms: int + request_completed_ts_ms: int + response_body_sha256: str + request_body_sha256: str | None + body: bytes + payload: object + + +@dataclass(frozen=True, slots=True) +class BtcFiveMinuteMarket: + condition_id: str + slug: str + open_epoch_seconds: int + up_token_id: str + down_token_id: str + + +def _canonical_json(value: object) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _validate_public_endpoint(method: str, url: str) -> None: + parsed = urlparse(url) + if parsed.scheme != "https" or parsed.username or parsed.password: + raise PublicEndpointViolation("only credential-free HTTPS endpoints are allowed") + normalized_method = method.upper() + if parsed.netloc == "gamma-api.polymarket.com": + allowed = normalized_method == "GET" and parsed.path.startswith("/markets/slug/") + elif parsed.netloc == "clob.polymarket.com": + allowed = normalized_method == "POST" and parsed.path == "/books" + else: + allowed = False + if not allowed: + raise PublicEndpointViolation(f"endpoint is outside the frozen public allowlist: {method} {url}") + + +class PublicHttpClient: + def __init__( + self, + *, + requester: Callable[..., Any] = requests.request, + clock_ms: Callable[[], int] | None = None, + timeout_seconds: float = 5.0, + ) -> None: + self.requester = requester + self.clock_ms = clock_ms or (lambda: time.time_ns() // 1_000_000) + self.timeout_seconds = float(timeout_seconds) + if self.timeout_seconds <= 0: + raise ValueError("timeout_seconds must be positive") + + def request_json( + self, + method: str, + url: str, + *, + json_body: object | None = None, + ) -> PublicHttpEvidence: + normalized_method = method.upper() + _validate_public_endpoint(normalized_method, url) + headers = { + "Accept": "application/json", + "User-Agent": "btc-only-v3-prospective-shadow/1", + } + if json_body is not None: + headers["Content-Type"] = "application/json" + started = int(self.clock_ms()) + response = self.requester( + normalized_method, + url, + headers=headers, + timeout=self.timeout_seconds, + json=json_body, + ) + completed = int(self.clock_ms()) + status = int(response.status_code) + body = bytes(response.content) + if status != 200: + raise PublicResponseError(f"public endpoint returned HTTP {status}: {url}") + try: + payload = json.loads(body) + except json.JSONDecodeError as exc: + raise PublicResponseError(f"public endpoint returned invalid JSON: {url}") from exc + request_hash = None if json_body is None else hashlib.sha256(_canonical_json(json_body)).hexdigest() + return PublicHttpEvidence( + method=normalized_method, + url=url, + status_code=status, + request_started_ts_ms=started, + request_completed_ts_ms=completed, + response_body_sha256=hashlib.sha256(body).hexdigest(), + request_body_sha256=request_hash, + body=body, + payload=payload, + ) + + +class RawEvidenceStore: + def __init__(self, root: str | Path) -> None: + self.root = Path(root) + self.manifest_path = self.root / "manifest.jsonl" + self.records: list[dict[str, Any]] = [] + self.head_hash = "0" * 64 + if self.manifest_path.exists(): + self._load_and_verify() + + def _load_and_verify(self) -> None: + previous = "0" * 64 + for line_number, line in enumerate( + self.manifest_path.read_text(encoding="utf-8").splitlines(), start=1 + ): + if not line.strip(): + continue + try: + row = json.loads(line) + except json.JSONDecodeError as exc: + raise RuntimeError(f"invalid evidence manifest JSON at line {line_number}") from exc + stored = str(row.get("record_hash", "")) + without = dict(row) + without.pop("record_hash", None) + if without.get("previous_hash") != previous: + raise RuntimeError(f"evidence hash chain mismatch at line {line_number}") + expected = hashlib.sha256(_canonical_json(without)).hexdigest() + if stored != expected: + raise RuntimeError(f"evidence record hash mismatch at line {line_number}") + body_path = self.root / str(row["body_path"]) + if not body_path.is_file(): + raise RuntimeError(f"missing evidence body at line {line_number}") + if hashlib.sha256(body_path.read_bytes()).hexdigest() != row["response_body_sha256"]: + raise RuntimeError(f"evidence body hash mismatch at line {line_number}") + self.records.append(row) + self.head_hash = stored + previous = stored + + def append(self, evidence: PublicHttpEvidence, *, purpose: str) -> dict[str, Any]: + if not purpose: + raise ValueError("purpose must be non-empty") + body_rel = Path("bodies") / f"{evidence.response_body_sha256}.json" + body_path = self.root / body_rel + body_path.parent.mkdir(parents=True, exist_ok=True) + if body_path.exists() and hashlib.sha256(body_path.read_bytes()).hexdigest() != evidence.response_body_sha256: + raise RuntimeError("existing evidence body conflicts with response hash") + if not body_path.exists(): + body_path.write_bytes(evidence.body) + without = { + "body_path": body_rel.as_posix(), + "method": evidence.method, + "previous_hash": self.head_hash, + "purpose": str(purpose), + "request_body_sha256": evidence.request_body_sha256, + "request_completed_ts_ms": evidence.request_completed_ts_ms, + "request_started_ts_ms": evidence.request_started_ts_ms, + "response_body_sha256": evidence.response_body_sha256, + "status_code": evidence.status_code, + "url": evidence.url, + } + row = {**without, "record_hash": hashlib.sha256(_canonical_json(without)).hexdigest()} + self.root.mkdir(parents=True, exist_ok=True) + with self.manifest_path.open("a", encoding="utf-8", newline="\n") as handle: + handle.write(json.dumps(row, sort_keys=True, separators=(",", ":")) + "\n") + handle.flush() + self.records.append(row) + self.head_hash = row["record_hash"] + return dict(row) + + +def _json_list(value: object, name: str) -> list[str]: + if isinstance(value, str): + try: + value = json.loads(value) + except json.JSONDecodeError as exc: + raise ValueError(f"{name} is not valid JSON") from exc + if not isinstance(value, list) or not all(isinstance(item, str) for item in value): + raise ValueError(f"{name} must be a string list") + return list(value) + + +def parse_btc_five_minute_market( + payload: Mapping[str, Any], expected_open_epoch_seconds: int +) -> BtcFiveMinuteMarket: + opening = int(expected_open_epoch_seconds) + expected_slug = f"btc-updown-5m-{opening}" + if str(payload.get("slug")) != expected_slug: + raise ValueError("Gamma market slug does not match requested BTC window") + if payload.get("active") is not True or payload.get("closed") is not False: + raise ValueError("Gamma market is not active and open") + if payload.get("enableOrderBook") is not True: + raise ValueError("Gamma market is not CLOB-enabled") + condition_id = str(payload.get("conditionId") or "") + if not condition_id: + raise ValueError("Gamma market is missing conditionId") + outcomes = _json_list(payload.get("outcomes"), "outcomes") + token_ids = _json_list(payload.get("clobTokenIds"), "clobTokenIds") + if len(outcomes) != 2 or len(token_ids) != 2 or set(outcomes) != {"Up", "Down"}: + raise ValueError("Gamma market must contain exactly Up and Down outcomes") + mapping = dict(zip(outcomes, token_ids, strict=True)) + if not mapping["Up"] or not mapping["Down"] or mapping["Up"] == mapping["Down"]: + raise ValueError("Gamma market contains invalid token IDs") + return BtcFiveMinuteMarket( + condition_id=condition_id, + slug=expected_slug, + open_epoch_seconds=opening, + up_token_id=mapping["Up"], + down_token_id=mapping["Down"], + ) + + +def _parse_books(payload: object, requested_token_ids: Sequence[str]) -> dict[str, Mapping[str, Any]]: + if not isinstance(payload, list): + raise PublicResponseError("CLOB /books response must be a list") + requested = [str(token) for token in requested_token_ids] + found: dict[str, Mapping[str, Any]] = {} + for row in payload: + if not isinstance(row, Mapping): + raise PublicResponseError("CLOB /books row must be an object") + token = str(row.get("asset_id") or "") + if token in found: + raise PublicResponseError(f"duplicate CLOB book for token {token}") + found[token] = row + if set(found) != set(requested): + raise PublicResponseError("CLOB /books response token set mismatch") + return found + + +def next_collectable_open(now_ts_ms: int, cutoff_epoch_seconds: int) -> int: + now_ms = int(now_ts_ms) + opening = (now_ms // 1000 // 300) * 300 + if now_ms > opening * 1000 + 210_000: + opening += 300 + while opening <= int(cutoff_epoch_seconds): + opening += 300 + return opening + + +class SingleMarketCollector: + def __init__( + self, + *, + client: PublicHttpClient, + evidence_store: RawEvidenceStore, + ledger: CaptureBoundLifecycleLedger, + trading_policy: FrozenPolicy, + capture_policy: CapturePolicy, + sleep_until_ms: Callable[[int], None], + ) -> None: + self.client = client + self.evidence_store = evidence_store + self.ledger = ledger + self.trading_policy = trading_policy + self.capture_policy = capture_policy + self.sleep_until_ms = sleep_until_ms + + def capture(self, market_open_epoch_seconds: int) -> dict[str, Any]: + opening = int(market_open_epoch_seconds) + slug = f"btc-updown-5m-{opening}" + discovery = self.client.request_json("GET", f"{GAMMA_BASE}/markets/slug/{slug}") + discovery_manifest = self.evidence_store.append(discovery, purpose="market_discovery") + if not isinstance(discovery.payload, Mapping): + raise PublicResponseError("Gamma market response must be an object") + market = parse_btc_five_minute_market(discovery.payload, opening) + self.ledger.append( + condition_id=market.condition_id, + market_open_epoch_seconds=opening, + event_type="discovered", + observed_ts_ms=discovery.request_completed_ts_ms, + payload={ + "slug": market.slug, + "up_token_id": market.up_token_id, + "down_token_id": market.down_token_id, + "gamma_response_body_sha256": discovery.response_body_sha256, + "evidence_record_hash": discovery_manifest["record_hash"], + }, + ) + signal_target = opening * 1000 + self.capture_policy.signal_target_offset_ms + self.sleep_until_ms(signal_target) + signal_body = [{"token_id": market.up_token_id}, {"token_id": market.down_token_id}] + signal_http = self.client.request_json( + "POST", f"{CLOB_BASE}/books", json_body=signal_body + ) + self.evidence_store.append(signal_http, purpose="signal_books") + signal_books = _parse_books(signal_http.payload, [market.up_token_id, market.down_token_id]) + signal = evaluate_signal_capture( + self.trading_policy, + self.capture_policy, + condition_id=market.condition_id, + market_open_epoch_seconds=opening, + up_token_id=market.up_token_id, + down_token_id=market.down_token_id, + up_book=signal_books[market.up_token_id], + down_book=signal_books[market.down_token_id], + request_started_ts_ms=signal_http.request_started_ts_ms, + request_completed_ts_ms=signal_http.request_completed_ts_ms, + response_payload_sha256=signal_http.response_body_sha256, + ) + self.ledger.append( + condition_id=market.condition_id, + market_open_epoch_seconds=opening, + event_type="signal", + observed_ts_ms=signal["observed_ts_ms"], + payload=signal, + ) + if signal["decision"] != "signal": + return signal + arrival_target = opening * 1000 + self.capture_policy.arrival_target_offset_ms + self.sleep_until_ms(arrival_target) + selected_token = str(signal["selected_token_id"]) + arrival_body = [{"token_id": selected_token}] + arrival_http = self.client.request_json( + "POST", f"{CLOB_BASE}/books", json_body=arrival_body + ) + self.evidence_store.append(arrival_http, purpose="arrival_book") + arrival_books = _parse_books(arrival_http.payload, [selected_token]) + arrival = evaluate_arrival_capture( + self.trading_policy, + self.capture_policy, + signal, + arrival_books[selected_token], + request_started_ts_ms=arrival_http.request_started_ts_ms, + request_completed_ts_ms=arrival_http.request_completed_ts_ms, + response_payload_sha256=arrival_http.response_body_sha256, + ) + self.ledger.append( + condition_id=market.condition_id, + market_open_epoch_seconds=opening, + event_type="arrival", + observed_ts_ms=arrival["observed_ts_ms"], + payload=arrival, + ) + return arrival diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector/resolution.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector/resolution.py new file mode 100644 index 00000000..9e5ee980 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/collector/resolution.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +import json +from decimal import Decimal, InvalidOperation +from typing import Any, Mapping + +from capture_policy import CaptureBoundLifecycleLedger, CapturePolicy +from late_favorite_v3 import FrozenPolicy, settle_fill +from public_collector import ( + GAMMA_BASE, + PublicHttpClient, + PublicResponseError, + RawEvidenceStore, +) + + +def _json_string_list(value: object, name: str) -> list[str]: + if isinstance(value, str): + try: + value = json.loads(value) + except json.JSONDecodeError as exc: + raise ValueError(f"{name} is not valid JSON") from exc + if not isinstance(value, list) or not all(isinstance(item, str) for item in value): + raise ValueError(f"{name} must be a string list") + return list(value) + + +def parse_terminal_outcome( + payload: Mapping[str, Any], + expected_open_epoch_seconds: int, + expected_condition_id: str, +) -> str: + opening = int(expected_open_epoch_seconds) + if str(payload.get("slug")) != f"btc-updown-5m-{opening}": + raise ValueError("terminal Gamma slug mismatch") + if str(payload.get("conditionId") or "") != str(expected_condition_id): + raise ValueError("terminal Gamma condition mismatch") + if payload.get("closed") is not True: + raise ValueError("Gamma market is not closed") + outcomes = _json_string_list(payload.get("outcomes"), "outcomes") + raw_prices = payload.get("outcomePrices") + if isinstance(raw_prices, str): + try: + raw_prices = json.loads(raw_prices) + except json.JSONDecodeError as exc: + raise ValueError("outcomePrices is not valid JSON") from exc + if not isinstance(raw_prices, list) or len(raw_prices) != len(outcomes): + raise ValueError("outcomePrices must align with outcomes") + try: + prices = [Decimal(str(value)) for value in raw_prices] + except (InvalidOperation, ValueError) as exc: + raise ValueError("outcomePrices contains a non-decimal value") from exc + if set(outcomes) != {"Up", "Down"} or len(outcomes) != 2: + raise ValueError("terminal market must contain Up and Down") + winners = [outcome for outcome, price in zip(outcomes, prices, strict=True) if price == 1] + losers = [outcome for outcome, price in zip(outcomes, prices, strict=True) if price == 0] + if len(winners) != 1 or len(losers) != 1: + raise ValueError("outcomePrices are not terminal binary prices") + return winners[0] + + +class SingleMarketResolver: + def __init__( + self, + *, + client: PublicHttpClient, + evidence_store: RawEvidenceStore, + ledger: CaptureBoundLifecycleLedger, + trading_policy: FrozenPolicy, + capture_policy: CapturePolicy, + ) -> None: + self.client = client + self.evidence_store = evidence_store + self.ledger = ledger + self.trading_policy = trading_policy + self.capture_policy = capture_policy + + def _event_payload(self, condition_id: str, event_type: str) -> Mapping[str, Any] | None: + for record in reversed(self.ledger.records): + if record["condition_id"] == condition_id and record["event_type"] == event_type: + payload = record["payload"] + if not isinstance(payload, Mapping): + raise RuntimeError("lifecycle payload is not an object") + return payload + return None + + def _opening(self, condition_id: str) -> int: + for record in self.ledger.records: + if record["condition_id"] == condition_id: + return int(record["market_open_epoch_seconds"]) + raise KeyError(f"unknown condition_id: {condition_id}") + + def resolve(self, condition_id: str) -> dict[str, Any]: + condition = str(condition_id) + opening = self._opening(condition) + discovered = self._event_payload(condition, "discovered") + signal = self._event_payload(condition, "signal") + arrival = self._event_payload(condition, "arrival") + if discovered is None or signal is None: + raise RuntimeError("resolution requires discovered and signal lifecycle events") + slug = str(discovered.get("slug") or f"btc-updown-5m-{opening}") + cache_bust = len(self.evidence_store.records) + evidence = self.client.request_json( + "GET", f"{GAMMA_BASE}/markets/slug/{slug}?cache_bust={cache_bust}" + ) + manifest = self.evidence_store.append(evidence, purpose="official_resolution") + if not isinstance(evidence.payload, Mapping): + raise PublicResponseError("Gamma terminal response must be an object") + outcome = parse_terminal_outcome(evidence.payload, opening, condition) + if arrival is not None and arrival.get("decision") == "hypothetical_fok_fill": + resolution = settle_fill( + arrival, + official_outcome=outcome, + resolution_payload_sha256=evidence.response_body_sha256, + ) + resolution["decision"] = "resolved_fill" + else: + resolution = { + "event_type": "resolution", + "condition_id": condition, + "market_open_epoch_seconds": opening, + "policy_sha256": self.trading_policy.policy_sha256, + "source_sha256": self.trading_policy.source_sha256, + "capture_policy_sha256": self.capture_policy.capture_policy_sha256, + "decision": "resolved_no_fill", + "official_outcome": outcome, + "official_won": None, + "resolution_payload_sha256": evidence.response_body_sha256, + "pnl_per_share": None, + "pnl_total": None, + } + resolution["gamma_evidence_record_hash"] = manifest["record_hash"] + self.ledger.append( + condition_id=condition, + market_open_epoch_seconds=opening, + event_type="resolution", + observed_ts_ms=evidence.request_completed_ts_ms, + payload=resolution, + ) + return resolution diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v1/BLOCK_SUMMARY.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v1/BLOCK_SUMMARY.json new file mode 100644 index 00000000..4916e62c --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v1/BLOCK_SUMMARY.json @@ -0,0 +1,45 @@ +{ + "authenticated_requests": 0, + "capture_decision_counts": { + "no_fill_ask_above_limit": 1, + "no_signal_below_threshold": 2 + }, + "command": "collect-block", + "condition_ids": [ + "0x5452c4c60c9d3ed96739f7a46a58533244f2221ec504c352a808c7e3e90b379b", + "0x7bd9fe5bfea03a72553b721d0824512a3e9dee6a0fec44992135d5cf08b3e352", + "0x1dcecaa2f4ee994fd4ab1b9f309bb18a56f24a5f9c84b82c290a7489d9cfd27d" + ], + "conditions_resolved": 3, + "credentials_used": 0, + "first_market_open_epoch_seconds": 1785101400, + "hypothetical_fok_fills": 0, + "last_market_open_epoch_seconds": 1785102000, + "last_resolution_errors": {}, + "lifecycle_head_sha256": "1f6f21cf25c1acd11ea5b06e4bd3dec617d69d84c8b25fb5471be22fc9430c11", + "lifecycle_records": 10, + "live_submission": "physically_absent", + "market_open_epoch_seconds": [ + 1785101400, + 1785101700, + 1785102000 + ], + "markets_collected": 3, + "markets_requested": 3, + "order_submissions": 0, + "policy_changes": 0, + "prospective_pnl_total": "0", + "raw_evidence_head_sha256": "9b43483f4a310d1ad84a3b6f5a029dc772f2181cb06dce08db476aab00dc2cf0", + "raw_evidence_records": 44, + "resolution_attempts_by_condition": { + "0x1dcecaa2f4ee994fd4ab1b9f309bb18a56f24a5f9c84b82c290a7489d9cfd27d": 31, + "0x5452c4c60c9d3ed96739f7a46a58533244f2221ec504c352a808c7e3e90b379b": 1, + "0x7bd9fe5bfea03a72553b721d0824512a3e9dee6a0fec44992135d5cf08b3e352": 5 + }, + "resolution_decision_counts": { + "resolved_no_fill": 3 + }, + "source_sha256": "81a13d2aeb3df3c2088b8aa99272b10e217a392eabc839a01a8f2b71f570ecbb", + "status": "complete", + "unresolved_condition_ids": [] +} diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v1/CHECKPOINT.md b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v1/CHECKPOINT.md new file mode 100644 index 00000000..a59c0ec9 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v1/CHECKPOINT.md @@ -0,0 +1,115 @@ +# BTC-Only v3 Prospective Block v1 + +## Classification + +**Valid prospective block evidence; not strategy admission.** + +This checkpoint covers three consecutive post-freeze BTC five-minute markets collected under the unchanged `late_favorite_btc_only_v3_prospective_shadow` candidate and capture policies. Every market was captured before any resolution request was made. All three conditions later received authoritative Gamma terminal outcomes. + +The block produced **zero hypothetical FOK fills** and **$0.00 prospective P&L**. It increases the cumulative prospective market count from 1 to 4, but does not satisfy the minimum market, fill, weekly-block, confidence-bound, concentration, or release gates. + +## Block identity + +- Windows: **2026-07-26 21:30–21:45 UTC** / **4:30–4:45 PM Jamaica** +- Consecutive market openings: `1785101400`, `1785101700`, `1785102000` +- GitHub Actions run: `30221188931` +- Workflow head SHA: `be327b99530d9a5b1142e0c9896ae01716d4b070` +- Artifact ID: `8637433570` +- Artifact ZIP SHA-256: `222d8261ea92dfdfcd3fbb9ccb12dbf6948d03f6bed1e7558f3d9e4bdc6a5580` +- Runtime source SHA-256: `81a13d2aeb3df3c2088b8aa99272b10e217a392eabc839a01a8f2b71f570ecbb` +- Candidate policy SHA-256: `82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba` +- Capture policy SHA-256: `888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc` + +## Market results + +| Jamaica window | Selected side | Signal ask | Arrival | Frozen decision | Official outcome | P&L | +|---|---:|---:|---:|---|---:|---:| +| 4:30–4:35 PM | Up | $0.66 | Not requested | `no_signal_below_threshold` | Up | $0.00 | +| 4:35–4:40 PM | Down | $0.76 | Not requested | `no_signal_below_threshold` | Down | $0.00 | +| 4:40–4:45 PM | Up | $0.95 | $0.98 best ask | `no_fill_ask_above_limit` | Down | $0.00 | + +The third market is decision-useful. Up qualified at a 95-cent signal ask, but one second later its best ask was 98 cents—above the frozen 95-cent FOK limit. The observer refused to chase the move. The market later resolved Down, so the no-fill rule avoided a losing position. + +## Capture timing + +### Market 1 — `btc-updown-5m-1785101400` + +- Condition: `0x5452c4c60c9d3ed96739f7a46a58533244f2221ec504c352a808c7e3e90b379b` +- Signal request start: exactly +210,000 ms +- Signal request duration: 323 ms +- Signal book timestamp: +210,256 ms +- Decision: below 85-cent threshold + +### Market 2 — `btc-updown-5m-1785101700` + +- Condition: `0x7bd9fe5bfea03a72553b721d0824512a3e9dee6a0fec44992135d5cf08b3e352` +- Signal request start: +210,001 ms +- Signal request duration: 152 ms +- Signal book timestamp: +210,083 ms +- Decision: below 85-cent threshold + +### Market 3 — `btc-updown-5m-1785102000` + +- Condition: `0x1dcecaa2f4ee994fd4ab1b9f309bb18a56f24a5f9c84b82c290a7489d9cfd27d` +- Signal request start: exactly +210,000 ms +- Signal request duration: 170 ms +- Signal book timestamp: +210,112 ms +- Arrival request start: exactly +211,000 ms +- Arrival request duration: 265 ms +- Arrival book timestamp: +211,206 ms +- Decision: arrival ask above frozen limit + +All request starts, durations, and book timestamps remained within the frozen capture-policy bounds. + +## Authoritative outcomes + +| Market | Resolution polls | Official outcome | Terminal prices | Gamma `updatedAt` | Terminal payload SHA-256 | +|---|---:|---:|---:|---|---| +| `1785101400` | 1 | Up | `[1, 0]` | `2026-07-26T21:37:05.248916Z` | `52737e9eb6279c33689c1044cf9074dad2dd98511b6fcc8e48439f5dea81e8ae` | +| `1785101700` | 5 | Down | `[0, 1]` | `2026-07-26T21:43:14.819851Z` | `a6fe3e627f6ac977e727d5fceb9fa47fd1b6ac9d2e1fc98a61aea9eccfa5aa86` | +| `1785102000` | 31 | Down | `[0, 1]` | `2026-07-26T21:46:24.754458Z` | `8693fbc94d74fbd218e18c3a8d65961efd2e997223509135b3b97395af6e89ca` | + +Resolution polling began only after all three captures completed. Every poll used the cache-busted public Gamma endpoint and was preserved in the raw-evidence chain. + +## Integrity verification + +- Internal `SHA256SUMS`: **passed** +- Deterministic TAR sidecar: **passed** +- ZIP/TAR directory round-trip: **byte-for-byte passed** +- Lifecycle records: **10** + - discovered: 3 + - signal: 3 + - arrival: 1 + - resolution: 3 +- Lifecycle chain: **passed** +- Lifecycle head: `1f6f21cf25c1acd11ea5b06e4bd3dec617d69d84c8b25fb5471be22fc9430c11` +- Raw evidence records: **44** + - market discovery: 3 + - signal books: 3 + - arrival books: 1 + - official resolution payloads: 37 +- Raw evidence chain and body hashes: **passed** +- Raw evidence head: `9b43483f4a310d1ad84a3b6f5a029dc772f2181cb06dce08db476aab00dc2cf0` + +## Admission ledger after this block + +- Prospective markets observed: **4 / 500 minimum** +- Official outcomes available: **4 / 4** +- Hypothetical FOK fills: **0 / 100 minimum** +- Prospective P&L: **$0.00** +- Complete untouched weekly blocks: **0 / 4 minimum** +- Credentials used: **0** +- Authenticated requests: **0** +- Order submissions: **0** +- Historical admission credit: **0** +- Live submission: **physically absent** + +The first one-shot observation and this block used different orchestration-source hashes, while the frozen candidate and capture policies remained identical. No weekly block receives completion credit yet. + +## Next valid work + +1. Continue capture-first prospective blocks without modifying candidate or capture thresholds. +2. Preserve every no-signal, no-fill, fill, resolution, and transport failure. +3. Accumulate at least 500 observed markets, 100 hypothetical fills, and four untouched weekly blocks. +4. Require zero unresolved states, acceptable concentration, measured execution evidence, and a positive lower confidence bound after fees and costs. +5. Keep real-money order submission physically absent. diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v1/MARKETS.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v1/MARKETS.json new file mode 100644 index 00000000..9a8a0406 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v1/MARKETS.json @@ -0,0 +1,80 @@ +[ + { + "arrival": null, + "condition_id": "0x5452c4c60c9d3ed96739f7a46a58533244f2221ec504c352a808c7e3e90b379b", + "decision": "no_signal_below_threshold", + "market_open_epoch_seconds": 1785101400, + "official_outcome": "Up", + "pnl_total": null, + "resolution_payload_sha256": "52737e9eb6279c33689c1044cf9074dad2dd98511b6fcc8e48439f5dea81e8ae", + "resolution_polls": 1, + "selected_side": "Up", + "signal": { + "book_timestamp_ms": 1785101610256, + "request_completed_ts_ms": 1785101610323, + "request_duration_ms": 323, + "request_started_ts_ms": 1785101610000, + "signal_ask": "0.66" + }, + "slug": "btc-updown-5m-1785101400", + "terminal_outcome_prices": [ + "1", + "0" + ], + "terminal_updated_at": "2026-07-26T21:37:05.248916Z" + }, + { + "arrival": null, + "condition_id": "0x7bd9fe5bfea03a72553b721d0824512a3e9dee6a0fec44992135d5cf08b3e352", + "decision": "no_signal_below_threshold", + "market_open_epoch_seconds": 1785101700, + "official_outcome": "Down", + "pnl_total": null, + "resolution_payload_sha256": "a6fe3e627f6ac977e727d5fceb9fa47fd1b6ac9d2e1fc98a61aea9eccfa5aa86", + "resolution_polls": 5, + "selected_side": "Down", + "signal": { + "book_timestamp_ms": 1785101910083, + "request_completed_ts_ms": 1785101910153, + "request_duration_ms": 152, + "request_started_ts_ms": 1785101910001, + "signal_ask": "0.76" + }, + "slug": "btc-updown-5m-1785101700", + "terminal_outcome_prices": [ + "0", + "1" + ], + "terminal_updated_at": "2026-07-26T21:43:14.819851Z" + }, + { + "arrival": { + "best_ask": "0.98", + "book_timestamp_ms": 1785102211206, + "request_completed_ts_ms": 1785102211265, + "request_duration_ms": 265, + "request_started_ts_ms": 1785102211000 + }, + "condition_id": "0x1dcecaa2f4ee994fd4ab1b9f309bb18a56f24a5f9c84b82c290a7489d9cfd27d", + "decision": "no_fill_ask_above_limit", + "market_open_epoch_seconds": 1785102000, + "official_outcome": "Down", + "pnl_total": null, + "resolution_payload_sha256": "8693fbc94d74fbd218e18c3a8d65961efd2e997223509135b3b97395af6e89ca", + "resolution_polls": 31, + "selected_side": "Up", + "signal": { + "book_timestamp_ms": 1785102210112, + "request_completed_ts_ms": 1785102210170, + "request_duration_ms": 170, + "request_started_ts_ms": 1785102210000, + "signal_ask": "0.95" + }, + "slug": "btc-updown-5m-1785102000", + "terminal_outcome_prices": [ + "0", + "1" + ], + "terminal_updated_at": "2026-07-26T21:46:24.754458Z" + } +] diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v1/VERIFICATION.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v1/VERIFICATION.json new file mode 100644 index 00000000..291f22da --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v1/VERIFICATION.json @@ -0,0 +1,47 @@ +{ + "artifact": { + "artifact_id": 8637433570, + "artifact_name": "btc-only-v3-prospective-block-v1", + "artifact_zip_sha256": "222d8261ea92dfdfcd3fbb9ccb12dbf6948d03f6bed1e7558f3d9e4bdc6a5580", + "deterministic_tar_round_trip": "passed", + "internal_sha256sums": "passed", + "tar_sidecar": "passed" + }, + "candidate_policy_sha256": "82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba", + "capture_policy_sha256": "888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc", + "integrity": { + "lifecycle_chain": "passed", + "lifecycle_event_counts": { + "arrival": 1, + "discovered": 3, + "resolution": 3, + "signal": 3 + }, + "lifecycle_head_sha256": "1f6f21cf25c1acd11ea5b06e4bd3dec617d69d84c8b25fb5471be22fc9430c11", + "lifecycle_records": 10, + "raw_body_hashes": "passed", + "raw_evidence_chain": "passed", + "raw_evidence_head_sha256": "9b43483f4a310d1ad84a3b6f5a029dc772f2181cb06dce08db476aab00dc2cf0", + "raw_evidence_purpose_counts": { + "arrival_book": 1, + "market_discovery": 3, + "official_resolution": 37, + "signal_books": 3 + }, + "raw_evidence_records": 44 + }, + "run": { + "github_actions_run_id": 30221188931, + "github_head_sha": "be327b99530d9a5b1142e0c9896ae01716d4b070", + "runtime_source_sha256": "81a13d2aeb3df3c2088b8aa99272b10e217a392eabc839a01a8f2b71f570ecbb" + }, + "safety": { + "authenticated_requests": 0, + "credentials_used": 0, + "historical_admission_credit": 0, + "live_submission": "physically_absent", + "order_submissions": 0, + "policy_changes": 0 + }, + "status": "valid_complete_prospective_block_not_admission" +} diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v2/BLOCK_SUMMARY.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v2/BLOCK_SUMMARY.json new file mode 100644 index 00000000..8b06c017 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v2/BLOCK_SUMMARY.json @@ -0,0 +1,50 @@ +{ + "authenticated_requests": 0, + "candidate_policy_sha256": "82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba", + "capture_decision_counts": { + "hypothetical_fok_fill": 1, + "no_signal_below_threshold": 2 + }, + "capture_policy_sha256": "888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc", + "command": "collect-block", + "condition_ids": [ + "0x32d793352dbae3fd435a200c1cbcb9cea3dfb6c03030c891ae21e562ee7db11b", + "0x24846fbe7d9babc0f24d8a470322b84e0bef0f689b4a1f4e69e72ff2c2fe3233", + "0xc5a4c25812fc718b99cb34694ae389abeacf4d03d158720be4a8a53358e0cef2" + ], + "conditions_resolved": 3, + "credentials_used": 0, + "first_market_open_epoch_seconds": 1785111000, + "historical_admission_credit": 0, + "hypothetical_fok_fills": 1, + "last_market_open_epoch_seconds": 1785111600, + "last_resolution_errors": {}, + "lifecycle_head_sha256": "a04aac52de49a03fca237b888cbb284788a6dd5a53560147fc6b561489c9c87b", + "lifecycle_records": 10, + "live_submission": "physically_absent", + "market_open_epoch_seconds": [ + 1785111000, + 1785111300, + 1785111600 + ], + "markets_collected": 3, + "markets_requested": 3, + "order_submissions": 0, + "policy_changes": 0, + "prospective_pnl_total": "0.233375", + "raw_evidence_head_sha256": "b44fc443b8c80537ae9f40243034194d45633e34f3d46694f0b3a895b9928606", + "raw_evidence_records": 44, + "resolution_attempts_by_condition": { + "0x24846fbe7d9babc0f24d8a470322b84e0bef0f689b4a1f4e69e72ff2c2fe3233": 7, + "0x32d793352dbae3fd435a200c1cbcb9cea3dfb6c03030c891ae21e562ee7db11b": 1, + "0xc5a4c25812fc718b99cb34694ae389abeacf4d03d158720be4a8a53358e0cef2": 29 + }, + "resolution_decision_counts": { + "resolved_fill": 1, + "resolved_no_fill": 2 + }, + "runtime_source_sha256": "81a13d2aeb3df3c2088b8aa99272b10e217a392eabc839a01a8f2b71f570ecbb", + "source_sha256": "81a13d2aeb3df3c2088b8aa99272b10e217a392eabc839a01a8f2b71f570ecbb", + "status": "complete", + "unresolved_condition_ids": [] +} diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v2/CHECKPOINT.md b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v2/CHECKPOINT.md new file mode 100644 index 00000000..07475e0b --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v2/CHECKPOINT.md @@ -0,0 +1,130 @@ +# BTC-Only v3 Prospective Block v2 + +## Classification + +**Valid prospective block evidence; not strategy admission.** + +This checkpoint covers three consecutive post-freeze BTC five-minute markets collected under the unchanged `late_favorite_btc_only_v3_prospective_shadow` candidate and capture policies. All three markets were captured before the runner made any resolution request. Every condition later received an authoritative Gamma terminal outcome. + +The block produced the first genuine prospective hypothetical FOK fill: **one winning five-share Down fill and +$0.233375 P&L**. This is a useful observation, but one fill is nowhere near sufficient for admission. + +## Block identity + +- Windows: **2026-07-27 00:10–00:25 UTC** / **2026-07-26 7:10–7:25 PM Jamaica** +- Consecutive openings: `1785111000`, `1785111300`, `1785111600` +- GitHub Actions run: `30226869672` +- Workflow head SHA: `afc14b93c69bed593f4d51ce92fcbdb900d4a145` +- Artifact ID: `8638983600` +- Artifact ZIP SHA-256: `69ee1bb111f8aec0b52f982f305136a5e835ca2b2e50f4b66a129d3370b5ac58` +- Deterministic TAR SHA-256: `4fc0e0b68e7261899e133e1ba9af05939a72303d26609f73391967b5251c73cc` +- Runtime source SHA-256: `81a13d2aeb3df3c2088b8aa99272b10e217a392eabc839a01a8f2b71f570ecbb` +- Candidate policy SHA-256: `82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba` +- Capture policy SHA-256: `888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc` + +## Market results + +| Jamaica window | Selected side | Signal ask | Arrival | Frozen decision | Official outcome | P&L | +|---|---:|---:|---:|---|---:|---:| +| 7:10–7:15 PM | Up | $0.64 | Not requested | `no_signal_below_threshold` | Up | $0.00 | +| 7:15–7:20 PM | Down | $0.95 | $0.95; five shares | `hypothetical_fok_fill` | Down | **+$0.233375** | +| 7:20–7:25 PM | Up | $0.66 | Not requested | `no_signal_below_threshold` | Up | $0.00 | + +## First prospective fill + +The middle market, `btc-updown-5m-1785111300`, qualified with Down as the favorite at a 95-cent executable ask. + +One second later: + +- arrival best ask: **$0.95**; +- displayed executable size at or below the frozen limit: at least **5 shares**; +- execution: **5 shares at $0.95**; +- execution VWAP: **$0.95**; +- taker fee/share: **$0.003325**; +- all-in cost/share: **$0.953325**. + +The market officially resolved Down. Therefore: + +- official win: **yes**; +- P&L/share: **$0.046675**; +- P&L at five shares: **+$0.233375**. + +No threshold or execution rule was changed before or during this market. + +## Capture timing + +### Market 1 — `btc-updown-5m-1785111000` + +- Signal request start: +210,001 ms +- Signal request duration: 161 ms +- Signal book timestamps: +210,102 ms +- Decision: below the frozen 85-cent threshold + +### Market 2 — `btc-updown-5m-1785111300` + +- Signal request start: exactly +210,000 ms +- Signal request duration: 122 ms +- Signal book timestamps: +210,070 ms and +210,067 ms +- Arrival request start: exactly +211,000 ms +- Arrival request duration: 143 ms +- Arrival book timestamp: +211,095 ms +- Decision: hypothetical FOK fill + +### Market 3 — `btc-updown-5m-1785111600` + +- Signal request start: exactly +210,000 ms +- Signal request duration: 129 ms +- Signal book timestamps: +210,080 ms +- Decision: below the frozen 85-cent threshold + +Every request start, request duration, and book timestamp remained within the frozen capture-policy bounds. + +## Authoritative outcomes + +| Market opening | Resolution polls | Official outcome | Terminal prices | Gamma `updatedAt` | Terminal payload SHA-256 | +|---:|---:|---:|---|---|---| +| `1785111000` | 1 | Up | `[1, 0]` | `2026-07-27T00:21:14.688273Z` | `c504beb6519ee8965f84c8a3122c701659672e7fa0ce0882689681325d77130b` | +| `1785111300` | 7 | Down | `[0, 1]` | `2026-07-27T00:23:15.362879Z` | `2863a051c35dd91192df4fc4088313937b328949e2bd0ac909597f2fe722d482` | +| `1785111600` | 29 | Up | `[1, 0]` | `2026-07-27T00:28:59.884201Z` | `a2ad563e7d0f24894088ca5c14206739b8f894041591462ff085b54136ff4ece` | + +Resolution began only after all three capture windows completed. Every cache-busted public Gamma payload was preserved in the raw-evidence chain. + +## Integrity verification + +- Internal `SHA256SUMS`: **passed** +- Deterministic TAR sidecar: **passed** +- ZIP/TAR directory round-trip: **byte-for-byte passed** +- Lifecycle records: **10** + - discovered: 3 + - signal: 3 + - arrival: 1 + - resolution: 3 +- Lifecycle chain: **passed** +- Lifecycle head: `a04aac52de49a03fca237b888cbb284788a6dd5a53560147fc6b561489c9c87b` +- Raw evidence records: **44** +- Raw evidence chain and body hashes: **passed** +- Raw evidence head: `b44fc443b8c80537ae9f40243034194d45633e34f3d46694f0b3a895b9928606` +- Unresolved conditions: **0** +- Credentials used: **0** +- Authenticated requests: **0** +- Order submissions: **0** +- Historical admission credit: **0** +- Live submission: **physically absent** + +## Cumulative admission ledger + +- Prospective markets observed: **7 / 500 minimum** +- Official outcomes available: **7 / 7** +- Hypothetical FOK fills: **1 / 100 minimum** +- Prospective P&L: **+$0.233375** +- Unresolved states: **0** +- Complete untouched weekly blocks: **0 / 4 minimum** + +The positive result is a single observation. It cannot support a confidence bound, concentration assessment, weekly-block pass, or strategy admission. + +## Next valid work + +1. Continue bounded capture-first blocks without modifying candidate or capture thresholds. +2. Preserve every no-signal, no-fill, fill, resolution, and transport-failure record. +3. Accumulate at least 500 observed markets, 100 hypothetical fills, and four untouched weekly blocks. +4. Require zero unresolved states, measured execution evidence, acceptable concentration, and a positive lower confidence bound after fees and costs. +5. Keep real-money order submission physically absent. diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v2/MARKETS.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v2/MARKETS.json new file mode 100644 index 00000000..470cfca8 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v2/MARKETS.json @@ -0,0 +1,117 @@ +[ + { + "arrival": null, + "condition_id": "0x32d793352dbae3fd435a200c1cbcb9cea3dfb6c03030c891ae21e562ee7db11b", + "decision": "no_signal_below_threshold", + "jamaica_window": "2026-07-26 07:10–07:15 PM Jamaica", + "market_open_epoch_seconds": 1785111000, + "official_outcome": "Up", + "official_won": null, + "pnl_per_share": null, + "pnl_total": null, + "resolution_payload_sha256": "c504beb6519ee8965f84c8a3122c701659672e7fa0ce0882689681325d77130b", + "resolution_polls": 1, + "selected_side": "Up", + "signal": { + "book_timestamps_ms": [ + 1785111210102, + 1785111210102 + ], + "request_completed_ts_ms": 1785111210162, + "request_duration_ms": 161, + "request_started_ts_ms": 1785111210001, + "target_ts_ms": 1785111210000 + }, + "signal_ask": "0.64", + "slug": "btc-updown-5m-1785111000", + "terminal_outcome_prices": [ + "1", + "0" + ], + "terminal_updated_at": "2026-07-27T00:21:14.688273Z", + "utc_window": "2026-07-27 00:10–00:15 UTC" + }, + { + "arrival": { + "all_in_cost_per_share": "0.953325", + "best_ask": "0.95", + "book_timestamps_ms": [ + 1785111511095 + ], + "execution_levels": [ + { + "price": "0.95", + "size": "5" + } + ], + "execution_vwap": "0.95", + "fee_per_share": "0.003325", + "filled_shares": "5", + "request_completed_ts_ms": 1785111511143, + "request_duration_ms": 143, + "request_started_ts_ms": 1785111511000, + "target_ts_ms": 1785111511000 + }, + "condition_id": "0x24846fbe7d9babc0f24d8a470322b84e0bef0f689b4a1f4e69e72ff2c2fe3233", + "decision": "hypothetical_fok_fill", + "jamaica_window": "2026-07-26 07:15–07:20 PM Jamaica", + "market_open_epoch_seconds": 1785111300, + "official_outcome": "Down", + "official_won": true, + "pnl_per_share": "0.046675", + "pnl_total": "0.233375", + "resolution_payload_sha256": "2863a051c35dd91192df4fc4088313937b328949e2bd0ac909597f2fe722d482", + "resolution_polls": 7, + "selected_side": "Down", + "signal": { + "book_timestamps_ms": [ + 1785111510070, + 1785111510067 + ], + "request_completed_ts_ms": 1785111510122, + "request_duration_ms": 122, + "request_started_ts_ms": 1785111510000, + "target_ts_ms": 1785111510000 + }, + "signal_ask": "0.95", + "slug": "btc-updown-5m-1785111300", + "terminal_outcome_prices": [ + "0", + "1" + ], + "terminal_updated_at": "2026-07-27T00:23:15.362879Z", + "utc_window": "2026-07-27 00:15–00:20 UTC" + }, + { + "arrival": null, + "condition_id": "0xc5a4c25812fc718b99cb34694ae389abeacf4d03d158720be4a8a53358e0cef2", + "decision": "no_signal_below_threshold", + "jamaica_window": "2026-07-26 07:20–07:25 PM Jamaica", + "market_open_epoch_seconds": 1785111600, + "official_outcome": "Up", + "official_won": null, + "pnl_per_share": null, + "pnl_total": null, + "resolution_payload_sha256": "a2ad563e7d0f24894088ca5c14206739b8f894041591462ff085b54136ff4ece", + "resolution_polls": 29, + "selected_side": "Up", + "signal": { + "book_timestamps_ms": [ + 1785111810080, + 1785111810080 + ], + "request_completed_ts_ms": 1785111810129, + "request_duration_ms": 129, + "request_started_ts_ms": 1785111810000, + "target_ts_ms": 1785111810000 + }, + "signal_ask": "0.66", + "slug": "btc-updown-5m-1785111600", + "terminal_outcome_prices": [ + "1", + "0" + ], + "terminal_updated_at": "2026-07-27T00:28:59.884201Z", + "utc_window": "2026-07-27 00:20–00:25 UTC" + } +] diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v2/VERIFICATION.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v2/VERIFICATION.json new file mode 100644 index 00000000..e0daf68b --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v2/VERIFICATION.json @@ -0,0 +1,31 @@ +{ + "artifact_id": 8638983600, + "artifact_zip_sha256": "69ee1bb111f8aec0b52f982f305136a5e835ca2b2e50f4b66a129d3370b5ac58", + "authenticated_requests": 0, + "candidate_policy_sha256": "82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba", + "capture_policy_sha256": "888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc", + "credentials_used": 0, + "deterministic_tar_sha256": "4fc0e0b68e7261899e133e1ba9af05939a72303d26609f73391967b5251c73cc", + "historical_admission_credit": 0, + "hypothetical_fok_fills": 1, + "internal_sha256sums_verified": true, + "lifecycle_chain_verified": true, + "lifecycle_head_sha256": "a04aac52de49a03fca237b888cbb284788a6dd5a53560147fc6b561489c9c87b", + "lifecycle_records": 10, + "live_submission": "physically_absent", + "markets": 3, + "official_outcomes": { + "denominator": 3, + "numerator": 3 + }, + "order_submissions": 0, + "prospective_pnl_total": "0.233375", + "raw_evidence_chain_and_bodies_verified": true, + "raw_evidence_head_sha256": "b44fc443b8c80537ae9f40243034194d45633e34f3d46694f0b3a895b9928606", + "raw_evidence_records": 44, + "runtime_source_sha256": "81a13d2aeb3df3c2088b8aa99272b10e217a392eabc839a01a8f2b71f570ecbb", + "tar_round_trip_verified": true, + "unresolved_states": 0, + "workflow_head_sha": "afc14b93c69bed593f4d51ce92fcbdb900d4a145", + "workflow_run_id": 30226869672 +} diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v3/BLOCK_SUMMARY.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v3/BLOCK_SUMMARY.json new file mode 100644 index 00000000..ab4c9766 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v3/BLOCK_SUMMARY.json @@ -0,0 +1,50 @@ +{ + "authenticated_requests": 0, + "candidate_policy_sha256": "82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba", + "capture_decision_counts": { + "hypothetical_fok_fill": 1, + "no_signal_below_threshold": 2 + }, + "capture_policy_sha256": "888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc", + "command": "collect-block", + "condition_ids": [ + "0x9e511bc8afffe4491ac7d943976763e9edcc6acac5c7fdf2de6ddccf9729536d", + "0xe7ac80be371db56b3a8c1a682427f2cc3b887f4bdccb7b9b0f2f4933b4147f8e", + "0xf739b004a67e777b0ef3de33bfa417782e70c81595bd707ad7fcdf26a40eaaa8" + ], + "conditions_resolved": 3, + "credentials_used": 0, + "first_market_open_epoch_seconds": 1785112800, + "historical_admission_credit": 0, + "hypothetical_fok_fills": 1, + "last_market_open_epoch_seconds": 1785113400, + "last_resolution_errors": {}, + "lifecycle_head_sha256": "fa080692b88dbfea6233fb6d2dc0f510008c7ff3c10342aec0b476afdb2f7579", + "lifecycle_records": 10, + "live_submission": "physically_absent", + "market_open_epoch_seconds": [ + 1785112800, + 1785113100, + 1785113400 + ], + "markets_collected": 3, + "markets_requested": 3, + "order_submissions": 0, + "policy_changes": 0, + "prospective_pnl_total": "0.610415", + "raw_evidence_head_sha256": "5b24cf59091c974219c7d38abe776743b25fe445b41ecd00aa8812d47977ff26", + "raw_evidence_records": 44, + "resolution_attempts_by_condition": { + "0x9e511bc8afffe4491ac7d943976763e9edcc6acac5c7fdf2de6ddccf9729536d": 1, + "0xe7ac80be371db56b3a8c1a682427f2cc3b887f4bdccb7b9b0f2f4933b4147f8e": 1, + "0xf739b004a67e777b0ef3de33bfa417782e70c81595bd707ad7fcdf26a40eaaa8": 35 + }, + "resolution_decision_counts": { + "resolved_fill": 1, + "resolved_no_fill": 2 + }, + "runtime_source_sha256": "81a13d2aeb3df3c2088b8aa99272b10e217a392eabc839a01a8f2b71f570ecbb", + "source_sha256": "81a13d2aeb3df3c2088b8aa99272b10e217a392eabc839a01a8f2b71f570ecbb", + "status": "complete", + "unresolved_condition_ids": [] +} diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v3/CHECKPOINT.md b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v3/CHECKPOINT.md new file mode 100644 index 00000000..870b626b --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v3/CHECKPOINT.md @@ -0,0 +1,121 @@ +# BTC-Only v3 Prospective Block v3 + +## Classification + +**Valid prospective block evidence; not strategy admission.** + +This checkpoint covers three consecutive post-freeze BTC five-minute markets collected under the unchanged candidate and capture policies. The runner captured all three windows before sending any resolution request, then resolved every condition from authoritative cache-busted Gamma terminal responses. + +The block produced one winning hypothetical FOK fill: **five Down shares at $0.87, +$0.610415**. Cumulatively, the prospective track now has two winning fills, but two observations remain far below every admission threshold. + +## Block identity + +- Windows: **2026-07-27 00:40–00:55 UTC** / **2026-07-26 7:40–7:55 PM Jamaica** +- Consecutive openings: `1785112800`, `1785113100`, `1785113400` +- GitHub Actions run: `30228071397` +- Workflow head SHA: `e5463d3ecb2b150de857ac73bf31e08d3317cd2f` +- Artifact ID: `8639355447` +- Artifact ZIP SHA-256: `b30e9e10de839070ad1ae88f98c952ff2d8d4378ab828bed1533cff0e588cc0e` +- Deterministic TAR SHA-256: `5af345750399cde55f782140c8ba0427b74d39c113ce864e7a045c3f246cc6ee` +- Runtime source SHA-256: `81a13d2aeb3df3c2088b8aa99272b10e217a392eabc839a01a8f2b71f570ecbb` +- Candidate policy SHA-256: `82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba` +- Capture policy SHA-256: `888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc` + +## Market results + +| Jamaica window | Selected side | Signal ask | Arrival | Frozen decision | Official outcome | P&L | +|---|---:|---:|---:|---|---:|---:| +| 7:40–7:45 PM | Up | $0.64 | Not requested | `no_signal_below_threshold` | Up | $0.00 | +| 7:45–7:50 PM | Down | $0.60 | Not requested | `no_signal_below_threshold` | Up | $0.00 | +| 7:50–7:55 PM | Down | $0.88 | $0.87; five shares | `hypothetical_fok_fill` | Down | **+$0.610415** | + +## Filled-market economics + +The final market, `btc-updown-5m-1785113400`, selected Down at an 88-cent signal ask. One second later, the best ask improved to 87 cents—exactly one cent below the signal ask, which remains eligible under the frozen adverse-move rule. + +- signal ask: **$0.88** +- arrival best ask: **$0.87** +- execution: **5 shares at $0.87** +- execution VWAP: **$0.87** +- fee/share: **$0.007917** +- all-in cost/share: **$0.877917** +- official outcome: **Down** +- P&L/share: **$0.122083** +- five-share P&L: **+$0.610415** + +No policy, threshold, size, fee, or execution rule changed. + +## Capture timing + +### Market 1 — `btc-updown-5m-1785112800` + +- Signal request start: exactly +210,000 ms +- Signal request duration: 183 ms +- Signal book timestamps: +210,110 ms +- Decision: below the frozen 85-cent threshold + +### Market 2 — `btc-updown-5m-1785113100` + +- Signal request start: exactly +210,000 ms +- Signal request duration: 187 ms +- Signal book timestamps: +210,110 ms and +210,109 ms +- Decision: below the frozen 85-cent threshold + +### Market 3 — `btc-updown-5m-1785113400` + +- Signal request start: +210,001 ms +- Signal request duration: 198 ms +- Signal book timestamps: +210,122 ms +- Arrival request start: exactly +211,000 ms +- Arrival request duration: 163 ms +- Arrival book timestamp: +211,088 ms +- Decision: hypothetical FOK fill + +Every request and book timestamp remained inside the frozen capture-policy limits. + +## Authoritative outcomes + +| Market opening | Resolution polls | Official outcome | Terminal prices | Gamma `updatedAt` | Terminal payload SHA-256 | +|---:|---:|---:|---|---|---| +| `1785112800` | 1 | Up | `[1, 0]` | `2026-07-27T00:51:14.942925Z` | `51fd814498aae2f743d5b6354eba77461066c333042c8a6614b263df0a85c8aa` | +| `1785113100` | 1 | Up | `[1, 0]` | `2026-07-27T00:51:35.707758Z` | `dbdcd84b4eb322624a45ae22543ed369be91021958779f9a52ca16c45e2d3423` | +| `1785113400` | 35 | Down | `[0, 1]` | `2026-07-27T00:56:26.240184Z` | `c37484c919bf833195bad9c90ecf508eafeda0cfc7af7560f6aff21fb465992a` | + +## Integrity verification + +- Internal `SHA256SUMS`: **passed** +- Deterministic TAR sidecar: **passed** +- ZIP/TAR directory round-trip: **byte-for-byte passed** +- Lifecycle records: **10** + - discovered: 3 + - signal: 3 + - arrival: 1 + - resolution: 3 +- Lifecycle chain: **passed** +- Lifecycle head: `fa080692b88dbfea6233fb6d2dc0f510008c7ff3c10342aec0b476afdb2f7579` +- Raw evidence records: **44** +- Raw evidence chain and body hashes: **passed** +- Raw evidence head: `5b24cf59091c974219c7d38abe776743b25fe445b41ecd00aa8812d47977ff26` +- Unresolved conditions: **0** +- Credentials/authenticated requests/order submissions: **0 / 0 / 0** +- Historical admission credit: **0** +- Live submission: **physically absent** + +## Cumulative admission ledger + +- Prospective markets observed: **10 / 500 minimum** +- Official outcomes available: **10 / 10** +- Hypothetical FOK fills: **2 / 100 minimum** +- Prospective P&L: **+$0.843790** +- Unresolved states: **0** +- Complete untouched weekly blocks: **0 / 4 minimum** + +Both fills won, but a two-trade record has no meaningful statistical power and cannot satisfy confidence, concentration, weekly-block, or admission gates. + +## Next valid work + +1. Continue bounded capture-first blocks without modifying candidate or capture thresholds. +2. Preserve every no-signal, no-fill, fill, resolution, and transport-failure record. +3. Accumulate at least 500 markets, 100 hypothetical fills, and four untouched weekly blocks. +4. Require zero unresolved states, measured execution evidence, acceptable concentration, and a positive lower confidence bound after fees and costs. +5. Keep real-money order submission physically absent. diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v3/MARKETS.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v3/MARKETS.json new file mode 100644 index 00000000..73a1ec79 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v3/MARKETS.json @@ -0,0 +1,117 @@ +[ + { + "arrival": null, + "condition_id": "0x9e511bc8afffe4491ac7d943976763e9edcc6acac5c7fdf2de6ddccf9729536d", + "decision": "no_signal_below_threshold", + "jamaica_window": "2026-07-26 07:40–07:45 PM Jamaica", + "market_open_epoch_seconds": 1785112800, + "official_outcome": "Up", + "official_won": null, + "pnl_per_share": null, + "pnl_total": null, + "resolution_payload_sha256": "51fd814498aae2f743d5b6354eba77461066c333042c8a6614b263df0a85c8aa", + "resolution_polls": 1, + "selected_side": "Up", + "signal": { + "book_timestamps_ms": [ + 1785113010110, + 1785113010110 + ], + "request_completed_ts_ms": 1785113010183, + "request_duration_ms": 183, + "request_started_ts_ms": 1785113010000, + "target_ts_ms": 1785113010000 + }, + "signal_ask": "0.64", + "slug": "btc-updown-5m-1785112800", + "terminal_outcome_prices": [ + "1", + "0" + ], + "terminal_updated_at": "2026-07-27T00:51:14.942925Z", + "utc_window": "2026-07-27 00:40–00:45 UTC" + }, + { + "arrival": null, + "condition_id": "0xe7ac80be371db56b3a8c1a682427f2cc3b887f4bdccb7b9b0f2f4933b4147f8e", + "decision": "no_signal_below_threshold", + "jamaica_window": "2026-07-26 07:45–07:50 PM Jamaica", + "market_open_epoch_seconds": 1785113100, + "official_outcome": "Up", + "official_won": null, + "pnl_per_share": null, + "pnl_total": null, + "resolution_payload_sha256": "dbdcd84b4eb322624a45ae22543ed369be91021958779f9a52ca16c45e2d3423", + "resolution_polls": 1, + "selected_side": "Down", + "signal": { + "book_timestamps_ms": [ + 1785113310110, + 1785113310109 + ], + "request_completed_ts_ms": 1785113310187, + "request_duration_ms": 187, + "request_started_ts_ms": 1785113310000, + "target_ts_ms": 1785113310000 + }, + "signal_ask": "0.6", + "slug": "btc-updown-5m-1785113100", + "terminal_outcome_prices": [ + "1", + "0" + ], + "terminal_updated_at": "2026-07-27T00:51:35.707758Z", + "utc_window": "2026-07-27 00:45–00:50 UTC" + }, + { + "arrival": { + "all_in_cost_per_share": "0.877917", + "best_ask": "0.87", + "book_timestamps_ms": [ + 1785113611088 + ], + "execution_levels": [ + { + "price": "0.87", + "size": "5" + } + ], + "execution_vwap": "0.87", + "fee_per_share": "0.007917", + "filled_shares": "5", + "request_completed_ts_ms": 1785113611163, + "request_duration_ms": 163, + "request_started_ts_ms": 1785113611000, + "target_ts_ms": 1785113611000 + }, + "condition_id": "0xf739b004a67e777b0ef3de33bfa417782e70c81595bd707ad7fcdf26a40eaaa8", + "decision": "hypothetical_fok_fill", + "jamaica_window": "2026-07-26 07:50–07:55 PM Jamaica", + "market_open_epoch_seconds": 1785113400, + "official_outcome": "Down", + "official_won": true, + "pnl_per_share": "0.122083", + "pnl_total": "0.610415", + "resolution_payload_sha256": "c37484c919bf833195bad9c90ecf508eafeda0cfc7af7560f6aff21fb465992a", + "resolution_polls": 35, + "selected_side": "Down", + "signal": { + "book_timestamps_ms": [ + 1785113610122, + 1785113610122 + ], + "request_completed_ts_ms": 1785113610199, + "request_duration_ms": 198, + "request_started_ts_ms": 1785113610001, + "target_ts_ms": 1785113610000 + }, + "signal_ask": "0.88", + "slug": "btc-updown-5m-1785113400", + "terminal_outcome_prices": [ + "0", + "1" + ], + "terminal_updated_at": "2026-07-27T00:56:26.240184Z", + "utc_window": "2026-07-27 00:50–00:55 UTC" + } +] diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v3/VERIFICATION.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v3/VERIFICATION.json new file mode 100644 index 00000000..f9cc82cd --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-block-v3/VERIFICATION.json @@ -0,0 +1,31 @@ +{ + "artifact_id": 8639355447, + "artifact_zip_sha256": "b30e9e10de839070ad1ae88f98c952ff2d8d4378ab828bed1533cff0e588cc0e", + "authenticated_requests": 0, + "candidate_policy_sha256": "82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba", + "capture_policy_sha256": "888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc", + "credentials_used": 0, + "deterministic_tar_sha256": "5af345750399cde55f782140c8ba0427b74d39c113ce864e7a045c3f246cc6ee", + "historical_admission_credit": 0, + "hypothetical_fok_fills": 1, + "internal_sha256sums_verified": true, + "lifecycle_chain_verified": true, + "lifecycle_head_sha256": "fa080692b88dbfea6233fb6d2dc0f510008c7ff3c10342aec0b476afdb2f7579", + "lifecycle_records": 10, + "live_submission": "physically_absent", + "markets": 3, + "official_outcomes": { + "denominator": 3, + "numerator": 3 + }, + "order_submissions": 0, + "prospective_pnl_total": "0.610415", + "raw_evidence_chain_and_bodies_verified": true, + "raw_evidence_head_sha256": "5b24cf59091c974219c7d38abe776743b25fe445b41ecd00aa8812d47977ff26", + "raw_evidence_records": 44, + "runtime_source_sha256": "81a13d2aeb3df3c2088b8aa99272b10e217a392eabc839a01a8f2b71f570ecbb", + "tar_round_trip_verified": true, + "unresolved_states": 0, + "workflow_head_sha": "e5463d3ecb2b150de857ac73bf31e08d3317cd2f", + "workflow_run_id": 30228071397 +} diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-run-v1/CAPTURE_SUMMARY.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-run-v1/CAPTURE_SUMMARY.json new file mode 100644 index 00000000..2ffbd7ac --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-run-v1/CAPTURE_SUMMARY.json @@ -0,0 +1,37 @@ +{ + "arrival": { + "arrival_best_ask": "0.99", + "arrival_book_sha256": "b58474a8562bb06af33e7dce84be1b27af50c482ac945c2a0226f0c61836996d", + "book_timestamp_ms": 1785099811069, + "decision": "no_fill_ask_above_limit", + "hypothetical_fok_fill": false, + "request_completed_ts_ms": 1785099811152, + "request_duration_ms": 152, + "request_started_ts_ms": 1785099811000, + "signal_ask": "0.98" + }, + "condition_id": "0x4c02c25039452fa49c84a0a8d10f59390ac9b26b8b11f52a58ecfcec7962f13c", + "market_open_epoch_seconds": 1785099600, + "resolution": { + "decision": "resolved_no_fill", + "official_outcome": "Up", + "official_won": null, + "pnl_per_share": null, + "pnl_total": null, + "resolution_payload_sha256": "aa55f228e06bcdbcfb71679b5d417e7571bd8e97bd35ef78fbaa3498e984df14" + }, + "signal": { + "book_timestamps_ms": [ + 1785099810080, + 1785099810080 + ], + "decision": "signal", + "down_book_sha256": "02fe0740029381cbab1dcb7c6c697b7790f3a8e451ba5fb017adee0bce6438b2", + "request_completed_ts_ms": 1785099810155, + "request_duration_ms": 155, + "request_started_ts_ms": 1785099810000, + "selected_side": "Up", + "signal_ask": "0.98", + "up_book_sha256": "da0b50980004035a111c91241adf5cd0381eec5a5345a561e86396039c5b3f42" + } +} diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-run-v1/CHECKPOINT.md b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-run-v1/CHECKPOINT.md new file mode 100644 index 00000000..cd9c78c4 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-run-v1/CHECKPOINT.md @@ -0,0 +1,80 @@ +# BTC-Only v3 Prospective Run v1 + +## Classification + +**Valid prospective observation; not strategy admission.** + +This is the first complete post-freeze market collected by `late_favorite_btc_only_v3_prospective_shadow` using the frozen candidate and capture policies. It counts as one observed prospective market and zero hypothetical FOK fills. It does not satisfy any weekly, fill-count, confidence-bound, concentration, or release gate. + +## Market and decision + +- Window: **2026-07-26 21:00–21:05 UTC** / **4:00–4:05 PM Jamaica** +- Slug: `btc-updown-5m-1785099600` +- Condition ID: `0x4c02c25039452fa49c84a0a8d10f59390ac9b26b8b11f52a58ecfcec7962f13c` +- Frozen signal time: market open +210 seconds +- Frozen arrival time: market open +211 seconds +- Selected side: **Up** +- Signal ask: **$0.98** +- Arrival best ask: **$0.99** +- Decision: **`no_fill_ask_above_limit`** +- Official outcome: **Up** +- Hypothetical fill: **No** +- P&L: **Not applicable; no fill** + +The observer did not chase the one-cent adverse price move. Although Up ultimately won, the arrival ask exceeded the frozen 98-cent FOK limit, so no position and no P&L were recorded. + +## Timing and market-data evidence + +- Signal request: started exactly at target; completed in **155 ms**. +- Signal books: timestamped **80 ms** after target and **75 ms** before request completion. +- Arrival request: started exactly at target; completed in **152 ms**. +- Arrival book: timestamped **69 ms** after target and **83 ms** before request completion. +- Full Up/Down signal books and selected-token arrival book were preserved and hashed. + +## Authoritative resolution + +Gamma returned 20 non-terminal payloads followed by one terminal payload. Every poll used a unique deterministic cache-busting query. The terminal response reported: + +- `closed: true` +- outcomes: `["Up", "Down"]` +- terminal prices: `["1", "0"]` +- `updatedAt`: `2026-07-26T21:06:32.32566Z` + +A stale-CDN defect was discovered during the preceding discarded attempt: the unqualified Gamma slug URL could continue returning a pre-close body after terminalization. Resolution requests now append `?cache_bust=`. This changes only transport freshness; it does not alter the candidate, timing, entry, FOK, fee, or outcome rules. + +## Integrity verification + +- GitHub Actions run: `30220121542` +- Artifact ID: `8637040765` +- Artifact ZIP SHA-256: `8d00c88d6046806a2abc07f43da8b2f0ab08449890ee8a7f9e1f7071b92c4e21` +- Workflow head SHA: `257e3b5f1bfa4118a4375fbbfc4693152a296c55` +- Runtime source SHA-256: `ce477bf97589997899c91ad22af9025de521520f8ee7aa46a21b50a313958bd7` +- Lifecycle records: **4**, chain verified +- Lifecycle head: `a2c123f6ba2413d7dddee24ef20d75b096503e0d31a7b1051d8118cfd9911a1d` +- Raw evidence records: **24**, chain and bodies verified +- Raw evidence head: `baa3454eab1f1d8417a0b494a34a0a5ab87a7b7a1ab0e96e058400fafab8d6e0` +- Internal `SHA256SUMS`: passed +- Deterministic TAR sidecar: passed +- ZIP/TAR directory round-trip: byte-for-byte passed +- Credentials used: **0** +- Authenticated Polymarket requests: **0** +- Order submissions: **0** +- Historical admission credit: **0** +- Live submission: **physically absent** + +## Admission ledger after this run + +- Prospective markets observed: **1 / 500 minimum** +- Hypothetical FOK fills: **0 / 100 minimum** +- Official outcome coverage: **1 / 1** +- Complete untouched weekly blocks: **0 / 4 minimum** +- Realized prospective P&L: **$0.00**, because no fill occurred +- Candidate status: **prospective shadow only** + +## Next valid work + +1. Merge the cache-busting resolver correction after the 25-test clean-room checkpoint passes. +2. Continue collecting every eligible BTC five-minute market, including no-signal and no-fill decisions. +3. Preserve authoritative outcomes and complete raw/lifecycle chains for every market. +4. Do not inspect or tune thresholds during the prospective block. +5. Keep live order submission physically absent. diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-run-v1/PROSPECTIVE_RUN.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-run-v1/PROSPECTIVE_RUN.json new file mode 100644 index 00000000..2e1ca3e9 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-run-v1/PROSPECTIVE_RUN.json @@ -0,0 +1,43 @@ +{ + "authenticated_requests": 0, + "candidate": "late_favorite_btc_only_v3_prospective_shadow", + "candidate_policy_sha256": "82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba", + "capture_decision": "no_fill_ask_above_limit", + "capture_policy_sha256": "888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc", + "combined_source_sha256": "ce477bf97589997899c91ad22af9025de521520f8ee7aa46a21b50a313958bd7", + "condition_id": "0x4c02c25039452fa49c84a0a8d10f59390ac9b26b8b11f52a58ecfcec7962f13c", + "credentials_used": 0, + "github_sha": "257e3b5f1bfa4118a4375fbbfc4693152a296c55", + "historical_admission_credit": 0, + "lifecycle_event_types": [ + "discovered", + "signal", + "arrival", + "resolution" + ], + "lifecycle_head_sha256": "a2c123f6ba2413d7dddee24ef20d75b096503e0d31a7b1051d8118cfd9911a1d", + "lifecycle_records": 4, + "live_submission": "physically_absent", + "market_open_epoch_seconds": 1785099600, + "order_submissions": 0, + "raw_evidence_head_sha256": "baa3454eab1f1d8417a0b494a34a0a5ab87a7b7a1ab0e96e058400fafab8d6e0", + "raw_evidence_records": 24, + "resolution": { + "capture_policy_sha256": "888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc", + "condition_id": "0x4c02c25039452fa49c84a0a8d10f59390ac9b26b8b11f52a58ecfcec7962f13c", + "decision": "resolved_no_fill", + "event_type": "resolution", + "gamma_evidence_record_hash": "baa3454eab1f1d8417a0b494a34a0a5ab87a7b7a1ab0e96e058400fafab8d6e0", + "market_open_epoch_seconds": 1785099600, + "official_outcome": "Up", + "official_won": null, + "pnl_per_share": null, + "pnl_total": null, + "policy_sha256": "82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba", + "resolution_payload_sha256": "aa55f228e06bcdbcfb71679b5d417e7571bd8e97bd35ef78fbaa3498e984df14", + "source_sha256": "ce477bf97589997899c91ad22af9025de521520f8ee7aa46a21b50a313958bd7" + }, + "resolution_attempts": 21, + "resolution_status": "terminal", + "run_label": "first-post-cutoff-observation" +} diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-run-v1/VERIFICATION.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-run-v1/VERIFICATION.json new file mode 100644 index 00000000..18a13b40 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/prospective-run-v1/VERIFICATION.json @@ -0,0 +1,54 @@ +{ + "artifact": { + "artifact_id": 8637040765, + "artifact_name": "btc-only-v3-prospective-run-v1", + "artifact_zip_sha256": "8d00c88d6046806a2abc07f43da8b2f0ab08449890ee8a7f9e1f7071b92c4e21", + "deterministic_tar_round_trip": "passed", + "internal_sha256sums": "passed", + "tar_sidecar": "passed" + }, + "candidate_policy_sha256": "82b923b0d4034d801156b77a213db6084be719f27491478247e5354ea93e92ba", + "capture_policy_sha256": "888810ae61a0ab3e067c68850faeba5f2709a57bb69ef0c2ad708e128b311edc", + "integrity": { + "lifecycle_chain": "passed", + "lifecycle_event_types": [ + "discovered", + "signal", + "arrival", + "resolution" + ], + "lifecycle_head_sha256": "a2c123f6ba2413d7dddee24ef20d75b096503e0d31a7b1051d8118cfd9911a1d", + "lifecycle_records": 4, + "raw_body_hashes": "passed", + "raw_evidence_chain": "passed", + "raw_evidence_head_sha256": "baa3454eab1f1d8417a0b494a34a0a5ab87a7b7a1ab0e96e058400fafab8d6e0", + "raw_evidence_records": 24 + }, + "resolution_poll_audit": { + "cache_bust_first": 3, + "cache_bust_last": 23, + "cache_bust_sequence_complete": true, + "nonterminal_polls": 20, + "resolution_polls": 21, + "terminal_closed": true, + "terminal_outcome_prices": [ + "1", + "0" + ], + "terminal_payload_sha256": "aa55f228e06bcdbcfb71679b5d417e7571bd8e97bd35ef78fbaa3498e984df14", + "terminal_updated_at": "2026-07-26T21:06:32.32566Z" + }, + "run": { + "github_actions_run_id": 30220121542, + "github_head_sha": "257e3b5f1bfa4118a4375fbbfc4693152a296c55", + "runtime_source_sha256": "ce477bf97589997899c91ad22af9025de521520f8ee7aa46a21b50a313958bd7" + }, + "safety": { + "authenticated_requests": 0, + "credentials_used": 0, + "historical_admission_credit": 0, + "live_submission": "physically_absent", + "order_submissions": 0 + }, + "status": "valid_prospective_observation_not_admission" +} diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/CHECKPOINT.md b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/CHECKPOINT.md new file mode 100644 index 00000000..3443a79d --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/CHECKPOINT.md @@ -0,0 +1,53 @@ +# BTC-Only v3 One-Shot Runner Checkpoint + +## Status + +The credential-free one-shot runner for `late_favorite_btc_only_v3_prospective_shadow` is implemented, clean-room verified, independently artifact-checked, and merged into the durable live-readiness branch. + +This checkpoint adds orchestration only. It does not change the frozen candidate, capture policy, execution model, or admission gates. + +## Commands + +- `collect-next`: choose the next eligible post-cutoff BTC five-minute window, persist discovery/signal/arrival evidence, and write a non-overwriting summary. +- `resolve --condition-id`: fetch authoritative terminal resolution and append official fill P&L or no-fill closure. +- `status`: report lifecycle/evidence heads, event counts, and unresolved condition IDs. + +There is no live, trading, signing, or order-submission command. + +## Persistence + +The runner loads the two frozen policies, binds all five runtime modules into the lifecycle source hash, and persists: + +- `lifecycle.jsonl` — append-only candidate/capture/source-bound lifecycle chain; +- `raw_evidence/manifest.jsonl` and content-addressed response bodies; +- exclusive `summaries/collect-.json` files; +- exclusive `summaries/resolve-.json` files. + +Duplicate summaries fail closed rather than overwriting evidence. + +## Verification + +- Runner PR: #67 +- Merge commit: `9e794700600ba8a9173040711d09b7feb4817ff9` +- Workflow run: `30218421705` +- Artifact ID: `8636471934` +- Artifact ZIP SHA-256: `762294c4141c25d6dd0a1f6678e2f90795bfbecf909deaf130d3001358ca7b88` +- Tests: **5 passed** +- Combined runtime source SHA-256: `b398e7b9893170cb47ea17923448a87a10b017249a950ca670f9b09081b0d845` +- Internal `SHA256SUMS`: all files verified +- Deterministic TAR sidecar: verified +- Archive extraction round-trip: byte-for-byte verified + +The unchanged observer core also passed its independent 22-contract checkpoint on the runner PR. + +## Safety boundary + +- credentials used: **0** +- authenticated requests: **0** +- order submissions: **0** +- live submission: **physically absent** +- historical admission credit: **0** + +## Next valid work + +Run `collect-next` in a one-shot GitHub workflow, upload the resulting append-only state artifact, then resolve that condition after official closure. Only post-capture-freeze markets may count. diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/RUNNER_VERIFICATION.json b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/RUNNER_VERIFICATION.json new file mode 100644 index 00000000..ded85564 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/RUNNER_VERIFICATION.json @@ -0,0 +1,22 @@ +{ + "authenticated_requests": 0, + "candidate": "late_favorite_btc_only_v3_prospective_shadow", + "combined_runtime_source_sha256": "b398e7b9893170cb47ea17923448a87a10b017249a950ca670f9b09081b0d845", + "credentials_used": 0, + "historical_admission_credit": 0, + "live_submission": "physically_absent", + "order_submissions": 0, + "runner_commands": [ + "collect-next", + "resolve", + "status" + ], + "source_files_sha256": { + "collector/public_collector.py": "85c16a1e9805b89a234139f15140c4f0f39658887419d70c2caf4c238dd5cc6b", + "collector/resolution.py": "bb4a643d4618e78442c11ef21ac15483c048a89e5bc7e900500af661f6a8c91c", + "runner/observer_cli.py": "07025cd07f79787c69d6ff6cd189f01c846e5b8c824a727cea63fe1e288125e7", + "source/capture_policy.py": "bda91808405d5b9e6ee6a09bd464cecd751c23573584bc2624891f5bf091e955", + "source/late_favorite_v3.py": "190ad8d358229f448b42a4dd8dc0e25532b440ce88a8e897fb9cb3c835cc42ee" + }, + "tests_collected": 5 +} diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/observer_cli.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/observer_cli.py new file mode 100644 index 00000000..fdc7040c --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/observer_cli.py @@ -0,0 +1,402 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import time +from dataclasses import dataclass +from decimal import Decimal, InvalidOperation +from pathlib import Path +from typing import Any, Callable + +from capture_policy import CaptureBoundLifecycleLedger, load_capture_policy +from late_favorite_v3 import load_frozen_policy +from public_collector import ( + PublicHttpClient, + RawEvidenceStore, + SingleMarketCollector, + next_collectable_open, +) +from resolution import SingleMarketResolver + +_RUNTIME_SOURCE_PATHS = ( + "source/late_favorite_v3.py", + "source/capture_policy.py", + "collector/public_collector.py", + "collector/resolution.py", + "runner/observer_cli.py", +) + +MAX_BLOCK_MARKETS = 12 +DEFAULT_RESOLUTION_ATTEMPTS = 120 +DEFAULT_RETRY_DELAY_MS = 10_000 + + +@dataclass(slots=True) +class ObserverRuntime: + project_root: Path + state_dir: Path + capture_policy: Any + ledger: Any + evidence_store: Any + collector: Any + resolver: Any + clock_ms: Callable[[], int] + sleep_until_ms: Callable[[int], None] + source_sha256: str + trading_policy: Any = None + client: Any = None + + +def compute_combined_source_sha256(project_root: str | Path) -> str: + root = Path(project_root) + digest = hashlib.sha256() + for relative in _RUNTIME_SOURCE_PATHS: + path = root / relative + raw = path.read_bytes() + digest.update(relative.encode("utf-8") + b"\0" + raw) + return digest.hexdigest() + + +def _real_clock_ms() -> int: + return time.time_ns() // 1_000_000 + + +def make_sleep_until_ms( + clock_ms: Callable[[], int], + sleeper: Callable[[float], None] = time.sleep, +) -> Callable[[int], None]: + def sleep_until(target_ts_ms: int) -> None: + target = int(target_ts_ms) + while True: + remaining_ms = target - int(clock_ms()) + if remaining_ms <= 0: + return + sleeper(min(remaining_ms / 1000.0, 0.25)) + + return sleep_until + + +def build_runtime( + project_root: str | Path, + state_dir: str | Path, + *, + clock_ms: Callable[[], int] | None = None, + sleep_until_ms: Callable[[int], None] | None = None, + requester: Callable[..., Any] | None = None, +) -> ObserverRuntime: + root = Path(project_root).resolve() + state = Path(state_dir).resolve() + source_sha256 = compute_combined_source_sha256(root) + trading_policy = load_frozen_policy( + root / "CANDIDATE_SPEC.json", + root / "FREEZE_MANIFEST.json", + source_sha256=source_sha256, + ) + capture_policy = load_capture_policy( + root / "CAPTURE_POLICY.json", + root / "CAPTURE_FREEZE_MANIFEST.json", + ) + clock = clock_ms or _real_clock_ms + sleep_until = sleep_until_ms or make_sleep_until_ms(clock) + ledger = CaptureBoundLifecycleLedger( + state / "lifecycle.jsonl", + trading_policy, + capture_policy, + ) + evidence = RawEvidenceStore(state / "raw_evidence") + client_kwargs: dict[str, Any] = {"clock_ms": clock} + if requester is not None: + client_kwargs["requester"] = requester + client = PublicHttpClient(**client_kwargs) + collector = SingleMarketCollector( + client=client, + evidence_store=evidence, + ledger=ledger, + trading_policy=trading_policy, + capture_policy=capture_policy, + sleep_until_ms=sleep_until, + ) + resolver = SingleMarketResolver( + client=client, + evidence_store=evidence, + ledger=ledger, + trading_policy=trading_policy, + capture_policy=capture_policy, + ) + return ObserverRuntime( + project_root=root, + state_dir=state, + capture_policy=capture_policy, + ledger=ledger, + evidence_store=evidence, + collector=collector, + resolver=resolver, + clock_ms=clock, + sleep_until_ms=sleep_until, + source_sha256=source_sha256, + trading_policy=trading_policy, + client=client, + ) + + +def _summary_path(runtime: ObserverRuntime, name: str) -> Path: + safe = re.sub(r"[^A-Za-z0-9._-]+", "_", str(name)).strip("._") + if not safe: + raise ValueError("summary name is empty after sanitization") + return runtime.state_dir / "summaries" / f"{safe}.json" + + +def _write_summary(runtime: ObserverRuntime, name: str, summary: dict[str, Any]) -> dict[str, Any]: + path = _summary_path(runtime, name) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("x", encoding="utf-8", newline="\n") as handle: + json.dump(summary, handle, indent=2, sort_keys=True) + handle.write("\n") + return summary + + +def _evidence_head(runtime: ObserverRuntime) -> str: + return str(getattr(runtime.evidence_store, "head_hash", "0" * 64)) + + +def _collect_opening(runtime: ObserverRuntime, opening: int) -> dict[str, Any]: + selected_opening = int(opening) + discovery_target = selected_opening * 1000 + 1_000 + if int(runtime.clock_ms()) < discovery_target: + runtime.sleep_until_ms(discovery_target) + result = dict(runtime.collector.capture(selected_opening)) + summary = { + **result, + "command": "collect-next", + "market_open_epoch_seconds": selected_opening, + "source_sha256": runtime.source_sha256, + "lifecycle_head_sha256": str(runtime.ledger.head_hash), + "raw_evidence_head_sha256": _evidence_head(runtime), + "lifecycle_records": len(getattr(runtime.ledger, "records", [])), + "raw_evidence_records": len(getattr(runtime.evidence_store, "records", [])), + } + return _write_summary(runtime, f"collect-{selected_opening}", summary) + + +def collect_next(runtime: ObserverRuntime, *, now_ts_ms: int | None = None) -> dict[str, Any]: + now = int(runtime.clock_ms() if now_ts_ms is None else now_ts_ms) + opening = next_collectable_open( + now, + int(runtime.capture_policy.valid_market_open_after_epoch_seconds), + ) + return _collect_opening(runtime, opening) + + +def resolve_condition(runtime: ObserverRuntime, condition_id: str) -> dict[str, Any]: + condition = str(condition_id).strip() + if not condition: + raise ValueError("condition_id must be non-empty") + result = dict(runtime.resolver.resolve(condition)) + summary = { + **result, + "command": "resolve", + "source_sha256": runtime.source_sha256, + "lifecycle_head_sha256": str(runtime.ledger.head_hash), + "raw_evidence_head_sha256": _evidence_head(runtime), + "lifecycle_records": len(getattr(runtime.ledger, "records", [])), + "raw_evidence_records": len(getattr(runtime.evidence_store, "records", [])), + } + return _write_summary(runtime, f"resolve-{condition}", summary) + + +def _positive_int(name: str, value: int) -> int: + parsed = int(value) + if parsed <= 0: + raise ValueError(f"{name} must be positive") + return parsed + + +def _pnl_total(resolution: dict[str, Any]) -> Decimal: + value = resolution.get("pnl_total") + if value is None: + return Decimal("0") + try: + return Decimal(str(value)) + except (InvalidOperation, ValueError) as exc: + raise ValueError(f"invalid pnl_total in resolution: {value!r}") from exc + + +def collect_block( + runtime: ObserverRuntime, + *, + markets: int, + now_ts_ms: int | None = None, + max_resolution_attempts: int = DEFAULT_RESOLUTION_ATTEMPTS, + retry_delay_ms: int = DEFAULT_RETRY_DELAY_MS, +) -> dict[str, Any]: + market_count = _positive_int("markets", markets) + if market_count > MAX_BLOCK_MARKETS: + raise ValueError(f"markets must be <= {MAX_BLOCK_MARKETS}") + attempt_limit = _positive_int("max_resolution_attempts", max_resolution_attempts) + retry_delay = _positive_int("retry_delay_ms", retry_delay_ms) + + now = int(runtime.clock_ms() if now_ts_ms is None else now_ts_ms) + first_opening = next_collectable_open( + now, + int(runtime.capture_policy.valid_market_open_after_epoch_seconds), + ) + openings = [first_opening + 300 * index for index in range(market_count)] + + # Capture every requested consecutive window before making any resolution + # request. This keeps terminal latency from causing a missed +210s signal. + captures = [_collect_opening(runtime, opening) for opening in openings] + condition_ids = [str(capture["condition_id"]) for capture in captures] + + resolutions: list[dict[str, Any]] = [] + unresolved: list[str] = [] + attempts_by_condition: dict[str, int] = {} + last_errors: dict[str, dict[str, str]] = {} + + for condition in condition_ids: + resolved: dict[str, Any] | None = None + for attempt in range(1, attempt_limit + 1): + attempts_by_condition[condition] = attempt + try: + resolved = resolve_condition(runtime, condition) + break + except Exception as exc: # fail closed and preserve the block evidence + last_errors[condition] = { + "type": type(exc).__name__, + "message": str(exc), + } + if attempt < attempt_limit: + runtime.sleep_until_ms(int(runtime.clock_ms()) + retry_delay) + if resolved is None: + unresolved.append(condition) + else: + resolutions.append(resolved) + last_errors.pop(condition, None) + + decision_counts: dict[str, int] = {} + hypothetical_fok_fills = 0 + for capture in captures: + decision = str(capture.get("decision", "unknown")) + decision_counts[decision] = decision_counts.get(decision, 0) + 1 + if capture.get("hypothetical_fok_fill") is True or decision == "hypothetical_fok_fill": + hypothetical_fok_fills += 1 + + total_pnl = sum((_pnl_total(resolution) for resolution in resolutions), Decimal("0")) + summary = { + "command": "collect-block", + "status": "complete" if not unresolved else "unresolved", + "markets_requested": market_count, + "markets_collected": len(captures), + "first_market_open_epoch_seconds": openings[0], + "last_market_open_epoch_seconds": openings[-1], + "market_open_epoch_seconds": openings, + "condition_ids": condition_ids, + "capture_decision_counts": dict(sorted(decision_counts.items())), + "hypothetical_fok_fills": hypothetical_fok_fills, + "conditions_resolved": len(resolutions), + "unresolved_condition_ids": unresolved, + "resolution_attempts_by_condition": attempts_by_condition, + "last_resolution_errors": last_errors, + "resolution_decision_counts": { + decision: sum(1 for row in resolutions if str(row.get("decision")) == decision) + for decision in sorted({str(row.get("decision")) for row in resolutions}) + }, + "prospective_pnl_total": format(total_pnl, "f"), + "source_sha256": runtime.source_sha256, + "lifecycle_head_sha256": str(runtime.ledger.head_hash), + "raw_evidence_head_sha256": _evidence_head(runtime), + "lifecycle_records": len(getattr(runtime.ledger, "records", [])), + "raw_evidence_records": len(getattr(runtime.evidence_store, "records", [])), + "credentials_used": 0, + "authenticated_requests": 0, + "order_submissions": 0, + "live_submission": "physically_absent", + "policy_changes": 0, + } + return _write_summary( + runtime, + f"block-{openings[0]}-{openings[-1]}", + summary, + ) + + +def runtime_status(runtime: ObserverRuntime) -> dict[str, Any]: + condition_ids: set[str] = set() + resolved: set[str] = set() + event_counts: dict[str, int] = {} + for record in getattr(runtime.ledger, "records", []): + condition = str(record.get("condition_id", "")) + event_type = str(record.get("event_type", "")) + if condition: + condition_ids.add(condition) + if event_type == "resolution": + resolved.add(condition) + event_counts[event_type] = event_counts.get(event_type, 0) + 1 + return { + "command": "status", + "conditions_seen": len(condition_ids), + "conditions_resolved": len(resolved), + "unresolved_condition_ids": sorted(condition_ids - resolved), + "event_counts": dict(sorted(event_counts.items())), + "source_sha256": runtime.source_sha256, + "lifecycle_head_sha256": str(runtime.ledger.head_hash), + "raw_evidence_head_sha256": _evidence_head(runtime), + "lifecycle_records": len(getattr(runtime.ledger, "records", [])), + "raw_evidence_records": len(getattr(runtime.evidence_store, "records", [])), + } + + +def build_parser() -> argparse.ArgumentParser: + default_root = Path(__file__).resolve().parents[1] + parser = argparse.ArgumentParser( + description="Credential-free BTC-only v3 prospective shadow runner" + ) + parser.add_argument("--project-root", type=Path, default=default_root) + parser.add_argument("--state-dir", type=Path, default=Path(".prospective/btc-only-v3")) + commands = parser.add_subparsers(dest="command", required=True) + collect = commands.add_parser("collect-next", help="collect the next eligible BTC window") + collect.add_argument("--now-ms", type=int) + block = commands.add_parser( + "collect-block", + help="capture consecutive BTC windows, then resolve the captured queue", + ) + block.add_argument("--markets", type=int, required=True) + block.add_argument("--now-ms", type=int) + block.add_argument( + "--max-resolution-attempts", + type=int, + default=DEFAULT_RESOLUTION_ATTEMPTS, + ) + block.add_argument("--retry-delay-ms", type=int, default=DEFAULT_RETRY_DELAY_MS) + resolve = commands.add_parser("resolve", help="resolve one collected condition") + resolve.add_argument("--condition-id", required=True) + commands.add_parser("status", help="show ledger and unresolved-condition status") + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + runtime = build_runtime(args.project_root, args.state_dir) + if args.command == "collect-next": + output = collect_next(runtime, now_ts_ms=args.now_ms) + elif args.command == "collect-block": + output = collect_block( + runtime, + markets=args.markets, + now_ts_ms=args.now_ms, + max_resolution_attempts=args.max_resolution_attempts, + retry_delay_ms=args.retry_delay_ms, + ) + elif args.command == "resolve": + output = resolve_condition(runtime, args.condition_id) + elif args.command == "status": + output = runtime_status(runtime) + else: # pragma: no cover + raise RuntimeError(f"unsupported command: {args.command}") + print(json.dumps(output, indent=2, sort_keys=True)) + return 0 if output.get("status") != "unresolved" else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/tests/test_observer_block.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/tests/test_observer_block.py new file mode 100644 index 00000000..3f2988e0 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/tests/test_observer_block.py @@ -0,0 +1,236 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path + +import pytest + +from observer_cli import ( + MAX_BLOCK_MARKETS, + ObserverRuntime, + build_parser, + collect_block, +) + + +@dataclass +class FakeCapturePolicy: + valid_market_open_after_epoch_seconds: int = 1_800_000_100 + + +@dataclass +class FakeLedger: + operations: list[str] + head_hash: str = "a" * 64 + records: list[dict[str, object]] = field(default_factory=list) + + +@dataclass +class FakeEvidence: + head_hash: str = "b" * 64 + records: list[dict[str, object]] = field(default_factory=list) + + +class FakeCollector: + def __init__(self, operations: list[str], ledger: FakeLedger, evidence: FakeEvidence) -> None: + self.operations = operations + self.ledger = ledger + self.evidence = evidence + self.openings: list[int] = [] + + def capture(self, opening: int) -> dict[str, object]: + self.openings.append(opening) + self.operations.append(f"capture:{opening}") + condition = f"condition-{opening}" + decision = ( + "hypothetical_fok_fill" + if len(self.openings) == 1 + else "no_signal_below_threshold" + if len(self.openings) == 2 + else "no_fill_ask_above_limit" + ) + self.ledger.records.append( + { + "condition_id": condition, + "event_type": "arrival" if decision != "no_signal_below_threshold" else "signal", + } + ) + self.evidence.records.append({"opening": opening}) + return { + "condition_id": condition, + "market_open_epoch_seconds": opening, + "decision": decision, + "hypothetical_fok_fill": decision == "hypothetical_fok_fill", + } + + +class FakeResolver: + def __init__( + self, + operations: list[str], + ledger: FakeLedger, + evidence: FakeEvidence, + *, + failures_before_success: int = 0, + never_resolve: bool = False, + ) -> None: + self.operations = operations + self.ledger = ledger + self.evidence = evidence + self.failures_before_success = failures_before_success + self.never_resolve = never_resolve + self.attempts: dict[str, int] = {} + + def resolve(self, condition_id: str) -> dict[str, object]: + self.operations.append(f"resolve:{condition_id}") + attempt = self.attempts.get(condition_id, 0) + 1 + self.attempts[condition_id] = attempt + self.evidence.records.append({"condition_id": condition_id, "attempt": attempt}) + if self.never_resolve or attempt <= self.failures_before_success: + raise ValueError("Gamma market is not closed") + self.ledger.records.append( + {"condition_id": condition_id, "event_type": "resolution"} + ) + return { + "condition_id": condition_id, + "decision": "resolved_fill" if condition_id.endswith("300") else "resolved_no_fill", + "official_outcome": "Up", + "pnl_total": "0.45" if condition_id.endswith("300") else None, + } + + +def runtime( + tmp_path: Path, + *, + now_ms: int, + failures_before_success: int = 0, + never_resolve: bool = False, +) -> tuple[ObserverRuntime, list[str], list[int]]: + operations: list[str] = [] + sleeps: list[int] = [] + ledger = FakeLedger(operations) + evidence = FakeEvidence() + collector = FakeCollector(operations, ledger, evidence) + resolver = FakeResolver( + operations, + ledger, + evidence, + failures_before_success=failures_before_success, + never_resolve=never_resolve, + ) + return ( + ObserverRuntime( + project_root=tmp_path, + state_dir=tmp_path / "state", + capture_policy=FakeCapturePolicy(), + ledger=ledger, + evidence_store=evidence, + collector=collector, + resolver=resolver, + clock_ms=lambda: now_ms, + sleep_until_ms=sleeps.append, + source_sha256="c" * 64, + ), + operations, + sleeps, + ) + + +def test_collect_block_captures_consecutive_windows_before_any_resolution(tmp_path: Path) -> None: + first_opening = 1_800_000_300 + now_ms = first_opening * 1000 + 500 + rt, operations, sleeps = runtime(tmp_path, now_ms=now_ms, failures_before_success=1) + + summary = collect_block( + rt, + markets=3, + now_ts_ms=now_ms, + max_resolution_attempts=3, + retry_delay_ms=10_000, + ) + + openings = [first_opening, first_opening + 300, first_opening + 600] + assert rt.collector.openings == openings + first_resolve_index = next( + index for index, operation in enumerate(operations) if operation.startswith("resolve:") + ) + assert all(operation.startswith("capture:") for operation in operations[:first_resolve_index]) + assert operations[:3] == [f"capture:{opening}" for opening in openings] + assert sleeps[:3] == [opening * 1000 + 1_000 for opening in openings] + assert summary["status"] == "complete" + assert summary["markets_requested"] == 3 + assert summary["markets_collected"] == 3 + assert summary["conditions_resolved"] == 3 + assert summary["unresolved_condition_ids"] == [] + assert summary["hypothetical_fok_fills"] == 1 + assert summary["capture_decision_counts"] == { + "hypothetical_fok_fill": 1, + "no_fill_ask_above_limit": 1, + "no_signal_below_threshold": 1, + } + assert summary["resolution_attempts_by_condition"] == { + f"condition-{opening}": 2 for opening in openings + } + saved = json.loads( + ( + rt.state_dir + / "summaries" + / f"block-{openings[0]}-{openings[-1]}.json" + ).read_text() + ) + assert saved == summary + + +def test_collect_block_records_unresolved_conditions_after_bounded_attempts(tmp_path: Path) -> None: + opening = 1_800_000_300 + now_ms = opening * 1000 + 500 + rt, operations, sleeps = runtime(tmp_path, now_ms=now_ms, never_resolve=True) + + summary = collect_block( + rt, + markets=1, + now_ts_ms=now_ms, + max_resolution_attempts=2, + retry_delay_ms=5_000, + ) + + condition = f"condition-{opening}" + assert summary["status"] == "unresolved" + assert summary["conditions_resolved"] == 0 + assert summary["unresolved_condition_ids"] == [condition] + assert summary["resolution_attempts_by_condition"] == {condition: 2} + assert operations == [ + f"capture:{opening}", + f"resolve:{condition}", + f"resolve:{condition}", + ] + assert sleeps == [opening * 1000 + 1_000, now_ms + 5_000] + + +@pytest.mark.parametrize("markets", [0, -1, MAX_BLOCK_MARKETS + 1]) +def test_collect_block_rejects_unbounded_market_counts(tmp_path: Path, markets: int) -> None: + rt, _, _ = runtime(tmp_path, now_ms=1_800_000_300_500) + with pytest.raises(ValueError): + collect_block(rt, markets=markets) + + +def test_cli_exposes_bounded_collect_block_without_live_commands() -> None: + parser = build_parser() + args = parser.parse_args( + [ + "collect-block", + "--markets", + "3", + "--max-resolution-attempts", + "120", + "--retry-delay-ms", + "10000", + ] + ) + assert args.command == "collect-block" + assert args.markets == 3 + assert args.max_resolution_attempts == 120 + assert args.retry_delay_ms == 10_000 + with pytest.raises(SystemExit): + parser.parse_args(["submit-block"]) diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/tests/test_observer_cli.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/tests/test_observer_cli.py new file mode 100644 index 00000000..f9b75a58 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/runner/tests/test_observer_cli.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path + +import pytest + +from observer_cli import ( + ObserverRuntime, + build_parser, + collect_next, + compute_combined_source_sha256, + resolve_condition, + runtime_status, +) + + +@dataclass +class FakeCapturePolicy: + valid_market_open_after_epoch_seconds: int = 1_800_000_100 + + +class FakeCollector: + def __init__(self) -> None: + self.openings: list[int] = [] + + def capture(self, opening: int) -> dict[str, object]: + self.openings.append(opening) + return { + "condition_id": f"condition-{opening}", + "market_open_epoch_seconds": opening, + "decision": "hypothetical_fok_fill", + } + + +class FakeResolver: + def __init__(self) -> None: + self.conditions: list[str] = [] + + def resolve(self, condition_id: str) -> dict[str, object]: + self.conditions.append(condition_id) + return { + "condition_id": condition_id, + "decision": "resolved_fill", + "official_outcome": "Up", + "pnl_total": "0.45", + } + + +class FakeLedger: + def __init__(self) -> None: + self.head_hash = "a" * 64 + self.records = [ + { + "condition_id": "resolved", + "event_type": "resolution", + "record_hash": "1" * 64, + }, + { + "condition_id": "unresolved", + "event_type": "signal", + "record_hash": "2" * 64, + }, + ] + + +class FakeEvidence: + head_hash = "b" * 64 + records = [{"record_hash": "3" * 64}] + + +def runtime(tmp_path: Path, *, now_ms: int, sleeps: list[int]) -> ObserverRuntime: + return ObserverRuntime( + project_root=tmp_path, + state_dir=tmp_path / "state", + capture_policy=FakeCapturePolicy(), + ledger=FakeLedger(), + evidence_store=FakeEvidence(), + collector=FakeCollector(), + resolver=FakeResolver(), + clock_ms=lambda: now_ms, + sleep_until_ms=sleeps.append, + source_sha256="c" * 64, + ) + + +def test_combined_source_hash_binds_all_runtime_modules(tmp_path: Path) -> None: + root = tmp_path / "candidate" + (root / "source").mkdir(parents=True) + (root / "collector").mkdir(parents=True) + files = { + root / "source/late_favorite_v3.py": b"core\n", + root / "source/capture_policy.py": b"capture\n", + root / "collector/public_collector.py": b"collector\n", + root / "collector/resolution.py": b"resolution\n", + root / "runner/observer_cli.py": b"runner\n", + } + (root / "runner").mkdir(parents=True) + for path, content in files.items(): + path.write_bytes(content) + first = compute_combined_source_sha256(root) + assert len(first) == 64 + files[root / "runner/observer_cli.py"] = b"runner changed\n" + (root / "runner/observer_cli.py").write_bytes(files[root / "runner/observer_cli.py"]) + assert compute_combined_source_sha256(root) != first + + +def test_collect_next_selects_future_window_and_persists_summary(tmp_path: Path) -> None: + opening = 1_800_000_300 + now_ms = opening * 1000 + 210_001 + sleeps: list[int] = [] + rt = runtime(tmp_path, now_ms=now_ms, sleeps=sleeps) + summary = collect_next(rt, now_ts_ms=now_ms) + selected = opening + 300 + assert rt.collector.openings == [selected] + assert sleeps == [selected * 1000 + 1_000] + assert summary["condition_id"] == f"condition-{selected}" + assert summary["lifecycle_head_sha256"] == "a" * 64 + assert summary["raw_evidence_head_sha256"] == "b" * 64 + saved = json.loads((rt.state_dir / "summaries" / f"collect-{selected}.json").read_text()) + assert saved == summary + with pytest.raises(FileExistsError): + collect_next(rt, now_ts_ms=now_ms) + + +def test_resolve_condition_persists_official_summary(tmp_path: Path) -> None: + rt = runtime(tmp_path, now_ms=1_800_000_000_000, sleeps=[]) + summary = resolve_condition(rt, "condition-1") + assert rt.resolver.conditions == ["condition-1"] + assert summary["official_outcome"] == "Up" + path = rt.state_dir / "summaries" / "resolve-condition-1.json" + assert json.loads(path.read_text()) == summary + + +def test_runtime_status_lists_unresolved_conditions(tmp_path: Path) -> None: + status = runtime_status(runtime(tmp_path, now_ms=0, sleeps=[])) + assert status["conditions_seen"] == 2 + assert status["conditions_resolved"] == 1 + assert status["unresolved_condition_ids"] == ["unresolved"] + assert status["lifecycle_head_sha256"] == "a" * 64 + + +def test_cli_exposes_only_collect_resolve_and_status() -> None: + parser = build_parser() + assert parser.parse_args(["collect-next"]).command == "collect-next" + assert parser.parse_args(["resolve", "--condition-id", "c1"]).condition_id == "c1" + assert parser.parse_args(["status"]).command == "status" + with pytest.raises(SystemExit): + parser.parse_args(["live"]) + with pytest.raises(SystemExit): + parser.parse_args(["submit-order"]) diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source/capture_policy.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source/capture_policy.py new file mode 100644 index 00000000..baf4417a --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source/capture_policy.py @@ -0,0 +1,372 @@ +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Any, Mapping, Sequence + +from late_favorite_v3 import ( + FreezeViolation, + FrozenPolicy, + LedgerIntegrityError, + LifecycleError, + ProspectiveLifecycleLedger, + evaluate_arrival, + evaluate_signal, +) + + +class CaptureWindowError(LifecycleError): + pass + + +def _validate_sha256(name: str, value: str) -> str: + text = str(value).lower() + if len(text) != 64 or any(character not in "0123456789abcdef" for character in text): + raise ValueError(f"{name} must be a 64-character SHA-256 hex digest") + return text + + +def _canonical_json(value: object) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +@dataclass(frozen=True, slots=True) +class CapturePolicy: + capture_policy_sha256: str + valid_market_open_after_epoch_seconds: int + signal_target_offset_ms: int + arrival_target_offset_ms: int + max_request_start_lateness_ms: int + max_request_duration_ms: int + max_book_age_at_request_completion_ms: int + max_book_future_skew_ms: int + + def __post_init__(self) -> None: + object.__setattr__( + self, + "capture_policy_sha256", + _validate_sha256("capture_policy_sha256", self.capture_policy_sha256), + ) + for name in ( + "valid_market_open_after_epoch_seconds", + "signal_target_offset_ms", + "arrival_target_offset_ms", + "max_request_start_lateness_ms", + "max_request_duration_ms", + "max_book_age_at_request_completion_ms", + "max_book_future_skew_ms", + ): + if int(getattr(self, name)) < 0: + raise ValueError(f"{name} must be non-negative") + if self.arrival_target_offset_ms <= self.signal_target_offset_ms: + raise ValueError("arrival target must be after signal target") + if self.max_request_duration_ms <= 0: + raise ValueError("max_request_duration_ms must be positive") + + +def load_capture_policy( + capture_policy_path: str | Path, + capture_manifest_path: str | Path, +) -> CapturePolicy: + policy_path = Path(capture_policy_path) + raw = policy_path.read_bytes() + payload = json.loads(raw) + manifest = json.loads(Path(capture_manifest_path).read_text(encoding="utf-8")) + got = hashlib.sha256(raw).hexdigest() + wanted = _validate_sha256( + "capture_policy_sha256", str(manifest.get("capture_policy_sha256", "")) + ) + if got != wanted: + raise FreezeViolation(f"capture policy SHA-256 mismatch: {got} != {wanted}") + if payload.get("name") != "late_favorite_btc_only_v3_capture_policy": + raise FreezeViolation("unexpected capture policy name") + if payload.get("status") != "frozen_before_prospective_collection": + raise FreezeViolation("capture policy is not frozen") + transport = payload.get("transport", {}) + privileged = ("credentials_used", "authenticated_requests", "order_submissions") + if any(int(transport.get(name, -1)) != 0 for name in privileged): + raise FreezeViolation("capture policy permits privileged transport") + freeze_text = str(manifest["valid_market_open_after_utc"]) + freeze_epoch = int(datetime.fromisoformat(freeze_text.replace("Z", "+00:00")).timestamp()) + event_time = payload["event_time"] + windows = payload["request_windows"] + freshness = payload["book_freshness"] + return CapturePolicy( + capture_policy_sha256=got, + valid_market_open_after_epoch_seconds=freeze_epoch, + signal_target_offset_ms=int(event_time["signal_target_offset_ms"]), + arrival_target_offset_ms=int(event_time["arrival_target_offset_ms"]), + max_request_start_lateness_ms=int(windows["max_request_start_lateness_ms"]), + max_request_duration_ms=int(windows["max_request_duration_ms"]), + max_book_age_at_request_completion_ms=int( + freshness["max_book_age_at_request_completion_ms"] + ), + max_book_future_skew_ms=int(freshness["max_book_future_skew_ms"]), + ) + + +def _target_ms( + capture_policy: CapturePolicy, + market_open_epoch_seconds: int, + *, + phase: str, +) -> int: + opening = int(market_open_epoch_seconds) + if opening <= capture_policy.valid_market_open_after_epoch_seconds: + raise FreezeViolation( + f"market open {opening} is not strictly after capture freeze " + f"{capture_policy.valid_market_open_after_epoch_seconds}" + ) + offset = ( + capture_policy.signal_target_offset_ms + if phase == "signal" + else capture_policy.arrival_target_offset_ms + ) + return opening * 1000 + offset + + +def _transport_evidence( + capture_policy: CapturePolicy, + *, + market_open_epoch_seconds: int, + phase: str, + request_started_ts_ms: int, + request_completed_ts_ms: int, + response_payload_sha256: str, +) -> tuple[dict[str, Any], str | None]: + target = _target_ms(capture_policy, market_open_epoch_seconds, phase=phase) + started = int(request_started_ts_ms) + completed = int(request_completed_ts_ms) + response_sha = _validate_sha256("response_payload_sha256", response_payload_sha256) + if started < target: + raise CaptureWindowError( + f"{phase} request started before target: {started} < {target}" + ) + if completed < started: + raise CaptureWindowError("request completion precedes request start") + base = { + "capture_policy_sha256": capture_policy.capture_policy_sha256, + "target_ts_ms": target, + "request_started_ts_ms": started, + "request_completed_ts_ms": completed, + "request_duration_ms": completed - started, + "response_payload_sha256": response_sha, + } + if started - target > capture_policy.max_request_start_lateness_ms: + return base, f"missed_{phase}_window" + if completed - started > capture_policy.max_request_duration_ms: + return base, f"{phase}_request_timeout" + return base, None + + +def _book_timestamp_decision( + capture_policy: CapturePolicy, + books: Sequence[Mapping[str, Any]], + *, + request_completed_ts_ms: int, + phase: str, +) -> tuple[list[int], str | None]: + timestamps: list[int] = [] + for book in books: + try: + timestamp = int(str(book.get("timestamp"))) + except (TypeError, ValueError): + return [], f"no_{phase}_invalid_book_timestamp" + timestamps.append(timestamp) + age = int(request_completed_ts_ms) - timestamp + if age > capture_policy.max_book_age_at_request_completion_ms: + return timestamps, f"no_{phase}_stale_book" + if timestamp - int(request_completed_ts_ms) > capture_policy.max_book_future_skew_ms: + return timestamps, f"no_{phase}_future_book" + return timestamps, None + + +def evaluate_signal_capture( + policy: FrozenPolicy, + capture_policy: CapturePolicy, + *, + condition_id: str, + market_open_epoch_seconds: int, + up_token_id: str, + down_token_id: str, + up_book: Mapping[str, Any], + down_book: Mapping[str, Any], + request_started_ts_ms: int, + request_completed_ts_ms: int, + response_payload_sha256: str, +) -> dict[str, Any]: + transport, transport_decision = _transport_evidence( + capture_policy, + market_open_epoch_seconds=market_open_epoch_seconds, + phase="signal", + request_started_ts_ms=request_started_ts_ms, + request_completed_ts_ms=request_completed_ts_ms, + response_payload_sha256=response_payload_sha256, + ) + fail_base = { + "event_type": "signal", + "condition_id": str(condition_id), + "market_open_epoch_seconds": int(market_open_epoch_seconds), + "observed_ts_ms": transport["target_ts_ms"], + "policy_sha256": policy.policy_sha256, + "source_sha256": policy.source_sha256, + "signal": False, + "selected_side": None, + "selected_token_id": None, + "signal_ask": None, + **transport, + } + if transport_decision is not None: + return {**fail_base, "decision": transport_decision} + timestamps, book_decision = _book_timestamp_decision( + capture_policy, + [up_book, down_book], + request_completed_ts_ms=request_completed_ts_ms, + phase="signal", + ) + if book_decision is not None: + return {**fail_base, "decision": book_decision, "book_timestamps_ms": timestamps} + result = evaluate_signal( + policy, + condition_id=condition_id, + market_open_epoch_seconds=market_open_epoch_seconds, + up_token_id=up_token_id, + down_token_id=down_token_id, + up_book=up_book, + down_book=down_book, + observed_ts_ms=transport["target_ts_ms"], + ) + return {**result, **transport, "book_timestamps_ms": timestamps} + + +def evaluate_arrival_capture( + policy: FrozenPolicy, + capture_policy: CapturePolicy, + signal_record: Mapping[str, Any], + selected_book: Mapping[str, Any], + *, + request_started_ts_ms: int, + request_completed_ts_ms: int, + response_payload_sha256: str, +) -> dict[str, Any]: + opening = int(signal_record["market_open_epoch_seconds"]) + transport, transport_decision = _transport_evidence( + capture_policy, + market_open_epoch_seconds=opening, + phase="arrival", + request_started_ts_ms=request_started_ts_ms, + request_completed_ts_ms=request_completed_ts_ms, + response_payload_sha256=response_payload_sha256, + ) + fail_base = { + "event_type": "arrival", + "condition_id": str(signal_record["condition_id"]), + "market_open_epoch_seconds": opening, + "observed_ts_ms": transport["target_ts_ms"], + "policy_sha256": policy.policy_sha256, + "source_sha256": policy.source_sha256, + "selected_side": signal_record.get("selected_side"), + "selected_token_id": signal_record.get("selected_token_id"), + "signal_ask": signal_record.get("signal_ask"), + "hypothetical_fok_fill": False, + "filled_shares": "0", + "execution_levels": [], + "execution_vwap": None, + "fee_per_share": None, + "all_in_cost_per_share": None, + **transport, + } + if transport_decision is not None: + return {**fail_base, "decision": transport_decision} + timestamps, book_decision = _book_timestamp_decision( + capture_policy, + [selected_book], + request_completed_ts_ms=request_completed_ts_ms, + phase="arrival", + ) + if book_decision is not None: + decision = book_decision.replace("no_arrival_", "no_fill_") + return {**fail_base, "decision": decision, "book_timestamps_ms": timestamps} + result = evaluate_arrival( + policy, + signal_record, + selected_book, + observed_ts_ms=transport["target_ts_ms"], + ) + return {**result, **transport, "book_timestamps_ms": timestamps} + + +class CaptureBoundLifecycleLedger(ProspectiveLifecycleLedger): + def __init__( + self, + path: str | Path, + policy: FrozenPolicy, + capture_policy: CapturePolicy, + ) -> None: + super().__init__(path, policy) + self.capture_policy = capture_policy + for line_number, record in enumerate(self.records, start=1): + if record.get("capture_policy_sha256") != capture_policy.capture_policy_sha256: + raise LedgerIntegrityError( + f"capture policy hash mismatch at line {line_number}" + ) + if int(record["market_open_epoch_seconds"]) <= capture_policy.valid_market_open_after_epoch_seconds: + raise LedgerIntegrityError( + f"capture freeze violation at line {line_number}" + ) + + def append( + self, + *, + condition_id: str, + market_open_epoch_seconds: int, + event_type: str, + observed_ts_ms: int, + payload: Mapping[str, Any], + ) -> dict[str, Any]: + condition = str(condition_id) + if not condition: + raise ValueError("condition_id must be non-empty") + opening = int(market_open_epoch_seconds) + _target_ms(self.capture_policy, opening, phase="signal") + event = str(event_type) + if event not in {"discovered", "signal", "arrival", "resolution"}: + raise LifecycleError(f"unsupported event_type: {event}") + observed = int(observed_ts_ms) + if observed < 0: + raise ValueError("observed_ts_ms must be non-negative") + try: + canonical_payload = json.loads(_canonical_json(dict(payload)).decode("utf-8")) + except (TypeError, ValueError) as exc: + raise ValueError("payload must be JSON serializable") from exc + self._validate_order(condition, event, canonical_payload) + payload_sha256 = hashlib.sha256(_canonical_json(canonical_payload)).hexdigest() + event_id = hashlib.sha256( + f"{condition}\0{event}\0{observed}\0{payload_sha256}".encode("utf-8") + ).hexdigest() + record_without_hash = { + "capture_policy_sha256": self.capture_policy.capture_policy_sha256, + "condition_id": condition, + "event_id": event_id, + "event_type": event, + "market_open_epoch_seconds": opening, + "observed_ts_ms": observed, + "payload": canonical_payload, + "payload_sha256": payload_sha256, + "policy_sha256": self.policy.policy_sha256, + "previous_hash": self.head_hash, + "source_sha256": self.policy.source_sha256, + } + record = { + **record_without_hash, + "record_hash": hashlib.sha256(_canonical_json(record_without_hash)).hexdigest(), + } + self.path.parent.mkdir(parents=True, exist_ok=True) + with self.path.open("a", encoding="utf-8", newline="\n") as handle: + handle.write(json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n") + handle.flush() + self._accept_record(record) + return dict(record) diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source/late_favorite_v3.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source/late_favorite_v3.py new file mode 100644 index 00000000..5a233a0c --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/source/late_favorite_v3.py @@ -0,0 +1,485 @@ +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from decimal import Decimal, InvalidOperation +from pathlib import Path +from typing import Any, Mapping, Sequence + + +class FreezeViolation(RuntimeError): + pass + + +class LifecycleError(RuntimeError): + pass + + +class LedgerIntegrityError(RuntimeError): + pass + + +def _validate_sha256(name: str, value: str) -> str: + text = str(value).lower() + if len(text) != 64 or any(character not in "0123456789abcdef" for character in text): + raise ValueError(f"{name} must be a 64-character SHA-256 hex digest") + return text + + +def _canonical_json(value: object) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _decimal(value: object, name: str) -> Decimal: + try: + number = Decimal(str(value)) + except (InvalidOperation, ValueError) as exc: + raise ValueError(f"{name} must be a decimal") from exc + if not number.is_finite(): + raise ValueError(f"{name} must be finite") + return number + + +def _decimal_text(value: Decimal) -> str: + return format(value, "f") + + +@dataclass(frozen=True, slots=True) +class FrozenPolicy: + policy_sha256: str + source_sha256: str + valid_market_open_after_epoch_seconds: int + signal_ask_min: Decimal + entry_second: int + latency_seconds: int + shares: Decimal + adverse_move_cancel: Decimal + fee_rate: Decimal + + def __post_init__(self) -> None: + object.__setattr__(self, "policy_sha256", _validate_sha256("policy_sha256", self.policy_sha256)) + object.__setattr__(self, "source_sha256", _validate_sha256("source_sha256", self.source_sha256)) + if int(self.valid_market_open_after_epoch_seconds) < 0: + raise ValueError("valid_market_open_after_epoch_seconds must be non-negative") + if int(self.entry_second) < 0: + raise ValueError("entry_second must be non-negative") + if int(self.latency_seconds) <= 0: + raise ValueError("latency_seconds must be positive") + for name in ("signal_ask_min", "shares", "adverse_move_cancel", "fee_rate"): + value = getattr(self, name) + if not isinstance(value, Decimal) or not value.is_finite() or value <= 0: + raise ValueError(f"{name} must be a positive finite Decimal") + if not Decimal("0") < self.signal_ask_min < Decimal("1"): + raise ValueError("signal_ask_min must be between zero and one") + + +def load_frozen_policy( + candidate_spec_path: str | Path, + freeze_manifest_path: str | Path, + *, + source_sha256: str, +) -> FrozenPolicy: + from datetime import datetime + + spec_path = Path(candidate_spec_path) + manifest_path = Path(freeze_manifest_path) + spec_raw = spec_path.read_bytes() + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + spec = json.loads(spec_raw) + got = hashlib.sha256(spec_raw).hexdigest() + wanted = _validate_sha256("manifest policy_sha256", str(manifest.get("policy_sha256", ""))) + if got != wanted: + raise FreezeViolation(f"candidate policy SHA-256 mismatch: {got} != {wanted}") + if spec.get("name") != manifest.get("candidate_name"): + raise FreezeViolation("candidate name does not match freeze manifest") + if spec.get("status") != "prospective_shadow_only": + raise FreezeViolation("candidate is not prospective_shadow_only") + if spec.get("observation_contract", {}).get("live_submission") != "physically_absent": + raise FreezeViolation("live submission boundary is not physically absent") + freeze_text = str(manifest["contamination_boundary"]["valid_market_open_after_utc"]) + freeze_epoch = int(datetime.fromisoformat(freeze_text.replace("Z", "+00:00")).timestamp()) + return FrozenPolicy( + policy_sha256=got, + source_sha256=source_sha256, + valid_market_open_after_epoch_seconds=freeze_epoch, + signal_ask_min=_decimal(spec["signal"]["signal_ask_min"], "signal_ask_min"), + entry_second=int(spec["signal"]["entry_second"]), + latency_seconds=int(spec["arrival"]["latency_seconds"]), + shares=_decimal(spec["arrival"]["shares"], "shares"), + adverse_move_cancel=Decimal("0.01"), + fee_rate=Decimal("0.07"), + ) + + +def _normalized_book( + payload: Mapping[str, Any], + *, + expected_condition_id: str, + expected_token_id: str, +) -> dict[str, Any]: + condition_id = str(payload.get("market") or "") + token_id = str(payload.get("asset_id") or "") + if condition_id != expected_condition_id: + raise LifecycleError(f"book condition mismatch: {condition_id} != {expected_condition_id}") + if token_id != expected_token_id: + raise LifecycleError(f"book token mismatch: {token_id} != {expected_token_id}") + raw_asks = payload.get("asks", []) + if not isinstance(raw_asks, Sequence) or isinstance(raw_asks, (str, bytes)): + raise LifecycleError("book asks must be a sequence") + asks: list[dict[str, str]] = [] + for index, row in enumerate(raw_asks): + if not isinstance(row, Mapping): + raise LifecycleError(f"ask level {index} is not an object") + price = _decimal(row.get("price"), f"ask[{index}].price") + size = _decimal(row.get("size"), f"ask[{index}].size") + if not Decimal("0") < price < Decimal("1"): + raise LifecycleError("ask price must be strictly between zero and one") + if size < 0: + raise LifecycleError("ask size must be non-negative") + if size > 0: + asks.append({"price": _decimal_text(price), "size": _decimal_text(size)}) + asks.sort(key=lambda row: Decimal(row["price"])) + normalized = { + "market": condition_id, + "asset_id": token_id, + "timestamp": str(payload.get("timestamp") or ""), + "hash": str(payload.get("hash") or ""), + "asks": asks, + "bids": payload.get("bids", []), + "min_order_size": str(payload.get("min_order_size") or "0"), + "tick_size": str(payload.get("tick_size") or "0.01"), + "neg_risk": bool(payload.get("neg_risk", False)), + "last_trade_price": None if payload.get("last_trade_price") in (None, "") else str(payload.get("last_trade_price")), + } + normalized["payload_sha256"] = hashlib.sha256(_canonical_json(dict(payload))).hexdigest() + return normalized + + +def _best_ask(book: Mapping[str, Any]) -> Decimal | None: + asks = book.get("asks", []) + if not asks: + return None + return Decimal(str(asks[0]["price"])) + + +def _assert_post_freeze(policy: FrozenPolicy, market_open_epoch_seconds: int) -> int: + opening = int(market_open_epoch_seconds) + if opening <= int(policy.valid_market_open_after_epoch_seconds): + raise FreezeViolation( + f"market open {opening} is not strictly after freeze {policy.valid_market_open_after_epoch_seconds}" + ) + return opening + + +def evaluate_signal( + policy: FrozenPolicy, + *, + condition_id: str, + market_open_epoch_seconds: int, + up_token_id: str, + down_token_id: str, + up_book: Mapping[str, Any], + down_book: Mapping[str, Any], + observed_ts_ms: int, +) -> dict[str, Any]: + opening = _assert_post_freeze(policy, market_open_epoch_seconds) + expected_ts_ms = (opening + int(policy.entry_second)) * 1000 + observed = int(observed_ts_ms) + if observed != expected_ts_ms: + raise LifecycleError(f"signal observation must be at {expected_ts_ms}, got {observed}") + condition = str(condition_id) + if not condition: + raise ValueError("condition_id must be non-empty") + up = _normalized_book(up_book, expected_condition_id=condition, expected_token_id=str(up_token_id)) + down = _normalized_book(down_book, expected_condition_id=condition, expected_token_id=str(down_token_id)) + up_ask = _best_ask(up) + down_ask = _best_ask(down) + base = { + "event_type": "signal", + "condition_id": condition, + "market_open_epoch_seconds": opening, + "observed_ts_ms": observed, + "policy_sha256": policy.policy_sha256, + "source_sha256": policy.source_sha256, + "up_token_id": str(up_token_id), + "down_token_id": str(down_token_id), + "up_book": up, + "down_book": down, + "up_book_sha256": up["payload_sha256"], + "down_book_sha256": down["payload_sha256"], + "signal": False, + "selected_side": None, + "selected_token_id": None, + "signal_ask": None, + } + if up_ask is None or down_ask is None: + return {**base, "decision": "no_signal_empty_book"} + if up_ask == down_ask: + return {**base, "decision": "no_signal_tied_favorite"} + selected_side = "Up" if up_ask > down_ask else "Down" + selected_token_id = str(up_token_id) if selected_side == "Up" else str(down_token_id) + signal_ask = up_ask if selected_side == "Up" else down_ask + selected = { + **base, + "selected_side": selected_side, + "selected_token_id": selected_token_id, + "signal_ask": _decimal_text(signal_ask), + } + if signal_ask < policy.signal_ask_min: + return {**selected, "decision": "no_signal_below_threshold"} + return {**selected, "decision": "signal", "signal": True} + + +def _fee_per_share(policy: FrozenPolicy, price: Decimal) -> Decimal: + return policy.fee_rate * price * (Decimal("1") - price) + + +def evaluate_arrival( + policy: FrozenPolicy, + signal_record: Mapping[str, Any], + selected_book: Mapping[str, Any], + *, + observed_ts_ms: int, +) -> dict[str, Any]: + if signal_record.get("decision") != "signal" or signal_record.get("signal") is not True: + raise LifecycleError("arrival requires an active signal") + if str(signal_record.get("policy_sha256")) != policy.policy_sha256: + raise LifecycleError("signal policy hash mismatch") + expected = int(signal_record["observed_ts_ms"]) + int(policy.latency_seconds) * 1000 + observed = int(observed_ts_ms) + if observed != expected: + raise LifecycleError(f"arrival observation must be at {expected}, got {observed}") + condition = str(signal_record["condition_id"]) + token_id = str(signal_record["selected_token_id"]) + book = _normalized_book(selected_book, expected_condition_id=condition, expected_token_id=token_id) + best_ask = _best_ask(book) + base = { + "event_type": "arrival", + "condition_id": condition, + "market_open_epoch_seconds": int(signal_record["market_open_epoch_seconds"]), + "observed_ts_ms": observed, + "policy_sha256": policy.policy_sha256, + "source_sha256": policy.source_sha256, + "selected_side": str(signal_record["selected_side"]), + "selected_token_id": token_id, + "signal_ask": str(signal_record["signal_ask"]), + "arrival_book": book, + "arrival_book_sha256": book["payload_sha256"], + "arrival_best_ask": None if best_ask is None else _decimal_text(best_ask), + "hypothetical_fok_fill": False, + "filled_shares": "0", + "execution_levels": [], + "execution_vwap": None, + "fee_per_share": None, + "all_in_cost_per_share": None, + } + if best_ask is None: + return {**base, "decision": "no_fill_empty_arrival_book"} + signal_ask = Decimal(str(signal_record["signal_ask"])) + if best_ask < signal_ask - policy.adverse_move_cancel: + return {**base, "decision": "cancel_adverse_move"} + if best_ask > signal_ask: + return {**base, "decision": "no_fill_ask_above_limit"} + + remaining = policy.shares + notional = Decimal("0") + fee_total = Decimal("0") + execution_levels: list[dict[str, str]] = [] + for row in book["asks"]: + price = Decimal(row["price"]) + if price > signal_ask: + break + available = Decimal(row["size"]) + take = min(remaining, available) + if take <= 0: + continue + execution_levels.append({"price": _decimal_text(price), "size": _decimal_text(take)}) + notional += price * take + fee_total += _fee_per_share(policy, price) * take + remaining -= take + if remaining == 0: + break + if remaining > 0: + return {**base, "decision": "no_fill_insufficient_depth"} + vwap = notional / policy.shares + fee = fee_total / policy.shares + return { + **base, + "decision": "hypothetical_fok_fill", + "hypothetical_fok_fill": True, + "filled_shares": _decimal_text(policy.shares), + "execution_levels": execution_levels, + "execution_vwap": _decimal_text(vwap), + "fee_per_share": _decimal_text(fee), + "all_in_cost_per_share": _decimal_text(vwap + fee), + } + + +def settle_fill( + fill_record: Mapping[str, Any], + *, + official_outcome: str, + resolution_payload_sha256: str, +) -> dict[str, Any]: + if fill_record.get("decision") != "hypothetical_fok_fill": + raise LifecycleError("settlement requires a hypothetical FOK fill") + outcome = str(official_outcome).title() + if outcome not in {"Up", "Down"}: + raise ValueError("official_outcome must be Up or Down") + resolution_sha = _validate_sha256("resolution_payload_sha256", resolution_payload_sha256) + selected_side = str(fill_record["selected_side"]) + won = selected_side == outcome + vwap = Decimal(str(fill_record["execution_vwap"])) + fee = Decimal(str(fill_record["fee_per_share"])) + shares = Decimal(str(fill_record["filled_shares"])) + pnl_per_share = Decimal(int(won)) - vwap - fee + return { + "event_type": "resolution", + "condition_id": str(fill_record["condition_id"]), + "market_open_epoch_seconds": int(fill_record["market_open_epoch_seconds"]), + "policy_sha256": str(fill_record["policy_sha256"]), + "source_sha256": str(fill_record["source_sha256"]), + "selected_side": selected_side, + "official_outcome": outcome, + "official_won": won, + "resolution_payload_sha256": resolution_sha, + "execution_vwap": _decimal_text(vwap), + "fee_per_share": _decimal_text(fee), + "pnl_per_share": _decimal_text(pnl_per_share), + "pnl_total": _decimal_text(pnl_per_share * shares), + } + + +_ALLOWED_EVENT_TYPES = {"discovered", "signal", "arrival", "resolution"} + + +class ProspectiveLifecycleLedger: + def __init__(self, path: str | Path, policy: FrozenPolicy) -> None: + self.path = Path(path) + self.policy = policy + self.records: list[dict[str, Any]] = [] + self.head_hash = "0" * 64 + self._events: dict[str, list[str]] = {} + self._payloads: dict[str, dict[str, Mapping[str, Any]]] = {} + if self.path.exists(): + self._load_and_verify() + + def _validate_order(self, condition_id: str, event_type: str, payload: Mapping[str, Any]) -> None: + sequence = self._events.get(condition_id, []) + if event_type in sequence: + raise LifecycleError(f"duplicate {event_type} event for {condition_id}") + if event_type == "discovered": + if sequence: + raise LifecycleError("discovered must be the first event") + return + if not sequence or sequence[0] != "discovered": + raise LifecycleError(f"{event_type} requires a discovered event") + if event_type == "signal": + if sequence != ["discovered"]: + raise LifecycleError("signal must follow discovered") + return + signal_payload = self._payloads.get(condition_id, {}).get("signal", {}) + if event_type == "arrival": + if sequence != ["discovered", "signal"]: + raise LifecycleError("arrival must follow signal") + if signal_payload.get("decision") != "signal": + raise LifecycleError("arrival is invalid after a no-signal decision") + return + if event_type == "resolution": + if signal_payload.get("decision") == "signal": + if "arrival" not in sequence: + raise LifecycleError("resolution requires arrival after an active signal") + elif sequence != ["discovered", "signal"]: + raise LifecycleError("no-signal resolution must follow signal") + return + raise LifecycleError(f"unsupported event_type: {event_type}") + + def _accept_record(self, record: Mapping[str, Any]) -> None: + condition = str(record["condition_id"]) + event_type = str(record["event_type"]) + payload = record["payload"] + self._validate_order(condition, event_type, payload) + self._events.setdefault(condition, []).append(event_type) + self._payloads.setdefault(condition, {})[event_type] = payload + self.records.append(dict(record)) + self.head_hash = str(record["record_hash"]) + + def _load_and_verify(self) -> None: + previous = "0" * 64 + for line_number, line in enumerate(self.path.read_text(encoding="utf-8").splitlines(), start=1): + if not line.strip(): + continue + try: + record = json.loads(line) + except json.JSONDecodeError as exc: + raise LedgerIntegrityError(f"invalid JSON at line {line_number}") from exc + stored = str(record.get("record_hash", "")) + payload_for_hash = dict(record) + payload_for_hash.pop("record_hash", None) + if payload_for_hash.get("previous_hash") != previous: + raise LedgerIntegrityError(f"hash chain mismatch at line {line_number}") + expected = hashlib.sha256(_canonical_json(payload_for_hash)).hexdigest() + if stored != expected: + raise LedgerIntegrityError(f"record hash mismatch at line {line_number}") + if record.get("policy_sha256") != self.policy.policy_sha256: + raise LedgerIntegrityError(f"policy hash mismatch at line {line_number}") + if record.get("source_sha256") != self.policy.source_sha256: + raise LedgerIntegrityError(f"source hash mismatch at line {line_number}") + try: + _assert_post_freeze(self.policy, int(record["market_open_epoch_seconds"])) + self._accept_record(record) + except (FreezeViolation, LifecycleError, KeyError, TypeError, ValueError) as exc: + raise LedgerIntegrityError(f"invalid lifecycle record at line {line_number}") from exc + previous = stored + + def append( + self, + *, + condition_id: str, + market_open_epoch_seconds: int, + event_type: str, + observed_ts_ms: int, + payload: Mapping[str, Any], + ) -> dict[str, Any]: + condition = str(condition_id) + if not condition: + raise ValueError("condition_id must be non-empty") + opening = _assert_post_freeze(self.policy, market_open_epoch_seconds) + event = str(event_type) + if event not in _ALLOWED_EVENT_TYPES: + raise LifecycleError(f"unsupported event_type: {event}") + observed = int(observed_ts_ms) + if observed < 0: + raise ValueError("observed_ts_ms must be non-negative") + try: + canonical_payload = json.loads(_canonical_json(dict(payload)).decode("utf-8")) + except (TypeError, ValueError) as exc: + raise ValueError("payload must be JSON serializable") from exc + self._validate_order(condition, event, canonical_payload) + payload_sha256 = hashlib.sha256(_canonical_json(canonical_payload)).hexdigest() + event_id = hashlib.sha256( + f"{condition}\0{event}\0{observed}\0{payload_sha256}".encode("utf-8") + ).hexdigest() + record_without_hash = { + "condition_id": condition, + "event_id": event_id, + "event_type": event, + "market_open_epoch_seconds": opening, + "observed_ts_ms": observed, + "payload": canonical_payload, + "payload_sha256": payload_sha256, + "policy_sha256": self.policy.policy_sha256, + "previous_hash": self.head_hash, + "source_sha256": self.policy.source_sha256, + } + record = { + **record_without_hash, + "record_hash": hashlib.sha256(_canonical_json(record_without_hash)).hexdigest(), + } + self.path.parent.mkdir(parents=True, exist_ok=True) + with self.path.open("a", encoding="utf-8", newline="\n") as handle: + handle.write(json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n") + handle.flush() + self._accept_record(record) + return dict(record) diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_capture_policy.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_capture_policy.py new file mode 100644 index 00000000..af061a05 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_capture_policy.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +from decimal import Decimal + +import pytest + +from capture_policy import ( + CaptureBoundLifecycleLedger, + CapturePolicy, + CaptureWindowError, + evaluate_arrival_capture, + evaluate_signal_capture, +) +from late_favorite_v3 import FrozenPolicy + + +def trading_policy() -> FrozenPolicy: + return FrozenPolicy( + policy_sha256="1" * 64, + source_sha256="2" * 64, + valid_market_open_after_epoch_seconds=1_800_000_000, + signal_ask_min=Decimal("0.85"), + entry_second=210, + latency_seconds=1, + shares=Decimal("5"), + adverse_move_cancel=Decimal("0.01"), + fee_rate=Decimal("0.07"), + ) + + +def capture_policy() -> CapturePolicy: + return CapturePolicy( + capture_policy_sha256="3" * 64, + valid_market_open_after_epoch_seconds=1_800_000_100, + signal_target_offset_ms=210_000, + arrival_target_offset_ms=211_000, + max_request_start_lateness_ms=250, + max_request_duration_ms=1_000, + max_book_age_at_request_completion_ms=2_000, + max_book_future_skew_ms=1_000, + ) + + +def book(condition_id: str, token_id: str, ask: str, *, timestamp_ms: int) -> dict[str, object]: + return { + "market": condition_id, + "asset_id": token_id, + "timestamp": str(timestamp_ms), + "hash": f"book-{token_id}-{timestamp_ms}", + "bids": [{"price": "0.10", "size": "10"}], + "asks": [{"price": ask, "size": "10"}], + "min_order_size": "1", + "tick_size": "0.01", + "neg_risk": False, + "last_trade_price": "0.50", + } + + +def test_signal_capture_rejects_pre_target_request() -> None: + opening = 1_800_000_300 + target = opening * 1000 + 210_000 + with pytest.raises(CaptureWindowError): + evaluate_signal_capture( + trading_policy(), capture_policy(), condition_id="c1", + market_open_epoch_seconds=opening, up_token_id="up", down_token_id="down", + up_book=book("c1", "up", "0.90", timestamp_ms=target), + down_book=book("c1", "down", "0.12", timestamp_ms=target), + request_started_ts_ms=target - 1, request_completed_ts_ms=target + 20, + response_payload_sha256="4" * 64, + ) + + +def test_signal_capture_records_late_and_stale_fail_closed() -> None: + opening = 1_800_000_300 + target = opening * 1000 + 210_000 + late = evaluate_signal_capture( + trading_policy(), capture_policy(), condition_id="c1", + market_open_epoch_seconds=opening, up_token_id="up", down_token_id="down", + up_book=book("c1", "up", "0.90", timestamp_ms=target), + down_book=book("c1", "down", "0.12", timestamp_ms=target), + request_started_ts_ms=target + 251, request_completed_ts_ms=target + 300, + response_payload_sha256="4" * 64, + ) + assert late["decision"] == "missed_signal_window" + assert late["signal"] is False + stale = evaluate_signal_capture( + trading_policy(), capture_policy(), condition_id="c1", + market_open_epoch_seconds=opening, up_token_id="up", down_token_id="down", + up_book=book("c1", "up", "0.90", timestamp_ms=target - 2_001), + down_book=book("c1", "down", "0.12", timestamp_ms=target), + request_started_ts_ms=target, request_completed_ts_ms=target + 100, + response_payload_sha256="4" * 64, + ) + assert stale["decision"] == "no_signal_stale_book" + assert stale["signal"] is False + + +def test_valid_signal_capture_preserves_target_and_transport_evidence() -> None: + opening = 1_800_000_300 + target = opening * 1000 + 210_000 + result = evaluate_signal_capture( + trading_policy(), capture_policy(), condition_id="c1", + market_open_epoch_seconds=opening, up_token_id="up", down_token_id="down", + up_book=book("c1", "up", "0.90", timestamp_ms=target + 10), + down_book=book("c1", "down", "0.12", timestamp_ms=target + 10), + request_started_ts_ms=target + 25, request_completed_ts_ms=target + 100, + response_payload_sha256="4" * 64, + ) + assert result["decision"] == "signal" + assert result["observed_ts_ms"] == target + assert result["target_ts_ms"] == target + assert result["request_started_ts_ms"] == target + 25 + assert result["request_completed_ts_ms"] == target + 100 + assert result["capture_policy_sha256"] == "3" * 64 + assert result["response_payload_sha256"] == "4" * 64 + + +def test_arrival_target_is_market_open_plus_211_seconds() -> None: + opening = 1_800_000_300 + signal_target = opening * 1000 + 210_000 + arrival_target = opening * 1000 + 211_000 + signal = evaluate_signal_capture( + trading_policy(), capture_policy(), condition_id="c1", + market_open_epoch_seconds=opening, up_token_id="up", down_token_id="down", + up_book=book("c1", "up", "0.90", timestamp_ms=signal_target), + down_book=book("c1", "down", "0.12", timestamp_ms=signal_target), + request_started_ts_ms=signal_target + 200, request_completed_ts_ms=signal_target + 900, + response_payload_sha256="4" * 64, + ) + arrival = evaluate_arrival_capture( + trading_policy(), capture_policy(), signal, + book("c1", "up", "0.90", timestamp_ms=arrival_target), + request_started_ts_ms=arrival_target, request_completed_ts_ms=arrival_target + 100, + response_payload_sha256="5" * 64, + ) + assert arrival["decision"] == "hypothetical_fok_fill" + assert arrival["target_ts_ms"] == arrival_target + assert arrival["observed_ts_ms"] == arrival_target + + +def test_lifecycle_record_binds_capture_policy_hash(tmp_path) -> None: + ledger = CaptureBoundLifecycleLedger( + tmp_path / "ledger.jsonl", trading_policy(), capture_policy() + ) + row = ledger.append( + condition_id="c1", market_open_epoch_seconds=1_800_000_300, + event_type="discovered", observed_ts_ms=1_800_000_301_000, + payload={"slug": "btc-updown-5m-1800000300"}, + ) + assert row["capture_policy_sha256"] == "3" * 64 + reloaded = CaptureBoundLifecycleLedger( + tmp_path / "ledger.jsonl", trading_policy(), capture_policy() + ) + assert reloaded.head_hash == ledger.head_hash diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_cli.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_cli.py new file mode 100644 index 00000000..965ac2d7 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_cli.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from cli import capture_once, resolve_once +from public_collector import PublicHttpClient + + +class FakeResponse: + def __init__(self, payload: object) -> None: + self.content = json.dumps(payload, separators=(",", ":")).encode() + self.status_code = 200 + + +class FakeRequester: + def __init__(self, responses: list[FakeResponse]) -> None: + self.responses = list(responses) + self.calls: list[dict[str, object]] = [] + + def __call__(self, method: str, url: str, **kwargs: object) -> FakeResponse: + self.calls.append({"method": method, "url": url, **kwargs}) + return self.responses.pop(0) + + +def market_payload(opening: int) -> dict[str, object]: + return { + "conditionId": "condition-1", + "slug": f"btc-updown-5m-{opening}", + "active": True, + "closed": False, + "enableOrderBook": True, + "outcomes": json.dumps(["Up", "Down"]), + "clobTokenIds": json.dumps(["token-up", "token-down"]), + } + + +def terminal_payload(opening: int) -> dict[str, object]: + return { + "conditionId": "condition-1", + "slug": f"btc-updown-5m-{opening}", + "closed": True, + "outcomes": json.dumps(["Up", "Down"]), + "outcomePrices": json.dumps(["1", "0"]), + } + + +def book(token: str, ask: str, timestamp_ms: int) -> dict[str, object]: + return { + "market": "condition-1", + "asset_id": token, + "timestamp": str(timestamp_ms), + "hash": f"hash-{token}-{timestamp_ms}", + "bids": [{"price": "0.10", "size": "10"}], + "asks": [{"price": ask, "size": "10"}], + "min_order_size": "1", + "tick_size": "0.01", + "neg_risk": False, + "last_trade_price": "0.50", + } + + +def test_capture_and_resolve_persist_complete_lifecycle(tmp_path: Path) -> None: + opening = 1_800_000_300 + signal_target = opening * 1000 + 210_000 + arrival_target = opening * 1000 + 211_000 + requester = FakeRequester([ + FakeResponse(market_payload(opening)), + FakeResponse([ + book("token-up", "0.90", signal_target), + book("token-down", "0.12", signal_target), + ]), + FakeResponse([book("token-up", "0.90", arrival_target)]), + FakeResponse(terminal_payload(opening)), + ]) + times = iter([ + opening * 1000 + 1_000, opening * 1000 + 1_020, + signal_target, signal_target + 100, + arrival_target, arrival_target + 100, + opening * 1000 + 301_000, opening * 1000 + 301_050, + ]) + client = PublicHttpClient(requester=requester, clock_ms=lambda: next(times)) + slept: list[int] = [] + root = Path(__file__).parents[1] + + capture = capture_once( + root=root, + output_dir=tmp_path, + market_open_epoch_seconds=opening, + client=client, + sleep_until_ms=slept.append, + ) + assert capture["decision"] == "hypothetical_fok_fill" + assert capture["condition_id"] == "condition-1" + assert slept == [signal_target, arrival_target] + assert (tmp_path / "lifecycle.jsonl").is_file() + assert (tmp_path / "raw" / "manifest.jsonl").is_file() + assert json.loads((tmp_path / "last_capture.json").read_text())["decision"] == "hypothetical_fok_fill" + + resolution = resolve_once( + root=root, + output_dir=tmp_path, + condition_id="condition-1", + client=client, + ) + assert resolution["decision"] == "resolved_fill" + assert resolution["official_outcome"] == "Up" + assert resolution["official_won"] is True + assert json.loads((tmp_path / "last_resolution.json").read_text())["decision"] == "resolved_fill" + rows = [json.loads(line) for line in (tmp_path / "lifecycle.jsonl").read_text().splitlines()] + assert [row["event_type"] for row in rows] == ["discovered", "signal", "arrival", "resolution"] + + +def test_capture_selects_next_eligible_window_when_opening_is_omitted(tmp_path: Path) -> None: + now_ms = 1_800_000_300_000 + 210_001 + opening = 1_800_000_600 + signal_target = opening * 1000 + 210_000 + requester = FakeRequester([ + FakeResponse(market_payload(opening)), + FakeResponse([ + book("token-up", "0.84", signal_target), + book("token-down", "0.18", signal_target), + ]), + ]) + times = iter([ + opening * 1000 + 1_000, opening * 1000 + 1_020, + signal_target, signal_target + 100, + ]) + client = PublicHttpClient(requester=requester, clock_ms=lambda: next(times)) + root = Path(__file__).parents[1] + result = capture_once( + root=root, + output_dir=tmp_path, + market_open_epoch_seconds=None, + now_ms=lambda: now_ms, + client=client, + sleep_until_ms=lambda _: None, + ) + assert result["market_open_epoch_seconds"] == opening + assert result["decision"] == "no_signal_below_threshold" diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_late_favorite_v3.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_late_favorite_v3.py new file mode 100644 index 00000000..94b519a7 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_late_favorite_v3.py @@ -0,0 +1,258 @@ +from __future__ import annotations + +import json +from decimal import Decimal + +import pytest + +from late_favorite_v3 import ( + FreezeViolation, + FrozenPolicy, + LedgerIntegrityError, + LifecycleError, + ProspectiveLifecycleLedger, + evaluate_arrival, + evaluate_signal, + settle_fill, +) + + +def policy() -> FrozenPolicy: + return FrozenPolicy( + policy_sha256="1" * 64, + source_sha256="2" * 64, + valid_market_open_after_epoch_seconds=1_800_000_000, + signal_ask_min=Decimal("0.85"), + entry_second=210, + latency_seconds=1, + shares=Decimal("5"), + adverse_move_cancel=Decimal("0.01"), + fee_rate=Decimal("0.07"), + ) + + +def book(condition_id: str, token_id: str, asks: list[tuple[str, str]]) -> dict[str, object]: + return { + "market": condition_id, + "asset_id": token_id, + "timestamp": "1800000210000", + "hash": f"book-{token_id}", + "bids": [{"price": "0.10", "size": "10"}], + "asks": [{"price": price, "size": size} for price, size in asks], + "min_order_size": "1", + "tick_size": "0.01", + "neg_risk": False, + "last_trade_price": "0.50", + } + + +def test_market_must_open_strictly_after_freeze() -> None: + with pytest.raises(FreezeViolation): + evaluate_signal( + policy(), + condition_id="c1", + market_open_epoch_seconds=1_800_000_000, + up_token_id="up", + down_token_id="down", + up_book=book("c1", "up", [("0.90", "10")]), + down_book=book("c1", "down", [("0.12", "10")]), + observed_ts_ms=1_800_000_210_000, + ) + + +def test_signal_selects_higher_executable_ask_as_favorite() -> None: + result = evaluate_signal( + policy(), + condition_id="c1", + market_open_epoch_seconds=1_800_000_300, + up_token_id="up", + down_token_id="down", + up_book=book("c1", "up", [("0.90", "10")]), + down_book=book("c1", "down", [("0.12", "10")]), + observed_ts_ms=1_800_000_510_000, + ) + assert result["decision"] == "signal" + assert result["selected_side"] == "Up" + assert result["selected_token_id"] == "up" + assert result["signal_ask"] == "0.90" + assert result["signal"] is True + assert len(result["up_book_sha256"]) == 64 + assert len(result["down_book_sha256"]) == 64 + + +def test_tied_or_subthreshold_favorite_fails_closed() -> None: + tied = evaluate_signal( + policy(), + condition_id="c1", + market_open_epoch_seconds=1_800_000_300, + up_token_id="up", + down_token_id="down", + up_book=book("c1", "up", [("0.50", "10")]), + down_book=book("c1", "down", [("0.50", "10")]), + observed_ts_ms=1_800_000_510_000, + ) + assert tied["decision"] == "no_signal_tied_favorite" + + below = evaluate_signal( + policy(), + condition_id="c2", + market_open_epoch_seconds=1_800_000_300, + up_token_id="up2", + down_token_id="down2", + up_book=book("c2", "up2", [("0.84", "10")]), + down_book=book("c2", "down2", [("0.18", "10")]), + observed_ts_ms=1_800_000_510_000, + ) + assert below["decision"] == "no_signal_below_threshold" + + +def test_arrival_cancels_toxic_improvement_and_rejects_price_above_limit() -> None: + signal = evaluate_signal( + policy(), + condition_id="c1", + market_open_epoch_seconds=1_800_000_300, + up_token_id="up", + down_token_id="down", + up_book=book("c1", "up", [("0.90", "10")]), + down_book=book("c1", "down", [("0.12", "10")]), + observed_ts_ms=1_800_000_510_000, + ) + toxic = evaluate_arrival( + policy(), signal, book("c1", "up", [("0.88", "10")]), observed_ts_ms=1_800_000_511_000 + ) + assert toxic["decision"] == "cancel_adverse_move" + + above = evaluate_arrival( + policy(), signal, book("c1", "up", [("0.91", "10")]), observed_ts_ms=1_800_000_511_000 + ) + assert above["decision"] == "no_fill_ask_above_limit" + + +def test_arrival_uses_full_depth_vwap_and_level_fees() -> None: + signal = evaluate_signal( + policy(), + condition_id="c1", + market_open_epoch_seconds=1_800_000_300, + up_token_id="up", + down_token_id="down", + up_book=book("c1", "up", [("0.90", "10")]), + down_book=book("c1", "down", [("0.12", "10")]), + observed_ts_ms=1_800_000_510_000, + ) + fill = evaluate_arrival( + policy(), + signal, + book("c1", "up", [("0.89", "2"), ("0.90", "3"), ("0.91", "100")]), + observed_ts_ms=1_800_000_511_000, + ) + assert fill["decision"] == "hypothetical_fok_fill" + assert fill["filled_shares"] == "5" + assert fill["execution_vwap"] == "0.896" + expected_fee = ( + Decimal("2") * Decimal("0.07") * Decimal("0.89") * Decimal("0.11") + + Decimal("3") * Decimal("0.07") * Decimal("0.90") * Decimal("0.10") + ) / Decimal("5") + assert Decimal(fill["fee_per_share"]) == expected_fee + assert fill["execution_levels"] == [ + {"price": "0.89", "size": "2"}, + {"price": "0.90", "size": "3"}, + ] + + +def test_arrival_requires_exact_latency_and_sufficient_depth() -> None: + signal = evaluate_signal( + policy(), + condition_id="c1", + market_open_epoch_seconds=1_800_000_300, + up_token_id="up", + down_token_id="down", + up_book=book("c1", "up", [("0.90", "10")]), + down_book=book("c1", "down", [("0.12", "10")]), + observed_ts_ms=1_800_000_510_000, + ) + with pytest.raises(LifecycleError): + evaluate_arrival( + policy(), signal, book("c1", "up", [("0.90", "10")]), observed_ts_ms=1_800_000_512_000 + ) + shallow = evaluate_arrival( + policy(), signal, book("c1", "up", [("0.90", "4.99")]), observed_ts_ms=1_800_000_511_000 + ) + assert shallow["decision"] == "no_fill_insufficient_depth" + + +def test_official_settlement_uses_frozen_fill_economics() -> None: + signal = evaluate_signal( + policy(), + condition_id="c1", + market_open_epoch_seconds=1_800_000_300, + up_token_id="up", + down_token_id="down", + up_book=book("c1", "up", [("0.90", "10")]), + down_book=book("c1", "down", [("0.12", "10")]), + observed_ts_ms=1_800_000_510_000, + ) + fill = evaluate_arrival( + policy(), signal, book("c1", "up", [("0.90", "5")]), observed_ts_ms=1_800_000_511_000 + ) + win = settle_fill(fill, official_outcome="Up", resolution_payload_sha256="3" * 64) + loss = settle_fill(fill, official_outcome="Down", resolution_payload_sha256="4" * 64) + assert Decimal(win["pnl_per_share"]) > 0 + assert Decimal(loss["pnl_per_share"]) < Decimal("-0.90") + assert win["official_won"] is True + assert loss["official_won"] is False + + +def test_lifecycle_ledger_is_append_only_and_enforces_order(tmp_path) -> None: + path = tmp_path / "ledger.jsonl" + ledger = ProspectiveLifecycleLedger(path, policy()) + discovered = ledger.append( + condition_id="c1", + market_open_epoch_seconds=1_800_000_300, + event_type="discovered", + observed_ts_ms=1_800_000_301_000, + payload={"slug": "btc-updown-5m-1800000300"}, + ) + assert discovered["previous_hash"] == "0" * 64 + with pytest.raises(LifecycleError): + ledger.append( + condition_id="c2", + market_open_epoch_seconds=1_800_000_300, + event_type="arrival", + observed_ts_ms=1_800_000_511_000, + payload={"decision": "no_fill"}, + ) + ledger.append( + condition_id="c1", + market_open_epoch_seconds=1_800_000_300, + event_type="signal", + observed_ts_ms=1_800_000_510_000, + payload={"decision": "signal"}, + ) + with pytest.raises(LifecycleError): + ledger.append( + condition_id="c1", + market_open_epoch_seconds=1_800_000_300, + event_type="signal", + observed_ts_ms=1_800_000_510_001, + payload={"decision": "signal"}, + ) + reloaded = ProspectiveLifecycleLedger(path, policy()) + assert reloaded.head_hash == ledger.head_hash + assert len(reloaded.records) == 2 + + +def test_lifecycle_ledger_detects_tampering(tmp_path) -> None: + path = tmp_path / "ledger.jsonl" + ledger = ProspectiveLifecycleLedger(path, policy()) + ledger.append( + condition_id="c1", + market_open_epoch_seconds=1_800_000_300, + event_type="discovered", + observed_ts_ms=1_800_000_301_000, + payload={"slug": "btc-updown-5m-1800000300"}, + ) + row = json.loads(path.read_text()) + row["payload"]["slug"] = "tampered" + path.write_text(json.dumps(row) + "\n") + with pytest.raises(LedgerIntegrityError): + ProspectiveLifecycleLedger(path, policy()) diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_public_collector.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_public_collector.py new file mode 100644 index 00000000..15d47791 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_public_collector.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +import json +from decimal import Decimal + +import pytest + +from capture_policy import CaptureBoundLifecycleLedger, CapturePolicy +from late_favorite_v3 import FrozenPolicy +from public_collector import ( + PublicEndpointViolation, + PublicHttpClient, + RawEvidenceStore, + SingleMarketCollector, + next_collectable_open, + parse_btc_five_minute_market, +) + + +class FakeResponse: + def __init__(self, payload: object, status_code: int = 200) -> None: + self.content = json.dumps(payload, separators=(",", ":")).encode() + self.status_code = status_code + + +class FakeRequester: + def __init__(self, responses: list[FakeResponse]) -> None: + self.responses = list(responses) + self.calls: list[dict[str, object]] = [] + + def __call__(self, method: str, url: str, **kwargs: object) -> FakeResponse: + self.calls.append({"method": method, "url": url, **kwargs}) + return self.responses.pop(0) + + +def trading_policy() -> FrozenPolicy: + return FrozenPolicy( + policy_sha256="1" * 64, + source_sha256="2" * 64, + valid_market_open_after_epoch_seconds=1_800_000_000, + signal_ask_min=Decimal("0.85"), + entry_second=210, + latency_seconds=1, + shares=Decimal("5"), + adverse_move_cancel=Decimal("0.01"), + fee_rate=Decimal("0.07"), + ) + + +def capture_policy() -> CapturePolicy: + return CapturePolicy( + capture_policy_sha256="3" * 64, + valid_market_open_after_epoch_seconds=1_800_000_100, + signal_target_offset_ms=210_000, + arrival_target_offset_ms=211_000, + max_request_start_lateness_ms=250, + max_request_duration_ms=1_000, + max_book_age_at_request_completion_ms=2_000, + max_book_future_skew_ms=1_000, + ) + + +def market_payload(opening: int) -> dict[str, object]: + return { + "conditionId": "condition-1", + "slug": f"btc-updown-5m-{opening}", + "active": True, + "closed": False, + "enableOrderBook": True, + "outcomes": json.dumps(["Up", "Down"]), + "clobTokenIds": json.dumps(["token-up", "token-down"]), + } + + +def book(condition: str, token: str, ask: str, timestamp_ms: int) -> dict[str, object]: + return { + "market": condition, + "asset_id": token, + "timestamp": str(timestamp_ms), + "hash": f"hash-{token}-{timestamp_ms}", + "bids": [{"price": "0.10", "size": "10"}], + "asks": [{"price": ask, "size": "10"}], + "min_order_size": "1", + "tick_size": "0.01", + "neg_risk": False, + "last_trade_price": "0.50", + } + + +def test_parse_market_maps_up_and_down_tokens() -> None: + market = parse_btc_five_minute_market(market_payload(1_800_000_300), 1_800_000_300) + assert market.condition_id == "condition-1" + assert market.up_token_id == "token-up" + assert market.down_token_id == "token-down" + + +def test_public_http_client_allows_only_read_only_public_endpoints() -> None: + requester = FakeRequester([FakeResponse({"ok": True})]) + clock = iter([1000, 1025]) + client = PublicHttpClient(requester=requester, clock_ms=lambda: next(clock)) + evidence = client.request_json( + "GET", "https://gamma-api.polymarket.com/markets/slug/test" + ) + assert evidence.status_code == 200 + call = requester.calls[0] + assert "Authorization" not in call["headers"] + assert not any(str(key).startswith("POLY_") for key in call["headers"]) + with pytest.raises(PublicEndpointViolation): + client.request_json("POST", "https://clob.polymarket.com/order", json_body={}) + + +def test_raw_evidence_store_is_hash_chained_and_tamper_evident(tmp_path) -> None: + requester = FakeRequester([FakeResponse({"value": 1})]) + clock = iter([1000, 1010]) + evidence = PublicHttpClient(requester=requester, clock_ms=lambda: next(clock)).request_json( + "GET", "https://gamma-api.polymarket.com/markets/slug/test" + ) + store = RawEvidenceStore(tmp_path) + row = store.append(evidence, purpose="market_discovery") + assert len(row["record_hash"]) == 64 + assert RawEvidenceStore(tmp_path).head_hash == row["record_hash"] + body = tmp_path / row["body_path"] + body.write_bytes(b"tampered") + with pytest.raises(RuntimeError): + RawEvidenceStore(tmp_path) + + +def test_next_collectable_open_uses_current_window_only_before_signal_target() -> None: + opening = 1_800_000_300 + assert next_collectable_open(opening * 1000 + 209_000, 1_800_000_100) == opening + assert next_collectable_open(opening * 1000 + 210_001, 1_800_000_100) == opening + 300 + + +def test_single_market_collector_records_discovery_signal_and_arrival(tmp_path) -> None: + opening = 1_800_000_300 + signal_target = opening * 1000 + 210_000 + arrival_target = opening * 1000 + 211_000 + responses = [ + FakeResponse(market_payload(opening)), + FakeResponse([ + book("condition-1", "token-up", "0.90", signal_target + 10), + book("condition-1", "token-down", "0.12", signal_target + 10), + ]), + FakeResponse([book("condition-1", "token-up", "0.90", arrival_target + 10)]), + ] + times = iter([ + opening * 1000 + 1_000, opening * 1000 + 1_020, + signal_target + 20, signal_target + 100, + arrival_target + 20, arrival_target + 100, + ]) + requester = FakeRequester(responses) + client = PublicHttpClient(requester=requester, clock_ms=lambda: next(times)) + ledger = CaptureBoundLifecycleLedger( + tmp_path / "lifecycle.jsonl", trading_policy(), capture_policy() + ) + store = RawEvidenceStore(tmp_path / "raw") + slept: list[int] = [] + collector = SingleMarketCollector( + client=client, + evidence_store=store, + ledger=ledger, + trading_policy=trading_policy(), + capture_policy=capture_policy(), + sleep_until_ms=slept.append, + ) + result = collector.capture(opening) + assert result["decision"] == "hypothetical_fok_fill" + assert [row["event_type"] for row in ledger.records] == ["discovered", "signal", "arrival"] + assert slept == [signal_target, arrival_target] + assert len(store.records) == 3 + assert requester.calls[1]["method"] == "POST" + assert requester.calls[1]["url"] == "https://clob.polymarket.com/books" + assert requester.calls[1]["json"] == [ + {"token_id": "token-up"}, {"token_id": "token-down"} + ] diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_resolution.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_resolution.py new file mode 100644 index 00000000..2fd2c815 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_resolution.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +import json +from decimal import Decimal + +import pytest + +from capture_policy import CaptureBoundLifecycleLedger, CapturePolicy +from late_favorite_v3 import FrozenPolicy +from public_collector import PublicHttpClient, RawEvidenceStore, SingleMarketCollector +from resolution import SingleMarketResolver, parse_terminal_outcome + + +class FakeResponse: + def __init__(self, payload: object) -> None: + self.content = json.dumps(payload, separators=(",", ":")).encode() + self.status_code = 200 + + +class FakeRequester: + def __init__(self, responses: list[FakeResponse]) -> None: + self.responses = list(responses) + + def __call__(self, method: str, url: str, **kwargs: object) -> FakeResponse: + return self.responses.pop(0) + + +def trading_policy() -> FrozenPolicy: + return FrozenPolicy( + policy_sha256="1" * 64, + source_sha256="2" * 64, + valid_market_open_after_epoch_seconds=1_800_000_000, + signal_ask_min=Decimal("0.85"), + entry_second=210, + latency_seconds=1, + shares=Decimal("5"), + adverse_move_cancel=Decimal("0.01"), + fee_rate=Decimal("0.07"), + ) + + +def capture_policy() -> CapturePolicy: + return CapturePolicy( + capture_policy_sha256="3" * 64, + valid_market_open_after_epoch_seconds=1_800_000_100, + signal_target_offset_ms=210_000, + arrival_target_offset_ms=211_000, + max_request_start_lateness_ms=250, + max_request_duration_ms=1_000, + max_book_age_at_request_completion_ms=2_000, + max_book_future_skew_ms=1_000, + ) + + +def terminal_payload(opening: int, *, winner: str = "Up") -> dict[str, object]: + prices = ["1", "0"] if winner == "Up" else ["0", "1"] + return { + "conditionId": "condition-1", + "slug": f"btc-updown-5m-{opening}", + "closed": True, + "outcomes": json.dumps(["Up", "Down"]), + "outcomePrices": json.dumps(prices), + } + + +def test_parse_terminal_outcome_requires_closed_binary_terminal_prices() -> None: + opening = 1_800_000_300 + assert parse_terminal_outcome(terminal_payload(opening, winner="Down"), opening, "condition-1") == "Down" + not_closed = terminal_payload(opening) + not_closed["closed"] = False + with pytest.raises(ValueError): + parse_terminal_outcome(not_closed, opening, "condition-1") + ambiguous = terminal_payload(opening) + ambiguous["outcomePrices"] = json.dumps(["0.5", "0.5"]) + with pytest.raises(ValueError): + parse_terminal_outcome(ambiguous, opening, "condition-1") + + +def test_resolver_appends_official_fill_settlement(tmp_path) -> None: + opening = 1_800_000_300 + signal_target = opening * 1000 + 210_000 + arrival_target = opening * 1000 + 211_000 + market = { + "conditionId": "condition-1", "slug": f"btc-updown-5m-{opening}", + "active": True, "closed": False, "enableOrderBook": True, + "outcomes": json.dumps(["Up", "Down"]), + "clobTokenIds": json.dumps(["token-up", "token-down"]), + } + def book(token: str, ask: str, timestamp: int) -> dict[str, object]: + return { + "market": "condition-1", "asset_id": token, "timestamp": str(timestamp), + "hash": f"hash-{token}", "bids": [{"price": "0.10", "size": "10"}], + "asks": [{"price": ask, "size": "10"}], "min_order_size": "1", + "tick_size": "0.01", "neg_risk": False, "last_trade_price": "0.50", + } + responses = [ + FakeResponse(market), + FakeResponse([book("token-up", "0.90", signal_target), book("token-down", "0.12", signal_target)]), + FakeResponse([book("token-up", "0.90", arrival_target)]), + FakeResponse(terminal_payload(opening, winner="Up")), + ] + times = iter([ + opening * 1000 + 1000, opening * 1000 + 1020, + signal_target, signal_target + 100, + arrival_target, arrival_target + 100, + opening * 1000 + 301_000, opening * 1000 + 301_050, + ]) + client = PublicHttpClient(requester=FakeRequester(responses), clock_ms=lambda: next(times)) + ledger = CaptureBoundLifecycleLedger(tmp_path / "ledger.jsonl", trading_policy(), capture_policy()) + store = RawEvidenceStore(tmp_path / "raw") + collector = SingleMarketCollector( + client=client, evidence_store=store, ledger=ledger, + trading_policy=trading_policy(), capture_policy=capture_policy(), + sleep_until_ms=lambda _: None, + ) + assert collector.capture(opening)["decision"] == "hypothetical_fok_fill" + resolver = SingleMarketResolver( + client=client, evidence_store=store, ledger=ledger, + trading_policy=trading_policy(), capture_policy=capture_policy(), + ) + resolution = resolver.resolve("condition-1") + assert resolution["official_outcome"] == "Up" + assert resolution["official_won"] is True + assert Decimal(resolution["pnl_total"]) > 0 + assert ledger.records[-1]["event_type"] == "resolution" + assert len(store.records) == 4 + + +def test_resolver_closes_no_signal_market_without_pnl(tmp_path) -> None: + opening = 1_800_000_300 + ledger = CaptureBoundLifecycleLedger(tmp_path / "ledger.jsonl", trading_policy(), capture_policy()) + ledger.append( + condition_id="condition-1", market_open_epoch_seconds=opening, + event_type="discovered", observed_ts_ms=opening * 1000 + 1000, + payload={"slug": f"btc-updown-5m-{opening}"}, + ) + ledger.append( + condition_id="condition-1", market_open_epoch_seconds=opening, + event_type="signal", observed_ts_ms=opening * 1000 + 210_000, + payload={"decision": "no_signal_below_threshold", "signal": False}, + ) + times = iter([opening * 1000 + 301_000, opening * 1000 + 301_050]) + client = PublicHttpClient( + requester=FakeRequester([FakeResponse(terminal_payload(opening, winner="Down"))]), + clock_ms=lambda: next(times), + ) + resolver = SingleMarketResolver( + client=client, evidence_store=RawEvidenceStore(tmp_path / "raw"), ledger=ledger, + trading_policy=trading_policy(), capture_policy=capture_policy(), + ) + resolution = resolver.resolve("condition-1") + assert resolution["decision"] == "resolved_no_fill" + assert resolution["official_outcome"] == "Down" + assert resolution["pnl_total"] is None diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_resolution_cache_bust.py b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_resolution_cache_bust.py new file mode 100644 index 00000000..728770f7 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-btc-only-v3/tests/test_resolution_cache_bust.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from types import SimpleNamespace + +from resolution import SingleMarketResolver + + +class FakeClient: + def __init__(self) -> None: + self.urls: list[str] = [] + + def request_json(self, method: str, url: str): + self.urls.append(url) + return SimpleNamespace( + payload={ + "conditionId": "c1", + "slug": "btc-updown-5m-1800000300", + "closed": True, + "outcomes": '["Up", "Down"]', + "outcomePrices": '["1", "0"]', + }, + response_body_sha256="4" * 64, + request_completed_ts_ms=1_800_000_600_000, + ) + + +class FakeEvidenceStore: + def __init__(self) -> None: + self.records = [object()] * 7 + + def append(self, evidence, *, purpose: str): + return {"record_hash": "5" * 64} + + +class FakeLedger: + def __init__(self) -> None: + self.records = [ + { + "condition_id": "c1", + "market_open_epoch_seconds": 1_800_000_300, + "event_type": "discovered", + "payload": {"slug": "btc-updown-5m-1800000300"}, + }, + { + "condition_id": "c1", + "market_open_epoch_seconds": 1_800_000_300, + "event_type": "signal", + "payload": {"decision": "no_signal_below_threshold", "signal": False}, + }, + ] + self.appended: list[dict[str, object]] = [] + + def append(self, **kwargs) -> None: + self.appended.append(kwargs) + + +def test_resolution_request_cache_busts_gamma_slug_url() -> None: + client = FakeClient() + resolver = SingleMarketResolver( + client=client, + evidence_store=FakeEvidenceStore(), + ledger=FakeLedger(), + trading_policy=SimpleNamespace(policy_sha256="1" * 64, source_sha256="2" * 64), + capture_policy=SimpleNamespace(capture_policy_sha256="3" * 64), + ) + + resolver.resolve("c1") + + assert client.urls == [ + "https://gamma-api.polymarket.com/markets/slug/" + "btc-updown-5m-1800000300?cache_bust=7" + ] diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/BTC_ONLY_ASSESSMENT.md b/ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/BTC_ONLY_ASSESSMENT.md new file mode 100644 index 00000000..86e6b957 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/BTC_ONLY_ASSESSMENT.md @@ -0,0 +1,64 @@ +# BTC-Only Post-Transfer Diagnostic Assessment + +## Classification + +**Marginal prospective lead only. Not admitted. Not a historical pass.** + +BTC was part of the inspected BTC+ETH candidate. Retaining BTC after observing the official asset split is post-hoc. The statistics below may justify low-cost prospective shadowing under a newly frozen version, but they cannot inherit any validation status from the rejected BTC+ETH rule. + +## Fixed-rule official result + +The entry geometry was not changed: second 210, favorite ask at least $0.85, one-second latency, five-share FOK at the signal ask, cancel when arrival ask is more than $0.01 lower, fee `0.07*p*(1-p)`, settlement hold. + +- Fills: **4,844** +- Official wins: **4,587** +- Official win rate: **94.6945%** +- Mean P&L/share: **+$0.00264236** (**0.2642 cents/share**) +- P&L at five shares: **+$64.00** +- Official days: **56** +- Positive days: **30** (53.57%) +- Daily 95% bootstrap interval for mean P&L: **-$1.73 to +$3.93/day** +- Positive calendar weeks: **6 of 9** +- Maximum cumulative P&L: **+$67.92** +- Maximum dollar drawdown: **-$56.15** +- Approximately **13 additional winner-to-loss flips** erase the observed profit. + +## Cost fragility + +| Additional cost/share | BTC P&L at five shares | +|---:|---:| +| 0.00 cents | +$64.00 | +| 0.10 cents | +$39.78 | +| 0.20 cents | +$15.56 | +| 0.25 cents | +$3.45 | +| 0.30 cents | -$8.66 | +| 0.50 cents | -$57.10 | + +The observed edge is therefore too small to tolerate even modest unmodeled execution friction. Public top-of-book crossability is not proof of queue-position or realized FOK fill probability. + +## Stability diagnostics + +Chronological quartile P&L was `[-$11.31, +$11.47, +$24.16, +$39.67]`, so the result was not produced solely by the earliest segment. Every leave-one-week-out total remained positive. However, weekly outcomes still included three meaningful losing weeks, and the daily confidence interval crosses zero. + +Both selected directions were positive under the unchanged rule: + +- Down favorite: +$55.52 across 2,418 fills. +- Up favorite: +$8.47 across 2,426 fills. + +This split must not be used to introduce a new side filter on the inspected corpus. Price-bucket results were also irregular; no bucket optimization is authorized. + +## Decision + +BTC-only is worth **shadow observation only** because the official point estimate is positive and broadly distributed, but the economic margin is nearly exhausted by 0.25 cents/share of additional cost and statistical uncertainty remains material. + +Any continuation must: + +1. use a new version identifier and policy hash; +2. start after the policy is frozen; +3. record every BTC 5-minute window, including no-signal and no-fill outcomes; +4. preserve signal and one-second-arrival full-depth books; +5. resolve every market from authoritative terminal outcomes; +6. measure actual crossability and displayed depth without assuming queue priority; +7. run at least four untouched weekly blocks; +8. require a positive lower confidence bound after fees and measured execution costs; +9. keep real-money submission physically absent. diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/CHECKPOINT.md b/ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/CHECKPOINT.md new file mode 100644 index 00000000..2351f2c7 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/CHECKPOINT.md @@ -0,0 +1,71 @@ +# Late-Favorite BTC/ETH Official-Outcome Decision Checkpoint v1 + +## Verdict + +The frozen `late_favorite_btc_eth_v2_diagnostic` candidate is **Rejected / falsified** after transfer to authoritative Polymarket terminal outcomes. + +- Frozen fills: **9,268** +- Official coverage: **100.00%** +- Official win rate: **94.3893%** +- Official mean P&L/share: **-$0.00127314** +- Official P&L at five shares: **-$59.00** +- Additional-cost headroom: **negative**; even zero additional slippage is unprofitable. + +No threshold, asset, timing, latency, fee, depth, FOK, or adverse-move rule was changed during transfer. Real-money submission remains physically absent. + +## Why the diagnostic result reversed + +The earlier inferred-label diagnostic reported +$428.93 on 8,883 labeled fills. Official resolution shows that the nine direct label disagreements account for only **-$15.00** of the correction. The decisive defect was the excluded cohort: + +- Previously unlabeled fills: **385** (4.15% of fills) +- Official win rate in that cohort: **67.53%** +- Official P&L in that cohort: **-$472.92** +- Losses in that cohort: **125**, or **24.04%** of all official losses +- Date span: **2026-03-25T04:20:00+00:00** through **2026-05-18T09:30:00+00:00** + +The public dataset explicitly labels outcomes by inference from the final recorded book and permits null outcomes near edge cases. Those nulls were not missing at random: excluding them removed difficult, reversal-prone markets and materially inflated the apparent edge. + +## Asset decomposition + +| Asset | Fills | Win rate | P&L at 5 shares | Daily 95% bootstrap interval | +|---|---:|---:|---:|---:| +| BTC | 4,844 | 94.6945% | +$64.00 | -$1.72 to +$3.93 | +| ETH | 4,424 | 94.0552% | -$123.00 | -$5.90 to +$0.18 | +| Combined | 9,268 | 94.3893% | -$59.00 | -$5.18 to +$3.00 | + +BTC-only is **not promoted**. BTC was selected for inspection together with ETH, and retaining BTC after seeing official outcomes would be post-hoc. Its daily confidence interval also crosses zero. Any BTC-only continuation must be frozen as a new prospective version with no inherited holdout status. + +## Risk findings + +- 56 official calendar days; 27 positive days (48.21%). +- Daily t-statistic: -0.50; sign-flip p-value: 0.691. +- Nine calendar weeks; five positive. +- BTC/ETH loss correlation in simultaneous windows: 0.522. +- ETH win probability was 97.80% when BTC won, but only 44.57% when BTC lost. +- Additional cost of only 0.25 cents/share worsens total P&L to approximately -$174.85. + +## Evidence identity + +- Filled condition-set SHA-256: `bf276fbce187d9651d56ecae976520d4251140c1d5b0d440696c5d077838a67a` +- Official transfer source run: `30213818908` +- Full official evidence artifact SHA-256: `210984541bcd64597ca48d480ad59102427f56b4ac204ab1308ad8f8a4f93ce5` +- Compact derivative artifact SHA-256: `fb5dfb4621caa8b62ebaefc0b5c15d04619f773c3686cc4f088767ca023416b9` +- Raw-response manifest SHA-256: `80283073bf2fbed3b5f9ff565852c4303c1ed18f1c6df0845a63abf478059655` +- Gamma terminal payloads: 9,268; failures: 0. +- CLOB corroborated 5,615 outcomes; 3,653 were Gamma-only and retained as such. + +## Classification + +- **Candidate:** Rejected / falsified. +- **Cause:** non-random inferred-label censoring plus weak/negative official economics. +- **BTC-only:** post-hoc diagnostic lead only; prospective-only. +- **ETH:** rejected. +- **Live mode:** disabled; no order submitter or credentials. + +## Next valid research + +1. Preserve this rejection as immutable diagnostic evidence. +2. Correct the reproducer so execution fees are recorded even when inferred labels are null; this is an audit-field correction only and does not change fills or the official verdict. +3. Freeze any BTC-only or replacement hypothesis under a new version before observing new data. +4. Run a credential-free, append-only prospective observer with authoritative terminal outcomes and complete no-trade/fill records. +5. Do not use the inspected historical corpus as admission evidence again. diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/DECISION_AUDIT.json b/ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/DECISION_AUDIT.json new file mode 100644 index 00000000..81320a4f --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/DECISION_AUDIT.json @@ -0,0 +1,87 @@ +{ + "additional_cost_stress_at_five_shares": { + "0.0000": -58.99744399999921, + "0.0025": -174.84744399999923, + "0.0050": -290.6974439999992, + "0.0100": -522.3974439999993, + "0.0150": -754.0974439999992 + }, + "break_even_additional_cost_per_share": -0.00127314294346138, + "by_asset": { + "btc": { + "daily_bootstrap_mean_ci95": [-1.7190766216517774, 3.9265995022544726], + "daily_t_statistic": 0.7865088750981918, + "days": 56, + "fills": 4844, + "mean_pnl_per_share": 0.0026423640338563096, + "pnl_at_five_shares": 63.99805690000045, + "positive_day_rate": 0.5357142857142857, + "win_rate": 0.9469446738232865, + "wins": 4587 + }, + "eth": { + "daily_bootstrap_mean_ci95": [-5.903782505852264, 0.18052209448864426], + "daily_t_statistic": -1.7849776737948304, + "days": 44, + "fills": 4424, + "mean_pnl_per_share": -0.005560375266726951, + "pnl_at_five_shares": -122.99550089999961, + "positive_day_rate": 0.4318181818181818, + "win_rate": 0.9405515370705244, + "wins": 4161 + } + }, + "comparable_to_inferred": 8883, + "cross_asset": { + "btc_losses_in_simultaneous_windows": 92, + "eth_losses_in_simultaneous_windows": 96, + "eth_win_given_btc_loss": 0.44565217391304346, + "eth_win_given_btc_win": 0.9780166096726918, + "loss_correlation": 0.5216654451050746, + "simultaneous_official_windows": 2139 + }, + "daily": { + "best_day_pnl": 32.51204815000002, + "bootstrap_mean_ci95": [-5.1750155101785555, 3.0011215983259074], + "days": 56, + "mean_pnl": -1.05352578571427, + "median_pnl": -1.997734124999985, + "positive_day_rate": 0.48214285714285715, + "positive_days": 27, + "sign_flip_p_value_mc": 0.6905366189267621, + "t_statistic": -0.5012307613775938, + "top_two_day_concentration": null, + "worst_day_pnl": -36.45123499999998 + }, + "fills": 9268, + "inferred_comparable_pnl_at_five_shares": 428.92574600000006, + "official_classification_counts": {"terminal_confirmed_both": 5615, "terminal_gamma_only": 3653}, + "official_comparable_pnl_at_five_shares": 413.9257460000008, + "official_coverage": 1.0, + "official_mean_pnl_per_share": -0.00127314294346138, + "official_pnl_at_five_shares": -58.99744399999921, + "official_usable": 9268, + "official_vs_inferred_agreement": 0.9989868287740629, + "official_vs_inferred_disagreements": 9, + "official_win_rate": 0.9438929650410013, + "official_wins": 8748, + "total_pnl_all_unavailable_losses": -58.99744399999921, + "total_pnl_all_unavailable_wins": -58.99744399999921, + "unavailable_fills": 0, + "weekly": { + "blocks": 9, + "pnl_by_week": [ + {"fills": 374, "pnl": 12.639917150000034, "week": "2026-03-23/2026-03-29", "wins": 354}, + {"fills": 624, "pnl": -7.385373599999932, "week": "2026-03-30/2026-04-05", "wins": 588}, + {"fills": 1279, "pnl": 30.049487200000122, "week": "2026-04-06/2026-04-12", "wins": 1216}, + {"fills": 1351, "pnl": -72.57941159999986, "week": "2026-04-13/2026-04-19", "wins": 1262}, + {"fills": 1320, "pnl": -42.79659704999987, "week": "2026-04-20/2026-04-26", "wins": 1234}, + {"fills": 1328, "pnl": 39.82094355000012, "week": "2026-04-27/2026-05-03", "wins": 1264}, + {"fills": 1486, "pnl": -56.12428954999987, "week": "2026-05-04/2026-05-10", "wins": 1399}, + {"fills": 1422, "pnl": 25.568809900000133, "week": "2026-05-11/2026-05-17", "wins": 1349}, + {"fills": 84, "pnl": 11.809070000000007, "week": "2026-05-18/2026-05-24", "wins": 82} + ], + "positive_block_rate": 0.5555555555555556, + "positive_blocks": 5 + } +} diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/DISAGREEMENTS.csv b/ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/DISAGREEMENTS.csv new file mode 100644 index 00000000..440dabd7 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/DISAGREEMENTS.csv @@ -0,0 +1,10 @@ +asset,condition_id,slug,market_start,selected_side,signal_ask,arrival_ask,arrival_ask_size,inferred_outcome,official_outcome,official_classification,official_won,official_pnl_at_five_shares,gamma_url,gamma_response_body_sha256,clob_url,clob_response_body_sha256 +btc,0x877164694b1e24e500bc57f6fddc491a710d592a437a6601dcd0a15397689e01,btc-updown-5m-1774638300,2026-03-27 19:05:00+00:00,Down,0.92,0.92,58.02,Down,Up,terminal_confirmed_both,False,-4.62576,https://gamma-api.polymarket.com/markets/slug/btc-updown-5m-1774638300,982bbcd818737974bcca97318105d5e2322a682280900af7586226d8d8984161,https://clob.polymarket.com/simplified-markets?next_cursor=Njk1MDAw,3a1ced842fbc4ce5c6f103a818dce87510d1d6bd138e2019712595d161f1760b +eth,0xab8463fdee7b57ca4c12d269f6b79dc9a616d7de5c5d652fcd2cb373e458bf9b,eth-updown-5m-1775580300,2026-04-07 16:45:00+00:00,Up,0.99,0.99,791.08,Up,Down,terminal_confirmed_both,False,-4.9534650000000005,https://gamma-api.polymarket.com/markets/slug/eth-updown-5m-1775580300,d5a8a2cf4f4388a47a9e356c9da852fd89ce5e51aebefc1c7781b93552032a3a,https://clob.polymarket.com/simplified-markets?next_cursor=Nzk2MDAw,0df5b94bd5b975865dfcb3eb6d8cfb0621984721b75bb544d87f2cadf58d7a39 +btc,0x9843de3787b17d3e8a56e088055f3be92c67cf9e003ece193129a339c1913039,btc-updown-5m-1775823900,2026-04-10 12:25:00+00:00,Down,0.92,0.91,136.09,Down,Up,terminal_confirmed_both,False,-4.578664999999999,https://gamma-api.polymarket.com/markets/slug/btc-updown-5m-1775823900,0dea7c6750bc31a522ea7ee441e87c7574e20b816d98532482313734c3bff024,https://clob.polymarket.com/simplified-markets?next_cursor=ODIwMDAw,3f53d41926de2b839c3fdd88f964d03fcda30daeac7ea444c1b6295aab5e3d12 +eth,0x2644f6586b1ff3dcc66fc8d902d6ac4e7d89fa7d7184452f222fc97f40a0bcf9,eth-updown-5m-1777034700,2026-04-24 12:45:00+00:00,Up,0.92,0.92,65.63,Up,Down,terminal_confirmed_both,False,-4.62576,https://gamma-api.polymarket.com/markets/slug/eth-updown-5m-1777034700,8410e6a3070397b0a58466a57a5c335cae87f879f7cbd5e44f95d95fc61ebb13,https://clob.polymarket.com/simplified-markets?next_cursor=OTQzMDAw,7c616bc55f639efc3ad5e5c63a6836bbcc3c3c7c2a796d3758e2350dbdf9a8d6 +eth,0x0922e6d8a580df0063dceadc6f9d33480fac5b6121b5933bc4d3609089052f2c,eth-updown-5m-1777642500,2026-05-01 13:35:00+00:00,Down,0.85,0.85,23.83,Up,Down,terminal_gamma_only,True,0.7053750000000002,https://gamma-api.polymarket.com/markets/slug/eth-updown-5m-1777642500,55f6b730aa1687f10d91455dee01df6ea11afd144bc2a48352dad85a77449699,, +eth,0x64a0b8bc82ce31dc6992f56a6f9e0037660835d1fd8af4808a00ed775a4e6ffc,eth-updown-5m-1777659000,2026-05-01 18:10:00+00:00,Up,0.91,0.91,77.16,Down,Up,terminal_gamma_only,True,0.4213350000000003,https://gamma-api.polymarket.com/markets/slug/eth-updown-5m-1777659000,4745638c609e852ffd6a96d7e23fd1a624261a3af39c3a6d1acd83fa54899d30,, +eth,0x4f7d402a972e69b25030ba76a195161cfe851936feb7e7daa79c70ca8546d630,eth-updown-5m-1777684800,2026-05-02 01:20:00+00:00,Up,0.96,0.96,250.0,Down,Up,terminal_gamma_only,True,0.1865600000000001,https://gamma-api.polymarket.com/markets/slug/eth-updown-5m-1777684800,f29109108eee28043e1d7b10063f0d06f5c5932b678f3c872454321a11197ed7,, +btc,0x7cba6e626c5335d2843bdffada0f5d5a5d7a2d6571ca4412d9635151e1ad0c0e,btc-updown-5m-1777857300,2026-05-04 01:15:00+00:00,Up,0.96,0.96,20.0,Up,Down,terminal_gamma_only,False,-4.81344,https://gamma-api.polymarket.com/markets/slug/btc-updown-5m-1777857300,711acbdade53c307071dcfc3af205774a0e93d8d4f94ad74e4e92ab2c3920937,, +eth,0x363c52b17b934dc97ed90043a8c1cfae5923e317409180b92fe79cb8fdceee32,eth-updown-5m-1778319300,2026-05-09 09:35:00+00:00,Up,0.86,0.86,10.0,Up,Down,terminal_gamma_only,False,-4.34214,https://gamma-api.polymarket.com/markets/slug/eth-updown-5m-1778319300,b01d74f0190071fa182cbf0fb461fe9445684a37ddad074bc0302a74d9317f01,, diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/ROOT_CAUSE_AUDIT.json b/ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/ROOT_CAUSE_AUDIT.json new file mode 100644 index 00000000..eef178e3 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-official-decision-v1/ROOT_CAUSE_AUDIT.json @@ -0,0 +1,43 @@ +{ + "btc_only_boundary": "BTC is positive in this inspected official transfer but its daily confidence interval crosses zero. Selecting BTC-only now is post-hoc and must be treated as a new prospective version, not inherited validation.", + "by_arrival_ask_bucket": [ + {"arrival_ask_bucket": "(0.849, 0.89]", "fills": 142, "official_pnl_at_five_shares": -201.98393, "win_rate": 0.5915492957746479, "wins": 84}, + {"arrival_ask_bucket": "(0.89, 0.92]", "fills": 62, "official_pnl_at_five_shares": -78.545885, "win_rate": 0.6612903225806451, "wins": 41}, + {"arrival_ask_bucket": "(0.92, 0.95]", "fills": 86, "official_pnl_at_five_shares": -71.13024500000002, "win_rate": 0.7790697674418605, "wins": 67}, + {"arrival_ask_bucket": "(0.95, 0.97]", "fills": 40, "official_pnl_at_five_shares": -43.70622499999999, "win_rate": 0.75, "wins": 30}, + {"arrival_ask_bucket": "(0.97, 0.99]", "fills": 50, "official_pnl_at_five_shares": -56.321704999999994, "win_rate": 0.76, "wins": 38}, + {"arrival_ask_bucket": "nan", "fills": 5, "official_pnl_at_five_shares": -21.2352, "win_rate": 0.0, "wins": 0} + ], + "by_asset": { + "btc": {"fills": 179, "first_market_start": "2026-03-25T04:20:00+00:00", "last_market_start": "2026-05-17T20:55:00+00:00", "mean_arrival_ask": 0.9121787709497207, "official_pnl_at_five_shares": -256.29615, "win_rate": 0.6312849162011173, "wins": 113}, + "eth": {"fills": 206, "first_market_start": "2026-04-05T23:45:00+00:00", "last_market_start": "2026-05-18T09:30:00+00:00", "mean_arrival_ask": 0.9188349514563106, "official_pnl_at_five_shares": -216.62704000000002, "win_rate": 0.7135922330097088, "wins": 147} + }, + "by_selected_side": { + "Down": {"fills": 189, "official_pnl_at_five_shares": -208.916355, "win_rate": 0.6984126984126984, "wins": 132}, + "Up": {"fills": 196, "official_pnl_at_five_shares": -264.006835, "win_rate": 0.6530612244897959, "wins": 128} + }, + "candidate": "late_favorite_btc_eth_v2_diagnostic", + "diagnosis": "Non-random label-availability bias: dataset outcomes inferred from final-book state exclude hard edge cases; official resolution restores them and reverses P&L.", + "fills": 9268, + "inferred_comparable_fills": 8883, + "inferred_comparable_pnl_at_five_shares": 428.92574600000006, + "inferred_labeled_mean_arrival_ask": 0.9425450861195542, + "label_disagreements": 9, + "live_submission": "physically_absent", + "official_comparable_pnl_at_five_shares": 413.9257460000008, + "official_coverage": 1.0, + "official_pnl_at_five_shares": -58.99744399999921, + "official_win_rate": 0.9438929650410013, + "pnl_change_from_nine_disagreements": -14.999999999999261, + "previously_unlabeled_fills": 385, + "previously_unlabeled_first_market_start": "2026-03-25T04:20:00+00:00", + "previously_unlabeled_last_market_start": "2026-05-18T09:30:00+00:00", + "previously_unlabeled_mean_arrival_ask": 0.9157402597402597, + "previously_unlabeled_official_losses": 125, + "previously_unlabeled_official_pnl_at_five_shares": -472.92319, + "previously_unlabeled_official_win_rate": 0.6753246753246753, + "previously_unlabeled_official_wins": 260, + "previously_unlabeled_share_of_all_losses": 0.2403846153846154, + "previously_unlabeled_share_of_fills": 0.04154078549848943, + "verdict": "Rejected / falsified on official outcomes" +} diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/official_transfer.py b/ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/official_transfer.py new file mode 100644 index 00000000..ced72ac2 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/official_transfer.py @@ -0,0 +1,695 @@ +from __future__ import annotations + +import argparse +import gzip +import hashlib +import json +import time +from dataclasses import dataclass, field +from datetime import datetime, timezone +from decimal import Decimal, InvalidOperation +from pathlib import Path +from typing import Any, Iterable, Mapping, Sequence + +import pandas as pd +import requests + +GAMMA_MARKETS_URL = "https://gamma-api.polymarket.com/markets" +GAMMA_MARKET_BY_SLUG_URL = "https://gamma-api.polymarket.com/markets/slug/{slug}" +CLOB_SIMPLIFIED_MARKETS_URL = "https://clob.polymarket.com/simplified-markets" +EXPECTED_FILLED_CONDITION_SET_SHA256 = "bf276fbce187d9651d56ecae976520d4251140c1d5b0d440696c5d077838a67a" +TERMINAL_EPSILON = Decimal("1e-12") +USER_AGENT = "late-favorite-official-transfer/1.0 research-only" + + +def _sha256_bytes(raw: bytes) -> str: + return hashlib.sha256(raw).hexdigest() + + +def _canonical_json_bytes(value: Any) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _payload_for_hash(value: Any) -> Any: + if isinstance(value, Mapping): + return {key: item for key, item in value.items() if not str(key).startswith("_")} + return value + + +def _normalize_condition_id(value: Any) -> str: + return str(value or "").strip().lower() + + +def _condition_set_sha256(values: Iterable[Any]) -> str: + ids = sorted({_normalize_condition_id(value) for value in values if _normalize_condition_id(value)}) + canonical = (("\n".join(ids) + "\n") if ids else "").encode("utf-8") + return _sha256_bytes(canonical) + + +def _decode_array(value: Any) -> list[Any]: + if isinstance(value, str): + value = json.loads(value) + if not isinstance(value, (list, tuple)): + raise ValueError("value is not an array") + return list(value) + + +def _base_parse_result(source: str, classification: str) -> dict[str, Any]: + return { + "source": source, + "classification": classification, + "official_outcome": None, + "condition_id": None, + "closed": None, + "uma_resolution_status": None, + "payload_sha256": None, + } + + +def parse_gamma_terminal(payload: Any, *, expected_condition_id: str) -> dict[str, Any]: + result = _base_parse_result("gamma", "malformed_gamma_payload") + result["payload_sha256"] = _sha256_bytes(_canonical_json_bytes(_payload_for_hash(payload))) + if not isinstance(payload, Mapping): + return result + + actual = _normalize_condition_id(payload.get("conditionId")) + expected = _normalize_condition_id(expected_condition_id) + result.update( + { + "condition_id": actual or None, + "closed": payload.get("closed"), + "uma_resolution_status": payload.get("umaResolutionStatus"), + } + ) + if actual != expected: + result["classification"] = "mismatched_condition_id" + return result + if payload.get("closed") is not True: + result["classification"] = "not_closed" + return result + + try: + outcomes = [str(value).strip().title() for value in _decode_array(payload.get("outcomes"))] + raw_prices = _decode_array(payload.get("outcomePrices")) + prices = [Decimal(str(value)) for value in raw_prices] + except (ValueError, TypeError, json.JSONDecodeError, InvalidOperation): + return result + + if len(outcomes) != len(prices) or len(outcomes) != 2: + return result + if len(set(outcomes)) != len(outcomes) or set(outcomes) != {"Up", "Down"}: + return result + if any((not price.is_finite()) or price < 0 or price > 1 for price in prices): + return result + + winner_indexes = [index for index, price in enumerate(prices) if abs(price - Decimal(1)) <= TERMINAL_EPSILON] + zero_indexes = [index for index, price in enumerate(prices) if abs(price) <= TERMINAL_EPSILON] + if len(winner_indexes) != 1 or len(zero_indexes) != len(prices) - 1: + result["classification"] = "ambiguous_terminal_prices" + return result + + result["classification"] = "terminal" + result["official_outcome"] = outcomes[winner_indexes[0]] + return result + + +def parse_clob_terminal(payload: Any, *, expected_condition_id: str) -> dict[str, Any]: + result = _base_parse_result("clob", "malformed_clob_payload") + result["payload_sha256"] = _sha256_bytes(_canonical_json_bytes(_payload_for_hash(payload))) + if not isinstance(payload, Mapping): + return result + + actual = _normalize_condition_id(payload.get("condition_id")) + expected = _normalize_condition_id(expected_condition_id) + result.update({"condition_id": actual or None, "closed": payload.get("closed")}) + if actual != expected: + result["classification"] = "mismatched_condition_id" + return result + if payload.get("closed") is not True: + result["classification"] = "not_closed" + return result + + tokens = payload.get("tokens") + if not isinstance(tokens, list) or len(tokens) != 2 or not all(isinstance(token, Mapping) for token in tokens): + return result + outcomes = [str(token.get("outcome") or "").strip().title() for token in tokens] + if len(set(outcomes)) != len(outcomes) or set(outcomes) != {"Up", "Down"}: + return result + winner_indexes = [index for index, token in enumerate(tokens) if token.get("winner") is True] + if len(winner_indexes) != 1: + result["classification"] = "ambiguous_winner_flags" + return result + + result["classification"] = "terminal" + result["official_outcome"] = outcomes[winner_indexes[0]] + return result + + +def reconcile_terminal_sources(gamma: Mapping[str, Any], clob: Mapping[str, Any]) -> dict[str, Any]: + gamma_class = str(gamma.get("classification") or "unavailable") + clob_class = str(clob.get("classification") or "unavailable") + gamma_outcome = gamma.get("official_outcome") + clob_outcome = clob.get("official_outcome") + result = { + "classification": "unavailable", + "official_outcome": None, + "gamma_classification": gamma_class, + "clob_classification": clob_class, + "gamma_outcome": gamma_outcome, + "clob_outcome": clob_outcome, + } + if gamma_class == "terminal": + if clob_class == "terminal": + if gamma_outcome == clob_outcome: + result.update( + {"classification": "terminal_confirmed_both", "official_outcome": gamma_outcome} + ) + else: + result["classification"] = "source_disagreement" + else: + result.update({"classification": "terminal_gamma_only", "official_outcome": gamma_outcome}) + return result + if clob_class == "terminal": + result["classification"] = "clob_only_unconfirmed" + return result + result["classification"] = gamma_class if gamma_class != "unavailable" else clob_class + return result + + +@dataclass +class ResponseArchive: + root: Path + records: list[dict[str, Any]] = field(default_factory=list) + sequence: int = 0 + + def __post_init__(self) -> None: + self.root.mkdir(parents=True, exist_ok=True) + + def save(self, *, source: str, kind: str, response: requests.Response) -> dict[str, Any]: + self.sequence += 1 + relative = Path("raw") / source / f"{self.sequence:06d}_{kind}.body" + path = self.root / relative + path.parent.mkdir(parents=True, exist_ok=True) + raw = bytes(response.content) + path.write_bytes(raw) + record = { + "sequence": self.sequence, + "source": source, + "kind": kind, + "url": response.url, + "status_code": int(response.status_code), + "body_path": str(relative), + "body_sha256": _sha256_bytes(raw), + "content_type": response.headers.get("content-type"), + } + self.records.append(record) + return record + + def save_exception(self, *, source: str, kind: str, url: str, error: Exception) -> dict[str, Any]: + self.sequence += 1 + record = { + "sequence": self.sequence, + "source": source, + "kind": kind, + "url": url, + "status_code": None, + "body_path": None, + "body_sha256": None, + "content_type": None, + "error_type": type(error).__name__, + "error": str(error), + } + self.records.append(record) + return record + + def write_manifest(self) -> Path: + path = self.root / "raw_response_manifest.jsonl.gz" + raw = "".join( + json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n" + for record in self.records + ).encode("utf-8") + with path.open("wb") as handle: + with gzip.GzipFile(filename="", mode="wb", fileobj=handle, mtime=0) as compressed: + compressed.write(raw) + return path + + +def _session() -> requests.Session: + session = requests.Session() + session.headers.update({"User-Agent": USER_AGENT, "Accept": "application/json"}) + return session + + +def _request_with_retry( + session: requests.Session, + url: str, + *, + params: Any = None, + timeout: float = 30.0, + attempts: int = 5, +) -> requests.Response: + last_error: Exception | None = None + for attempt in range(attempts): + try: + response = session.get(url, params=params, timeout=timeout) + if response.status_code not in {429, 500, 502, 503, 504}: + return response + last_error = RuntimeError(f"retryable HTTP status {response.status_code}") + except requests.RequestException as error: + last_error = error + if attempt + 1 < attempts: + time.sleep(min(8.0, 0.5 * (2**attempt))) + assert last_error is not None + raise last_error + + +def _chunks(values: Sequence[str], size: int) -> Iterable[list[str]]: + for start in range(0, len(values), size): + yield list(values[start : start + size]) + + +def _extract_gamma_payloads(value: Any) -> list[dict[str, Any]]: + if isinstance(value, Mapping): + return [dict(value)] + if isinstance(value, list): + return [dict(item) for item in value if isinstance(item, Mapping)] + return [] + + +def fetch_gamma_payloads( + targets: pd.DataFrame, + *, + archive: ResponseArchive, + session: requests.Session, + batch_size: int, +) -> tuple[dict[str, dict[str, Any]], dict[str, dict[str, Any]]]: + target_ids = sorted({_normalize_condition_id(value) for value in targets["condition_id"]}) + slugs = { + _normalize_condition_id(row.condition_id): str(row.slug) + for row in targets[["condition_id", "slug"]].drop_duplicates("condition_id").itertuples(index=False) + } + payloads: dict[str, dict[str, Any]] = {} + failures: dict[str, dict[str, Any]] = {} + + def absorb(response: requests.Response, requested: set[str], kind: str) -> set[str]: + evidence = archive.save(source="gamma", kind=kind, response=response) + if response.status_code != 200: + return set() + try: + decoded = response.json() + except ValueError: + return set() + matched: set[str] = set() + for payload in _extract_gamma_payloads(decoded): + condition_id = _normalize_condition_id(payload.get("conditionId")) + if condition_id not in requested: + continue + if condition_id in payloads: + failures[condition_id] = { + "classification": "duplicate_gamma_records", + "official_outcome": None, + } + continue + payload = dict(payload) + payload["_response_body_sha256"] = evidence["body_sha256"] + payload["_response_url"] = evidence["url"] + payloads[condition_id] = payload + matched.add(condition_id) + return matched + + for batch_number, batch in enumerate(_chunks(target_ids, batch_size), start=1): + requested = set(batch) + repeated_params: list[tuple[str, str | int]] = [("limit", len(batch)), ("offset", 0)] + repeated_params.extend(("condition_ids", condition_id) for condition_id in batch) + try: + response = _request_with_retry(session, GAMMA_MARKETS_URL, params=repeated_params) + matched = absorb(response, requested, f"batch_{batch_number:04d}_repeated") + except Exception as error: + archive.save_exception( + source="gamma", kind=f"batch_{batch_number:04d}_repeated", url=GAMMA_MARKETS_URL, error=error + ) + matched = set() + + missing = requested - matched - set(payloads) + if missing: + comma_params = { + "limit": len(missing), + "offset": 0, + "condition_ids": ",".join(sorted(missing)), + } + try: + response = _request_with_retry(session, GAMMA_MARKETS_URL, params=comma_params) + absorb(response, missing, f"batch_{batch_number:04d}_comma") + except Exception as error: + archive.save_exception( + source="gamma", kind=f"batch_{batch_number:04d}_comma", url=GAMMA_MARKETS_URL, error=error + ) + + remaining = [condition_id for condition_id in target_ids if condition_id not in payloads] + for index, condition_id in enumerate(remaining, start=1): + slug = slugs.get(condition_id, "").strip() + if not slug or slug.lower() == "nan": + failures[condition_id] = {"classification": "missing_slug", "official_outcome": None} + continue + url = GAMMA_MARKET_BY_SLUG_URL.format(slug=slug) + try: + response = _request_with_retry(session, url) + evidence = archive.save(source="gamma", kind=f"slug_{index:05d}", response=response) + if response.status_code != 200: + failures[condition_id] = { + "classification": f"gamma_http_{response.status_code}", + "official_outcome": None, + } + continue + decoded = response.json() + candidates = _extract_gamma_payloads(decoded) + if len(candidates) != 1: + failures[condition_id] = { + "classification": "gamma_slug_cardinality_error", + "official_outcome": None, + } + continue + payload = candidates[0] + payload["_response_body_sha256"] = evidence["body_sha256"] + payload["_response_url"] = evidence["url"] + payloads[condition_id] = payload + except Exception as error: + archive.save_exception(source="gamma", kind=f"slug_{index:05d}", url=url, error=error) + failures[condition_id] = { + "classification": "gamma_request_exception", + "official_outcome": None, + } + return payloads, failures + + +def fetch_clob_payloads( + target_ids: set[str], + *, + archive: ResponseArchive, + session: requests.Session, + max_pages: int, +) -> tuple[dict[str, dict[str, Any]], dict[str, Any]]: + payloads: dict[str, dict[str, Any]] = {} + cursor: str | None = None + seen_cursors: set[str] = set() + audit: dict[str, Any] = {"pages": 0, "terminated": None, "last_cursor": None} + + for page in range(1, max_pages + 1): + params = None if cursor is None else {"next_cursor": cursor} + try: + response = _request_with_retry(session, CLOB_SIMPLIFIED_MARKETS_URL, params=params) + except Exception as error: + archive.save_exception( + source="clob", kind=f"page_{page:05d}", url=CLOB_SIMPLIFIED_MARKETS_URL, error=error + ) + audit.update({"pages": page - 1, "terminated": "request_exception", "last_cursor": cursor}) + break + evidence = archive.save(source="clob", kind=f"page_{page:05d}", response=response) + if response.status_code != 200: + audit.update( + {"pages": page, "terminated": f"http_{response.status_code}", "last_cursor": cursor} + ) + break + try: + decoded = response.json() + except ValueError: + audit.update({"pages": page, "terminated": "malformed_json", "last_cursor": cursor}) + break + if not isinstance(decoded, Mapping): + audit.update({"pages": page, "terminated": "malformed_payload", "last_cursor": cursor}) + break + data = decoded.get("data") + if not isinstance(data, list): + audit.update({"pages": page, "terminated": "malformed_data", "last_cursor": cursor}) + break + for item in data: + if not isinstance(item, Mapping): + continue + condition_id = _normalize_condition_id(item.get("condition_id")) + if condition_id not in target_ids or condition_id in payloads: + continue + payload = dict(item) + payload["_response_body_sha256"] = evidence["body_sha256"] + payload["_response_url"] = evidence["url"] + payloads[condition_id] = payload + next_cursor = decoded.get("next_cursor") + audit.update({"pages": page, "last_cursor": next_cursor}) + if not next_cursor or next_cursor == "LTE=": + audit["terminated"] = "end_cursor" + break + next_cursor = str(next_cursor) + if next_cursor in seen_cursors: + audit["terminated"] = "repeated_cursor" + break + seen_cursors.add(next_cursor) + cursor = next_cursor + else: + audit["terminated"] = "max_pages" + audit["matched_targets"] = len(payloads) + return payloads, audit + + +def _unavailable(source: str, classification: str = "unavailable") -> dict[str, Any]: + return _base_parse_result(source, classification) + + +def build_transfer_ledger( + eligible: pd.DataFrame, + gamma_payloads: Mapping[str, Mapping[str, Any]], + gamma_failures: Mapping[str, Mapping[str, Any]], + clob_payloads: Mapping[str, Mapping[str, Any]], +) -> pd.DataFrame: + rows: list[dict[str, Any]] = [] + for trade in eligible.itertuples(index=False): + condition_id = _normalize_condition_id(trade.condition_id) + gamma_payload = gamma_payloads.get(condition_id) + if gamma_payload is not None: + gamma = parse_gamma_terminal(gamma_payload, expected_condition_id=condition_id) + gamma_url = gamma_payload.get("_response_url") + gamma_body_sha256 = gamma_payload.get("_response_body_sha256") + else: + failure = gamma_failures.get(condition_id) or {} + gamma = _unavailable("gamma", str(failure.get("classification") or "unavailable")) + gamma_url = None + gamma_body_sha256 = None + + clob_payload = clob_payloads.get(condition_id) + if clob_payload is not None: + clob = parse_clob_terminal(clob_payload, expected_condition_id=condition_id) + clob_url = clob_payload.get("_response_url") + clob_body_sha256 = clob_payload.get("_response_body_sha256") + else: + clob = _unavailable("clob") + clob_url = None + clob_body_sha256 = None + + reconciled = reconcile_terminal_sources(gamma, clob) + official_outcome = reconciled["official_outcome"] + official_usable = official_outcome in {"Up", "Down"} + official_won = bool(str(trade.selected_side).title() == official_outcome) if official_usable else None + arrival_ask = float(trade.arrival_ask) + fee = float(trade.fee_per_share) if pd.notna(trade.fee_per_share) else 0.07 * arrival_ask * (1.0 - arrival_ask) + official_pnl_per_share = (float(official_won) - arrival_ask - fee) if official_usable else None + inferred = str(trade.inferred_outcome).title() if pd.notna(trade.inferred_outcome) else None + inferred = inferred if inferred in {"Up", "Down"} else None + + rows.append( + { + **trade._asdict(), + "condition_id": condition_id, + "gamma_classification": gamma["classification"], + "gamma_outcome": gamma.get("official_outcome"), + "gamma_payload_sha256": gamma.get("payload_sha256"), + "gamma_response_body_sha256": gamma_body_sha256, + "gamma_url": gamma_url, + "gamma_closed": gamma.get("closed"), + "gamma_uma_resolution_status": gamma.get("uma_resolution_status"), + "clob_classification": clob["classification"], + "clob_outcome": clob.get("official_outcome"), + "clob_payload_sha256": clob.get("payload_sha256"), + "clob_response_body_sha256": clob_body_sha256, + "clob_url": clob_url, + "official_classification": reconciled["classification"], + "official_outcome": official_outcome, + "official_label_usable": official_usable, + "official_won": official_won, + "official_pnl_per_share": official_pnl_per_share, + "official_pnl_at_five_shares": official_pnl_per_share * 5 if official_pnl_per_share is not None else None, + "inferred_label_available": inferred is not None, + "official_vs_inferred_agree": (official_outcome == inferred) if official_usable and inferred else None, + } + ) + return pd.DataFrame(rows) + + +def _counts(series: pd.Series) -> dict[str, int]: + return {str(key): int(value) for key, value in series.value_counts(dropna=False).sort_index().items()} + + +def summarize_transfer(ledger: pd.DataFrame) -> dict[str, Any]: + usable = ledger.loc[ledger["official_label_usable"]].copy() + comparable = usable.loc[usable["inferred_label_available"]].copy() + by_asset: dict[str, Any] = {} + for asset, frame in ledger.groupby("asset", sort=True): + asset_usable = frame.loc[frame["official_label_usable"]] + asset_comparable = asset_usable.loc[asset_usable["inferred_label_available"]] + by_asset[str(asset)] = { + "fills": int(len(frame)), + "official_usable": int(len(asset_usable)), + "official_coverage": float(len(asset_usable) / len(frame)) if len(frame) else None, + "official_wins": int(asset_usable["official_won"].sum()), + "official_win_rate": float(asset_usable["official_won"].mean()) if len(asset_usable) else None, + "official_pnl_at_five_shares": float(asset_usable["official_pnl_at_five_shares"].sum()), + "comparable_to_inferred": int(len(asset_comparable)), + "official_vs_inferred_agreement": ( + float(asset_comparable["official_vs_inferred_agree"].mean()) if len(asset_comparable) else None + ), + } + return { + "fills": int(len(ledger)), + "official_usable": int(len(usable)), + "official_coverage": float(len(usable) / len(ledger)) if len(ledger) else None, + "official_wins": int(usable["official_won"].sum()), + "official_win_rate": float(usable["official_won"].mean()) if len(usable) else None, + "official_mean_pnl_per_share": float(usable["official_pnl_per_share"].mean()) if len(usable) else None, + "official_pnl_at_five_shares": float(usable["official_pnl_at_five_shares"].sum()), + "comparable_to_inferred": int(len(comparable)), + "official_vs_inferred_agreement": ( + float(comparable["official_vs_inferred_agree"].mean()) if len(comparable) else None + ), + "official_vs_inferred_disagreements": int((comparable["official_vs_inferred_agree"] == False).sum()), + "classification_counts": _counts(ledger["official_classification"]), + "gamma_classification_counts": _counts(ledger["gamma_classification"]), + "clob_classification_counts": _counts(ledger["clob_classification"]), + "by_asset": by_asset, + } + + +def _format_percent(value: float | None) -> str: + return "n/a" if value is None else f"{value:.4%}" + + +def _format_decimal(value: float | None, places: int = 6) -> str: + return "n/a" if value is None else f"{value:.{places}f}" + + +def run_transfer( + *, + eligible_path: Path, + out_dir: Path, + batch_size: int, + clob_max_pages: int, +) -> dict[str, Any]: + out_dir.mkdir(parents=True, exist_ok=True) + eligible = pd.read_csv(eligible_path) + required = { + "condition_id", + "slug", + "asset", + "selected_side", + "arrival_ask", + "fee_per_share", + "inferred_outcome", + } + missing = sorted(required - set(eligible.columns)) + if missing: + raise RuntimeError(f"eligible ledger missing columns: {missing}") + condition_set_sha256 = _condition_set_sha256(eligible["condition_id"]) + if condition_set_sha256 != EXPECTED_FILLED_CONDITION_SET_SHA256: + raise RuntimeError( + f"filled condition-set SHA-256 mismatch: {condition_set_sha256} != " + f"{EXPECTED_FILLED_CONDITION_SET_SHA256}" + ) + if eligible["condition_id"].duplicated().any(): + raise RuntimeError("eligible ledger contains duplicate condition IDs") + + archive = ResponseArchive(out_dir) + session = _session() + targets = eligible[["condition_id", "slug"]].copy() + gamma_payloads, gamma_failures = fetch_gamma_payloads( + targets, archive=archive, session=session, batch_size=batch_size + ) + clob_payloads, clob_scan = fetch_clob_payloads( + {_normalize_condition_id(value) for value in eligible["condition_id"]}, + archive=archive, + session=session, + max_pages=clob_max_pages, + ) + raw_manifest = archive.write_manifest() + + ledger = build_transfer_ledger(eligible, gamma_payloads, gamma_failures, clob_payloads) + ledger = ledger.sort_values(["market_start", "asset", "condition_id"]) + ledger_path = out_dir / "official_transfer_ledger.csv.gz" + ledger.to_csv(ledger_path, index=False, compression={"method": "gzip", "mtime": 0}) + + summary = summarize_transfer(ledger) + audit = { + "candidate": "late_favorite_btc_eth_v2_diagnostic", + "collected_at_utc": datetime.now(timezone.utc).isoformat(), + "filled_condition_set_sha256": condition_set_sha256, + "expected_filled_condition_set_sha256": EXPECTED_FILLED_CONDITION_SET_SHA256, + "eligible_input_sha256": _sha256_bytes(eligible_path.read_bytes()), + "source_module_sha256": _sha256_bytes(Path(__file__).read_bytes()), + "gamma_endpoint": GAMMA_MARKETS_URL, + "clob_endpoint": CLOB_SIMPLIFIED_MARKETS_URL, + "gamma_payloads": len(gamma_payloads), + "gamma_failures": len(gamma_failures), + "clob_payloads": len(clob_payloads), + "clob_scan": clob_scan, + "raw_responses": len(archive.records), + "raw_response_manifest_sha256": _sha256_bytes(raw_manifest.read_bytes()), + "live_submission": "physically_absent", + "credentials_used": 0, + "thresholds_changed": 0, + **summary, + } + audit_path = out_dir / "AUDIT.json" + audit_path.write_text(json.dumps(audit, indent=2, sort_keys=True) + "\n") + + report = ( + "# Late-Favorite BTC/ETH Official Outcome Transfer\n\n" + "## Result\n\n" + f"- Frozen filled condition set: `{condition_set_sha256}`\n" + f"- Fills: {audit['fills']:,}\n" + f"- Official usable labels: {audit['official_usable']:,} ({_format_percent(audit['official_coverage'])})\n" + f"- Official win rate: {_format_percent(audit['official_win_rate'])}\n" + f"- Official mean P&L/share: {_format_decimal(audit['official_mean_pnl_per_share'])}\n" + f"- Official P&L at five shares: ${_format_decimal(audit['official_pnl_at_five_shares'])}\n" + f"- Comparable inferred labels: {audit['comparable_to_inferred']:,}\n" + f"- Official/inferred disagreements: {audit['official_vs_inferred_disagreements']:,}\n" + f"- Gamma payloads: {audit['gamma_payloads']:,}\n" + f"- CLOB corroborating payloads: {audit['clob_payloads']:,}\n\n" + "## Evidence boundary\n\n" + "Gamma is the primary official terminal source. CLOB winner flags corroborate Gamma where available. CLOB-only outcomes are not promoted. Every HTTP response body is retained with its URL, status code, and SHA-256. This job contains no credentials or order-submission implementation.\n" + ) + report_path = out_dir / "REPORT.md" + report_path.write_text(report) + + top_level_outputs = [audit_path, ledger_path, raw_manifest, report_path] + sums_path = out_dir / "SHA256SUMS" + sums_path.write_text( + "".join(f"{_sha256_bytes(path.read_bytes())} {path.name}\n" for path in sorted(top_level_outputs)) + ) + return audit + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--eligible-trades", type=Path, required=True) + parser.add_argument("--out-dir", type=Path, required=True) + parser.add_argument("--batch-size", type=int, default=25) + parser.add_argument("--clob-max-pages", type=int, default=5000) + args = parser.parse_args() + if args.batch_size < 1 or args.batch_size > 100: + raise SystemExit("batch size must be between 1 and 100") + audit = run_transfer( + eligible_path=args.eligible_trades, + out_dir=args.out_dir, + batch_size=args.batch_size, + clob_max_pages=args.clob_max_pages, + ) + print(json.dumps(audit, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/test_official_transfer.py b/ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/test_official_transfer.py new file mode 100644 index 00000000..7712e588 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-official-transfer-source/test_official_transfer.py @@ -0,0 +1,158 @@ +from __future__ import annotations + +import pytest + +from official_transfer import ( + parse_clob_terminal, + parse_gamma_terminal, + reconcile_terminal_sources, +) + + +CONDITION = "0x" + "ab" * 32 + + +def gamma_payload(**overrides): + payload = { + "conditionId": CONDITION, + "closed": True, + "umaResolutionStatus": "resolved", + "outcomes": '["Up", "Down"]', + "outcomePrices": '["1", "0"]', + } + payload.update(overrides) + return payload + + +def clob_payload(**overrides): + payload = { + "condition_id": CONDITION, + "closed": True, + "tokens": [ + {"outcome": "Up", "price": 1, "winner": True}, + {"outcome": "Down", "price": 0, "winner": False}, + ], + } + payload.update(overrides) + return payload + + +def test_gamma_accepts_json_string_arrays_and_one_hot_terminal_prices() -> None: + result = parse_gamma_terminal(gamma_payload(), expected_condition_id=CONDITION) + assert result["classification"] == "terminal" + assert result["official_outcome"] == "Up" + assert result["closed"] is True + assert result["uma_resolution_status"] == "resolved" + + +def test_gamma_accepts_native_arrays() -> None: + result = parse_gamma_terminal( + gamma_payload(outcomes=["Up", "Down"], outcomePrices=[0, 1]), + expected_condition_id=CONDITION, + ) + assert result["classification"] == "terminal" + assert result["official_outcome"] == "Down" + + +def test_gamma_fails_closed_on_condition_id_mismatch() -> None: + result = parse_gamma_terminal(gamma_payload(conditionId="0x" + "cd" * 32), expected_condition_id=CONDITION) + assert result["classification"] == "mismatched_condition_id" + assert result["official_outcome"] is None + + +def test_gamma_fails_closed_when_market_is_not_closed() -> None: + result = parse_gamma_terminal(gamma_payload(closed=False), expected_condition_id=CONDITION) + assert result["classification"] == "not_closed" + + +def test_gamma_rejects_non_one_hot_prices() -> None: + result = parse_gamma_terminal( + gamma_payload(outcomePrices='["0.999", "0.001"]'), + expected_condition_id=CONDITION, + ) + assert result["classification"] == "ambiguous_terminal_prices" + assert result["official_outcome"] is None + + +def test_gamma_rejects_malformed_array_encoding() -> None: + result = parse_gamma_terminal(gamma_payload(outcomes="not-json"), expected_condition_id=CONDITION) + assert result["classification"] == "malformed_gamma_payload" + + +def test_gamma_rejects_duplicate_outcome_labels() -> None: + result = parse_gamma_terminal( + gamma_payload(outcomes='["Up", "Up"]'), + expected_condition_id=CONDITION, + ) + assert result["classification"] == "malformed_gamma_payload" + + +def test_clob_requires_exactly_one_winner() -> None: + result = parse_clob_terminal(clob_payload(), expected_condition_id=CONDITION) + assert result["classification"] == "terminal" + assert result["official_outcome"] == "Up" + + +def test_clob_rejects_multiple_winners() -> None: + result = parse_clob_terminal( + clob_payload( + tokens=[ + {"outcome": "Up", "winner": True}, + {"outcome": "Down", "winner": True}, + ] + ), + expected_condition_id=CONDITION, + ) + assert result["classification"] == "ambiguous_winner_flags" + + +def test_clob_fails_closed_on_condition_id_mismatch() -> None: + result = parse_clob_terminal( + clob_payload(condition_id="0x" + "ef" * 32), + expected_condition_id=CONDITION, + ) + assert result["classification"] == "mismatched_condition_id" + + +def test_reconciliation_confirms_matching_terminal_sources() -> None: + result = reconcile_terminal_sources( + parse_gamma_terminal(gamma_payload(), expected_condition_id=CONDITION), + parse_clob_terminal(clob_payload(), expected_condition_id=CONDITION), + ) + assert result["classification"] == "terminal_confirmed_both" + assert result["official_outcome"] == "Up" + + +def test_reconciliation_marks_source_disagreement() -> None: + result = reconcile_terminal_sources( + parse_gamma_terminal(gamma_payload(), expected_condition_id=CONDITION), + parse_clob_terminal( + clob_payload( + tokens=[ + {"outcome": "Up", "winner": False}, + {"outcome": "Down", "winner": True}, + ] + ), + expected_condition_id=CONDITION, + ), + ) + assert result["classification"] == "source_disagreement" + assert result["official_outcome"] is None + + +def test_reconciliation_allows_gamma_terminal_when_clob_is_unavailable() -> None: + result = reconcile_terminal_sources( + parse_gamma_terminal(gamma_payload(), expected_condition_id=CONDITION), + {"classification": "unavailable", "official_outcome": None}, + ) + assert result["classification"] == "terminal_gamma_only" + assert result["official_outcome"] == "Up" + + +def test_reconciliation_never_promotes_clob_only_to_primary_official_label() -> None: + result = reconcile_terminal_sources( + {"classification": "unavailable", "official_outcome": None}, + parse_clob_terminal(clob_payload(), expected_condition_id=CONDITION), + ) + assert result["classification"] == "clob_only_unconfirmed" + assert result["official_outcome"] is None diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py b/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py new file mode 100644 index 00000000..fb189089 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/late_favorite_transfer.py @@ -0,0 +1,346 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import math +from pathlib import Path +from typing import Any, Mapping + +import pandas as pd +from huggingface_hub import hf_hub_download + +DATASET_REPO = "kachoio/polymarket-5-minute-crypto-up-down-markets" +ASSETS = ("btc", "eth") +POLICY = { + "assets": ASSETS, + "entry_second": 210, + "signal_ask_min": 0.85, + "latency_seconds": 1, + "fok_limit": "signal_ask", + "shares": 5, + "cancel_below_signal_by_more_than": 0.01, + "hold": "settlement", + "fee_rate": 0.07, + "live_submission": "disabled", +} + + +def frozen_policy() -> dict[str, Any]: + return dict(POLICY) + + +def fee_per_share(price: float) -> float: + p = float(price) + if not math.isfinite(p) or not 0.0 < p < 1.0: + raise ValueError("price must be finite and strictly between zero and one") + return POLICY["fee_rate"] * p * (1.0 - p) + + +def _finite(value: Any) -> float | None: + try: + parsed = float(value) + except (TypeError, ValueError): + return None + return parsed if math.isfinite(parsed) else None + + +def evaluate_market( + signal: Mapping[str, Any] | None, + arrival: Mapping[str, Any] | None, + *, + outcome: str | None, +) -> dict[str, Any]: + result: dict[str, Any] = { + "selected_side": None, + "signal_ask": None, + "arrival_ask": None, + "arrival_ask_size": None, + "signal": False, + "hypothetical_fok_fill": False, + "decision": "no_signal_missing_signal_book", + "won": None, + "fee_per_share": None, + "pnl_per_share": None, + "pnl_at_five_shares": None, + } + if signal is None: + return result + + up_ask = _finite(signal.get("au")) + down_ask = _finite(signal.get("ad")) + if up_ask is None or down_ask is None or not (0.0 < up_ask < 1.0 and 0.0 < down_ask < 1.0): + result["decision"] = "no_signal_invalid_signal_book" + return result + if math.isclose(up_ask, down_ask, rel_tol=0.0, abs_tol=1e-12): + result["decision"] = "no_signal_tied_favorite" + return result + + selected_side = "Up" if up_ask > down_ask else "Down" + signal_ask = up_ask if selected_side == "Up" else down_ask + result.update({"selected_side": selected_side, "signal_ask": signal_ask}) + if signal_ask < POLICY["signal_ask_min"]: + result["decision"] = "no_signal_below_threshold" + return result + result["signal"] = True + + if arrival is None: + result["decision"] = "signal_missing_arrival_book" + return result + ask_column = "au" if selected_side == "Up" else "ad" + size_column = "sau" if selected_side == "Up" else "sad" + arrival_ask = _finite(arrival.get(ask_column)) + arrival_size = _finite(arrival.get(size_column)) + result.update({"arrival_ask": arrival_ask, "arrival_ask_size": arrival_size}) + if arrival_ask is None or not 0.0 < arrival_ask < 1.0: + result["decision"] = "signal_invalid_arrival_ask" + return result + + cancel_floor = signal_ask - POLICY["cancel_below_signal_by_more_than"] + if arrival_ask < cancel_floor - 1e-12: + result["decision"] = "cancel_adverse_move" + return result + if arrival_ask > signal_ask + 1e-12: + result["decision"] = "no_fill_ask_above_frozen_limit" + return result + if arrival_size is None or arrival_size < POLICY["shares"]: + result["decision"] = "no_fill_insufficient_displayed_depth" + return result + + result["hypothetical_fok_fill"] = True + result["decision"] = "hypothetical_fok_fill" + fee = fee_per_share(arrival_ask) + result["fee_per_share"] = fee + normalized_outcome = str(outcome).title() if outcome is not None else None + if normalized_outcome in {"Up", "Down"}: + won = selected_side == normalized_outcome + pnl_per_share = float(won) - arrival_ask - fee + result.update( + { + "won": won, + "pnl_per_share": pnl_per_share, + "pnl_at_five_shares": pnl_per_share * POLICY["shares"], + } + ) + return result + + +def _sha256_bytes(raw: bytes) -> str: + return hashlib.sha256(raw).hexdigest() + + +def _canonical_id_hash(values: pd.Series) -> str: + ids = sorted({str(value) for value in values.dropna()}) + canonical = (("\n".join(ids) + "\n") if ids else "").encode() + return _sha256_bytes(canonical) + + +def verify_candidate_spec(path: Path) -> str: + raw = path.read_bytes() + payload = json.loads(raw) + expected = { + "assets": ["btc", "eth"], + "entry_second": 210, + "signal_ask_min": 0.85, + "latency_seconds": 1, + "fok_limit": "signal_ask", + "shares": 5, + "hold": "settlement", + "fee_per_share": "0.07*p*(1-p)", + "live_submission": "disabled", + "max_positions_per_asset_per_window": 1, + "max_total_positions_per_window": 2, + } + for key, value in expected.items(): + if payload.get(key) != value: + raise RuntimeError(f"candidate spec mismatch for {key}: {payload.get(key)!r} != {value!r}") + cancel = payload.get("arrival_adverse_move_cancel") or {} + if cancel.get("max_price_improvement") != 0.01: + raise RuntimeError("candidate spec cancel threshold mismatch") + return _sha256_bytes(raw) + + +def build_market_ledger(asset: str, markets_path: Path, ticks_path: Path) -> pd.DataFrame: + asset = asset.lower() + if asset not in ASSETS: + raise ValueError(f"unsupported asset: {asset}") + + markets = pd.read_parquet(markets_path) + required_market_columns = {"condition_id", "market_start", "outcome"} + missing_market_columns = sorted(required_market_columns - set(markets.columns)) + if missing_market_columns: + raise RuntimeError(f"market source missing columns: {missing_market_columns}") + for optional in ("event_id", "slug", "market_end"): + if optional not in markets.columns: + markets[optional] = None + markets = markets[["condition_id", "event_id", "slug", "market_start", "market_end", "outcome"]].copy() + markets["market_start"] = pd.to_datetime(markets["market_start"], utc=True) + markets["market_start_s"] = markets["market_start"].astype("int64") // 1_000_000_000 + markets = markets.sort_values(["market_start", "condition_id"]).drop_duplicates("condition_id", keep="last") + + tick_columns = ["condition_id", "t", "bu", "au", "bd", "ad", "sau", "sad"] + ticks = pd.read_parquet(ticks_path, columns=tick_columns) + starts = markets.set_index("condition_id")["market_start_s"] + ticks["market_start_s"] = ticks["condition_id"].map(starts) + ticks = ticks.loc[ticks["market_start_s"].notna()].copy() + ticks["elapsed"] = ticks["t"].astype("int64") - ticks["market_start_s"].astype("int64") + signal_second = POLICY["entry_second"] + arrival_second = signal_second + POLICY["latency_seconds"] + ticks = ticks.loc[ticks["elapsed"].isin({signal_second, arrival_second})].copy() + duplicates = ticks.duplicated(["condition_id", "elapsed"], keep=False) + if duplicates.any(): + examples = ticks.loc[duplicates, ["condition_id", "elapsed"]].head(10).to_dict("records") + raise RuntimeError(f"duplicate exact-second observations: {examples}") + indexed = {(str(row.condition_id), int(row.elapsed)): row._asdict() for row in ticks.itertuples(index=False)} + + rows: list[dict[str, Any]] = [] + for market in markets.itertuples(index=False): + condition_id = str(market.condition_id) + decision = evaluate_market( + indexed.get((condition_id, signal_second)), + indexed.get((condition_id, arrival_second)), + outcome=market.outcome, + ) + rows.append( + { + "asset": asset, + "condition_id": condition_id, + "event_id": None if market.event_id is None else str(market.event_id), + "slug": None if market.slug is None else str(market.slug), + "market_start": market.market_start, + "market_end": market.market_end, + "inferred_outcome": market.outcome, + **decision, + } + ) + return pd.DataFrame(rows) + + +def summarize_asset(frame: pd.DataFrame) -> dict[str, Any]: + fills = frame.loc[frame["hypothetical_fok_fill"]].copy() + labeled = fills.loc[fills["won"].notna()].copy() + win_rate = float(labeled["won"].astype(bool).mean()) if len(labeled) else None + mean_pnl = float(labeled["pnl_per_share"].mean()) if len(labeled) else None + return { + "markets": int(len(frame)), + "signals": int(frame["signal"].sum()), + "hypothetical_fok_fills": int(len(fills)), + "labeled_fills": int(len(labeled)), + "unlabeled_fills": int(len(fills) - len(labeled)), + "inferred_label_wins": int(labeled["won"].astype(bool).sum()), + "inferred_label_win_rate": win_rate, + "inferred_label_mean_pnl_per_share": mean_pnl, + "inferred_label_pnl_at_five_shares": float(labeled["pnl_at_five_shares"].sum()), + "first_market_start": str(frame["market_start"].min()), + "last_market_start": str(frame["market_start"].max()), + "all_condition_ids_sha256": _canonical_id_hash(frame["condition_id"]), + "filled_condition_ids_sha256": _canonical_id_hash(fills["condition_id"]), + "labeled_filled_condition_ids_sha256": _canonical_id_hash(labeled["condition_id"]), + } + + +def _download_asset(asset: str, cache_dir: Path) -> tuple[Path, Path]: + markets = hf_hub_download( + repo_id=DATASET_REPO, + repo_type="dataset", + filename=f"{asset}_markets.parquet", + local_dir=cache_dir, + ) + ticks = hf_hub_download( + repo_id=DATASET_REPO, + repo_type="dataset", + filename=f"{asset}_ticks.parquet", + local_dir=cache_dir, + ) + return Path(markets), Path(ticks) + + +def run_audit(candidate_spec: Path, out_dir: Path, cache_dir: Path) -> dict[str, Any]: + out_dir.mkdir(parents=True, exist_ok=True) + cache_dir.mkdir(parents=True, exist_ok=True) + policy_sha256 = verify_candidate_spec(candidate_spec) + ledgers: list[pd.DataFrame] = [] + source_files: dict[str, str] = {} + for asset in ASSETS: + markets_path, ticks_path = _download_asset(asset, cache_dir) + source_files[f"{asset}_markets"] = _sha256_bytes(markets_path.read_bytes()) + source_files[f"{asset}_ticks"] = _sha256_bytes(ticks_path.read_bytes()) + ledgers.append(build_market_ledger(asset, markets_path, ticks_path)) + + ledger = pd.concat(ledgers, ignore_index=True).sort_values(["market_start", "asset", "condition_id"]) + eligible = ledger.loc[ledger["hypothetical_fok_fill"]].copy() + ledger_path = out_dir / "market_ledger.csv.gz" + eligible_path = out_dir / "eligible_trades.csv.gz" + compression = {"method": "gzip", "mtime": 0} + ledger.to_csv(ledger_path, index=False, compression=compression) + eligible.to_csv(eligible_path, index=False, compression=compression) + + by_asset = {asset: summarize_asset(frame) for asset, frame in ledger.groupby("asset", sort=True)} + overall = summarize_asset(ledger) + audit: dict[str, Any] = { + "candidate": "late_favorite_btc_eth_v2_diagnostic", + "candidate_spec_sha256": policy_sha256, + "dataset_repo": DATASET_REPO, + "source_files_sha256": source_files, + "source_module_sha256": _sha256_bytes(Path(__file__).read_bytes()), + "policy": frozen_policy(), + "markets": int(len(ledger)), + "signals": int(ledger["signal"].sum()), + "hypothetical_fok_fills": int(ledger["hypothetical_fok_fill"].sum()), + "labeled_fills": overall["labeled_fills"], + "unlabeled_fills": overall["unlabeled_fills"], + "inferred_label_wins": overall["inferred_label_wins"], + "inferred_label_win_rate": overall["inferred_label_win_rate"], + "inferred_label_mean_pnl_per_share": overall["inferred_label_mean_pnl_per_share"], + "inferred_label_pnl_at_five_shares": overall["inferred_label_pnl_at_five_shares"], + "all_condition_ids_sha256": _canonical_id_hash(ledger["condition_id"]), + "filled_condition_ids_sha256": _canonical_id_hash(eligible["condition_id"]), + "decision_counts": { + str(key): int(value) + for key, value in ledger["decision"].value_counts(dropna=False).sort_index().items() + }, + "by_asset": by_asset, + "outcomes": "dataset-inferred; not admission evidence", + "official_outcomes_used": 0, + "live_submission": "physically_absent", + "split_reproduction": "not claimed: original deep-EDA split manifest/source is absent", + "future_outcomes_used_for_signal": 0, + } + audit_path = out_dir / "AUDIT.json" + audit_path.write_text(json.dumps(audit, indent=2, sort_keys=True, default=list) + "\n") + + report = ( + "# Late-Favorite BTC/ETH Reproducibility Audit\n\n" + "## Scope\n\n" + "This checkpoint applies the frozen 210-second BTC/ETH rule to every market in the public source corpus and binds the resulting market sets by SHA-256. It does **not** claim to reproduce the previously reported final-test split because the original split manifest and exact deep-EDA source were not preserved in the branch.\n\n" + f"- Markets: {len(ledger):,}\n" + f"- Signals: {int(ledger['signal'].sum()):,}\n" + f"- Hypothetical five-share FOK fills: {int(ledger['hypothetical_fok_fill'].sum()):,}\n" + f"- Labeled / unlabeled fills: {overall['labeled_fills']:,} / {overall['unlabeled_fills']:,}\n" + f"- Inferred-label win rate: {overall['inferred_label_win_rate']:.4%}\n" + f"- All-condition-set SHA-256: `{audit['all_condition_ids_sha256']}`\n" + f"- Filled-condition-set SHA-256: `{audit['filled_condition_ids_sha256']}`\n\n" + "## Evidence boundary\n\n" + "PnL in this artifact uses the dataset's inferred outcomes only. It is diagnostic and cannot satisfy official-label transfer or prospective admission. No credentials, authenticated request, or order-submission implementation exists in this checkpoint.\n" + ) + report_path = out_dir / "REPORT.md" + report_path.write_text(report) + + outputs = [audit_path, eligible_path, ledger_path, report_path] + sums = "".join(f"{_sha256_bytes(path.read_bytes())} {path.name}\n" for path in sorted(outputs)) + (out_dir / "SHA256SUMS").write_text(sums) + return audit + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--candidate-spec", type=Path, required=True) + parser.add_argument("--out-dir", type=Path, required=True) + parser.add_argument("--cache-dir", type=Path, required=True) + args = parser.parse_args() + audit = run_audit(args.candidate_spec, args.out_dir, args.cache_dir) + print(json.dumps(audit, indent=2, sort_keys=True, default=list)) + + +if __name__ == "__main__": + main() diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_fee_field_regression.py b/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_fee_field_regression.py new file mode 100644 index 00000000..5838425d --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_fee_field_regression.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import pytest + +from late_favorite_transfer import evaluate_market, fee_per_share + + +def test_unlabeled_hypothetical_fill_still_records_execution_fee() -> None: + result = evaluate_market( + {"au": 0.90, "ad": 0.12}, + {"au": 0.90, "ad": 0.12, "sau": 5.0, "sad": 5.0}, + outcome=None, + ) + + assert result["decision"] == "hypothetical_fok_fill" + assert result["hypothetical_fok_fill"] is True + assert result["fee_per_share"] == pytest.approx(fee_per_share(0.90)) + assert result["won"] is None + assert result["pnl_per_share"] is None + assert result["pnl_at_five_shares"] is None diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_late_favorite_transfer.py b/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_late_favorite_transfer.py new file mode 100644 index 00000000..9b333e32 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-repro-source/test_late_favorite_transfer.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +import math + +import pandas as pd +import pytest + +from late_favorite_transfer import ( + evaluate_market, + fee_per_share, + frozen_policy, + summarize_asset, +) + + +def signal(up: float, down: float) -> dict[str, float]: + return {"au": up, "ad": down} + + +def arrival(up: float, down: float, up_size: float = 5.0, down_size: float = 5.0) -> dict[str, float]: + return {"au": up, "ad": down, "sau": up_size, "sad": down_size} + + +def test_frozen_policy_matches_the_reviewed_candidate() -> None: + assert frozen_policy() == { + "assets": ("btc", "eth"), + "entry_second": 210, + "signal_ask_min": 0.85, + "latency_seconds": 1, + "fok_limit": "signal_ask", + "shares": 5, + "cancel_below_signal_by_more_than": 0.01, + "hold": "settlement", + "fee_rate": 0.07, + "live_submission": "disabled", + } + + +def test_fee_uses_reviewed_polymarket_curve() -> None: + assert fee_per_share(0.9) == pytest.approx(0.07 * 0.9 * 0.1) + + +def test_fills_at_frozen_limit_with_sufficient_displayed_depth() -> None: + result = evaluate_market(signal(0.90, 0.12), arrival(0.90, 0.12), outcome="Up") + assert result["decision"] == "hypothetical_fok_fill" + assert result["selected_side"] == "Up" + assert result["hypothetical_fok_fill"] is True + assert result["pnl_per_share"] == pytest.approx(1.0 - 0.90 - fee_per_share(0.90)) + + +def test_cancels_when_price_improves_by_more_than_one_cent() -> None: + result = evaluate_market(signal(0.90, 0.12), arrival(0.88, 0.12), outcome="Up") + assert result["decision"] == "cancel_adverse_move" + assert result["hypothetical_fok_fill"] is False + + +def test_one_cent_improvement_remains_eligible() -> None: + result = evaluate_market(signal(0.90, 0.12), arrival(0.89, 0.12), outcome="Up") + assert result["decision"] == "hypothetical_fok_fill" + + +def test_rejects_arrival_ask_above_frozen_limit() -> None: + result = evaluate_market(signal(0.90, 0.12), arrival(0.91, 0.12), outcome="Up") + assert result["decision"] == "no_fill_ask_above_frozen_limit" + + +def test_rejects_insufficient_displayed_depth() -> None: + result = evaluate_market(signal(0.90, 0.12), arrival(0.90, 0.12, up_size=4.99), outcome="Up") + assert result["decision"] == "no_fill_insufficient_displayed_depth" + + +def test_tied_favorite_fails_closed() -> None: + result = evaluate_market(signal(0.50, 0.50), arrival(0.50, 0.50), outcome="Up") + assert result["decision"] == "no_signal_tied_favorite" + assert result["signal"] is False + + +def test_signal_below_threshold_is_not_traded() -> None: + result = evaluate_market(signal(0.84, 0.18), arrival(0.84, 0.18), outcome="Up") + assert result["decision"] == "no_signal_below_threshold" + + +def test_loss_pnl_uses_executable_ask_and_fee() -> None: + result = evaluate_market(signal(0.90, 0.12), arrival(0.90, 0.12), outcome="Down") + assert result["won"] is False + assert result["pnl_per_share"] == pytest.approx(-0.90 - fee_per_share(0.90)) + assert math.isclose(result["pnl_at_five_shares"], result["pnl_per_share"] * 5) + + +def test_asset_summary_exposes_missing_labels_in_denominators() -> None: + frame = pd.DataFrame( + { + "condition_id": ["a", "b", "c"], + "market_start": pd.to_datetime( + ["2026-05-01T00:00:00Z", "2026-05-01T00:05:00Z", "2026-05-01T00:10:00Z"] + ), + "signal": [True, True, True], + "hypothetical_fok_fill": [True, True, True], + "won": [True, False, None], + "pnl_at_five_shares": [0.4, -4.5, None], + "pnl_per_share": [0.08, -0.9, None], + } + ) + summary = summarize_asset(frame) + assert summary["hypothetical_fok_fills"] == 3 + assert summary["labeled_fills"] == 2 + assert summary["unlabeled_fills"] == 1 + assert summary["inferred_label_wins"] == 1 + assert summary["inferred_label_win_rate"] == pytest.approx(0.5) + assert summary["inferred_label_pnl_at_five_shares"] == pytest.approx(-4.1) diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-repro-v2/AUDIT.json b/ci_payload/digital-surface-live-readiness/late-favorite-repro-v2/AUDIT.json new file mode 100644 index 00000000..ebdb4948 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-repro-v2/AUDIT.json @@ -0,0 +1,68 @@ +{ + "all_condition_ids_sha256": "ed051d81e9d5d4aa57cd598205b34cdd0af804f5fa0ddec45aeca4bb85b919a1", + "artifact_archive_sha256": "0cc4a640d411d3b85130c858def8a84034cf7c293d7c25af6b8a9098382c8afc", + "by_asset": { + "btc": { + "all_condition_ids_sha256": "153b73d0fbf938b562018c9d3d3af7aac1c2a182e2957e0887086c570f8cc50e", + "filled_condition_ids_sha256": "64ba2f09192a5dca87b18dd466b7d7d6b8e48c6f7629272bddeb10a5090a8f88", + "first_market_start": "2026-03-24 22:10:00+00:00", + "hypothetical_fok_fills": 4844, + "inferred_label_mean_pnl_per_share": 0.014374885612004295, + "inferred_label_pnl_at_five_shares": 335.2942069000002, + "inferred_label_win_rate": 0.9596998928188639, + "inferred_label_wins": 4477, + "labeled_filled_condition_ids_sha256": "d070649c7f7cc1614b76301dec4750a379288d709fdb7d6b26f663815bfd3618", + "labeled_fills": 4665, + "last_market_start": "2026-05-18 10:30:00+00:00", + "markets": 15682, + "signals": 6847, + "unlabeled_fills": 179 + }, + "eth": { + "all_condition_ids_sha256": "d051614a07d368c17ae6792eadc4696a52db42cd90f47422a58ca888277ffc75", + "filled_condition_ids_sha256": "8fce860d459b12357547283181e7bb5b570cc4e92cffb403d6b2d9ceca9df5b2", + "first_market_start": "2026-04-05 19:25:00+00:00", + "hypothetical_fok_fills": 4424, + "inferred_label_mean_pnl_per_share": 0.004439617785680423, + "inferred_label_pnl_at_five_shares": 93.63153910000014, + "inferred_label_win_rate": 0.9516358463726885, + "inferred_label_wins": 4014, + "labeled_filled_condition_ids_sha256": "4cb9935e7816ed65d8e5412415ecf522243ed675532aa1ee8d0e8a211029fcef", + "labeled_fills": 4218, + "last_market_start": "2026-05-18 10:30:00+00:00", + "markets": 12258, + "signals": 5848, + "unlabeled_fills": 206 + } + }, + "candidate": "late_favorite_btc_eth_v2_diagnostic", + "candidate_spec_sha256": "e8f09194678ff203b9906b0462c411e327c1c1e1a57fc9e7ce1a1c77c28f7cd3", + "dataset_repo": "kachoio/polymarket-5-minute-crypto-up-down-markets", + "decision_counts": { + "cancel_adverse_move": 527, + "hypothetical_fok_fill": 9268, + "no_fill_ask_above_frozen_limit": 2591, + "no_fill_insufficient_displayed_depth": 238, + "no_signal_below_threshold": 13721, + "no_signal_invalid_signal_book": 1488, + "no_signal_tied_favorite": 36, + "signal_invalid_arrival_ask": 71 + }, + "filled_condition_ids_sha256": "bf276fbce187d9651d56ecae976520d4251140c1d5b0d440696c5d077838a67a", + "future_outcomes_used_for_signal": 0, + "hypothetical_fok_fills": 9268, + "inferred_label_mean_pnl_per_share": 0.00965722719801869, + "inferred_label_pnl_at_five_shares": 428.92574600000034, + "inferred_label_win_rate": 0.9558707643814027, + "inferred_label_wins": 8491, + "labeled_fills": 8883, + "live_submission": "physically_absent", + "markets": 27940, + "official_outcomes_used": 0, + "outcomes": "dataset-inferred; not admission evidence", + "source_commit": "6230f9060f463b1c687be8a10201c416fd7eb178", + "source_module_sha256": "9fee982c6934614ecef52679b49f81826f34f41e0a04fc7e251856ee79b47237", + "split_reproduction": "not claimed: original deep-EDA split manifest/source is absent", + "unlabeled_fills": 385, + "workflow_run_id": 30213239532 +} diff --git a/ci_payload/digital-surface-live-readiness/late-favorite-repro-v2/CHECKPOINT.md b/ci_payload/digital-surface-live-readiness/late-favorite-repro-v2/CHECKPOINT.md new file mode 100644 index 00000000..156218be --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/late-favorite-repro-v2/CHECKPOINT.md @@ -0,0 +1,75 @@ +# Late-Favorite BTC/ETH Reproducibility Checkpoint v2 + +## Verdict + +The frozen late-favorite rule now has a clean-room, hash-bound full-corpus reproducer and a complete per-market ledger. This is a successful **reproducibility checkpoint**, not strategy admission. + +The exact previously reported final-test result of 2,054 BTC/ETH trades and +$139.72 is **not yet independently reproducible** because the branch does not contain the original deep-EDA source or its exact train/validation/final-test split manifest. No replacement split was tuned to force a match. + +## Clean-room verification + +- Workflow: `Late Favorite Repro Audit` +- Run: `30213239532` +- Source commit: `6230f9060f463b1c687be8a10201c416fd7eb178` +- Result: green +- Behavior tests: 11 passed +- Source compilation: passed +- Live-submission absence check: passed +- Full BTC/ETH ledger build: passed +- Evidence-boundary and checksum verification: passed +- GitHub artifact ZIP digest: `sha256:0cc4a640d411d3b85130c858def8a84034cf7c293d7c25af6b8a9098382c8afc` +- Deterministic tar digest: `703365cd593e8cf38f08a5af4ece1a947c2393ece68a30583b5ba357a3110cdb` + +## Bound market universe + +- Markets: 27,940 +- Signals at second 210: 12,695 +- Hypothetical five-share FOK fills after the frozen one-second latency, limit, depth, and adverse-move rules: 9,268 +- Labeled fills: 8,883 +- Unlabeled fills: 385 +- Inferred-label win rate among labeled fills: 95.5871% +- Diagnostic inferred-label P&L at five shares: +$428.925746 +- Mean diagnostic P&L: +0.9657 cents per labeled share + +Asset detail: + +| Asset | Markets | Signals | Fills | Labeled | Unlabeled | Inferred win rate | Diagnostic P&L at 5 shares | +|---|---:|---:|---:|---:|---:|---:|---:| +| BTC | 15,682 | 6,847 | 4,844 | 4,665 | 179 | 95.9700% | +$335.294207 | +| ETH | 12,258 | 5,848 | 4,424 | 4,218 | 206 | 95.1636% | +$93.631539 | + +## Cryptographic identities + +- Candidate spec SHA-256: `e8f09194678ff203b9906b0462c411e327c1c1e1a57fc9e7ce1a1c77c28f7cd3` +- Source module SHA-256: `9fee982c6934614ecef52679b49f81826f34f41e0a04fc7e251856ee79b47237` +- All-condition-set SHA-256: `ed051d81e9d5d4aa57cd598205b34cdd0af804f5fa0ddec45aeca4bb85b919a1` +- Filled-condition-set SHA-256: `bf276fbce187d9651d56ecae976520d4251140c1d5b0d440696c5d077838a67a` +- BTC filled set: `64ba2f09192a5dca87b18dd466b7d7d6b8e48c6f7629272bddeb10a5090a8f88` +- ETH filled set: `8fce860d459b12357547283181e7bb5b570cc4e92cffb403d6b2d9ceca9df5b2` + +Output file checksums: + +- `AUDIT.json`: `eb05935840395e34601d95954a20a167f16b431cde83beeb463ceba10ebb0c7c` +- `REPORT.md`: `290bf383f91ed01debfe3fe0a910dadcbaef433ff68e17d3753be971344ea9e5` +- `eligible_trades.csv.gz`: `3438e739c24bc4523e36b439c2fad0804d7018f11082787ce490b3abc8b5915a` +- `market_ledger.csv.gz`: `2e46aa667ddae8a0c5ea24d3329b4b0bba084fcb90dab760dca349e4ca154c62` + +## Data-quality finding + +The first green artifact exposed 385 filled rows with no inferred terminal label. The v2 audit now reports labeled and unlabeled denominators separately and tests that they reconcile exactly to total fills. Diagnostic P&L and win rate use only labeled fills. + +Those 385 rows are not assumed winners, losers, or unresolved Polymarket settlements. They must be transferred to authoritative terminal outcomes or explicitly classified as unavailable. + +## Reproducibility limitations + +1. Official outcomes used: **0**. +2. The dataset outcomes remain inferred and cannot count toward admission. +3. Exact deep-EDA split boundaries and the original implementation source were not preserved. +4. This v2 reproducer selects the favorite as the side with the higher signal ask. That is consistent with the frozen `signal_ask` description but is not proven byte-for-byte identical to the missing original deep-EDA implementation. +5. The post-inspection adverse-move cancel remains a diagnostic/prospective rule; it does not inherit untouched holdout status. +6. Top-of-book depth still does not establish live queue position or realized FOK execution. +7. Live submission remains physically absent. + +## Next bounded job + +Transfer the hash-bound filled condition set to official Polymarket terminal outcomes, preserve every lookup response and failure classification, and recompute the same fixed-rule ledger without modifying thresholds, assets, timing, fees, latency, depth, or cancel logic. diff --git a/ci_payload/digital-surface-live-readiness/market-state-observer-source/research/digital_surface/public_state_observer.py b/ci_payload/digital-surface-live-readiness/market-state-observer-source/research/digital_surface/public_state_observer.py new file mode 100644 index 00000000..2d467db1 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/market-state-observer-source/research/digital_surface/public_state_observer.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import time +from pathlib import Path +from typing import Callable, Sequence + +from .binance_public import BTCMarketState, BINANCE_MARKET_DATA_BASE, collect_btc_market_state +from .diagnostic_filter import DiagnosticBloomFilter +from .market_discovery import GammaMarketRecord, discover_target_markets +from .prospective import ProspectiveLedger +from .public_observer import fetch_gamma_page +from .spot_ledger import SpotStateLedger + + +def _sha256_file(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def observe_public_btc_state( + markets: Sequence[GammaMarketRecord], + *, + prospective_market_ids: set[str], + prospective_head_sha256: str, + ledger: SpotStateLedger, + observed_ts_ms: int, + policy_sha256: str, + source_sha256: str, + collect: Callable[..., BTCMarketState] = collect_btc_market_state, +) -> dict[str, object]: + registered = [market for market in markets if market.condition_id in prospective_market_ids] + boundaries = sorted({int(market.open_epoch_seconds) * 1_000 for market in registered}) + state = collect(boundaries, observed_ts_ms=int(observed_ts_ms)) + if set(state.strikes) != set(boundaries): + raise ValueError( + f"BTC state strike set does not match registered market boundaries: " + f"{sorted(state.strikes)} != {boundaries}" + ) + record = ledger.append( + state, + policy_sha256=policy_sha256, + source_sha256=source_sha256, + prospective_head_sha256=prospective_head_sha256, + ) + return { + "markets_discovered": len(markets), + "markets_registered": len(registered), + "markets_skipped_unregistered": len(markets) - len(registered), + "strike_boundaries": boundaries, + "spot": str(state.spot), + "vol_30s": state.vol_30s, + "vol_120s": state.vol_120s, + "server_time_ms": state.server_time_ms, + "observed_ts_ms": state.observed_ts_ms, + "spot_state_head_sha256": ledger.head_hash, + "spot_state_record_sha256": record["record_hash"], + "spot_state_observations_total": len(ledger.rows), + "prospective_head_sha256": prospective_head_sha256, + "binance_market_data_base": BINANCE_MARKET_DATA_BASE, + "authenticated_requests": 0, + "order_submissions": 0, + "trading_mode": "observation_only", + } + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Collect public BTC market state without credentials") + parser.add_argument("--diagnostic-filter", type=Path, required=True) + parser.add_argument("--prospective-ledger", type=Path, required=True) + parser.add_argument("--spot-ledger", type=Path, required=True) + parser.add_argument("--policy", type=Path, required=True) + parser.add_argument("--source-manifest", type=Path, required=True) + parser.add_argument("--summary", type=Path, required=True) + parser.add_argument("--page-size", type=int, default=100) + parser.add_argument("--max-pages", type=int, default=20) + args = parser.parse_args(argv) + + bloom = DiagnosticBloomFilter.from_path(args.diagnostic_filter) + prospective = ProspectiveLedger(args.prospective_ledger, diagnostic_market_ids=bloom) + markets = discover_target_markets( + lambda params: fetch_gamma_page(params), + page_size=args.page_size, + max_pages=args.max_pages, + ) + summary = observe_public_btc_state( + markets, + prospective_market_ids=prospective.market_ids, + prospective_head_sha256=prospective.head_hash, + ledger=SpotStateLedger(args.spot_ledger), + observed_ts_ms=int(time.time() * 1_000), + policy_sha256=_sha256_file(args.policy), + source_sha256=_sha256_file(args.source_manifest), + ) + args.summary.parent.mkdir(parents=True, exist_ok=True) + args.summary.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ci_payload/digital-surface-live-readiness/market-state-observer-source/tests/digital_surface/test_public_state_observer.py b/ci_payload/digital-surface-live-readiness/market-state-observer-source/tests/digital_surface/test_public_state_observer.py new file mode 100644 index 00000000..39c9473e --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/market-state-observer-source/tests/digital_surface/test_public_state_observer.py @@ -0,0 +1,104 @@ +from decimal import Decimal + +from research.digital_surface.binance_public import BTCMarketState +from research.digital_surface.market_discovery import GammaMarketRecord +from research.digital_surface.public_state_observer import observe_public_btc_state +from research.digital_surface.spot_ledger import SpotStateLedger + + +def market(condition: str, epoch: int, duration: int = 300) -> GammaMarketRecord: + return GammaMarketRecord( + condition_id=condition, + slug=f"btc-updown-{duration // 60}m-{epoch}", + question="Bitcoin Up or Down", + yes_token_id=f"yes-{condition}", + no_token_id=f"no-{condition}", + duration_seconds=duration, + open_epoch_seconds=epoch, + end_date="2030-01-01T00:00:00Z", + tick_size=Decimal("0.01"), + ) + + +def test_collects_unique_boundaries_only_for_registered_prospective_markets(tmp_path): + markets = [ + market("registered-5", 1_800), + market("registered-15", 1_800, duration=900), + market("unregistered", 2_100), + ] + calls = [] + + def collect(boundaries, *, observed_ts_ms): + calls.append((list(boundaries), observed_ts_ms)) + return BTCMarketState( + symbol="BTCUSDT", + server_time_ms=5_000, + observed_ts_ms=observed_ts_ms, + spot=Decimal("100.50"), + vol_30s=0.001, + vol_120s=0.002, + strikes={1_800_000: Decimal("99.00")}, + closed_one_second_bars=121, + raw_response_sha256={ + "server_time": "a" * 64, + "one_second_klines": "b" * 64, + "strike:1800000": "c" * 64, + }, + ) + + ledger = SpotStateLedger(tmp_path / "btc_state.jsonl") + summary = observe_public_btc_state( + markets, + prospective_market_ids={"registered-5", "registered-15"}, + prospective_head_sha256="3" * 64, + ledger=ledger, + observed_ts_ms=5_100, + policy_sha256="1" * 64, + source_sha256="2" * 64, + collect=collect, + ) + + assert calls == [([1_800_000], 5_100)] + assert summary["markets_registered"] == 2 + assert summary["markets_skipped_unregistered"] == 1 + assert summary["strike_boundaries"] == [1_800_000] + assert summary["spot_state_head_sha256"] == ledger.head_hash + assert summary["authenticated_requests"] == 0 + assert summary["order_submissions"] == 0 + + +def test_empty_registered_set_still_records_causal_spot_without_strikes(tmp_path): + calls = [] + + def collect(boundaries, *, observed_ts_ms): + calls.append(list(boundaries)) + return BTCMarketState( + symbol="BTCUSDT", + server_time_ms=5_000, + observed_ts_ms=observed_ts_ms, + spot=Decimal("100.50"), + vol_30s=0.001, + vol_120s=0.002, + strikes={}, + closed_one_second_bars=121, + raw_response_sha256={ + "server_time": "a" * 64, + "one_second_klines": "b" * 64, + }, + ) + + ledger = SpotStateLedger(tmp_path / "btc_state.jsonl") + summary = observe_public_btc_state( + [market("unregistered", 2_100)], + prospective_market_ids=set(), + prospective_head_sha256="3" * 64, + ledger=ledger, + observed_ts_ms=5_100, + policy_sha256="1" * 64, + source_sha256="2" * 64, + collect=collect, + ) + assert calls == [[]] + assert summary["markets_registered"] == 0 + assert summary["strike_boundaries"] == [] + assert len(ledger.rows) == 1 diff --git a/ci_payload/digital-surface-live-readiness/market-state-source/research/digital_surface/binance_public.py b/ci_payload/digital-surface-live-readiness/market-state-source/research/digital_surface/binance_public.py new file mode 100644 index 00000000..f9dbb3d8 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/market-state-source/research/digital_surface/binance_public.py @@ -0,0 +1,220 @@ +from __future__ import annotations + +import hashlib +import json +import math +import statistics +import time +from dataclasses import dataclass +from decimal import Decimal +from typing import Any, Callable, Mapping, Sequence + + +BINANCE_MARKET_DATA_BASE = "https://data-api.binance.vision" +BINANCE_SYMBOL = "BTCUSDT" +_INTERVAL_MS = {"1s": 1_000, "1m": 60_000} + + +def _canonical_json(value: object) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _payload_sha256(value: object) -> str: + return hashlib.sha256(_canonical_json(value)).hexdigest() + + +@dataclass(frozen=True, slots=True) +class BinanceKline: + open_time_ms: int + open: Decimal + high: Decimal + low: Decimal + close: Decimal + volume: Decimal + close_time_ms: int + + +@dataclass(frozen=True, slots=True) +class BTCMarketState: + symbol: str + server_time_ms: int + observed_ts_ms: int + spot: Decimal + vol_30s: float + vol_120s: float + strikes: dict[int, Decimal] + closed_one_second_bars: int + raw_response_sha256: dict[str, str] + + def as_json(self) -> dict[str, object]: + return { + "symbol": self.symbol, + "server_time_ms": int(self.server_time_ms), + "observed_ts_ms": int(self.observed_ts_ms), + "spot": str(self.spot), + "vol_30s": float(self.vol_30s), + "vol_120s": float(self.vol_120s), + "strikes": {str(key): str(value) for key, value in sorted(self.strikes.items())}, + "closed_one_second_bars": int(self.closed_one_second_bars), + "raw_response_sha256": dict(sorted(self.raw_response_sha256.items())), + } + + +def parse_klines(payload: object, *, interval: str) -> tuple[BinanceKline, ...]: + if interval not in _INTERVAL_MS: + raise ValueError(f"unsupported kline interval: {interval}") + if not isinstance(payload, Sequence) or isinstance(payload, (str, bytes)): + raise ValueError("Binance klines payload must be a sequence") + interval_ms = _INTERVAL_MS[interval] + parsed: list[BinanceKline] = [] + for index, raw in enumerate(payload): + if not isinstance(raw, Sequence) or isinstance(raw, (str, bytes)) or len(raw) < 7: + raise ValueError(f"Binance kline row {index} is malformed") + open_time = int(raw[0]) + open_px = Decimal(str(raw[1])) + high = Decimal(str(raw[2])) + low = Decimal(str(raw[3])) + close = Decimal(str(raw[4])) + volume = Decimal(str(raw[5])) + close_time = int(raw[6]) + if min(open_px, high, low, close) <= 0: + raise ValueError("Binance kline prices must be positive") + if volume < 0: + raise ValueError("Binance kline volume must be non-negative") + if high < max(open_px, close) or low > min(open_px, close) or low > high: + raise ValueError("Binance kline OHLC values are inconsistent") + if close_time < open_time: + raise ValueError("Binance kline close time precedes open time") + parsed.append( + BinanceKline( + open_time_ms=open_time, + open=open_px, + high=high, + low=low, + close=close, + volume=volume, + close_time_ms=close_time, + ) + ) + parsed.sort(key=lambda item: item.open_time_ms) + for previous, current in zip(parsed, parsed[1:]): + if current.open_time_ms - previous.open_time_ms != interval_ms: + raise ValueError(f"Binance {interval} klines must be contiguous") + return tuple(parsed) + + +def realized_log_volatility(closes: Sequence[Decimal], *, returns: int) -> float: + if returns <= 0: + raise ValueError("returns must be positive") + required = returns + 1 + if len(closes) < required: + raise ValueError(f"realized volatility requires at least {required} close prices") + trailing = [Decimal(value) for value in closes[-required:]] + if any(value <= 0 for value in trailing): + raise ValueError("close prices must be positive") + log_returns = [math.log(float(current / previous)) for previous, current in zip(trailing, trailing[1:])] + return float(statistics.pstdev(log_returns)) + + +def _default_get(url: str, **kwargs: Any): + import requests + + return requests.get(url, **kwargs) + + +def _get_json( + get: Callable[..., Any], + url: str, + *, + params: Mapping[str, object] | None = None, + timeout_seconds: float = 15.0, +) -> object: + kwargs: dict[str, object] = {"timeout": timeout_seconds} + if params is not None: + kwargs["params"] = dict(params) + response = get(url, **kwargs) + response.raise_for_status() + return response.json() + + +def collect_btc_market_state( + boundary_open_ms: Sequence[int], + *, + get: Callable[..., Any] | None = None, + observed_ts_ms: int | None = None, + timeout_seconds: float = 15.0, +) -> BTCMarketState: + fetch = get or _default_get + server_payload = _get_json( + fetch, + f"{BINANCE_MARKET_DATA_BASE}/api/v3/time", + timeout_seconds=timeout_seconds, + ) + if not isinstance(server_payload, Mapping) or "serverTime" not in server_payload: + raise ValueError("Binance server-time response is malformed") + server_time_ms = int(server_payload["serverTime"]) + if server_time_ms < 0: + raise ValueError("Binance server time must be non-negative") + + one_second_payload = _get_json( + fetch, + f"{BINANCE_MARKET_DATA_BASE}/api/v3/klines", + params={ + "symbol": BINANCE_SYMBOL, + "interval": "1s", + "endTime": server_time_ms, + "limit": 122, + }, + timeout_seconds=timeout_seconds, + ) + one_second_all = parse_klines(one_second_payload, interval="1s") + closed = [item for item in one_second_all if item.close_time_ms <= server_time_ms] + if len(closed) < 121: + raise ValueError(f"need at least 121 closed one-second bars, found {len(closed)}") + closed = closed[-121:] + for previous, current in zip(closed, closed[1:]): + if current.open_time_ms - previous.open_time_ms != 1_000: + raise ValueError("closed one-second klines must be contiguous") + closes = [item.close for item in closed] + + strikes: dict[int, Decimal] = {} + raw_hashes = { + "server_time": _payload_sha256(server_payload), + "one_second_klines": _payload_sha256(one_second_payload), + } + for boundary in sorted({int(value) for value in boundary_open_ms}): + if boundary < 0 or boundary % 60_000: + raise ValueError(f"strike boundary must be a non-negative minute epoch in milliseconds: {boundary}") + strike_payload = _get_json( + fetch, + f"{BINANCE_MARKET_DATA_BASE}/api/v3/klines", + params={ + "symbol": BINANCE_SYMBOL, + "interval": "1m", + "startTime": boundary, + "endTime": boundary + 59_999, + "limit": 1, + }, + timeout_seconds=timeout_seconds, + ) + strike_rows = parse_klines(strike_payload, interval="1m") + if len(strike_rows) != 1 or strike_rows[0].open_time_ms != boundary: + returned = None if not strike_rows else strike_rows[0].open_time_ms + raise ValueError(f"Binance did not return the exact boundary {boundary}; returned {returned}") + strikes[boundary] = strike_rows[0].open + raw_hashes[f"strike:{boundary}"] = _payload_sha256(strike_payload) + + observed = int(time.time() * 1000) if observed_ts_ms is None else int(observed_ts_ms) + if observed < 0: + raise ValueError("observed_ts_ms must be non-negative") + return BTCMarketState( + symbol=BINANCE_SYMBOL, + server_time_ms=server_time_ms, + observed_ts_ms=observed, + spot=closes[-1], + vol_30s=realized_log_volatility(closes, returns=30), + vol_120s=realized_log_volatility(closes, returns=120), + strikes=strikes, + closed_one_second_bars=len(closed), + raw_response_sha256=raw_hashes, + ) diff --git a/ci_payload/digital-surface-live-readiness/market-state-source/research/digital_surface/spot_ledger.py b/ci_payload/digital-surface-live-readiness/market-state-source/research/digital_surface/spot_ledger.py new file mode 100644 index 00000000..e49df8d7 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/market-state-source/research/digital_surface/spot_ledger.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +from .binance_public import BTCMarketState + + +class SpotStateIntegrityError(RuntimeError): + """Raised when the append-only BTC state ledger cannot be verified.""" + + +def _canonical_json(value: dict[str, Any]) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _sha256(name: str, value: str) -> str: + text = str(value).lower() + if len(text) != 64 or any(character not in "0123456789abcdef" for character in text): + raise ValueError(f"{name} must be a 64-character SHA-256 hex digest") + return text + + +def _verify_state(state: object, *, line_number: int | None = None) -> dict[str, Any]: + suffix = "" if line_number is None else f" at line {line_number}" + if not isinstance(state, dict): + raise SpotStateIntegrityError(f"missing BTC state{suffix}") + required = { + "symbol", + "server_time_ms", + "observed_ts_ms", + "spot", + "vol_30s", + "vol_120s", + "strikes", + "closed_one_second_bars", + "raw_response_sha256", + } + missing = sorted(required - set(state)) + if missing: + raise SpotStateIntegrityError(f"BTC state missing fields {missing}{suffix}") + if str(state["symbol"]) != "BTCUSDT": + raise SpotStateIntegrityError(f"unexpected BTC state symbol{suffix}") + if int(state["server_time_ms"]) < 0 or int(state["observed_ts_ms"]) < 0: + raise SpotStateIntegrityError(f"invalid BTC state timestamps{suffix}") + if int(state["closed_one_second_bars"]) < 121: + raise SpotStateIntegrityError(f"insufficient closed one-second bars{suffix}") + if not isinstance(state["strikes"], dict) or not isinstance(state["raw_response_sha256"], dict): + raise SpotStateIntegrityError(f"invalid BTC state maps{suffix}") + try: + for value in state["raw_response_sha256"].values(): + _sha256("raw_response_sha256", str(value)) + except ValueError as exc: + raise SpotStateIntegrityError(f"invalid raw response hash{suffix}") from exc + return state + + +class SpotStateLedger: + def __init__(self, path: str | Path) -> None: + self.path = Path(path) + self.rows: list[dict[str, Any]] = [] + self.observation_ids: set[str] = set() + self.head_hash = "0" * 64 + if self.path.exists(): + self._load_and_verify() + + def _load_and_verify(self) -> None: + previous = "0" * 64 + rows: list[dict[str, Any]] = [] + seen: set[str] = set() + for line_number, line in enumerate(self.path.read_text(encoding="utf-8").splitlines(), start=1): + if not line.strip(): + continue + try: + record = json.loads(line) + except json.JSONDecodeError as exc: + raise SpotStateIntegrityError(f"invalid JSON at line {line_number}") from exc + stored_hash = str(record.get("record_hash", "")) + payload = dict(record) + payload.pop("record_hash", None) + if payload.get("previous_hash") != previous: + raise SpotStateIntegrityError(f"BTC state chain mismatch at line {line_number}") + expected_record_hash = hashlib.sha256(_canonical_json(payload)).hexdigest() + if stored_hash != expected_record_hash: + raise SpotStateIntegrityError(f"record hash mismatch at line {line_number}") + state = _verify_state(payload.get("state"), line_number=line_number) + expected_state_hash = hashlib.sha256(_canonical_json(state)).hexdigest() + if payload.get("state_sha256") != expected_state_hash: + raise SpotStateIntegrityError(f"state hash mismatch at line {line_number}") + observation_id = str(payload.get("observation_id") or "") + expected_observation = hashlib.sha256( + f"{state['server_time_ms']}\0{state['observed_ts_ms']}\0{expected_state_hash}".encode() + ).hexdigest() + if observation_id != expected_observation: + raise SpotStateIntegrityError(f"observation identity mismatch at line {line_number}") + if observation_id in seen: + raise SpotStateIntegrityError(f"duplicate BTC state snapshot at line {line_number}") + for field in ("policy_sha256", "source_sha256", "prospective_head_sha256"): + try: + _sha256(field, str(payload.get(field, ""))) + except ValueError as exc: + raise SpotStateIntegrityError(f"invalid {field} at line {line_number}") from exc + rows.append(record) + seen.add(observation_id) + previous = stored_hash + self.rows = rows + self.observation_ids = seen + self.head_hash = previous + + def append( + self, + state: BTCMarketState, + *, + policy_sha256: str, + source_sha256: str, + prospective_head_sha256: str, + ) -> dict[str, Any]: + state_json = state.as_json() + _verify_state(state_json) + state_hash = hashlib.sha256(_canonical_json(state_json)).hexdigest() + observation_id = hashlib.sha256( + f"{state.server_time_ms}\0{state.observed_ts_ms}\0{state_hash}".encode() + ).hexdigest() + if observation_id in self.observation_ids: + raise ValueError("duplicate BTC state snapshot") + payload = { + "observation_id": observation_id, + "policy_sha256": _sha256("policy_sha256", policy_sha256), + "previous_hash": self.head_hash, + "prospective_head_sha256": _sha256("prospective_head_sha256", prospective_head_sha256), + "source_sha256": _sha256("source_sha256", source_sha256), + "state": state_json, + "state_sha256": state_hash, + } + record = {**payload, "record_hash": hashlib.sha256(_canonical_json(payload)).hexdigest()} + self.path.parent.mkdir(parents=True, exist_ok=True) + with self.path.open("a", encoding="utf-8", newline="\n") as handle: + handle.write(json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n") + handle.flush() + self.rows.append(record) + self.observation_ids.add(observation_id) + self.head_hash = record["record_hash"] + return dict(record) diff --git a/ci_payload/digital-surface-live-readiness/market-state-source/tests/digital_surface/test_binance_public.py b/ci_payload/digital-surface-live-readiness/market-state-source/tests/digital_surface/test_binance_public.py new file mode 100644 index 00000000..af86c484 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/market-state-source/tests/digital_surface/test_binance_public.py @@ -0,0 +1,130 @@ +import hashlib +import json +import math +from decimal import Decimal + +import pytest + +from research.digital_surface.binance_public import ( + BINANCE_MARKET_DATA_BASE, + collect_btc_market_state, + parse_klines, + realized_log_volatility, +) + + +class Response: + def __init__(self, payload): + self._payload = payload + + def raise_for_status(self): + return None + + def json(self): + return self._payload + + +def row(open_ms: int, close: str, *, interval_ms: int = 1_000, open_price: str | None = None): + price = open_price or close + return [ + open_ms, + price, + str(Decimal(price) + Decimal("1")), + str(Decimal(price) - Decimal("1")), + close, + "1.0", + open_ms + interval_ms - 1, + "1.0", + 1, + "0.5", + "0.5", + "0", + ] + + +def test_parse_klines_rejects_nonpositive_prices_and_time_gaps(): + with pytest.raises(ValueError, match="positive"): + parse_klines([row(0, "0")], interval="1s") + + with pytest.raises(ValueError, match="contiguous"): + parse_klines([row(0, "100"), row(2_000, "101")], interval="1s") + + +def test_realized_log_volatility_uses_exact_trailing_return_count(): + closes = [Decimal(str(math.exp(index * 0.001))) for index in range(121)] + assert realized_log_volatility(closes, returns=30) == pytest.approx(0.0, abs=1e-15) + assert realized_log_volatility(closes, returns=120) == pytest.approx(0.0, abs=1e-15) + with pytest.raises(ValueError, match="121 close prices"): + realized_log_volatility(closes[:-1], returns=120) + + +def test_collects_closed_one_second_state_and_exact_boundary_strikes_without_auth_headers(): + boundary_a = 1_800_000 + boundary_b = 2_100_000 + server_time = 5_000_000 + observed_time = 5_000_123 + calls = [] + + closes = [Decimal("100") + Decimal(index) / Decimal("10") for index in range(122)] + one_second = [row(4_878_000 + index * 1_000, str(price)) for index, price in enumerate(closes)] + # The newest candle is still open at server_time and must be excluded. + one_second[-1][6] = server_time + 999 + + one_minute = { + boundary_a: [row(boundary_a, "101.25", interval_ms=60_000, open_price="101.25")], + boundary_b: [row(boundary_b, "102.50", interval_ms=60_000, open_price="102.50")], + } + + def get(url, **kwargs): + calls.append((url, kwargs)) + assert "headers" not in kwargs + if url == f"{BINANCE_MARKET_DATA_BASE}/api/v3/time": + return Response({"serverTime": server_time}) + assert url == f"{BINANCE_MARKET_DATA_BASE}/api/v3/klines" + params = kwargs["params"] + assert params["symbol"] == "BTCUSDT" + if params["interval"] == "1s": + assert params["limit"] == 122 + assert params["endTime"] == server_time + return Response(one_second) + assert params["interval"] == "1m" + assert params["limit"] == 1 + assert params["endTime"] == params["startTime"] + 59_999 + return Response(one_minute[params["startTime"]]) + + state = collect_btc_market_state( + [boundary_b, boundary_a, boundary_a], + get=get, + observed_ts_ms=observed_time, + ) + + assert state.server_time_ms == server_time + assert state.observed_ts_ms == observed_time + assert state.spot == closes[-2] + assert state.strikes == { + boundary_a: Decimal("101.25"), + boundary_b: Decimal("102.50"), + } + assert state.closed_one_second_bars == 121 + assert state.vol_30s >= 0 + assert state.vol_120s >= 0 + assert set(state.raw_response_sha256) == {"server_time", "one_second_klines", f"strike:{boundary_a}", f"strike:{boundary_b}"} + assert all(len(value) == 64 for value in state.raw_response_sha256.values()) + assert len(calls) == 4 + + +def test_exact_boundary_strike_fails_closed_when_binance_returns_another_minute(): + boundary = 1_800_000 + server_time = 5_000_000 + one_second = [row(4_879_000 + index * 1_000, str(100 + index / 10)) for index in range(121)] + + def get(url, **kwargs): + if url.endswith("/time"): + return Response({"serverTime": server_time}) + params = kwargs["params"] + if params["interval"] == "1s": + return Response(one_second) + return Response([row(boundary + 60_000, "100", interval_ms=60_000)]) + + with pytest.raises(ValueError, match="exact boundary"): + collect_btc_market_state([boundary], get=get, observed_ts_ms=server_time) diff --git a/ci_payload/digital-surface-live-readiness/market-state-source/tests/digital_surface/test_spot_ledger.py b/ci_payload/digital-surface-live-readiness/market-state-source/tests/digital_surface/test_spot_ledger.py new file mode 100644 index 00000000..dbec3a1e --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/market-state-source/tests/digital_surface/test_spot_ledger.py @@ -0,0 +1,72 @@ +import json + +import pytest + +from research.digital_surface.binance_public import BTCMarketState +from research.digital_surface.spot_ledger import SpotStateIntegrityError, SpotStateLedger + + +def state(): + return BTCMarketState( + symbol="BTCUSDT", + server_time_ms=2_000, + observed_ts_ms=2_100, + spot=__import__("decimal").Decimal("100.50"), + vol_30s=0.001, + vol_120s=0.002, + strikes={1_800_000: __import__("decimal").Decimal("99.00")}, + closed_one_second_bars=121, + raw_response_sha256={ + "server_time": "a" * 64, + "one_second_klines": "b" * 64, + "strike:1800000": "c" * 64, + }, + ) + + +def test_appends_state_bound_to_policy_source_and_prospective_head(tmp_path): + path = tmp_path / "btc_state.jsonl" + ledger = SpotStateLedger(path) + first = ledger.append( + state(), + policy_sha256="1" * 64, + source_sha256="2" * 64, + prospective_head_sha256="3" * 64, + ) + assert first["previous_hash"] == "0" * 64 + assert first["state"]["spot"] == "100.50" + assert first["state"]["strikes"] == {"1800000": "99.00"} + assert len(first["state_sha256"]) == 64 + + reloaded = SpotStateLedger(path) + assert reloaded.head_hash == first["record_hash"] + assert len(reloaded.rows) == 1 + + +def test_detects_state_or_digest_tampering(tmp_path): + path = tmp_path / "btc_state.jsonl" + ledger = SpotStateLedger(path) + ledger.append( + state(), + policy_sha256="1" * 64, + source_sha256="2" * 64, + prospective_head_sha256="3" * 64, + ) + row = json.loads(path.read_text()) + row["state"]["spot"] = "999999" + path.write_text(json.dumps(row) + "\n") + with pytest.raises(SpotStateIntegrityError, match="record hash mismatch|state hash mismatch"): + SpotStateLedger(path) + + +def test_rejects_duplicate_snapshot_identity(tmp_path): + path = tmp_path / "btc_state.jsonl" + ledger = SpotStateLedger(path) + kwargs = { + "policy_sha256": "1" * 64, + "source_sha256": "2" * 64, + "prospective_head_sha256": "3" * 64, + } + ledger.append(state(), **kwargs) + with pytest.raises(ValueError, match="duplicate BTC state snapshot"): + ledger.append(state(), **kwargs) diff --git a/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/book_ledger.py b/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/book_ledger.py new file mode 100644 index 00000000..9e02e6a2 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/book_ledger.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +from .public_books import PublicOrderBook + + +class BookLedgerIntegrityError(RuntimeError): + pass + + +def _canonical_json(value: dict[str, Any]) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _sha256(name: str, value: str) -> str: + text = str(value).lower() + if len(text) != 64 or any(ch not in "0123456789abcdef" for ch in text): + raise ValueError(f"{name} must be a 64-character SHA-256 hex digest") + return text + + +class PublicBookLedger: + def __init__(self, path: str | Path) -> None: + self.path = Path(path) + self.rows: list[dict[str, Any]] = [] + self.head_hash = "0" * 64 + self.observation_ids: set[str] = set() + if self.path.exists(): + self._load_and_verify() + + def _load_and_verify(self) -> None: + previous = "0" * 64 + rows: list[dict[str, Any]] = [] + seen: set[str] = set() + for line_number, line in enumerate(self.path.read_text(encoding="utf-8").splitlines(), start=1): + if not line.strip(): + continue + try: + record = json.loads(line) + except json.JSONDecodeError as exc: + raise BookLedgerIntegrityError(f"invalid JSON at line {line_number}") from exc + stored = str(record.get("record_hash", "")) + payload = dict(record) + payload.pop("record_hash", None) + if payload.get("previous_hash") != previous: + raise BookLedgerIntegrityError(f"book chain mismatch at line {line_number}") + expected = hashlib.sha256(_canonical_json(payload)).hexdigest() + if stored != expected: + raise BookLedgerIntegrityError(f"book record hash mismatch at line {line_number}") + observation_id = str(payload.get("observation_id") or "") + if not observation_id or observation_id in seen: + raise BookLedgerIntegrityError(f"duplicate or empty observation_id at line {line_number}") + for field in ("policy_sha256", "source_sha256", "prospective_head_sha256"): + try: + _sha256(field, str(payload.get(field, ""))) + except ValueError as exc: + raise BookLedgerIntegrityError(f"invalid {field} at line {line_number}") from exc + book = payload.get("book") + if not isinstance(book, dict): + raise BookLedgerIntegrityError(f"missing book payload at line {line_number}") + payload_sha = str(book.get("payload_sha256") or "") + _sha256("payload_sha256", payload_sha) + rows.append(record) + seen.add(observation_id) + previous = stored + self.rows = rows + self.observation_ids = seen + self.head_hash = previous + + def append( + self, + *, + condition_id: str, + outcome: str, + observed_ts_ms: int, + book: PublicOrderBook, + policy_sha256: str, + source_sha256: str, + prospective_head_sha256: str, + ) -> dict[str, Any]: + timestamp = int(observed_ts_ms) + if timestamp < 0: + raise ValueError("observed_ts_ms must be non-negative") + side = str(outcome).lower() + if side not in {"yes", "no"}: + raise ValueError("outcome must be yes or no") + condition = str(condition_id) + if book.condition_id != condition: + raise ValueError(f"book condition mismatch: {book.condition_id} != {condition}") + observation_id = hashlib.sha256( + f"{condition}\0{book.token_id}\0{timestamp}\0{book.payload_sha256}".encode() + ).hexdigest() + if observation_id in self.observation_ids: + raise ValueError("duplicate book observation") + payload = { + "book": book.as_json(), + "condition_id": condition, + "observation_id": observation_id, + "observed_ts_ms": timestamp, + "outcome": side, + "policy_sha256": _sha256("policy_sha256", policy_sha256), + "previous_hash": self.head_hash, + "prospective_head_sha256": _sha256("prospective_head_sha256", prospective_head_sha256), + "source_sha256": _sha256("source_sha256", source_sha256), + } + record = {**payload, "record_hash": hashlib.sha256(_canonical_json(payload)).hexdigest()} + self.path.parent.mkdir(parents=True, exist_ok=True) + with self.path.open("a", encoding="utf-8", newline="\n") as handle: + handle.write(json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n") + handle.flush() + self.rows.append(record) + self.observation_ids.add(observation_id) + self.head_hash = record["record_hash"] + return dict(record) diff --git a/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/diagnostic_filter.py b/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/diagnostic_filter.py new file mode 100644 index 00000000..a95e97ae --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/diagnostic_filter.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import base64 +import hashlib +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable + + +def _sha256(name: str, value: str) -> str: + text = str(value).strip().lower() + if len(text) != 64 or any(ch not in "0123456789abcdef" for ch in text): + raise ValueError(f"{name} must be a 64-character SHA-256 hex digest") + return text + + +def _normalise(value: str) -> bytes: + return str(value).strip().lower().encode("utf-8") + + +def _positions(value: str, *, m_bits: int, k_hashes: int) -> tuple[int, ...]: + digest = hashlib.sha256(_normalise(value)).digest() + h1 = int.from_bytes(digest[:8], "big") + h2 = int.from_bytes(digest[8:16], "big") or 0x9E3779B97F4A7C15 + return tuple((h1 + index * h2) % m_bits for index in range(k_hashes)) + + +def build_diagnostic_bloom( + market_ids: Iterable[str], + *, + source_sha256: str, + m_bits: int = 262_144, + k_hashes: int = 7, +) -> dict[str, object]: + if m_bits <= 0 or m_bits % 8: + raise ValueError("m_bits must be a positive multiple of 8") + if k_hashes <= 0: + raise ValueError("k_hashes must be positive") + ids = sorted({str(value).strip().lower() for value in market_ids if str(value).strip()}) + bitset = bytearray(m_bits // 8) + for value in ids: + for position in _positions(value, m_bits=m_bits, k_hashes=k_hashes): + bitset[position // 8] |= 1 << (position % 8) + raw = bytes(bitset) + return { + "algorithm": "sha256-double-hash-v1", + "bitset_base64": base64.b64encode(raw).decode("ascii"), + "bitset_sha256": hashlib.sha256(raw).hexdigest(), + "item_count": len(ids), + "k_hashes": int(k_hashes), + "m_bits": int(m_bits), + "source_sha256": _sha256("source_sha256", source_sha256), + "version": 1, + } + + +@dataclass(frozen=True, slots=True) +class DiagnosticBloomFilter: + bitset: bytes + m_bits: int + k_hashes: int + item_count: int + source_sha256: str + + @classmethod + def from_payload(cls, payload: dict[str, object]) -> "DiagnosticBloomFilter": + if int(payload.get("version", -1)) != 1: + raise ValueError("unsupported diagnostic Bloom filter version") + if payload.get("algorithm") != "sha256-double-hash-v1": + raise ValueError("unsupported diagnostic Bloom filter algorithm") + m_bits = int(payload["m_bits"]) + k_hashes = int(payload["k_hashes"]) + item_count = int(payload["item_count"]) + if m_bits <= 0 or m_bits % 8: + raise ValueError("m_bits must be a positive multiple of 8") + if k_hashes <= 0 or item_count < 0: + raise ValueError("invalid Bloom filter dimensions") + try: + bitset = base64.b64decode(str(payload["bitset_base64"]), validate=True) + except Exception as exc: + raise ValueError("invalid Bloom filter base64") from exc + if len(bitset) != m_bits // 8: + raise ValueError("Bloom filter bitset length mismatch") + expected = _sha256("bitset_sha256", str(payload["bitset_sha256"])) + got = hashlib.sha256(bitset).hexdigest() + if got != expected: + raise ValueError(f"bitset SHA-256 mismatch: {got} != {expected}") + return cls( + bitset=bitset, + m_bits=m_bits, + k_hashes=k_hashes, + item_count=item_count, + source_sha256=_sha256("source_sha256", str(payload["source_sha256"])), + ) + + @classmethod + def from_path(cls, path: str | Path) -> "DiagnosticBloomFilter": + return cls.from_payload(json.loads(Path(path).read_text(encoding="utf-8"))) + + def __contains__(self, market_id: object) -> bool: + value = str(market_id) + return all( + bool(self.bitset[position // 8] & (1 << (position % 8))) + for position in _positions(value, m_bits=self.m_bits, k_hashes=self.k_hashes) + ) diff --git a/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/market_discovery.py b/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/market_discovery.py new file mode 100644 index 00000000..4f9ff8e2 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/market_discovery.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from decimal import Decimal +from typing import Any, Callable, Mapping, Sequence + + +_TARGET_SLUG = re.compile(r"^btc-updown-(5m|15m)-(\d{10,13})$") + + +def _bool(value: object) -> bool: + if isinstance(value, bool): + return value + return str(value).strip().lower() in {"1", "true", "yes"} + + +def _list(value: object, name: str) -> list[str]: + parsed = json.loads(value) if isinstance(value, str) else value + if not isinstance(parsed, Sequence) or isinstance(parsed, (str, bytes)): + raise ValueError(f"{name} must be a JSON array or sequence") + return [str(item) for item in parsed] + + +@dataclass(frozen=True, slots=True) +class GammaMarketRecord: + condition_id: str + slug: str + question: str + yes_token_id: str + no_token_id: str + duration_seconds: int + open_epoch_seconds: int + end_date: str + tick_size: Decimal + + +def parse_gamma_market(payload: Mapping[str, Any]) -> GammaMarketRecord: + condition_id = str(payload.get("conditionId") or payload.get("condition_id") or "").strip() + slug = str(payload.get("slug") or "").strip().lower() + match = _TARGET_SLUG.fullmatch(slug) + if not condition_id or match is None: + raise ValueError("market is not a BTC 5m/15m up/down contract") + token_ids = _list(payload.get("clobTokenIds") or payload.get("clob_token_ids"), "clobTokenIds") + if len(token_ids) != 2 or not all(token_ids): + raise ValueError("clobTokenIds must contain exactly two token IDs") + outcomes = _list(payload.get("outcomes", '["Up","Down"]'), "outcomes") + if len(outcomes) != 2: + raise ValueError("outcomes must contain exactly two values") + first = outcomes[0].strip().lower() + second = outcomes[1].strip().lower() + if first not in {"up", "yes"} or second not in {"down", "no"}: + raise ValueError(f"unexpected outcome order: {outcomes}") + epoch = int(match.group(2)) + if epoch >= 100_000_000_000: + epoch //= 1000 + return GammaMarketRecord( + condition_id=condition_id, + slug=slug, + question=str(payload.get("question") or ""), + yes_token_id=token_ids[0], + no_token_id=token_ids[1], + duration_seconds=300 if match.group(1) == "5m" else 900, + open_epoch_seconds=epoch, + end_date=str(payload.get("endDate") or payload.get("end_date") or ""), + tick_size=Decimal(str(payload.get("orderPriceMinTickSize") or payload.get("order_price_min_tick_size") or "0.01")), + ) + + +def _is_target(payload: Mapping[str, Any]) -> bool: + slug = str(payload.get("slug") or "").strip().lower() + return ( + _TARGET_SLUG.fullmatch(slug) is not None + and _bool(payload.get("active", False)) + and not _bool(payload.get("closed", False)) + and _bool(payload.get("enableOrderBook", payload.get("enable_order_book", False))) + ) + + +def discover_target_markets( + fetch_page: Callable[[dict[str, object]], Sequence[Mapping[str, Any]]], + *, + page_size: int = 100, + max_pages: int = 20, +) -> list[GammaMarketRecord]: + if page_size <= 0 or max_pages <= 0: + raise ValueError("page_size and max_pages must be positive") + records: dict[str, GammaMarketRecord] = {} + for page in range(max_pages): + offset = page * page_size + payloads = list(fetch_page({ + "active": "true", + "closed": "false", + "limit": page_size, + "offset": offset, + })) + for payload in payloads: + if _is_target(payload): + record = parse_gamma_market(payload) + records.setdefault(record.condition_id, record) + if len(payloads) < page_size: + break + return sorted(records.values(), key=lambda row: (row.open_epoch_seconds, row.duration_seconds, row.condition_id)) diff --git a/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/public_book_observer.py b/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/public_book_observer.py new file mode 100644 index 00000000..3a2c6c1a --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/public_book_observer.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import time +from pathlib import Path +from typing import Sequence + +from .book_ledger import PublicBookLedger +from .diagnostic_filter import DiagnosticBloomFilter +from .market_discovery import GammaMarketRecord, discover_target_markets +from .prospective import ProspectiveLedger +from .public_books import PublicOrderBook, fetch_public_order_books +from .public_observer import fetch_gamma_page + + +def _sha256_file(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def observe_public_books( + markets: Sequence[GammaMarketRecord], + books: dict[str, PublicOrderBook], + *, + ledger: PublicBookLedger, + prospective_market_ids: set[str], + prospective_head_sha256: str, + observed_ts_ms: int, + policy_sha256: str, + source_sha256: str, +) -> dict[str, object]: + requested = 0 + appended = 0 + missing: list[str] = [] + skipped_unregistered = 0 + for market in markets: + if market.condition_id not in prospective_market_ids: + skipped_unregistered += 1 + continue + for outcome, token_id in (("yes", market.yes_token_id), ("no", market.no_token_id)): + requested += 1 + book = books.get(token_id) + if book is None: + missing.append(token_id) + continue + ledger.append( + condition_id=market.condition_id, + outcome=outcome, + observed_ts_ms=observed_ts_ms, + book=book, + policy_sha256=policy_sha256, + source_sha256=source_sha256, + prospective_head_sha256=prospective_head_sha256, + ) + appended += 1 + return { + "markets_discovered": len(markets), + "markets_registered": len(prospective_market_ids), + "markets_skipped_unregistered": skipped_unregistered, + "tokens_requested": requested, + "books_received": len(books), + "books_appended": appended, + "missing_token_ids": sorted(missing), + "book_ledger_head_sha256": ledger.head_hash, + "book_observations_total": len(ledger.rows), + "prospective_head_sha256": prospective_head_sha256, + "observed_ts_ms": int(observed_ts_ms), + } + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Collect public Polymarket orderbooks without credentials") + parser.add_argument("--diagnostic-filter", type=Path, required=True) + parser.add_argument("--prospective-ledger", type=Path, required=True) + parser.add_argument("--book-ledger", type=Path, required=True) + parser.add_argument("--policy", type=Path, required=True) + parser.add_argument("--source-manifest", type=Path, required=True) + parser.add_argument("--summary", type=Path, required=True) + parser.add_argument("--page-size", type=int, default=100) + parser.add_argument("--max-pages", type=int, default=20) + args = parser.parse_args(argv) + + bloom = DiagnosticBloomFilter.from_path(args.diagnostic_filter) + prospective = ProspectiveLedger(args.prospective_ledger, diagnostic_market_ids=bloom) + markets = discover_target_markets( + lambda params: fetch_gamma_page(params), + page_size=args.page_size, + max_pages=args.max_pages, + ) + registered = [market for market in markets if market.condition_id in prospective.market_ids] + token_ids = [token for market in registered for token in (market.yes_token_id, market.no_token_id)] + books = fetch_public_order_books(token_ids) + book_ledger = PublicBookLedger(args.book_ledger) + summary = observe_public_books( + markets, + books, + ledger=book_ledger, + prospective_market_ids=prospective.market_ids, + prospective_head_sha256=prospective.head_hash, + observed_ts_ms=int(time.time() * 1000), + policy_sha256=_sha256_file(args.policy), + source_sha256=_sha256_file(args.source_manifest), + ) + summary.update({ + "authenticated_requests": 0, + "order_submissions": 0, + "trading_mode": "observation_only", + "public_books_endpoint": "https://clob.polymarket.com/books", + }) + args.summary.parent.mkdir(parents=True, exist_ok=True) + args.summary.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/public_books.py b/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/public_books.py new file mode 100644 index 00000000..c4f0ba1c --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/public_books.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from decimal import Decimal +from typing import Any, Callable, Mapping, Sequence + + +CLOB_BOOKS_URL = "https://clob.polymarket.com/books" + + +def _canonical_json(value: object) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +@dataclass(frozen=True, slots=True) +class BookLevel: + price: Decimal + size: Decimal + + def as_json(self) -> dict[str, str]: + return {"price": str(self.price), "size": str(self.size)} + + +@dataclass(frozen=True, slots=True) +class PublicOrderBook: + condition_id: str + token_id: str + timestamp: str + book_hash: str + bids: tuple[BookLevel, ...] + asks: tuple[BookLevel, ...] + min_order_size: Decimal + tick_size: Decimal + neg_risk: bool + last_trade_price: Decimal | None + payload_sha256: str + + @property + def best_bid(self) -> BookLevel | None: + return self.bids[0] if self.bids else None + + @property + def best_ask(self) -> BookLevel | None: + return self.asks[0] if self.asks else None + + def as_json(self) -> dict[str, object]: + return { + "condition_id": self.condition_id, + "token_id": self.token_id, + "timestamp": self.timestamp, + "book_hash": self.book_hash, + "bids": [level.as_json() for level in self.bids], + "asks": [level.as_json() for level in self.asks], + "min_order_size": str(self.min_order_size), + "tick_size": str(self.tick_size), + "neg_risk": self.neg_risk, + "last_trade_price": None if self.last_trade_price is None else str(self.last_trade_price), + "payload_sha256": self.payload_sha256, + } + + +def _levels(value: object, *, side: str) -> tuple[BookLevel, ...]: + if value is None: + return () + if not isinstance(value, Sequence) or isinstance(value, (str, bytes)): + raise ValueError(f"{side} must be a sequence") + levels: list[BookLevel] = [] + for row in value: + if not isinstance(row, Mapping): + raise ValueError(f"{side} level must be an object") + price = Decimal(str(row.get("price"))) + size = Decimal(str(row.get("size"))) + if not (Decimal("0") < price < Decimal("1")): + raise ValueError(f"{side} price must be between zero and one") + if size < 0: + raise ValueError(f"{side} size must be non-negative") + if size: + levels.append(BookLevel(price=price, size=size)) + reverse = side == "bids" + return tuple(sorted(levels, key=lambda level: level.price, reverse=reverse)) + + +def parse_public_order_book( + payload: Mapping[str, Any], + *, + expected_token_id: str | None = None, +) -> PublicOrderBook: + token_id = str(payload.get("asset_id") or "").strip() + condition_id = str(payload.get("market") or "").strip() + if not token_id or not condition_id: + raise ValueError("orderbook is missing market or asset_id") + if expected_token_id is not None and token_id != str(expected_token_id): + raise ValueError(f"orderbook asset mismatch: {token_id} != {expected_token_id}") + raw_last = payload.get("last_trade_price") + last_trade = None if raw_last in (None, "") else Decimal(str(raw_last)) + return PublicOrderBook( + condition_id=condition_id, + token_id=token_id, + timestamp=str(payload.get("timestamp") or ""), + book_hash=str(payload.get("hash") or ""), + bids=_levels(payload.get("bids", []), side="bids"), + asks=_levels(payload.get("asks", []), side="asks"), + min_order_size=Decimal(str(payload.get("min_order_size") or "0")), + tick_size=Decimal(str(payload.get("tick_size") or "0.01")), + neg_risk=bool(payload.get("neg_risk", False)), + last_trade_price=last_trade, + payload_sha256=hashlib.sha256(_canonical_json(dict(payload))).hexdigest(), + ) + + +def fetch_public_order_books( + token_ids: Sequence[str], + *, + post: Callable[..., Any] | None = None, + timeout_seconds: float = 15.0, + batch_size: int = 500, +) -> dict[str, PublicOrderBook]: + if batch_size <= 0 or batch_size > 500: + raise ValueError("batch_size must be between 1 and 500") + ordered = list(dict.fromkeys(str(token).strip() for token in token_ids if str(token).strip())) + if not ordered: + return {} + if post is None: + import requests + post = requests.post + result: dict[str, PublicOrderBook] = {} + for start in range(0, len(ordered), batch_size): + batch = ordered[start : start + batch_size] + response = post( + CLOB_BOOKS_URL, + json=[{"token_id": token} for token in batch], + timeout=timeout_seconds, + ) + response.raise_for_status() + payload = response.json() + if not isinstance(payload, list): + raise ValueError("public books response must be a list") + expected = set(batch) + for raw in payload: + book = parse_public_order_book(raw) + if book.token_id not in expected: + raise ValueError(f"unexpected orderbook asset: {book.token_id}") + if book.token_id in result: + raise ValueError(f"duplicate orderbook asset: {book.token_id}") + result[book.token_id] = book + return result diff --git a/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/public_observer.py b/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/public_observer.py new file mode 100644 index 00000000..10330a41 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/public_observer.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import time +from pathlib import Path +from typing import Any, Mapping, Sequence + +from .diagnostic_filter import DiagnosticBloomFilter +from .market_discovery import GammaMarketRecord, discover_target_markets +from .prospective import ProspectiveLedger + + +GAMMA_MARKETS_URL = "https://gamma-api.polymarket.com/markets" + + +def fetch_gamma_page(params: Mapping[str, object], *, timeout_seconds: float = 15.0) -> Sequence[Mapping[str, Any]]: + """Fetch a public Gamma page. No credentials or authenticated endpoint is used.""" + import requests + + response = requests.get(GAMMA_MARKETS_URL, params=dict(params), timeout=timeout_seconds) + response.raise_for_status() + payload = response.json() + if not isinstance(payload, list): + raise ValueError("Gamma markets response must be a list") + return payload + + +def _metadata(record: GammaMarketRecord) -> dict[str, object]: + return { + "duration_seconds": int(record.duration_seconds), + "end_date": record.end_date, + "no_token_id": record.no_token_id, + "open_epoch_seconds": int(record.open_epoch_seconds), + "question": record.question, + "slug": record.slug, + "tick_size": str(record.tick_size), + "yes_token_id": record.yes_token_id, + } + + +def observe_markets( + markets: Sequence[GammaMarketRecord], + *, + ledger: ProspectiveLedger, + observed_ts_ms: int, + policy_sha256: str, + source_sha256: str, +) -> dict[str, object]: + counters = { + "discovered": len(markets), + "appended": 0, + "skipped_diagnostic": 0, + "skipped_existing": 0, + } + appended: list[str] = [] + for market in markets: + condition_id = str(market.condition_id) + if condition_id in ledger.diagnostic_market_ids: + counters["skipped_diagnostic"] += 1 + continue + if condition_id in ledger.market_ids: + counters["skipped_existing"] += 1 + continue + ledger.append_observation( + market_id=condition_id, + first_seen_ts_ms=int(observed_ts_ms), + policy_sha256=policy_sha256, + source_sha256=source_sha256, + metadata=_metadata(market), + ) + appended.append(condition_id) + counters["appended"] += 1 + return { + **counters, + "appended_condition_ids": appended, + "ledger_head_sha256": ledger.head_hash, + "markets_total_in_ledger": len(ledger.market_ids), + "observed_ts_ms": int(observed_ts_ms), + } + + +def _sha256_file(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Observe public BTC 5m/15m Polymarket markets without credentials") + parser.add_argument("--diagnostic-filter", type=Path, required=True) + parser.add_argument("--ledger", type=Path, required=True) + parser.add_argument("--policy", type=Path, required=True) + parser.add_argument("--source-manifest", type=Path, required=True) + parser.add_argument("--summary", type=Path, required=True) + parser.add_argument("--page-size", type=int, default=100) + parser.add_argument("--max-pages", type=int, default=20) + args = parser.parse_args(argv) + + bloom = DiagnosticBloomFilter.from_path(args.diagnostic_filter) + policy_sha256 = _sha256_file(args.policy) + source_sha256 = _sha256_file(args.source_manifest) + ledger = ProspectiveLedger(args.ledger, diagnostic_market_ids=bloom) + markets = discover_target_markets( + lambda params: fetch_gamma_page(params), + page_size=args.page_size, + max_pages=args.max_pages, + ) + summary = observe_markets( + markets, + ledger=ledger, + observed_ts_ms=int(time.time() * 1000), + policy_sha256=policy_sha256, + source_sha256=source_sha256, + ) + summary.update({ + "diagnostic_filter_source_sha256": bloom.source_sha256, + "policy_sha256": policy_sha256, + "source_sha256": source_sha256, + "trading_mode": "observation_only", + "authenticated_requests": 0, + "order_submissions": 0, + }) + args.summary.parent.mkdir(parents=True, exist_ok=True) + args.summary.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/shadow_observation_policy_v1.json b/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/shadow_observation_policy_v1.json new file mode 100644 index 00000000..48b8409e --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/observer-source/research/digital_surface/shadow_observation_policy_v1.json @@ -0,0 +1,11 @@ +{ + "authenticated_requests": false, + "market_endpoint": "https://gamma-api.polymarket.com/markets", + "mode": "observation_only", + "order_submission": false, + "target_slugs": [ + "btc-updown-5m-", + "btc-updown-15m-" + ], + "version": 1 +} diff --git a/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_book_ledger.py b/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_book_ledger.py new file mode 100644 index 00000000..c99b02ea --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_book_ledger.py @@ -0,0 +1,42 @@ +import json + +import pytest + +from research.digital_surface.book_ledger import BookLedgerIntegrityError, PublicBookLedger +from research.digital_surface.public_books import parse_public_order_book + + +def book(): + return parse_public_order_book({ + "market": "c1", "asset_id": "t1", "timestamp": "1", "hash": "h", + "bids": [{"price": "0.40", "size": "10"}], + "asks": [{"price": "0.50", "size": "10"}], + "min_order_size": "1", "tick_size": "0.01", "neg_risk": False, + }) + + +def test_appends_and_reloads_book_hash_chain(tmp_path): + path = tmp_path / "books.jsonl" + ledger = PublicBookLedger(path) + first = ledger.append( + condition_id="c1", outcome="yes", observed_ts_ms=1, book=book(), + policy_sha256="1"*64, source_sha256="2"*64, prospective_head_sha256="3"*64, + ) + assert first["previous_hash"] == "0"*64 + reloaded = PublicBookLedger(path) + assert reloaded.head_hash == first["record_hash"] + assert len(reloaded.rows) == 1 + + +def test_detects_book_ledger_tampering(tmp_path): + path = tmp_path / "books.jsonl" + ledger = PublicBookLedger(path) + ledger.append( + condition_id="c1", outcome="yes", observed_ts_ms=1, book=book(), + policy_sha256="1"*64, source_sha256="2"*64, prospective_head_sha256="3"*64, + ) + row = json.loads(path.read_text()) + row["book"]["asks"][0]["size"] = "999" + path.write_text(json.dumps(row) + "\n") + with pytest.raises(BookLedgerIntegrityError, match="record hash mismatch"): + PublicBookLedger(path) diff --git a/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_diagnostic_filter.py b/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_diagnostic_filter.py new file mode 100644 index 00000000..aa09d99f --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_diagnostic_filter.py @@ -0,0 +1,38 @@ +import json + +import pytest + +from research.digital_surface.diagnostic_filter import DiagnosticBloomFilter, build_diagnostic_bloom +from research.digital_surface.prospective import ProspectiveContaminationError, ProspectiveLedger + + +def test_bloom_round_trip_has_no_false_negatives_for_supplied_ids(tmp_path): + ids = [f"0x{index:064x}" for index in range(100)] + payload = build_diagnostic_bloom(ids, source_sha256="a" * 64, m_bits=4096, k_hashes=5) + path = tmp_path / "filter.json" + path.write_text(json.dumps(payload)) + loaded = DiagnosticBloomFilter.from_path(path) + assert all(value in loaded for value in ids) + assert loaded.item_count == 100 + assert loaded.source_sha256 == "a" * 64 + + +def test_bloom_can_enforce_prospective_contamination_rejection(tmp_path): + known = "0x" + "1" * 64 + payload = build_diagnostic_bloom([known], source_sha256="a" * 64, m_bits=1024, k_hashes=4) + bloom = DiagnosticBloomFilter.from_payload(payload) + ledger = ProspectiveLedger(tmp_path / "ledger.jsonl", diagnostic_market_ids=bloom) + with pytest.raises(ProspectiveContaminationError, match="appeared in diagnostic data"): + ledger.append_observation( + market_id=known, + first_seen_ts_ms=1, + policy_sha256="b" * 64, + source_sha256="c" * 64, + ) + + +def test_rejects_corrupt_filter_bitset(): + payload = build_diagnostic_bloom(["known"], source_sha256="a" * 64, m_bits=1024, k_hashes=4) + payload["bitset_sha256"] = "0" * 64 + with pytest.raises(ValueError, match="bitset SHA-256 mismatch"): + DiagnosticBloomFilter.from_payload(payload) diff --git a/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_market_discovery.py b/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_market_discovery.py new file mode 100644 index 00000000..71370e9d --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_market_discovery.py @@ -0,0 +1,45 @@ +import json + +from research.digital_surface.market_discovery import discover_target_markets, parse_gamma_market + + +def market(slug, condition, token_ids, *, active=True, closed=False, enable=True): + return { + "conditionId": condition, + "slug": slug, + "question": "Bitcoin Up or Down", + "active": active, + "closed": closed, + "enableOrderBook": enable, + "clobTokenIds": json.dumps(token_ids), + "outcomes": '["Up", "Down"]', + "orderPriceMinTickSize": 0.01, + "endDate": "2026-07-25T15:00:00Z", + } + + +def test_parses_gamma_token_order_and_target_duration(): + record = parse_gamma_market(market("btc-updown-5m-1774450800", "c1", ["yes", "no"])) + assert record.condition_id == "c1" + assert record.yes_token_id == "yes" + assert record.no_token_id == "no" + assert record.duration_seconds == 300 + assert record.open_epoch_seconds == 1774450800 + + +def test_discovery_filters_non_target_closed_and_duplicate_markets(): + pages = { + 0: [ + market("btc-updown-5m-1774450800", "c1", ["yes1", "no1"]), + market("btc-updown-15m-1774450800", "c2", ["yes2", "no2"]), + market("eth-updown-5m-1774450800", "c3", ["yes3", "no3"]), + market("btc-updown-5m-1774451100", "c4", ["yes4", "no4"], closed=True), + ], + 4: [market("btc-updown-5m-1774450800", "c1", ["yes1", "no1"])], + } + + def fetch(params): + return pages.get(params["offset"], []) + + found = discover_target_markets(fetch, page_size=4, max_pages=3) + assert [row.condition_id for row in found] == ["c1", "c2"] diff --git a/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_public_book_observer.py b/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_public_book_observer.py new file mode 100644 index 00000000..b200778c --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_public_book_observer.py @@ -0,0 +1,56 @@ +from decimal import Decimal + +from research.digital_surface.book_ledger import PublicBookLedger +from research.digital_surface.market_discovery import GammaMarketRecord +from research.digital_surface.public_book_observer import observe_public_books +from research.digital_surface.public_books import parse_public_order_book + + +def market(condition="c1"): + return GammaMarketRecord( + condition_id=condition, slug="btc-updown-5m-1774450800", question="Bitcoin Up or Down", + yes_token_id=f"yes-{condition}", no_token_id=f"no-{condition}", duration_seconds=300, + open_epoch_seconds=1774450800, end_date="2026-07-25T15:00:00Z", tick_size=Decimal("0.01"), + ) + + +def book(token, condition): + return parse_public_order_book({ + "market": condition, "asset_id": token, "timestamp": "1", "hash": f"h-{token}", + "bids": [{"price": "0.40", "size": "10"}], + "asks": [{"price": "0.50", "size": "10"}], + "min_order_size": "1", "tick_size": "0.01", "neg_risk": False, + }) + + +def test_observer_records_both_registered_token_books_and_binds_prospective_head(tmp_path): + item = market() + books = { + item.yes_token_id: book(item.yes_token_id, item.condition_id), + item.no_token_id: book(item.no_token_id, item.condition_id), + } + ledger = PublicBookLedger(tmp_path / "books.jsonl") + summary = observe_public_books( + [item], books, ledger=ledger, prospective_market_ids={"c1"}, + prospective_head_sha256="3"*64, observed_ts_ms=10, + policy_sha256="1"*64, source_sha256="2"*64, + ) + assert summary["books_appended"] == 2 + assert summary["missing_token_ids"] == [] + assert {row["outcome"] for row in ledger.rows} == {"yes", "no"} + assert all(row["prospective_head_sha256"] == "3"*64 for row in ledger.rows) + + +def test_observer_reports_missing_books_and_skips_unregistered_markets(tmp_path): + registered = market("c1") + unregistered = market("c2") + books = {registered.yes_token_id: book(registered.yes_token_id, registered.condition_id)} + ledger = PublicBookLedger(tmp_path / "books.jsonl") + summary = observe_public_books( + [registered, unregistered], books, ledger=ledger, prospective_market_ids={"c1"}, + prospective_head_sha256="3"*64, observed_ts_ms=10, + policy_sha256="1"*64, source_sha256="2"*64, + ) + assert summary["books_appended"] == 1 + assert summary["markets_skipped_unregistered"] == 1 + assert summary["missing_token_ids"] == [registered.no_token_id] diff --git a/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_public_books.py b/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_public_books.py new file mode 100644 index 00000000..b940e731 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_public_books.py @@ -0,0 +1,63 @@ +from decimal import Decimal + +import pytest + +from research.digital_surface.public_books import ( + CLOB_BOOKS_URL, + fetch_public_order_books, + parse_public_order_book, +) + + +class Response: + def __init__(self, payload): + self.payload = payload + def raise_for_status(self): + return None + def json(self): + return self.payload + + +def payload(token="t1", condition="c1"): + return { + "market": condition, + "asset_id": token, + "timestamp": "1234", + "hash": "book-hash", + "bids": [{"price": "0.44", "size": "2"}, {"price": "0.46", "size": "3"}], + "asks": [{"price": "0.55", "size": "4"}, {"price": "0.52", "size": "5"}], + "min_order_size": "1", + "tick_size": "0.01", + "neg_risk": False, + "last_trade_price": "0.50", + } + + +def test_parses_and_sorts_full_public_orderbook_depth(): + book = parse_public_order_book(payload(), expected_token_id="t1") + assert book.best_bid.price == Decimal("0.46") + assert book.best_bid.size == Decimal("3") + assert book.best_ask.price == Decimal("0.52") + assert [level.price for level in book.asks] == [Decimal("0.52"), Decimal("0.55")] + assert len(book.payload_sha256) == 64 + + +def test_rejects_asset_mismatch_and_invalid_levels(): + with pytest.raises(ValueError, match="asset mismatch"): + parse_public_order_book(payload(), expected_token_id="other") + broken = payload() + broken["asks"] = [{"price": "1.2", "size": "1"}] + with pytest.raises(ValueError, match="between zero and one"): + parse_public_order_book(broken) + + +def test_batch_fetch_uses_only_public_books_endpoint_without_headers(): + calls = [] + def post(url, **kwargs): + calls.append((url, kwargs)) + return Response([payload("t1"), payload("t2", "c2")]) + books = fetch_public_order_books(["t1", "t2"], post=post) + assert set(books) == {"t1", "t2"} + assert calls[0][0] == CLOB_BOOKS_URL + assert calls[0][1]["json"] == [{"token_id": "t1"}, {"token_id": "t2"}] + assert "headers" not in calls[0][1] diff --git a/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_public_observer.py b/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_public_observer.py new file mode 100644 index 00000000..305e2058 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/observer-source/tests/digital_surface/test_public_observer.py @@ -0,0 +1,63 @@ +import json + +from research.digital_surface.diagnostic_filter import DiagnosticBloomFilter, build_diagnostic_bloom +from research.digital_surface.market_discovery import GammaMarketRecord +from research.digital_surface.prospective import ProspectiveLedger +from research.digital_surface.public_observer import observe_markets + + +def record(condition, *, duration=300, epoch=1774450800): + return GammaMarketRecord( + condition_id=condition, + slug=f"btc-updown-{duration // 60}m-{epoch}", + question="Bitcoin Up or Down", + yes_token_id=f"yes-{condition}", + no_token_id=f"no-{condition}", + duration_seconds=duration, + open_epoch_seconds=epoch, + end_date="2026-07-25T15:00:00Z", + tick_size=__import__("decimal").Decimal("0.01"), + ) + + +def test_observer_rejects_historical_and_appends_full_new_market_metadata(tmp_path): + known = "known" + bloom = DiagnosticBloomFilter.from_payload( + build_diagnostic_bloom([known], source_sha256="a" * 64, m_bits=1024, k_hashes=4) + ) + ledger = ProspectiveLedger(tmp_path / "ledger.jsonl", diagnostic_market_ids=bloom) + summary = observe_markets( + [record(known), record("fresh")], + ledger=ledger, + observed_ts_ms=123, + policy_sha256="b" * 64, + source_sha256="c" * 64, + ) + assert summary["discovered"] == 2 + assert summary["appended"] == 1 + assert summary["skipped_diagnostic"] == 1 + row = json.loads((tmp_path / "ledger.jsonl").read_text()) + assert row["market_id"] == "fresh" + assert row["metadata"]["yes_token_id"] == "yes-fresh" + assert row["metadata"]["no_token_id"] == "no-fresh" + assert row["metadata"]["duration_seconds"] == 300 + assert row["metadata"]["tick_size"] == "0.01" + + +def test_observer_is_idempotent_for_already_observed_markets(tmp_path): + bloom = DiagnosticBloomFilter.from_payload( + build_diagnostic_bloom([], source_sha256="a" * 64, m_bits=1024, k_hashes=4) + ) + ledger = ProspectiveLedger(tmp_path / "ledger.jsonl", diagnostic_market_ids=bloom) + first = observe_markets( + [record("fresh")], ledger=ledger, observed_ts_ms=123, + policy_sha256="b" * 64, source_sha256="c" * 64, + ) + second = observe_markets( + [record("fresh")], ledger=ledger, observed_ts_ms=124, + policy_sha256="b" * 64, source_sha256="c" * 64, + ) + assert first["appended"] == 1 + assert second["appended"] == 0 + assert second["skipped_existing"] == 1 + assert len((tmp_path / "ledger.jsonl").read_text().splitlines()) == 1 diff --git a/ci_payload/digital-surface-live-readiness/prospective-signal-source/research/digital_surface/prospective_signal.py b/ci_payload/digital-surface-live-readiness/prospective-signal-source/research/digital_surface/prospective_signal.py new file mode 100644 index 00000000..af767d5a --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/prospective-signal-source/research/digital_surface/prospective_signal.py @@ -0,0 +1,242 @@ +from __future__ import annotations + +import hashlib +import json +import math +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping, Sequence + +import numpy as np +import pandas as pd + +from .binance_public import BTCMarketState +from .candidates import CandidateConfig, generate_cross_horizon_candidates, select_one_candidate_per_expiry +from .market_discovery import GammaMarketRecord +from .public_books import PublicOrderBook + + +_EPS = 1e-6 + + +def _sha256(name: str, value: str) -> str: + text = str(value).lower() + if len(text) != 64 or any(character not in "0123456789abcdef" for character in text): + raise ValueError(f"{name} must be a 64-character SHA-256 hex digest") + return text + + +def _normal_cdf(value: float) -> float: + return 0.5 * (1.0 + math.erf(float(value) / math.sqrt(2.0))) + + +def _logit(probability: float) -> float: + p = min(max(float(probability), _EPS), 1.0 - _EPS) + return math.log(p / (1.0 - p)) + + +def _expit(value: float) -> float: + if value >= 0: + inverse = math.exp(-value) + return 1.0 / (1.0 + inverse) + exponent = math.exp(value) + return exponent / (1.0 + exponent) + + +@dataclass(frozen=True, slots=True) +class FrozenSurfacePolicy: + coefficients: tuple[float, float, float, float] + training_rows: int + training_contracts: int + config: CandidateConfig + policy_sha256: str + selection: str = "first_trigger_per_expiry" + flow_assumption: str = "zero_without_public_causal_trade_flow" + + @classmethod + def from_payload(cls, payload: Mapping[str, Any], *, policy_sha256: str) -> "FrozenSurfacePolicy": + if int(payload.get("version", -1)) != 1: + raise ValueError("unsupported frozen surface policy version") + model = payload.get("model") + candidate = payload.get("candidate") + if not isinstance(model, Mapping) or not isinstance(candidate, Mapping): + raise ValueError("frozen policy requires model and candidate objects") + coefficients = tuple(float(value) for value in model.get("coefficients", [])) + if len(coefficients) != 4 or not all(math.isfinite(value) for value in coefficients): + raise ValueError("surface coefficients must contain four finite values") + if coefficients[1] < 0.05: + raise ValueError("structural slope must preserve positive spot monotonicity") + training_rows = int(model.get("training_rows", 0)) + training_contracts = int(model.get("training_contracts", 0)) + if training_rows < 0 or training_contracts <= 0: + raise ValueError("frozen policy requires positive training_contracts") + config = CandidateConfig( + min_edge=float(candidate["min_edge"]), + max_total_cost=float(candidate["max_total_cost"]), + min_tau_s=float(candidate.get("min_tau_s", 3.0)), + max_tau_s=float(candidate["max_tau_s"]), + decision_cadence_s=int(candidate.get("decision_cadence_s", 5)), + pair_gap_ms=int(candidate.get("pair_gap_ms", 2_000)), + slippage=float(candidate.get("slippage", 0.01)), + uncertainty_penalty=float(candidate.get("uncertainty_penalty", 0.0)), + ) + selection = str(payload.get("selection") or "") + if selection != "first_trigger_per_expiry": + raise ValueError("only the causal first-trigger selection policy is supported") + flow_assumption = str(payload.get("flow_assumption") or "") + if flow_assumption != "zero_without_public_causal_trade_flow": + raise ValueError("prospective policy must freeze the zero-flow assumption") + return cls( + coefficients=(coefficients[0], coefficients[1], coefficients[2], coefficients[3]), + training_rows=training_rows, + training_contracts=training_contracts, + config=config, + policy_sha256=_sha256("policy_sha256", policy_sha256), + selection=selection, + flow_assumption=flow_assumption, + ) + + def fair_yes(self, *, spot: float, strike: float, tau_s: float, volatility_per_s: float, horizon_s: int) -> tuple[float, float, float]: + if spot <= 0 or strike <= 0 or tau_s <= 0 or volatility_per_s <= 0: + raise ValueError("surface inputs must be positive") + z_moneyness = math.log(spot / strike) / (volatility_per_s * math.sqrt(tau_s)) + structural = min(max(_normal_cdf(z_moneyness), _EPS), 1.0 - _EPS) + intercept, structural_slope, horizon_offset, flow_coefficient = self.coefficients + eta = intercept + structural_slope * _logit(structural) + horizon_offset * (1.0 if horizon_s == 900 else 0.0) + eta += flow_coefficient * 0.0 + fair = min(max(_expit(eta), _EPS), 1.0 - _EPS) + uncertainty = math.sqrt(fair * (1.0 - fair) / self.training_contracts) + return fair, uncertainty, z_moneyness + + +def load_frozen_surface_policy(path: str | Path) -> FrozenSurfacePolicy: + policy_path = Path(path) + raw = policy_path.read_bytes() + payload = json.loads(raw) + if not isinstance(payload, Mapping): + raise ValueError("frozen surface policy must be a JSON object") + return FrozenSurfacePolicy.from_payload(payload, policy_sha256=hashlib.sha256(raw).hexdigest()) + + +def generate_prospective_candidates( + markets: Sequence[GammaMarketRecord], + books: Mapping[str, PublicOrderBook], + state: BTCMarketState, + *, + book_observed_ts_ms: int, + policy: FrozenSurfacePolicy, + prospective_head_sha256: str, + book_ledger_head_sha256: str, + spot_state_head_sha256: str, + max_evidence_age_ms: int = 2_000, +) -> tuple[pd.DataFrame, dict[str, object]]: + evidence_hashes = { + "policy_sha256": _sha256("policy_sha256", policy.policy_sha256), + "prospective_head_sha256": _sha256("prospective_head_sha256", prospective_head_sha256), + "book_ledger_head_sha256": _sha256("book_ledger_head_sha256", book_ledger_head_sha256), + "spot_state_head_sha256": _sha256("spot_state_head_sha256", spot_state_head_sha256), + } + book_ts = int(book_observed_ts_ms) + decision_ts_ms = max(book_ts, int(state.observed_ts_ms)) + if max_evidence_age_ms < 0: + raise ValueError("max_evidence_age_ms must be non-negative") + book_age = decision_ts_ms - book_ts + state_age = decision_ts_ms - int(state.observed_ts_ms) + if book_age > max_evidence_age_ms or state_age > max_evidence_age_ms: + raise ValueError( + f"stale evidence: book_age_ms={book_age}, state_age_ms={state_age}, max={max_evidence_age_ms}" + ) + + volatility = float(state.vol_30s) + if not math.isfinite(volatility) or volatility <= 0: + volatility = float(state.vol_120s) + if not math.isfinite(volatility) or volatility <= 0: + raise ValueError("BTC state does not contain positive causal volatility") + + rows: list[dict[str, object]] = [] + skipped_missing_books = 0 + skipped_empty_books = 0 + for market in markets: + duration = {300: "5m", 900: "15m"}.get(int(market.duration_seconds)) + if duration is None: + continue + boundary_ms = int(market.open_epoch_seconds) * 1_000 + if boundary_ms not in state.strikes: + raise ValueError(f"missing exact strike for {market.condition_id} at boundary {boundary_ms}") + yes_book = books.get(market.yes_token_id) + no_book = books.get(market.no_token_id) + if yes_book is None or no_book is None: + skipped_missing_books += 1 + continue + if yes_book.condition_id != market.condition_id or no_book.condition_id != market.condition_id: + raise ValueError(f"book condition mismatch for {market.condition_id}") + if yes_book.best_ask is None or no_book.best_ask is None: + skipped_empty_books += 1 + continue + close_ts = int(market.open_epoch_seconds) + int(market.duration_seconds) + tau_s = (close_ts * 1_000 - decision_ts_ms) / 1_000.0 + if tau_s <= 0: + continue + strike = float(state.strikes[boundary_ms]) + fair, uncertainty, z_moneyness = policy.fair_yes( + spot=float(state.spot), + strike=strike, + tau_s=tau_s, + volatility_per_s=volatility, + horizon_s=int(market.duration_seconds), + ) + rows.append( + { + "condition_id": market.condition_id, + "duration": duration, + "close_ts": close_ts, + "ts_ms": decision_ts_ms, + "strike": strike, + "yes_token_id": market.yes_token_id, + "no_token_id": market.no_token_id, + "yes_ask": float(yes_book.best_ask.price), + "no_ask": float(no_book.best_ask.price), + "yes_ask_size": float(yes_book.best_ask.size), + "no_ask_size": float(no_book.best_ask.size), + "fair_yes": fair, + "surface_uncertainty": uncertainty, + "resolved_yes": False, + "tau_s": tau_s, + "z_moneyness": z_moneyness, + "horizon_s": int(market.duration_seconds), + "signed_trade_flow_5s": 0.0, + } + ) + + surface = pd.DataFrame(rows) + if surface.empty: + candidates = pd.DataFrame() + else: + candidates = select_one_candidate_per_expiry( + generate_cross_horizon_candidates(surface, config=policy.config) + ) + if not candidates.empty: + for name, value in evidence_hashes.items(): + candidates[name] = value + candidates["book_observed_ts_ms"] = book_ts + candidates["spot_observed_ts_ms"] = int(state.observed_ts_ms) + candidates["decision_ts_ms"] = decision_ts_ms + candidates["fill_model"] = "public_book_arrival_shadow" + candidates["transactionally_atomic"] = False + candidates["future_outcomes_used"] = 0 + + summary = { + "markets_seen": len(markets), + "surface_rows": len(surface), + "candidates_selected": len(candidates), + "skipped_missing_books": skipped_missing_books, + "skipped_empty_books": skipped_empty_books, + "decision_ts_ms": decision_ts_ms, + "book_age_ms": book_age, + "spot_state_age_ms": state_age, + "policy_config_id": policy.config.config_id, + "future_outcomes_used": 0, + "transactionally_atomic": False, + **evidence_hashes, + } + return candidates.reset_index(drop=True), summary diff --git a/ci_payload/digital-surface-live-readiness/prospective-signal-source/tests/digital_surface/test_prospective_signal.py b/ci_payload/digital-surface-live-readiness/prospective-signal-source/tests/digital_surface/test_prospective_signal.py new file mode 100644 index 00000000..fe5bb75a --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/prospective-signal-source/tests/digital_surface/test_prospective_signal.py @@ -0,0 +1,167 @@ +import json +from decimal import Decimal + +from research.digital_surface.binance_public import BTCMarketState +from research.digital_surface.market_discovery import GammaMarketRecord +from research.digital_surface.prospective_signal import ( + FrozenSurfacePolicy, + generate_prospective_candidates, + load_frozen_surface_policy, +) +from research.digital_surface.public_books import parse_public_order_book + + +def market(condition, *, epoch, duration, token_prefix): + return GammaMarketRecord( + condition_id=condition, + slug=f"btc-updown-{duration // 60}m-{epoch}", + question="Bitcoin Up or Down", + yes_token_id=f"yes-{token_prefix}", + no_token_id=f"no-{token_prefix}", + duration_seconds=duration, + open_epoch_seconds=epoch, + end_date="2030-01-01T00:00:00Z", + tick_size=Decimal("0.01"), + ) + + +def book(token, condition, yes_ask="0.45", no_ask="0.45", size="10"): + ask = yes_ask if token.startswith("yes-") else no_ask + return parse_public_order_book({ + "market": condition, + "asset_id": token, + "timestamp": "1250000", + "hash": f"hash-{token}", + "bids": [{"price": "0.40", "size": size}], + "asks": [{"price": ask, "size": size}], + "min_order_size": "1", + "tick_size": "0.01", + "neg_risk": False, + }) + + +def policy_payload(): + return { + "version": 1, + "model": { + "coefficients": [0.0, 1.0, 0.0, 0.0], + "training_rows": 1000, + "training_contracts": 500, + }, + "candidate": { + "min_edge": 0.02, + "max_total_cost": 1.25, + "min_tau_s": 3.0, + "max_tau_s": 60.0, + "decision_cadence_s": 5, + "pair_gap_ms": 2000, + "slippage": 0.01, + "uncertainty_penalty": 0.0, + }, + "selection": "first_trigger_per_expiry", + "flow_assumption": "zero_without_public_causal_trade_flow", + } + + +def setup_case(): + five = market("five", epoch=1_000, duration=300, token_prefix="five") + fifteen = market("fifteen", epoch=400, duration=900, token_prefix="fifteen") + books = { + five.yes_token_id: book(five.yes_token_id, five.condition_id), + five.no_token_id: book(five.no_token_id, five.condition_id), + fifteen.yes_token_id: book(fifteen.yes_token_id, fifteen.condition_id), + fifteen.no_token_id: book(fifteen.no_token_id, fifteen.condition_id), + } + state = BTCMarketState( + symbol="BTCUSDT", + server_time_ms=1_250_000, + observed_ts_ms=1_250_000, + spot=Decimal("100"), + vol_30s=0.001, + vol_120s=0.0015, + strikes={1_000_000: Decimal("99"), 400_000: Decimal("101")}, + closed_one_second_bars=121, + raw_response_sha256={ + "server_time": "a"*64, + "one_second_klines": "b"*64, + "strike:1000000": "c"*64, + "strike:400000": "d"*64, + }, + ) + return [five, fifteen], books, state + + +def test_loads_frozen_policy_and_hashes_exact_file_bytes(tmp_path): + path = tmp_path / "policy.json" + path.write_text(json.dumps(policy_payload(), indent=2, sort_keys=True) + "\n") + policy = load_frozen_surface_policy(path) + assert isinstance(policy, FrozenSurfacePolicy) + assert policy.config.config_id == "e0.0200-c1.2500-tau3-60-cad5-slip0.010-u0.00" + assert len(policy.policy_sha256) == 64 + assert policy.training_contracts == 500 + + +def test_generates_original_same_expiry_lower_yes_higher_no_candidate(): + markets, books, state = setup_case() + policy = FrozenSurfacePolicy.from_payload(policy_payload(), policy_sha256="1"*64) + candidates, summary = generate_prospective_candidates( + markets, + books, + state, + book_observed_ts_ms=1_250_000, + policy=policy, + prospective_head_sha256="2"*64, + book_ledger_head_sha256="3"*64, + spot_state_head_sha256="4"*64, + ) + assert len(candidates) == 1 + row = candidates.iloc[0] + assert row["low_condition_id"] == "five" + assert row["high_condition_id"] == "fifteen" + assert row["low_yes_token_id"] == "yes-five" + assert row["high_no_token_id"] == "no-fifteen" + assert row["low_strike"] == 99.0 + assert row["high_strike"] == 101.0 + assert row["theoretical_edge"] >= 0.02 + assert row["policy_sha256"] == "1"*64 + assert row["prospective_head_sha256"] == "2"*64 + assert bool(row["transactionally_atomic"]) is False + assert summary["candidates_selected"] == 1 + assert summary["future_outcomes_used"] == 0 + + +def test_fails_closed_on_missing_exact_strike_or_stale_book_evidence(): + markets, books, state = setup_case() + policy = FrozenSurfacePolicy.from_payload(policy_payload(), policy_sha256="1"*64) + broken = BTCMarketState( + symbol=state.symbol, + server_time_ms=state.server_time_ms, + observed_ts_ms=state.observed_ts_ms, + spot=state.spot, + vol_30s=state.vol_30s, + vol_120s=state.vol_120s, + strikes={1_000_000: Decimal("99")}, + closed_one_second_bars=state.closed_one_second_bars, + raw_response_sha256=state.raw_response_sha256, + ) + try: + generate_prospective_candidates( + markets, books, broken, book_observed_ts_ms=1_250_000, policy=policy, + prospective_head_sha256="2"*64, book_ledger_head_sha256="3"*64, + spot_state_head_sha256="4"*64, + ) + except ValueError as exc: + assert "missing exact strike" in str(exc) + else: + raise AssertionError("missing strike did not fail closed") + + try: + generate_prospective_candidates( + markets, books, state, book_observed_ts_ms=1_240_000, policy=policy, + prospective_head_sha256="2"*64, book_ledger_head_sha256="3"*64, + spot_state_head_sha256="4"*64, max_evidence_age_ms=2_000, + ) + except ValueError as exc: + assert "stale evidence" in str(exc) + else: + raise AssertionError("stale evidence did not fail closed") diff --git a/ci_payload/digital-surface-live-readiness/prospective-source/research/digital_surface/prospective.py b/ci_payload/digital-surface-live-readiness/prospective-source/research/digital_surface/prospective.py new file mode 100644 index 00000000..59844fca --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/prospective-source/research/digital_surface/prospective.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from collections.abc import Container, Iterable +from typing import Any, Mapping + + +class ProspectiveContaminationError(RuntimeError): + pass + + +class ProspectiveIntegrityError(RuntimeError): + pass + + +def _canonical_json(value: dict[str, Any]) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _validate_sha256(name: str, value: str) -> str: + text = str(value).lower() + if len(text) != 64 or any(ch not in "0123456789abcdef" for ch in text): + raise ValueError(f"{name} must be a 64-character SHA-256 hex digest") + return text + + +class ProspectiveLedger: + def __init__(self, path: str | Path, *, diagnostic_market_ids: Iterable[str] | Container[str]) -> None: + self.path = Path(path) + if isinstance(diagnostic_market_ids, (str, bytes)): + raise TypeError("diagnostic_market_ids must be an iterable/container of IDs, not a string") + if isinstance(diagnostic_market_ids, Container): + self.diagnostic_market_ids = diagnostic_market_ids + else: + self.diagnostic_market_ids = {str(value) for value in diagnostic_market_ids} + self._rows: list[dict[str, Any]] = [] + self.market_ids: set[str] = set() + self.head_hash = "0" * 64 + if self.path.exists(): + self._load_and_verify() + + def _load_and_verify(self) -> None: + previous = "0" * 64 + seen: set[str] = set() + rows: list[dict[str, Any]] = [] + for line_number, line in enumerate(self.path.read_text(encoding="utf-8").splitlines(), start=1): + if not line.strip(): + continue + try: + row = json.loads(line) + except json.JSONDecodeError as exc: + raise ProspectiveIntegrityError(f"invalid JSON at line {line_number}") from exc + stored_hash = str(row.get("record_hash", "")) + payload = dict(row) + payload.pop("record_hash", None) + if payload.get("previous_hash") != previous: + raise ProspectiveIntegrityError(f"hash chain mismatch at line {line_number}") + expected = hashlib.sha256(_canonical_json(payload)).hexdigest() + if stored_hash != expected: + raise ProspectiveIntegrityError(f"record hash mismatch at line {line_number}") + market_id = str(payload.get("market_id", "")) + if not market_id or market_id in seen: + raise ProspectiveIntegrityError(f"duplicate or empty market_id at line {line_number}") + _validate_sha256("policy_sha256", str(payload.get("policy_sha256", ""))) + _validate_sha256("source_sha256", str(payload.get("source_sha256", ""))) + seen.add(market_id) + rows.append(row) + previous = stored_hash + self._rows = rows + self.market_ids = seen + self.head_hash = previous + + def append_observation( + self, + *, + market_id: str, + first_seen_ts_ms: int, + policy_sha256: str, + source_sha256: str, + metadata: Mapping[str, Any] | None = None, + ) -> dict[str, Any]: + market = str(market_id) + if not market: + raise ValueError("market_id must be non-empty") + if market in self.diagnostic_market_ids: + raise ProspectiveContaminationError(f"market {market!r} appeared in diagnostic data") + if market in self.market_ids: + raise ProspectiveContaminationError(f"market {market!r} was already observed") + policy = _validate_sha256("policy_sha256", policy_sha256) + source = _validate_sha256("source_sha256", source_sha256) + timestamp = int(first_seen_ts_ms) + if timestamp < 0: + raise ValueError("first_seen_ts_ms must be non-negative") + payload = { + "first_seen_ts_ms": timestamp, + "market_id": market, + "policy_sha256": policy, + "previous_hash": self.head_hash, + "source_sha256": source, + "state": "observed", + } + if metadata is not None: + try: + canonical_metadata = json.loads(_canonical_json(dict(metadata)).decode("utf-8")) + except (TypeError, ValueError) as exc: + raise ValueError("metadata must be JSON serializable") from exc + payload["metadata"] = canonical_metadata + record = {**payload, "record_hash": hashlib.sha256(_canonical_json(payload)).hexdigest()} + self.path.parent.mkdir(parents=True, exist_ok=True) + with self.path.open("a", encoding="utf-8", newline="\n") as handle: + handle.write(json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n") + handle.flush() + self._rows.append(record) + self.market_ids.add(market) + self.head_hash = record["record_hash"] + return dict(record) diff --git a/ci_payload/digital-surface-live-readiness/prospective-source/tests/digital_surface/test_prospective.py b/ci_payload/digital-surface-live-readiness/prospective-source/tests/digital_surface/test_prospective.py new file mode 100644 index 00000000..0df6a5f3 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/prospective-source/tests/digital_surface/test_prospective.py @@ -0,0 +1,94 @@ +import json + +import pytest + +from research.digital_surface.prospective import ( + ProspectiveContaminationError, + ProspectiveIntegrityError, + ProspectiveLedger, +) + + +def test_rejects_diagnostic_markets_and_appends_new_observations_with_hash_chain(tmp_path): + path = tmp_path / "prospective.jsonl" + ledger = ProspectiveLedger(path, diagnostic_market_ids={"known"}) + + with pytest.raises(ProspectiveContaminationError, match="known"): + ledger.append_observation( + market_id="known", + first_seen_ts_ms=100, + policy_sha256="1" * 64, + source_sha256="2" * 64, + ) + + first = ledger.append_observation( + market_id="new-1", + first_seen_ts_ms=101, + policy_sha256="1" * 64, + source_sha256="2" * 64, + ) + second = ledger.append_observation( + market_id="new-2", + first_seen_ts_ms=102, + policy_sha256="1" * 64, + source_sha256="2" * 64, + ) + + assert first["previous_hash"] == "0" * 64 + assert second["previous_hash"] == first["record_hash"] + assert ledger.market_ids == {"new-1", "new-2"} + + reloaded = ProspectiveLedger(path, diagnostic_market_ids={"known"}) + assert reloaded.market_ids == ledger.market_ids + assert reloaded.head_hash == second["record_hash"] + + +def test_rejects_duplicate_market_ids_and_noncanonical_hashes(tmp_path): + path = tmp_path / "prospective.jsonl" + ledger = ProspectiveLedger(path, diagnostic_market_ids=set()) + ledger.append_observation( + market_id="new-1", + first_seen_ts_ms=101, + policy_sha256="1" * 64, + source_sha256="2" * 64, + ) + + with pytest.raises(ProspectiveContaminationError, match="already observed"): + ledger.append_observation( + market_id="new-1", + first_seen_ts_ms=102, + policy_sha256="1" * 64, + source_sha256="2" * 64, + ) + + rows = [json.loads(line) for line in path.read_text().splitlines()] + rows[0]["market_id"] = "tampered" + path.write_text("\n".join(json.dumps(row) for row in rows) + "\n") + + with pytest.raises(ProspectiveIntegrityError, match="record hash mismatch"): + ProspectiveLedger(path, diagnostic_market_ids=set()) + + +def test_requires_fixed_length_source_and_policy_hashes(tmp_path): + ledger = ProspectiveLedger(tmp_path / "prospective.jsonl", diagnostic_market_ids=set()) + with pytest.raises(ValueError, match="policy_sha256"): + ledger.append_observation( + market_id="new", + first_seen_ts_ms=1, + policy_sha256="short", + source_sha256="2" * 64, + ) + + +def test_preserves_canonical_market_metadata_in_hash_chain(tmp_path): + path = tmp_path / "prospective.jsonl" + ledger = ProspectiveLedger(path, diagnostic_market_ids=set()) + record = ledger.append_observation( + market_id="new-meta", + first_seen_ts_ms=10, + policy_sha256="1" * 64, + source_sha256="2" * 64, + metadata={"slug": "btc-updown-5m-10", "tick_size": "0.01"}, + ) + assert record["metadata"]["slug"] == "btc-updown-5m-10" + assert ProspectiveLedger(path, diagnostic_market_ids=set()).head_hash == record["record_hash"] diff --git a/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/__init__.py b/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/__init__.py new file mode 100644 index 00000000..ca848daa --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/__init__.py @@ -0,0 +1,25 @@ +from .config import LiveTradingDisabledError, TradingConfig, load_config +from .gateway import ShadowGateway, quantize_down +from .report import ShadowCanaryMetrics, evaluate_shadow_admission +from .report_chain import ShadowReportIntegrityError, ShadowReportLedger +from .risk import evaluate_pair_risk +from .types import BookQuote, PreparedOrder, PreparedPair, RiskDecision, RiskState, TradeIntent + +__all__ = [ + "BookQuote", + "LiveTradingDisabledError", + "PreparedOrder", + "PreparedPair", + "RiskDecision", + "ShadowCanaryMetrics", + "ShadowReportIntegrityError", + "ShadowReportLedger", + "RiskState", + "ShadowGateway", + "TradeIntent", + "TradingConfig", + "evaluate_pair_risk", + "evaluate_shadow_admission", + "load_config", + "quantize_down", +] diff --git a/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/config.py b/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/config.py new file mode 100644 index 00000000..a11a7553 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/config.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from dataclasses import dataclass +from decimal import Decimal +from typing import Mapping + + +class LiveTradingDisabledError(RuntimeError): + """Raised whenever the process is asked to enable real-money submission.""" + + +@dataclass(frozen=True, slots=True) +class TradingConfig: + mode: str = "shadow" + tick_size: Decimal = Decimal("0.01") + max_book_age_ms: int = 2_000 + max_feed_age_ms: int = 5_000 + max_daily_loss: Decimal = Decimal("25") + max_orphan_exposure: Decimal = Decimal("0") + + +def load_config(env: Mapping[str, str] | None = None) -> TradingConfig: + values = dict(env or {}) + mode = str(values.get("TRADING_MODE", "shadow")).strip().lower() + if mode not in {"shadow", "live"}: + raise ValueError("TRADING_MODE must be 'shadow' or 'live'") + # Deliberately stronger than an arm token: this research checkpoint has no + # network-capable live implementation. Credentials cannot override it. + if mode == "live": + raise LiveTradingDisabledError( + "live trading is disabled until prospective admission and a separately reviewed release" + ) + return TradingConfig( + mode="shadow", + tick_size=Decimal(str(values.get("TICK_SIZE", "0.01"))), + max_book_age_ms=int(values.get("MAX_BOOK_AGE_MS", "2000")), + max_feed_age_ms=int(values.get("MAX_FEED_AGE_MS", "5000")), + max_daily_loss=Decimal(str(values.get("MAX_DAILY_LOSS", "25"))), + max_orphan_exposure=Decimal(str(values.get("MAX_ORPHAN_EXPOSURE", "0"))), + ) diff --git a/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/gateway.py b/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/gateway.py new file mode 100644 index 00000000..901736c4 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/gateway.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from decimal import Decimal, ROUND_DOWN +from typing import Any + +from .config import TradingConfig +from .risk import evaluate_pair_risk +from .types import BookQuote, PreparedOrder, PreparedPair, RiskState, TradeIntent + + +def quantize_down(value: Decimal, tick: Decimal) -> Decimal: + if tick <= 0: + raise ValueError("tick must be positive") + units = (value / tick).to_integral_value(rounding=ROUND_DOWN) + return units * tick + + +class ShadowGateway: + """Prepare auditable FOK parameters without signing or sending a request.""" + + def __init__(self, config: TradingConfig, transport: Any | None = None) -> None: + if config.mode != "shadow": + raise ValueError("ShadowGateway requires shadow mode") + self.config = config + self._transport = transport + self.prepared: list[PreparedPair] = [] + + def prepare_pair( + self, + *, + intent: TradeIntent, + low_book: BookQuote, + high_book: BookQuote, + state: RiskState, + now_ms: int, + ) -> PreparedPair: + risk = evaluate_pair_risk( + intent=intent, + low_book=low_book, + high_book=high_book, + state=state, + config=self.config, + now_ms=now_ms, + ) + if not risk.allowed: + raise RuntimeError(f"risk rejected pair: {risk.reason}") + low_price = quantize_down(min(low_book.best_ask, intent.max_low_price), self.config.tick_size) + high_price = quantize_down(min(high_book.best_ask, intent.max_high_price), self.config.tick_size) + pair = PreparedPair( + mode="shadow", + intent=intent, + low_order=PreparedOrder( + token_id=intent.token_id_low_yes, + side="BUY", + order_type="FOK", + price=low_price, + size=intent.shares, + ), + high_order=PreparedOrder( + token_id=intent.token_id_high_no, + side="BUY", + order_type="FOK", + price=high_price, + size=intent.shares, + ), + risk=risk, + ) + self.prepared.append(pair) + # No method on the supplied transport is invoked. The object exists only + # so the test can prove preparation has zero network side effects. + return pair + + def submit_pair(self, pair: PreparedPair) -> None: + raise RuntimeError("network submission is not implemented in this shadow-only checkpoint") diff --git a/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/report.py b/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/report.py new file mode 100644 index 00000000..374cba0c --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/report.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass +from typing import Any + + +@dataclass(frozen=True, slots=True) +class ShadowCanaryMetrics: + markets_observed: int + qualified_portfolios: int + simulated_atomic: int + orphaned: int + unresolved_order_states: int + feed_outages: int + weekly_passes: int + + +def evaluate_shadow_admission(metrics: ShadowCanaryMetrics) -> dict[str, Any]: + values = asdict(metrics) + if any(int(value) < 0 for value in values.values()): + raise ValueError("shadow canary metrics must be non-negative") + attempts = int(metrics.simulated_atomic) + int(metrics.orphaned) + orphan_rate = float(metrics.orphaned / attempts) if attempts else 0.0 + reasons: list[str] = [] + if metrics.markets_observed < 500: + reasons.append("requires at least 500 observed markets") + if metrics.qualified_portfolios < 100: + reasons.append("requires at least 100 qualified portfolios") + if metrics.weekly_passes < 4: + reasons.append("requires four untouched weekly passes") + if metrics.unresolved_order_states: + reasons.append("unresolved order state exists") + if orphan_rate > 0.05: + reasons.append("orphan rate exceeds 5%") + classification = "Admissible" if not reasons else "Rejected" + return { + "classification": classification, + "reasons": reasons, + "markets_observed": int(metrics.markets_observed), + "qualified_portfolios": int(metrics.qualified_portfolios), + "simulated_atomic": int(metrics.simulated_atomic), + "orphaned": int(metrics.orphaned), + "orphan_rate": orphan_rate, + "unresolved_order_states": int(metrics.unresolved_order_states), + "feed_outages": int(metrics.feed_outages), + "weekly_passes": int(metrics.weekly_passes), + # Even an admissible research report cannot arm the current build. + "live_mode": "disabled_pending_operator_release", + } diff --git a/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/report_chain.py b/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/report_chain.py new file mode 100644 index 00000000..8875f8d8 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/report_chain.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import hashlib +import json +from dataclasses import asdict +from pathlib import Path +from typing import Any + +from .report import ShadowCanaryMetrics, evaluate_shadow_admission + + +class ShadowReportIntegrityError(RuntimeError): + """Raised when an append-only shadow report ledger has been altered.""" + + +def _canonical_json(value: dict[str, Any]) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _sha256(name: str, value: str) -> str: + text = str(value).lower() + if len(text) != 64 or any(ch not in "0123456789abcdef" for ch in text): + raise ValueError(f"{name} must be a 64-character SHA-256 hex digest") + return text + + +class ShadowReportLedger: + """Append-only reports bound to immutable policy, source, and market-ledger state.""" + + def __init__(self, path: str | Path) -> None: + self.path = Path(path) + self.head_hash = "0" * 64 + self.rows: list[dict[str, Any]] = [] + if self.path.exists(): + self._load_and_verify() + + def _load_and_verify(self) -> None: + previous = "0" * 64 + rows: list[dict[str, Any]] = [] + for line_number, line in enumerate(self.path.read_text(encoding="utf-8").splitlines(), start=1): + if not line.strip(): + continue + try: + record = json.loads(line) + except json.JSONDecodeError as exc: + raise ShadowReportIntegrityError(f"invalid JSON at line {line_number}") from exc + stored = str(record.get("report_hash", "")) + payload = dict(record) + payload.pop("report_hash", None) + if payload.get("previous_report_hash") != previous: + raise ShadowReportIntegrityError(f"report chain mismatch at line {line_number}") + for field in ("policy_sha256", "source_sha256", "prospective_head_sha256"): + try: + _sha256(field, str(payload.get(field, ""))) + except ValueError as exc: + raise ShadowReportIntegrityError(f"invalid {field} at line {line_number}") from exc + expected = hashlib.sha256(_canonical_json(payload)).hexdigest() + if stored != expected: + raise ShadowReportIntegrityError(f"report hash mismatch at line {line_number}") + metrics = ShadowCanaryMetrics(**payload["metrics"]) + expected_decision = evaluate_shadow_admission(metrics) + if payload.get("decision") != expected_decision: + raise ShadowReportIntegrityError(f"decision mismatch at line {line_number}") + rows.append(record) + previous = stored + self.rows = rows + self.head_hash = previous + + def append( + self, + *, + generated_ts_ms: int, + metrics: ShadowCanaryMetrics, + policy_sha256: str, + source_sha256: str, + prospective_head_sha256: str, + ) -> dict[str, Any]: + timestamp = int(generated_ts_ms) + if timestamp < 0: + raise ValueError("generated_ts_ms must be non-negative") + policy = _sha256("policy_sha256", policy_sha256) + source = _sha256("source_sha256", source_sha256) + prospective = _sha256("prospective_head_sha256", prospective_head_sha256) + payload = { + "decision": evaluate_shadow_admission(metrics), + "generated_ts_ms": timestamp, + "metrics": asdict(metrics), + "policy_sha256": policy, + "previous_report_hash": self.head_hash, + "prospective_head_sha256": prospective, + "source_sha256": source, + } + report_hash = hashlib.sha256(_canonical_json(payload)).hexdigest() + record = {**payload, "report_hash": report_hash} + self.path.parent.mkdir(parents=True, exist_ok=True) + with self.path.open("a", encoding="utf-8", newline="\n") as handle: + handle.write(json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n") + handle.flush() + self.rows.append(record) + self.head_hash = report_hash + return dict(record) diff --git a/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/risk.py b/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/risk.py new file mode 100644 index 00000000..094a673a --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/risk.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from decimal import Decimal + +from .config import TradingConfig +from .types import BookQuote, RiskDecision, RiskState, TradeIntent + + +def evaluate_pair_risk( + *, + intent: TradeIntent, + low_book: BookQuote, + high_book: BookQuote, + state: RiskState, + config: TradingConfig, + now_ms: int, +) -> RiskDecision: + if now_ms >= intent.expires_ts_ms: + return RiskDecision(False, "intent_expired") + if state.ambiguous_pair_state: + return RiskDecision(False, "ambiguous_pair_state") + if state.unresolved_orders: + return RiskDecision(False, "unresolved_prior_orders") + if state.orphan_exposure > config.max_orphan_exposure: + return RiskDecision(False, "orphan_exposure") + if state.daily_pnl <= -abs(config.max_daily_loss): + return RiskDecision(False, "daily_loss_limit") + if now_ms - state.feed_heartbeat_ts_ms > config.max_feed_age_ms: + return RiskDecision(False, "stale_feed") + if now_ms - low_book.observed_ts_ms > config.max_book_age_ms: + return RiskDecision(False, "stale_low_book") + if now_ms - high_book.observed_ts_ms > config.max_book_age_ms: + return RiskDecision(False, "stale_high_book") + if low_book.token_id != intent.token_id_low_yes: + return RiskDecision(False, "low_token_mismatch") + if high_book.token_id != intent.token_id_high_no: + return RiskDecision(False, "high_token_mismatch") + if low_book.best_ask > intent.max_low_price: + return RiskDecision(False, "low_ask_above_limit") + if high_book.best_ask > intent.max_high_price: + return RiskDecision(False, "high_ask_above_limit") + if low_book.ask_size < intent.shares: + return RiskDecision(False, "low_depth_insufficient") + if high_book.ask_size < intent.shares: + return RiskDecision(False, "high_depth_insufficient") + if intent.shares <= Decimal("0"): + return RiskDecision(False, "invalid_size") + return RiskDecision(True, "allowed") diff --git a/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/types.py b/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/types.py new file mode 100644 index 00000000..204cfc2c --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/shadow-source/research/digital_surface/live/types.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from dataclasses import dataclass +from decimal import Decimal + + +@dataclass(frozen=True, slots=True) +class TradeIntent: + condition_id_low: str + token_id_low_yes: str + condition_id_high: str + token_id_high_no: str + max_low_price: Decimal + max_high_price: Decimal + shares: Decimal + decision_ts_ms: int + expires_ts_ms: int + + +@dataclass(frozen=True, slots=True) +class BookQuote: + token_id: str + best_ask: Decimal + ask_size: Decimal + observed_ts_ms: int + + +@dataclass(frozen=True, slots=True) +class RiskState: + feed_heartbeat_ts_ms: int + daily_pnl: Decimal = Decimal("0") + unresolved_orders: int = 0 + orphan_exposure: Decimal = Decimal("0") + ambiguous_pair_state: bool = False + + +@dataclass(frozen=True, slots=True) +class RiskDecision: + allowed: bool + reason: str + + +@dataclass(frozen=True, slots=True) +class PreparedOrder: + token_id: str + side: str + order_type: str + price: Decimal + size: Decimal + + +@dataclass(frozen=True, slots=True) +class PreparedPair: + mode: str + intent: TradeIntent + low_order: PreparedOrder + high_order: PreparedOrder + risk: RiskDecision diff --git a/ci_payload/digital-surface-live-readiness/shadow-source/tests/digital_surface/live/test_report.py b/ci_payload/digital-surface-live-readiness/shadow-source/tests/digital_surface/live/test_report.py new file mode 100644 index 00000000..491e309e --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/shadow-source/tests/digital_surface/live/test_report.py @@ -0,0 +1,47 @@ +from research.digital_surface.live.report import ShadowCanaryMetrics, evaluate_shadow_admission + + +def test_rejects_insufficient_prospective_sample_and_weekly_evidence(): + decision = evaluate_shadow_admission(ShadowCanaryMetrics( + markets_observed=499, + qualified_portfolios=99, + simulated_atomic=95, + orphaned=5, + unresolved_order_states=0, + feed_outages=0, + weekly_passes=3, + )) + assert decision["classification"] == "Rejected" + assert "requires at least 500 observed markets" in decision["reasons"] + assert "requires at least 100 qualified portfolios" in decision["reasons"] + assert "requires four untouched weekly passes" in decision["reasons"] + + +def test_rejects_unresolved_orders_and_excess_orphaning(): + decision = evaluate_shadow_admission(ShadowCanaryMetrics( + markets_observed=600, + qualified_portfolios=120, + simulated_atomic=100, + orphaned=8, + unresolved_order_states=1, + feed_outages=0, + weekly_passes=4, + )) + assert decision["classification"] == "Rejected" + assert "unresolved order state exists" in decision["reasons"] + assert "orphan rate exceeds 5%" in decision["reasons"] + + +def test_admits_only_when_every_prospective_gate_clears(): + decision = evaluate_shadow_admission(ShadowCanaryMetrics( + markets_observed=650, + qualified_portfolios=120, + simulated_atomic=116, + orphaned=4, + unresolved_order_states=0, + feed_outages=1, + weekly_passes=4, + )) + assert decision["classification"] == "Admissible" + assert decision["orphan_rate"] == 4 / 120 + assert decision["live_mode"] == "disabled_pending_operator_release" diff --git a/ci_payload/digital-surface-live-readiness/shadow-source/tests/digital_surface/live/test_report_chain.py b/ci_payload/digital-surface-live-readiness/shadow-source/tests/digital_surface/live/test_report_chain.py new file mode 100644 index 00000000..a56cb02c --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/shadow-source/tests/digital_surface/live/test_report_chain.py @@ -0,0 +1,72 @@ +import json + +import pytest + +from research.digital_surface.live.report import ShadowCanaryMetrics +from research.digital_surface.live.report_chain import ShadowReportIntegrityError, ShadowReportLedger + + +def metrics(markets=650, qualified=120, atomic=116, orphaned=4, unresolved=0, weeks=4): + return ShadowCanaryMetrics( + markets_observed=markets, + qualified_portfolios=qualified, + simulated_atomic=atomic, + orphaned=orphaned, + unresolved_order_states=unresolved, + feed_outages=0, + weekly_passes=weeks, + ) + + +def test_appends_hash_chained_reports_bound_to_policy_source_and_prospective_head(tmp_path): + path = tmp_path / "reports.jsonl" + ledger = ShadowReportLedger(path) + first = ledger.append( + generated_ts_ms=100, + metrics=metrics(), + policy_sha256="1" * 64, + source_sha256="2" * 64, + prospective_head_sha256="3" * 64, + ) + second = ledger.append( + generated_ts_ms=200, + metrics=metrics(markets=700), + policy_sha256="1" * 64, + source_sha256="2" * 64, + prospective_head_sha256="4" * 64, + ) + assert first["decision"]["classification"] == "Admissible" + assert first["decision"]["live_mode"] == "disabled_pending_operator_release" + assert second["previous_report_hash"] == first["report_hash"] + reloaded = ShadowReportLedger(path) + assert reloaded.head_hash == second["report_hash"] + assert len(reloaded.rows) == 2 + + +def test_recomputes_decision_and_detects_tampering(tmp_path): + path = tmp_path / "reports.jsonl" + ledger = ShadowReportLedger(path) + ledger.append( + generated_ts_ms=100, + metrics=metrics(markets=499), + policy_sha256="1" * 64, + source_sha256="2" * 64, + prospective_head_sha256="3" * 64, + ) + row = json.loads(path.read_text()) + row["decision"]["classification"] = "Admissible" + path.write_text(json.dumps(row) + "\n") + with pytest.raises(ShadowReportIntegrityError, match="report hash mismatch|decision mismatch"): + ShadowReportLedger(path) + + +def test_rejects_unbound_or_invalid_digests(tmp_path): + ledger = ShadowReportLedger(tmp_path / "reports.jsonl") + with pytest.raises(ValueError, match="prospective_head_sha256"): + ledger.append( + generated_ts_ms=100, + metrics=metrics(), + policy_sha256="1" * 64, + source_sha256="2" * 64, + prospective_head_sha256="not-a-hash", + ) diff --git a/ci_payload/digital-surface-live-readiness/shadow-source/tests/digital_surface/live/test_shadow_gateway.py b/ci_payload/digital-surface-live-readiness/shadow-source/tests/digital_surface/live/test_shadow_gateway.py new file mode 100644 index 00000000..448b0c44 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/shadow-source/tests/digital_surface/live/test_shadow_gateway.py @@ -0,0 +1,99 @@ +from decimal import Decimal + +import pytest + +from research.digital_surface.live import ( + BookQuote, + LiveTradingDisabledError, + RiskState, + ShadowGateway, + TradeIntent, + load_config, +) + + +class ExplodingTransport: + def __getattr__(self, name): + raise AssertionError(f"shadow path attempted network transport method {name}") + + +def intent() -> TradeIntent: + return TradeIntent( + condition_id_low="low", + token_id_low_yes="yes-low", + condition_id_high="high", + token_id_high_no="no-high", + max_low_price=Decimal("0.51"), + max_high_price=Decimal("0.51"), + shares=Decimal("5"), + decision_ts_ms=1_000, + expires_ts_ms=10_000, + ) + + +def books(now=2_000): + return ( + BookQuote("yes-low", Decimal("0.5039"), Decimal("10"), now), + BookQuote("no-high", Decimal("0.4979"), Decimal("10"), now), + ) + + +def state(now=2_000): + return RiskState(feed_heartbeat_ts_ms=now) + + +def test_defaults_to_shadow_and_live_is_physically_disabled_even_with_credentials(): + assert load_config({}).mode == "shadow" + with pytest.raises(LiveTradingDisabledError): + load_config({ + "TRADING_MODE": "live", + "POLYMARKET_API_KEY": "present", + "POLYMARKET_PRIVATE_KEY": "present", + "LIVE_ARM_TOKEN": "present", + }) + + +def test_shadow_prepare_quantizes_exactly_and_never_calls_transport(): + config = load_config({"TICK_SIZE": "0.01"}) + gateway = ShadowGateway(config, transport=ExplodingTransport()) + low, high = books() + pair = gateway.prepare_pair(intent=intent(), low_book=low, high_book=high, state=state(), now_ms=2_000) + assert pair.mode == "shadow" + assert pair.low_order.price == Decimal("0.50") + assert pair.high_order.price == Decimal("0.49") + assert pair.low_order.order_type == pair.high_order.order_type == "FOK" + assert len(gateway.prepared) == 1 + + +def test_stale_feed_books_and_unresolved_exposure_fail_closed(): + gateway = ShadowGateway(load_config({})) + low, high = books(now=1_000) + with pytest.raises(RuntimeError, match="stale_feed"): + gateway.prepare_pair(intent=intent(), low_book=low, high_book=high, state=state(now=0), now_ms=6_000) + low, high = books(now=0) + with pytest.raises(RuntimeError, match="stale_low_book"): + gateway.prepare_pair(intent=intent(), low_book=low, high_book=high, state=state(now=6_000), now_ms=6_000) + low, high = books(now=2_000) + with pytest.raises(RuntimeError, match="unresolved_prior_orders"): + gateway.prepare_pair( + intent=intent(), low_book=low, high_book=high, + state=RiskState(feed_heartbeat_ts_ms=2_000, unresolved_orders=1), now_ms=2_000, + ) + with pytest.raises(RuntimeError, match="orphan_exposure"): + gateway.prepare_pair( + intent=intent(), low_book=low, high_book=high, + state=RiskState(feed_heartbeat_ts_ms=2_000, orphan_exposure=Decimal("0.01")), now_ms=2_000, + ) + + +def test_ambiguous_pair_state_blocks_and_submit_is_unavailable(): + gateway = ShadowGateway(load_config({})) + low, high = books() + with pytest.raises(RuntimeError, match="ambiguous_pair_state"): + gateway.prepare_pair( + intent=intent(), low_book=low, high_book=high, + state=RiskState(feed_heartbeat_ts_ms=2_000, ambiguous_pair_state=True), now_ms=2_000, + ) + pair = gateway.prepare_pair(intent=intent(), low_book=low, high_book=high, state=state(), now_ms=2_000) + with pytest.raises(RuntimeError, match="not implemented"): + gateway.submit_pair(pair) diff --git a/ci_payload/digital-surface-live-readiness/simulation-source/research/digital_surface/shadow_fill.py b/ci_payload/digital-surface-live-readiness/simulation-source/research/digital_surface/shadow_fill.py new file mode 100644 index 00000000..f17793d7 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/simulation-source/research/digital_surface/shadow_fill.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +from dataclasses import dataclass +from decimal import Decimal +from typing import Any, Mapping + +from .public_books import BookLevel, PublicOrderBook + + +@dataclass(frozen=True, slots=True) +class BuyFokEstimate: + filled: bool + shares_requested: Decimal + shares_available: Decimal + gross_cost: Decimal + average_price: Decimal | None + worst_price: Decimal | None + + +@dataclass(frozen=True, slots=True) +class PairedFokEstimate: + paired_crossable: bool + orphanable_leg: str | None + low: BuyFokEstimate + high: BuyFokEstimate + combined_gross_cost: Decimal | None + transactionally_atomic: bool = False + + +def estimate_buy_fok( + asks: tuple[BookLevel, ...], + *, + shares: Decimal, + max_price: Decimal, +) -> BuyFokEstimate: + requested = Decimal(shares) + limit = Decimal(max_price) + if requested <= 0: + raise ValueError("shares must be positive") + if not (Decimal("0") < limit < Decimal("1")): + raise ValueError("max_price must be between zero and one") + remaining = requested + available = Decimal("0") + gross_cost = Decimal("0") + worst: Decimal | None = None + for level in asks: + if level.price > limit: + break + take = min(remaining, level.size) + if take <= 0: + continue + available += take + gross_cost += take * level.price + worst = level.price + remaining -= take + if remaining == 0: + break + filled = remaining == 0 + if not filled: + return BuyFokEstimate( + filled=False, + shares_requested=requested, + shares_available=available, + gross_cost=Decimal("0"), + average_price=None, + worst_price=None, + ) + return BuyFokEstimate( + filled=True, + shares_requested=requested, + shares_available=available, + gross_cost=gross_cost, + average_price=gross_cost / requested, + worst_price=worst, + ) + + +def estimate_paired_fok( + low_book: PublicOrderBook, + high_book: PublicOrderBook, + *, + shares: Decimal, + low_max_price: Decimal, + high_max_price: Decimal, +) -> PairedFokEstimate: + if low_book.token_id == high_book.token_id: + raise ValueError("paired legs must use distinct tokens") + low = estimate_buy_fok(low_book.asks, shares=shares, max_price=low_max_price) + high = estimate_buy_fok(high_book.asks, shares=shares, max_price=high_max_price) + orphanable_leg: str | None = None + if low.filled != high.filled: + orphanable_leg = "low_yes" if low.filled else "high_no" + paired = low.filled and high.filled + return PairedFokEstimate( + paired_crossable=paired, + orphanable_leg=orphanable_leg, + low=low, + high=high, + combined_gross_cost=(low.gross_cost + high.gross_cost) if paired else None, + transactionally_atomic=False, + ) + + +def estimate_pair_from_ledger_records( + low_record: Mapping[str, Any], + high_record: Mapping[str, Any], + *, + shares: Decimal, + low_max_price: Decimal, + high_max_price: Decimal, +) -> PairedFokEstimate: + for field in ("observed_ts_ms", "policy_sha256", "source_sha256", "prospective_head_sha256"): + if low_record.get(field) != high_record.get(field): + raise ValueError(f"paired evidence mismatch for {field}") + if str(low_record.get("outcome")) != "yes": + raise ValueError("low record must be the YES leg") + if str(high_record.get("outcome")) != "no": + raise ValueError("high record must be the NO leg") + if str(low_record.get("condition_id")) == str(high_record.get("condition_id")): + raise ValueError("paired legs must use distinct conditions") + return estimate_paired_fok( + _book_from_record(low_record), + _book_from_record(high_record), + shares=shares, + low_max_price=low_max_price, + high_max_price=high_max_price, + ) + + +def _book_from_record(record: Mapping[str, Any]) -> PublicOrderBook: + payload = record.get("book") + if not isinstance(payload, Mapping): + raise ValueError("ledger record is missing book data") + def levels(name: str) -> tuple[BookLevel, ...]: + raw = payload.get(name, []) + if not isinstance(raw, list): + raise ValueError(f"book {name} must be a list") + return tuple(BookLevel(Decimal(str(row["price"])), Decimal(str(row["size"]))) for row in raw) + last = payload.get("last_trade_price") + return PublicOrderBook( + condition_id=str(payload["condition_id"]), + token_id=str(payload["token_id"]), + timestamp=str(payload.get("timestamp") or ""), + book_hash=str(payload.get("book_hash") or ""), + bids=levels("bids"), + asks=levels("asks"), + min_order_size=Decimal(str(payload.get("min_order_size") or "0")), + tick_size=Decimal(str(payload.get("tick_size") or "0.01")), + neg_risk=bool(payload.get("neg_risk", False)), + last_trade_price=None if last in (None, "") else Decimal(str(last)), + payload_sha256=str(payload["payload_sha256"]), + ) diff --git a/ci_payload/digital-surface-live-readiness/simulation-source/tests/digital_surface/test_shadow_fill.py b/ci_payload/digital-surface-live-readiness/simulation-source/tests/digital_surface/test_shadow_fill.py new file mode 100644 index 00000000..01619224 --- /dev/null +++ b/ci_payload/digital-surface-live-readiness/simulation-source/tests/digital_surface/test_shadow_fill.py @@ -0,0 +1,84 @@ +from decimal import Decimal + +import pytest + +from research.digital_surface.public_books import BookLevel, parse_public_order_book +from research.digital_surface.shadow_fill import ( + estimate_buy_fok, + estimate_pair_from_ledger_records, + estimate_paired_fok, +) + + +def book(token, condition, asks): + return parse_public_order_book({ + "market": condition, + "asset_id": token, + "timestamp": "1", + "hash": f"h-{token}", + "bids": [], + "asks": [{"price": str(price), "size": str(size)} for price, size in asks], + "min_order_size": "1", + "tick_size": "0.01", + "neg_risk": False, + }) + + +def record(item, outcome, observed=10, policy="1", source="2", prospective="3"): + return { + "condition_id": item.condition_id, + "outcome": outcome, + "observed_ts_ms": observed, + "policy_sha256": policy * 64, + "source_sha256": source * 64, + "prospective_head_sha256": prospective * 64, + "book": item.as_json(), + } + + +def test_walks_multiple_ask_levels_under_frozen_limit(): + result = estimate_buy_fok( + (BookLevel(Decimal("0.50"), Decimal("2")), BookLevel(Decimal("0.52"), Decimal("4"))), + shares=Decimal("5"), + max_price=Decimal("0.52"), + ) + assert result.filled is True + assert result.gross_cost == Decimal("2.56") + assert result.average_price == Decimal("0.512") + assert result.worst_price == Decimal("0.52") + + +def test_fok_estimate_discards_partial_cost_when_full_size_is_unavailable(): + result = estimate_buy_fok( + (BookLevel(Decimal("0.50"), Decimal("2")),), + shares=Decimal("5"), + max_price=Decimal("0.50"), + ) + assert result.filled is False + assert result.shares_available == Decimal("2") + assert result.gross_cost == Decimal("0") + assert result.average_price is None + + +def test_pair_is_crossable_but_never_claimed_transactionally_atomic(): + low = book("yes-low", "low", [("0.50", "5")]) + high = book("no-high", "high", [("0.49", "5")]) + result = estimate_paired_fok( + low, high, shares=Decimal("5"), + low_max_price=Decimal("0.51"), high_max_price=Decimal("0.50"), + ) + assert result.paired_crossable is True + assert result.combined_gross_cost == Decimal("4.95") + assert result.transactionally_atomic is False + assert result.orphanable_leg is None + + +def test_pair_records_require_same_causal_observation_and_bound_digests(): + low = book("yes-low", "low", [("0.50", "5")]) + high = book("no-high", "high", [("0.49", "5")]) + with pytest.raises(ValueError, match="observed_ts_ms"): + estimate_pair_from_ledger_records( + record(low, "yes", observed=10), + record(high, "no", observed=11), + shares=Decimal("5"), low_max_price=Decimal("0.51"), high_max_price=Decimal("0.50"), + ) diff --git a/ci_payload/digital-surface-official-resolution-patch/.github/workflows/digital-surface-arbitrage.yml b/ci_payload/digital-surface-official-resolution-patch/.github/workflows/digital-surface-arbitrage.yml new file mode 100644 index 00000000..76ea1931 --- /dev/null +++ b/ci_payload/digital-surface-official-resolution-patch/.github/workflows/digital-surface-arbitrage.yml @@ -0,0 +1,108 @@ +name: Digital Surface Arbitrage + +on: + push: + branches: + - digital-surface-arbitrage-implementation-20260723 + paths: + - research/digital_surface/** + - tests/digital_surface/** + - .github/workflows/digital-surface-arbitrage.yml + - docs/superpowers/plans/2026-07-23-digital-surface-arbitrage-implementation.md + pull_request: + branches: + - main + paths: + - research/digital_surface/** + - tests/digital_surface/** + - .github/workflows/digital-surface-arbitrage.yml + workflow_dispatch: + +permissions: + contents: read + +env: + PYTHONHASHSEED: '17' + PYTHONPATH: '.' + PYTHONUNBUFFERED: '1' + +jobs: + test-and-research: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Cache immutable source data + uses: actions/cache@v4 + with: + path: .cache/digital-surface + key: digital-surface-hybrid-obadiaha-11793901-kinzik-eb4e9fc7-v1 + + - name: Install pinned research stack + run: | + python -m pip install --upgrade pip + python -m pip install \ + numpy==2.2.6 \ + pandas==2.2.3 \ + scipy==1.15.3 \ + scikit-learn==1.6.1 \ + pyarrow==19.0.1 \ + requests==2.32.3 \ + pytest==8.3.5 \ + tabulate==0.9.0 + + - name: Compile package + run: | + python -m compileall -q research/digital_surface tests/digital_surface + + - name: Run unit, property, regression, and integration tests + run: | + pytest tests/digital_surface -q + + - name: Run immutable rolling-origin research + shell: bash + run: | + set -o pipefail + python -m research.digital_surface.run_research \ + --cache-dir .cache/digital-surface \ + --output-dir artifacts/digital-surface-arbitrage \ + 2>&1 | tee digital-surface-arbitrage.log + + - name: Verify internal checksums + shell: bash + run: | + set -o pipefail + python - <<'PY' + import hashlib, json + from pathlib import Path + root=Path('artifacts/digital-surface-arbitrage') + expected=json.loads((root/'SHA256SUMS.json').read_text()) + for name,want in expected.items(): + path=root/name + h=hashlib.sha256(path.read_bytes()).hexdigest() + assert h==want,(name,h,want) + assert (root/'REPORT.md').exists() + assert (root/'admission.json').exists() + print(f'verified {len(expected)} artifact checksums') + PY + + - name: Show decision + if: always() + run: | + cat artifacts/digital-surface-arbitrage/admission.json || true + tail -200 digital-surface-arbitrage.log || true + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: digital-surface-arbitrage-results + path: | + artifacts/digital-surface-arbitrage/ + digital-surface-arbitrage.log + if-no-files-found: warn + retention-days: 30 diff --git a/ci_payload/digital-surface-official-resolution-patch/asset-merge-fix.patch b/ci_payload/digital-surface-official-resolution-patch/asset-merge-fix.patch new file mode 100644 index 00000000..dc3d84e2 --- /dev/null +++ b/ci_payload/digital-surface-official-resolution-patch/asset-merge-fix.patch @@ -0,0 +1,39 @@ +diff --git a/research/digital_surface/causal.py b/research/digital_surface/causal.py +--- a/research/digital_surface/causal.py ++++ b/research/digital_surface/causal.py +@@ -137,7 +137,22 @@ + if state.empty: + raise ValueError("book filter produced zero rows") + state = state.drop_duplicates(["condition_id", "ts_ms"], keep="last") +- state = state.merge(registry, on="condition_id", how="inner", validate="many_to_one") ++ registry_for_merge = registry.copy() ++ has_book_asset = "asset" in state.columns ++ has_registry_asset = "asset" in registry_for_merge.columns ++ if has_book_asset and has_registry_asset: ++ state = state.rename(columns={"asset": "book_asset"}) ++ state = state.merge(registry_for_merge, on="condition_id", how="inner", validate="many_to_one") ++ if has_book_asset and has_registry_asset: ++ book_asset = state["book_asset"].astype(str).str.lower() ++ registry_asset = state["asset"].astype(str).str.lower() ++ mismatch = book_asset.ne(registry_asset) ++ if mismatch.any(): ++ examples = state.loc[mismatch, ["condition_id", "book_asset", "asset"]].head(10).to_dict("records") ++ raise ValueError(f"book asset disagrees with registry asset; examples: {examples}") ++ state = state.drop(columns="book_asset") ++ if "asset" not in state.columns: ++ raise ValueError("registry and books do not provide an asset column") + state["close_ts_ms"] = state["close_ts"].astype("int64") * 1000 + state["open_ts_ms"] = state["open_ts"].astype("int64") * 1000 + state["tau_s"] = (state["close_ts_ms"] - state["ts_ms"]) / 1000.0 +diff --git a/tests/digital_surface/test_causal.py b/tests/digital_surface/test_causal.py +--- a/tests/digital_surface/test_causal.py ++++ b/tests/digital_surface/test_causal.py +@@ -28,7 +28,7 @@ + for ts in range(350_000, 381_000, 1000): + spot = 100 + (ts - 350_000) / 100_000 + for cid in ("c5", "c15"): +- rows.append({"condition_id": cid, "ts_ms": ts, "yes_bid": .45, "yes_ask": .47, "yes_bid_size": 20, "yes_ask_size": 20, "no_bid": .52, "no_ask": .54, "no_bid_size": 20, "no_ask_size": 20, "spot": spot}) ++ rows.append({"condition_id": cid, "asset": "btc", "ts_ms": ts, "yes_bid": .45, "yes_ask": .47, "yes_bid_size": 20, "yes_ask_size": 20, "no_bid": .52, "no_ask": .54, "no_bid_size": 20, "no_ask_size": 20, "spot": spot}) + return registry, pd.DataFrame(rows) + + diff --git a/ci_payload/digital-surface-official-resolution-patch/full-universe-gate-fix.patch b/ci_payload/digital-surface-official-resolution-patch/full-universe-gate-fix.patch new file mode 100644 index 00000000..5b2cef85 --- /dev/null +++ b/ci_payload/digital-surface-official-resolution-patch/full-universe-gate-fix.patch @@ -0,0 +1,98 @@ +diff --git a/research/digital_surface/official_resolution.py b/research/digital_surface/official_resolution.py +index 1b180fc..61be096 100644 +--- a/research/digital_surface/official_resolution.py ++++ b/research/digital_surface/official_resolution.py +@@ -162,8 +162,9 @@ def install(data_module: ModuleType) -> None: + ) + binance_direction_agreement = float(audit["oracle_agreement"]) + ++ source_ids = patched_markets["condition_id"].astype(str).drop_duplicates() + official = ( +- load_official_resolution_snapshot(slots["condition_id"]) ++ load_official_resolution_snapshot(source_ids) + if official_resolutions is None + else official_resolutions.copy() + ) +@@ -187,8 +188,7 @@ def install(data_module: ModuleType) -> None: + recorded["condition_id"] = recorded["condition_id"].astype(str) + recorded["outcome"] = recorded["outcome"].astype(str).str.title() + recorded = recorded.drop_duplicates("condition_id", keep="last") +- admitted_ids = slots["condition_id"].astype(str).drop_duplicates() +- evidence = pd.DataFrame({"condition_id": admitted_ids}).merge( ++ evidence = pd.DataFrame({"condition_id": source_ids}).merge( + recorded, + on="condition_id", + how="left", +@@ -214,6 +214,8 @@ def install(data_module: ModuleType) -> None: + audit["oracle_agreement"] = agreement + audit["official_resolution_coverage"] = coverage + audit["official_resolution_agreement"] = agreement ++ audit["official_resolution_source_contracts"] = int(len(source_ids)) ++ audit["official_resolution_panel_contracts"] = int(slots["condition_id"].nunique()) + audit["binance_direction_agreement_diagnostic"] = binance_direction_agreement + return slots, books, trades, audit + +diff --git a/tests/digital_surface/test_data_adapter.py b/tests/digital_surface/test_data_adapter.py +index 4042b1d..122666b 100644 +--- a/tests/digital_surface/test_data_adapter.py ++++ b/tests/digital_surface/test_data_adapter.py +@@ -180,14 +180,26 @@ def test_normalize_obadiaha_uses_official_token_order_and_causal_open_strike(): + "market_type": "crypto_15m", + "slug": "btc-updown-15m-1772755200", + }, ++ { ++ "market_id": "btc-updown-5m-1772755500", ++ "asset": "BTC", ++ "condition_id": "cgap", ++ "question": "Bitcoin Up or Down", ++ "start_time": pd.Timestamp("2026-03-06T00:10:00Z"), ++ "end_time": pd.Timestamp("2026-03-06T00:10:00Z"), ++ "market_type": "crypto_5m", ++ "slug": "btc-updown-5m-1772755500", ++ }, + ]) + resolutions = pd.DataFrame([ + {"condition_id": "c5", "outcome": "Up"}, + {"condition_id": "c15", "outcome": "Down"}, ++ {"condition_id": "cgap", "outcome": "Up"}, + ]) + gamma = pd.DataFrame([ + {"condition_id": "c5", "yes_token_id": "yes5", "no_token_id": "no5"}, + {"condition_id": "c15", "yes_token_id": "yes15", "no_token_id": "no15"}, ++ {"condition_id": "cgap", "yes_token_id": "yesgap", "no_token_id": "nogap"}, + ]) + spot = pd.DataFrame([ + {"timestamp": pd.Timestamp("2026-03-06T00:00:00Z"), "asset": "BTC", "open": 100.0, "close": 100.5}, +@@ -208,6 +220,7 @@ def test_normalize_obadiaha_uses_official_token_order_and_causal_open_strike(): + official = pd.DataFrame([ + {"condition_id": "c5", "official_outcome": "Up", "url": "https://gamma.example/c5", "payload_sha256": "1" * 64}, + {"condition_id": "c15", "official_outcome": "Down", "url": "https://gamma.example/c15", "payload_sha256": "2" * 64}, ++ {"condition_id": "cgap", "official_outcome": "Down", "url": "https://gamma.example/cgap", "payload_sha256": "3" * 64}, + ]) + + # The immutable parquet corpus stores timestamps at microsecond resolution. +@@ -222,6 +235,15 @@ def test_normalize_obadiaha_uses_official_token_order_and_causal_open_strike(): + for column in columns: + frame[column] = frame[column].dt.tz_localize(None).astype("datetime64[us]") + ++ # The source gate covers every immutable market label, including contracts ++ # later excluded from the trading panel because a causal Binance boundary ++ # bar is unavailable. A disagreement in that source-only row must fail. ++ with pytest.raises(ValueError, match="official Polymarket resolution agreement"): ++ normalize_obadiaha_source( ++ markets, resolutions, gamma, spot, raw_books, raw_trades, official_resolutions=official ++ ) ++ official.loc[official["condition_id"] == "cgap", "official_outcome"] = "Up" ++ + slots, books, trades, audit = normalize_obadiaha_source( + markets, resolutions, gamma, spot, raw_books, raw_trades, official_resolutions=official + ) +@@ -237,6 +259,8 @@ def test_normalize_obadiaha_uses_official_token_order_and_causal_open_strike(): + assert slots.set_index("condition_id").loc["c15", "resolved_side"] == "No" + assert audit["official_resolution_coverage"] == 1.0 + assert audit["official_resolution_agreement"] == 1.0 ++ assert audit["official_resolution_source_contracts"] == 3 ++ assert audit["official_resolution_panel_contracts"] == 2 + assert audit["binance_direction_agreement_diagnostic"] == 0.5 + b5 = books.set_index("condition_id").loc["c5"] + assert b5["yes_ask_size"] == 8.0 diff --git a/ci_payload/digital-surface-official-resolution-patch/research/digital_surface/__init__.py b/ci_payload/digital-surface-official-resolution-patch/research/digital_surface/__init__.py new file mode 100644 index 00000000..9900e97a --- /dev/null +++ b/ci_payload/digital-surface-official-resolution-patch/research/digital_surface/__init__.py @@ -0,0 +1,11 @@ +"""Causal digital-surface relative-value research engine.""" + +from . import data as _data +from .official_resolution import install as _install_official_resolution + +_install_official_resolution(_data) + +from .types import AtomicFill, Contract, FoldResult, PairCandidate, SurfacePoint + +__all__ = ["AtomicFill", "Contract", "FoldResult", "PairCandidate", "SurfacePoint"] +__version__ = "0.1.0" diff --git a/ci_payload/digital-surface-official-resolution-patch/research/digital_surface/obadiaha_official_resolution_snapshot.json b/ci_payload/digital-surface-official-resolution-patch/research/digital_surface/obadiaha_official_resolution_snapshot.json new file mode 100644 index 00000000..a2990363 --- /dev/null +++ b/ci_payload/digital-surface-official-resolution-patch/research/digital_surface/obadiaha_official_resolution_snapshot.json @@ -0,0 +1,12 @@ +{ + "agreement": 1.0, + "canonical_labels_sha256": "fadc6558cf546e158eb7b4d806452a7864649446300178f5f5cf7e6cd72f1665", + "condition_ids_sha256": "ca8bd425d9766abda4a2858c41dca1de0ac70e57f00c174a4b1578f1f376e0f1", + "contracts": 5228, + "coverage": 1.0, + "encoding": "MSB-first outcome bitset; 1=Up, 0=Down; ordered by lexicographically sorted condition_id", + "full_evidence_snapshot_sha256": "bada3fc328d150390220f3b5e6440a21b399f21b3cb376e93682bbdb340884aa", + "outcome_bits_base64": "AZs9yM5vsi9cDpXrFbDJFqrexAcU5BwDIodD91+RpZdAWhO3GmseEe4aNZK2dTKdyJH5r3Va6EsL3jb7DWESzDaHqe3mtzbYNc6e1w042YmQ+1b2BrJ0q5z/GlO2JFqCItWI32BQLGPDlRZeb8gJVxpG/sYmmARa/KZN64oBjbtAsKLqdo9fqjz+GYFJhaHx9SbGIw7xdEYC+2do3J00cGKu3NpqgQ/TPRjKFwtnlv63MRaram1E7zP0BjN0ATrZWZv+Y6ffZP+FFKSBFsVaSoCGgesEqThvkGhzy0f5/fbW9GAcZUYvvbZNxN2ARNonz2BWgWmrj2Hm1+xnANqPFGao3bbjcVH7sFRDyLSq1vty30SyasAxXminU02XvglKIQ52/cNASbC1tpZ0FhtdgbNhpVTHQK+TE//din21xcRqvERjjmE6jpzNJx7olwSlWpbcOKc5gMAbGMs7l6NKFnHqFPdPEYgaxyA9uTr6lz/Z7EC5+e8cPzmEPUr7WVWS6BPpViOdCNyZpP2OdKivkIpUsdT2qXVfBUtx9Jlp9r1Al7Wm1ORVX0kgDdvJWWDeOI0upVawV5neaDdcHm4RGHNTtkzKgsVGBY71TD/aAvf0Ox5QDLFY/KmLKbtyGekSSM1Sajtr6+hPxdBzuTIO3lOxWBG9x+LRZEpIaMZY+SK+iRYamMiau88yTiQTR4Y9CLhNyMuIvVKNzHLNSa2aEHGKssvF3AxMzD9a4lvMfdxkFNGRRt0Gd4RjDHU+DGszcy1YSCUnPo9ByBMNZtzqfSYI1q/hrewC7SNkqraixDhLGOx4pb3tHFkKv/T1Ccpr2wpQ4QESNlb3VltU25rY7FZJVmt6Zh2yucZE3bog", + "source": "official Polymarket Gamma event terminal outcomes", + "upstream_full_audit_artifact_sha256": "3b86cebd50910e76e1e3d222e4ded9cfe5fd8df9dcd4c929fcb5a34c53a44723" +} diff --git a/ci_payload/digital-surface-official-resolution-patch/research/digital_surface/official_resolution.py b/ci_payload/digital-surface-official-resolution-patch/research/digital_surface/official_resolution.py new file mode 100644 index 00000000..3536d29e --- /dev/null +++ b/ci_payload/digital-surface-official-resolution-patch/research/digital_surface/official_resolution.py @@ -0,0 +1,228 @@ +from __future__ import annotations + +import hashlib +from pathlib import Path +from types import ModuleType +from typing import Any + +import pandas as pd + +SNAPSHOT_METADATA_PATH = Path(__file__).with_name("obadiaha_official_resolution_snapshot.json") +SNAPSHOT_METADATA_SHA256 = "36f34ff7ee5c2f82e89b4d8674ce4371aea27ad1a63287ba537a6939d997d728" +FULL_EVIDENCE_SNAPSHOT_SHA256 = "bada3fc328d150390220f3b5e6440a21b399f21b3cb376e93682bbdb340884aa" +UPSTREAM_FULL_AUDIT_ARTIFACT_SHA256 = "3b86cebd50910e76e1e3d222e4ded9cfe5fd8df9dcd4c929fcb5a34c53a44723" +RESOLUTION_SOURCE = ( + "Official Polymarket terminal outcome; Gamma [Yes, No] token map; " + "causal Binance opening strike" +) + + +def _sha256_file(path: str | Path) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + for block in iter(lambda: handle.read(1 << 20), b""): + digest.update(block) + return digest.hexdigest() + + +def load_official_resolution_snapshot( + condition_ids: Any = None, + *, + path: str | Path = SNAPSHOT_METADATA_PATH, + expected_sha256: str = SNAPSHOT_METADATA_SHA256, +) -> pd.DataFrame: + import base64 + import json + + metadata_path = Path(path) + got = _sha256_file(metadata_path) + if got != expected_sha256: + raise RuntimeError( + f"official resolution metadata SHA-256 mismatch: {got} != {expected_sha256}" + ) + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + if metadata.get("upstream_full_audit_artifact_sha256") != UPSTREAM_FULL_AUDIT_ARTIFACT_SHA256: + raise RuntimeError("official resolution upstream audit artifact identity mismatch") + if metadata.get("full_evidence_snapshot_sha256") != FULL_EVIDENCE_SNAPSHOT_SHA256: + raise RuntimeError("official resolution full evidence snapshot identity mismatch") + count = int(metadata.get("contracts", -1)) + packed = base64.b64decode(str(metadata.get("outcome_bits_base64") or ""), validate=True) + if len(packed) != (count + 7) // 8: + raise RuntimeError("official resolution outcome bitset length mismatch") + outcomes = [ + "Up" if ((packed[index // 8] >> (7 - index % 8)) & 1) else "Down" + for index in range(count) + ] + if condition_ids is None: + ordered_ids = [f"ordinal:{index:06d}" for index in range(count)] + else: + ordered_ids = sorted({str(value) for value in condition_ids}) + if len(ordered_ids) != count: + raise RuntimeError( + f"official resolution condition count mismatch: {len(ordered_ids)} != {count}" + ) + canonical_ids = ("\n".join(ordered_ids) + "\n").encode("utf-8") + ids_sha256 = hashlib.sha256(canonical_ids).hexdigest() + if ids_sha256 != str(metadata.get("condition_ids_sha256")): + raise RuntimeError( + f"official resolution condition-set SHA-256 mismatch: {ids_sha256} " + f"!= {metadata.get('condition_ids_sha256')}" + ) + frame = pd.DataFrame({"condition_id": ordered_ids, "official_outcome": outcomes}) + if not frame["official_outcome"].isin(["Up", "Down"]).all(): + raise ValueError("official resolution snapshot contains non-terminal outcomes") + # The compact bitset is cryptographically bound to the sorted condition-ID + # set. The checksum-pinned full audit artifact retains every Gamma URL and + # response payload hash used to derive these labels. + frame["url"] = "artifact://digital-surface-full-chainlink-source-audit" + frame["payload_sha256"] = UPSTREAM_FULL_AUDIT_ARTIFACT_SHA256 + return frame + + +def _slug_epoch_seconds(series: pd.Series) -> pd.Series: + raw = series.astype(str).str.extract(r"(\d{10,13})(?:\D*)$", expand=False) + values = pd.to_numeric(raw, errors="coerce") + seconds = values.where(values < 100_000_000_000, values // 1000) + return seconds.astype("Int64") + + +def install(data_module: ModuleType) -> None: + """Install the corrected source gate without changing strategy geometry. + + The recovered adapter used Binance minute bars for two different roles: + causal trading state and settlement-label validation. Binance remains the + causal strike/diagnostic source. The unchanged 99% pre-fold gate now checks + immutable labels against checksum-pinned official Polymarket terminal + outcomes, which are the actual market-resolution record. + """ + if getattr(data_module, "_OFFICIAL_RESOLUTION_PATCH_INSTALLED", False): + return + + original_normalize = data_module.normalize_obadiaha_source + original_load_validation = data_module._load_obadiaha_validation + + def normalize_obadiaha_source( + markets: pd.DataFrame, + resolutions: pd.DataFrame, + gamma_tokens: pd.DataFrame, + spot_bars: pd.DataFrame, + raw_books: pd.DataFrame, + raw_trades: pd.DataFrame, + *, + official_resolutions: pd.DataFrame | None = None, + minimum_oracle_agreement: float = 0.99, + ) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, dict[str, Any]]: + if "slug" not in markets.columns: + raise ValueError("Obadiaha markets missing columns: ['slug']") + patched_markets = markets.copy() + slug_open = _slug_epoch_seconds(patched_markets["slug"]) + if slug_open.isna().any(): + examples = patched_markets.loc[slug_open.isna(), "slug"].astype(str).head(10).tolist() + raise ValueError( + f"Obadiaha slugs missing terminal Unix epochs; examples: {examples}" + ) + duration_seconds = patched_markets["market_type"].map( + {"crypto_5m": 300, "crypto_15m": 900} + ) + if duration_seconds.isna().any(): + examples = patched_markets.loc[duration_seconds.isna(), "market_type"].head(10).tolist() + raise ValueError(f"unsupported Obadiaha market types; examples: {examples}") + open_ts = slug_open.astype("int64") + patched_markets["start_time"] = pd.to_datetime(open_ts, unit="s", utc=True) + patched_markets["end_time"] = pd.to_datetime( + open_ts + duration_seconds.astype("int64"), unit="s", utc=True + ) + + # Preserve Binance as causal state and diagnostic, but do not use a + # different exchange as the settlement oracle. + slots, books, trades, audit = original_normalize( + patched_markets, + resolutions, + gamma_tokens, + spot_bars, + raw_books, + raw_trades, + minimum_oracle_agreement=0.0, + ) + binance_direction_agreement = float(audit["oracle_agreement"]) + + official = ( + load_official_resolution_snapshot(slots["condition_id"]) + if official_resolutions is None + else official_resolutions.copy() + ) + required = {"condition_id", "official_outcome", "url", "payload_sha256"} + missing = sorted(required - set(official.columns)) + if missing: + raise ValueError(f"official resolution evidence missing columns: {missing}") + official = official[list(required)].copy() + official["condition_id"] = official["condition_id"].astype(str) + official["official_outcome"] = official["official_outcome"].astype(str).str.title() + if official["condition_id"].duplicated().any(): + raise ValueError("official resolution evidence contains duplicate condition IDs") + if not official["official_outcome"].isin(["Up", "Down"]).all(): + raise ValueError("official resolution evidence contains non-terminal outcomes") + if official["url"].astype(str).str.strip().eq("").any(): + raise ValueError("official resolution evidence contains empty verification URLs") + if not official["payload_sha256"].astype(str).str.fullmatch(r"[0-9a-fA-F]{64}").all(): + raise ValueError("official resolution evidence contains invalid payload SHA-256 values") + + recorded = resolutions[["condition_id", "outcome"]].copy() + recorded["condition_id"] = recorded["condition_id"].astype(str) + recorded["outcome"] = recorded["outcome"].astype(str).str.title() + recorded = recorded.drop_duplicates("condition_id", keep="last") + admitted_ids = slots["condition_id"].astype(str).drop_duplicates() + evidence = pd.DataFrame({"condition_id": admitted_ids}).merge( + recorded, + on="condition_id", + how="left", + validate="one_to_one", + ).merge( + official, + on="condition_id", + how="left", + validate="one_to_one", + ) + coverage = float(evidence["official_outcome"].notna().mean()) + if coverage < 1.0: + raise ValueError( + f"official Polymarket resolution coverage {coverage:.4%} is below 100.00%" + ) + agreement = float((evidence["official_outcome"] == evidence["outcome"]).mean()) + if agreement < float(minimum_oracle_agreement): + raise ValueError( + f"official Polymarket resolution agreement {agreement:.4%} " + f"is below {minimum_oracle_agreement:.2%}" + ) + + audit["oracle_agreement"] = agreement + audit["official_resolution_coverage"] = coverage + audit["official_resolution_agreement"] = agreement + audit["binance_direction_agreement_diagnostic"] = binance_direction_agreement + return slots, books, trades, audit + + def _load_obadiaha_validation(cache_dir: Path): + panel, audit = original_load_validation(cache_dir) + registry, state, books, trades = panel + registry = registry.copy() + if "resolution_source" in registry.columns: + registry["resolution_source"] = RESOLUTION_SOURCE + audit["official_resolution_snapshot_sha256"] = FULL_EVIDENCE_SNAPSHOT_SHA256 + audit["official_resolution_metadata_sha256"] = SNAPSHOT_METADATA_SHA256 + audit["official_resolution_upstream_artifact_sha256"] = UPSTREAM_FULL_AUDIT_ARTIFACT_SHA256 + audit["official_resolution_rows"] = int(len(load_official_resolution_snapshot())) + return (registry, state, books, trades), audit + + data_module.normalize_obadiaha_source = normalize_obadiaha_source + data_module.load_official_resolution_snapshot = load_official_resolution_snapshot + data_module._load_obadiaha_validation = _load_obadiaha_validation + data_module.OBADIAHA_OFFICIAL_RESOLUTION_PATH = SNAPSHOT_METADATA_PATH + data_module.OBADIAHA_OFFICIAL_RESOLUTION_SHA256 = SNAPSHOT_METADATA_SHA256 + try: + data_module.SOURCE_SPEC["validation_source"]["resolution"] = ( + "Obadiaha recorded outcome with >=99% official Polymarket " + "terminal-outcome agreement" + ) + except (KeyError, TypeError): + pass + data_module._OFFICIAL_RESOLUTION_PATCH_INSTALLED = True diff --git a/ci_payload/digital-surface-official-resolution-patch/resolved-universe-fix.patch b/ci_payload/digital-surface-official-resolution-patch/resolved-universe-fix.patch new file mode 100644 index 00000000..607bc5f9 --- /dev/null +++ b/ci_payload/digital-surface-official-resolution-patch/resolved-universe-fix.patch @@ -0,0 +1,78 @@ +diff --git a/research/digital_surface/official_resolution.py b/research/digital_surface/official_resolution.py +--- a/research/digital_surface/official_resolution.py ++++ b/research/digital_surface/official_resolution.py +@@ -162,7 +162,15 @@ + ) + binance_direction_agreement = float(audit["oracle_agreement"]) + +- source_ids = patched_markets["condition_id"].astype(str).drop_duplicates() ++ market_ids = patched_markets["condition_id"].astype(str).drop_duplicates() ++ resolved_source = resolutions[["condition_id", "outcome"]].copy() ++ resolved_source["condition_id"] = resolved_source["condition_id"].astype(str) ++ resolved_source["outcome"] = resolved_source["outcome"].astype(str).str.title() ++ resolved_source = resolved_source[ ++ resolved_source["outcome"].isin(["Up", "Down"]) ++ ].drop_duplicates("condition_id", keep="last") ++ resolved_ids = set(resolved_source["condition_id"]) ++ source_ids = market_ids[market_ids.isin(resolved_ids)] + official = ( + load_official_resolution_snapshot(source_ids) + if official_resolutions is None +@@ -214,7 +222,9 @@ + audit["oracle_agreement"] = agreement + audit["official_resolution_coverage"] = coverage + audit["official_resolution_agreement"] = agreement ++ audit["official_resolution_market_contracts"] = int(len(market_ids)) + audit["official_resolution_source_contracts"] = int(len(source_ids)) ++ audit["official_resolution_unresolved_market_contracts"] = int(len(market_ids) - len(source_ids)) + audit["official_resolution_panel_contracts"] = int(slots["condition_id"].nunique()) + audit["binance_direction_agreement_diagnostic"] = binance_direction_agreement + return slots, books, trades, audit +diff --git a/tests/digital_surface/test_data_adapter.py b/tests/digital_surface/test_data_adapter.py +--- a/tests/digital_surface/test_data_adapter.py ++++ b/tests/digital_surface/test_data_adapter.py +@@ -190,6 +190,16 @@ + "market_type": "crypto_5m", + "slug": "btc-updown-5m-1772755500", + }, ++ { ++ "market_id": "btc-updown-5m-1772755800", ++ "asset": "BTC", ++ "condition_id": "cpending", ++ "question": "Bitcoin Up or Down", ++ "start_time": pd.Timestamp("2026-03-06T00:15:00Z"), ++ "end_time": pd.Timestamp("2026-03-06T00:15:00Z"), ++ "market_type": "crypto_5m", ++ "slug": "btc-updown-5m-1772755800", ++ }, + ]) + resolutions = pd.DataFrame([ + {"condition_id": "c5", "outcome": "Up"}, +@@ -200,6 +210,7 @@ + {"condition_id": "c5", "yes_token_id": "yes5", "no_token_id": "no5"}, + {"condition_id": "c15", "yes_token_id": "yes15", "no_token_id": "no15"}, + {"condition_id": "cgap", "yes_token_id": "yesgap", "no_token_id": "nogap"}, ++ {"condition_id": "cpending", "yes_token_id": "yespending", "no_token_id": "nopending"}, + ]) + spot = pd.DataFrame([ + {"timestamp": pd.Timestamp("2026-03-06T00:00:00Z"), "asset": "BTC", "open": 100.0, "close": 100.5}, +@@ -244,6 +255,9 @@ + ) + official.loc[official["condition_id"] == "cgap", "official_outcome"] = "Up" + ++ # Markets without a terminal Up/Down resolution are not label-bearing source ++ # contracts. They remain visible in the raw market audit but must not dilute ++ # official-outcome coverage or alter the 5,228 resolved-contract universe. + slots, books, trades, audit = normalize_obadiaha_source( + markets, resolutions, gamma, spot, raw_books, raw_trades, official_resolutions=official + ) +@@ -259,7 +273,9 @@ + assert slots.set_index("condition_id").loc["c15", "resolved_side"] == "No" + assert audit["official_resolution_coverage"] == 1.0 + assert audit["official_resolution_agreement"] == 1.0 ++ assert audit["official_resolution_market_contracts"] == 4 + assert audit["official_resolution_source_contracts"] == 3 ++ assert audit["official_resolution_unresolved_market_contracts"] == 1 + assert audit["official_resolution_panel_contracts"] == 2 + assert audit["binance_direction_agreement_diagnostic"] == 0.5 + b5 = books.set_index("condition_id").loc["c5"] diff --git a/ci_payload/digital-surface-official-resolution-patch/tests/digital_surface/test_data_adapter.py b/ci_payload/digital-surface-official-resolution-patch/tests/digital_surface/test_data_adapter.py new file mode 100644 index 00000000..3a9cd8c9 --- /dev/null +++ b/ci_payload/digital-surface-official-resolution-patch/tests/digital_surface/test_data_adapter.py @@ -0,0 +1,296 @@ +from pathlib import Path +import hashlib +import json + +import pytest + +from research.digital_surface.data import download_verified + + +def test_download_verified_rejects_hash_mismatch(tmp_path, monkeypatch): + class Response: + def __enter__(self): return self + def __exit__(self, *args): pass + def raise_for_status(self): pass + def iter_content(self, size): yield b"bad" + monkeypatch.setattr("research.digital_surface.data.requests.get", lambda *a, **k: Response()) + spec = {"repo": "x", "source_revision": "r", "resolution_source": "o", "files": {"slots": {"url": "https://example.invalid", "sha256": "0" * 64}}} + with pytest.raises(RuntimeError, match="hash mismatch"): + download_verified(spec, tmp_path) + + +def test_download_verified_writes_manifest(tmp_path, monkeypatch): + payload = b"good" + digest = hashlib.sha256(payload).hexdigest() + class Response: + def __enter__(self): return self + def __exit__(self, *args): pass + def raise_for_status(self): pass + def iter_content(self, size): yield payload + monkeypatch.setattr("research.digital_surface.data.requests.get", lambda *a, **k: Response()) + spec = {"repo": "x", "source_revision": "r", "resolution_source": "o", "files": {"slots": {"url": "https://example.invalid", "sha256": digest}}} + paths = download_verified(spec, tmp_path) + assert paths["slots"].read_bytes() == payload + manifest = json.loads((tmp_path / "SOURCE_MANIFEST.json").read_text()) + assert manifest["files"]["slots"]["sha256"] == digest + +import pandas as pd + +from research.digital_surface.data import ( + assign_hybrid_week_labels, + git_blob_sha1_bytes, + load_official_resolution_snapshot, + normalize_obadiaha_source, +) + + +def test_git_blob_sha1_matches_git_object_definition(): + payload = b"hello\n" + assert git_blob_sha1_bytes(payload) == "ce013625030ba8dba906f756967f9e9ca394464a" + + +def test_lfs_download_records_pointer_identity_separately_from_payload_hash(tmp_path, monkeypatch): + from research.digital_surface.data import download_git_blob_verified + + payload = b"parquet-payload" + + class Response: + def __enter__(self): return self + def __exit__(self, *args): pass + def raise_for_status(self): pass + def iter_content(self, size): yield payload + + monkeypatch.setattr("research.digital_surface.data.requests.get", lambda *a, **k: Response()) + spec = { + "repo": "owner/dataset", + "revision": "a" * 40, + "files": [{ + "path": "data/file.parquet", + "size": len(payload), + "git_blob_sha1": "b" * 40, + }], + } + paths = download_git_blob_verified(spec, tmp_path) + assert paths["data/file.parquet"].read_bytes() == payload + manifest = json.loads((tmp_path / "SOURCE_MANIFEST.json").read_text()) + row = manifest["files"]["data/file.parquet"] + assert row["repository_git_blob_sha1"] == "b" * 40 + assert row["sha256"] == hashlib.sha256(payload).hexdigest() + assert row["bytes"] == len(payload) + + + +def test_gamma_historical_lookup_batches_exact_condition_ids_and_includes_closed(tmp_path, monkeypatch): + from research.digital_surface.data import fetch_gamma_token_map + + markets = pd.DataFrame([ + { + "condition_id": "c1", + "slug": "btc-updown-5m-1", + "start_time": pd.Timestamp("2026-03-06T00:05:00Z"), + "end_time": pd.Timestamp("2026-03-06T00:05:00Z"), + }, + { + "condition_id": "c2", + "slug": "btc-updown-15m-1", + "start_time": pd.Timestamp("2026-03-06T00:15:00Z"), + "end_time": pd.Timestamp("2026-03-06T00:15:00Z"), + }, + ]) + calls = [] + + def fake_request(url, *, params): + calls.append(params) + assert params["closed"] == "true" + assert params["condition_ids"] == ["c1", "c2"] + return [ + {"id": "1", "conditionId": "c1", "slug": "btc-updown-5m-1", "outcomes": '["Yes","No"]', "clobTokenIds": '["yes1","no1"]'}, + {"id": "2", "conditionId": "c2", "slug": "btc-updown-15m-1", "outcomes": '["Yes","No"]', "clobTokenIds": '["yes2","no2"]'}, + ] + + monkeypatch.setattr("research.digital_surface.data._request_json", fake_request) + result = fetch_gamma_token_map(markets, tmp_path / "gamma.json").set_index("condition_id") + assert result.loc["c1", "yes_token_id"] == "yes1" + assert result.loc["c2", "no_token_id"] == "no2" + assert len(calls) == 1 + + +def test_resumable_download_retries_from_exact_partial_offset(tmp_path, monkeypatch): + import requests + from research.digital_surface.data import _download_resumable + + payload = b"abcdefghij" + calls = [] + + class FakeResponse: + def __init__(self, status_code, blocks): + self.status_code = status_code + self.blocks = blocks + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def raise_for_status(self): + return None + + def iter_content(self, _chunk_size): + for block in self.blocks: + if isinstance(block, Exception): + raise block + yield block + + responses = [ + FakeResponse(200, [payload[:4], requests.exceptions.ChunkedEncodingError("broken")]), + FakeResponse(206, [payload[4:]]), + ] + + def fake_get(url, *, stream, timeout, headers): + calls.append(dict(headers)) + return responses.pop(0) + + monkeypatch.setattr("research.digital_surface.data.requests.get", fake_get) + monkeypatch.setattr("research.digital_surface.data.time.sleep", lambda _seconds: None) + target = tmp_path / "payload.part" + _download_resumable("https://example.invalid/file", target, len(payload), attempts=2) + assert target.read_bytes() == payload + assert calls == [{}, {"Range": "bytes=4-"}] + +def test_normalize_obadiaha_uses_official_token_order_and_causal_open_strike(): + markets = pd.DataFrame([ + { + "market_id": "btc-updown-5m-1772755200", + "asset": "BTC", + "condition_id": "c5", + "question": "Bitcoin Up or Down", + "start_time": pd.Timestamp("2026-03-06T00:05:00Z"), + "end_time": pd.Timestamp("2026-03-06T00:05:00Z"), + "market_type": "crypto_5m", + "slug": "btc-updown-5m-1772755200", + }, + { + "market_id": "btc-updown-15m-1772755200", + "asset": "BTC", + "condition_id": "c15", + "question": "Bitcoin Up or Down", + "start_time": pd.Timestamp("2026-03-06T00:15:00Z"), + "end_time": pd.Timestamp("2026-03-06T00:15:00Z"), + "market_type": "crypto_15m", + "slug": "btc-updown-15m-1772755200", + }, + ]) + resolutions = pd.DataFrame([ + {"condition_id": "c5", "outcome": "Up"}, + {"condition_id": "c15", "outcome": "Down"}, + ]) + gamma = pd.DataFrame([ + {"condition_id": "c5", "yes_token_id": "yes5", "no_token_id": "no5"}, + {"condition_id": "c15", "yes_token_id": "yes15", "no_token_id": "no15"}, + ]) + spot = pd.DataFrame([ + {"timestamp": pd.Timestamp("2026-03-06T00:00:00Z"), "asset": "BTC", "open": 100.0, "close": 100.5}, + {"timestamp": pd.Timestamp("2026-03-06T00:04:00Z"), "asset": "BTC", "open": 101.0, "close": 99.5}, + {"timestamp": pd.Timestamp("2026-03-06T00:14:00Z"), "asset": "BTC", "open": 99.5, "close": 99.0}, + ]) + raw_books = pd.DataFrame([ + {"timestamp": pd.Timestamp("2026-03-06T00:03:00Z"), "asset": "BTC", "market_id": "btc-updown-5m-1772755200", "condition_id": "c5", "token_id": "yes5", "best_bid": .40, "best_ask": .41, "bid_levels": '[{"price":0.4,"size":7}]', "ask_levels": '[{"price":0.41,"size":8}]'}, + {"timestamp": pd.Timestamp("2026-03-06T00:03:00Z"), "asset": "BTC", "market_id": "btc-updown-5m-1772755200", "condition_id": "c5", "token_id": "no5", "best_bid": .59, "best_ask": .60, "bid_levels": '[{"price":0.59,"size":9}]', "ask_levels": '[{"price":0.6,"size":10}]'}, + {"timestamp": pd.Timestamp("2026-03-06T00:13:00Z"), "asset": "BTC", "market_id": "btc-updown-15m-1772755200", "condition_id": "c15", "token_id": "yes15", "best_bid": .30, "best_ask": .31, "bid_levels": '[{"price":0.3,"size":11}]', "ask_levels": '[{"price":0.31,"size":12}]'}, + {"timestamp": pd.Timestamp("2026-03-06T00:13:00Z"), "asset": "BTC", "market_id": "btc-updown-15m-1772755200", "condition_id": "c15", "token_id": "no15", "best_bid": .69, "best_ask": .70, "bid_levels": '[{"price":0.69,"size":13}]', "ask_levels": '[{"price":0.7,"size":14}]'}, + ]) + raw_trades = pd.DataFrame([ + {"timestamp": pd.Timestamp("2026-03-06T00:03:01Z"), "asset": "BTC", "condition_id": "c5", "token_id": "yes5", "side": "BUY", "price": .41, "size": 5.0}, + {"timestamp": pd.Timestamp("2026-03-06T00:03:01Z"), "asset": "BTC", "condition_id": "c5", "token_id": "no5", "side": "SELL", "price": .59, "size": 2.0}, + ]) + + official = pd.DataFrame([ + {"condition_id": "c5", "official_outcome": "Up", "url": "https://gamma.example/c5", "payload_sha256": "1" * 64}, + {"condition_id": "c15", "official_outcome": "Down", "url": "https://gamma.example/c15", "payload_sha256": "2" * 64}, + ]) + + slots, books, trades, audit = normalize_obadiaha_source( + markets, resolutions, gamma, spot, raw_books, raw_trades, official_resolutions=official + ) + + by_condition = slots.set_index("condition_id") + assert by_condition.loc["c5", "open_ts"] == 1772755200 + assert by_condition.loc["c5", "close_ts"] == 1772755500 + assert by_condition.loc["c15", "close_ts"] == 1772756100 + assert by_condition.loc["c5", "strike"] == 100.0 + assert slots.set_index("condition_id").loc["c5", "yes_token_id"] == "yes5" + assert slots.set_index("condition_id").loc["c15", "no_token_id"] == "no15" + assert slots.set_index("condition_id").loc["c5", "resolved_side"] == "Yes" + assert slots.set_index("condition_id").loc["c15", "resolved_side"] == "No" + assert audit["official_resolution_coverage"] == 1.0 + assert audit["official_resolution_agreement"] == 1.0 + assert audit["binance_direction_agreement_diagnostic"] == 0.5 + b5 = books.set_index("condition_id").loc["c5"] + assert b5["yes_ask_size"] == 8.0 + assert b5["no_ask_size"] == 10.0 + assert b5["spot"] == 100.0 # backward minute open, never the future 00:04 bar + assert trades.loc[0, "taker_buy"] + assert not trades.loc[1, "taker_buy"] + pinned = load_official_resolution_snapshot() + assert len(pinned) == 5228 + assert pinned["official_outcome"].isin(["Up", "Down"]).all() + + +def test_assign_hybrid_week_labels_orders_validation_before_four_transfer_tests(): + frame = pd.DataFrame({ + "source": ["obadiaha", "obadiaha", "kinzik", "kinzik", "kinzik", "kinzik"], + "close_ts": [ + int(pd.Timestamp("2026-03-07T00:00:00Z").timestamp()), + int(pd.Timestamp("2026-03-14T00:00:00Z").timestamp()), + int(pd.Timestamp("2026-05-28T00:00:00Z").timestamp()), + int(pd.Timestamp("2026-06-04T00:00:00Z").timestamp()), + int(pd.Timestamp("2026-06-11T00:00:00Z").timestamp()), + int(pd.Timestamp("2026-06-18T00:00:00Z").timestamp()), + ], + }) + out = assign_hybrid_week_labels(frame) + assert out["week"].tolist() == [ + "00-obadiaha-2026-03-06_12", + "01-obadiaha-2026-03-13_19", + "02-kinzik-2026-05-26_06-01", + "03-kinzik-2026-06-02_06-08", + "04-kinzik-2026-06-09_06-15", + "05-kinzik-2026-06-16_06-22", + ] + +from research.digital_surface.data import combine_hybrid_panels + + +def test_combine_hybrid_panels_preserves_two_validation_and_four_transfer_blocks(): + def panel(source, closes): + registry = pd.DataFrame({ + "condition_id": [f"{source}-{i}" for i in range(len(closes))], + "close_ts": closes, + "source": source, + }) + state = registry.copy() + state["ts_ms"] = state["close_ts"] * 1000 - 1000 + books = pd.DataFrame({"condition_id": registry["condition_id"], "ts_ms": state["ts_ms"], "source": source}) + trades = pd.DataFrame({"token_id": [f"t-{source}-{i}" for i in range(len(closes))], "ts_ms": state["ts_ms"], "source": source}) + return registry, state, books, trades + + ob_closes = [ + int(pd.Timestamp("2026-03-07T00:00:00Z").timestamp()), + int(pd.Timestamp("2026-03-14T00:00:00Z").timestamp()), + ] + kin_closes = [ + int(pd.Timestamp("2026-05-28T00:00:00Z").timestamp()), + int(pd.Timestamp("2026-06-04T00:00:00Z").timestamp()), + int(pd.Timestamp("2026-06-11T00:00:00Z").timestamp()), + int(pd.Timestamp("2026-06-18T00:00:00Z").timestamp()), + ] + registry, state, books, trades = combine_hybrid_panels(panel("obadiaha", ob_closes), panel("kinzik", kin_closes)) + assert state["week"].nunique() == 6 + assert sorted(state["week"].unique())[:2] == [ + "00-obadiaha-2026-03-06_12", + "01-obadiaha-2026-03-13_19", + ] + assert all(label.startswith(("00-", "01-")) for label in registry[registry.source == "obadiaha"].week) + assert len(books) == 6 + assert len(trades) == 6 diff --git a/ci_payload/digital-surface-official-resolution-patch/timestamp-unit-fix.patch b/ci_payload/digital-surface-official-resolution-patch/timestamp-unit-fix.patch new file mode 100644 index 00000000..0179e7e5 --- /dev/null +++ b/ci_payload/digital-surface-official-resolution-patch/timestamp-unit-fix.patch @@ -0,0 +1,50 @@ +diff --git a/research/digital_surface/official_resolution.py b/research/digital_surface/official_resolution.py +index 3536d29..1b180fc 100644 +--- a/research/digital_surface/official_resolution.py ++++ b/research/digital_surface/official_resolution.py +@@ -101,6 +101,22 @@ def install(data_module: ModuleType) -> None: + original_normalize = data_module.normalize_obadiaha_source + original_load_validation = data_module._load_obadiaha_validation + ++ def _timestamp_seconds(series: pd.Series) -> pd.Series: ++ values = pd.to_datetime(series, utc=True) ++ nanoseconds = values.dt.as_unit("ns").astype("int64") ++ return (nanoseconds // 1_000_000_000).astype("int64") ++ ++ def _timestamp_milliseconds(series: pd.Series) -> pd.Series: ++ values = pd.to_datetime(series, utc=True) ++ nanoseconds = values.dt.as_unit("ns").astype("int64") ++ return (nanoseconds // 1_000_000).astype("int64") ++ ++ # Parquet preserves datetime64[us]. The recovered adapter divided the raw ++ # physical integers as though they were nanoseconds, emptying every as-of ++ # join. Normalize to nanoseconds explicitly before epoch conversion. ++ data_module._timestamp_seconds = _timestamp_seconds ++ data_module._timestamp_milliseconds = _timestamp_milliseconds ++ + def normalize_obadiaha_source( + markets: pd.DataFrame, + resolutions: pd.DataFrame, +diff --git a/tests/digital_surface/test_data_adapter.py b/tests/digital_surface/test_data_adapter.py +index 3a9cd8c..4042b1d 100644 +--- a/tests/digital_surface/test_data_adapter.py ++++ b/tests/digital_surface/test_data_adapter.py +@@ -210,6 +210,18 @@ def test_normalize_obadiaha_uses_official_token_order_and_causal_open_strike(): + {"condition_id": "c15", "official_outcome": "Down", "url": "https://gamma.example/c15", "payload_sha256": "2" * 64}, + ]) + ++ # The immutable parquet corpus stores timestamps at microsecond resolution. ++ # The adapter must normalize the physical unit before converting to epoch ++ # seconds/milliseconds; treating these integers as nanoseconds empties the panel. ++ for frame, columns in ( ++ (markets, ("start_time", "end_time")), ++ (spot, ("timestamp",)), ++ (raw_books, ("timestamp",)), ++ (raw_trades, ("timestamp",)), ++ ): ++ for column in columns: ++ frame[column] = frame[column].dt.tz_localize(None).astype("datetime64[us]") ++ + slots, books, trades, audit = normalize_obadiaha_source( + markets, resolutions, gamma, spot, raw_books, raw_trades, official_resolutions=official + ) diff --git a/docs/superpowers/plans/2026-07-25-digital-surface-live-readiness.md b/docs/superpowers/plans/2026-07-25-digital-surface-live-readiness.md new file mode 100644 index 00000000..b0189bcb --- /dev/null +++ b/docs/superpowers/plans/2026-07-25-digital-surface-live-readiness.md @@ -0,0 +1,321 @@ +# Digital Surface Live-Readiness Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Diagnose and remove legitimate execution bottlenecks, create new uncontaminated acceptance folds, and build a fail-closed Polymarket CLOB V2 shadow bot that cannot submit real orders by default. + +**Architecture:** Keep the existing causal surface, candidate, execution, portfolio, and validation modules as the research core. Add focused diagnostics and liquidity-policy modules, then add a separate live package whose only dependency on research is an immutable `TradeIntent` interface. Historical known weeks remain diagnostic; new prospective weeks alone determine admission. + +**Tech Stack:** Python 3.11, pandas 2.2.3, NumPy 2.2.6, SciPy 1.15.3, scikit-learn 1.6.1, PyArrow 19.0.1, pytest 8.3.5, Polymarket CLOB V2 REST/WebSocket APIs, GitHub Actions. + +## Global Constraints + +- Preserve the 99% official-resolution agreement gate and 100% coverage requirement. +- Never tune against the four already observed Kinzik test weeks. +- Keep all execution evidence post-arrival and strictly causal. +- Do not claim two independent market orders are atomic. +- Default `TRADING_MODE` is `shadow`; CI may not set it to `live`. +- No secret, private key, API key, passphrase, or wallet material may be committed. +- Commit every meaningful diagnostic, test, fix, workflow, and result checkpoint. + +--- + +### Task 1: Deterministic execution failure attribution + +**Files:** +- Create: `research/digital_surface/diagnostics.py` +- Create: `tests/digital_surface/test_diagnostics.py` +- Modify: `research/digital_surface/run_research.py` +- Create: `.github/workflows/digital-surface-execution-diagnostics.yml` + +**Interfaces:** +- Consumes: `candidate_ledger.parquet`, `execution_ledger.parquet`, and existing candidate/execution columns. +- Produces: `attribute_execution_failures(candidates: pd.DataFrame, executions: pd.DataFrame) -> tuple[pd.DataFrame, dict[str, object]]` and artifacts `execution_failure_attribution.csv` plus `EXECUTION_DIAGNOSTICS.json`. + +- [ ] **Step 1: Write failing tests** + +```python +def test_attributes_each_leg_to_one_terminal_reason(): + candidates = pd.DataFrame([ + {"candidate_id": "a", "tau_s": 30.0, "expected_cost": 1.05, "theoretical_edge": 0.03}, + {"candidate_id": "b", "tau_s": 12.0, "expected_cost": 1.02, "theoretical_edge": 0.04}, + ]) + executions = pd.DataFrame([ + { + "candidate_id": "a", + "atomic": False, + "low_book_ts_ms": np.nan, + "low_book_crossable": False, + "low_book_full": False, + "low_tape_full": False, + "high_book_ts_ms": 2, + "high_book_crossable": True, + "high_book_full": True, + "high_tape_full": False, + }, + { + "candidate_id": "b", + "atomic": True, + "low_book_ts_ms": 2, + "low_book_crossable": True, + "low_book_full": True, + "low_tape_full": True, + "high_book_ts_ms": 2, + "high_book_crossable": True, + "high_book_full": True, + "high_tape_full": True, + }, + ]) + ledger, summary = attribute_execution_failures(candidates, executions) + assert ledger.set_index(["candidate_id", "leg"]).loc[("a", "low"), "reason"] == "missing_post_arrival_book" + assert ledger.set_index(["candidate_id", "leg"]).loc[("a", "high"), "reason"] == "insufficient_post_arrival_tape" + assert ledger[ledger["candidate_id"] == "b"]["reason"].tolist() == ["filled", "filled"] + assert summary["legs_attributed"] == 4 +``` + +- [ ] **Step 2: Run the test and verify RED** + +Run: `PYTHONPATH=. pytest tests/digital_surface/test_diagnostics.py -v` +Expected: FAIL because `research.digital_surface.diagnostics` does not exist. + +- [ ] **Step 3: Implement one-reason precedence** + +Reason precedence per leg: + +1. `missing_post_arrival_book` +2. `ask_above_limit` +3. `insufficient_displayed_depth` +4. `insufficient_post_arrival_tape` +5. `filled` + +Join candidates one-to-one to executions, emit exactly two rows per candidate, assert complete attribution, and summarize by week, leg, reason, tau bucket, expected-cost bucket, edge bucket, and requested shares. + +- [ ] **Step 4: Run focused and full tests** + +Run: + +```bash +PYTHONPATH=. pytest tests/digital_surface/test_diagnostics.py -v +PYTHONPATH=. pytest tests/digital_surface -q +``` + +Expected: focused PASS and exactly 29 existing tests plus the new diagnostics tests all PASS. + +- [ ] **Step 5: Wire diagnostics into `run_research.py`** + +Write the CSV and JSON only after candidate and execution ledgers exist. Include their hashes in `SHA256SUMS.json` and `MANIFEST.json`. + +- [ ] **Step 6: Add clean-room workflow and commit** + +The workflow reconstructs the checksum-pinned v5 source, applies the diagnostics patch, runs tests, runs research, validates that every candidate produces two attributed legs, archives results, and uploads them. + +Commit: `feat: attribute every candidate leg execution failure` + +--- + +### Task 2: Validation-only liquidity sizing policy + +**Files:** +- Create: `research/digital_surface/liquidity.py` +- Create: `tests/digital_surface/test_liquidity.py` +- Modify: `research/digital_surface/execution.py` +- Modify: `research/digital_surface/validation.py` + +**Interfaces:** +- Produces: `LiquidityPolicy(size_tiers: tuple[float, ...], min_joint_fill_probability: float)` and `fit_liquidity_policy(validation_candidates, validation_executions) -> LiquidityPolicy`. +- `replay_atomic_fok` accepts either scalar `shares` or a candidate column `requested_shares`. + +- [ ] **Step 1: Write failing tests** + +Test that size is selected only from supplied validation rows, never from test rows, and that lower size can convert a displayed-depth or tape failure into an atomic fill without changing timestamps or prices. + +- [ ] **Step 2: Verify RED** + +Run: `PYTHONPATH=. pytest tests/digital_surface/test_liquidity.py -v` +Expected: FAIL because the policy module and candidate-sized replay do not exist. + +- [ ] **Step 3: Implement minimal deterministic policy** + +Candidate size tiers are exactly `(1.0, 2.0, 3.0, 5.0)`. Estimate joint empirical fill probability by causal feature bins from validation data. Select the largest tier with Wilson 95% lower bound at least `0.80`; otherwise reject the candidate. + +- [ ] **Step 4: Verify focused and full suites** + +Run focused tests, then the complete suite. Do not inspect held-out fold outcomes during policy selection. + +- [ ] **Step 5: Commit** + +Commit: `feat: fit validation-only liquidity size tiers` + +--- + +### Task 3: Candidate selection that optimizes completed portfolios, not raw edge + +**Files:** +- Modify: `research/digital_surface/candidates.py` +- Modify: `research/digital_surface/validation.py` +- Test: `tests/digital_surface/test_candidates.py` +- Test: `tests/digital_surface/test_validation.py` + +**Interfaces:** +- Add candidate columns `joint_fill_score`, `net_edge_after_orphan_risk`, and `requested_shares`. +- `select_one_candidate_per_expiry` ranks first causal trigger satisfying the frozen liquidity policy; it never chooses a later timestamp based on realized fill. + +- [ ] **Step 1: Write a failing causal-selection regression** + +Construct two timestamps for one expiry where the early candidate fails the frozen policy and the later candidate passes it. Assert the later candidate is selected. Construct a second case where both pass and assert the first timestamp remains selected. + +- [ ] **Step 2: Verify RED** + +Run the focused candidate test and confirm the missing-policy behavior causes the failure. + +- [ ] **Step 3: Implement minimal policy-aware filtering** + +Apply only frozen, pre-test liquidity estimates. Do not use realized test execution fields. + +- [ ] **Step 4: Run all tests and commit** + +Commit: `feat: select first liquidity-qualified surface portfolio` + +--- + +### Task 4: Contamination guard and prospective fold ledger + +**Files:** +- Create: `research/digital_surface/prospective.py` +- Create: `tests/digital_surface/test_prospective.py` +- Modify: `research/digital_surface/run_research.py` + +**Interfaces:** +- Produces `ProspectiveLedger` with immutable market IDs, first-seen timestamps, policy hash, source hashes, and result state. +- Refuses to classify any week whose market IDs appeared in prior diagnostics or policy fitting. + +- [ ] **Step 1: Write failing contamination tests** + +Test that a market ID present in `diagnostic_market_ids` causes `ProspectiveContaminationError`, while a genuinely new market is accepted. + +- [ ] **Step 2: Verify RED** + +Run the focused test and confirm the missing guard is the failure. + +- [ ] **Step 3: Implement append-only ledger** + +Use canonical JSON, SHA-256 chaining, and explicit policy/source hashes. No row may be edited after observation. + +- [ ] **Step 4: Run all tests and commit** + +Commit: `feat: enforce uncontaminated prospective fold ledger` + +--- + +### Task 5: CLOB V2 shadow adapter and fail-closed risk controls + +**Files:** +- Create: `research/digital_surface/live/__init__.py` +- Create: `research/digital_surface/live/config.py` +- Create: `research/digital_surface/live/intents.py` +- Create: `research/digital_surface/live/market_stream.py` +- Create: `research/digital_surface/live/order_gateway.py` +- Create: `research/digital_surface/live/risk.py` +- Create: `research/digital_surface/live/runner.py` +- Create: `tests/digital_surface/live/test_config.py` +- Create: `tests/digital_surface/live/test_risk.py` +- Create: `tests/digital_surface/live/test_gateway.py` + +**Interfaces:** + +```python +@dataclass(frozen=True) +class TradeIntent: + condition_id_low: str + token_id_low_yes: str + condition_id_high: str + token_id_high_no: str + max_low_price: float + max_high_price: float + shares: float + decision_ts_ms: int + expires_ts_ms: int + +@dataclass(frozen=True) +class RiskDecision: + allowed: bool + reason: str +``` + +- [ ] **Step 1: Write failing default-shadow and risk-limit tests** + +Assert missing `TRADING_MODE` resolves to `shadow`; any live mode without an explicit arm token fails startup; stale books, heartbeat loss, daily-loss breach, unresolved prior orders, or orphan exposure block new intents. + +- [ ] **Step 2: Verify RED** + +Run the live tests and confirm imports or required behavior fail. + +- [ ] **Step 3: Implement config and risk kernel** + +No network call exists in the risk kernel. It is deterministic and independently tested. + +- [ ] **Step 4: Implement shadow gateway** + +The shadow gateway records the exact signed-order parameters it would submit but never accesses credentials or sends HTTP requests. + +- [ ] **Step 5: Implement authenticated gateway behind explicit arm gate** + +Use the current CLOB V2 SDK/API shape, batch up to two pre-signed FOK orders to reduce skew, reconcile every response and user-WebSocket event, and immediately cancel open orders on ambiguity. Document that batch posting is not atomic. + +- [ ] **Step 6: Run tests and commit** + +Commit: `feat: add fail-closed Polymarket shadow execution service` + +--- + +### Task 6: Shadow canary and final admission workflow + +**Files:** +- Create: `.github/workflows/digital-surface-shadow-canary.yml` +- Create: `research/digital_surface/live/report.py` +- Create: `tests/digital_surface/live/test_report.py` + +**Interfaces:** +- Produces a signed/hash-chained shadow report with market count, qualified intents, simulated paired fills, orphan rate, feed outages, order-state ambiguities, and risk-gate rejections. + +- [ ] **Step 1: Write failing report-gate tests** + +Assert admission fails below 500 observed markets, below 100 qualified portfolios, with any unresolved order state, above 5% orphan rate, or with fewer than four untouched weekly passes. + +- [ ] **Step 2: Verify RED** + +Run focused tests and confirm report-gate behavior is absent. + +- [ ] **Step 3: Implement report and workflow** + +The workflow runs on a schedule in shadow mode only, persists append-only artifacts, and never has access to live credentials. + +- [ ] **Step 4: Run full suite and commit** + +Commit: `ci: add prospective shadow canary admission gate` + +--- + +### Task 7: Verification and authoritative promotion + +**Files:** +- Modify: PR description and handoff documentation. +- No production behavior changes. + +- [ ] **Step 1: Run exact source reconstruction and full suite** + +Expected: all tests pass and collected count matches the manifest. + +- [ ] **Step 2: Verify prospective artifacts** + +Verify outer artifact digest, deterministic TAR sidecars, every internal SHA-256 entry, source hashes, policy hash, chronological fold identity, and live-mode-disabled evidence. + +- [ ] **Step 3: Promote atomically** + +Create one authoritative Git tree and commit containing the exact verified source. Fast-forward the isolated authoritative branch once and read critical blobs back. + +- [ ] **Step 4: Final classification** + +Classify `Rejected`, `Exploratory`, or `Admissible` strictly from the unchanged gates. Real-money mode remains disabled unless `Admissible` and the operator explicitly arms it. + +Commit: `chore: promote verified digital-surface live-readiness checkpoint` diff --git a/docs/superpowers/specs/2026-07-25-digital-surface-live-readiness-design.md b/docs/superpowers/specs/2026-07-25-digital-surface-live-readiness-design.md new file mode 100644 index 00000000..167d0537 --- /dev/null +++ b/docs/superpowers/specs/2026-07-25-digital-surface-live-readiness-design.md @@ -0,0 +1,94 @@ +# Digital Surface Live-Readiness Design + +## Objective + +Turn the verified BTC 5m-versus-15m surface-arbitrage research snapshot into a recoverable, production-shaped system without claiming profitability from contaminated holdouts or enabling real-money orders before prospective evidence exists. + +## Non-negotiable research integrity + +1. The four Kinzik weeks already examined are diagnostic data and cannot be reused as untouched acceptance folds. +2. Strategy, threshold, sizing, and execution-policy selection may use only chronologically prior data. +3. Final admission requires four newly reserved untouched weeks or four prospective weekly shadow folds. +4. No admission rule may be relaxed merely to make a fold pass. +5. Every source, test, workflow, diagnostic, and result checkpoint is committed to GitHub before the next experiment. + +## Workstreams + +### 1. Execution bottleneck diagnosis + +Produce deterministic candidate-to-fill attribution for each leg: + +- missing post-arrival book; +- ask above limit; +- displayed depth below requested shares; +- post-arrival taker tape below requested shares; +- atomic fill; +- orphaned low-YES leg; +- orphaned high-NO leg. + +Break results down by week, time-to-expiry, decision cadence, expected cost, edge, size, and leg orientation. + +### 2. Validation-only strategy redesign + +Evaluate changes only on chronologically prior training/validation blocks. Candidate approaches are: + +- liquidity-aware prequalification using causal book/tape features; +- smaller size tiers selected from validation liquidity; +- maker-first coordinated execution with bounded orphan inventory; +- multiple same-expiry strike pairs rather than only the first qualifying pair; +- stronger calibration and interval-probability uncertainty penalties. + +The selected policy must be frozen before any new test week. + +### 3. Production-shaped shadow bot + +Implement a Polymarket CLOB V2 adapter with: + +- public market WebSocket ingestion; +- authenticated user WebSocket reconciliation; +- signed FOK or explicitly bounded maker orders; +- heartbeat fail-safe; +- cancel-all kill switch; +- idempotent order state machine; +- balance, allowance, tick-size, fee-rate, and market-status checks; +- maximum daily loss and maximum orphan exposure limits; +- shadow mode as the default and only CI-enabled mode. + +There is no claim of atomic multi-market execution. Batch submission reduces timing skew but does not make two independent CLOB orders atomic. + +## Acceptance gates + +### Historical diagnostic gate + +- Existing 29 tests remain green. +- Execution attribution covers 100% of candidate legs. +- No future book or trade print is used. +- Any redesigned policy is selected without consulting its test week. + +### Prospective admission gate + +Across four newly untouched weekly folds: + +- all four weekly gates pass; +- at least 40 atomic or economically equivalent completed portfolios overall; +- at least 10 completed portfolios in every week; +- aggregate frequency at least 0.50 portfolios/hour; +- weakest-week frequency at least 0.20 portfolios/hour; +- orphan rate at most 5%; +- no single settlement timestamp contributes more than 20% of positive P&L; +- positive net growth after fees and one- and two-cent adverse slippage; +- bankroll and drawdown limits remain satisfied. + +### Live-arm gate + +Real-money order submission remains disabled until: + +- prospective admission passes; +- at least 500 shadow markets have been observed; +- at least 100 shadow-qualified portfolios have complete order-state evidence; +- credentials are supplied through secrets, never committed; +- the operator explicitly changes `TRADING_MODE=shadow` to `TRADING_MODE=live` after reviewing the final evidence artifact. + +## Failure behavior + +Any stale market feed, authentication error, heartbeat failure, unresolved order state, balance discrepancy, source-integrity failure, or risk-limit breach immediately disables new orders and cancels open orders. Existing positions are surfaced for manual resolution; the bot never hides or automatically doubles down on orphan exposure.