Unify bets playback, market parity, and keeper recovery

.github/actions/setup-hyperbet/action.yml

@@ -75,6 +75,19 @@ runs:
       if: ${{ inputs.install-foundry == 'true' }}
       uses: foundry-rs/foundry-toolchain@v1
 
+    - name: Initialize EVM submodules
+      if: ${{ inputs.install-foundry == 'true' }}
+      shell: bash
+      run: |
+        set -euo pipefail
+        if [[ ! -f .gitmodules ]]; then
+          exit 0
+        fi
+        if [[ -f packages/evm-contracts/lib/solady/src/utils/FixedPointMathLib.sol ]]; then
+          exit 0
+        fi
+        git submodule update --init --recursive packages/evm-contracts/lib/solady
+
     - name: Cache Solana CLI
       if: ${{ inputs.install-solana == 'true' }}
       uses: actions/cache@v5.0.4

.github/workflows/ci.yml

@@ -223,29 +223,41 @@ jobs:
         include:
           - package: hyperbet-solana
             workspace_target: hyperbet-solana
-            audit_target: pages:solana
+            audit_target: pages:legacy
             build_mode: mainnet-beta
             solana_cluster: mainnet-beta
+            keeper_url_var: HYPERBET_SOLANA_KEEPER_URL
+            keeper_ws_url_var: HYPERBET_SOLANA_KEEPER_WS_URL
+            keeper_fallback_url: "https://api.hyperbet.win"
+            keeper_fallback_ws_url: "wss://api.hyperbet.win/ws"
             avax_chain_id: ""
             avax_gold_clob_address: ""
           - package: hyperbet-bsc
             workspace_target: hyperbet-bsc
-            audit_target: pages:bsc
+            audit_target: pages:legacy
             build_mode: mainnet-beta
             solana_cluster: mainnet-beta
+            keeper_url_var: HYPERBET_BSC_KEEPER_URL
+            keeper_ws_url_var: HYPERBET_BSC_KEEPER_WS_URL
+            keeper_fallback_url: "https://bsc-api.hyperbet.win"
+            keeper_fallback_ws_url: "wss://bsc-api.hyperbet.win/ws"
             avax_chain_id: ""
             avax_gold_clob_address: ""
           - package: hyperbet-avax
             workspace_target: hyperbet-avax
             audit_target: app:avax
             build_mode: testnet
             solana_cluster: testnet
+            keeper_url_var: HYPERBET_AVAX_KEEPER_URL
+            keeper_ws_url_var: HYPERBET_AVAX_KEEPER_WS_URL
+            keeper_fallback_url: "https://avax-api.hyperbet.win"
+            keeper_fallback_ws_url: "wss://avax-api.hyperbet.win/ws"
             avax_chain_id: ""
             avax_gold_clob_address: ""
     env:
       CF_PAGES_COMMIT_SHA: ${{ github.sha }}
-      VITE_GAME_API_URL: https://api.hyperbet.win
-      VITE_GAME_WS_URL: wss://api.hyperbet.win/ws
+      VITE_GAME_API_URL: ${{ vars[matrix.keeper_url_var] || matrix.keeper_fallback_url }}
+      VITE_GAME_WS_URL: ${{ vars[matrix.keeper_ws_url_var] || matrix.keeper_fallback_ws_url }}
       VITE_SOLANA_CLUSTER: ${{ matrix.solana_cluster }}
       VITE_USE_GAME_RPC_PROXY: "true"
       VITE_USE_GAME_EVM_RPC_PROXY: "true"
@@ -335,8 +347,8 @@ jobs:
     needs: shared-validation
     env:
       CF_PAGES_COMMIT_SHA: ${{ github.sha }}
-      VITE_GAME_API_URL: https://api.hyperbet.win
-      VITE_GAME_WS_URL: wss://api.hyperbet.win/ws
+      VITE_GAME_API_URL: ${{ vars.HYPERBET_BSC_KEEPER_URL || 'https://bsc-api.hyperbet.win' }}
+      VITE_GAME_WS_URL: ${{ vars.HYPERBET_BSC_KEEPER_WS_URL || 'wss://bsc-api.hyperbet.win/ws' }}
       VITE_SOLANA_CLUSTER: mainnet-beta
       VITE_USE_GAME_RPC_PROXY: "true"
       VITE_USE_GAME_EVM_RPC_PROXY: "true"

.github/workflows/deploy-avax-pages.yml

@@ -1,17 +1,6 @@
 name: Deploy Hyperbet AVAX Pages
 
 on:
-  push:
-    branches: [main]
-    paths:
-      - "packages/hyperbet-avax/app/**"
-      - "packages/hyperbet-ui/**"
-      - "scripts/**"
-      - ".github/actions/setup-hyperbet/**"
-      - ".github/workflows/deploy-avax-pages.yml"
-      - "docs/hyperbet-production-deploy.md"
-      - "package.json"
-      - "bun.lock"
   workflow_dispatch:
     inputs:
       environment:
@@ -154,7 +143,7 @@ jobs:
           if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ inputs.environment }}" = "preview" ]; then
             BRANCH="preview-${{ github.run_id }}"
           elif [ "${DEPLOY_TARGET}" = "staging" ]; then
-            BRANCH="staging"
+            BRANCH="enoomian/staging"
           else
             BRANCH="main"
           fi

.github/workflows/deploy-bsc-pages.yml

@@ -1,18 +1,6 @@
-name: Deploy Hyperbet BSC Pages
+name: Rollback Legacy Hyperbet BSC Pages
 
 on:
-  push:
-    branches: [main]
-    paths:
-      - "packages/hyperbet-bsc/app/**"
-      - "packages/hyperbet-ui/**"
-      - "packages/hyperbet-chain-registry/**"
-      - "scripts/**"
-      - ".github/actions/setup-hyperbet/**"
-      - ".github/workflows/deploy-bsc-pages.yml"
-      - "docs/hyperbet-production-deploy.md"
-      - "package.json"
-      - "bun.lock"
   workflow_dispatch:
     inputs:
       environment:
@@ -24,6 +12,11 @@ on:
           - production
           - staging
           - preview
+      rollback_only:
+        description: "Confirm this legacy workflow is being used for rollback only"
+        required: true
+        default: false
+        type: boolean
 
 permissions:
   contents: read
@@ -34,7 +27,7 @@ concurrency:
 
 jobs:
   deploy:
-    if: ${{ github.event_name == 'workflow_dispatch' || vars.HYPERBET_BSC_PAGES_DEPLOY_ENABLED == 'true' || (vars.HYPERBET_BSC_PAGES_DEPLOY_ENABLED == '' && vars.HYPERBET_BSC_DEPLOY_ENABLED == 'true') }}
+    if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.rollback_only == 'true' }}
     name: Deploy hyperbet-bsc pages
     runs-on: ubuntu-latest
     environment: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.environment == 'staging' && 'staging' || 'production' }}
@@ -120,7 +113,7 @@ jobs:
           bash scripts/ci-install-verified.sh root hyperbet-bsc-app
 
       - name: Audit deploy env
-        run: node --import tsx scripts/ci-env-audit.ts --target=pages:bsc --deployment=${DEPLOY_TARGET}
+        run: node --import tsx scripts/ci-env-audit.ts --target=pages:legacy --deployment=${DEPLOY_TARGET}
         env:
           VITE_GAME_API_URL: ${{ env.HYPERBET_API_URL }}
           VITE_GAME_WS_URL: ${{ env.HYPERBET_WS_URL }}
@@ -164,7 +157,7 @@ jobs:
           if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ inputs.environment }}" = "preview" ]; then
             BRANCH="preview-${{ github.run_id }}"
           elif [ "${DEPLOY_TARGET}" = "staging" ]; then
-            BRANCH="staging"
+            BRANCH="enoomian/staging"
           else
             BRANCH="main"
           fi

.github/workflows/deploy-hyperbet-keeper.yml

@@ -0,0 +1,233 @@
+name: Deploy Hyperbet Keeper
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "packages/hyperbet-evm/keeper/**"
+      - "packages/hyperbet-evm/deployments/**"
+      - "packages/hyperbet-chain-registry/**"
+      - "packages/hyperbet-mm-core/**"
+      - "packages/hyperbet-evm-keeper-core/**"
+      - "scripts/**"
+      - ".github/actions/setup-hyperbet/**"
+      - ".github/workflows/deploy-hyperbet-keeper.yml"
+      - "docs/hyperbet-production-deploy.md"
+      - "package.json"
+      - "bun.lock"
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: "Deployment target"
+        required: true
+        default: production
+        type: choice
+        options:
+          - production
+          - staging
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  deploy:
+    name: Deploy hyperbet keeper
+    runs-on: ubuntu-latest
+    environment: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.environment == 'staging' && 'enoomian-staging' || 'production' }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6.0.2
+        with:
+          fetch-depth: 0
+
+      - name: Resolve deployment target
+        run: |
+          deploy_target="production"
+          pages_url="${{ vars.HYPERBET_PAGES_URL }}"
+          keeper_url="${{ vars.HYPERBET_KEEPER_URL }}"
+          keeper_ws_url="${{ vars.HYPERBET_KEEPER_WS_URL }}"
+          railway_project_id="${{ vars.HYPERBET_RAILWAY_PROJECT_ID }}"
+          railway_environment_id="${{ vars.HYPERBET_RAILWAY_PRODUCTION_ENVIRONMENT_ID }}"
+          railway_service_id="${{ vars.HYPERBET_RAILWAY_KEEPER_SERVICE_ID }}"
+          enabled_evm_chains="bsc,avax"
+          avax_rpc_url="https://api.avax.network/ext/bc/C/rpc"
+          avax_gold_clob_address="${{ vars.HYPERBET_AVAX_GOLD_CLOB_ADDRESS }}"
+          keeper_status_filter='.ok == true and .proxies.solanaRpc == true and .proxies.bscRpc == true and .proxies.avaxRpc == true'
+
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ inputs.environment }}" = "staging" ]; then
+            deploy_target="staging"
+            pages_url="${{ vars.ENOOMIAN_HYPERBET_PAGES_URL }}"
+            keeper_url="${{ vars.ENOOMIAN_HYPERBET_KEEPER_URL }}"
+            keeper_ws_url="${{ vars.ENOOMIAN_HYPERBET_KEEPER_WS_URL }}"
+            railway_project_id="${{ vars.ENOOMIAN_RAILWAY_PROJECT_ID }}"
+            railway_environment_id="${{ vars.ENOOMIAN_RAILWAY_ENVIRONMENT_ID }}"
+            railway_service_id="${{ vars.ENOOMIAN_HYPERBET_KEEPER_SERVICE_ID }}"
+            enabled_evm_chains="bsc"
+            avax_rpc_url=""
+            avax_gold_clob_address=""
+            keeper_status_filter='.ok == true and .proxies.solanaRpc == true and .proxies.bscRpc == true'
+          fi
+
+          if [ -z "${pages_url}" ] || [ -z "${keeper_url}" ] || [ -z "${keeper_ws_url}" ] || [ -z "${railway_project_id}" ] || [ -z "${railway_environment_id}" ] || [ -z "${railway_service_id}" ]; then
+            echo "Missing Hyperbet keeper configuration for ${deploy_target}" >&2
+            exit 1
+          fi
+
+          echo "DEPLOY_TARGET=${deploy_target}" >> "${GITHUB_ENV}"
+          echo "HYPERBET_PAGES_URL=${pages_url}" >> "${GITHUB_ENV}"
+          echo "HYPERBET_KEEPER_URL=${keeper_url}" >> "${GITHUB_ENV}"
+          echo "HYPERBET_KEEPER_WS_URL=${keeper_ws_url}" >> "${GITHUB_ENV}"
+          echo "RAILWAY_PROJECT_ID=${railway_project_id}" >> "${GITHUB_ENV}"
+          echo "RAILWAY_ENVIRONMENT_ID=${railway_environment_id}" >> "${GITHUB_ENV}"
+          echo "RAILWAY_KEEPER_SERVICE_ID=${railway_service_id}" >> "${GITHUB_ENV}"
+          echo "HYPERBET_ENABLED_EVM_CHAINS=${enabled_evm_chains}" >> "${GITHUB_ENV}"
+          echo "HYPERBET_AVAX_RPC_URL=${avax_rpc_url}" >> "${GITHUB_ENV}"
+          echo "HYPERBET_AVAX_GOLD_CLOB_ADDRESS=${avax_gold_clob_address}" >> "${GITHUB_ENV}"
+          echo "KEEPER_STATUS_FILTER=${keeper_status_filter}" >> "${GITHUB_ENV}"
+
+      - name: Setup Hyperbet toolchain
+        uses: ./.github/actions/setup-hyperbet
+
+      - name: Detect keeper source changes
+        id: keeper_changes
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            echo "Manual dispatch requested; forcing keeper deploy."
+            echo "keeper_source_changed=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          base_ref="${{ github.event.before }}"
+          if [ -z "$base_ref" ] || [ "$base_ref" = "0000000000000000000000000000000000000000" ] || ! git cat-file -e "${base_ref}^{commit}" 2>/dev/null; then
+            base_ref="$(git rev-parse HEAD^ 2>/dev/null || true)"
+          fi
+
+          if [ -n "$base_ref" ]; then
+            changed_files="$(git diff --name-only "$base_ref" "${{ github.sha }}")"
+          else
+            changed_files="$(git show --pretty='' --name-only "${{ github.sha }}")"
+          fi
+          printf '%s\n' "$changed_files"
+
+          keeper_change_pattern='^(packages/hyperbet-evm/keeper/|packages/hyperbet-evm/deployments/|packages/hyperbet-chain-registry/|packages/hyperbet-mm-core/|packages/hyperbet-evm-keeper-core/|scripts/|\.github/actions/setup-hyperbet/)'
+          keeper_source_changed=false
+          if printf '%s\n' "$changed_files" | grep -Eq "${keeper_change_pattern}"; then
+            keeper_source_changed=true
+          fi
+
+          echo "keeper_source_changed=${keeper_source_changed}" >> "$GITHUB_OUTPUT"
+
+      - name: Install dependencies
+        run: bash scripts/ci-install-verified.sh root hyperbet-evm-keeper
+
+      - name: Audit deploy env
+        run: node --import tsx scripts/ci-env-audit.ts --target=keeper:unified --deployment=${DEPLOY_TARGET}
+        env:
+          ENOOMIAN_HYPERBET_PAGES_URL: ${{ env.HYPERBET_PAGES_URL }}
+          ENOOMIAN_HYPERBET_KEEPER_URL: ${{ env.HYPERBET_KEEPER_URL }}
+          ENOOMIAN_HYPERBET_KEEPER_WS_URL: ${{ env.HYPERBET_KEEPER_WS_URL }}
+          CI_AUDIT_REQUIRE_RUNTIME: "true"
+          HYPERBET_KEEPER_URL: ${{ env.HYPERBET_KEEPER_URL }}
+          RAILWAY_PROJECT_ID: ${{ env.RAILWAY_PROJECT_ID }}
+          RAILWAY_ENVIRONMENT_ID: ${{ env.RAILWAY_ENVIRONMENT_ID }}
+          RAILWAY_KEEPER_SERVICE_ID: ${{ env.RAILWAY_KEEPER_SERVICE_ID }}
+          SOLANA_RPC_URL: https://api.mainnet-beta.solana.com
+          BSC_RPC_URL: https://bsc-dataseed.binance.org
+          BSC_GOLD_CLOB_ADDRESS: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.environment == 'staging' && (vars.ENOOMIAN_BSC_GOLD_CLOB_ADDRESS != '' && vars.ENOOMIAN_BSC_GOLD_CLOB_ADDRESS || vars.HYPERBET_BSC_STAGING_GOLD_CLOB_ADDRESS) || vars.HYPERBET_BSC_GOLD_CLOB_ADDRESS }}
+          AVAX_RPC_URL: ${{ env.HYPERBET_AVAX_RPC_URL }}
+          AVAX_GOLD_CLOB_ADDRESS: ${{ env.HYPERBET_AVAX_GOLD_CLOB_ADDRESS }}
+          EVM_KEEPER_CHAINS: ${{ env.HYPERBET_ENABLED_EVM_CHAINS }}
+
+      - name: Typecheck keeper
+        run: bunx tsc --noEmit -p packages/hyperbet-evm/keeper/tsconfig.json
+
+      - name: Smoke test keeper service
+        run: |
+          PORT=18080 \
+          ENABLE_KEEPER_BOT=false \
+          EVM_KEEPER_CHAINS="${HYPERBET_ENABLED_EVM_CHAINS}" \
+          SOLANA_RPC_URL=https://api.mainnet-beta.solana.com \
+          BSC_RPC_URL=https://bsc-dataseed.binance.org \
+          AVAX_RPC_URL="${HYPERBET_AVAX_RPC_URL}" \
+          BSC_GOLD_CLOB_ADDRESS=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.environment == 'staging' && (vars.ENOOMIAN_BSC_GOLD_CLOB_ADDRESS != '' && vars.ENOOMIAN_BSC_GOLD_CLOB_ADDRESS || vars.HYPERBET_BSC_STAGING_GOLD_CLOB_ADDRESS) || vars.HYPERBET_BSC_GOLD_CLOB_ADDRESS }} \
+          AVAX_GOLD_CLOB_ADDRESS="${HYPERBET_AVAX_GOLD_CLOB_ADDRESS}" \
+          bun --bun packages/hyperbet-evm/keeper/src/service.ts >/tmp/hyperbet-keeper.log 2>&1 &
+          KEEPER_PID=$!
+          trap 'kill "$KEEPER_PID" >/dev/null 2>&1 || true' EXIT
+
+          for attempt in $(seq 1 20); do
+            if curl -fsSL http://127.0.0.1:18080/status | jq -e '.ok == true and .proxies != null' >/dev/null; then
+              exit 0
+            fi
+            sleep 1
+          done
+
+          cat /tmp/hyperbet-keeper.log
+          exit 1
+
+      - name: Install Railway CLI
+        if: steps.keeper_changes.outputs.keeper_source_changed == 'true'
+        run: bun add -g @railway/cli
+
+      - name: Write Railway keeper context
+        if: steps.keeper_changes.outputs.keeper_source_changed == 'true'
+        working-directory: packages/hyperbet-evm/keeper
+        run: |
+          keeper_path="$(pwd)"
+          mkdir -p "${HOME}/.railway"
+          cat > "${HOME}/.railway/config.json" <<EOF
+          {
+            "projects": {
+              "${keeper_path}": {
+                "projectPath": "${keeper_path}",
+                "name": "hyperbet",
+                "project": "${RAILWAY_PROJECT_ID}",
+                "environment": "${RAILWAY_ENVIRONMENT_ID}",
+                "environmentName": "${DEPLOY_TARGET}",
+                "service": "${RAILWAY_KEEPER_SERVICE_ID}"
+              }
+            },
+            "user": {
+              "token": "${RAILWAY_TOKEN}"
+            }
+          }
+          EOF
+        env:
+          RAILWAY_TOKEN: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.environment == 'staging' && secrets.ENOOMIAN_RAILWAY_TOKEN || secrets.RAILWAY_TOKEN }}
+
+      - name: Stage keeper workspace context
+        if: steps.keeper_changes.outputs.keeper_source_changed == 'true'
+        run: node --import tsx scripts/stage-deploy-workspace.ts --target=keeper:shared
+
+      - name: Deploy keeper
+        if: steps.keeper_changes.outputs.keeper_source_changed == 'true'
+        working-directory: packages/hyperbet-evm/keeper
+        run: |
+          railway up . \
+            --path-as-root \
+            --detach \
+            --service "${RAILWAY_KEEPER_SERVICE_ID}" \
+            --environment "${RAILWAY_ENVIRONMENT_ID}"
+        env:
+          RAILWAY_TOKEN: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.environment == 'staging' && secrets.ENOOMIAN_RAILWAY_TOKEN || secrets.RAILWAY_TOKEN }}
+          CI: "true"
+
+      - name: Verify keeper endpoints
+        if: steps.keeper_changes.outputs.keeper_source_changed == 'true'
+        run: |
+          for attempt in $(seq 1 20); do
+            status_payload="$(curl -fsSL "${HYPERBET_KEEPER_URL}/status" || true)"
+            if [ -n "$status_payload" ] && echo "$status_payload" | jq -e "${KEEPER_STATUS_FILTER}" >/dev/null; then
+              break
+            fi
+            sleep 15
+          done
+
+          curl -fsSL "${HYPERBET_KEEPER_URL}/status" | jq -e '.ok == true'
+          curl -fsSL "${HYPERBET_KEEPER_URL}/api/arena/prediction-markets/active" | jq -e '.duel != null and .markets != null'
+          curl -fsSL "${HYPERBET_KEEPER_URL}/api/arena/prediction-markets/overview" | jq -e '.duel != null and .markets != null'
+          curl -fsSL "${HYPERBET_KEEPER_URL}/api/keeper/bot-health" | jq -e '.ok == true and .markets != null'