diff --git a/.github/workflows/prediction-market-gates.yml b/.github/workflows/prediction-market-gates.yml index cbcb272a..9509909f 100644 --- a/.github/workflows/prediction-market-gates.yml +++ b/.github/workflows/prediction-market-gates.yml @@ -258,7 +258,14 @@ jobs: run: bun run dev:bootstrap - name: Install Playwright browser - run: bunx playwright install --with-deps chromium + # Use the workspace-pinned Playwright (`@playwright/test` in each + # app's package.json) rather than `bunx playwright install`, which + # resolves the latest CLI from npm and downloads a newer browser + # bundle (e.g. v1217) than the pinned test runner (v1208) expects. + # That version skew is what caused the Apr-23 Cross-Chain E2E + # regression after npm republished the Playwright CLI. + working-directory: packages/hyperbet-${{ matrix.chain }}/app + run: ./node_modules/.bin/playwright install --with-deps chromium - name: AVAX gate posture note if: matrix.chain == 'avax' diff --git a/docs/runbooks/README.md b/docs/runbooks/README.md index 5bf7d4a5..0e0934ff 100644 --- a/docs/runbooks/README.md +++ b/docs/runbooks/README.md @@ -8,6 +8,7 @@ Primary runbooks: - [PM Confidence Soak](pm-confidence-soak.md) - [Staged Live Proof](staged-live-proof.md) - [Hyperscapes Local PM Integration](hyperscapes-local-pm-integration.md) +- [Enoomian AWS GPU Migration](enoomian-aws-gpu-migration.md) - [Prediction-Market Governance And Emergency Controls](prediction-market-governance-and-emergency-controls.md) - [Signer Policy And Key Rotation](signer-policy-and-key-rotation.md) - [Quote Disablement And Safe Restart](quote-disablement-and-safe-restart.md) @@ -17,3 +18,4 @@ Primary runbooks: - [Claim Backlog Drainage](claim-backlog-drainage.md) - [Oracle Finality Fail-Closed](oracle-finality-fail-closed.md) - [AVAX Fuji Bootstrap Market Smoke](avax-fuji-bootstrap.md) +- [Enoomian Staging Stream Authority](enoomian-staging-stream-authority.md) diff --git a/docs/runbooks/enoomian-aws-gpu-migration.md b/docs/runbooks/enoomian-aws-gpu-migration.md new file mode 100644 index 00000000..fde6943a --- /dev/null +++ b/docs/runbooks/enoomian-aws-gpu-migration.md @@ -0,0 +1,350 @@ +# Enoomian AWS GPU Migration Runbook + +This runbook is intentionally scoped to `enoomian/staging`. + +The goal is to move the Hyperscapes GPU capture host from Hetzner to AWS without +regressing the staging stream, bettor playback, or keeper readiness surfaces. Do +not treat this as a production deploy-path replacement until enoomian staging has +proven it under soak. + +## Why This Is Separate + +The current enoomian staging lane contains deployment topology and operational +choices that are not yet accepted as the canonical production standard. This +runbook documents the version that works for enoomian staging so the team can +test it end-to-end before deciding what belongs in the permanent production +runbooks and PRs. + +This is not meant to fork the architecture forever. It is a staging proof lane: + +- prove the AWS GPU host can render, encode, and publish the same feed as the + current Hetzner host +- preserve the Cloudflare Stream playback contract when Cloudflare ingest is + healthy, but allow an explicit enoomian-only self-HLS fallback during AWS + migration proof +- preserve the enoomian public authority path and status APIs +- avoid changing GitHub production or generic staging deploy paths during proof +- only promote the working pieces into the effective PRs after soak + +## Current Working Contract + +Before migration, capture is expected to work like this: + +- Hyperscapes source worker runs on the dedicated GPU host. +- The source worker renders the stream page in a GPU-capable browser. +- Capture mode is `x11_nvenc` when the host supports NVIDIA NVENC correctly. +- FFmpeg publishes RTMPS to the configured Cloudflare Stream live input. +- Cloudflare Stream distributes the public LL-HLS playback used by Hyperbet. +- Enoomian staging status and readiness endpoints must report the real source + worker state, not a separate Railway-only view. + +Do not restart or replace the healthy live rail during a migration test unless +the migration window explicitly calls for cutover. + +## Migration Principle + +AWS should be introduced as a replacement GPU source host, not as a broad +replacement for Cloudflare distribution. + +The desired result is: + +- AWS renders and encodes the same Hyperscapes source view. +- AWS publishes to the same canonical Cloudflare Stream lane or a deliberate + canary lane when Cloudflare ingest is healthy. +- If Cloudflare Stream ingest rejects AWS publishers, AWS may temporarily expose + self-HLS for enoomian staging only while Cloudflare ingest is investigated. +- Hyperbet consumes the canonical playback URL stamped by enoomian staging, + either Cloudflare LL-HLS or the explicit AWS self-HLS fallback. +- Public enoomian staging APIs continue to expose one truthful authority path. +- Rollback is immediate: stop AWS publisher, restore the previous enoomian + playback/API env, and redeploy the enoomian staging surfaces. + +## Deployment Targets + +The enoomian staging deploy script now owns the AWS GPU migration path. Use these +targets from the Hyperbet repo root: + +```bash +scripts/enoomian-staging/deploy.sh hyperscapes-aws-gpu-plan +scripts/enoomian-staging/deploy.sh hyperscapes-aws-gpu-preflight +scripts/enoomian-staging/deploy.sh hyperscapes-aws-gpu-deploy +scripts/enoomian-staging/deploy.sh hyperscapes-aws-gpu-status +``` + +The deploy target prepares the AWS GPU host and starts the Hyperscapes duel API. +It does not redeploy Cloudflare Pages, change Hyperbet playback selection, move +Railway services, or start the AWS stream publisher by default. + +The AWS stream publisher is opt-in: + +```bash +ENOOMIAN_AWS_GPU_START_SOURCE=1 \ +scripts/enoomian-staging/deploy.sh hyperscapes-aws-gpu-deploy +``` + +Only use `ENOOMIAN_AWS_GPU_START_SOURCE=1` for an intentional canary live input +or a scheduled canonical cutover. Do not run it against the canonical Cloudflare +input while the Hetzner source is still connected. + +Public authority cutover is a separate explicit step: + +```bash +ENOOMIAN_AWS_GPU_ACTIVATE_PUBLIC_AUTHORITY=1 \ +scripts/enoomian-staging/deploy.sh hyperscapes-aws-gpu-activate +``` + +Activation requires `ENOOMIAN_AWS_GPU_PUBLIC_API_URL` and +`ENOOMIAN_AWS_GPU_PUBLIC_WS_URL`. It updates only enoomian staging authority +inputs and then redeploys the staging surfaces that consume those inputs. When +`ENOOMIAN_AWS_GPU_STREAM_DELIVERY_MODE=self_hls`, activation also stamps +`ENOOMIAN_STREAM_PLAYBACK_URL` from `ENOOMIAN_AWS_GPU_PUBLIC_HLS_URL` so +keepers and Pages do not keep using a stale Cloudflare playback URL. + +By default the AWS activation target does not sync Hyperscapes Railway variables. +That is intentional for this proof lane: Hyperscapes public authority is AWS + +Pages. Hyperbet is different: the public app mounts the canonical stream session +published by the keeper, so activation must reconcile the Hyperbet keeper env and +redeploy the unified keeper by default. Otherwise the Pages bundle can contain +the AWS HLS URL while the live canonical session still advertises stale +Cloudflare playback. + +Optional overrides: + +- `ENOOMIAN_AWS_GPU_SYNC_HYPERSCAPES_RAILWAY_ENV=1` also reconciles the + Hyperscapes Railway env during activation. +- `ENOOMIAN_AWS_GPU_SYNC_HYPERBET_KEEPER_ENV=0` skips Hyperbet keeper env sync; + use only for Pages-only proof because the app may keep the previous canonical + session. +- `ENOOMIAN_AWS_GPU_DEPLOY_HYPERBET_KEEPER=0` skips the keeper deploy; use only + when the keeper has already been restarted with reconciled env. + +## AWS Host Requirements + +Minimum functional requirements: + +- NVIDIA GPU with working driver and NVENC support. +- Linux display stack capable of running headful Chrome/Chromium against X11. +- Browser launch flags compatible with Hyperscapes WebGPU on Linux. +- FFmpeg with `h264_nvenc` available. +- Stable outbound RTMPS to Cloudflare Stream. +- Enough CPU, memory, disk, and network headroom for continuous 720p stream + capture plus API/status processes. +- Stable process supervision for `hyperscape-stream-source` and related + services. + +Deployment env required by the AWS GPU target: + +- `ENOOMIAN_AWS_GPU_SSH_TARGET`, or both `ENOOMIAN_AWS_GPU_HOST` and optional + `ENOOMIAN_AWS_GPU_SSH_USER` +- optional `ENOOMIAN_AWS_GPU_SSH_KEY` +- optional `ENOOMIAN_AWS_GPU_SSH_PORT` +- optional `ENOOMIAN_AWS_GPU_REMOTE_ROOT`, default `/root/hyperscape` for + `root` SSH and `/home//hyperscape` for non-root SSH +- existing enoomian stream and Cloudflare env from `personal-staging.env` + +The exact AWS instance family and availability zone are not hardcoded in this +runbook. Select them from actual AWS availability during the migration window, +then record the chosen region, zone, instance type, AMI, volume size, public IP, +and security group in the staging proof notes. + +## Current AWS Proof Host + +Current enoomian staging proof host as of 2026-04-28: + +- AWS account: `364947027011` +- AWS profile: `stream-admin` +- Region: `us-east-2` +- Instance id: `i-0c212958b7b092968` +- Instance type: `g6.xlarge` +- GPU: NVIDIA L4, 24 GB class memory, NVENC available +- AMI: `Deep Learning Base OSS Nvidia Driver GPU AMI (Ubuntu 22.04) 20260421` +- Root volume: 75 GB gp3 +- Public IP: `3.129.45.146` +- Security group: `sg-0583d9b7f2735aabf` +- SSH user: `ubuntu` +- Remote root: `/home/ubuntu/hyperscape` +- Local API port: `5560` +- Source publisher default: disabled with `ENOOMIAN_AWS_GPU_START_SOURCE=0` +- Public API proof URL: `https://3.129.45.146.sslip.io` +- Public self-HLS proof URL: `https://3.129.45.146.sslip.io/live/stream.m3u8` + +Initial proof result: + +- `nvidia-smi` works and reports the L4 GPU. +- `ffmpeg -encoders` includes `h264_nvenc`. +- The enoomian AWS deploy can sync/build the Hyperscapes app on the host. +- `hyperscape-duel-api` starts successfully on the AWS host. +- `/health` returns `ok` locally on the AWS host. +- Direct Cloudflare Stream ingest from the AWS proof host failed for canonical, + canary, and fresh live inputs with `The specified session has been + invalidated for some reason`. That was reproduced with the Hyperscapes source + worker and direct FFmpeg, including a software-encode probe, so the failure is + outside the Hyperscapes renderer/NVENC path. +- AWS self-HLS is currently the working enoomian staging fallback. The stream + source writes fresh two-second segments to `/live/stream.m3u8`; the public + Caddy rail serves the manifest over HTTPS. + +## Do Not Change During Proof + +Do not change these unless the migration test specifically calls for it: + +- Cloudflare Stream account, live input, or playback policy +- Hyperbet app playback selection rules +- canonical enoomian Pages env values +- production deploy scripts +- generic staging deploy scripts +- duel lifecycle, keeper polling, or betting settlement logic +- Cloudflare cleanup jobs + +The migration is about the GPU source host first. + +## Proof Phases + +### Phase 1: Inventory + +Document the current Hetzner source host before provisioning AWS: + +- process manager and process names +- source worker command and environment +- browser path and launch flags +- capture mode +- FFmpeg version and encoders +- Caddy or reverse proxy config +- public status endpoints +- Cloudflare live input id +- playback URL used by Hyperbet +- current health/status payload samples +- rollback command + +If direct host access is unavailable, use existing local runbooks, env manifests, +deployment scripts, and public status endpoints to fill the inventory. Mark any +field as `unknown` instead of guessing. + +### Phase 2: AWS Provision + +Provision a GPU host in AWS as a canary first: + +- create or select a GPU-capable EC2 instance +- attach persistent storage sized for logs, app builds, browser cache, and HLS + scratch space if used +- install NVIDIA driver and verify `nvidia-smi` +- install Chrome/Chromium channel required for WebGPU +- install FFmpeg and verify `ffmpeg -encoders` includes `h264_nvenc` +- install Bun/Node/runtime dependencies matching the current source worker +- install process supervisor +- configure firewall/security group for only required inbound ports +- configure outbound access to Cloudflare RTMPS + +Do not point public enoomian authority to AWS yet. + +Then run: + +```bash +scripts/enoomian-staging/deploy.sh hyperscapes-aws-gpu-preflight +scripts/enoomian-staging/deploy.sh hyperscapes-aws-gpu-deploy +``` + +### Phase 3: Canary Source Worker + +Run the AWS source worker against a canary Cloudflare live input first. + +Before enabling the source publisher, confirm the env points to a canary live +input or that a canonical cutover window is active. The safe deploy default is +`ENOOMIAN_AWS_GPU_START_SOURCE=0`; change it to `1` only for this phase. + +Required checks: + +- browser starts with WebGPU available +- source worker reports ready +- capture mode is `x11_nvenc` or an explicitly approved fallback +- FFmpeg emits frames continuously +- Cloudflare canary live input shows connected +- canary playback shows visible world/fighters +- source status payload includes accurate capture mode, readiness, heartbeat, + and playback URL + +If canary cannot produce a stable feed, stop. Do not touch the canonical +Cloudflare live input or public enoomian authority path. + +Use this status check after deploy: + +```bash +scripts/enoomian-staging/deploy.sh hyperscapes-aws-gpu-status +``` + +### Phase 4: Public Authority Cutover + +Only after canary is stable: + +- decide whether AWS will publish to canonical Cloudflare directly or first + continue on canary +- update enoomian staging-only env and proxy authority to point status/control + endpoints at the AWS source host if needed +- keep Hyperbet playback on Cloudflare, not direct browser capture +- verify `/api/streaming/capture/status` reports the AWS source worker truth +- verify Hyperbet pages still play the Cloudflare stream +- verify keeper readiness changes propagate without stale same-duel state + +This phase must not modify production or generic staging deploy paths. + +Only after canary status is healthy, set the AWS public authority env values and +run the explicit activation target. + +### Phase 5: Soak + +Run soak before declaring AWS acceptable: + +- direct Cloudflare playback +- Hyperbet embed playback +- public capture/status endpoint +- keeper stream-state publication +- source worker heartbeat +- Cloudflare input connected state +- frame continuity +- recovery after AWS process restart +- recovery after browser restart +- recovery after FFmpeg restart + +Record exact timestamps, commands, status payloads, and any drift observed. + +### Phase 6: Promotion Decision + +After soak, split the proven work into effective PRs: + +- Hyperscapes PR: source-worker/runtime changes required for AWS GPU capture. +- Hyperbet PR: enoomian staging deploy/runbook changes required to operate the + AWS host and public authority lane. +- Production runbook update: only after the team agrees the staging topology is + the future-forward standard. + +Anything that is only a personal enoomian staging secret, host note, or local +proof artifact stays out of the public PR and remains documented in the local +staging manifest. + +## Acceptance Criteria + +The AWS migration is not accepted until all of these are true: + +- Cloudflare playback remains the public bettor distribution rail. +- AWS source worker can run continuously without black frames or stale readiness. +- `sourceRuntime.captureMode` is truthful. +- public capture status matches the active source worker. +- Hyperbet embed plays the expected duel feed. +- no stale canonical session can mount for the wrong duel. +- no same-duel readiness or authority update is suppressed by keeper dedupe. +- rollback to Hetzner is documented and tested. + +## Rollback + +Rollback must be simple: + +1. Stop the AWS source worker. +2. Restore the previous enoomian staging authority/status target if it changed. +3. Resume or keep the Hetzner source worker on the canonical Cloudflare input. +4. Verify Cloudflare playback. +5. Verify Hyperbet embed playback. +6. Record why AWS failed and whether the failure is host, driver, browser, + capture, RTMPS, Cloudflare, proxy, or keeper/status related. + +Do not debug by changing Hyperbet playback rules unless playback selection is +directly proven to be the failing layer. diff --git a/docs/runbooks/enoomian-staging-stream-authority.md b/docs/runbooks/enoomian-staging-stream-authority.md new file mode 100644 index 00000000..3bb815b1 --- /dev/null +++ b/docs/runbooks/enoomian-staging-stream-authority.md @@ -0,0 +1,141 @@ +# Enoomian Staging Stream Authority + +## Goal + +Protect the enoomian staging product lane from one specific class of outage: + +- Cloudflare playback is healthy +- the GPU source worker is healthy +- but Hyperbet blocks before mounting the player because keeper authority is reading the wrong upstream truth + +This runbook exists to catch and fix that class of failure before it becomes another long investigation. + +## Core Invariant + +On enoomian staging, the public authority path for: + +- `/api/streaming/*` +- `/api/internal/bet-sync/*` + +must resolve to the same upstream truth for the active stream lane. + +If those two surfaces disagree, Hyperbet can block playback even when the live stream itself is healthy. + +## Known Failure Signature + +The specific incident on 2026-04-22 presented as: + +- Cloudflare LL-HLS manifest returned `200` +- GPU-host source worker was healthy +- `capture/status` reported `sourceRuntime.ready = true` +- Hyperbet showed stream-authority blocking copy or stayed stuck before useful playback +- the player either never mounted or stayed stuck in bootstrap/catch-up behavior driven by false upstream authority + +Root cause: + +- `46.4.80.150.sslip.io` was split at the proxy layer +- `/api/streaming/*` resolved to the healthy GPU-host service +- `/api/internal/bet-sync/*` resolved elsewhere +- keeper trusted the false-negative bet-sync authority and blocked playback + +## What Matters + +Do not treat these as interchangeable: + +1. source runtime truth +2. public betting authority truth +3. keeper canonical state +4. Cloudflare media transport + +The product lane is healthy only when all four align enough for Hyperbet to mount and play the canonical Cloudflare stream. + +## Fast Audit + +Run: + +```bash +bun run enoomian:authority:audit +``` + +Optional overrides: + +```bash +bun run enoomian:authority:audit -- \ + --public-origin=https://46.4.80.150.sslip.io \ + --keeper-url=https://hyperbet-keeper-staging-production.up.railway.app \ + --timeout-ms=12000 +``` + +The audit checks: + +- public bet-sync state +- public capture/status +- keeper canonical state +- playback manifest reachability + +If direct `bet-sync/state` is not readable from the current environment and returns +`401` or `403`, the audit falls back to: + +- `capture/status` +- `keeper/state` + +That fallback is still useful because keeper is the surface that blocks or allows +Hyperbet playback. + +It fails if: + +- bet-sync `sourceRuntime` disagrees with `capture/status` +- bet-sync blocks `publicReadiness` while capture reports a healthy source +- keeper disagrees with bet-sync on readiness or playback URL +- playback URL is missing or dead while authority says ready + +## Manual Spot Check + +If the audit reports failure, inspect all of these explicitly: + +```bash +curl -fsSL https://46.4.80.150.sslip.io/api/internal/bet-sync/state | jq +curl -fsSL https://46.4.80.150.sslip.io/api/streaming/capture/status | jq +curl -fsSL https://hyperbet-keeper-staging-production.up.railway.app/api/streaming/state | jq +curl -fsSI 'https://customer-mmgn3z7blyfeyxx5.cloudflarestream.com//manifest/video.m3u8?protocol=llhls' +``` + +If direct bet-sync returns `401` or `403`, continue with: + +- `capture/status` +- `keeper/state` + +and treat the keeper response as the public bettor-authority proxy. + +Expected healthy pattern: + +- bet-sync `publicReadiness.ready = true` +- bet-sync `sourceRuntime.ready = true` +- capture/status `sourceRuntime.ready = true` +- keeper `canonicalAuthority.decision = ready` +- keeper `channel.publicPlaybackUrl` present +- Cloudflare manifest returns `200` + +## Recovery Order + +1. Confirm Cloudflare playback is actually healthy. +2. Confirm source worker truth on the GPU host. +3. Compare public bet-sync state against public capture/status. +4. Fix public authority routing before touching player code. +5. Restart the keeper only after the public authority path is corrected. + +Do not start by changing Hyperbet UI code if: + +- Cloudflare is healthy +- capture/status is healthy +- keeper is blocked by false upstream readiness + +That is an authority topology problem, not a player problem. + +## Promotion Rule + +If this runbook ever changes because of a new incident, update it together with: + +- the authority audit script +- the local enoomian incident record +- any proxy/deploy topology documentation that explains the change diff --git a/package.json b/package.json index 13bf1cef..c494bcf5 100644 --- a/package.json +++ b/package.json @@ -72,6 +72,7 @@ "pm:stream:probe": "node --import tsx scripts/probe-hyperscapes-stream-headless.ts", "pm:soak": "node --import tsx scripts/pm-soak-monitor.ts", "pm:soak:harness": "node --import tsx scripts/soak-harness.ts", + "enoomian:authority:audit": "node --import tsx scripts/enoomian-staging-authority-audit.ts", "ci:prepr": "node scripts/check-pr-ready.mjs" }, "devDependencies": { diff --git a/scripts/enoomian-staging-authority-audit.test.ts b/scripts/enoomian-staging-authority-audit.test.ts new file mode 100644 index 00000000..bd742cd0 --- /dev/null +++ b/scripts/enoomian-staging-authority-audit.test.ts @@ -0,0 +1,103 @@ +import { afterEach, describe, expect, it } from "bun:test"; + +import { + compareTruths, + fetchJsonSettled, + probePlaybackSettled, + summarizePayload, +} from "./enoomian-staging-authority-audit"; + +const originalFetch = globalThis.fetch; + +afterEach(() => { + globalThis.fetch = originalFetch; +}); + +describe("enoomian staging authority audit", () => { + it("preserves endpoint evidence when a JSON probe throws", async () => { + globalThis.fetch = (async () => { + throw new Error("capture timed out"); + }) as typeof fetch; + + const result = await fetchJsonSettled("https://example.com/capture", 100); + + expect(result.ok).toBe(false); + expect(result.status).toBe(0); + expect(result.error).toContain("capture timed out"); + expect(result.body).toEqual({ error: "capture timed out" }); + }); + + it("preserves endpoint evidence when a playback probe throws", async () => { + globalThis.fetch = (async () => { + throw new Error("socket hang up"); + }) as typeof fetch; + + const result = await probePlaybackSettled("https://example.com/video.m3u8", 100); + + expect(result.ok).toBe(false); + expect(result.status).toBe(0); + expect(result.error).toContain("socket hang up"); + }); + + it("reports probe failures alongside surviving truth comparisons", () => { + const betSync = { + url: "https://example.com/bet-sync", + status: 200, + ok: true, + headers: {}, + body: {}, + error: null, + }; + const capture = { + url: "https://example.com/capture", + status: 0, + ok: false, + headers: {}, + body: { error: "timeout" }, + error: "timeout", + }; + const keeper = { + url: "https://example.com/keeper", + status: 200, + ok: true, + headers: {}, + body: {}, + error: null, + }; + + const betSyncSummary = summarizePayload({ + publicReadiness: { ready: false, reason: "source_unready" }, + sourceRuntime: { ready: false, captureMode: "none" }, + canonicalAuthority: { decision: "blocked" }, + }); + const captureSummary = summarizePayload({ + publicReadiness: { ready: true }, + sourceRuntime: { ready: true, captureMode: "x11_nvenc" }, + delivery: { playbackUrl: "https://example.com/video.m3u8" }, + }); + const keeperSummary = summarizePayload({ + publicReadiness: { ready: false, reason: "source_unready" }, + sourceRuntime: { ready: true, captureMode: "x11_nvenc" }, + canonicalAuthority: { decision: "blocked" }, + delivery: { playbackUrl: "https://example.com/video.m3u8" }, + }); + + const issues = compareTruths( + betSync, + capture, + keeper, + captureSummary, + betSyncSummary, + keeperSummary, + ); + + expect(issues).toContain("capture/status probe failed: timeout"); + expect(issues).toContain("capture/status returned HTTP 0"); + expect(issues).toContain( + "bet-sync blocks public readiness while capture/status reports sourceReady=true", + ); + expect(issues).toContain( + "keeper blocks public readiness while capture/status reports sourceReady=true", + ); + }); +}); diff --git a/scripts/enoomian-staging-authority-audit.ts b/scripts/enoomian-staging-authority-audit.ts new file mode 100644 index 00000000..332afd38 --- /dev/null +++ b/scripts/enoomian-staging-authority-audit.ts @@ -0,0 +1,441 @@ +import { URL, pathToFileURL } from "node:url"; + +type JsonRecord = Record; + +type EndpointResult = { + url: string; + status: number; + ok: boolean; + headers: Record; + body: unknown; + error: string | null; +}; + +type StreamSummary = { + publicReady: boolean | null; + publicReason: string | null; + channelReady: boolean | null; + sourceReady: boolean | null; + captureMode: string | null; + sourceStatusSource: string | null; + canonicalDecision: string | null; + canonicalLiveInputId: string | null; + playbackUrl: string | null; + rendererReady: boolean | null; +}; + +type AuditConfig = { + publicOrigin: string; + keeperUrl: string; + timeoutMs: number; + checkPlayback: boolean; +}; + +function getArg(name: string): string | null { + const prefix = `${name}=`; + const direct = process.argv.find((entry) => entry.startsWith(prefix)); + return direct ? direct.slice(prefix.length).trim() : null; +} + +function hasFlag(name: string): boolean { + return process.argv.includes(name); +} + +function normalizeBaseUrl(value: string): string { + return value.trim().replace(/\/+$/, ""); +} + +function parseConfig(): AuditConfig { + const publicOrigin = + getArg("--public-origin") ?? + process.env.ENOOMIAN_PUBLIC_ORIGIN ?? + "https://46.4.80.150.sslip.io"; + const keeperUrl = + getArg("--keeper-url") ?? + process.env.ENOOMIAN_KEEPER_URL ?? + "https://hyperbet-keeper-staging-production.up.railway.app"; + const timeoutMs = Number.parseInt( + getArg("--timeout-ms") ?? process.env.ENOOMIAN_AUDIT_TIMEOUT_MS ?? "12000", + 10, + ); + + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { + throw new Error(`invalid timeout ${timeoutMs}`); + } + + return { + publicOrigin: normalizeBaseUrl(publicOrigin), + keeperUrl: normalizeBaseUrl(keeperUrl), + timeoutMs, + checkPlayback: !hasFlag("--skip-playback"), + }; +} + +function asRecord(value: unknown): JsonRecord | null { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as JsonRecord) + : null; +} + +function asBoolean(value: unknown): boolean | null { + return typeof value === "boolean" ? value : null; +} + +function asString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value : null; +} + +function getPath(record: JsonRecord | null, ...path: string[]): unknown { + let cursor: unknown = record; + for (const key of path) { + const next = asRecord(cursor); + if (!next || !(key in next)) return null; + cursor = next[key]; + } + return cursor; +} + +function pickPlaybackUrl(record: JsonRecord | null): string | null { + const candidates = [ + getPath(record, "channel", "publicPlaybackUrl"), + getPath(record, "delivery", "playbackUrl"), + getPath(record, "delivery", "llhlsUrl"), + getPath(record, "canonicalAuthority", "playbackUrl"), + ]; + for (const candidate of candidates) { + const value = asString(candidate); + if (value) return value; + } + return null; +} + +function summarizePayload(payload: unknown): StreamSummary { + const root = asRecord(payload); + return { + publicReady: asBoolean( + getPath(root, "publicReadiness", "ready") ?? + getPath(root, "channel", "publicReadiness", "ready"), + ), + publicReason: asString( + getPath(root, "publicReadiness", "reason") ?? + getPath(root, "channel", "publicReadiness", "reason"), + ), + channelReady: asBoolean(getPath(root, "channel", "ready")), + sourceReady: asBoolean(getPath(root, "sourceRuntime", "ready")), + captureMode: asString(getPath(root, "sourceRuntime", "captureMode")), + sourceStatusSource: asString(getPath(root, "sourceRuntime", "statusSource")), + canonicalDecision: asString(getPath(root, "canonicalAuthority", "decision")), + canonicalLiveInputId: asString(getPath(root, "canonicalAuthority", "liveInputId")), + playbackUrl: pickPlaybackUrl(root), + rendererReady: asBoolean(getPath(root, "rendererHealth", "ready")), + }; +} + +async function fetchJson(url: string, timeoutMs: number): Promise { + const response = await fetch(url, { + headers: { accept: "application/json" }, + signal: AbortSignal.timeout(timeoutMs), + }); + const text = await response.text(); + let body: unknown = text; + try { + body = JSON.parse(text); + } catch { + // keep raw body for diagnosis + } + const headers = Object.fromEntries(response.headers.entries()); + return { + url, + status: response.status, + ok: response.ok, + headers, + body, + error: null, + }; +} + +async function probePlayback(url: string, timeoutMs: number): Promise { + const response = await fetch(url, { + method: "HEAD", + signal: AbortSignal.timeout(timeoutMs), + }); + return { + url, + status: response.status, + ok: response.ok, + headers: Object.fromEntries(response.headers.entries()), + body: null, + error: null, + }; +} + +function formatError(error: unknown): string { + if (error instanceof Error) return error.message; + return String(error); +} + +function buildFailedEndpoint(url: string, error: unknown): EndpointResult { + const message = formatError(error); + return { + url, + status: 0, + ok: false, + headers: {}, + body: { error: message }, + error: message, + }; +} + +async function fetchJsonSettled(url: string, timeoutMs: number): Promise { + try { + return await fetchJson(url, timeoutMs); + } catch (error) { + return buildFailedEndpoint(url, error); + } +} + +async function probePlaybackSettled( + url: string, + timeoutMs: number, +): Promise { + try { + return await probePlayback(url, timeoutMs); + } catch (error) { + return buildFailedEndpoint(url, error); + } +} + +function formatSummary(label: string, summary: StreamSummary): string[] { + return [ + `${label}:`, + ` publicReady=${summary.publicReady ?? "null"} reason=${summary.publicReason ?? "null"}`, + ` channelReady=${summary.channelReady ?? "null"} rendererReady=${summary.rendererReady ?? "null"}`, + ` sourceReady=${summary.sourceReady ?? "null"} captureMode=${summary.captureMode ?? "null"} statusSource=${summary.sourceStatusSource ?? "null"}`, + ` canonicalDecision=${summary.canonicalDecision ?? "null"} liveInputId=${summary.canonicalLiveInputId ?? "null"}`, + ` playbackUrl=${summary.playbackUrl ?? "null"}`, + ]; +} + +function compareTruths( + betSyncResult: EndpointResult, + captureResult: EndpointResult, + keeperResult: EndpointResult, + capture: StreamSummary, + betSync: StreamSummary, + keeper: StreamSummary, +): string[] { + const issues: string[] = []; + + if (captureResult.error) { + issues.push(`capture/status probe failed: ${captureResult.error}`); + } + + if (!captureResult.ok) { + issues.push(`capture/status returned HTTP ${captureResult.status}`); + } + + if (keeperResult.error) { + issues.push(`keeper/state probe failed: ${keeperResult.error}`); + } + + if (!keeperResult.ok) { + issues.push(`keeper/state returned HTTP ${keeperResult.status}`); + } + + if (betSyncResult.error) { + issues.push(`bet-sync/state probe failed: ${betSyncResult.error}`); + } + + if (!betSyncResult.ok && betSyncResult.status !== 401 && betSyncResult.status !== 403) { + issues.push(`bet-sync/state returned HTTP ${betSyncResult.status}`); + } + + if (capture.sourceReady !== null && betSync.sourceReady !== null && capture.sourceReady !== betSync.sourceReady) { + issues.push( + `bet-sync sourceReady ${betSync.sourceReady} disagrees with capture/status ${capture.sourceReady}`, + ); + } + + if ( + capture.captureMode && + betSync.captureMode && + capture.captureMode !== betSync.captureMode + ) { + issues.push( + `bet-sync captureMode ${betSync.captureMode} disagrees with capture/status ${capture.captureMode}`, + ); + } + + if ( + capture.sourceReady === true && + betSync.publicReady === false + ) { + issues.push( + `bet-sync blocks public readiness while capture/status reports sourceReady=true`, + ); + } + + if ( + capture.sourceReady === true && + keeper.publicReady === false + ) { + issues.push( + `keeper blocks public readiness while capture/status reports sourceReady=true`, + ); + } + + if ( + betSync.playbackUrl && + keeper.playbackUrl && + betSync.playbackUrl !== keeper.playbackUrl + ) { + issues.push( + `keeper playbackUrl disagrees with bet-sync playbackUrl`, + ); + } + + if ( + capture.playbackUrl && + keeper.playbackUrl && + capture.playbackUrl !== keeper.playbackUrl + ) { + issues.push(`keeper playbackUrl disagrees with capture/status playbackUrl`); + } + + if ( + betSync.publicReady !== null && + keeper.publicReady !== null && + betSync.publicReady !== keeper.publicReady + ) { + issues.push( + `keeper publicReadiness ${keeper.publicReady} disagrees with bet-sync ${betSync.publicReady}`, + ); + } + + if ( + betSync.canonicalDecision && + keeper.canonicalDecision && + betSync.canonicalDecision !== keeper.canonicalDecision + ) { + issues.push( + `keeper canonicalAuthority.decision ${keeper.canonicalDecision} disagrees with bet-sync ${betSync.canonicalDecision}`, + ); + } + + if (betSync.publicReady === true && !betSync.playbackUrl) { + issues.push(`bet-sync reports publicReady=true without a playback URL`); + } + + if (keeper.publicReady === true && !keeper.playbackUrl) { + issues.push(`keeper reports publicReady=true without a playback URL`); + } + + return issues; +} + +function printEndpoint(label: string, result: EndpointResult): void { + console.log(`${label} endpoint: ${result.url}`); + console.log(` http=${result.status} ok=${result.ok}`); + const server = result.headers.server ?? "null"; + const via = result.headers.via ?? "null"; + console.log(` server=${server} via=${via}`); + if (result.error) { + console.log(` error=${result.error}`); + } +} + +async function main(): Promise { + const config = parseConfig(); + + const betSyncUrl = new URL("/api/internal/bet-sync/state", config.publicOrigin).toString(); + const captureUrl = new URL("/api/streaming/capture/status", config.publicOrigin).toString(); + const keeperStateUrl = new URL("/api/streaming/state", config.keeperUrl).toString(); + + const [betSync, capture, keeper] = await Promise.all([ + fetchJsonSettled(betSyncUrl, config.timeoutMs), + fetchJsonSettled(captureUrl, config.timeoutMs), + fetchJsonSettled(keeperStateUrl, config.timeoutMs), + ]); + + const betSyncSummary = summarizePayload(betSync.body); + const captureSummary = summarizePayload(capture.body); + const keeperSummary = summarizePayload(keeper.body); + + printEndpoint("bet-sync", betSync); + printEndpoint("capture/status", capture); + printEndpoint("keeper/state", keeper); + console.log(""); + for (const line of formatSummary("bet-sync", betSyncSummary)) console.log(line); + for (const line of formatSummary("capture/status", captureSummary)) console.log(line); + for (const line of formatSummary("keeper/state", keeperSummary)) console.log(line); + + const issues = compareTruths( + betSync, + capture, + keeper, + captureSummary, + betSyncSummary, + keeperSummary, + ); + + if (!betSync.ok && (betSync.status === 401 || betSync.status === 403)) { + console.log(""); + console.log( + `note: direct bet-sync access returned HTTP ${betSync.status}; using keeper/state as the public authority proxy`, + ); + } + + if (config.checkPlayback) { + const playbackUrl = + keeperSummary.playbackUrl ?? betSyncSummary.playbackUrl ?? captureSummary.playbackUrl; + if (!playbackUrl) { + issues.push("no playback URL available to probe"); + } else { + const playback = await probePlaybackSettled(playbackUrl, config.timeoutMs); + console.log(""); + console.log(`playback probe: ${playback.url}`); + console.log(` http=${playback.status} ok=${playback.ok}`); + console.log(` server=${playback.headers.server ?? "null"} via=${playback.headers.via ?? "null"}`); + if (playback.error) { + console.log(` error=${playback.error}`); + } + if (playback.error) { + issues.push(`playback probe failed: ${playback.error}`); + } + if (!playback.ok) { + issues.push(`playback manifest probe returned HTTP ${playback.status}`); + } + } + } + + console.log(""); + if (issues.length === 0) { + console.log("authority audit: PASS"); + return; + } + + console.log("authority audit: FAIL"); + for (const issue of issues) { + console.log(`- ${issue}`); + } + process.exitCode = 1; +} + +export { + buildFailedEndpoint, + compareTruths, + fetchJsonSettled, + probePlaybackSettled, + summarizePayload, +}; + +const isMain = + typeof process.argv[1] === "string" && + pathToFileURL(process.argv[1]).href === import.meta.url; + +if (isMain) { + main().catch((error) => { + console.error(`authority audit failed: ${formatError(error)}`); + process.exit(1); + }); +} diff --git a/scripts/enoomian-staging-secret-manifest.sh b/scripts/enoomian-staging-secret-manifest.sh new file mode 100644 index 00000000..5f8061fc --- /dev/null +++ b/scripts/enoomian-staging-secret-manifest.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash + +# Variable names only. Real values must live in a local untracked env file. + +ENOOMIAN_PERSONAL_SECRET_REQUIRED_KEYS=( + ENOOMIAN_HYPERSCAPES_ROOT + ENOOMIAN_RAILWAY_PROJECT_ID + ENOOMIAN_RAILWAY_ENVIRONMENT_ID + ENOOMIAN_HYPERSCAPES_SERVICE_ID + ENOOMIAN_HYPERSCAPES_API_URL + ENOOMIAN_HYPERSCAPES_WS_URL + ENOOMIAN_HYPERSCAPES_PAGES_URL + ENOOMIAN_HYPERSCAPES_PAGES_PROJECT_NAME + ENOOMIAN_DATABASE_URL + ENOOMIAN_HYPERSCAPES_JWT_SECRET + ENOOMIAN_HYPERSCAPES_AUTOMATION_TOKEN + ENOOMIAN_HYPERBET_SOLANA_KEEPER_SERVICE_ID + ENOOMIAN_HYPERBET_SOLANA_KEEPER_URL + ENOOMIAN_HYPERBET_BSC_KEEPER_SERVICE_ID + ENOOMIAN_HYPERBET_BSC_KEEPER_URL + ENOOMIAN_HYPERBET_KEEPER_SERVICE_ID + ENOOMIAN_HYPERBET_KEEPER_URL + ENOOMIAN_HYPERBET_KEEPER_WS_URL + ENOOMIAN_HYPERBET_PAGES_URL + ENOOMIAN_HYPERBET_PAGES_PROJECT_NAME + ENOOMIAN_SOLANA_CLUSTER + ENOOMIAN_SOLANA_RPC_URL + ENOOMIAN_SOLANA_FIGHT_ORACLE_PROGRAM_ID + ENOOMIAN_SOLANA_GOLD_CLOB_PROGRAM_ID + ENOOMIAN_SOLANA_GOLD_AMM_PROGRAM_ID + ENOOMIAN_SOLANA_GOLD_PERPS_PROGRAM_ID + ENOOMIAN_HYPERBET_SOLANA_STREAM_PUBLISH_KEY + ENOOMIAN_HYPERBET_SOLANA_ORACLE_AUTHORITY_KEYPAIR + ENOOMIAN_HYPERBET_SOLANA_CANARY_KEYPAIR + ENOOMIAN_HYPERBET_SOLANA_MARKET_MAKER_KEYPAIR + ENOOMIAN_BSC_RPC_URL + ENOOMIAN_BSC_DUEL_ORACLE_ADDRESS + ENOOMIAN_BSC_GOLD_CLOB_ADDRESS + ENOOMIAN_HYPERBET_BSC_STREAM_PUBLISH_KEY + ENOOMIAN_HYPERBET_BSC_REPORTER_PRIVATE_KEY + ENOOMIAN_HYPERBET_BSC_MARKET_OPERATOR_PRIVATE_KEY + ENOOMIAN_HYPERBET_BSC_CANARY_PRIVATE_KEY +) + +ENOOMIAN_PERSONAL_SECRET_OPTIONAL_KEYS=( + ENOOMIAN_HYPERBET_BSC_FINALIZER_PRIVATE_KEY + ENOOMIAN_HYPERBET_SOLANA_PAGES_URL + ENOOMIAN_HYPERBET_SOLANA_PAGES_PROJECT_NAME + ENOOMIAN_HYPERBET_BSC_PAGES_URL + ENOOMIAN_HYPERBET_BSC_PAGES_PROJECT_NAME + ENOOMIAN_BETTING_FEED_ACCESS_TOKEN + ENOOMIAN_STREAMING_VIEWER_ACCESS_TOKEN + ENOOMIAN_PUBLIC_PRIVY_APP_ID + ENOOMIAN_PRIVY_APP_SECRET + ENOOMIAN_STREAM_INGEST_STREAM_KEY + ENOOMIAN_STREAM_INGEST_SRT_PASSPHRASE + ENOOMIAN_STREAM_RENDERER_HEALTH_BEARER_TOKEN + ENOOMIAN_AWS_GPU_SSH_TARGET + ENOOMIAN_AWS_GPU_HOST + ENOOMIAN_AWS_GPU_SSH_USER + ENOOMIAN_AWS_GPU_SSH_KEY + ENOOMIAN_AWS_GPU_SSH_PORT + ENOOMIAN_AWS_GPU_REMOTE_ROOT + ENOOMIAN_AWS_GPU_API_PORT + ENOOMIAN_AWS_GPU_UWS_PORT + ENOOMIAN_AWS_GPU_DATABASE_URL + ENOOMIAN_AWS_GPU_START_SOURCE + ENOOMIAN_AWS_GPU_STREAM_DELIVERY_MODE + ENOOMIAN_AWS_GPU_STREAM_DELIVERY_PROVIDER + ENOOMIAN_AWS_GPU_STREAM_ENABLED_DESTINATIONS + ENOOMIAN_AWS_GPU_STREAM_CANONICAL_PROVIDER_PRIORITY + ENOOMIAN_AWS_GPU_PUBLIC_HLS_URL + ENOOMIAN_AWS_GPU_PUBLIC_LLHLS_URL + ENOOMIAN_AWS_GPU_PUBLIC_API_URL + ENOOMIAN_AWS_GPU_PUBLIC_WS_URL + ENOOMIAN_AWS_GPU_ACTIVATE_PUBLIC_AUTHORITY + ENOOMIAN_AWS_GPU_SYNC_HYPERSCAPES_RAILWAY_ENV + ENOOMIAN_AWS_GPU_SYNC_HYPERBET_KEEPER_ENV +) + +ENOOMIAN_PERSONAL_SECRET_RESERVED_CLOUDFLARE_KEYS=( + ENOOMIAN_CLOUDFLARE_API_TOKEN + ENOOMIAN_CLOUDFLARE_CDN_API_TOKEN + ENOOMIAN_CLOUDFLARE_R2_BUCKET_API_TOKEN + ENOOMIAN_CLOUDFLARE_STREAM_API_TOKEN + STREAM_CLOUDFLARE_LIVE_INPUT_ID + STREAM_CLOUDFLARE_WEBHOOK_SECRET + STREAM_CLOUDFLARE_ACCOUNT_ID +) diff --git a/scripts/enoomian-staging/aws-gpu.sh b/scripts/enoomian-staging/aws-gpu.sh new file mode 100644 index 00000000..c863d13e --- /dev/null +++ b/scripts/enoomian-staging/aws-gpu.sh @@ -0,0 +1,502 @@ +#!/usr/bin/env bash + +set -euo pipefail + +enoomian_aws_gpu_remote_root() { + if [[ -n "${ENOOMIAN_AWS_GPU_REMOTE_ROOT:-}" ]]; then + printf '%s\n' "${ENOOMIAN_AWS_GPU_REMOTE_ROOT}" + return 0 + fi + + local user="${ENOOMIAN_AWS_GPU_SSH_USER:-root}" + if [[ -n "${ENOOMIAN_AWS_GPU_SSH_TARGET:-}" && "${ENOOMIAN_AWS_GPU_SSH_TARGET}" == *@* ]]; then + user="${ENOOMIAN_AWS_GPU_SSH_TARGET%@*}" + fi + if [[ "${user}" == "root" ]]; then + printf '%s\n' "/root/hyperscape" + else + printf '%s\n' "/home/${user}/hyperscape" + fi +} + +enoomian_aws_gpu_ssh_target() { + if [[ -n "${ENOOMIAN_AWS_GPU_SSH_TARGET:-}" ]]; then + printf '%s\n' "${ENOOMIAN_AWS_GPU_SSH_TARGET}" + return 0 + fi + + enoomian_require_env ENOOMIAN_AWS_GPU_HOST + printf '%s@%s\n' "${ENOOMIAN_AWS_GPU_SSH_USER:-root}" "${ENOOMIAN_AWS_GPU_HOST}" +} + +enoomian_aws_gpu_ssh_args() { + local out_name="$1" + eval "${out_name}=(-o BatchMode=yes -o StrictHostKeyChecking=accept-new)" + if [[ -n "${ENOOMIAN_AWS_GPU_SSH_KEY:-}" ]]; then + eval "${out_name}+=( -i \"\${ENOOMIAN_AWS_GPU_SSH_KEY}\" )" + fi + if [[ -n "${ENOOMIAN_AWS_GPU_SSH_PORT:-}" ]]; then + eval "${out_name}+=( -p \"\${ENOOMIAN_AWS_GPU_SSH_PORT}\" )" + fi +} + +enoomian_aws_gpu_ssh() { + local target + local ssh_args=() + target="$(enoomian_aws_gpu_ssh_target)" + enoomian_aws_gpu_ssh_args ssh_args + ssh "${ssh_args[@]}" "${target}" "$@" +} + +enoomian_aws_gpu_rsync_rsh() { + local ssh_cmd="ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new" + if [[ -n "${ENOOMIAN_AWS_GPU_SSH_KEY:-}" ]]; then + ssh_cmd+=" -i $(enoomian_aws_gpu_shell_quote "${ENOOMIAN_AWS_GPU_SSH_KEY}")" + fi + if [[ -n "${ENOOMIAN_AWS_GPU_SSH_PORT:-}" ]]; then + ssh_cmd+=" -p $(enoomian_aws_gpu_shell_quote "${ENOOMIAN_AWS_GPU_SSH_PORT}")" + fi + printf '%s\n' "${ssh_cmd}" +} + +enoomian_aws_gpu_shell_quote() { + local value="$1" + printf "'%s'" "${value//\'/\'\\\'\'}" +} + +enoomian_aws_gpu_write_env_line() { + local file="$1" + local key="$2" + local value="${3:-}" + printf '%s=' "${key}" >>"${file}" + enoomian_aws_gpu_shell_quote "${value}" >>"${file}" + printf '\n' >>"${file}" +} + +enoomian_aws_gpu_trim_trailing_slashes() { + local value="${1:-}" + while [[ "${value}" == */ ]]; do + value="${value%/}" + done + printf '%s\n' "${value}" +} + +enoomian_aws_gpu_write_runtime_env() { + local env_file="$1" + local stream_url + local public_cdn_url + local stream_delivery_mode + local stream_delivery_provider + local stream_playback_url + local stream_playback_llhls_url + local stream_external_delivery_provider + local stream_external_playback_hls_url + local stream_external_playback_llhls_url + local stream_external_ingest_rtmps_url + local stream_ingest_rtmps_url + local stream_enabled_destinations + local stream_canonical_provider_priority + local stream_ingest_transport + local stream_capture_width + local stream_capture_height + local database_url + local remote_root + local api_port + + enoomian_require_env \ + ENOOMIAN_HYPERSCAPES_ROOT \ + ENOOMIAN_HYPERSCAPES_API_URL \ + ENOOMIAN_HYPERSCAPES_WS_URL \ + ENOOMIAN_HYPERSCAPES_PAGES_URL \ + ENOOMIAN_HYPERSCAPES_JWT_SECRET \ + ENOOMIAN_BETTING_FEED_ACCESS_TOKEN \ + ENOOMIAN_STREAMING_VIEWER_ACCESS_TOKEN + + database_url="${ENOOMIAN_AWS_GPU_DATABASE_URL:-${ENOOMIAN_EXTERNAL_DATABASE_URL:-${ENOOMIAN_LOCAL_DATABASE_URL:-${ENOOMIAN_DATABASE_URL:-}}}}" + [[ -n "${database_url}" ]] || enoomian_die "missing AWS-reachable database url: set ENOOMIAN_AWS_GPU_DATABASE_URL or ENOOMIAN_LOCAL_DATABASE_URL" + + stream_url="$(enoomian_hyperscapes_stream_url)" + enoomian_require_tokenized_hyperscapes_stream_url "${stream_url}" + + public_cdn_url="${ENOOMIAN_PUBLIC_CDN_URL:-${ENOOMIAN_HYPERSCAPES_CDN_URL:-${ENOOMIAN_HYPERSCAPES_API_URL%/}/game-assets}}" + stream_delivery_mode="${ENOOMIAN_AWS_GPU_STREAM_DELIVERY_MODE:-${ENOOMIAN_STREAM_DELIVERY_MODE:-external_hls}}" + stream_delivery_provider="${ENOOMIAN_AWS_GPU_STREAM_DELIVERY_PROVIDER:-${ENOOMIAN_STREAM_DELIVERY_PROVIDER:-cloudflare_stream}}" + stream_playback_url="${ENOOMIAN_AWS_GPU_PUBLIC_HLS_URL:-${ENOOMIAN_STREAM_PLAYBACK_URL:-}}" + stream_playback_llhls_url="${ENOOMIAN_STREAM_PLAYBACK_LLHLS_URL:-}" + stream_external_delivery_provider="${ENOOMIAN_STREAM_EXTERNAL_DELIVERY_PROVIDER:-}" + stream_external_playback_hls_url="${ENOOMIAN_STREAM_EXTERNAL_PLAYBACK_HLS_URL:-}" + stream_external_playback_llhls_url="${ENOOMIAN_STREAM_EXTERNAL_PLAYBACK_LLHLS_URL:-}" + if [[ "${stream_delivery_mode}" == "external_hls" ]]; then + stream_external_delivery_provider="${stream_external_delivery_provider:-${stream_delivery_provider:-cloudflare_stream}}" + stream_external_playback_hls_url="${stream_external_playback_hls_url:-${stream_playback_url}}" + stream_external_playback_llhls_url="${stream_external_playback_llhls_url:-${stream_playback_llhls_url}}" + fi + stream_external_ingest_rtmps_url="$(enoomian_aws_gpu_trim_trailing_slashes "${ENOOMIAN_STREAM_EXTERNAL_INGEST_RTMPS_URL:-}")" + stream_ingest_rtmps_url="$(enoomian_aws_gpu_trim_trailing_slashes "${ENOOMIAN_STREAM_INGEST_RTMPS_URL:-}")" + stream_enabled_destinations="${ENOOMIAN_AWS_GPU_STREAM_ENABLED_DESTINATIONS:-${ENOOMIAN_STREAM_ENABLED_DESTINATIONS:-}}" + stream_canonical_provider_priority="${ENOOMIAN_AWS_GPU_STREAM_CANONICAL_PROVIDER_PRIORITY:-${ENOOMIAN_STREAM_CANONICAL_PROVIDER_PRIORITY:-cloudflare_stream}}" + stream_ingest_transport="${ENOOMIAN_STREAM_INGEST_TRANSPORT:-rtmps}" + stream_capture_width="${ENOOMIAN_STREAM_CAPTURE_WIDTH:-${ENOOMIAN_STREAM_OUTPUT_WIDTH:-1280}}" + stream_capture_height="${ENOOMIAN_STREAM_CAPTURE_HEIGHT:-${ENOOMIAN_STREAM_OUTPUT_HEIGHT:-720}}" + remote_root="$(enoomian_aws_gpu_remote_root)" + api_port="${ENOOMIAN_AWS_GPU_API_PORT:-5560}" + + : >"${env_file}" + enoomian_aws_gpu_write_env_line "${env_file}" NODE_ENV production + enoomian_aws_gpu_write_env_line "${env_file}" PORT "${api_port}" + enoomian_aws_gpu_write_env_line "${env_file}" UWS_PORT "${ENOOMIAN_AWS_GPU_UWS_PORT:-5561}" + enoomian_aws_gpu_write_env_line "${env_file}" DUEL_DATABASE_MODE remote + enoomian_aws_gpu_write_env_line "${env_file}" USE_LOCAL_POSTGRES false + enoomian_aws_gpu_write_env_line "${env_file}" DATABASE_URL "${database_url}" + enoomian_aws_gpu_write_env_line "${env_file}" JWT_SECRET "${ENOOMIAN_HYPERSCAPES_JWT_SECRET}" + enoomian_aws_gpu_write_env_line "${env_file}" SKIP_MIGRATIONS true + enoomian_aws_gpu_write_env_line "${env_file}" DISABLE_RATE_LIMIT true + enoomian_aws_gpu_write_env_line "${env_file}" UWS_ENABLED false + enoomian_aws_gpu_write_env_line "${env_file}" HEALTH_CHECK_DATABASE false + enoomian_aws_gpu_write_env_line "${env_file}" STREAMING_DUEL_ENABLED true + enoomian_aws_gpu_write_env_line "${env_file}" STREAMING_CAPTURE_ENABLED true + enoomian_aws_gpu_write_env_line "${env_file}" STREAMING_LOCAL_CAPTURE_WORKER_ENABLED false + enoomian_aws_gpu_write_env_line "${env_file}" STREAMING_LOCAL_CAPTURE_USE_PUBLIC_URL false + enoomian_aws_gpu_write_env_line "${env_file}" DUEL_OWNS_STREAM_CAPTURE false + enoomian_aws_gpu_write_env_line "${env_file}" GAME_URL "${stream_url}" + enoomian_aws_gpu_write_env_line "${env_file}" GAME_FALLBACK_URLS "${ENOOMIAN_HYPERSCAPES_PAGES_URL%/}/stream.html,${ENOOMIAN_HYPERSCAPES_PAGES_URL%/}/?page=stream" + enoomian_aws_gpu_write_env_line "${env_file}" PUBLIC_API_URL "${ENOOMIAN_HYPERSCAPES_API_URL}" + enoomian_aws_gpu_write_env_line "${env_file}" PUBLIC_WS_URL "${ENOOMIAN_HYPERSCAPES_WS_URL}" + enoomian_aws_gpu_write_env_line "${env_file}" PUBLIC_APP_URL "${ENOOMIAN_HYPERSCAPES_PAGES_URL}" + enoomian_aws_gpu_write_env_line "${env_file}" PUBLIC_CDN_URL "${public_cdn_url}" + enoomian_aws_gpu_write_env_line "${env_file}" PUBLIC_ASSETS_URL "${public_cdn_url}" + enoomian_aws_gpu_write_env_line "${env_file}" BETTING_FEED_ACCESS_TOKEN "${ENOOMIAN_BETTING_FEED_ACCESS_TOKEN}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAMING_VIEWER_ACCESS_TOKEN "${ENOOMIAN_STREAMING_VIEWER_ACCESS_TOKEN}" + + enoomian_aws_gpu_write_env_line "${env_file}" DISPLAY "${ENOOMIAN_AWS_GPU_DISPLAY:-:99}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_CAPTURE_MODE x11_nvenc + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_CAPTURE_HEADLESS false + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_CAPTURE_CHANNEL "${ENOOMIAN_AWS_GPU_CHROME_CHANNEL:-chrome-beta}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_CAPTURE_ANGLE vulkan + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_CAPTURE_WIDTH "${stream_capture_width}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_CAPTURE_HEIGHT "${stream_capture_height}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_OUTPUT_WIDTH "${ENOOMIAN_STREAM_OUTPUT_WIDTH:-1280}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_OUTPUT_HEIGHT "${ENOOMIAN_STREAM_OUTPUT_HEIGHT:-720}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_VIDEO_BITRATE_KBPS "${ENOOMIAN_STREAM_VIDEO_BITRATE_KBPS:-4500}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_AUDIO_BITRATE_KBPS "${ENOOMIAN_STREAM_AUDIO_BITRATE_KBPS:-128}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_FPS "${ENOOMIAN_STREAM_FPS:-30}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_GOP_SIZE "${ENOOMIAN_STREAM_GOP_SIZE:-60}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_AUDIO_SAMPLE_RATE "${ENOOMIAN_STREAM_AUDIO_SAMPLE_RATE:-48000}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_LOW_LATENCY "${ENOOMIAN_STREAM_LOW_LATENCY:-true}" + enoomian_aws_gpu_write_env_line "${env_file}" FFMPEG_HWACCEL nvidia + enoomian_aws_gpu_write_env_line "${env_file}" RTMP_STATUS_FILE "${ENOOMIAN_AWS_GPU_RTMP_STATUS_FILE:-${remote_root}/.runtime-locks/rtmp-status.json}" + + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_DELIVERY_MODE "${stream_delivery_mode}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_DELIVERY_PROVIDER "${stream_delivery_provider}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_PLAYBACK_URL "${stream_playback_url}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_PLAYBACK_HLS_URL "${stream_playback_url}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_PLAYBACK_LLHLS_URL "${stream_playback_llhls_url}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_EXTERNAL_DELIVERY_PROVIDER "${stream_external_delivery_provider}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_EXTERNAL_PLAYBACK_HLS_URL "${stream_external_playback_hls_url}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_EXTERNAL_PLAYBACK_LLHLS_URL "${stream_external_playback_llhls_url}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_CANONICAL_PROVIDER_PRIORITY "${stream_canonical_provider_priority}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_ENABLED_DESTINATIONS "${stream_enabled_destinations}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_ENABLE_AUTOMATIC_FAILOVER "${ENOOMIAN_STREAM_ENABLE_AUTOMATIC_FAILOVER:-false}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_CLOUDFLARE_PROBE_ONLY "${ENOOMIAN_STREAM_CLOUDFLARE_PROBE_ONLY:-false}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_CLOUDFLARE_LIVE_INPUT_ID "${STREAM_CLOUDFLARE_LIVE_INPUT_ID:-${ENOOMIAN_STREAM_CLOUDFLARE_LIVE_INPUT_ID:-${ENOOMIAN_STREAM_INGEST_SRT_STREAM_ID:-}}}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_CLOUDFLARE_ACCOUNT_ID "${STREAM_CLOUDFLARE_ACCOUNT_ID:-${ENOOMIAN_CLOUDFLARE_ACCOUNT_ID:-}}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_INGEST_PROFILE "${ENOOMIAN_STREAM_INGEST_PROFILE:-cloudflare_live}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_INGEST_TRANSPORT "${stream_ingest_transport}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_EXTERNAL_INGEST_RTMPS_URL "${stream_external_ingest_rtmps_url}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_INGEST_RTMPS_URL "${stream_ingest_rtmps_url}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_INGEST_STREAM_KEY "${ENOOMIAN_STREAM_INGEST_STREAM_KEY:-}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_INGEST_SRT_URL "${ENOOMIAN_STREAM_INGEST_SRT_URL:-}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_INGEST_SRT_STREAM_ID "${ENOOMIAN_STREAM_INGEST_SRT_STREAM_ID:-}" + enoomian_aws_gpu_write_env_line "${env_file}" STREAM_INGEST_SRT_PASSPHRASE "${ENOOMIAN_STREAM_INGEST_SRT_PASSPHRASE:-}" +} + +enoomian_aws_gpu_plan() { + local target="(unset)" + local remote_root + local public_api="${ENOOMIAN_AWS_GPU_PUBLIC_API_URL:-}" + remote_root="$(enoomian_aws_gpu_remote_root)" + if [[ -n "${ENOOMIAN_AWS_GPU_SSH_TARGET:-${ENOOMIAN_AWS_GPU_HOST:-}}" ]]; then + target="$(enoomian_aws_gpu_ssh_target)" + fi + + cat </dev/null + enoomian_log "local AWS GPU deploy preflight passed" +} + +enoomian_aws_gpu_remote_preflight() { + enoomian_aws_gpu_preflight + enoomian_log "checking AWS GPU host prerequisites" + enoomian_aws_gpu_ssh 'bash -lc '"'"' + set -euo pipefail + command -v nvidia-smi >/dev/null + nvidia-smi >/dev/null + command -v ffmpeg >/dev/null + ffmpeg -hide_banner -encoders 2>/dev/null > /tmp/hyperscape-ffmpeg-encoders.txt + grep -q h264_nvenc /tmp/hyperscape-ffmpeg-encoders.txt + command -v Xvfb >/dev/null + command -v xdpyinfo >/dev/null + command -v bun >/dev/null + if ! command -v google-chrome-beta >/dev/null && ! command -v google-chrome >/dev/null && ! command -v chromium >/dev/null; then + echo "missing Chrome/Chromium browser" >&2 + exit 1 + fi + '"'" + enoomian_log "AWS GPU remote preflight passed" +} + +enoomian_aws_gpu_sync_code() { + local target + local remote_root + local rsync_rsh + target="$(enoomian_aws_gpu_ssh_target)" + remote_root="$(enoomian_aws_gpu_remote_root)" + rsync_rsh="$(enoomian_aws_gpu_rsync_rsh)" + + enoomian_log "creating remote root ${target}:${remote_root}" + enoomian_aws_gpu_ssh "mkdir -p '${remote_root}'" + + enoomian_log "syncing Hyperscapes checkout to AWS GPU host" + rsync -az --delete \ + --exclude '.git' \ + --exclude 'node_modules' \ + --exclude 'packages/*/node_modules' \ + --exclude 'packages/client/dist' \ + --exclude 'packages/server/dist' \ + --exclude '.runtime-locks' \ + --exclude 'logs' \ + -e "${rsync_rsh}" \ + "${ENOOMIAN_HYPERSCAPES_ROOT%/}/" \ + "${target}:${remote_root}/" +} + +enoomian_aws_gpu_push_runtime_env() { + local env_tmp + local target + local remote_root + local rsync_rsh + env_tmp="$(mktemp "$(enoomian_tmp_root)/aws-gpu-env.XXXXXX")" + target="$(enoomian_aws_gpu_ssh_target)" + remote_root="$(enoomian_aws_gpu_remote_root)" + rsync_rsh="$(enoomian_aws_gpu_rsync_rsh)" + + enoomian_aws_gpu_write_runtime_env "${env_tmp}" + chmod 600 "${env_tmp}" + enoomian_log "pushing AWS GPU runtime env to ${target} (secret values not printed)" + rsync -az -e "${rsync_rsh}" "${env_tmp}" "${target}:/tmp/hyperscape-secrets.env" + enoomian_aws_gpu_ssh "mkdir -p '${remote_root}' && install -m 600 /tmp/hyperscape-secrets.env '${remote_root}/.env.production' && if [ \"\$(id -u)\" = 0 ]; then install -m 600 /tmp/hyperscape-secrets.env /root/hyperscape-secrets.env; fi" + rm -f "${env_tmp}" +} + +enoomian_aws_gpu_remote_bootstrap() { + local remote_root + local start_source + remote_root="$(enoomian_aws_gpu_remote_root)" + start_source="${ENOOMIAN_AWS_GPU_START_SOURCE:-0}" + + enoomian_log "bootstrapping AWS GPU host runtime" + enoomian_aws_gpu_ssh "REMOTE_ROOT='${remote_root}' ENOOMIAN_AWS_GPU_START_SOURCE='${start_source}' bash -s" <<'REMOTE_SCRIPT' +set -euo pipefail + +cd "${REMOTE_ROOT}" +export PATH="${HOME}/.bun/bin:/root/.bun/bin:${PATH}" + +SUDO="" +if [ "$(id -u)" -ne 0 ]; then + command -v sudo >/dev/null + SUDO="sudo" +fi + +${SUDO} apt-get update +${SUDO} apt-get install -y build-essential python3 socat xvfb git-lfs ffmpeg wget gnupg iproute2 lsof postgresql-client x11-utils jq curl + +if ! command -v google-chrome-beta >/dev/null 2>&1; then + wget -q -O /tmp/google-linux-signing-key.pub https://dl.google.com/linux/linux_signing_key.pub + ${SUDO} apt-key add /tmp/google-linux-signing-key.pub || true + echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" | ${SUDO} tee /etc/apt/sources.list.d/google-chrome.list >/dev/null + ${SUDO} apt-get update + ${SUDO} apt-get install -y google-chrome-beta +fi + +if ! command -v bun >/dev/null 2>&1; then + curl -fsSL https://bun.sh/install | bash + export PATH="${HOME}/.bun/bin:/root/.bun/bin:${PATH}" +fi + +mkdir -p logs .runtime-locks +bun install + +bun run build:shared +(cd packages/server && bun run build) +(cd packages/client && bun run build:cf) + +set -a +. "${REMOTE_ROOT}/.env.production" +set +a + +bunx pm2 delete hyperscape-duel-api hyperscape-stream-source 2>/dev/null || true + +display="${DISPLAY:-:99}" +display_num="${display#:}" +pkill -f "Xvfb ${display}" 2>/dev/null || true +pkill -f "ffmpeg .*${display}\\.0" 2>/dev/null || true +pkill -f "chrome.*${display}" 2>/dev/null || true +rm -f "/tmp/.X${display_num}-lock" "/tmp/.X11-unix/X${display_num}" +xvfb_log="${REMOTE_ROOT}/logs/xvfb.log" +nohup Xvfb "${display}" -screen 0 "${STREAM_CAPTURE_WIDTH:-1280}x${STREAM_CAPTURE_HEIGHT:-720}x24" -ac +extension GLX +render -noreset >"${xvfb_log}" 2>&1 & +for attempt in $(seq 1 10); do + if xdpyinfo -display "${display}" >/dev/null 2>&1; then + break + fi + sleep 1 +done +if ! xdpyinfo -display "${display}" >/dev/null 2>&1; then + echo "Xvfb did not become ready on ${display}" >&2 + cat "${xvfb_log}" >&2 || true + exit 1 +fi +ffmpeg -hide_banner -encoders 2>/dev/null > /tmp/hyperscape-ffmpeg-encoders.txt +grep -q h264_nvenc /tmp/hyperscape-ffmpeg-encoders.txt + +if [ "${ENOOMIAN_AWS_GPU_START_SOURCE:-0}" = "1" ]; then + bunx pm2 start ecosystem.config.cjs --update-env +else + bunx pm2 start ecosystem.config.cjs --only hyperscape-duel-api --update-env +fi +bunx pm2 save +bunx pm2 status +REMOTE_SCRIPT +} + +enoomian_aws_gpu_deploy() { + enoomian_check_personal_secrets + enoomian_aws_gpu_preflight + enoomian_aws_gpu_sync_code + enoomian_aws_gpu_push_runtime_env + enoomian_aws_gpu_remote_bootstrap + enoomian_log "AWS GPU source deploy complete; Cloudflare Pages and Railway were not changed" +} + +enoomian_aws_gpu_status() { + local remote_root + local api_port + local start_source + remote_root="$(enoomian_aws_gpu_remote_root)" + api_port="${ENOOMIAN_AWS_GPU_API_PORT:-5560}" + start_source="${ENOOMIAN_AWS_GPU_START_SOURCE:-0}" + enoomian_aws_gpu_preflight + enoomian_aws_gpu_ssh "REMOTE_ROOT='${remote_root}' API_PORT='${api_port}' START_SOURCE='${start_source}' bash -s" <<'REMOTE_SCRIPT' +set -euo pipefail + +cd "${REMOTE_ROOT}" +health_json="$(curl -fsS "http://127.0.0.1:${API_PORT}/health")" +status_tmp="$(mktemp)" +status_code="$(curl -sS -o "${status_tmp}" -w "%{http_code}" "http://127.0.0.1:${API_PORT}/api/streaming/capture/status" || true)" + +if [ "${status_code}" = "200" ]; then + jq \ + --argjson health "${health_json}" \ + --arg startSource "${START_SOURCE}" \ + '{ + apiHealth: $health, + sourcePublisherConfigured: ($startSource == "1"), + captureStatusCode: 200, + running, + rendererReady: .rendererHealth.ready, + sourceReady: .sourceRuntime.ready, + captureMode: .sourceRuntime.captureMode, + playbackUrl: (.delivery.playbackUrl // .cloudflare.playbackUrl // null), + workerHeartbeatAt: .sourceRuntime.workerHeartbeatAt + }' <"${status_tmp}" +else + jq -n \ + --argjson health "${health_json}" \ + --arg startSource "${START_SOURCE}" \ + --arg statusCode "${status_code}" \ + '{ + apiHealth: $health, + sourcePublisherConfigured: ($startSource == "1"), + captureStatusCode: ($statusCode | tonumber? // 0), + running: false, + rendererReady: false, + sourceReady: false, + captureMode: null, + playbackUrl: null, + workerHeartbeatAt: null + }' +fi + +rm -f "${status_tmp}" +REMOTE_SCRIPT +} + +enoomian_aws_gpu_activate() { + if [[ "${ENOOMIAN_AWS_GPU_ACTIVATE_PUBLIC_AUTHORITY:-0}" != "1" ]]; then + enoomian_die "refusing to activate AWS GPU authority without ENOOMIAN_AWS_GPU_ACTIVATE_PUBLIC_AUTHORITY=1" + return 1 + fi + enoomian_require_env ENOOMIAN_AWS_GPU_PUBLIC_API_URL ENOOMIAN_AWS_GPU_PUBLIC_WS_URL + + enoomian_log "activating AWS GPU authority for enoomian staging only" + export ENOOMIAN_HYPERSCAPES_API_URL="${ENOOMIAN_AWS_GPU_PUBLIC_API_URL%/}" + export ENOOMIAN_HYPERSCAPES_WS_URL="${ENOOMIAN_AWS_GPU_PUBLIC_WS_URL}" + export ENOOMIAN_HYPERSCAPES_PUBLIC_API_URL="${ENOOMIAN_HYPERSCAPES_API_URL}" + export ENOOMIAN_HYPERSCAPES_PUBLIC_WS_URL="${ENOOMIAN_HYPERSCAPES_WS_URL}" + export ENOOMIAN_STREAM_DELIVERY_MODE="${ENOOMIAN_AWS_GPU_STREAM_DELIVERY_MODE:-self_hls}" + export ENOOMIAN_STREAM_DELIVERY_PROVIDER="${ENOOMIAN_AWS_GPU_STREAM_DELIVERY_PROVIDER:-self_hls}" + export ENOOMIAN_STREAM_PLAYBACK_URL="${ENOOMIAN_AWS_GPU_PUBLIC_HLS_URL:-${ENOOMIAN_HYPERSCAPES_API_URL%/}/live/stream.m3u8}" + export ENOOMIAN_STREAM_PLAYBACK_LLHLS_URL="${ENOOMIAN_AWS_GPU_PUBLIC_LLHLS_URL:-}" + export ENOOMIAN_STREAM_CANONICAL_PROVIDER_PRIORITY="${ENOOMIAN_AWS_GPU_STREAM_CANONICAL_PROVIDER_PRIORITY:-self_hls,cloudflare_stream}" + export ENOOMIAN_STREAM_RENDERER_HEALTH_URL="${ENOOMIAN_AWS_GPU_PUBLIC_API_URL%/}/api/streaming/capture/status" + + if [[ "${ENOOMIAN_AWS_GPU_SYNC_HYPERSCAPES_RAILWAY_ENV:-0}" == "1" ]]; then + enoomian_sync_hyperscapes_railway_env + else + enoomian_log "skipping Hyperscapes Railway env sync for AWS GPU authority; public authority is AWS + Pages" + fi + enoomian_deploy_hyperscapes_pages + if [[ "${ENOOMIAN_AWS_GPU_SYNC_HYPERBET_KEEPER_ENV:-1}" == "1" ]]; then + enoomian_sync_hyperbet_keepers_env + else + enoomian_log "skipping Hyperbet keeper Railway env sync for AWS GPU authority by explicit override; Hyperbet app may keep the previous canonical session" + fi + if [[ "${ENOOMIAN_AWS_GPU_DEPLOY_HYPERBET_KEEPER:-1}" == "1" ]]; then + enoomian_deploy_hyperbet_keeper + else + enoomian_log "skipping Hyperbet keeper deploy for AWS GPU authority by explicit override; Railway --skip-deploys env changes may not be live" + fi + enoomian_deploy_hyperbet_pages + enoomian_log "AWS GPU authority activation complete for enoomian staging" +} diff --git a/scripts/enoomian-staging/deploy.sh b/scripts/enoomian-staging/deploy.sh new file mode 100755 index 00000000..2773f2c5 --- /dev/null +++ b/scripts/enoomian-staging/deploy.sh @@ -0,0 +1,1577 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=./lib.sh +source "${SCRIPT_DIR}/lib.sh" + +# Lock the deploy lane to `enoomian/staging` before touching anything +# branch-dependent. The secret manifest below is tracked on +# enoomian/staging only, so running from any other branch would +# otherwise surface as a confusing "No such file or directory" at +# the next source line. enoomian_require_enoomian_staging_branch +# respects ENOOMIAN_ALLOW_OFFBRANCH_DEPLOY=1 as an escape hatch; see +# lib.sh for the helper definition. +enoomian_require_enoomian_staging_branch + +# shellcheck source=../enoomian-staging-secret-manifest.sh +source "${SCRIPT_DIR}/../enoomian-staging-secret-manifest.sh" +# shellcheck source=./aws-gpu.sh +source "${SCRIPT_DIR}/aws-gpu.sh" + +enoomian_require_dist_dir() { + local dir="$1" + local label="$2" + if [[ ! -d "${dir}" ]]; then + enoomian_die "${label} dist directory not found: ${dir}" + fi +} + +enoomian_print_manifest_section() { + local title="$1" + shift + + printf '%s\n' "${title}:" + local key + for key in "$@"; do + printf ' %s\n' "${key}" + done +} + +enoomian_print_personal_secret_manifest() { + cat </dev/null 2>&1; then + commit_sha="$(git -C "${repo_root}" rev-parse HEAD)" + else + commit_sha="local-enoomian-$(date +%s)" + fi + + "${sink}" "${workdir}" "${service_id}" plain NODE_ENV production + "${sink}" "${workdir}" "${service_id}" plain USE_LOCAL_POSTGRES false + "${sink}" "${workdir}" "${service_id}" plain STREAMING_DUEL_ENABLED true + "${sink}" "${workdir}" "${service_id}" plain STREAMING_LOCAL_CAPTURE_WORKER_ENABLED false + "${sink}" "${workdir}" "${service_id}" plain STREAMING_LOCAL_CAPTURE_USE_PUBLIC_URL false + "${sink}" "${workdir}" "${service_id}" plain STREAMING_CAPTURE_ENABLED false + "${sink}" "${workdir}" "${service_id}" plain DISABLE_RATE_LIMIT true + "${sink}" "${workdir}" "${service_id}" plain UWS_ENABLED false + "${sink}" "${workdir}" "${service_id}" plain HEALTH_CHECK_DATABASE false + "${sink}" "${workdir}" "${service_id}" plain SKIP_MIGRATIONS true + "${sink}" "${workdir}" "${service_id}" plain SKIP_CDN_MANIFEST_FETCH true + "${sink}" "${workdir}" "${service_id}" plain GAME_URL "${public_stream_url}" + "${sink}" "${workdir}" "${service_id}" plain GAME_FALLBACK_URLS "${public_legacy_stream_url},${public_stream_html_url}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_CAPTURE_MODE cdp + "${sink}" "${workdir}" "${service_id}" plain STREAM_CAPTURE_HEADLESS false + "${sink}" "${workdir}" "${service_id}" plain HLS_OUTPUT_PATH "" + "${sink}" "${workdir}" "${service_id}" plain HLS_SEGMENT_PATTERN "" + "${sink}" "${workdir}" "${service_id}" plain HLS_TIME_SECONDS "${hls_time_seconds}" + "${sink}" "${workdir}" "${service_id}" plain HLS_LIST_SIZE "${hls_list_size}" + "${sink}" "${workdir}" "${service_id}" plain HLS_DELETE_THRESHOLD "${hls_delete_threshold}" + "${sink}" "${workdir}" "${service_id}" plain HLS_ALLOW_CACHE 0 + "${sink}" "${workdir}" "${service_id}" plain STREAM_LOW_LATENCY "${stream_low_latency}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_FPS "${stream_fps}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_OUTPUT_WIDTH "${stream_output_width}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_OUTPUT_HEIGHT "${stream_output_height}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_VIDEO_BITRATE_KBPS "${stream_video_bitrate_kbps}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_AUDIO_BITRATE_KBPS "${stream_audio_bitrate_kbps}" + "${sink}" "${workdir}" "${service_id}" plain RTMP_STATUS_FILE "" + "${sink}" "${workdir}" "${service_id}" plain STREAM_DELIVERY_MODE "${stream_delivery_mode}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_DELIVERY_PROVIDER "${stream_delivery_provider}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_PLAYBACK_HLS_URL "${stream_playback_hls_url}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_PLAYBACK_LLHLS_URL "${stream_playback_llhls_url}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_CANONICAL_PROVIDER_PRIORITY "${stream_canonical_provider_priority}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_ENABLE_AUTOMATIC_FAILOVER "${stream_enable_automatic_failover}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_FAILBACK_SOAK_MS "${stream_failback_soak_ms}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_EXTERNAL_DELIVERY_PROVIDER "${stream_external_delivery_provider}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_EXTERNAL_PLAYBACK_HLS_URL "${stream_external_playback_hls_url}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_EXTERNAL_PLAYBACK_LLHLS_URL "${stream_external_playback_llhls_url}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_EXTERNAL_INGEST_RTMPS_URL "${stream_external_ingest_rtmps_url}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_INGEST_RTMPS_URL "${stream_ingest_rtmps_url}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_INGEST_PROFILE "${stream_ingest_profile}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_INGEST_TRANSPORT "${stream_ingest_transport}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_INGEST_SRT_URL "${stream_ingest_srt_url}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_INGEST_SRT_STREAM_ID "${stream_ingest_srt_stream_id}" + "${sink}" "${workdir}" "${service_id}" stdin STREAM_INGEST_SRT_PASSPHRASE "${stream_ingest_srt_passphrase}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_AUDIO_SAMPLE_RATE "${stream_audio_sample_rate}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_GOP_SIZE "${stream_gop_size}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_CLOUDFLARE_PROBE_ONLY "${stream_cloudflare_probe_only}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_CLOUDFLARE_LIVE_INPUT_ID "${stream_cloudflare_live_input_id}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_CLOUDFLARE_ACCOUNT_ID "${stream_cloudflare_account_id}" + if [[ -n "${stream_cloudflare_webhook_secret}" ]]; then + "${sink}" "${workdir}" "${service_id}" stdin STREAM_CLOUDFLARE_WEBHOOK_SECRET "${stream_cloudflare_webhook_secret}" + fi + "${sink}" "${workdir}" "${service_id}" plain PUBLIC_API_URL "${ENOOMIAN_HYPERSCAPES_API_URL}" + "${sink}" "${workdir}" "${service_id}" plain PUBLIC_WS_URL "${ENOOMIAN_HYPERSCAPES_WS_URL}" + "${sink}" "${workdir}" "${service_id}" plain PUBLIC_APP_URL "${ENOOMIAN_HYPERSCAPES_PAGES_URL}" + "${sink}" "${workdir}" "${service_id}" plain PUBLIC_CDN_URL "${ENOOMIAN_PUBLIC_CDN_URL:-${ENOOMIAN_HYPERSCAPES_CDN_URL:-${ENOOMIAN_HYPERSCAPES_API_URL%/}/game-assets}}" + "${sink}" "${workdir}" "${service_id}" plain PUBLIC_ASSETS_URL "${ENOOMIAN_PUBLIC_CDN_URL:-${ENOOMIAN_HYPERSCAPES_CDN_URL:-${ENOOMIAN_HYPERSCAPES_API_URL%/}/game-assets}}" + if [[ -n "${ENOOMIAN_HYPERSCAPES_GAME_ASSETS_FALLBACK_URL:-}" ]]; then + "${sink}" "${workdir}" "${service_id}" plain GAME_ASSETS_FALLBACK_URL "${ENOOMIAN_HYPERSCAPES_GAME_ASSETS_FALLBACK_URL}" + else + "${sink}" "${workdir}" "${service_id}" plain GAME_ASSETS_FALLBACK_URL "" + fi + "${sink}" "${workdir}" "${service_id}" plain PUBLIC_ASSET_VERSION "${commit_sha}" + "${sink}" "${workdir}" "${service_id}" stdin DATABASE_URL "${ENOOMIAN_DATABASE_URL}" + "${sink}" "${workdir}" "${service_id}" stdin JWT_SECRET "${ENOOMIAN_HYPERSCAPES_JWT_SECRET}" + if [[ -n "${stream_ingest_stream_key}" ]]; then + "${sink}" "${workdir}" "${service_id}" stdin STREAM_INGEST_STREAM_KEY "${stream_ingest_stream_key}" + else + "${sink}" "${workdir}" "${service_id}" plain STREAM_INGEST_STREAM_KEY "" + fi + if [[ -n "${ENOOMIAN_PUBLIC_PRIVY_APP_ID:-}" ]]; then + "${sink}" "${workdir}" "${service_id}" stdin PUBLIC_PRIVY_APP_ID "${ENOOMIAN_PUBLIC_PRIVY_APP_ID}" + fi + if [[ -n "${ENOOMIAN_PRIVY_APP_SECRET:-}" ]]; then + "${sink}" "${workdir}" "${service_id}" stdin PRIVY_APP_SECRET "${ENOOMIAN_PRIVY_APP_SECRET}" + fi + if [[ -n "${ENOOMIAN_BETTING_FEED_ACCESS_TOKEN:-}" ]]; then + "${sink}" "${workdir}" "${service_id}" stdin BETTING_FEED_ACCESS_TOKEN "${ENOOMIAN_BETTING_FEED_ACCESS_TOKEN}" + fi + if [[ -n "${ENOOMIAN_STREAMING_VIEWER_ACCESS_TOKEN:-}" ]]; then + "${sink}" "${workdir}" "${service_id}" stdin STREAMING_VIEWER_ACCESS_TOKEN "${ENOOMIAN_STREAMING_VIEWER_ACCESS_TOKEN}" + fi + if [[ -n "${ENOOMIAN_STREAMING_PUBLIC_DELAY_MS:-}" ]]; then + "${sink}" "${workdir}" "${service_id}" plain STREAMING_PUBLIC_DELAY_MS "${ENOOMIAN_STREAMING_PUBLIC_DELAY_MS}" + fi + "${sink}" "${workdir}" "${service_id}" plain STREAMING_EMIT_RAW_SOURCE_TIME "${ENOOMIAN_STREAMING_EMIT_RAW_SOURCE_TIME:-false}" + "${sink}" "${workdir}" "${service_id}" stdin STREAMING_AUTOMATION_TOKEN "${ENOOMIAN_HYPERSCAPES_AUTOMATION_TOKEN}" +} + +enoomian_sync_hyperscapes_railway_env() { + enoomian_require_hyperscapes_root + enoomian_require_env \ + ENOOMIAN_RAILWAY_PROJECT_ID \ + ENOOMIAN_RAILWAY_ENVIRONMENT_ID \ + ENOOMIAN_HYPERSCAPES_SERVICE_ID \ + ENOOMIAN_HYPERSCAPES_API_URL \ + ENOOMIAN_HYPERSCAPES_WS_URL \ + ENOOMIAN_HYPERSCAPES_PAGES_URL \ + ENOOMIAN_DATABASE_URL \ + ENOOMIAN_HYPERSCAPES_JWT_SECRET \ + ENOOMIAN_HYPERSCAPES_AUTOMATION_TOKEN + + enoomian_log "syncing Hyperscapes Railway service variables" + enoomian_emit_hyperscapes_railway_env_manifest \ + enoomian_apply_railway_manifest_entry \ + "${ENOOMIAN_HYPERSCAPES_ROOT}" \ + "${ENOOMIAN_HYPERSCAPES_SERVICE_ID}" +} + +enoomian_emit_hyperbet_solana_keeper_env_manifest() { + local sink="$1" + local workdir="$2" + local service_id="$3" + local stream_playback_url="${ENOOMIAN_STREAM_PLAYBACK_URL:-${ENOOMIAN_HYPERSCAPES_API_URL%/}/live/stream.m3u8}" + local stream_playback_llhls_url="${ENOOMIAN_STREAM_PLAYBACK_LLHLS_URL:-}" + local stream_delivery_mode="${ENOOMIAN_STREAM_DELIVERY_MODE:-self_hls}" + local stream_delivery_provider="${ENOOMIAN_STREAM_DELIVERY_PROVIDER:-}" + local stream_renderer_health_url="${ENOOMIAN_STREAM_RENDERER_HEALTH_URL:-}" + local stream_renderer_health_bearer_token="${ENOOMIAN_STREAM_RENDERER_HEALTH_BEARER_TOKEN:-}" + local stream_renderer_health_poll_ms="${ENOOMIAN_STREAM_RENDERER_HEALTH_POLL_MS:-2000}" + local stream_renderer_hls_freshness_ms="${ENOOMIAN_STREAM_RENDERER_HLS_FRESHNESS_MS:-15000}" + local stream_ingest_rtmps_url="${ENOOMIAN_STREAM_INGEST_RTMPS_URL:-}" + local oracle_config_authority_keypair + local clob_config_authority_keypair + local oracle_authority_keypair + local bot_keypair + + oracle_config_authority_keypair="$(enoomian_normalize_keypair_ref "${ENOOMIAN_HYPERBET_SOLANA_ORACLE_CONFIG_AUTHORITY_KEYPAIR}")" + clob_config_authority_keypair="$(enoomian_normalize_keypair_ref "${ENOOMIAN_HYPERBET_SOLANA_CLOB_CONFIG_AUTHORITY_KEYPAIR}")" + oracle_authority_keypair="$(enoomian_normalize_keypair_ref "${ENOOMIAN_HYPERBET_SOLANA_ORACLE_AUTHORITY_KEYPAIR}")" + bot_keypair="$(enoomian_normalize_keypair_ref "${ENOOMIAN_HYPERBET_SOLANA_CANARY_KEYPAIR}")" + + "${sink}" "${workdir}" "${service_id}" plain NODE_ENV production + "${sink}" "${workdir}" "${service_id}" plain ENABLE_KEEPER_BOT true + "${sink}" "${workdir}" "${service_id}" plain CORS_ORIGINS "$(enoomian_hyperbet_pages_cors_origins)" + "${sink}" "${workdir}" "${service_id}" plain BET_SYNC_SOURCE_EVENTS_URL "${ENOOMIAN_HYPERSCAPES_API_URL}/api/internal/bet-sync/events" + "${sink}" "${workdir}" "${service_id}" plain BET_SYNC_SOURCE_STATE_URL "${ENOOMIAN_HYPERSCAPES_API_URL}/api/internal/bet-sync/state" + "${sink}" "${workdir}" "${service_id}" plain STREAM_STATE_SOURCE_URL "${ENOOMIAN_HYPERSCAPES_API_URL}/api/streaming/state" + if [[ -n "${ENOOMIAN_BETTING_FEED_ACCESS_TOKEN:-}" ]]; then + "${sink}" "${workdir}" "${service_id}" stdin BET_SYNC_SOURCE_BEARER_TOKEN "${ENOOMIAN_BETTING_FEED_ACCESS_TOKEN}" + "${sink}" "${workdir}" "${service_id}" stdin STREAM_STATE_SOURCE_BEARER_TOKEN "${ENOOMIAN_BETTING_FEED_ACCESS_TOKEN}" + fi + "${sink}" "${workdir}" "${service_id}" plain STREAM_PLAYBACK_KIND hls + "${sink}" "${workdir}" "${service_id}" plain STREAM_PLAYBACK_URL "${stream_playback_url}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_DELIVERY_MODE "${stream_delivery_mode}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_DELIVERY_PROVIDER "${stream_delivery_provider}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_PLAYBACK_HLS_URL "${stream_playback_url}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_PLAYBACK_LLHLS_URL "${stream_playback_llhls_url}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_INGEST_RTMPS_URL "${stream_ingest_rtmps_url}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_RENDERER_HEALTH_URL "${stream_renderer_health_url}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_RENDERER_HEALTH_POLL_MS "${stream_renderer_health_poll_ms}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_RENDERER_HLS_FRESHNESS_MS "${stream_renderer_hls_freshness_ms}" + if [[ -n "${ENOOMIAN_STREAMING_PUBLIC_DELAY_MS:-}" ]]; then + "${sink}" "${workdir}" "${service_id}" plain STREAM_PRESENTATION_DELAY_MS "${ENOOMIAN_STREAMING_PUBLIC_DELAY_MS}" + fi + if [[ -n "${stream_renderer_health_bearer_token}" ]]; then + "${sink}" "${workdir}" "${service_id}" stdin STREAM_RENDERER_HEALTH_BEARER_TOKEN "${stream_renderer_health_bearer_token}" + else + "${sink}" "${workdir}" "${service_id}" plain STREAM_RENDERER_HEALTH_BEARER_TOKEN "" + fi + "${sink}" "${workdir}" "${service_id}" plain SOLANA_CLUSTER "${ENOOMIAN_SOLANA_CLUSTER}" + "${sink}" "${workdir}" "${service_id}" plain SOLANA_RPC_URL "${ENOOMIAN_SOLANA_RPC_URL}" + "${sink}" "${workdir}" "${service_id}" plain SOLANA_ORACLE_DISPUTE_WINDOW_SECS "${ENOOMIAN_SOLANA_ORACLE_DISPUTE_WINDOW_SECS}" + "${sink}" "${workdir}" "${service_id}" plain FIGHT_ORACLE_PROGRAM_ID "${ENOOMIAN_SOLANA_FIGHT_ORACLE_PROGRAM_ID}" + "${sink}" "${workdir}" "${service_id}" plain GOLD_CLOB_MARKET_PROGRAM_ID "${ENOOMIAN_SOLANA_GOLD_CLOB_PROGRAM_ID}" + "${sink}" "${workdir}" "${service_id}" plain GOLD_AMM_MARKET_PROGRAM_ID "${ENOOMIAN_SOLANA_GOLD_AMM_PROGRAM_ID}" + "${sink}" "${workdir}" "${service_id}" plain GOLD_PERPS_MARKET_PROGRAM_ID "${ENOOMIAN_SOLANA_GOLD_PERPS_PROGRAM_ID}" + "${sink}" "${workdir}" "${service_id}" stdin STREAM_PUBLISH_KEY "${ENOOMIAN_HYPERBET_SOLANA_STREAM_PUBLISH_KEY}" + "${sink}" "${workdir}" "${service_id}" stdin ARENA_EXTERNAL_BET_WRITE_KEY "${ENOOMIAN_HYPERBET_SOLANA_STREAM_PUBLISH_KEY}" + "${sink}" "${workdir}" "${service_id}" stdin ORACLE_CONFIG_AUTHORITY_KEYPAIR "${oracle_config_authority_keypair}" + "${sink}" "${workdir}" "${service_id}" stdin CLOB_CONFIG_AUTHORITY_KEYPAIR "${clob_config_authority_keypair}" + "${sink}" "${workdir}" "${service_id}" stdin ORACLE_AUTHORITY_KEYPAIR "${oracle_authority_keypair}" + "${sink}" "${workdir}" "${service_id}" stdin ORACLE_REPORTER_KEYPAIR "${oracle_authority_keypair}" + "${sink}" "${workdir}" "${service_id}" stdin ORACLE_FINALIZER_KEYPAIR "${oracle_authority_keypair}" + "${sink}" "${workdir}" "${service_id}" stdin ORACLE_CHALLENGER_KEYPAIR "${oracle_authority_keypair}" + "${sink}" "${workdir}" "${service_id}" stdin CLOB_MARKET_OPERATOR_KEYPAIR "${oracle_authority_keypair}" + "${sink}" "${workdir}" "${service_id}" stdin BOT_KEYPAIR "${bot_keypair}" +} + +enoomian_sync_hyperbet_solana_keeper_env() { + enoomian_require_env \ + ENOOMIAN_RAILWAY_PROJECT_ID \ + ENOOMIAN_RAILWAY_ENVIRONMENT_ID \ + ENOOMIAN_HYPERBET_SOLANA_KEEPER_SERVICE_ID \ + ENOOMIAN_HYPERBET_SOLANA_KEEPER_URL \ + ENOOMIAN_HYPERBET_PAGES_URL \ + ENOOMIAN_HYPERSCAPES_API_URL \ + ENOOMIAN_SOLANA_CLUSTER \ + ENOOMIAN_SOLANA_RPC_URL \ + ENOOMIAN_SOLANA_ORACLE_DISPUTE_WINDOW_SECS \ + ENOOMIAN_SOLANA_FIGHT_ORACLE_PROGRAM_ID \ + ENOOMIAN_SOLANA_GOLD_CLOB_PROGRAM_ID \ + ENOOMIAN_SOLANA_GOLD_AMM_PROGRAM_ID \ + ENOOMIAN_SOLANA_GOLD_PERPS_PROGRAM_ID \ + ENOOMIAN_HYPERBET_SOLANA_STREAM_PUBLISH_KEY \ + ENOOMIAN_HYPERBET_SOLANA_ORACLE_CONFIG_AUTHORITY_KEYPAIR \ + ENOOMIAN_HYPERBET_SOLANA_CLOB_CONFIG_AUTHORITY_KEYPAIR \ + ENOOMIAN_HYPERBET_SOLANA_ORACLE_AUTHORITY_KEYPAIR \ + ENOOMIAN_HYPERBET_SOLANA_CANARY_KEYPAIR + + enoomian_log "syncing Hyperbet Solana keeper Railway variables" + enoomian_emit_hyperbet_solana_keeper_env_manifest \ + enoomian_apply_railway_manifest_entry \ + "${ENOOMIAN_REPO_ROOT}/packages/hyperbet-solana/keeper" \ + "${ENOOMIAN_HYPERBET_SOLANA_KEEPER_SERVICE_ID}" +} + +enoomian_emit_hyperbet_bsc_keeper_env_manifest() { + local sink="$1" + local workdir="$2" + local service_id="$3" + local stream_playback_url="${ENOOMIAN_STREAM_PLAYBACK_URL:-${ENOOMIAN_HYPERSCAPES_API_URL%/}/live/stream.m3u8}" + local stream_playback_llhls_url="${ENOOMIAN_STREAM_PLAYBACK_LLHLS_URL:-}" + local stream_delivery_mode="${ENOOMIAN_STREAM_DELIVERY_MODE:-self_hls}" + local stream_delivery_provider="${ENOOMIAN_STREAM_DELIVERY_PROVIDER:-}" + local stream_renderer_health_url="${ENOOMIAN_STREAM_RENDERER_HEALTH_URL:-}" + local stream_renderer_health_bearer_token="${ENOOMIAN_STREAM_RENDERER_HEALTH_BEARER_TOKEN:-}" + local stream_renderer_health_poll_ms="${ENOOMIAN_STREAM_RENDERER_HEALTH_POLL_MS:-2000}" + local stream_renderer_hls_freshness_ms="${ENOOMIAN_STREAM_RENDERER_HLS_FRESHNESS_MS:-15000}" + local stream_ingest_rtmps_url="${ENOOMIAN_STREAM_INGEST_RTMPS_URL:-}" + local oracle_authority_keypair + local evm_keeper_chains + local base_rpc_url="" + local base_duel_oracle_address="" + local base_gold_clob_address="" + + oracle_authority_keypair="$(enoomian_normalize_keypair_ref "${ENOOMIAN_HYPERBET_SOLANA_ORACLE_AUTHORITY_KEYPAIR}")" + evm_keeper_chains="$(enoomian_resolve_bsc_keeper_chains)" + if enoomian_base_lane_enabled; then + base_rpc_url="${ENOOMIAN_BASE_RPC_URL}" + base_duel_oracle_address="${ENOOMIAN_BASE_DUEL_ORACLE_ADDRESS}" + base_gold_clob_address="${ENOOMIAN_BASE_GOLD_CLOB_ADDRESS}" + fi + + "${sink}" "${workdir}" "${service_id}" plain NODE_ENV production + "${sink}" "${workdir}" "${service_id}" plain ENABLE_KEEPER_BOT true + "${sink}" "${workdir}" "${service_id}" plain SOLANA_CLUSTER "${ENOOMIAN_SOLANA_CLUSTER}" + "${sink}" "${workdir}" "${service_id}" plain EVM_KEEPER_CHAINS "${evm_keeper_chains}" + "${sink}" "${workdir}" "${service_id}" plain CORS_ORIGINS "$(enoomian_hyperbet_pages_cors_origins)" + "${sink}" "${workdir}" "${service_id}" plain BET_SYNC_SOURCE_EVENTS_URL "${ENOOMIAN_HYPERSCAPES_API_URL}/api/internal/bet-sync/events" + "${sink}" "${workdir}" "${service_id}" plain BET_SYNC_SOURCE_STATE_URL "${ENOOMIAN_HYPERSCAPES_API_URL}/api/internal/bet-sync/state" + "${sink}" "${workdir}" "${service_id}" plain STREAM_STATE_SOURCE_URL "${ENOOMIAN_HYPERSCAPES_API_URL}/api/streaming/state" + if [[ -n "${ENOOMIAN_BETTING_FEED_ACCESS_TOKEN:-}" ]]; then + "${sink}" "${workdir}" "${service_id}" stdin BET_SYNC_SOURCE_BEARER_TOKEN "${ENOOMIAN_BETTING_FEED_ACCESS_TOKEN}" + "${sink}" "${workdir}" "${service_id}" stdin STREAM_STATE_SOURCE_BEARER_TOKEN "${ENOOMIAN_BETTING_FEED_ACCESS_TOKEN}" + fi + "${sink}" "${workdir}" "${service_id}" plain STREAM_PLAYBACK_KIND hls + "${sink}" "${workdir}" "${service_id}" plain STREAM_PLAYBACK_URL "${stream_playback_url}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_DELIVERY_MODE "${stream_delivery_mode}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_DELIVERY_PROVIDER "${stream_delivery_provider}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_PLAYBACK_HLS_URL "${stream_playback_url}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_PLAYBACK_LLHLS_URL "${stream_playback_llhls_url}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_INGEST_RTMPS_URL "${stream_ingest_rtmps_url}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_RENDERER_HEALTH_URL "${stream_renderer_health_url}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_RENDERER_HEALTH_POLL_MS "${stream_renderer_health_poll_ms}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_RENDERER_HLS_FRESHNESS_MS "${stream_renderer_hls_freshness_ms}" + if [[ -n "${ENOOMIAN_STREAMING_PUBLIC_DELAY_MS:-}" ]]; then + "${sink}" "${workdir}" "${service_id}" plain STREAM_PRESENTATION_DELAY_MS "${ENOOMIAN_STREAMING_PUBLIC_DELAY_MS}" + fi + if [[ -n "${stream_renderer_health_bearer_token}" ]]; then + "${sink}" "${workdir}" "${service_id}" stdin STREAM_RENDERER_HEALTH_BEARER_TOKEN "${stream_renderer_health_bearer_token}" + else + "${sink}" "${workdir}" "${service_id}" plain STREAM_RENDERER_HEALTH_BEARER_TOKEN "" + fi + "${sink}" "${workdir}" "${service_id}" plain SOLANA_RPC_URL "${ENOOMIAN_SOLANA_RPC_URL}" + "${sink}" "${workdir}" "${service_id}" plain SOLANA_ORACLE_DISPUTE_WINDOW_SECS "${ENOOMIAN_SOLANA_ORACLE_DISPUTE_WINDOW_SECS}" + "${sink}" "${workdir}" "${service_id}" plain FIGHT_ORACLE_PROGRAM_ID "${ENOOMIAN_SOLANA_FIGHT_ORACLE_PROGRAM_ID}" + "${sink}" "${workdir}" "${service_id}" plain GOLD_CLOB_MARKET_PROGRAM_ID "${ENOOMIAN_SOLANA_GOLD_CLOB_PROGRAM_ID}" + "${sink}" "${workdir}" "${service_id}" plain BSC_RPC_URL "${ENOOMIAN_BSC_RPC_URL}" + "${sink}" "${workdir}" "${service_id}" plain BSC_DUEL_ORACLE_ADDRESS "${ENOOMIAN_BSC_DUEL_ORACLE_ADDRESS}" + "${sink}" "${workdir}" "${service_id}" plain BSC_GOLD_CLOB_ADDRESS "${ENOOMIAN_BSC_GOLD_CLOB_ADDRESS}" + "${sink}" "${workdir}" "${service_id}" plain BASE_RPC_URL "${base_rpc_url}" + "${sink}" "${workdir}" "${service_id}" plain BASE_DUEL_ORACLE_ADDRESS "${base_duel_oracle_address}" + "${sink}" "${workdir}" "${service_id}" plain BASE_GOLD_CLOB_ADDRESS "${base_gold_clob_address}" + "${sink}" "${workdir}" "${service_id}" stdin STREAM_PUBLISH_KEY "${ENOOMIAN_HYPERBET_BSC_STREAM_PUBLISH_KEY}" + "${sink}" "${workdir}" "${service_id}" stdin ARENA_EXTERNAL_BET_WRITE_KEY "${ENOOMIAN_HYPERBET_BSC_STREAM_PUBLISH_KEY}" + "${sink}" "${workdir}" "${service_id}" stdin ORACLE_AUTHORITY_KEYPAIR "${oracle_authority_keypair}" +} + +enoomian_sync_hyperbet_bsc_keeper_env() { + enoomian_require_env \ + ENOOMIAN_RAILWAY_PROJECT_ID \ + ENOOMIAN_RAILWAY_ENVIRONMENT_ID \ + ENOOMIAN_HYPERBET_BSC_KEEPER_SERVICE_ID \ + ENOOMIAN_HYPERBET_BSC_KEEPER_URL \ + ENOOMIAN_HYPERBET_PAGES_URL \ + ENOOMIAN_HYPERSCAPES_API_URL \ + ENOOMIAN_SOLANA_RPC_URL \ + ENOOMIAN_SOLANA_ORACLE_DISPUTE_WINDOW_SECS \ + ENOOMIAN_SOLANA_FIGHT_ORACLE_PROGRAM_ID \ + ENOOMIAN_SOLANA_GOLD_CLOB_PROGRAM_ID \ + ENOOMIAN_BSC_RPC_URL \ + ENOOMIAN_BSC_DUEL_ORACLE_ADDRESS \ + ENOOMIAN_BSC_GOLD_CLOB_ADDRESS \ + ENOOMIAN_HYPERBET_BSC_STREAM_PUBLISH_KEY \ + ENOOMIAN_HYPERBET_SOLANA_ORACLE_AUTHORITY_KEYPAIR + + enoomian_log "syncing Hyperbet BSC keeper Railway variables" + enoomian_emit_hyperbet_bsc_keeper_env_manifest \ + enoomian_apply_railway_manifest_entry \ + "${ENOOMIAN_REPO_ROOT}/packages/hyperbet-bsc/keeper" \ + "${ENOOMIAN_HYPERBET_BSC_KEEPER_SERVICE_ID}" +} + +enoomian_emit_hyperbet_keeper_env_manifest() { + local sink="$1" + local workdir="$2" + local service_id="$3" + local stream_playback_url="${ENOOMIAN_STREAM_PLAYBACK_URL:-${ENOOMIAN_HYPERSCAPES_API_URL%/}/live/stream.m3u8}" + local stream_playback_llhls_url="${ENOOMIAN_STREAM_PLAYBACK_LLHLS_URL:-}" + local stream_delivery_mode="${ENOOMIAN_STREAM_DELIVERY_MODE:-self_hls}" + local stream_delivery_provider="${ENOOMIAN_STREAM_DELIVERY_PROVIDER:-}" + local stream_renderer_health_url="${ENOOMIAN_STREAM_RENDERER_HEALTH_URL:-}" + local stream_renderer_health_bearer_token="${ENOOMIAN_STREAM_RENDERER_HEALTH_BEARER_TOKEN:-}" + local stream_renderer_health_poll_ms="${ENOOMIAN_STREAM_RENDERER_HEALTH_POLL_MS:-2000}" + local stream_renderer_hls_freshness_ms="${ENOOMIAN_STREAM_RENDERER_HLS_FRESHNESS_MS:-15000}" + local stream_ingest_rtmps_url="${ENOOMIAN_STREAM_INGEST_RTMPS_URL:-}" + local evm_keeper_chains + local keeper_pages_url + local keeper_pages_origins + local keeper_url + local keeper_ws_url + local oracle_config_authority_keypair + local clob_config_authority_keypair + local oracle_authority_keypair + local bot_keypair + local market_maker_keypair + local bsc_finalizer_private_key + + keeper_pages_url="$(enoomian_hyperbet_pages_url)" + keeper_pages_origins="$(enoomian_hyperbet_pages_cors_origins)" + keeper_url="$(enoomian_hyperbet_keeper_url)" + keeper_ws_url="$(enoomian_hyperbet_keeper_ws_url)" + oracle_config_authority_keypair="$(enoomian_normalize_keypair_ref "${ENOOMIAN_HYPERBET_SOLANA_ORACLE_CONFIG_AUTHORITY_KEYPAIR}")" + clob_config_authority_keypair="$(enoomian_normalize_keypair_ref "${ENOOMIAN_HYPERBET_SOLANA_CLOB_CONFIG_AUTHORITY_KEYPAIR}")" + oracle_authority_keypair="$(enoomian_normalize_keypair_ref "${ENOOMIAN_HYPERBET_SOLANA_ORACLE_AUTHORITY_KEYPAIR}")" + bot_keypair="$(enoomian_normalize_keypair_ref "${ENOOMIAN_HYPERBET_SOLANA_CANARY_KEYPAIR}")" + market_maker_keypair="$(enoomian_normalize_keypair_ref "${ENOOMIAN_HYPERBET_SOLANA_MARKET_MAKER_KEYPAIR}")" + bsc_finalizer_private_key="${ENOOMIAN_HYPERBET_BSC_FINALIZER_PRIVATE_KEY:-${TESTNET_FINALIZER_PRIVATE_KEY:-}}" + [[ -n "${bsc_finalizer_private_key}" ]] || enoomian_die "missing required env: ENOOMIAN_HYPERBET_BSC_FINALIZER_PRIVATE_KEY or TESTNET_FINALIZER_PRIVATE_KEY" + evm_keeper_chains="bsc" + + "${sink}" "${workdir}" "${service_id}" plain NODE_ENV production + "${sink}" "${workdir}" "${service_id}" plain ENABLE_KEEPER_BOT true + "${sink}" "${workdir}" "${service_id}" plain EVM_KEEPER_CHAINS "${evm_keeper_chains}" + "${sink}" "${workdir}" "${service_id}" plain CORS_ORIGINS "${keeper_pages_origins}" + "${sink}" "${workdir}" "${service_id}" plain HYPERBET_KEEPER_URL "${keeper_url}" + "${sink}" "${workdir}" "${service_id}" plain HYPERBET_KEEPER_WS_URL "${keeper_ws_url}" + "${sink}" "${workdir}" "${service_id}" plain HYPERBET_SOLANA_KEEPER_URL "${ENOOMIAN_HYPERBET_SOLANA_KEEPER_URL}" + "${sink}" "${workdir}" "${service_id}" plain GAME_URL "${ENOOMIAN_HYPERSCAPES_API_URL}" + "${sink}" "${workdir}" "${service_id}" plain BET_SYNC_SOURCE_EVENTS_URL "${ENOOMIAN_HYPERSCAPES_API_URL}/api/internal/bet-sync/events" + "${sink}" "${workdir}" "${service_id}" plain BET_SYNC_SOURCE_STATE_URL "${ENOOMIAN_HYPERSCAPES_API_URL}/api/internal/bet-sync/state" + "${sink}" "${workdir}" "${service_id}" plain STREAM_STATE_SOURCE_URL "${ENOOMIAN_HYPERSCAPES_API_URL}/api/streaming/state" + if [[ -n "${ENOOMIAN_BETTING_FEED_ACCESS_TOKEN:-}" ]]; then + "${sink}" "${workdir}" "${service_id}" stdin BET_SYNC_SOURCE_BEARER_TOKEN "${ENOOMIAN_BETTING_FEED_ACCESS_TOKEN}" + "${sink}" "${workdir}" "${service_id}" stdin STREAM_STATE_SOURCE_BEARER_TOKEN "${ENOOMIAN_BETTING_FEED_ACCESS_TOKEN}" + fi + "${sink}" "${workdir}" "${service_id}" plain STREAM_PLAYBACK_KIND hls + "${sink}" "${workdir}" "${service_id}" plain STREAM_PLAYBACK_URL "${stream_playback_url}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_DELIVERY_MODE "${stream_delivery_mode}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_DELIVERY_PROVIDER "${stream_delivery_provider}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_PLAYBACK_HLS_URL "${stream_playback_url}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_PLAYBACK_LLHLS_URL "${stream_playback_llhls_url}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_INGEST_RTMPS_URL "${stream_ingest_rtmps_url}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_RENDERER_HEALTH_URL "${stream_renderer_health_url}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_RENDERER_HEALTH_POLL_MS "${stream_renderer_health_poll_ms}" + "${sink}" "${workdir}" "${service_id}" plain STREAM_RENDERER_HLS_FRESHNESS_MS "${stream_renderer_hls_freshness_ms}" + if [[ -n "${ENOOMIAN_STREAMING_PUBLIC_DELAY_MS:-}" ]]; then + "${sink}" "${workdir}" "${service_id}" plain STREAM_PRESENTATION_DELAY_MS "${ENOOMIAN_STREAMING_PUBLIC_DELAY_MS}" + fi + if [[ -n "${stream_renderer_health_bearer_token}" ]]; then + "${sink}" "${workdir}" "${service_id}" stdin STREAM_RENDERER_HEALTH_BEARER_TOKEN "${stream_renderer_health_bearer_token}" + else + "${sink}" "${workdir}" "${service_id}" plain STREAM_RENDERER_HEALTH_BEARER_TOKEN "" + fi + "${sink}" "${workdir}" "${service_id}" plain SOLANA_CLUSTER "${ENOOMIAN_SOLANA_CLUSTER}" + "${sink}" "${workdir}" "${service_id}" plain SOLANA_RPC_URL "${ENOOMIAN_SOLANA_RPC_URL}" + "${sink}" "${workdir}" "${service_id}" plain SOLANA_ORACLE_DISPUTE_WINDOW_SECS "${ENOOMIAN_SOLANA_ORACLE_DISPUTE_WINDOW_SECS}" + "${sink}" "${workdir}" "${service_id}" plain FIGHT_ORACLE_PROGRAM_ID "${ENOOMIAN_SOLANA_FIGHT_ORACLE_PROGRAM_ID}" + "${sink}" "${workdir}" "${service_id}" plain GOLD_CLOB_MARKET_PROGRAM_ID "${ENOOMIAN_SOLANA_GOLD_CLOB_PROGRAM_ID}" + "${sink}" "${workdir}" "${service_id}" plain BSC_RPC_URL "${ENOOMIAN_BSC_RPC_URL}" + "${sink}" "${workdir}" "${service_id}" plain BSC_DUEL_ORACLE_ADDRESS "${ENOOMIAN_BSC_DUEL_ORACLE_ADDRESS}" + "${sink}" "${workdir}" "${service_id}" plain BSC_GOLD_CLOB_ADDRESS "${ENOOMIAN_BSC_GOLD_CLOB_ADDRESS}" + "${sink}" "${workdir}" "${service_id}" plain BASE_RPC_URL "" + "${sink}" "${workdir}" "${service_id}" plain BASE_DUEL_ORACLE_ADDRESS "" + "${sink}" "${workdir}" "${service_id}" plain BASE_GOLD_CLOB_ADDRESS "" + "${sink}" "${workdir}" "${service_id}" stdin STREAM_PUBLISH_KEY "${ENOOMIAN_HYPERBET_BSC_STREAM_PUBLISH_KEY}" + "${sink}" "${workdir}" "${service_id}" stdin ARENA_EXTERNAL_BET_WRITE_KEY "${ENOOMIAN_HYPERBET_BSC_STREAM_PUBLISH_KEY}" + "${sink}" "${workdir}" "${service_id}" stdin HYPERBET_BSC_STAGING_REPORTER_PRIVATE_KEY "${ENOOMIAN_HYPERBET_BSC_REPORTER_PRIVATE_KEY}" + "${sink}" "${workdir}" "${service_id}" stdin HYPERBET_BSC_STAGING_MARKET_OPERATOR_PRIVATE_KEY "${ENOOMIAN_HYPERBET_BSC_MARKET_OPERATOR_PRIVATE_KEY}" + "${sink}" "${workdir}" "${service_id}" stdin HYPERBET_BSC_STAGING_FINALIZER_PRIVATE_KEY "${bsc_finalizer_private_key}" + "${sink}" "${workdir}" "${service_id}" stdin ORACLE_CONFIG_AUTHORITY_KEYPAIR "${oracle_config_authority_keypair}" + "${sink}" "${workdir}" "${service_id}" stdin CLOB_CONFIG_AUTHORITY_KEYPAIR "${clob_config_authority_keypair}" + "${sink}" "${workdir}" "${service_id}" stdin ORACLE_AUTHORITY_KEYPAIR "${oracle_authority_keypair}" + "${sink}" "${workdir}" "${service_id}" stdin ORACLE_REPORTER_KEYPAIR "${oracle_authority_keypair}" + "${sink}" "${workdir}" "${service_id}" stdin ORACLE_FINALIZER_KEYPAIR "${oracle_authority_keypair}" + "${sink}" "${workdir}" "${service_id}" stdin ORACLE_CHALLENGER_KEYPAIR "${oracle_authority_keypair}" + "${sink}" "${workdir}" "${service_id}" stdin CLOB_MARKET_OPERATOR_KEYPAIR "${oracle_authority_keypair}" + "${sink}" "${workdir}" "${service_id}" stdin BOT_KEYPAIR "${bot_keypair}" + "${sink}" "${workdir}" "${service_id}" stdin MARKET_MAKER_KEYPAIR "${market_maker_keypair}" + if [[ -n "${keeper_pages_url}" ]]; then + "${sink}" "${workdir}" "${service_id}" plain PUBLIC_APP_URL "${keeper_pages_url}" + fi +} + +enoomian_sync_hyperbet_keeper_env() { + enoomian_backfill_hyperbet_surface_aliases + enoomian_require_env \ + ENOOMIAN_RAILWAY_PROJECT_ID \ + ENOOMIAN_RAILWAY_ENVIRONMENT_ID \ + ENOOMIAN_HYPERBET_KEEPER_SERVICE_ID \ + ENOOMIAN_HYPERBET_KEEPER_URL \ + ENOOMIAN_HYPERBET_KEEPER_WS_URL \ + ENOOMIAN_HYPERBET_PAGES_URL \ + ENOOMIAN_HYPERBET_SOLANA_KEEPER_URL \ + ENOOMIAN_HYPERSCAPES_API_URL \ + ENOOMIAN_SOLANA_CLUSTER \ + ENOOMIAN_SOLANA_RPC_URL \ + ENOOMIAN_SOLANA_ORACLE_DISPUTE_WINDOW_SECS \ + ENOOMIAN_SOLANA_FIGHT_ORACLE_PROGRAM_ID \ + ENOOMIAN_SOLANA_GOLD_CLOB_PROGRAM_ID \ + ENOOMIAN_BSC_RPC_URL \ + ENOOMIAN_BSC_DUEL_ORACLE_ADDRESS \ + ENOOMIAN_BSC_GOLD_CLOB_ADDRESS \ + ENOOMIAN_HYPERBET_BSC_STREAM_PUBLISH_KEY \ + ENOOMIAN_HYPERBET_BSC_REPORTER_PRIVATE_KEY \ + ENOOMIAN_HYPERBET_BSC_MARKET_OPERATOR_PRIVATE_KEY \ + ENOOMIAN_HYPERBET_SOLANA_ORACLE_CONFIG_AUTHORITY_KEYPAIR \ + ENOOMIAN_HYPERBET_SOLANA_CLOB_CONFIG_AUTHORITY_KEYPAIR \ + ENOOMIAN_HYPERBET_SOLANA_ORACLE_AUTHORITY_KEYPAIR \ + ENOOMIAN_HYPERBET_SOLANA_CANARY_KEYPAIR \ + ENOOMIAN_HYPERBET_SOLANA_MARKET_MAKER_KEYPAIR + + enoomian_log "syncing Hyperbet unified keeper Railway variables" + enoomian_emit_hyperbet_keeper_env_manifest \ + enoomian_apply_railway_manifest_entry \ + "${ENOOMIAN_REPO_ROOT}/packages/hyperbet-evm/keeper" \ + "${ENOOMIAN_HYPERBET_KEEPER_SERVICE_ID}" +} + +enoomian_sync_hyperbet_keepers_env() { + enoomian_sync_hyperbet_keeper_env +} + +enoomian_deploy_hyperscapes_railway() { + enoomian_require_hyperscapes_root + enoomian_require_env \ + ENOOMIAN_RAILWAY_PROJECT_ID \ + ENOOMIAN_RAILWAY_ENVIRONMENT_ID \ + ENOOMIAN_HYPERSCAPES_SERVICE_ID \ + ENOOMIAN_HYPERSCAPES_API_URL \ + ENOOMIAN_HYPERSCAPES_WS_URL \ + ENOOMIAN_HYPERSCAPES_PAGES_URL \ + ENOOMIAN_DATABASE_URL \ + ENOOMIAN_HYPERSCAPES_JWT_SECRET \ + ENOOMIAN_HYPERSCAPES_AUTOMATION_TOKEN + + local repo_root="${ENOOMIAN_HYPERSCAPES_ROOT}" + local service_id="${ENOOMIAN_HYPERSCAPES_SERVICE_ID}" + local upload_root + local commit_sha + local public_stream_url="${ENOOMIAN_HYPERSCAPES_PAGES_URL%/}/stream" + local public_stream_html_url="${ENOOMIAN_HYPERSCAPES_PAGES_URL%/}/stream.html" + local public_legacy_stream_url="${ENOOMIAN_HYPERSCAPES_PAGES_URL%/}/?page=stream" + local stream_delivery_mode="${ENOOMIAN_STREAM_DELIVERY_MODE:-self_hls}" + local stream_delivery_provider="${ENOOMIAN_STREAM_DELIVERY_PROVIDER:-}" + local stream_playback_hls_url="${ENOOMIAN_STREAM_PLAYBACK_URL:-${ENOOMIAN_HYPERSCAPES_API_URL%/}/live/stream.m3u8}" + local stream_playback_llhls_url="${ENOOMIAN_STREAM_PLAYBACK_LLHLS_URL:-}" + local stream_ingest_rtmps_url="${ENOOMIAN_STREAM_INGEST_RTMPS_URL:-}" + local stream_ingest_stream_key="${ENOOMIAN_STREAM_INGEST_STREAM_KEY:-}" + local stream_ingest_profile="${ENOOMIAN_STREAM_INGEST_PROFILE:-default}" + local stream_ingest_transport="${ENOOMIAN_STREAM_INGEST_TRANSPORT:-rtmps}" + local stream_ingest_srt_url="${ENOOMIAN_STREAM_INGEST_SRT_URL:-}" + local stream_ingest_srt_stream_id="${ENOOMIAN_STREAM_INGEST_SRT_STREAM_ID:-}" + local stream_ingest_srt_passphrase="${ENOOMIAN_STREAM_INGEST_SRT_PASSPHRASE:-}" + local stream_audio_sample_rate="${ENOOMIAN_STREAM_AUDIO_SAMPLE_RATE:-}" + local stream_gop_size="${ENOOMIAN_STREAM_GOP_SIZE:-}" + local stream_cloudflare_probe_only="${ENOOMIAN_STREAM_CLOUDFLARE_PROBE_ONLY:-false}" + local stream_cloudflare_live_input_id="${STREAM_CLOUDFLARE_LIVE_INPUT_ID:-${ENOOMIAN_STREAM_CLOUDFLARE_LIVE_INPUT_ID:-${stream_ingest_srt_stream_id}}}" + local stream_cloudflare_account_id="${STREAM_CLOUDFLARE_ACCOUNT_ID:-${ENOOMIAN_CLOUDFLARE_ACCOUNT_ID:-}}" + local stream_cloudflare_webhook_secret="${STREAM_CLOUDFLARE_WEBHOOK_SECRET:-${ENOOMIAN_STREAM_CLOUDFLARE_WEBHOOK_SECRET:-}}" + local stream_low_latency="${ENOOMIAN_STREAM_LOW_LATENCY:-true}" + local stream_fps="${ENOOMIAN_STREAM_FPS:-30}" + local stream_output_width="${ENOOMIAN_STREAM_OUTPUT_WIDTH:-426}" + local stream_output_height="${ENOOMIAN_STREAM_OUTPUT_HEIGHT:-240}" + local stream_video_bitrate_kbps="${ENOOMIAN_STREAM_VIDEO_BITRATE_KBPS:-96}" + local stream_audio_bitrate_kbps="${ENOOMIAN_STREAM_AUDIO_BITRATE_KBPS:-32}" + local hls_time_seconds="${ENOOMIAN_HLS_TIME_SECONDS:-2}" + local hls_list_size="${ENOOMIAN_HLS_LIST_SIZE:-18}" + local hls_delete_threshold="${ENOOMIAN_HLS_DELETE_THRESHOLD:-54}" + + if git -C "${repo_root}" rev-parse HEAD >/dev/null 2>&1; then + commit_sha="$(git -C "${repo_root}" rev-parse HEAD)" + else + commit_sha="local-enoomian-$(date +%s)" + fi + + upload_root="$(enoomian_make_upload_alias "${repo_root}" hyperscapes)" + enoomian_patch_hyperscapes_upload_root "${upload_root}" + if [[ "${ENOOMIAN_HYPERSCAPES_ASSET_BUNDLE:-proxy}" == "full" ]]; then + enoomian_validate_hyperscapes_assets_root "${upload_root}" + else + enoomian_validate_hyperscapes_assets_root "${repo_root}" + enoomian_log "skipping trimmed upload-root asset validation for ${ENOOMIAN_HYPERSCAPES_ASSET_BUNDLE:-proxy} bundle" + fi + if [[ "${ENOOMIAN_SKIP_PREFLIGHT:-0}" != "1" ]]; then + enoomian_run_hyperscapes_preflight "${upload_root}" + else + enoomian_log "skipping Hyperscapes preflight (ENOOMIAN_SKIP_PREFLIGHT=1)" + fi + + enoomian_maybe_sync_railway_env \ + "Hyperscapes" \ + "hyperscapes-railway-env" \ + enoomian_sync_hyperscapes_railway_env \ + enoomian_emit_hyperscapes_railway_env_manifest \ + "${repo_root}" \ + "${service_id}" + + enoomian_log "deploying Hyperscapes to personal Railway" + ( + cd "${upload_root}" + enoomian_railway_up \ + "Hyperscapes Railway upload" \ + -p "${ENOOMIAN_RAILWAY_PROJECT_ID}" \ + -e "${ENOOMIAN_RAILWAY_ENVIRONMENT_ID}" \ + -s "${service_id}" \ + --ci \ + --verbose + ) + rm -rf "$(dirname "${upload_root}")" + + enoomian_log "verifying Hyperscapes status" + enoomian_wait_for_json "${ENOOMIAN_HYPERSCAPES_API_URL}/health" '.status == "ok"' 30 10 || enoomian_die "Hyperscapes /health did not become healthy" + enoomian_wait_for_json "${ENOOMIAN_HYPERSCAPES_API_URL}/status" '.uptime != null and .connectedUsers != null' 30 10 || enoomian_die "Hyperscapes /status did not become healthy" + enoomian_wait_for_json "${ENOOMIAN_HYPERSCAPES_API_URL}/api/streaming/state" '.type != null' 36 10 || enoomian_die "Hyperscapes streaming state did not become healthy" +} + +enoomian_write_hyperscapes_pages_env_js() { + local output_file="$1" + local public_api_url="$2" + local public_ws_url="$3" + local public_cdn_url="$4" + local public_app_url="$5" + local embed_allowed_origins="${6:-}" + local streaming_viewer_access_token="${7:-}" + local public_privy_app_id="${8:-}" + + python3 - "${output_file}" "${public_api_url}" "${public_ws_url}" "${public_cdn_url}" "${public_app_url}" "${embed_allowed_origins}" "${streaming_viewer_access_token}" "${public_privy_app_id}" <<'PY' +import json +import pathlib +import sys + +output_file, public_api_url, public_ws_url, public_cdn_url, public_app_url, embed_allowed_origins, streaming_viewer_access_token, public_privy_app_id = sys.argv[1:] + +env = { + "PUBLIC_API_URL": public_api_url, + "PUBLIC_WS_URL": public_ws_url, + "PUBLIC_CDN_URL": public_cdn_url, + "PUBLIC_ASSETS_URL": public_cdn_url, + "PUBLIC_APP_URL": public_app_url, +} + +if embed_allowed_origins: + env["PUBLIC_EMBED_ALLOWED_ORIGINS"] = embed_allowed_origins +if streaming_viewer_access_token: + env["PUBLIC_STREAMING_VIEWER_ACCESS_TOKEN"] = streaming_viewer_access_token +if public_privy_app_id: + env["PUBLIC_PRIVY_APP_ID"] = public_privy_app_id + +path = pathlib.Path(output_file) +path.parent.mkdir(parents=True, exist_ok=True) +path.write_text( + "// Generated by scripts/enoomian-staging/deploy.sh\n" + "(() => {\n" + f" window.env = {json.dumps(env, indent=2, sort_keys=True)};\n" + "})();\n", + encoding="utf-8", +) +PY +} + +enoomian_validate_hyperscapes_pages_env_js() { + local env_file="$1" + local expected_api_url="$2" + local expected_ws_url="$3" + local expected_cdn_url="$4" + + python3 - "${env_file}" "${expected_api_url}" "${expected_ws_url}" "${expected_cdn_url}" <<'PY' +import json +import pathlib +import re +import sys + +env_file, expected_api_url, expected_ws_url, expected_cdn_url = sys.argv[1:] +content = pathlib.Path(env_file).read_text(encoding="utf-8") +match = re.search(r"window\.env = (\{.*\});\n\}\)\(\);", content, re.S) +if not match: + raise SystemExit(f"Unable to parse {env_file}") + +env = json.loads(match.group(1)) +expected = { + "PUBLIC_API_URL": expected_api_url, + "PUBLIC_WS_URL": expected_ws_url, + "PUBLIC_CDN_URL": expected_cdn_url, + "PUBLIC_ASSETS_URL": expected_cdn_url, +} + +for key, value in expected.items(): + actual = env.get(key) + if actual != value: + raise SystemExit( + f"{env_file} drifted for {key}: expected {value!r}, found {actual!r}" + ) +PY +} + +enoomian_deploy_hyperscapes_pages() { + enoomian_require_hyperscapes_root + enoomian_require_env \ + ENOOMIAN_HYPERSCAPES_PAGES_PROJECT_NAME \ + ENOOMIAN_HYPERSCAPES_PAGES_URL \ + ENOOMIAN_HYPERSCAPES_API_URL \ + ENOOMIAN_HYPERSCAPES_WS_URL + + local repo_root="${ENOOMIAN_HYPERSCAPES_ROOT}" + local hyperscapes_public_api_url="${ENOOMIAN_HYPERSCAPES_PUBLIC_API_URL:-${ENOOMIAN_HYPERSCAPES_API_URL}}" + local hyperscapes_public_ws_url="${ENOOMIAN_HYPERSCAPES_PUBLIC_WS_URL:-${ENOOMIAN_HYPERSCAPES_WS_URL}}" + local hyperscapes_cdn_url + local commit_sha + local commit_msg + hyperscapes_cdn_url="${ENOOMIAN_PUBLIC_CDN_URL:-${ENOOMIAN_HYPERSCAPES_CDN_URL:-${ENOOMIAN_HYPERSCAPES_API_URL%/}/game-assets}}" + if [[ "${ENOOMIAN_ALLOW_SPLIT_HYPERSCAPES_PUBLIC_RAIL:-0}" != "1" ]]; then + if [[ "${hyperscapes_public_api_url%/}" != "${ENOOMIAN_HYPERSCAPES_API_URL%/}" ]]; then + enoomian_die "Refusing Hyperscapes Pages deploy: ENOOMIAN_HYPERSCAPES_PUBLIC_API_URL must match ENOOMIAN_HYPERSCAPES_API_URL for the enoomian staging canonical rail" + fi + if [[ "${hyperscapes_public_ws_url%/}" != "${ENOOMIAN_HYPERSCAPES_WS_URL%/}" ]]; then + enoomian_die "Refusing Hyperscapes Pages deploy: ENOOMIAN_HYPERSCAPES_PUBLIC_WS_URL must match ENOOMIAN_HYPERSCAPES_WS_URL for the enoomian staging canonical rail" + fi + fi + if git -C "${repo_root}" rev-parse HEAD >/dev/null 2>&1; then + commit_sha="$(git -C "${repo_root}" rev-parse HEAD)" + commit_msg="$(git -C "${repo_root}" log -1 --pretty=%s | tr -d '"' | cut -c1-100)" + else + commit_sha="local-enoomian-$(date +%s)" + commit_msg="local staging deploy" + fi + + enoomian_ensure_pages_project "${ENOOMIAN_HYPERSCAPES_PAGES_PROJECT_NAME}" + + enoomian_log "building Hyperscapes client" + ( + cd "${repo_root}" + bun install --frozen-lockfile + export NODE_OPTIONS='--max-old-space-size=4096' + export PUBLIC_API_URL="${hyperscapes_public_api_url}" + export PUBLIC_WS_URL="${hyperscapes_public_ws_url}" + export PUBLIC_APP_URL="${ENOOMIAN_HYPERSCAPES_PAGES_URL}" + export PUBLIC_CDN_URL="${hyperscapes_cdn_url}" + export PUBLIC_ASSET_VERSION="${commit_sha}" + export PUBLIC_EMBED_ALLOWED_ORIGINS="$(enoomian_hyperscapes_embed_allowed_origins)" + if [[ -n "${ENOOMIAN_STREAMING_VIEWER_ACCESS_TOKEN:-}" ]]; then + export PUBLIC_STREAMING_VIEWER_ACCESS_TOKEN="${ENOOMIAN_STREAMING_VIEWER_ACCESS_TOKEN}" + else + unset PUBLIC_STREAMING_VIEWER_ACCESS_TOKEN || true + fi + if [[ -n "${ENOOMIAN_PUBLIC_PRIVY_APP_ID:-}" ]]; then + export PUBLIC_PRIVY_APP_ID="${ENOOMIAN_PUBLIC_PRIVY_APP_ID}" + else + unset PUBLIC_PRIVY_APP_ID || true + fi + bun run build:client + ) + + enoomian_log "stamping Hyperscapes Pages runtime env" + enoomian_write_hyperscapes_pages_env_js \ + "${repo_root}/packages/client/dist/env.js" \ + "${hyperscapes_public_api_url}" \ + "${hyperscapes_public_ws_url}" \ + "${hyperscapes_cdn_url}" \ + "${ENOOMIAN_HYPERSCAPES_PAGES_URL}" \ + "$(enoomian_hyperscapes_embed_allowed_origins)" \ + "${ENOOMIAN_STREAMING_VIEWER_ACCESS_TOKEN:-}" \ + "${ENOOMIAN_PUBLIC_PRIVY_APP_ID:-}" + enoomian_validate_hyperscapes_pages_env_js \ + "${repo_root}/packages/client/dist/env.js" \ + "${hyperscapes_public_api_url}" \ + "${hyperscapes_public_ws_url}" \ + "${hyperscapes_cdn_url}" + + enoomian_log "deploying Hyperscapes Pages" + ( + cd "${repo_root}/packages/client" + enoomian_wrangler pages deploy dist \ + --project-name="${ENOOMIAN_HYPERSCAPES_PAGES_PROJECT_NAME}" \ + --branch="$(enoomian_hyperbet_pages_branch)" \ + --commit-hash="${commit_sha}" \ + --commit-message="${commit_msg}" + ) + + enoomian_wait_for_url "${ENOOMIAN_HYPERSCAPES_PAGES_URL}" || enoomian_die "Hyperscapes Pages URL did not become reachable" +} + +enoomian_pages_commit_message() { + if git -C "${ENOOMIAN_REPO_ROOT}" rev-parse HEAD >/dev/null 2>&1; then + git -C "${ENOOMIAN_REPO_ROOT}" log -1 --pretty=%s | tr -d '"' | cut -c1-100 + else + printf '%s\n' "local staging deploy" + fi +} + +enoomian_pages_deploy_dist() { + local dist_dir="$1" + local project_name="$2" + local commit_sha="$3" + local commit_msg="$4" + + enoomian_wrangler pages deploy "${dist_dir}" \ + --project-name="${project_name}" \ + --branch="$(enoomian_hyperbet_pages_branch)" \ + --commit-hash="${commit_sha}" \ + --commit-message="${commit_msg}" \ + --commit-dirty=true \ + --skip-caching +} + +enoomian_make_hyperbet_redirect_dist() { + local target_url="$1" + local label="$2" + local redirect_root + + redirect_root="$(enoomian_mktemp_dir "enoomian-${label}-redirect")" + target_url="${target_url%/}" + + cat >"${redirect_root}/_headers" <<'EOF' +/* + Cache-Control: no-store +EOF + + cat >"${redirect_root}/_redirects" <"${redirect_root}/index.html" < + + + + ${label} moved + + + +

This surface moved to ${target_url}/.

+ + +EOF + + printf '%s\n' "${redirect_root}" +} + +enoomian_deploy_hyperbet_keeper() { + enoomian_backfill_hyperbet_surface_aliases + enoomian_require_env \ + ENOOMIAN_RAILWAY_PROJECT_ID \ + ENOOMIAN_RAILWAY_ENVIRONMENT_ID \ + ENOOMIAN_HYPERBET_KEEPER_SERVICE_ID \ + ENOOMIAN_HYPERBET_KEEPER_URL \ + ENOOMIAN_HYPERBET_PAGES_URL \ + ENOOMIAN_HYPERBET_SOLANA_KEEPER_URL \ + ENOOMIAN_HYPERSCAPES_API_URL \ + ENOOMIAN_SOLANA_RPC_URL \ + ENOOMIAN_SOLANA_FIGHT_ORACLE_PROGRAM_ID \ + ENOOMIAN_SOLANA_GOLD_CLOB_PROGRAM_ID \ + ENOOMIAN_BSC_RPC_URL \ + ENOOMIAN_BSC_DUEL_ORACLE_ADDRESS \ + ENOOMIAN_BSC_GOLD_CLOB_ADDRESS \ + ENOOMIAN_HYPERBET_BSC_STREAM_PUBLISH_KEY + + local package_root="${ENOOMIAN_REPO_ROOT}/packages/hyperbet-evm" + local keeper_root="${package_root}/keeper" + local service_id="${ENOOMIAN_HYPERBET_KEEPER_SERVICE_ID}" + local upload_root + local keeper_health_filter='.ok == true and .proxies.solanaRpc == true and .proxies.bscRpc == true' + + enoomian_log "staging Hyperbet unified keeper workspace" + ( + cd "${ENOOMIAN_REPO_ROOT}" + node --import tsx scripts/stage-deploy-workspace.ts --target=keeper:shared + ) + mkdir -p "${keeper_root}/workspace-packages/packages/hyperbet-evm/deployments" + rsync -a \ + "${ENOOMIAN_REPO_ROOT}/packages/hyperbet-evm/deployments/" \ + "${keeper_root}/workspace-packages/packages/hyperbet-evm/deployments/" + + enoomian_maybe_sync_railway_env \ + "Hyperbet unified keeper" \ + "hyperbet-keeper-env" \ + enoomian_sync_hyperbet_keeper_env \ + enoomian_emit_hyperbet_keeper_env_manifest \ + "${keeper_root}" \ + "${service_id}" + + enoomian_log "deploying Hyperbet unified keeper" + upload_root="$(enoomian_make_upload_alias "${package_root}")" + enoomian_railway_up \ + "Hyperbet unified keeper upload" \ + "${upload_root}" \ + --path-as-root \ + -p "${ENOOMIAN_RAILWAY_PROJECT_ID}" \ + -e "${ENOOMIAN_RAILWAY_ENVIRONMENT_ID}" \ + -s "${service_id}" \ + --ci \ + --verbose + rm -rf "$(dirname "${upload_root}")" + + enoomian_wait_for_json "${ENOOMIAN_HYPERBET_KEEPER_URL}/status" "${keeper_health_filter}" || enoomian_die "Hyperbet unified keeper did not become healthy" +} + +enoomian_deploy_hyperbet_pages() { + enoomian_backfill_hyperbet_surface_aliases + enoomian_require_env \ + ENOOMIAN_HYPERBET_PAGES_PROJECT_NAME \ + ENOOMIAN_HYPERBET_PAGES_URL \ + ENOOMIAN_HYPERBET_KEEPER_URL \ + ENOOMIAN_HYPERBET_KEEPER_WS_URL \ + ENOOMIAN_SOLANA_CLUSTER \ + ENOOMIAN_BSC_RPC_URL \ + ENOOMIAN_BSC_CHAIN_ID \ + ENOOMIAN_BSC_GOLD_CLOB_ADDRESS + + local api_url="${ENOOMIAN_HYPERBET_KEEPER_URL}" + local ws_url="${ENOOMIAN_HYPERBET_KEEPER_WS_URL}" + local stream_url="" + local playback_url="" + local public_cdn_url + local commit_sha + local commit_msg + local use_game_evm_rpc_proxy + local base_rpc_url="" + local base_chain_id="" + local base_gold_clob_address="" + local avax_rpc_url="" + local avax_chain_id="" + local avax_gold_clob_address="" + local dist_dir="${ENOOMIAN_HYPERBET_PAGES_DIST_DIR:-}" + + public_cdn_url="${ENOOMIAN_PUBLIC_CDN_URL:-${ENOOMIAN_HYPERSCAPES_CDN_URL:-${ENOOMIAN_HYPERSCAPES_API_URL%/}/game-assets}}" + use_game_evm_rpc_proxy="${ENOOMIAN_HYPERBET_PAGES_USE_GAME_EVM_RPC_PROXY:-true}" + enoomian_log "Base and AVAX remain disabled on the unified staging Pages surface" + if [[ -n "${ENOOMIAN_HYPERSCAPES_PAGES_URL:-}" && -n "${ENOOMIAN_STREAMING_VIEWER_ACCESS_TOKEN:-}" ]]; then + stream_url="$(enoomian_hyperscapes_stream_url)" + fi + enoomian_require_tokenized_hyperscapes_stream_url "${stream_url}" + if [[ -n "${stream_url}" && "${stream_url}" == *twitch.tv* ]]; then + enoomian_die "expected a Hyperscapes stream URL for staging, got ${stream_url}" + fi + + # Prefer the Cloudflare Stream HLS manifest as the canonical bets-page + # stream URL. StreamPlayer.tsx sniffs the `.m3u8` suffix and mounts a + # native