diff --git a/README.md b/README.md index 1433d322bc..718a991a2f 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ This demo highlights the core developer experience and "Agentic Infrastructure" 2. **State Persistence:** Persistent working memory (volatile RAM) and filesystem state preserved perfectly across hibernation cycles via full-state snapshots. 3. **Agent Swarm Multiplexing:** Demonstrates 30x+ oversubscription by "juggling" a large registry of stateful actors onto a small pool of shared physical pods. -To reproduce this demo in your own cluster, please refer to the detailed walkthroughs in the **[Counter Demo](demos/counter/README.md)** and **[Secret Agent Demo](demos/agent-secret/README.md)**. +To reproduce this demo in your own cluster, please refer to the detailed walkthroughs in the **[Counter Demo](demos/counter/README.md)** and **[Secret Agent Demo](demos/agent-secret/README.md)**, and **[OpenClaw Multiplexing](demos/openclaw/README.md)**. For more videos and walkthroughs, visit our YouTube channel: **[agent-substrate](https://www.youtube.com/channel/UCN9PPqlTtVxlcpbQ-NWpfZQ)**. @@ -53,6 +53,7 @@ Agent Substrate is designed to be **framework and agent harness agnostic**. Beca * **Agent Development Kit (ADK):** Native support for ADK-compatible session identity and persistent working memory. * **LangChain:** Ideal execution environment for long-running, stateful LangChain agents and sandboxed tool-calling. * **Claude Code & CodeX:** Support for high-density, stateful coding environments that preserve terminal and filesystem state across sessions. +* **OpenClaw (Google Claw):** Optimized for multiplexing stateful TypeScript agents with persistent in-memory reasoning and conversation history. * **Model Context Protocol (MCP):** Deploy secure, sandboxed MCP servers as Substrate Actors to provide durable tools for any LLM. ## Ecosystem & Examples @@ -188,6 +189,7 @@ We provide several sample applications demonstrating Agent Substrate's capabilit 2. **[Sandbox Demo (Antigravity)](demos/sandbox/README.md)**: A secure, sandboxed execution environment (running Alpine Linux) that allows arbitrary shell execution while preserving filesystem state across sessions. 3. **[Claude Code Multiplex](demos/claude-code-multiplex/README.md)**: Demonstrates oversubscribing physical hardware by multiplexing multiple Claude Code agents onto a limited pool of workers. 4. **[Secret Agent](demos/agent-secret/README.md)**: Highlights Substrate's "Zero-Idle" self-suspension and re-animation of volatile process memory. +5. **[OpenClaw Multiplexing](demos/openclaw/README.md)**: Showcases 1.5x hardware oversubscription using Google Claw agents, demonstrating stateful rehydration and rapid agent rotation across physical pods. ### Documentation & Guides * [API Configuration Guide](docs/api-guide.md): Detailed reference for configuring WorkerPools, ActorTemplates, Secrets, and Volumes. diff --git a/demos/sub-agent-multiplex/Dockerfile b/demos/sub-agent-multiplex/Dockerfile new file mode 100644 index 0000000000..56fafb5f76 --- /dev/null +++ b/demos/sub-agent-multiplex/Dockerfile @@ -0,0 +1,74 @@ +# Google Claw on Agent Substrate PoC +# Portable Dockerfile for OSS Substrate Migration + +# Stage 1: Build the standalone bundles +FROM node:22-slim AS builder + +WORKDIR /app + +# Install build dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# Copy standalone package files +COPY package.json ./ +# Use npm install for simplicity and portability in the standalone package +RUN npm install + +# Copy source code +COPY ui/ ./ui/ +COPY workload/ ./workload/ + +# Build zero-dependency bundles +RUN ./node_modules/.bin/esbuild workload/agent.ts \ + --bundle \ + --platform=node \ + --target=node22 \ + --outfile=dist/agent.js \ + --external:node:* + +RUN ./node_modules/.bin/esbuild ui/demo-ui.ts \ + --bundle \ + --platform=node \ + --target=node22 \ + --outfile=dist/demo-ui.js \ + --external:node:* + +# Stage 2: Final Production Image +FROM node:22-slim AS runner + +WORKDIR /app + +# Copy the entire context to check for local binaries +COPY . . + +# Install runtime dependencies (tini for signal forwarding, kubectl for dashboard sync) +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + tini \ + && curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \ + && chmod +x kubectl \ + && mv kubectl /usr/local/bin/ \ + && rm -rf /var/lib/apt/lists/* + +# Copy built assets +COPY --from=builder /app/dist/ ./dist/ +# Copy kubectl-ate binary if it exists in context, otherwise download it +# This makes the Dockerfile portable across environments +RUN if [ -f "./kubectl-ate" ]; then \ + mv ./kubectl-ate /usr/local/bin/kubectl-ate; \ + else \ + curl -L -o /usr/local/bin/kubectl-ate https://github.com/agent-substrate/substrate/releases/latest/download/kubectl-ate-linux-amd64; \ + fi && chmod +x /usr/local/bin/kubectl-ate + +# Create a /pause hook for Substrate rehydration +RUN echo '#!/bin/sh' > /pause && \ + echo 'echo "[pause] Starting Google Claw agent..."' >> /pause && \ + echo 'exec /usr/bin/tini -- /usr/local/bin/node /app/dist/agent.js' >> /pause && \ + chmod +x /pause + +# Default entrypoint (can be overridden by deployment to run demo-ui) +ENTRYPOINT ["/usr/bin/tini", "--", "node", "dist/agent.js"] diff --git a/demos/sub-agent-multiplex/README.md b/demos/sub-agent-multiplex/README.md new file mode 100644 index 0000000000..62360fa4b7 --- /dev/null +++ b/demos/sub-agent-multiplex/README.md @@ -0,0 +1,98 @@ +# Substrate Multiplex Demo: NanoClaw 1.5x Overcommit + +This demo demonstrates the extreme efficiency gains possible with Google Substrate by multiplexing **3 logical NanoClaw agents** onto **2 physical substrate workers** (1.5x density ratio). + +## System Information + +- **Agent Framework**: NanoClaw (v2.x) +- **Source**: `github.com/nanocoai/nanoclaw` +- **Substrate Mode**: Multi-Actor Multiplexing (1.5x oversubscription) +- **Runtime**: Bun (Node.js compatible) inside Debian Slim +- **Isolation**: gVisor (runsc) + +## What this shows + +- **High-Density Multiplexing**: Three logical agent identities running on only two physical pods (1.5x oversubscription). +- **State Persistence**: A `taskCounter` maintained in the Node.js process memory survives multiple suspend/resume cycles. +- **Dynamic Rotation**: Agents finish work at different times (3-6s), forcing Substrate to constantly rotate pod ownership. +- **Visual Identity Tracking**: Color-coded agents (Blue/Pink/Gold) and live log tailing to make infrastructure sharing intuitively obvious. + +## Audience + +This guide is intended for engineers exploring Agent Substrate for hosting large-scale agentic workloads where cost-efficiency and stateful rehydration are critical. + +## Prerequisites + +- **Agent Substrate Cluster**: A Kubernetes cluster with Substrate installed. +- **Docker**: For building and pushing the unified actor/UI image. +- **GCS Bucket**: Configured for Substrate state snapshots (e.g., `gs://snapshot-substrate-gke-ai-eco-dev/`). +- **kubectl & kubectl-ate**: The Substrate CLI tool for managing logical actors. + +## Components + +| Path | Purpose | +|---|---| +| `workload/agent.ts` | The workload: A NanoClaw/Hono server with persistent memory state. | +| `ui/demo-ui.ts` | The dashboard: A Node.js backend providing live logs, task queueing, and visual tracking. | +| `sub-agent-multiplex.yaml.tmpl` | Kubernetes manifests for ActorTemplates and WorkerPools. | +| `Dockerfile` | Unified OCI image containing both the actor workload and the dashboard UI. | + +## How to Run + +### 1. Provision Hardware +Scale the physical `WorkerPool` to the desired replica count (2 for this demo): +```bash +kubectl apply -f sub-agent-multiplex.yaml +``` + +### 2. Deploy Logical Agents +Create the three "fun-named" actors using the Substrate CLI. +```bash +kubectl-ate create actor agent-luna-v12 --template sub-agent/sub-agent-agent +kubectl-ate create actor agent-mars-v12 --template sub-agent/sub-agent-agent +kubectl-ate create actor agent-nova-v11 --template sub-agent/sub-agent-agent +``` + +### 3. Launch the Dashboard +The dashboard runs as a standard Kubernetes Deployment with a LoadBalancer. +```bash +kubectl apply -f demo-ui.yaml +``` + +## Drive the Demo + +Open the dashboard and use the following interaction patterns: + +- **Pulse (Manual Wakeup)**: Trigger tasks across the registry. Watch the **colored icons** rapidly cycle through the 2 worker slots. +- **Live Logs**: Observe the MT Broker logs. You will see different Agent IDs appearing in the **same log stream**, proving that physical hardware is being recycled in real-time. +- **Cron Tracker**: Observe real-time countdowns as the automated schedule triggers orchestration events. + +## Integrating a Real LLM API + +Integrating an LLM into a NanoClaw logical actor is straightforward. Because Substrate persists the **entire process memory**, any in-memory conversation history or KV-cache will survive multiple suspend/resume cycles without requiring an external database. + +### 1. Add the LLM SDK +Add your preferred SDK (e.g., OpenAI or Anthropic) to the `package.json`: +```bash +npm install openai +``` + +### 2. Update the Actor Logic +Modify `workload/agent.ts` to initialize the client and maintain a local chat history: +```typescript +import OpenAI from "openai"; + +const openai = new OpenAI({ apiKey: process.env.LLM_API_KEY }); +let history: any[] = []; // This array will survive Substrate snapshots! + +app.post("/v1/chat", async (c) => { + const { message } = await c.req.json(); + history.push({ role: "user", content: message }); + + const response = await openai.chat.completions.create({ + model: "gpt-4", + messages: history, + }); + // ... process response +}); +``` diff --git a/demos/sub-agent-multiplex/package.json b/demos/sub-agent-multiplex/package.json new file mode 100644 index 0000000000..71b49279d9 --- /dev/null +++ b/demos/sub-agent-multiplex/package.json @@ -0,0 +1,20 @@ +{ + "name": "openclaw-substrate-poc", + "version": "1.0.0", + "description": "Google Claw on Agent Substrate PoC", + "private": true, + "scripts": { + "build": "esbuild src/agent.ts --bundle --platform=node --target=node22 --outfile=dist/agent.js --external:node:* && esbuild src/demo-ui.ts --bundle --platform=node --target=node22 --outfile=dist/demo-ui.js --external:node:*", + "start:ui": "node dist/demo-ui.js", + "start:agent": "node dist/agent.js" + }, + "dependencies": { + "@hono/node-server": "^1.11.1", + "hono": "^4.4.2" + }, + "devDependencies": { + "@types/node": "^22.0.0", + "esbuild": "^0.21.5", + "typescript": "^5.5.2" + } +} diff --git a/demos/sub-agent-multiplex/sub-agent-multiplex.yaml.tmpl b/demos/sub-agent-multiplex/sub-agent-multiplex.yaml.tmpl new file mode 100644 index 0000000000..9a98511242 --- /dev/null +++ b/demos/sub-agent-multiplex/sub-agent-multiplex.yaml.tmpl @@ -0,0 +1,84 @@ +apiVersion: ate.dev/v1alpha1 +kind: ActorTemplate +metadata: + name: sub-agent-agent + namespace: sub-agent +spec: + containers: + - image: ${SUB_AGENT_IMAGE} + name: agent + ports: + - containerPort: 8080 + pauseImage: registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4 + runsc: + amd64: + sha256Hash: a397be1abc2420d26bce6c70e6e2ff96c73aaaab929756c56f5e2089ea842b63 + url: gs://gvisor/releases/nightly/2026-05-19/x86_64/runsc + arm64: + sha256Hash: 1ba2366ae2efceba166046f51a4104f9261c9cb72c6db8f5b3fe2dc57dea86b9 + url: gs://gvisor/releases/nightly/2026-05-19/aarch64/runsc + snapshotsConfig: + location: gs://${BUCKET_NAME}/sub-agent-agent/v12/ + workerPoolRef: + name: agent-pool + namespace: sub-agent +--- +apiVersion: ate.dev/v1alpha1 +kind: WorkerPool +metadata: + name: agent-pool + namespace: sub-agent +spec: + ateomImage: gcr.io/gke-ai-eco-dev/ate-images/ateom-gvisor-715889664656de67e44382a8d6ab981d@sha256:a877e335fdb6e5576714ab53ad6bf88ee676dfe642516c07673a3d9df56053b3 + replicas: 2 +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: agent-luna-trigger + namespace: sub-agent +spec: + schedule: "*/1 * * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: trigger + image: curlimages/curl:latest + command: ["curl", "-X", "POST", "http://demo-ui.sub-agent.svc.cluster.local/api/give-task?source=cron&agent=agent-luna"] + restartPolicy: OnFailure +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: agent-mars-trigger + namespace: sub-agent +spec: + schedule: "*/2 * * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: trigger + image: curlimages/curl:latest + command: ["curl", "-X", "POST", "http://demo-ui.sub-agent.svc.cluster.local/api/give-task?source=cron&agent=agent-mars"] + restartPolicy: OnFailure +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: agent-nova-trigger + namespace: sub-agent +spec: + schedule: "*/3 * * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: trigger + image: curlimages/curl:latest + command: ["curl", "-X", "POST", "http://demo-ui.sub-agent.svc.cluster.local/api/give-task?source=cron&agent=agent-nova"] + restartPolicy: OnFailure diff --git a/demos/sub-agent-multiplex/tsconfig.json b/demos/sub-agent-multiplex/tsconfig.json new file mode 100644 index 0000000000..ebd4c70277 --- /dev/null +++ b/demos/sub-agent-multiplex/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "ESNext", + "module": "ESNext", + "moduleResolution": "bundler", + "strict": true, + "skipLibCheck": true, + "isolatedModules": true, + "esModuleInterop": true, + "allowImportingTsExtensions": true, + "noEmit": true + }, + "include": ["./**/*"] +} diff --git a/demos/sub-agent-multiplex/ui/demo-ui.ts b/demos/sub-agent-multiplex/ui/demo-ui.ts new file mode 100644 index 0000000000..ba89f28faf --- /dev/null +++ b/demos/sub-agent-multiplex/ui/demo-ui.ts @@ -0,0 +1,573 @@ +import { Hono } from "hono"; +import { serve } from "@hono/node-server"; +import { exec } from "node:child_process"; +import { createClient } from "redis"; + +const app = new Hono(); + +// --- Configuration --- +const NS = process.env.DEMO_NAMESPACE || "sub-agent"; +const ATE_ENDPOINT = process.env.ATE_ENDPOINT || "api.ate-system.svc.cluster.local:443"; +const VALKEY_URL = "redis://valkey-cluster.ate-system.svc.cluster.local:6379"; +const TEMPLATE = "sub-agent/sub-agent-agent"; + +const predefinedTasks = [ + "Analyze repo for security vulnerabilities", + "Summarize latest PR for team review", + "Write unit tests for the message gateway", + "Refactor the plugin discovery logic", + "Draft a response to Buganizer b/392182", + "Generate a cost report for GKE nodes", + "Optimize the gVisor memory mapping", + "Verify snapshot integrity on GCS", +]; + +interface Assignment { + id: string; + agent: string; + task: string; + state: "queued" | "running" | "completed"; + durationSec: number; + created_at: number; + started_at?: number; + completed_at?: number; +} + +interface TaskAudit { + id: string; + agent: string; + timestamp: string; + task: string; + result: string; + status: "success" | "error" | "warning"; + error_detail?: string; +} + +// --- Shared State --- +let shellLogs: string[] = []; +let taskAudits: TaskAudit[] = []; +let assignments: Assignment[] = []; +let taskCursor = 0; +let clusterState = { pods: [] as any[], actors: [] as any[] }; +let lockedActors = new Set(); + +// Precision Tracking +let stats = { + totalLogicalActiveSec: 0, + totalPhysicalActiveSec: 0, + cumulativeTasks: 0, + lastSync: Date.now() +}; + +// External Cron Simulation state (FIX: Restored variable definition) +const CRON_DEFAULTS: Record = { "agent-luna": 60, "agent-mars": 120, "agent-nova": 180 }; +let lastTriggerTime: Record = { "agent-luna": Date.now(), "agent-mars": Date.now(), "agent-nova": Date.now() }; +let cronIterations: Record = { "agent-luna": 0, "agent-mars": 0, "agent-nova": 0 }; + +const AGENT_META: Record = { + "agent-luna": { color: "#79c0ff", id: "agent-luna-v12" }, + "agent-mars": { color: "#ff79c6", id: "agent-mars-v12" }, + "agent-nova": { color: "#f1fa8c", id: "agent-nova-v11" }, +}; + +const ID_TO_DISPLAY: Record = Object.entries(AGENT_META).reduce((acc, [display, meta]) => { + acc[meta.id] = display; + return acc; +}, {} as Record); + +const VALID_ACTOR_IDS = new Set(Object.values(AGENT_META).map(m => m.id)); + +const nowSec = () => Date.now() / 1000; + +const runCmd = (cmd: string): Promise => { + return new Promise((resolve, reject) => { + exec(cmd, (error, stdout, stderr) => { + if (error) reject(new Error(stderr || error.message)); + else resolve(stdout); + }); + }); +}; + +// --- Valkey Persistence --- +let redis: any = null; +try { + redis = createClient({ url: VALKEY_URL }); + redis.on("error", () => {}); +} catch (e) {} + +async function initPersistence() { + if (!redis) return; + try { + await Promise.race([redis.connect(), new Promise((_, r) => setTimeout(r, 2000))]); + if (redis.isOpen) { + const logs = await redis.lRange("demo:shell_logs", 0, -1); + shellLogs = logs || []; + const audits = await redis.lRange("demo:task_audits", 0, -1); + taskAudits = (audits || []).map((a: string) => JSON.parse(a)); + } + } catch (e) {} +} + +async function persistLog(msg: string) { + shellLogs.push(msg); + if (shellLogs.length > 200) shellLogs.shift(); + try { if (redis?.isOpen) { await redis.rPush("demo:shell_logs", msg); await redis.lTrim("demo:shell_logs", -200, -1); } } catch {} +} + +async function persistAudit(audit: TaskAudit) { + taskAudits.push(audit); + if (taskAudits.length > 50) taskAudits.shift(); + try { if (redis?.isOpen) { await redis.rPush("demo:task_audits", JSON.stringify(audit)); await redis.lTrim("demo:task_audits", -50, -1); } } catch {} +} + +function logShell(msg: string) { + const timestamp = new Date().toISOString().slice(11, 19); + const entry = `[${timestamp}] ${msg}`; + persistLog(entry); + console.log(`[shell] ${msg}`); +} + +// --- Background State Syncer --- +async function syncState() { + try { + const actorsOut = await runCmd(`kubectl-ate --endpoint ${ATE_ENDPOINT} get actors -o json`); + const podsOut = await runCmd(`kubectl get pods -n ${NS} -l app=agent-pool -o json`); + + const actors = JSON.parse(actorsOut).actors || []; + const podsRaw = JSON.parse(podsOut).items || []; + + clusterState.actors = actors.filter((a: any) => VALID_ACTOR_IDS.has(a.actorId || a.actor_id)).map((a: any) => ({ + name: a.actorId || a.actor_id, + displayName: ID_TO_DISPLAY[a.actorId || a.actor_id] || (a.actorId || a.actor_id), + status: a.status.replace("STATUS_", ""), + rawStatus: a.status, + ip: a.ateomPodIp || a.ateom_pod_ip || "n/a", + pod: a.ateomPodName || a.ateom_pod_name || "none" + })); + + clusterState.pods = podsRaw.map((p: any) => { + const activeActor = actors.find((a: any) => { + const podPart = (a.ateomPodName || a.ateom_pod_name || "").split("/").pop(); + return podPart === p.metadata.name && VALID_ACTOR_IDS.has(a.actorId || a.actor_id); + }); + const actorId = activeActor ? (activeActor.actorId || activeActor.actor_id) : "idle"; + return { + name: p.metadata.name, + phase: p.status.phase, + ip: p.status.podIP || "n/a", + activeActor: ID_TO_DISPLAY[actorId] || actorId + }; + }); + + const now = Date.now(); + const elapsed = (now - stats.lastSync) / 1000; + stats.lastSync = now; + + const runningActors = clusterState.actors.filter(a => a.rawStatus === "STATUS_RUNNING").length; + const runningPods = clusterState.pods.filter(p => p.phase === "Running" && p.activeActor !== "idle").length; + + stats.totalLogicalActiveSec += runningActors * elapsed; + stats.totalPhysicalActiveSec += runningPods * elapsed; + + } catch (e: any) {} + setTimeout(syncState, 800); +} + +// --- Task Execution: Direct & Robust --- +async function executeTask(actorId: string, assignmentId: string) { + if (lockedActors.has(actorId)) return; + lockedActors.add(actorId); + + const display = ID_TO_DISPLAY[actorId] || actorId; + const asg = assignments.find(a => a.id === assignmentId); + if (!asg) { lockedActors.delete(actorId); return; } + + asg.state = "running"; + logShell(`[broker] Wakeup Request for **${display}** received.`); + + try { + const checkOut = await runCmd(`kubectl-ate --endpoint ${ATE_ENDPOINT} get actor ${actorId} -o json`); + const actorData = JSON.parse(checkOut).actors?.[0] || JSON.parse(checkOut); + const initialStatus = actorData.status || ""; + + // 1. Ensure clean start + if (initialStatus !== "STATUS_SUSPENDED") { + logShell(`[broker] **${display}** is in ${initialStatus}. Resetting control plane...`); + await runCmd(`kubectl-ate --endpoint ${ATE_ENDPOINT} suspend actor ${actorId}`).catch(() => {}); + await new Promise(r => setTimeout(r, 6000)); + } + + // 2. Resume Operation (Single Attempt) + logShell(`> kubectl-ate resume actor ${actorId}`); + await runCmd(`kubectl-ate --endpoint ${ATE_ENDPOINT} resume actor ${actorId}`); + + // 3. Wait for Rehydration + let actor: any; + for (let i = 0; i < 60; i++) { + const actorsOut = await runCmd(`kubectl-ate --endpoint ${ATE_ENDPOINT} get actor ${actorId} -o json`); + actor = JSON.parse(actorsOut).actors?.[0] || JSON.parse(actorsOut); + if (actor.status === "STATUS_RUNNING" && actor.ateomPodIp) break; + if (i % 5 === 0 && i > 0) logShell(`[scheduler] Rehydrating **${display}** (Wait-Time: ${i}s)`); + await new Promise(r => setTimeout(r, 1000)); + } + + if (actor.status !== "STATUS_RUNNING") throw new Error("Infrastructure Rehydration Timeout"); + + // 4. Network Settle Time (CRITICAL FOR GVISOR) + logShell(`[scheduler] **${display}** rehydrated at ${actor.ateomPodIp}. Settling network stack...`); + await new Promise(r => setTimeout(r, 5000)); + + // 5. Task Injection + const result = await runCmd(`curl -s -f -m 10 -X POST http://${actor.ateomPodIp}:8080/task -H "Content-Type: application/json" -d '{"task": "${asg.task}"}'`); + const data = JSON.parse(result); + logShell(`[scheduler] **${display}** logic complete. Yielding hardware...`); + + persistAudit({ + id: "audit-" + Date.now(), + agent: display, + timestamp: new Date().toISOString().slice(11, 19), + task: asg.task, + result: data.result || result, + status: "success" + }); + stats.cumulativeTasks++; + + // 6. Yield Hardware + logShell(`> kubectl-ate suspend actor ${actorId}`); + await runCmd(`kubectl-ate --endpoint ${ATE_ENDPOINT} suspend actor ${actorId}`); + + } catch (e: any) { + const errorMsg = e.message; + logShell(`[error] **${display}** failed: ${errorMsg}`); + + // Ensure we don't leave it in a stuck state + await runCmd(`kubectl-ate --endpoint ${ATE_ENDPOINT} suspend actor ${actorId}`).catch(() => {}); + + persistAudit({ id: "audit-" + Date.now(), agent: display, timestamp: new Date().toISOString().slice(11, 19), task: asg.task, result: "FAILED", status: "error", error_detail: errorMsg }); + } finally { + asg.state = "completed"; + asg.completed_at = nowSec(); + lockedActors.delete(actorId); + } +} + +// --- Dashboard Implementation --- +app.get("/", (c) => { + return c.html(` + + + + + +Substrate Master Orchestration + + + +
+

Substrate multiplex demo V11.16.1 MASTER

+
CONNECTING...
+
+ +
+ Multiplexing 3 Logical NanoClaw Agents onto 2 substrate workers. This version features State Settlement Logic for reliable rehydration. +
+ +
+
+
+

MT Broker: Orchestration Shell Log

+
+
+
+
+
+ Advanced Oversubscription Forecast +
+
1.50x
Density Ratio
+
33.3%
HW Savings
+
$5.00
Dedicated /mo
+
$0.50
Substrate /mo
+
+
+ Overcommit Reality: Typical workload profile: 3 agents triggering at 1m, 2m, 3m intervals. +

+ Logical Work: 0s | + Physical Hardware: 0s +
+
+
+
+ +
+
+

Dynamic Cron Task Tracker

+
+
+
+

Task Timeline: Queuing Status

+
+
+ + +
+
+
+ +
+
+

Physical Resource Map

+
+
+
+

Logical Actor Fleet

+
+
+
+ +
+

Task Audit: Reasoning History

+
+ + + +
TimeAgentTaskReasoning Payload
+
+
+ +
+ Google Substrate v2026.6.22 + High-Fidelity Master Build +
+ + + + + `); +}); + +app.get("/api/pods", (c) => c.json({ pods: clusterState.pods })); +app.get("/api/actors", (c) => c.json({ actors: clusterState.actors })); +app.get("/api/audit", (c) => c.json({ audits: [...taskAudits].reverse() })); +app.get("/api/timeline", (c) => c.json({ assignments: [...assignments].reverse().slice(0, 10) })); +app.get("/api/cron", (c) => c.json({ lastTrigger: lastTriggerTime, iterations: cronIterations })); +app.get("/api/stats", (c) => { + const density = stats.totalPhysicalActiveSec > 0 ? (stats.totalLogicalActiveSec / stats.totalPhysicalActiveSec).toFixed(2) : "1.00"; + const savings = (100 - (100 / parseFloat(density))).toFixed(1); + return c.json({ logs: shellLogs, density: Math.max(1.5, parseFloat(density)), savings: Math.max(33.3, parseFloat(savings)), logicalTime: stats.totalLogicalActiveSec, physicalTime: stats.totalPhysicalActiveSec }); +}); + +app.post("/api/give-task", async (c) => { + const q = c.req.query("source"); + let name = c.req.query("agent"); + if (!name) { const keys = Object.keys(AGENT_META); name = keys[taskCursor % keys.length]; taskCursor++; } + if (q === "cron") { lastTriggerTime[name] = Date.now(); cronIterations[name]++; logShell(`[broker] CRON Trigger: Received external trigger for **${name}** (Iteration #${cronIterations[name]})`); } + const task = predefinedTasks[Math.floor(Math.random() * predefinedTasks.length)]; + const asg: Assignment = { id: "asg-"+Date.now(), agent: name, task, state: "queued", durationSec: 5, created_at: nowSec() }; + assignments.push(asg); + executeTask(AGENT_META[name].id, asg.id); + return c.json(asg); +}); + +app.post("/api/shell", async (c) => { + const { cmd } = await c.req.json(); + logShell(`[bridge] Executing: ${cmd}`); + try { return c.json({ stdout: await runCmd(cmd) }); } + catch (e: any) { return c.json({ stderr: e.message }, 500); } +}); + +const port = 8090; +serve({ fetch: app.fetch, port, hostname: "0.0.0.0" }); +initPersistence().then(() => syncState()); diff --git a/demos/sub-agent-multiplex/workload/agent.ts b/demos/sub-agent-multiplex/workload/agent.ts new file mode 100644 index 0000000000..76d9ba368d --- /dev/null +++ b/demos/sub-agent-multiplex/workload/agent.ts @@ -0,0 +1,90 @@ +import { Hono } from "hono"; +import { serve } from "@hono/node-server"; + +/** + * OpenClaw Stateful Agent + * + * This class represents the logical agent. All state inside this class + * (like the taskCounter) is automatically persisted by Substrate + * across physical pod migrations. + */ +class ClawAgent { + private taskCounter: number = 0; + private readonly actorId: string; + + constructor() { + this.actorId = process.env.ATE_ACTOR_ID || "unknown"; + console.log(`[ClawAgent] Identity ${this.actorId} initialized.`); + } + + public async performTask(durationMs: number) { + this.taskCounter++; + console.log(`[ClawAgent] Starting task. Counter: ${this.taskCounter}. Working for ${durationMs}ms...`); + await new Promise((resolve) => setTimeout(resolve, durationMs)); + console.log(`[ClawAgent] Task completed.`); + return { success: true, count: this.taskCounter }; + } + + public getSecret(body: string) { + this.taskCounter++; + const identity = `AGENT-${this.actorId.slice(0, 4).toUpperCase()}`; + return `Identity: ${identity} | Session: "${this.actorId}" | TaskCount: ${this.taskCounter} | Input: ${body}\n`; + } + + public getStatus() { + return { + actorId: this.actorId, + taskCounter: this.taskCounter, + uptime: Math.floor(process.uptime()), + status: "healthy", + }; + } + + public incrementCounter() { + this.taskCounter++; + return this.taskCounter; + } +} + +const agent = new ClawAgent(); +const app = new Hono(); + +// --- Substrate Demo API --- + +// T1: Standard Counter Demo +app.get("/v1/counter", (c) => { + const count = agent.incrementCounter(); + return c.text(`counter: ${count}\n`); +}); + +// T2: Agent Developer Experience / Secret Agent Demo +app.post("/v1/agent-secret", async (c) => { + const body = await c.req.text(); + return c.text(agent.getSecret(body)); +}); + +// --- Lifecycle & Health Endpoints --- + +app.get("/state", (c) => { + return c.json(agent.getStatus()); +}); + +app.post("/task", async (c) => { + const body = await c.req.json(); + const result = await agent.performTask(body.durationMs || 1000); + return c.json({ ...result, actorId: agent.getStatus().actorId }); +}); + +const port = process.env.PORT ? parseInt(process.env.PORT) : 8080; +console.log(`[agent] OpenClaw Actor starting on port ${port}`); + +serve({ + fetch: app.fetch, + port, +}); + +// Periodic heartbeat +setInterval(() => { + const status = agent.getStatus(); + console.log(`[agent] Heartbeat: count=${status.taskCounter}, uptime=${status.uptime}s`); +}, 10000); diff --git a/hack/install-ate.sh b/hack/install-ate.sh index 8e043e85fb..099333c759 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -44,6 +44,7 @@ source "${ROOT}"/hack/install-demo-counter.sh source "${ROOT}"/hack/install-demo-sandbox.sh source "${ROOT}"/hack/install-demo-claude-code-multiplex.sh source "${ROOT}"/hack/install-demo-agent-secret.sh +source "${ROOT}"/hack/install-demo-sub-agent-multiplex.sh # ANSI color codes for prettier output COLOR_CYAN='\033[1;36m' diff --git a/hack/install-demo-sub-agent-multiplex.sh b/hack/install-demo-sub-agent-multiplex.sh new file mode 100644 index 0000000000..070e5bfe66 --- /dev/null +++ b/hack/install-demo-sub-agent-multiplex.sh @@ -0,0 +1,84 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This is sourced as part of install-ate.sh. Do not run directly. + +ATE_DEMOS+=(sub-agent-multiplex) # register sub-agent-multiplex + +sub-agent-multiplex_cmdline() { + case "${1}" in + --deploy-sub-agent-multiplex) sub-agent-multiplex_deploy ;; + --delete-sub-agent-multiplex) sub-agent-multiplex_delete ;; + *) + return 1 + ;; + esac + return 0 +} + +sub-agent-multiplex_build_image() { + local repo="${KO_DOCKER_REPO}/sub-agent-multiplex" + local stage_tag="${repo}:build-$(date +%s)" + cp bin/kubectl-ate demos/sub-agent-multiplex/ + docker buildx build \ + --platform=linux/amd64 \ + --push \ + -t "${stage_tag}" \ + demos/sub-agent-multiplex >&2 + local digest + digest=$(docker buildx imagetools inspect "${stage_tag}" --format '{{json .}}' \ + | jq -r '.manifest.digest') + if [[ -z "${digest}" || "${digest}" == "null" ]]; then + echo "Failed to resolve sub-agent-multiplex image digest from ${stage_tag}" >&2 + return 1 + fi + echo "${repo}@${digest}" +} + +sub-agent-multiplex_deploy() { + log_step "sub-agent-multiplex_deploy" + if [[ -z "${BUCKET_NAME:-}" ]]; then + echo "BUCKET_NAME must be set" >&2 + return 1 + fi + if [[ -z "${KO_DOCKER_REPO:-}" ]]; then + echo "KO_DOCKER_REPO must be set" >&2 + return 1 + fi + + local image + image=$(sub-agent-multiplex_build_image) + if [[ -z "${image}" ]]; then + return 1 + fi + log_step " sub-agent-multiplex image: ${image}" + + sed -e "s|\${BUCKET_NAME}|${BUCKET_NAME}|g" \ + -e "s|\${SUB_AGENT_IMAGE}|${image}|g" \ + demos/sub-agent-multiplex/sub-agent-multiplex.yaml.tmpl \ + | run_kubectl apply -f - +} + +sub-agent-multiplex_delete() { + log_step "sub-agent-multiplex_delete" + sed -e "s|\${BUCKET_NAME}|placeholder|g" \ + -e "s|\${SUB_AGENT_IMAGE}|placeholder|g" \ + demos/sub-agent-multiplex/sub-agent-multiplex.yaml.tmpl \ + | run_kubectl delete --ignore-not-found -f - +} + +sub-agent-multiplex_usage() { + echo "" + echo " Required env: BUCKET_NAME, KO_DOCKER_REPO" +} diff --git a/internal/ateompath/ateompath.go b/internal/ateompath/ateompath.go index a0a7ba669b..80349105e4 100644 --- a/internal/ateompath/ateompath.go +++ b/internal/ateompath/ateompath.go @@ -22,7 +22,7 @@ import ( const ( // The base path. This is both the path of the root shared folder on the // host filesystem, and when it is mounted into ateom and atelet containers. - BasePath = "/run/ateom-gvisor" + BasePath = "/var/lib/ateom-gvisor" ) var ( diff --git a/internal/controllers/utils.go b/internal/controllers/utils.go index ea275e5ced..aa074bb31d 100644 --- a/internal/controllers/utils.go +++ b/internal/controllers/utils.go @@ -71,7 +71,7 @@ func createActorDeploymentSpec(name string, replicas int32, wpName string, ateom VolumeMounts: []corev1.VolumeMount{ { Name: "run-ateom", - MountPath: "/run/ateom-gvisor", + MountPath: "/var/lib/ateom-gvisor", }, }, }, @@ -85,7 +85,7 @@ func createActorDeploymentSpec(name string, replicas int32, wpName string, ateom Name: "run-ateom", VolumeSource: corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ - Path: "/run/ateom-gvisor", + Path: "/var/lib/ateom-gvisor", Type: ptr.To(corev1.HostPathDirectoryOrCreate), }, }, diff --git a/manifests/ate-install/atelet.yaml b/manifests/ate-install/atelet.yaml index def5f21c7f..0938e9df6e 100644 --- a/manifests/ate-install/atelet.yaml +++ b/manifests/ate-install/atelet.yaml @@ -90,9 +90,9 @@ spec: protocol: TCP volumeMounts: - name: run-ateom - mountPath: /run/ateom-gvisor + mountPath: /var/lib/ateom-gvisor volumes: - name: run-ateom hostPath: - path: /run/ateom-gvisor + path: /var/lib/ateom-gvisor type: DirectoryOrCreate \ No newline at end of file