From 94f84a6ac4238c916b5ceac0689a2bb2d853bef5 Mon Sep 17 00:00:00 2001 From: sallyom Date: Sat, 10 Oct 2026 09:56:17 -0400 Subject: [PATCH 01/10] feat(openshell): deliver native Harness setup through runtime providers Signed-off-by: sallyom Co-authored-by: sallyom <11166065+sallyom@users.noreply.github.com> --- tests/helpers/openshell-kubernetes-real.mjs | 26 +- ...sandbox-driver-openshell-k3d-real.test.mjs | 974 ++---------------- 2 files changed, 115 insertions(+), 885 deletions(-) diff --git a/tests/helpers/openshell-kubernetes-real.mjs b/tests/helpers/openshell-kubernetes-real.mjs index 2509bab878..de69d48887 100644 --- a/tests/helpers/openshell-kubernetes-real.mjs +++ b/tests/helpers/openshell-kubernetes-real.mjs @@ -964,20 +964,26 @@ export function createOpenShellKubernetesFixture({ ); } - function harnessContainer(pod) { + function harnessContainer(pod, harnessId = "codex") { const container = pod.spec.containers.find(({ name }) => name === "agent"); - assert.ok(container, "provider-owned Pod must contain the Codex Harness container."); + assert.ok(container, "provider-owned Pod must contain its Harness container."); const environment = container.env ?? []; assert.equal( environment.some(({ name }) => name === "APP_SERVER_TOKEN"), false, "the raw app-server token must stay outside the OpenShell Harness Pod spec.", ); - assert.match( - environment.find(({ name }) => name === "APP_TOKEN_SHA")?.value ?? "", - /^[a-f0-9]{64}$/, - "the OpenShell Harness requires only its app-server token verifier.", - ); + const verifier = environment.find(({ name }) => name === "APP_TOKEN_SHA"); + if (harnessId === "openclaw") { + assert.equal(verifier, undefined, "native OpenClaw must not receive a Codex verifier."); + } else { + assert.equal(harnessId, "codex", "unsupported Harness privilege check"); + assert.match( + verifier?.value ?? "", + /^[a-f0-9]{64}$/, + "the OpenShell Codex Harness requires only its app-server token verifier.", + ); + } return container; } @@ -1102,7 +1108,7 @@ export function createOpenShellKubernetesFixture({ assert.equal(claims.sub, `system:serviceaccount:${namespace}:${pod.spec.serviceAccountName}`); } - function assertApprovedOpenShellPrivileges(pod, { compatibilityBridge = false } = {}) { + function assertApprovedOpenShellPrivileges(pod, harnessId = "codex") { assert.equal( pod.spec.runtimeClassName, openShellRuntimeClass, @@ -1126,9 +1132,7 @@ export function createOpenShellKubernetesFixture({ undefined, "OpenShell v0.1.4 must keep its network supervisor outside the workload Pod.", ); - const container = compatibilityBridge - ? pod.spec.containers.find(({ name }) => name === "agent") - : harnessContainer(pod); + const container = harnessContainer(pod, harnessId); assert.ok(container, "the provider-owned Pod must contain its Agent container."); assert.equal(container.securityContext?.allowPrivilegeEscalation, false); assert.deepEqual(container.securityContext?.capabilities?.drop, ["ALL"]); diff --git a/tests/integration/sandbox-driver-openshell-k3d-real.test.mjs b/tests/integration/sandbox-driver-openshell-k3d-real.test.mjs index c810456c7f..57aaf26c88 100644 --- a/tests/integration/sandbox-driver-openshell-k3d-real.test.mjs +++ b/tests/integration/sandbox-driver-openshell-k3d-real.test.mjs @@ -1,6 +1,4 @@ import { defaultAgentModel } from "../../apps/controller/src/console/agents/starter-model.mjs"; -import { RUNTIME_WRAPPER_COMMAND } from "../../apps/controller/src/drivers/compute/kubernetes/runtime-entrypoints.ts"; -import { nodeProgramArguments } from "../../apps/controller/src/drivers/compute/node-program.ts"; import { openShellProviderName } from "../../apps/controller/src/backends/openshell.ts"; import { failureSecrets, @@ -23,7 +21,6 @@ import test from "node:test"; import { setTimeout as delay } from "node:timers/promises"; import { connect as connectTls } from "node:tls"; import { promisify } from "node:util"; -import { inflateRawSync } from "node:zlib"; import { createAuthenticatedControllerRequest } from "../helpers/auth-session.mjs"; import { ensureDevelopmentBootstrap } from "../helpers/bootstrap-installation.mjs"; import { createHarnessConfiguration } from "../helpers/harness-configuration.mjs"; @@ -32,7 +29,6 @@ import { createOpenShellInstallationConfiguration, createOpenShellKubernetesFixture, createOpenShellServiceLoopbackLookup, - OPENSHELL_KUBERNETES_WORKLOAD_IDENTITY, openShellAgentName, openShellGatewayName, openshellHash as hash, @@ -138,17 +134,7 @@ const adminCredentials = Object.freeze({ email: "admin-openshell-sandboxdriver@example.test", password: "openshell-sandboxdriver-admin-password", }); -const credentialMountPath = "/run/enterprise-credentials"; -const pluginRuntimeMountPath = "/etc/openclaw/plugin-runtime"; -const runtimeAssetsMountPath = "/home/node/openclaw-runtime-assets"; -const nativeTemporaryMountPath = "/tmp/openclaw-native-worker"; -const nodeStateMountPath = "/home/node/.openclaw-node"; -// Kubernetes Compute backs all of /home/node with an emptyDir; stock OpenShell has no equivalent, -// so the bridge stages the native state root where the Agent entrypoint publishes plugin skills. -const openclawHomeMountPath = "/home/node/.openclaw"; -const bridgedNodeStateMountPath = "/openclaw-node-state"; const workspaceMountPath = "/home/node/workspace"; -const bridgedWorkspaceSubPath = "workspace/openshell-home"; const demoStatePath = process.env.OCC_K3D_DEMO_STATE?.trim() || undefined; const demoControlUiPort = 18_888; const demoConsolePort = 18_889; @@ -160,12 +146,8 @@ if (demoStatePath !== undefined) { process.once("SIGINT", resolveDemoStop); process.once("SIGTERM", resolveDemoStop); } -const portableCommandArgumentBytes = 30 * 1024; const diagnosticQueryTimeoutMs = 3_000; const observerPoolConnectionTimeoutMs = 5_000; -const controllerRequire = createRequire( - new URL("../../apps/controller/package.json", import.meta.url), -); const fixture = createOpenShellKubernetesFixture({ kubeconfigPath, @@ -796,320 +778,6 @@ async function assertWorkerCompleted(options) { } } -// TODO(OpenShell native workload projections): remove this integration-only bridge once -// OpenShell can receive SecretKeyRefs, projected ServiceAccount tokens, and immutable -// plugin-runtime ConfigMaps directly from HarnessWorkloadRequirements. -function credentialJobName(revisionId) { - return `openshell-cred-${hash(revisionId)}`; -} - -function bridgedNodeStateParent(agentId) { - return `.openclaw/openshell-bootstrap/nodes/${hash(agentId, 32)}`; -} - -function pluginRuntimeConfigMapName(context) { - return `plugin-runtime-${hash(context.revision.agentId)}-rev-${hash(context.revision.id)}`; -} - -function optionalSecretEnvironment(requirements, name) { - const variable = requirements.environment.find((entry) => entry.name === name); - if (variable === undefined) { - return undefined; - } - assert.ok(variable.valueFrom?.secretKeyRef, `${name} must come from an exact SecretKeyRef.`); - return variable; -} - -function literalEnvironment(requirements, name) { - const variable = requirements.environment.find((entry) => entry.name === name); - assert.equal(typeof variable?.value, "string", `${name} must be a literal environment value.`); - assert.equal(variable.valueFrom, undefined, `${name} must not reference a Secret.`); - return variable; -} - -async function waitForCredentialJob(operatorKubernetes, context, name, namespaceName) { - const deadline = Date.now() + 180_000; - while (Date.now() < deadline) { - context.signal.throwIfAborted(); - const job = await operatorKubernetes.read({ - apiVersion: "batch/v1", - kind: "Job", - metadata: { namespace: namespaceName, name }, - }); - const conditions = Array.isArray(job?.status?.conditions) ? job.status.conditions : []; - if ( - conditions.some((condition) => condition.type === "Complete" && condition.status === "True") - ) { - return; - } - const failed = conditions.find( - (condition) => condition.type === "Failed" && condition.status === "True", - ); - assert.equal(failed, undefined, `OpenShell bootstrap Job ${name} failed.`); - await delay(750, undefined, { signal: context.signal }); - } - assert.fail(`Timed out waiting for OpenShell bootstrap Job ${name}.`); -} - -async function applyCredentialJob(operatorKubernetes, resource) { - return operatorKubernetes.patch( - resource, - undefined, - undefined, - "openclaw-enterprise-compute", - false, - "application/apply-patch+yaml", - ); -} - -async function deleteCredentialJob( - operatorKubernetes, - context, - name, - namespace = context.namespace.name, -) { - await operatorKubernetes.delete({ - apiVersion: "batch/v1", - kind: "Job", - metadata: { namespace, name }, - }); -} - -async function waitForCredentialJobDeletion(operatorKubernetes, context, name) { - const deadline = Date.now() + 60_000; - while (Date.now() < deadline) { - context.signal.throwIfAborted(); - try { - await operatorKubernetes.read({ - apiVersion: "batch/v1", - kind: "Job", - metadata: { namespace: context.namespace.name, name }, - }); - } catch (error) { - if (error.code === 404 || error.statusCode === 404) { - return; - } - throw error; - } - await delay(250, undefined, { signal: context.signal }); - } - assert.fail(`Timed out waiting for OpenShell bootstrap Job ${name} deletion.`); -} - -function credentialBridgeResource(context, claimName, subPath) { - const { uid: runtimeUser, gid: runtimeGroup } = OPENSHELL_KUBERNETES_WORKLOAD_IDENTITY; - const namespaceName = context.namespace.name; - const name = credentialJobName(context.revision.id); - const workloadIdentity = context.requirements.workloadIdentity; - assert.ok(workloadIdentity, "the native compatibility bridge requires projected identity."); - const servicePrincipalToken = workloadIdentity.token; - assert.match( - servicePrincipalToken.path, - /^(?!.*(?:^|\/)\.\.(?:\/|$))[A-Za-z0-9._/-]+$/, - "the OpenShell bridge requires a safe relative ServiceAccount token path.", - ); - const tokenParent = servicePrincipalToken.path.includes("/") - ? servicePrincipalToken.path.slice(0, servicePrincipalToken.path.lastIndexOf("/")) - : "."; - const appServerToken = optionalSecretEnvironment(context.requirements, "APP_SERVER_TOKEN"); - const needsPluginRuntime = context.revision.harness.id === "codex"; - const needsNativeTemporary = context.revision.harness.id === "openclaw"; - if (needsNativeTemporary) { - assert.equal( - literalEnvironment(context.requirements, "TMPDIR").value, - nativeTemporaryMountPath, - ); - } - const nodeSetupCode = optionalSecretEnvironment(context.requirements, "OPENCLAW_NODE_SETUP_CODE"); - const nodeCa = literalEnvironment(context.requirements, "OPENCLAW_NODE_CA_PEM"); - return { - apiVersion: "batch/v1", - kind: "Job", - metadata: { - name, - namespace: namespaceName, - labels: { - ...context.requirements.labels, - "openclaw.dev/workload-role": "sandbox-bootstrap-bridge", - }, - annotations: { - "openclaw.dev/namespace-id": context.revision.namespaceId, - "openclaw.dev/agent-id": context.revision.agentId, - "openclaw.dev/revision-id": context.revision.id, - }, - }, - spec: { - backoffLimit: 0, - // The copied ServiceAccount token remains bound to this completed Job's Pod. - ttlSecondsAfterFinished: servicePrincipalToken.expirationSeconds, - template: { - metadata: { - labels: { - ...context.requirements.labels, - "openclaw.dev/workload-role": "sandbox-bootstrap-bridge", - }, - }, - spec: { - restartPolicy: "Never", - serviceAccountName: workloadIdentity.serviceAccountName, - automountServiceAccountToken: false, - securityContext: { - runAsNonRoot: true, - runAsUser: runtimeUser, - runAsGroup: runtimeGroup, - fsGroup: runtimeGroup, - seccompProfile: { type: "RuntimeDefault" }, - }, - containers: [ - { - name: "write-openshell-bootstrap", - image: context.requirements.image, - imagePullPolicy: "IfNotPresent", - command: [ - "sh", - "-ceu", - [ - "umask 077", - ...(needsPluginRuntime ? ["mkdir -p /bootstrap/plugin-runtime"] : []), - // Match Compute's Agent-scoped node identity so a replacement reconnects - // with the device already recorded by the Gateway instead of redeeming twice. - "mkdir -p /agent-node-state/node-state", - "mkdir -p /bootstrap/runtime-assets", - "mkdir -p /bootstrap/openclaw-home", - ...(needsNativeTemporary ? ["mkdir -p /bootstrap/native-tmp"] : []), - `mkdir -p /bootstrap/service-principal/${tokenParent}`, - "mkdir -p /workspace-home/openshell-home/.codex", - ...(needsPluginRuntime - ? ["chmod 0700 /bootstrap/plugin-runtime /bootstrap/service-principal"] - : ["chmod 0700 /bootstrap/service-principal"]), - "chmod 0700 /agent-node-state/node-state", - ...(needsNativeTemporary ? ["chmod 0700 /bootstrap/native-tmp"] : []), - "chmod 0600 /bootstrap/app-server-token /bootstrap/openclaw-node-setup-code /bootstrap/openclaw-node-ca.pem 2>/dev/null || true", - ...(needsPluginRuntime - ? [ - "chmod 0600 /bootstrap/plugin-runtime/runtime.json /bootstrap/plugin-runtime/config.toml 2>/dev/null || true", - ] - : []), - `chmod 0600 /bootstrap/service-principal/${servicePrincipalToken.path} 2>/dev/null || true`, - ...(appServerToken === undefined - ? [] - : ['printf "%s" "$APP_SERVER_TOKEN" > /bootstrap/app-server-token']), - ...(nodeSetupCode === undefined - ? [] - : [ - 'printf "%s" "$OPENCLAW_NODE_SETUP_CODE" > /bootstrap/openclaw-node-setup-code', - ]), - 'printf "%s" "$OPENCLAW_NODE_CA_PEM" > /bootstrap/openclaw-node-ca.pem', - ...(needsPluginRuntime - ? [ - "cp /source-plugin-runtime/runtime.json /bootstrap/plugin-runtime/runtime.json", - "cp /source-plugin-runtime/config.toml /bootstrap/plugin-runtime/config.toml", - ] - : []), - `cp /source-service-principal/${servicePrincipalToken.path} /bootstrap/service-principal/${servicePrincipalToken.path}`, - ...(appServerToken === undefined - ? [] - : ["chmod 0444 /bootstrap/app-server-token"]), - ...(nodeSetupCode === undefined - ? [] - : ["chmod 0444 /bootstrap/openclaw-node-setup-code"]), - "chmod 0444 /bootstrap/openclaw-node-ca.pem", - ...(needsPluginRuntime - ? [ - "chmod 0444 /bootstrap/plugin-runtime/runtime.json /bootstrap/plugin-runtime/config.toml", - ] - : []), - `chmod 0444 /bootstrap/service-principal/${servicePrincipalToken.path}`, - ...(needsPluginRuntime - ? ["chmod 0555 /bootstrap/plugin-runtime /bootstrap/service-principal"] - : ["chmod 0555 /bootstrap/service-principal"]), - "chmod 0700 /bootstrap/runtime-assets /bootstrap/openclaw-home", - "chmod 0700 /workspace-home/openshell-home /workspace-home/openshell-home/.codex", - ].join("\n"), - ], - env: [ - ...(appServerToken === undefined ? [] : [appServerToken]), - ...(nodeSetupCode === undefined ? [] : [nodeSetupCode]), - nodeCa, - ], - volumeMounts: [ - { name: "bootstrap", mountPath: "/bootstrap", subPath }, - { name: "bootstrap", mountPath: "/workspace-home", subPath: "workspace" }, - { - name: "node-state-bootstrap", - mountPath: "/agent-node-state", - subPath: bridgedNodeStateParent(context.revision.agentId), - }, - ...(needsPluginRuntime - ? [ - { - name: "plugin-runtime", - mountPath: "/source-plugin-runtime", - readOnly: true, - }, - ] - : []), - { - name: "service-principal", - mountPath: "/source-service-principal", - readOnly: true, - }, - ], - securityContext: { - allowPrivilegeEscalation: false, - readOnlyRootFilesystem: true, - capabilities: { drop: ["ALL"] }, - }, - }, - ], - volumes: [ - { name: "bootstrap", persistentVolumeClaim: { claimName } }, - { name: "node-state-bootstrap", persistentVolumeClaim: { claimName } }, - ...(needsPluginRuntime - ? [ - { - name: "plugin-runtime", - configMap: { - name: pluginRuntimeConfigMapName(context), - items: [ - { key: "runtime.json", path: "runtime.json" }, - { key: "config.toml", path: "config.toml" }, - ], - optional: false, - }, - }, - ] - : []), - { - name: "service-principal", - projected: { - sources: [ - { - serviceAccountToken: { - audience: servicePrincipalToken.audience, - expirationSeconds: servicePrincipalToken.expirationSeconds, - path: servicePrincipalToken.path, - }, - }, - ], - }, - }, - ], - }, - }, - }, - }; -} - -function inlineNodeProgramIndex(command) { - assert.deepEqual( - command.slice(0, RUNTIME_WRAPPER_COMMAND.length), - [...RUNTIME_WRAPPER_COMMAND], - "the OpenShell bridge expects the Compute runtime wrapper command.", - ); - return RUNTIME_WRAPPER_COMMAND.length; -} - function assertNoProjectedAgentIdentity(pod) { const container = pod.spec.containers.find(({ name }) => name === "agent"); assert.ok(container, "the OpenShell Sandbox must provide its Agent container."); @@ -1129,222 +797,6 @@ function assertNoProjectedAgentIdentity(pod) { ); } -function portableProgramPieces(program) { - const chunks = []; - let chunk = ""; - let bytes = 0; - for (const character of program) { - const characterBytes = Buffer.byteLength(character); - if (bytes + characterBytes > portableCommandArgumentBytes && chunk.length > 0) { - chunks.push(chunk); - chunk = ""; - bytes = 0; - } - chunk += character; - bytes += characterBytes; - } - if (chunk.length > 0) { - chunks.push(chunk); - } - assert.ok(chunks.length > 0, "the OpenShell bridge requires a non-empty runtime entrypoint."); - assert.equal( - chunks.every((entry) => Buffer.byteLength(entry) <= portableCommandArgumentBytes), - true, - "every OpenShell runtime command chunk must remain below the upstream argument limit.", - ); - return chunks; -} - -function portableRuntimeCommand(command) { - const programIndex = inlineNodeProgramIndex(command); - const runtimeArguments = command.slice(programIndex); - const nodeProgramLoader = nodeProgramArguments("")[0]; - if (runtimeArguments[0] === nodeProgramLoader) { - // Preserve Compute's fixed loader while splitting the compressed payload below - // OpenShell's smaller argument limit. - const payload = runtimeArguments.slice(1).join(""); - return [ - ...command.slice(0, programIndex), - runtimeArguments[0], - ...portableProgramPieces(payload), - ]; - } - assert.equal( - runtimeArguments.length, - 1, - "the OpenShell bridge expects one inline Node program or the bounded program loader.", - ); - return [ - ...command.slice(0, programIndex), - 'eval(process.argv.slice(1).join(""))', - ...portableProgramPieces(runtimeArguments[0]), - ]; -} - -function bridgeRequirements(context, claimName, subPath) { - // The model key reaches Codex only through the Credential Gateway: Compute renders no model - // Secret. The Sandbox receives the model source's attachment first, then one for each - // other source the revision still holds; a withdrawn one is never re-attached. - assert.equal( - context.requirements.environment.some(({ name }) => - ["OPENAI_API_KEY", "CODEX_ACCESS_TOKEN"].includes(name), - ), - false, - "Compute must not project model credentials when a credential source authenticates the Harness.", - ); - const attached = context.requirements.credentialAttachments.map(({ sourceId }) => sourceId); - assert.equal(attached[0], context.revision.harnessAuth.sourceId); - const admitted = (context.revision.credentialSources ?? []) - .map(({ sourceId }) => sourceId) - .filter((sourceId) => sourceId !== context.revision.harnessAuth.sourceId); - assert.deepEqual( - attached.slice(1), - admitted.filter((sourceId) => attached.includes(sourceId)), - "other attachments must follow the model source in admission order", - ); - const workloadIdentity = context.requirements.workloadIdentity; - assert.ok(workloadIdentity, "the native compatibility bridge requires projected identity."); - const servicePrincipalToken = workloadIdentity.token; - const appServerToken = optionalSecretEnvironment(context.requirements, "APP_SERVER_TOKEN"); - const needsPluginRuntime = context.revision.harness.id === "codex"; - const needsNativeTemporary = context.revision.harness.id === "openclaw"; - const effectiveNodeStateMountPath = needsNativeTemporary - ? bridgedNodeStateMountPath - : nodeStateMountPath; - if (needsNativeTemporary) { - assert.equal( - literalEnvironment(context.requirements, "TMPDIR").value, - nativeTemporaryMountPath, - ); - } - const nodeSetupCode = optionalSecretEnvironment(context.requirements, "OPENCLAW_NODE_SETUP_CODE"); - literalEnvironment(context.requirements, "OPENCLAW_NODE_CA_PEM"); - const credentialBootstrap = [ - // The Driver keeps launch-time TMPDIR at /tmp for the supervisor; after - // launch, the native worker must use the private mount it owns. - ...(needsNativeTemporary - ? [`process.env.TMPDIR = ${JSON.stringify(nativeTemporaryMountPath)};`] - : []), - ...(appServerToken === undefined - ? [] - : [ - `process.env.APP_SERVER_TOKEN = require("node:fs").readFileSync("${credentialMountPath}/app-server-token", "utf8");`, - ]), - ...(nodeSetupCode === undefined - ? [] - : [ - `process.env.OPENCLAW_NODE_SETUP_CODE = require("node:fs").readFileSync("${credentialMountPath}/openclaw-node-setup-code", "utf8");`, - ]), - `process.env.OPENCLAW_NODE_CA_PEM = require("node:fs").readFileSync("${credentialMountPath}/openclaw-node-ca.pem", "utf8");`, - ].join("\n"); - const runtimeCommand = structuredClone(context.requirements.command); - const programIndex = inlineNodeProgramIndex(runtimeCommand); - assert.equal(runtimeCommand[programIndex], nodeProgramArguments("")[0]); - const runtimeProgram = inflateRawSync( - Buffer.from(runtimeCommand.slice(programIndex + 1).join(""), "base64"), - ).toString("utf8"); - // The Driver admits only the fixed loader; bootstrap belongs in its compressed program. - runtimeCommand.splice( - programIndex, - runtimeCommand.length - programIndex, - ...nodeProgramArguments(`${credentialBootstrap}\n${runtimeProgram}`), - ); - const environment = context.requirements.environment - .filter( - ({ name }) => - !["APP_SERVER_TOKEN", "OPENCLAW_NODE_SETUP_CODE", "OPENCLAW_NODE_CA_PEM"].includes(name), - ) - .map((entry) => { - if (entry.name === "HOME") { - return { name: "HOME", value: "/home/node/workspace" }; - } - if (entry.name === "CODEX_HOME") { - return { name: "CODEX_HOME", value: "/home/node/workspace/.codex" }; - } - if (needsNativeTemporary && entry.name === "OPENCLAW_NODE_STATE_DIR") { - // Stock OpenShell runs the Agent as 10001 while /home/node belongs to the image user. - // A root-level mount keeps secure workspace-transfer ancestry root/current-user owned. - return { name: entry.name, value: bridgedNodeStateMountPath }; - } - if (needsNativeTemporary && entry.name === "NODE_COMPILE_CACHE") { - return { name: entry.name, value: `${bridgedNodeStateMountPath}/.cache/node-compile` }; - } - return entry; - }); - return { - ...context.requirements, - workloadIdentity: undefined, - command: portableRuntimeCommand(runtimeCommand), - environment, - workspaceMounts: [ - ...context.requirements.workspaceMounts.filter( - ({ mountPath }) => mountPath !== workspaceMountPath && mountPath !== nodeStateMountPath, - ), - { - claimName, - subPath: bridgedWorkspaceSubPath, - mountPath: workspaceMountPath, - readOnly: false, - }, - { - claimName, - subPath: `${bridgedNodeStateParent(context.revision.agentId)}/node-state`, - mountPath: effectiveNodeStateMountPath, - readOnly: false, - }, - { claimName, subPath, mountPath: credentialMountPath, readOnly: true }, - ...(needsPluginRuntime - ? [ - { - claimName, - subPath: `${subPath}/plugin-runtime`, - mountPath: pluginRuntimeMountPath, - readOnly: true, - }, - ] - : []), - { - claimName, - subPath: `${subPath}/service-principal`, - mountPath: servicePrincipalToken.mountPath, - readOnly: true, - }, - { - claimName, - subPath: `${subPath}/runtime-assets`, - mountPath: runtimeAssetsMountPath, - readOnly: false, - }, - ...(needsNativeTemporary - ? [ - { - claimName, - subPath: `${subPath}/native-tmp`, - mountPath: nativeTemporaryMountPath, - readOnly: false, - }, - ] - : []), - { - claimName, - subPath: `${subPath}/openclaw-home`, - mountPath: openclawHomeMountPath, - readOnly: false, - }, - ...(needsPluginRuntime - ? [ - { - claimName, - subPath: `${bridgedWorkspaceSubPath}/.codex`, - mountPath: "/home/node/.codex", - readOnly: false, - }, - ] - : []), - ], - }; -} - function integrationGatewayClient( GrpcOpenShellGatewayClient, endpoint, @@ -1440,96 +892,6 @@ async function assertModelKeyIsPlaceholderOnly(namespace, pod, modelKey) { ); } -async function assertBridgedServicePrincipalToken(namespace, pod, expected) { - const container = pod.spec.containers.find(({ name }) => name === "agent"); - assert.ok(container, "the OpenShell Sandbox must provide its Agent container."); - const mounts = (container.volumeMounts ?? []).filter( - ({ mountPath }) => mountPath === "/var/run/secrets/openclaw/service-principal", - ); - assert.equal(mounts.length, 1, "the bridged Harness requires its Agent token mount."); - assert.equal(mounts[0].readOnly, true); - const volume = (pod.spec.volumes ?? []).find(({ name }) => name === mounts[0].name); - assert.ok( - volume?.persistentVolumeClaim, - "stock OpenShell must receive the test-only ServiceAccount token through the shared PVC.", - ); - - // Validate the copied token without exposing it. This proves the bridge used the exact Agent - // ServiceAccount and audience, but intentionally does not claim native kubelet projection. - const script = [ - 'const token = require("node:fs").readFileSync(process.argv[1], "utf8").trim();', - 'const claims = JSON.parse(Buffer.from(token.split(".")[1], "base64url").toString());', - "process.stdout.write(JSON.stringify({ aud: claims.aud, sub: claims.sub }));", - ].join(" "); - const claims = JSON.parse( - await kubectl( - "exec", - pod.metadata.name, - "--namespace", - namespace, - "--container", - container.name, - "--", - "node", - "-e", - script, - "/var/run/secrets/openclaw/service-principal/token", - ), - ); - const audiences = Array.isArray(claims.aud) ? claims.aud : [claims.aud]; - assert.deepEqual(audiences, [expected.audience]); - assert.equal(claims.sub, `system:serviceaccount:${namespace}:${pod.spec.serviceAccountName}`); -} - -function bridgedHarnessContainer(pod) { - const container = pod.spec.containers.find(({ name }) => name === "agent"); - assert.ok(container, "the provider-owned Pod must contain the OpenShell Agent container."); - for (const name of ["APP_SERVER_TOKEN", "OPENAI_API_KEY", "CODEX_ACCESS_TOKEN"]) { - assert.equal( - (container.env ?? []).some((entry) => entry.name === name), - false, - `${name} must not remain in the stock OpenShell Pod environment.`, - ); - } - return container; -} - -function assertBridgedNativeStateMount(pod) { - const container = bridgedHarnessContainer(pod); - const environment = container.env ?? []; - assert.equal( - environment.find(({ name }) => name === "OPENCLAW_NODE_STATE_DIR")?.value, - bridgedNodeStateMountPath, - "the native bridge must keep secure state ancestry outside the image-owned home directory.", - ); - assert.equal( - environment.find(({ name }) => name === "NODE_COMPILE_CACHE")?.value, - `${bridgedNodeStateMountPath}/.cache/node-compile`, - ); - const nativeInference = JSON.parse( - environment.find(({ name }) => name === "OPENCLAW_NATIVE_INFERENCE_CONFIG")?.value, - ); - assert.ok(nativeInference.models.providers.openai.models.length > 0); - assert.deepEqual(nativeInference.models.providers.openai.apiKey, { - source: "env", - provider: "model", - id: "OPENAI_API_KEY", - }); - const mounts = container.volumeMounts ?? []; - assert.equal( - mounts.some( - ({ mountPath, readOnly }) => mountPath === bridgedNodeStateMountPath && readOnly !== true, - ), - true, - "the native bridge requires Agent-scoped writable node state.", - ); - assert.equal( - mounts.some(({ mountPath }) => mountPath === nodeStateMountPath), - false, - "the native bridge must not retain the mixed-owner image-home state mount.", - ); -} - function throwOpenShellAbortReason(signal) { if (!signal.aborted) { return; @@ -1541,8 +903,7 @@ function createIntegrationSandboxDriverFactory( OpenShellSandboxDriver, GrpcOpenShellGatewayClient, OpenShellGateway, - operatorKubernetes, - { enableCompatibilityBridges, gatewayTrustPem }, + { gatewayTrustPem }, ) { const gatewayState = new Map(); const endpointClients = new Map(); @@ -1611,7 +972,6 @@ function createIntegrationSandboxDriverFactory( }; const provisioningFailures = new Map(); const harnessServiceUrls = new Map(); - const credentialBridges = new Map(); async function stopGatewayForward(namespaceName, expectedState) { const state = gatewayState.get(namespaceName); @@ -1743,7 +1103,6 @@ function createIntegrationSandboxDriverFactory( context === undefined ? clientForEndpoint(endpoint) : integrationGatewayClient(GrpcOpenShellGatewayClient, endpoint, context, { - enableCompatibilityBridge: enableCompatibilityBridges, observeServiceUrl: (serviceUrl) => { harnessServiceUrls.set(context.revision.id, serviceUrl); }, @@ -1789,136 +1148,69 @@ function createIntegrationSandboxDriverFactory( } }, async provisionHarness(context) { - if ( - enableCompatibilityBridges && - optionalSecretEnvironment(context.requirements, "OPENCLAW_NODE_SETUP_CODE") === undefined - ) { - throw new Error( - "The OpenShell compatibility proof is waiting for the Gateway node setup projection.", - ); - } - let bridge; - try { - if (!enableCompatibilityBridges) { - // The gateway binds every Sandbox to one ServiceAccount, so reconfigure it for the - // Agent's own ServiceAccount, as the bridge path does. Without projected workload - // identity Compute renders no identity, but the Agent ServiceAccount still exists. - const endpoint = await endpointForNamespace(context, { - sandboxServiceAccountName: agentSandboxServiceAccount(context), - exact: context.requirements.workloadIdentity !== undefined, - }); - // The production Codex proof hands the real Driver exactly what Compute rendered; - // unsupported shapes fail closed before OpenShell creates provider resources. - return await delegate( - context.requirements, - context.namespace.name, - endpoint, - context, - ).provisionHarness(context); - } - - const claimName = context.requirements.workspaceMounts[0]?.claimName; - assert.ok(claimName, "OpenShell bootstrap requires the Agent shared workspace PVC."); - const subPath = `.openclaw/openshell-bootstrap/${hash(context.revision.id, 32)}`; - bridge = credentialBridgeResource(context, claimName, subPath); - const previousBridge = credentialBridges.get(context.revision.id); - if (previousBridge === undefined) { - await applyCredentialJob(operatorKubernetes, bridge); - await waitForCredentialJob( - operatorKubernetes, - context, - bridge.metadata.name, - context.namespace.name, - ); - // Readiness retries must not revoke the token by deleting its bound Pod. - credentialBridges.set(context.revision.id, bridge); - } - const requirements = bridgeRequirements(context, claimName, subPath); - const endpoint = await endpointForNamespace(context, { - sandboxServiceAccountName: context.requirements.workloadIdentity.serviceAccountName, - exact: true, - }); - const provisioning = { ...context, requirements }; - return await delegate( - requirements, - context.namespace.name, - endpoint, - provisioning, - ).provisionHarness(provisioning); - } catch (error) { - if (!provisioningFailures.has(context.revision.id)) { - process.stderr.write( - `OpenShell first provisioning failure for ${context.revision.id}: ${error.message}\n`, - ); - provisioningFailures.set(context.revision.id, error); - } - if (bridge !== undefined && !credentialBridges.has(context.revision.id)) { - await deleteCredentialJob(operatorKubernetes, context, bridge.metadata.name).catch( - () => undefined, - ); - await waitForCredentialJobDeletion( - operatorKubernetes, - context, - bridge.metadata.name, - ).catch(() => undefined); - } - throw error; - } + // The real Driver receives unmodified Compute requirements for both Harness kinds. + const endpoint = await endpointForNamespace(context, { + sandboxServiceAccountName: agentSandboxServiceAccount(context), + }); + return await delegate( + context.requirements, + context.namespace.name, + endpoint, + context, + ).provisionHarness(context); + }, + async harnessEndpoint(context) { + const endpoint = await endpointForNamespace(context, { + sandboxServiceAccountName: agentSandboxServiceAccount(context), + }); + return await delegate( + context.requirements, + context.namespace.name, + endpoint, + context, + ).harnessEndpoint(context); + }, + async harnessStatus(context) { + const endpoint = await endpointForNamespace(context, { + sandboxServiceAccountName: agentSandboxServiceAccount(context), + }); + return await delegate( + context.requirements, + context.namespace.name, + endpoint, + context, + ).harnessStatus(context); + }, + async readSandboxLogs(context, request) { + const endpoint = existingEndpointForNamespace(context); + return await delegate(undefined, context.namespace.name, endpoint).readSandboxLogs( + context, + request, + ); + }, + // Compute drops a native worker's exposure once its node has enrolled (finding 1657). + async withdrawHarnessExposure(context) { + const endpoint = await endpointForNamespace(context, { + sandboxServiceAccountName: agentSandboxServiceAccount(context), + exact: context.requirements.workloadIdentity !== undefined, + }); + await delegate( + context.requirements, + context.namespace.name, + endpoint, + context, + ).withdrawHarnessExposure(context); + }, + // Compute admits the workspace-node connection only from the revision's + // supervisor Pod, which the real Driver labels from the gateway's Sandbox record. + async workspaceNodePeerLabels(context) { + const endpoint = existingEndpointForNamespace(context); + return await delegate( + undefined, + context.namespace.name, + endpoint, + ).workspaceNodePeerLabels(context); }, - // Compute asks for the OpenShell-exposed Codex endpoint and points the Agent Gateway at it. - // The real Driver supports this only for dedicated Codex, so the bridge proof omits it. - ...(enableCompatibilityBridges - ? {} - : { - async harnessEndpoint(context) { - const endpoint = await endpointForNamespace(context, { - sandboxServiceAccountName: agentSandboxServiceAccount(context), - exact: context.requirements.workloadIdentity !== undefined, - }); - return await delegate( - context.requirements, - context.namespace.name, - endpoint, - context, - ).harnessEndpoint(context); - }, - // Compute activates only a Harness that completes the authenticated handshake. - async harnessStatus(context) { - const endpoint = await endpointForNamespace(context, { - sandboxServiceAccountName: agentSandboxServiceAccount(context), - exact: context.requirements.workloadIdentity !== undefined, - }); - return await delegate( - context.requirements, - context.namespace.name, - endpoint, - context, - ).harnessStatus(context); - }, - // Compute drops a native worker's exposure once its node has enrolled (finding 1657). - async withdrawHarnessExposure(context) { - const endpoint = await endpointForNamespace(context, { - sandboxServiceAccountName: agentSandboxServiceAccount(context), - exact: context.requirements.workloadIdentity !== undefined, - }); - await delegate( - context.requirements, - context.namespace.name, - endpoint, - context, - ).withdrawHarnessExposure(context); - }, - // Compute admits the workspace-node connection only from the revision's - // supervisor Pod, which the real Driver labels from the gateway's Sandbox record. - async workspaceNodePeerLabels(context) { - const endpoint = existingEndpointForNamespace(context); - return await delegate( - undefined, - context.namespace.name, - endpoint, - ).workspaceNodePeerLabels(context); - }, - }), async cleanup(context) { if (context.revision === undefined) { try { @@ -1929,68 +1221,8 @@ function createIntegrationSandboxDriverFactory( } return; } - try { - const endpoint = existingEndpointForNamespace(context); - await delegate(undefined, context.namespace.name, endpoint).cleanup(context); - } finally { - if (enableCompatibilityBridges) { - const bridge = credentialBridges.get(context.revision.id); - if (bridge !== undefined) { - const cleaner = structuredClone(bridge); - cleaner.metadata.name = `${bridge.metadata.name}-cleanup`; - const container = cleaner.spec.template.spec.containers[0]; - container.name = "delete-openshell-bootstrap"; - container.env = []; - // Cleanup contexts intentionally omit provisioning requirements. The bridge mount is - // revision-scoped, so remove its bounded contents without reconstructing token paths. - container.command = [ - "sh", - "-ceu", - [ - "if [ -d /bootstrap/plugin-runtime ]; then chmod -R u+w /bootstrap/plugin-runtime; fi", - "chmod -R u+w /bootstrap/service-principal", - "rm -f /bootstrap/app-server-token /bootstrap/openclaw-node-setup-code /bootstrap/openclaw-node-ca.pem", - "rm -rf /bootstrap/plugin-runtime /bootstrap/service-principal /bootstrap/runtime-assets /bootstrap/native-tmp /bootstrap/openclaw-home", - ].join("\n"), - ]; - container.volumeMounts = container.volumeMounts.filter( - ({ name }) => name === "bootstrap", - ); - cleaner.spec.template.spec.volumes = cleaner.spec.template.spec.volumes.filter( - ({ name }) => name === "bootstrap", - ); - try { - await applyCredentialJob(operatorKubernetes, cleaner); - await waitForCredentialJob( - operatorKubernetes, - context, - cleaner.metadata.name, - context.namespace.name, - ); - } finally { - await deleteCredentialJob( - operatorKubernetes, - context, - cleaner.metadata.name, - context.namespace.name, - ).catch(() => undefined); - await deleteCredentialJob( - operatorKubernetes, - context, - bridge.metadata.name, - context.namespace.name, - ).catch(() => undefined); - } - credentialBridges.delete(context.revision.id); - } - await deleteCredentialJob( - operatorKubernetes, - context, - credentialJobName(context.revision.id), - context.namespace.name, - ).catch(() => undefined); - } - } + const endpoint = existingEndpointForNamespace(context); + await delegate(undefined, context.namespace.name, endpoint).cleanup(context); }, }; }; @@ -2027,12 +1259,9 @@ function withFirstPrepareRevisionFailureDiagnostic(computeDriver) { } catch (error) { if (!reported) { reported = true; - const message = - error.message === - "The OpenShell compatibility proof is waiting for the Gateway node setup projection." - ? "OpenShell integration: waiting for the Gateway node setup projection; Compute will retry." - : `OpenShell integration: initial Compute prepareRevision retry: ${error.message}`; - process.stderr.write(`${message}\n`); + process.stderr.write( + `OpenShell integration: initial Compute prepareRevision retry: ${error.message}\n`, + ); } throw error; } @@ -2131,7 +1360,6 @@ async function prepareProductionInstallation( { OpenShellSandboxDriver }, { GrpcOpenShellGatewayClient }, { OpenShellGateway }, - { KubeConfig, KubernetesObjectApi }, ] = await Promise.all([ import("pg"), import("../../packages/occ/src/state/postgres-state.ts"), @@ -2142,16 +1370,8 @@ async function prepareProductionInstallation( import("../../apps/controller/src/drivers/sandbox/openshell.ts"), import("../../apps/controller/src/drivers/sandbox/openshell-gateway-client.ts"), import("../../apps/controller/src/backends/openshell.ts"), - import(controllerRequire.resolve("@kubernetes/client-node")), ]); - // The positive integration owns these compatibility mutations. Production controller clients - // remain restricted and receive no authority to patch provider-owned resources or read Secrets. - const operatorConfiguration = new KubeConfig(); - operatorConfiguration.loadFromFile(kubeconfigPath); - operatorConfiguration.setCurrentContext(kubernetesContext); - const operatorKubernetes = KubernetesObjectApi.makeApiClient(operatorConfiguration); - const directory = await mkdtemp(join(tmpdir(), "oce-openshell-sandboxdriver-")); const startupPath = join(directory, "installation.json"); const configuration = createOpenShellInstallationConfiguration({ @@ -2171,6 +1391,7 @@ async function prepareProductionInstallation( if (harnessId === "openclaw") { // scripts/k3d builds this runtime from an OpenClaw source with native worker support. configuration.runtime = { nativeWorkerSupport: "custom-image" }; + configuration.drivers.compute.configuration.servicePrincipalCredentials = { mode: "disabled" }; configuration.drivers.compute.configuration.runtime.nativeOpenClawSessionCapacity = 2; // The node and two retained session workers share one container; the single-Harness // 2 GiB budget OOM-kills the second turn. Keep Compute and Sandbox budgets aligned. @@ -2179,20 +1400,6 @@ async function prepareProductionInstallation( configuration.drivers.compute.configuration.resources.agent.limits.memory; configuration.drivers.compute.configuration.resources.namespace.quota["limits.memory"] = "16Gi"; configuration.drivers.credential_gateway.configuration.binaries = ["/usr/local/bin/node"]; - // Automatic enrollment egress belongs to the Driver's Codex runtime provider. The native - // verification bridge must admit only its Node executable and exact Gateway endpoint. - configuration.drivers.sandbox.configuration.policy.networkPolicies.push({ - name: "workspace-node-enrollment", - binaries: [{ path: "/usr/local/bin/node" }], - endpoints: [ - { - host: workspaceGateway.routing.hostname, - ports: [workspaceGateway.routing.endpointPort], - tls: "skip", - enforcement: "enforce", - }, - ], - }); } // A disposable Keycloak issues OAuth2 tokens over HTTPS signed by a private CA. The Namespace // gateway is configured to trust that CA so it can mint and refresh tokens itself. @@ -2249,11 +1456,7 @@ async function prepareProductionInstallation( OpenShellSandboxDriver, GrpcOpenShellGatewayClient, OpenShellGateway, - operatorKubernetes, - { - enableCompatibilityBridges: harnessId === "openclaw", - ...(keycloak === undefined ? {} : { gatewayTrustPem: keycloak.caPem }), - }, + { ...(keycloak === undefined ? {} : { gatewayTrustPem: keycloak.caPem }) }, ); const drivers = await loadInstallationConfiguration({ mode: "production", @@ -2750,15 +1953,18 @@ async function prepareProductionInstallation( "test -w /sandbox/.openclaw-runtime/home/.codex", ); assertNoProjectedAgentIdentity(harnessPod); - assertApprovedOpenShellPrivileges(harnessPod); + assertApprovedOpenShellPrivileges(harnessPod, harnessId); } else { - assertBridgedNativeStateMount(harnessPod); - await assertBridgedServicePrincipalToken( - placement, - harnessPod, - configuration.drivers.compute.configuration.servicePrincipalCredentials, + // Native delivery uses the production provider path without copied identity tokens or Jobs. + assertNoProjectedAgentIdentity(harnessPod); + assertApprovedOpenShellPrivileges(harnessPod, harnessId); + assert.deepEqual( + (await resources("jobs", placement)).filter(({ metadata }) => + metadata.name.startsWith("openshell-cred-"), + ), + [], + "native delivery must not create bootstrap Jobs", ); - assertApprovedOpenShellPrivileges(harnessPod, { compatibilityBridge: true }); } process.stderr.write( "OpenShell integration: approved mounts and privileges verified; checking secret exposure.\n", @@ -2769,6 +1975,26 @@ async function prepareProductionInstallation( "OpenShell integration: secret exposure checks passed; verifying gateway routing.\n", ); + // Exercise the Console's real API path: Pod/event reads and bounded logs must be + // authorized independently of the worker's successful workload provisioning. + const runtimePath = `/namespaces/${namespaceId}/agents/${agent.data.id}/deployments/${deployed.data.id}/runtime`; + const runtime = await request("GET", runtimePath); + assert.equal(runtime.status, 200, "the demo API must describe its Agent runtime"); + const gatewaySource = runtime.data.sources.find(({ id }) => id === "gateway"); + assert.ok(gatewaySource?.available, "the ready Gateway must offer runtime logs"); + const logs = await request("GET", `${runtimePath}/logs?source=gateway&tailLines=10`); + assert.equal(logs.status, 200, "the demo API must read bounded Gateway logs"); + assert.ok(Array.isArray(logs.data.records)); + assert.equal(JSON.stringify(logs).includes(process.env.OPENAI_API_KEY), false); + assert.ok( + runtime.data.sources.some(({ id }) => id === "sandbox"), + "the wrapper must preserve OpenShell policy-log support", + ); + const sandboxLogs = await request("GET", `${runtimePath}/logs?source=sandbox&tailLines=10`); + assert.equal(sandboxLogs.status, 200, "the demo API must read OpenShell policy logs"); + assert.ok(Array.isArray(sandboxLogs.data.records)); + assert.equal(JSON.stringify(sandboxLogs).includes(process.env.OPENAI_API_KEY), false); + const agentServiceName = openShellAgentName(agent.data.id); const gatewayPods = (await resources("pods", gatewayPlacement)).filter( (pod) => From 94e154931e4c5e04c29838ec85a550a9e55d0f3f Mon Sep 17 00:00:00 2001 From: sallyom Date: Sat, 10 Oct 2026 09:56:39 -0400 Subject: [PATCH 02/10] fix(openshell): temporarily prepare native PVC ownership Signed-off-by: sallyom Co-authored-by: sallyom <11166065+sallyom@users.noreply.github.com> --- .../sandbox-driver-openshell-k3d-real.test.mjs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/integration/sandbox-driver-openshell-k3d-real.test.mjs b/tests/integration/sandbox-driver-openshell-k3d-real.test.mjs index 57aaf26c88..16a3252cad 100644 --- a/tests/integration/sandbox-driver-openshell-k3d-real.test.mjs +++ b/tests/integration/sandbox-driver-openshell-k3d-real.test.mjs @@ -1075,6 +1075,19 @@ function createIntegrationSandboxDriverFactory( serviceName: `openshell-${hash(namespaceName, 10)}`, }; if (requirements !== undefined) { + // TEMPORARY: The production preparer reuses this gateway-configured identity; + // remove if OpenShell adds private subpath preparation and OCE adopts it. + options.gateway.operatorWorkspaceResources = [ + { + apiVersion: "v1", + kind: "ServiceAccount", + metadata: { + name: + requirements.workloadIdentity?.serviceAccountName ?? + openShellAgentName(requirements.labels["openclaw.dev/agent"]), + }, + }, + ]; const workspace = requirements.workspaceMounts.find( ({ mountPath }) => mountPath === workspaceMountPath, ); From dc80c64616e32639ed13689258eaea4bb672c3e0 Mon Sep 17 00:00:00 2001 From: roboclaw-bot <309084314+roboclaw-bot@users.noreply.github.com> Date: Mon, 12 Oct 2026 02:24:17 +0000 Subject: [PATCH 03/10] test(openshell): prove native Gateway authority boundaries Exercise real native workspace and successive model operations, cross-Agent setup and signed-device rejection, natural setup expiry, and live node revocation with an independent healthy Agent control. Keep authority probes and diagnostics credential-free. Co-authored-by: sallyom <11166065+sallyom@users.noreply.github.com> Signed-off-by: roboclaw-bot <309084314+roboclaw-bot@users.noreply.github.com> --- tests/helpers/native-gateway-authority.mjs | 113 ++++++ ...sandbox-driver-openshell-k3d-real.test.mjs | 381 +++++++++++++++++- 2 files changed, 489 insertions(+), 5 deletions(-) create mode 100644 tests/helpers/native-gateway-authority.mjs diff --git a/tests/helpers/native-gateway-authority.mjs b/tests/helpers/native-gateway-authority.mjs new file mode 100644 index 0000000000..5cc5c144e8 --- /dev/null +++ b/tests/helpers/native-gateway-authority.mjs @@ -0,0 +1,113 @@ +import assert from "node:assert/strict"; +import { createHash, createPublicKey, generateKeyPairSync, sign } from "node:crypto"; +import { createRequire } from "node:module"; +import { pathToFileURL } from "node:url"; + +// Resolve the same published client as the controller, not a second protocol implementation. +const controllerRequire = createRequire( + new URL("../../apps/controller/package.json", import.meta.url), +); + +export function createProbeDeviceIdentity() { + const { publicKey, privateKey } = generateKeyPairSync("ed25519"); + const publicKeyPem = publicKey.export({ type: "spki", format: "pem" }).toString(); + return { + deviceId: createHash("sha256").update(rawPublicKey(publicKeyPem)).digest("hex"), + publicKeyPem, + privateKeyPem: privateKey.export({ type: "pkcs8", format: "pem" }).toString(), + }; +} + +function rawPublicKey(pem) { + const der = createPublicKey(pem).export({ type: "spki", format: "der" }); + const prefix = Buffer.from("302a300506032b6570032100", "hex"); + assert.equal(der.length, prefix.length + 32); + assert.ok(der.subarray(0, prefix.length).equals(prefix)); + return der.subarray(prefix.length); +} + +// No persistent device store: every probe explicitly supplies its authority. The real +// Gateway validates the client's signed connect challenge and issues the device token. +export async function connectNativeGateway(options) { + const { GatewayClient } = await import( + pathToFileURL(controllerRequire.resolve("@openclaw/gateway-client")).href + ); + let accept; + let refuse; + const hello = new Promise((resolve, reject) => { + accept = resolve; + refuse = reject; + }); + const client = new GatewayClient({ + ...options, + hostDeps: { + signDevicePayload: (key, payload) => + sign(null, Buffer.from(payload), key).toString("base64url"), + publicKeyRawBase64UrlFromPem: (key) => rawPublicKey(key).toString("base64url"), + loadDeviceAuthToken: () => null, + storeDeviceAuthToken: () => {}, + clearDeviceAuthToken: () => {}, + // Handshake failures are reported below as codes only, never raw transport errors. + logDebug: () => {}, + logError: () => {}, + }, + onHelloOk: accept, + onConnectError: refuse, + }); + const timer = setTimeout( + () => refuse(new Error("Gateway authority handshake timed out")), + 15_000, + ); + const close = async () => { + client.stop(); + await client.stopAndWait({ timeoutMs: 1_000 }); + }; + try { + client.start(); + const connected = await hello; + return { client, hello: connected, close }; + } catch (error) { + await close(); + // Do not retain the raw error/cause: URLs, setup bearers and device tokens are private. + const rejected = new Error("Native Gateway rejected the authority handshake"); + rejected.code = error?.code; + rejected.detailCode = error?.details?.code; + throw rejected; + } finally { + clearTimeout(timer); + } +} + +export async function expectNativeAuthorityDenied(options, detailCode) { + let admitted; + let failure; + try { + admitted = await connectNativeGateway(options); + } catch (error) { + failure = error; + } finally { + await admitted?.close(); + } + assert.ok(failure, "the other Agent or expired authority must not connect"); + assert.equal(failure.code, "INVALID_REQUEST"); + assert.equal(failure.detailCode, detailCode, "require the Gateway's causal auth rejection"); + return { code: failure.code, detailCode: failure.detailCode }; +} + +export function decodeNativeSetup(setup) { + assert.equal(setup.access, "node"); + assert.ok(typeof setup.setupId === "string" && setup.setupId.length > 0); + const encoded = setup.setupCode.toLowerCase().startsWith("oc-pair://") + ? setup.setupCode.slice("oc-pair://".length) + : setup.setupCode; + let envelope; + try { + envelope = JSON.parse(Buffer.from(encoded, "base64url").toString("utf8")); + } catch { + throw new Error("The Gateway issued an invalid native setup envelope"); + } + assert.equal(envelope.expiresAtMs, setup.expiresAtMs); + assert.ok(typeof envelope.bootstrapToken === "string" && envelope.bootstrapToken.length > 0); + assert.ok(Number.isSafeInteger(setup.expiresAtMs) && setup.expiresAtMs > Date.now()); + return envelope; +} diff --git a/tests/integration/sandbox-driver-openshell-k3d-real.test.mjs b/tests/integration/sandbox-driver-openshell-k3d-real.test.mjs index 16a3252cad..ddd614f3e2 100644 --- a/tests/integration/sandbox-driver-openshell-k3d-real.test.mjs +++ b/tests/integration/sandbox-driver-openshell-k3d-real.test.mjs @@ -49,6 +49,12 @@ import { ensureEnvoyGatewayControllers, } from "../helpers/envoy-workspace-gateway.mjs"; import { availablePort } from "../helpers/available-port.mjs"; +import { + connectNativeGateway, + createProbeDeviceIdentity, + decodeNativeSetup, + expectNativeAuthorityDenied, +} from "../helpers/native-gateway-authority.mjs"; import { defaultKeycloakImage, issueServingCertificate, @@ -1411,7 +1417,9 @@ async function prepareProductionInstallation( configuration.drivers.compute.configuration.resources.agent.limits.memory = "6Gi"; configuration.drivers.sandbox.configuration.kubernetes.agentResources.limits.memory = configuration.drivers.compute.configuration.resources.agent.limits.memory; - configuration.drivers.compute.configuration.resources.namespace.quota["limits.memory"] = "16Gi"; + // Authority checks keep two Agents in this namespace: 2 * (4Gi Gateway + 6Gi Harness). + // Leave another 12Gi for OpenShell provider workloads and transient preparation Pods. + configuration.drivers.compute.configuration.resources.namespace.quota["limits.memory"] = "32Gi"; configuration.drivers.credential_gateway.configuration.binaries = ["/usr/local/bin/node"]; } // A disposable Keycloak issues OAuth2 tokens over HTTPS signed by a private CA. The Namespace @@ -2038,6 +2046,13 @@ async function prepareProductionInstallation( request, placement, namespaceId, + nativeProof: { + configurationId: agentConfiguration.data.id, + modelSourceId: modelSource.data.id, + sourceRoleId: sourceRole.data.id, + gatewayRouting: workspaceGateway.routing, + gatewayApiKey: workspaceGateway.apiKey, + }, agent: agent.data, revision: deployed.data, harnessPod, @@ -3825,16 +3840,371 @@ async function requestGatewayTurns(context, topology, harnessId) { } } +async function createNativeTurnRequester(context, topology) { + const passwordSecret = await resource( + "secret", + `gateway-password-${hash(topology.agent.id)}`, + topology.gatewayPlacement, + ); + const gatewayPassword = Buffer.from(passwordSecret.data["gateway-password"], "base64").toString(); + const forwarding = await startGatewayPortForward( + topology.gatewayPlacement, + openShellGatewayName(topology.agent.id), + ); + context.after(() => forwarding.stop()); + return { + nodeUrl: forwarding.url.replace("http:", "ws:") + "/node", + async failureDiagnostic(payload) { + const secrets = failureSecrets([ + process.env, + { GATEWAY_PASSWORD: gatewayPassword, APP_SERVER_TOKEN: topology.appServerToken }, + ]); + return ( + "\nNative Gateway response: " + + redactLogLine(JSON.stringify(payload), secrets, 1600) + + (await gatewayTurnFailureDiagnostic(topology, gatewayPassword)) + ); + }, + async turn(sessionKey) { + const nonce = "OCC-OPENSHELL-NATIVE-" + randomUUID(); + let response; + try { + response = await fetch(forwarding.url + "/v1/chat/completions", { + method: "POST", + headers: { + authorization: "Bearer " + gatewayPassword, + "content-type": "application/json", + "x-openclaw-session-key": sessionKey, + }, + body: JSON.stringify({ + model: "openclaw/default", + stream: false, + messages: [{ role: "user", content: "Reply with exactly " + nonce + "." }], + }), + signal: AbortSignal.timeout(180_000), + }); + } catch { + // A timeout/network error is not evidence that revocation rejected a model operation. + assert.fail( + "Native OpenClaw turn transport failed" + + (await gatewayTurnFailureDiagnostic(topology, gatewayPassword)), + ); + } + const body = await response.text(); + assert.equal( + body.includes(process.env.OPENAI_API_KEY), + false, + "model key leaked in response", + ); + assert.equal(body.includes(gatewayPassword), false, "Gateway password leaked in response"); + let payload; + try { + payload = JSON.parse(body); + } catch { + assert.fail("Native Gateway returned non-JSON model response; HTTP " + response.status); + } + return { status: response.status, payload, nonce }; + }, + }; +} + +async function assertNativeTurn(requester, sessionKey) { + const result = await requester.turn(sessionKey); + if (result.status !== 200) { + assert.fail( + `native model turn must complete; HTTP ${result.status}` + + (await requester.failureDiagnostic(result.payload)), + ); + } + if (!result.payload.choices?.[0]?.message?.content?.includes(result.nonce)) { + assert.fail( + "a real assistant completion must contain the fresh nonce" + + (await requester.failureDiagnostic(result.payload)), + ); + } +} + +async function requestNativeOpenClawTurns(context, topology) { + const requester = await createNativeTurnRequester(context, topology); + const sessions = ["openshell-native-" + randomUUID(), "openshell-native-" + randomUUID()]; + // Retained session workers share the same revision. The second session needs its own slot. + await assertNativeTurn(requester, sessions[0]); + await assertNativeTurn(requester, sessions[1]); + // Continue an existing worker too, rather than proving only first-turn startup. + await assertNativeTurn(requester, sessions[0]); + return { requester, sessions }; +} + async function assertNativeOpenClawWorkspaceFiles(topology) { const path = `/namespaces/${topology.namespaceId}/agents/${topology.agent.id}/workspace/files/AGENTS.md`; - const content = "# Dedicated native OpenClaw\n"; + const content = "# Dedicated native OpenClaw " + randomUUID() + "\n"; const written = await topology.request("PUT", path, { content }); - assert.equal(written.status, 200, JSON.stringify(written.error)); + assert.equal(written.status, 200, "native workspace write must succeed"); const read = await topology.request("GET", path); - assert.equal(read.status, 200, JSON.stringify(read.error)); + assert.equal(read.status, 200, "native workspace read must succeed"); assert.deepEqual(read.data, { name: "AGENTS.md", content }); } +async function createSecondNativeAgent(topology) { + const { request, namespaceId, nativeProof } = topology; + const created = await request("POST", "/namespaces/" + namespaceId + "/agents", { + name: "openshell-authority-peer-" + randomUUID(), + configurationId: nativeProof.configurationId, + executionMode: "dedicated", + harnessAuth: { method: "credential_source", sourceId: nativeProof.modelSourceId }, + credentialSources: [{ sourceId: nativeProof.modelSourceId }], + }); + assert.equal(created.status, 201, "create independent native Agent B through OCC"); + const agent = created.data; + const base = "/namespaces/" + namespaceId + "/agents/" + agent.id; + const credentials = await request("POST", base + "/runtime-credentials", {}); + assert.equal(credentials.status, 200); + const binding = await request("POST", "/namespaces/" + namespaceId + "/iam/access-bindings", { + subjectKind: "identity", + subjectId: agent.servicePrincipalId, + roleId: nativeProof.sourceRoleId, + resourceKind: "credential_source", + resourceId: nativeProof.modelSourceId, + }); + assert.equal(binding.status, 201); + const deployed = await request("POST", base + "/deploy"); + assert.equal(deployed.status, 202); + await waitFor("native Agent B activation", async () => { + const observed = await request("GET", base); + assert.equal(observed.status, 200); + return observed.data.activeRevisionId === deployed.data.id ? true : undefined; + }); + const sandbox = await waitForSandbox(topology.placement, deployed.data); + const harnessPod = await waitForProviderHarnessPod(topology.placement, deployed.data); + await assertProviderOwnedHarness(topology.placement, deployed.data, sandbox, harnessPod); + const gatewayPod = (await resources("pods", topology.gatewayPlacement)).find( + (pod) => + pod.metadata.labels?.["openclaw.dev/workload-role"] === "gateway" && + pod.metadata.labels?.["openclaw.dev/agent-id"] === agent.id, + ); + assert.ok(gatewayPod, "Agent B must have its own live Gateway"); + assert.notEqual(gatewayPod.metadata.uid, topology.gatewayPod.metadata.uid); + return { ...topology, agent, revision: deployed.data, harnessPod, sandbox, gatewayPod }; +} + +async function connectNativeAdministrator(context, topology) { + const { gatewayRouting, gatewayApiKey } = topology.nativeProof; + const port = gatewayRouting.endpointPort ?? 443; + const authority = gatewayRouting.hostname + (port === 443 ? "" : ":" + port); + const connected = await connectNativeGateway({ + url: + "wss://" + authority + "/namespaces/" + topology.namespaceId + "/agents/" + topology.agent.id, + clientName: "gateway-client", + mode: "backend", + role: "operator", + scopes: [], + deviceIdentity: null, + edgeAuthHeaders: { "x-api-key": gatewayApiKey }, + }); + context.after(connected.close); + assert.equal(connected.hello.auth?.role, "operator"); + assert.ok(connected.hello.auth.scopes.includes("operator.admin")); + // Never print raw RPC errors: setup/device authority can occur in their diagnostics. + return async (method, params) => { + try { + return await connected.client.request(method, params, { timeoutMs: 30_000 }); + } catch (error) { + // A raw cause can make Node's test reporter print setup/device credentials. + // eslint-disable-next-line preserve-caught-error + throw new Error("Native administrative RPC " + method + " failed (" + error?.code + ")"); + } + }; +} + +function nativeAuthorityTrace(context, stage, observations = {}) { + context.diagnostic("native-authority " + JSON.stringify({ stage, ...observations })); +} + +async function assertNativeAuthorityChain(context, topology, positive) { + const adminA = await connectNativeAdministrator(context, topology); + const issueSetup = async () => { + const issued = await adminA("device.pair.setupCode", { + bootstrapProfile: "node", + includeQr: false, + publicUrl: positive.requester.nodeUrl, + }); + return { issued, envelope: decodeNativeSetup(issued) }; + }; + // Keep an unredeemed, genuinely issued setup until its server-owned TTL elapses. No + // clock, database, envelope-expiry or provider configuration is changed for this case. + const expiry = await issueSetup(); + assert.ok( + expiry.issued.expiresAtMs - Date.now() <= 11 * 60_000, + "unexpected upstream setup TTL: review the live lane timeout instead of shortening authority", + ); + const peer = await createSecondNativeAgent(topology); + await assertNativeOpenClawWorkspaceFiles(peer); + const peerTurns = await requestNativeOpenClawTurns(context, peer); + const adminB = await connectNativeAdministrator(context, peer); + nativeAuthorityTrace(context, "positive-native-workspace-and-successive-turns", { agents: 2 }); + + const deviceIdentity = createProbeDeviceIdentity(); + const probe = { clientName: "node-host", mode: "node", role: "node", scopes: [], deviceIdentity }; + const setup = await issueSetup(); + const setupDenial = await expectNativeAuthorityDenied( + { ...probe, url: peerTurns.requester.nodeUrl, bootstrapToken: setup.envelope.bootstrapToken }, + "AUTH_BOOTSTRAP_TOKEN_INVALID", + ); + // Reuse the identical token at A: B's refusal must not consume A's one-use authority. + const admitted = await connectNativeGateway({ + ...probe, + url: positive.requester.nodeUrl, + bootstrapToken: setup.envelope.bootstrapToken, + }); + let deviceToken; + try { + deviceToken = admitted.hello.auth?.deviceToken; + assert.equal(admitted.hello.auth?.role, "node"); + assert.deepEqual(admitted.hello.auth.scopes, [], "node setup must grant no operator scopes"); + assert.ok( + typeof deviceToken === "string" && deviceToken.length > 0, + "A must issue real signed-device authority", + ); + } finally { + await admitted.close(); + } + const connectedAgain = await connectNativeGateway({ + ...probe, + url: positive.requester.nodeUrl, + deviceToken, + }); + await connectedAgain.close(); + const deviceDenial = await expectNativeAuthorityDenied( + { ...probe, url: peerTurns.requester.nodeUrl, deviceToken }, + "AUTH_DEVICE_TOKEN_MISMATCH", + ); + const peerSetupStatus = await adminB("device.pair.setupStatus", { + setupId: setup.issued.setupId, + }); + assert.deepEqual(peerSetupStatus, {}, "B must not record A's enrollment"); + const ownSetupStatus = await adminA("device.pair.setupStatus", { setupId: setup.issued.setupId }); + assert.equal( + (ownSetupStatus.completion ?? ownSetupStatus.deliveryUncertain)?.deviceId, + deviceIdentity.deviceId, + "A must record the probe enrollment under its own setup", + ); + nativeAuthorityTrace(context, "cross-agent-setup-and-signed-device-denied", { + setupDenial, + deviceDenial, + }); + // Remove the auth-only probe before testing the real, provider-owned node. It never + // advertises workspace/worker commands or substitutes for actual runtime execution. + await adminA("device.pair.remove", { deviceId: deviceIdentity.deviceId }); + + const waitMs = Math.max(0, expiry.issued.expiresAtMs - Date.now() + 1_000); + nativeAuthorityTrace(context, "waiting-for-real-setup-expiry", { waitMs }); + await delay(waitMs, undefined, { signal: context.signal }); + const fresh = await issueSetup(); + const expiryProbe = { + ...probe, + deviceIdentity: createProbeDeviceIdentity(), + url: positive.requester.nodeUrl, + }; + const expiryDenial = await expectNativeAuthorityDenied( + { ...expiryProbe, bootstrapToken: expiry.envelope.bootstrapToken }, + "AUTH_BOOTSTRAP_TOKEN_INVALID", + ); + const freshConnection = await connectNativeGateway({ + ...expiryProbe, + bootstrapToken: fresh.envelope.bootstrapToken, + }); + await freshConnection.close(); + await adminA("device.pair.remove", { deviceId: expiryProbe.deviceIdentity.deviceId }); + assert.deepEqual(await adminA("device.pair.setupStatus", { setupId: expiry.issued.setupId }), {}); + nativeAuthorityTrace(context, "expired-setup-denied-fresh-setup-admitted", { expiryDenial }); + + // Revoke the actual Compute-enrolled node, not the auth-only probe and not the + // shared model source. B remains an independent live control using that same source. + const nodeSecret = await resource( + "secret", + "workspace-node-" + hash(topology.agent.id) + "-" + hash("harness:openclaw"), + topology.placement, + ); + const deviceId = Buffer.from(nodeSecret.data.deviceId, "base64").toString(); + assert.ok(deviceId.length > 0); + assert.equal((await adminA("node.describe", { nodeId: deviceId })).connected, true); + await assertNativeOpenClawWorkspaceFiles(topology); + await assertNativeTurn(positive.requester, positive.sessions[0]); + const before = await resource("pod", topology.harnessPod.metadata.name, topology.placement); + const gatewayBefore = await resource( + "pod", + topology.gatewayPod.metadata.name, + topology.gatewayPlacement, + ); + const revoked = await adminA("device.token.revoke", { deviceId, role: "node" }); + assert.equal(revoked.deviceId, deviceId); + assert.equal(revoked.role, "node"); + assert.ok(Number.isSafeInteger(revoked.revokedAtMs)); + await waitFor("revoked native node disconnected", async () => + (await adminA("node.describe", { nodeId: deviceId })).connected === false ? true : undefined, + ); + nativeAuthorityTrace(context, "native-node-revoked", { + connected: false, + revokedAtMs: revoked.revokedAtMs, + }); + const path = `/namespaces/${topology.namespaceId}/agents/${topology.agent.id}/workspace/files/AGENTS.md`; + for (const method of ["GET", "PUT"]) { + const result = await topology.request( + method, + path, + method === "PUT" ? { content: "revoked-write-must-not-land-" + randomUUID() } : undefined, + ); + assert.equal(result.status, 503, "revoked native workspace " + method + " must be unavailable"); + nativeAuthorityTrace(context, "revoked-workspace-operation", { method, status: result.status }); + } + // Reuse the worker that just completed a turn. A new session could fail because + // both worker slots are occupied, independently of revocation. + const revokedTurn = await positive.requester.turn(positive.sessions[0]); + assert.ok( + revokedTurn.status >= 500 && revokedTurn.status <= 599, + "revoked retained worker must fail instead of completing or rejecting operator auth", + ); + assert.ok(revokedTurn.payload.error, "revoked turn must carry the native API error envelope"); + // The HTTP adapter may redact placement errors. Pair this failure with the + // disconnected node, immediate same-session success, unchanged Pods, and B's + // successful model turn below; a generic server error alone proves nothing. + assert.equal( + /capacity|billing|quota|rate.limit/iu.test(revokedTurn.payload.error.message ?? ""), + false, + "an unrelated model/worker capacity failure cannot prove revocation", + ); + assert.equal(JSON.stringify(revokedTurn.payload).includes(revokedTurn.nonce), false); + nativeAuthorityTrace(context, "revoked-retained-model-operation", { status: revokedTurn.status }); + await assertNativeOpenClawWorkspaceFiles(peer); + await assertNativeTurn(peerTurns.requester, peerTurns.sessions[0]); + const after = await resource("pod", topology.harnessPod.metadata.name, topology.placement); + assert.equal(after.metadata.uid, before.metadata.uid, "revocation must not replace the Harness"); + assert.deepEqual( + after.status.containerStatuses?.map(({ name, restartCount }) => ({ name, restartCount })), + before.status.containerStatuses?.map(({ name, restartCount }) => ({ name, restartCount })), + "the same running Harness must lose its authority", + ); + const gatewayAfter = await resource( + "pod", + topology.gatewayPod.metadata.name, + topology.gatewayPlacement, + ); + assert.equal(gatewayAfter.metadata.uid, gatewayBefore.metadata.uid); + assert.deepEqual( + gatewayAfter.status.containerStatuses?.map(({ name, restartCount }) => ({ + name, + restartCount, + })), + gatewayBefore.status.containerStatuses?.map(({ name, restartCount }) => ({ + name, + restartCount, + })), + "Gateway restarts cannot stand in for revocation enforcement", + ); + assert.equal((await adminA("node.describe", { nodeId: deviceId })).connected, false); + nativeAuthorityTrace(context, "revocation-complete-peer-still-working"); +} + async function holdNativeOpenClawDemo(context, topology) { assert.ok(demoStatePath, "OCC_K3D_DEMO_STATE is required for the browser demo."); assert.equal(topology.controllerUrl, `http://127.0.0.1:${demoConsolePort}`); @@ -5580,7 +5950,8 @@ test( process.stderr.write( "OpenShell integration: starting successive native worker sessions through Gateway.\n", ); - await requestGatewayTurns(context, topology, "openclaw"); + const positive = await requestNativeOpenClawTurns(context, topology); + await assertNativeAuthorityChain(context, topology, positive); await assertEmbeddedOpenShellFailsClosed(topology); } return; From 786d52ce6d171093ce2a413526ad58fc521fad58 Mon Sep 17 00:00:00 2001 From: sallyom Date: Tue, 13 Oct 2026 13:39:47 -0400 Subject: [PATCH 04/10] fix(openshell): verify native startup and two-agent authority Signed-off-by: sallyom Co-authored-by: sallyom <11166065+sallyom@users.noreply.github.com> --- tests/helpers/openshell-kubernetes-real.mjs | 10 ++++- ...sandbox-driver-openshell-k3d-real.test.mjs | 37 +++++++++++++++++-- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/tests/helpers/openshell-kubernetes-real.mjs b/tests/helpers/openshell-kubernetes-real.mjs index de69d48887..62da406bd4 100644 --- a/tests/helpers/openshell-kubernetes-real.mjs +++ b/tests/helpers/openshell-kubernetes-real.mjs @@ -937,11 +937,17 @@ export function createOpenShellKubernetesFixture({ return pod; } - async function assertProviderOwnedHarness(namespace, revision, sandbox, pod) { + async function assertProviderOwnedHarness( + namespace, + revision, + sandbox, + pod, + serviceAccountName = openShellAgentName(revision.agentId), + ) { assert.equal(sandbox.metadata.namespace, namespace); assert.equal(pod.metadata.namespace, namespace); assert.ok(sandbox.metadata.name, "Sandbox must have a stable resource name."); - assert.equal(pod.spec.serviceAccountName, openShellAgentName(revision.agentId)); + assert.equal(pod.spec.serviceAccountName, serviceAccountName); assert.equal(pod.metadata.labels?.["openclaw.dev/namespace"], revision.namespaceId); assert.equal(pod.metadata.labels?.["openclaw.dev/agent"], revision.agentId); assert.equal(pod.metadata.labels?.["openclaw.dev/revision"], revision.id); diff --git a/tests/integration/sandbox-driver-openshell-k3d-real.test.mjs b/tests/integration/sandbox-driver-openshell-k3d-real.test.mjs index ddd614f3e2..b7a647ae84 100644 --- a/tests/integration/sandbox-driver-openshell-k3d-real.test.mjs +++ b/tests/integration/sandbox-driver-openshell-k3d-real.test.mjs @@ -1200,6 +1200,12 @@ function createIntegrationSandboxDriverFactory( context, ).harnessStatus(context); }, + async workspaceNodePeerLabels(context) { + const endpoint = existingEndpointForNamespace(context); + return await delegate(undefined, context.namespace.name, endpoint).workspaceNodePeerLabels( + context, + ); + }, async readSandboxLogs(context, request) { const endpoint = existingEndpointForNamespace(context); return await delegate(undefined, context.namespace.name, endpoint).readSandboxLogs( @@ -2074,6 +2080,8 @@ async function prepareProductionInstallation( pauseWorker, resumeWorker: startWorker, provisioningFailure: (revisionId) => createSandboxDriver.provisioningFailures.get(revisionId), + waitForAgentWorkerCompleted: (agentId, revisionId) => + assertWorkerCompleted({ pool: observerPool, events, namespaceId, agentId, revisionId }), diagnoseAgentRevision: (agentId, revisionId) => writeWorkerCompletionDiagnostics({ pool: observerPool, @@ -3974,13 +3982,24 @@ async function createSecondNativeAgent(topology) { assert.equal(observed.status, 200); return observed.data.activeRevisionId === deployed.data.id ? true : undefined; }); + // Activation publishes the revision before the deployment worker finishes its + // Gateway rollout. Use the same completion boundary as Agent A's setup. + await topology.waitForAgentWorkerCompleted(agent.id, deployed.data.id); const sandbox = await waitForSandbox(topology.placement, deployed.data); const harnessPod = await waitForProviderHarnessPod(topology.placement, deployed.data); - await assertProviderOwnedHarness(topology.placement, deployed.data, sandbox, harnessPod); + // Native Harnesses request no projected identity. OpenShell retains the namespace + // gateway's Agent A ServiceAccount; Agent B remains isolated by Gateway authority. + await assertProviderOwnedHarness( + topology.placement, + deployed.data, + sandbox, + harnessPod, + openShellAgentName(topology.agent.id), + ); const gatewayPod = (await resources("pods", topology.gatewayPlacement)).find( (pod) => pod.metadata.labels?.["openclaw.dev/workload-role"] === "gateway" && - pod.metadata.labels?.["openclaw.dev/agent-id"] === agent.id, + pod.metadata.labels?.["openclaw.dev/agent"] === agent.id, ); assert.ok(gatewayPod, "Agent B must have its own live Gateway"); assert.notEqual(gatewayPod.metadata.uid, topology.gatewayPod.metadata.uid); @@ -4038,9 +4057,21 @@ async function assertNativeAuthorityChain(context, topology, positive) { "unexpected upstream setup TTL: review the live lane timeout instead of shortening authority", ); const peer = await createSecondNativeAgent(topology); + const adminB = await connectNativeAdministrator(context, peer); + const peerNodeSecret = await resource( + "secret", + "workspace-node-" + hash(peer.agent.id) + "-" + hash("harness:openclaw"), + peer.placement, + ); + const peerNodeId = Buffer.from(peerNodeSecret.data.deviceId, "base64").toString(); + assert.ok(peerNodeId.length > 0); + // Sandbox readiness precedes native node enrollment. Observe the Gateway's real + // connection before exercising workspace operations; do not retry those mutations. + await waitFor("native Agent B workspace node connected", async () => + (await adminB("node.describe", { nodeId: peerNodeId })).connected === true ? true : undefined, + ); await assertNativeOpenClawWorkspaceFiles(peer); const peerTurns = await requestNativeOpenClawTurns(context, peer); - const adminB = await connectNativeAdministrator(context, peer); nativeAuthorityTrace(context, "positive-native-workspace-and-successive-turns", { agents: 2 }); const deviceIdentity = createProbeDeviceIdentity(); From dd2fa142645f6b894404c9348c339b5dd641bfe5 Mon Sep 17 00:00:00 2001 From: roboclaw-bot <309084314+roboclaw-bot@users.noreply.github.com> Date: Sun, 18 Oct 2026 01:30:09 +0000 Subject: [PATCH 05/10] test(openshell): align native authority proof with current Drivers Keep main native provider, preparer, SDK normalization, status and exposure-withdrawal implementations unchanged. Adapt the remaining fixture bridge removal and authority checks to the current contracts, and register the native entry separately in the OpenShell CI suite. Co-authored-by: sallyom <11166065+sallyom@users.noreply.github.com> Signed-off-by: roboclaw-bot <309084314+roboclaw-bot@users.noreply.github.com> --- .github/workflows/full-integration.yml | 4 +- .github/workflows/qa-advisory.yml | 4 +- docs/testing/ci.md | 2 +- docs/testing/openshell-coverage.md | 24 +++++++ docs/testing/openshell.md | 17 +++-- scripts/ci/test-suites/openshell.json | 4 ++ tests/helpers/native-gateway-authority.mjs | 2 + tests/helpers/openshell-kubernetes-real.mjs | 24 +++---- tests/integration/ci-openshell.test.mjs | 27 +++++++ ...sandbox-driver-openshell-k3d-real.test.mjs | 72 ++++++++++--------- ...-driver-openshell-native-k3d-real.test.mjs | 4 ++ 11 files changed, 125 insertions(+), 59 deletions(-) create mode 100644 tests/integration/sandbox-driver-openshell-native-k3d-real.test.mjs diff --git a/.github/workflows/full-integration.yml b/.github/workflows/full-integration.yml index 126a444676..7d0a8f7a42 100644 --- a/.github/workflows/full-integration.yml +++ b/.github/workflows/full-integration.yml @@ -389,7 +389,7 @@ jobs: runs-on: ubuntu-24.04 needs: preflight if: inputs.lane == 'all' || inputs.lane == 'openshell' - timeout-minutes: 120 + timeout-minutes: 160 environment: integration-openshell env: NODE_BASE_IMAGE: ${{ vars.CONTAINER_NODE_BASE_IMAGE }} @@ -413,7 +413,7 @@ jobs: profile: full node-version: ${{ env.NODE_VERSION }} artifact-prefix: full-results - job-timeout-minutes: 120 + job-timeout-minutes: 160 k3d-otel: name: k3d OTEL diff --git a/.github/workflows/qa-advisory.yml b/.github/workflows/qa-advisory.yml index 6f20a1b29c..4c6f6449e4 100644 --- a/.github/workflows/qa-advisory.yml +++ b/.github/workflows/qa-advisory.yml @@ -117,7 +117,7 @@ jobs: if: needs.eligibility.outputs.trusted == 'true' # OpenShell requires an enabled Landlock LSM, absent on the Blacksmith kernel. runs-on: ubuntu-24.04 - timeout-minutes: 120 + timeout-minutes: 160 environment: integration-qa-pr env: NODE_BASE_IMAGE: ${{ vars.CONTAINER_NODE_BASE_IMAGE }} @@ -149,4 +149,4 @@ jobs: profile: full node-version: "24.16.0" artifact-prefix: qa-advisory - job-timeout-minutes: 120 + job-timeout-minutes: 160 diff --git a/docs/testing/ci.md b/docs/testing/ci.md index a4ba54ed6c..cd5faf6480 100644 --- a/docs/testing/ci.md +++ b/docs/testing/ci.md @@ -279,7 +279,7 @@ itself remains manual. | Docker Model Runtime; k3d Model Runtime | [Docker](docker.md) and [Kubernetes](kubernetes.md#kubernetes-model-turns-and-secrets) real models, Pod replacement, Secret rotation and first-Agent reuse | [Docker](../../scripts/ci/test-suites/docker-model.json), [k3d](../../scripts/ci/test-suites/k3d-model.json) | No | | Gateway Routing; Production TUI | [Envoy/OCC routing](kubernetes.md#private-gateway-routing), [Helm/TUI revision cutover](kubernetes.md#production-tui-tests) | [Routing](../../scripts/ci/test-suites/gateway-routing.json), [TUI](../../scripts/ci/test-suites/production-tui.json) | No | | Slack Live; Provider Account | [Live Slack isolation, issued credentials and model turns](slack.md) | [Slack](../../scripts/ci/test-suites/slack.json), [Account](../../scripts/ci/test-suites/provider-account.json) | No | -| OpenShell | [Sandbox enforcement and first-Agent flows](openshell.md) | [OpenShell](../../scripts/ci/test-suites/openshell.json) | Yes: QA advisory (OpenShell) | +| OpenShell | [Sandbox enforcement, native authority, and first-Agent flows](openshell.md) | [OpenShell](../../scripts/ci/test-suites/openshell.json) | Yes: QA advisory (OpenShell) | | Local Kubernetes Installation | [Installation/recovery, including Keycloak](kubernetes.md#local-kubernetes-installation) | [Installation](../../scripts/ci/test-suites/dev-up-k3d.json) | Yes: QA advisory (local installation) | | Logging Collector | [Collector forwarding and recovery](local.md#metrics-and-observability-tests) | [Collector](../../scripts/ci/test-suites/logging-collector.json) | Yes: CI | | k3d OTEL; Helper Timeout | [Real runtime OTLP logs](local.md#kubernetes-observability-acceptance); 300-second worker readiness deadline | [OTEL](../../scripts/ci/test-suites/k3d-otel.json), [Timeout](../../scripts/ci/test-suites/helper-timeout.json) | No | diff --git a/docs/testing/openshell-coverage.md b/docs/testing/openshell-coverage.md index c4bc451934..4d88d45b60 100644 --- a/docs/testing/openshell-coverage.md +++ b/docs/testing/openshell-coverage.md @@ -25,6 +25,30 @@ network, and credential enforcement. Exact workload identity, approved mounts and privileges, source withdrawal, replacement, and cleanup remain required. Missing prerequisites fail rather than skip. +## Native authority + +The native `--harness openclaw` case uses production provider delivery and no +credential-copy bridge. Its separate +`tests/integration/sandbox-driver-openshell-native-k3d-real.test.mjs` entry runs +in the registered OpenShell lane with `OPENAI_API_KEY`; the Codex entry cannot +substitute for it. It provisions independent Agents A and B using the same +model source. Each must read/write its workspace and complete fresh and retained +model sessions. Compute observes held startup failures through the temporary +provider exposure and withdraws that exposure once the node enrolls. + +A-issued setup and signed-device authority must work at A and fail at B. An +unused setup expires on the real Gateway clock; a fresh positive control still +connects. Auth-only probes connect through loopback-forwarded `/node`; they do not +prove NetworkPolicy enforcement or substitute for the running Harness. + +Revoking A's actual Compute-enrolled node must disconnect it, reject workspace +GET/PUT and stop a previously successful retained model session. B must remain +functional, with A's Gateway/Harness Pod identities and restart counts unchanged. +Transport errors and worker-capacity failures cannot stand in for revocation. +These checks require the selected upstream OpenClaw source to implement setup +issuance/status, signed device authentication and retained-worker revocation. +Source inspection and ordinary CI do not establish that this live case ran. + ## Credential route After the egress turn, the test writes a probe script into the shared workspace diff --git a/docs/testing/openshell.md b/docs/testing/openshell.md index ebce7c8f95..908dd85c4f 100644 --- a/docs/testing/openshell.md +++ b/docs/testing/openshell.md @@ -117,8 +117,8 @@ The Codex account must permit `:danger-full-access`; OpenShell supplies the oute sandbox. The live probe checks the effective profile before starting inference and fails immediately if managed policy selects a restricted sandbox. -The lane includes both setup tests, the Sandbox test, and first-Agent tests for -both control-plane variants. It fails when selected prerequisites are missing. +The lane includes both setup tests, separate Codex and native authority Sandbox +entries, and first-Agent tests for both control-plane variants. It fails when selected prerequisites are missing. Run cleanup even if preparation or a test fails: ```sh @@ -127,8 +127,8 @@ node scripts/ci/cleanup.mjs --state "$OCC_OPENSHELL_TEST_ROOT/state.json" The Codex Sandbox case and guided native OpenClaw first-Agent case use production Drivers, but neither qualifies a production installation. The separate native -Sandbox case uses a test-only compatibility bridge. Use the k3d helper below to -run that case. The [Sandbox Driver reference](../reference/drivers/openshell-sandbox.md#qualification-contract) +Sandbox case also uses production Driver delivery. Use the k3d helper below to +run its [authority checks](openshell-coverage.md#native-authority). The [Sandbox Driver reference](../reference/drivers/openshell-sandbox.md#qualification-contract) describes the remaining limits. ### Native OpenClaw with k3d @@ -142,10 +142,14 @@ the native test and builds the sibling `../openclaw` checkout. Set ./scripts/k3d demo --harness openclaw ``` -Both commands use the test-only bridge. `test` requires an authorized +Both commands use unmodified Compute requirements and the production Driver, +without bootstrap Jobs or copied identity tokens. `test` requires an authorized `OPENAI_API_KEY`; `demo` prompts for one if it is not set. `demo` leaves the proven environment running and opens the Control UI and OCE console on loopback. It leaves the dedicated worker slot free for the first browser session. +The native test reserves two worker slots and 6 GiB per Harness, with a 32 GiB +Namespace quota for two Agents and provider workloads. Genuine setup expiry +adds up to ten minutes; the test has a 40-minute cap. Run `./scripts/k3d down --harness openclaw` to remove it. Codex and native OpenClaw use separate helper-owned state; without `--harness`, `copy` selects the active demo and `down` removes both environments. @@ -172,7 +176,8 @@ directory when finished. and Kubernetes-only first-Agent stacks. It first proves Codex with a `codex-pat` source, then creates a separate `openai` source from `OPENAI_API_KEY` for guided native OpenClaw provisioning. The native case uses the pinned runtime and -production Drivers without the k3d helper's compatibility bridge. +production Drivers. The separate k3d Sandbox case covers cross-Agent authority +and node revocation rather than session continuity across replacement. The case edits `USER.md` through the workspace API and verifies its marker through real model execution and `exec`. That turn writes a unique file in its session diff --git a/scripts/ci/test-suites/openshell.json b/scripts/ci/test-suites/openshell.json index 75c34ca428..7fabc1d1bc 100644 --- a/scripts/ci/test-suites/openshell.json +++ b/scripts/ci/test-suites/openshell.json @@ -47,6 +47,10 @@ "path": "tests/integration/sandbox-driver-openshell-k3d-real.test.mjs", "expectedTests": ["OpenShell enforces provider delivery and authenticated Codex exposure"] }, + { + "path": "tests/integration/sandbox-driver-openshell-native-k3d-real.test.mjs", + "expectedTests": ["OpenShell enforces native provider delivery and Gateway authority"] + }, { "path": "tests/integration/local-first-agent-openshell-real.test.mjs", "expectedTests": [ diff --git a/tests/helpers/native-gateway-authority.mjs b/tests/helpers/native-gateway-authority.mjs index 5cc5c144e8..a1a27db74f 100644 --- a/tests/helpers/native-gateway-authority.mjs +++ b/tests/helpers/native-gateway-authority.mjs @@ -40,6 +40,8 @@ export async function connectNativeGateway(options) { }); const client = new GatewayClient({ ...options, + minProtocol: 4, + maxProtocol: 4, hostDeps: { signDevicePayload: (key, payload) => sign(null, Buffer.from(payload), key).toString("base64url"), diff --git a/tests/helpers/openshell-kubernetes-real.mjs b/tests/helpers/openshell-kubernetes-real.mjs index 62da406bd4..8d40fa2959 100644 --- a/tests/helpers/openshell-kubernetes-real.mjs +++ b/tests/helpers/openshell-kubernetes-real.mjs @@ -970,26 +970,20 @@ export function createOpenShellKubernetesFixture({ ); } - function harnessContainer(pod, harnessId = "codex") { + function harnessContainer(pod) { const container = pod.spec.containers.find(({ name }) => name === "agent"); - assert.ok(container, "provider-owned Pod must contain its Harness container."); + assert.ok(container, "provider-owned Pod must contain the Codex Harness container."); const environment = container.env ?? []; assert.equal( environment.some(({ name }) => name === "APP_SERVER_TOKEN"), false, "the raw app-server token must stay outside the OpenShell Harness Pod spec.", ); - const verifier = environment.find(({ name }) => name === "APP_TOKEN_SHA"); - if (harnessId === "openclaw") { - assert.equal(verifier, undefined, "native OpenClaw must not receive a Codex verifier."); - } else { - assert.equal(harnessId, "codex", "unsupported Harness privilege check"); - assert.match( - verifier?.value ?? "", - /^[a-f0-9]{64}$/, - "the OpenShell Codex Harness requires only its app-server token verifier.", - ); - } + assert.match( + environment.find(({ name }) => name === "APP_TOKEN_SHA")?.value ?? "", + /^[a-f0-9]{64}$/, + "the OpenShell Harness requires its transport/status token verifier.", + ); return container; } @@ -1114,7 +1108,7 @@ export function createOpenShellKubernetesFixture({ assert.equal(claims.sub, `system:serviceaccount:${namespace}:${pod.spec.serviceAccountName}`); } - function assertApprovedOpenShellPrivileges(pod, harnessId = "codex") { + function assertApprovedOpenShellPrivileges(pod) { assert.equal( pod.spec.runtimeClassName, openShellRuntimeClass, @@ -1138,7 +1132,7 @@ export function createOpenShellKubernetesFixture({ undefined, "OpenShell v0.1.4 must keep its network supervisor outside the workload Pod.", ); - const container = harnessContainer(pod, harnessId); + const container = harnessContainer(pod); assert.ok(container, "the provider-owned Pod must contain its Agent container."); assert.equal(container.securityContext?.allowPrivilegeEscalation, false); assert.deepEqual(container.securityContext?.capabilities?.drop, ["ALL"]); diff --git a/tests/integration/ci-openshell.test.mjs b/tests/integration/ci-openshell.test.mjs index 19bf869536..7616224ea4 100644 --- a/tests/integration/ci-openshell.test.mjs +++ b/tests/integration/ci-openshell.test.mjs @@ -155,6 +155,33 @@ test("OpenShell host probes retain the exposed hostname while connecting to loop assert.equal(observedHost, `${serviceHostname}:${address.port}`); }); +test("OpenShell privilege checks retain native status verification without a bridge bypass", () => { + const { assertApprovedOpenShellPrivileges: check } = createOpenShellKubernetesFixture({}); + const pod = { + spec: { + runtimeClassName: "openshell-sandbox", + securityContext: { runAsUser: 10001, runAsGroup: 10001 }, + containers: [ + { + name: "agent", + env: [{ name: "APP_TOKEN_SHA", value: "a".repeat(64) }], + securityContext: { allowPrivilegeEscalation: false, capabilities: { drop: ["ALL"] } }, + }, + ], + }, + }; + // Native Compute now requests a temporary held-failure exposure. It requires the + // same verifier boundary as Codex; the retired bridge must not bypass that check. + assert.doesNotThrow(() => check(pod)); + pod.spec.containers[0].env = []; + assert.throws( + () => check(pod, { compatibilityBridge: true }), + /transport\/status token verifier/, + ); + pod.spec.containers[0].env = [{ name: "APP_SERVER_TOKEN", value: "synthetic-token" }]; + assert.throws(() => check(pod), /raw app-server token must stay outside/); +}); + test("OpenShell probe observes its command after the turn ends, or preserves the unknown outcome", async (t) => { // Run the actual in-Pod client and probe decision. Only kubectl transport and the // app-server peer are fixtures. WebSocket message order, not a sleep, creates the boundary. diff --git a/tests/integration/sandbox-driver-openshell-k3d-real.test.mjs b/tests/integration/sandbox-driver-openshell-k3d-real.test.mjs index b7a647ae84..8fbfaa1046 100644 --- a/tests/integration/sandbox-driver-openshell-k3d-real.test.mjs +++ b/tests/integration/sandbox-driver-openshell-k3d-real.test.mjs @@ -1089,6 +1089,7 @@ function createIntegrationSandboxDriverFactory( kind: "ServiceAccount", metadata: { name: + gatewayState.get(namespaceName)?.sandboxServiceAccountName ?? requirements.workloadIdentity?.serviceAccountName ?? openShellAgentName(requirements.labels["openclaw.dev/agent"]), }, @@ -1170,6 +1171,7 @@ function createIntegrationSandboxDriverFactory( // The real Driver receives unmodified Compute requirements for both Harness kinds. const endpoint = await endpointForNamespace(context, { sandboxServiceAccountName: agentSandboxServiceAccount(context), + exact: context.requirements.workloadIdentity !== undefined, }); return await delegate( context.requirements, @@ -1181,6 +1183,7 @@ function createIntegrationSandboxDriverFactory( async harnessEndpoint(context) { const endpoint = await endpointForNamespace(context, { sandboxServiceAccountName: agentSandboxServiceAccount(context), + exact: context.requirements.workloadIdentity !== undefined, }); return await delegate( context.requirements, @@ -1192,6 +1195,7 @@ function createIntegrationSandboxDriverFactory( async harnessStatus(context) { const endpoint = await endpointForNamespace(context, { sandboxServiceAccountName: agentSandboxServiceAccount(context), + exact: context.requirements.workloadIdentity !== undefined, }); return await delegate( context.requirements, @@ -1200,12 +1204,6 @@ function createIntegrationSandboxDriverFactory( context, ).harnessStatus(context); }, - async workspaceNodePeerLabels(context) { - const endpoint = existingEndpointForNamespace(context); - return await delegate(undefined, context.namespace.name, endpoint).workspaceNodePeerLabels( - context, - ); - }, async readSandboxLogs(context, request) { const endpoint = existingEndpointForNamespace(context); return await delegate(undefined, context.namespace.name, endpoint).readSandboxLogs( @@ -1215,26 +1213,19 @@ function createIntegrationSandboxDriverFactory( }, // Compute drops a native worker's exposure once its node has enrolled (finding 1657). async withdrawHarnessExposure(context) { - const endpoint = await endpointForNamespace(context, { - sandboxServiceAccountName: agentSandboxServiceAccount(context), - exact: context.requirements.workloadIdentity !== undefined, - }); - await delegate( - context.requirements, - context.namespace.name, - endpoint, + // Sweeps also call this hook without provisioning requirements. + const endpoint = existingEndpointForNamespace(context); + await delegate(undefined, context.namespace.name, endpoint).withdrawHarnessExposure( context, - ).withdrawHarnessExposure(context); + ); }, // Compute admits the workspace-node connection only from the revision's // supervisor Pod, which the real Driver labels from the gateway's Sandbox record. async workspaceNodePeerLabels(context) { const endpoint = existingEndpointForNamespace(context); - return await delegate( - undefined, - context.namespace.name, - endpoint, - ).workspaceNodePeerLabels(context); + return await delegate(undefined, context.namespace.name, endpoint).workspaceNodePeerLabels( + context, + ); }, async cleanup(context) { if (context.revision === undefined) { @@ -1980,11 +1971,11 @@ async function prepareProductionInstallation( "test -w /sandbox/.openclaw-runtime/home/.codex", ); assertNoProjectedAgentIdentity(harnessPod); - assertApprovedOpenShellPrivileges(harnessPod, harnessId); + assertApprovedOpenShellPrivileges(harnessPod); } else { // Native delivery uses the production provider path without copied identity tokens or Jobs. assertNoProjectedAgentIdentity(harnessPod); - assertApprovedOpenShellPrivileges(harnessPod, harnessId); + assertApprovedOpenShellPrivileges(harnessPod); assert.deepEqual( (await resources("jobs", placement)).filter(({ metadata }) => metadata.name.startsWith("openshell-cred-"), @@ -3899,11 +3890,7 @@ async function createNativeTurnRequester(context, topology) { ); } const body = await response.text(); - assert.equal( - body.includes(process.env.OPENAI_API_KEY), - false, - "model key leaked in response", - ); + assert.equal(body.includes(modelCredential), false, "model key leaked in response"); assert.equal(body.includes(gatewayPassword), false, "Gateway password leaked in response"); let payload; try { @@ -3953,6 +3940,21 @@ async function assertNativeOpenClawWorkspaceFiles(topology) { assert.deepEqual(read.data, { name: "AGENTS.md", content }); } +async function assertNativeExposureWithdrawn(context, topology) { + const client = topology.gatewayClient(); + const workspace = topology.placement; + const name = topology.sandbox.metadata.name; + // A historical create URL can exist for held-startup diagnostics. Read live + // state, and confirm Workspace visibility rather than trusting a hidden 404. + assert.equal((await client.getWorkspace(workspace, context.signal))?.name, workspace); + assert.equal((await client.getSandbox({ workspace, name }, context.signal))?.name, name); + assert.equal( + await client.getService(workspace, name, "", context.signal), + undefined, + "an enrolled native Harness must have no remaining provider exposure", + ); +} + async function createSecondNativeAgent(topology) { const { request, namespaceId, nativeProof } = topology; const created = await request("POST", "/namespaces/" + namespaceId + "/agents", { @@ -4063,6 +4065,11 @@ async function assertNativeAuthorityChain(context, topology, positive) { "workspace-node-" + hash(peer.agent.id) + "-" + hash("harness:openclaw"), peer.placement, ); + assert.equal( + Buffer.from(peerNodeSecret.data.enrolledRevisionId, "base64").toString(), + peer.revision.id, + "the retained node identity must have enrolled for Agent B's active revision", + ); const peerNodeId = Buffer.from(peerNodeSecret.data.deviceId, "base64").toString(); assert.ok(peerNodeId.length > 0); // Sandbox readiness precedes native node enrollment. Observe the Gateway's real @@ -4070,6 +4077,7 @@ async function assertNativeAuthorityChain(context, topology, positive) { await waitFor("native Agent B workspace node connected", async () => (await adminB("node.describe", { nodeId: peerNodeId })).connected === true ? true : undefined, ); + await assertNativeExposureWithdrawn(context, peer); await assertNativeOpenClawWorkspaceFiles(peer); const peerTurns = await requestNativeOpenClawTurns(context, peer); nativeAuthorityTrace(context, "positive-native-workspace-and-successive-turns", { agents: 2 }); @@ -5935,7 +5943,9 @@ if (demoStatePath !== undefined) { } test( - "OpenShell enforces provider delivery and authenticated Codex exposure", + selectedHarness === "openclaw" + ? "OpenShell enforces native provider delivery and Gateway authority" + : "OpenShell enforces provider delivery and authenticated Codex exposure", { ...requiresOpenShellK3d, ...(demoStatePath === undefined ? { timeout: 2_400_000 } : {}), @@ -5963,11 +5973,7 @@ test( ); assert.ok(topology.sandbox.metadata.name); if (selectedHarness === "openclaw") { - assert.equal( - topology.harnessServiceUrl, - undefined, - "native OpenClaw must not request an inbound OpenShell service exposure.", - ); + await assertNativeExposureWithdrawn(context, topology); process.stderr.write( "OpenShell integration: verifying dedicated native workspace file access.\n", ); diff --git a/tests/integration/sandbox-driver-openshell-native-k3d-real.test.mjs b/tests/integration/sandbox-driver-openshell-native-k3d-real.test.mjs new file mode 100644 index 0000000000..9298bc5a08 --- /dev/null +++ b/tests/integration/sandbox-driver-openshell-native-k3d-real.test.mjs @@ -0,0 +1,4 @@ +// Run the shared production-Driver proof with native authority checks in its own +// test process; the ordinary Sandbox entry selects Codex and cannot cover these. +process.env.OCC_TEST_OPENSHELL_HARNESS = "openclaw"; +await import("./sandbox-driver-openshell-k3d-real.test.mjs"); From d6be1537640c1dbf86b438db8c0748b883f8e2dc Mon Sep 17 00:00:00 2001 From: roboclaw-bot <309084314+roboclaw-bot@users.noreply.github.com> Date: Sun, 18 Oct 2026 01:32:44 +0000 Subject: [PATCH 06/10] test(openshell): preserve first provisioning failure diagnostics Retain main first-failure collection for startup and second-Agent diagnostics after removing the credential-copy bridge. Co-authored-by: sallyom <11166065+sallyom@users.noreply.github.com> Signed-off-by: roboclaw-bot <309084314+roboclaw-bot@users.noreply.github.com> --- ...sandbox-driver-openshell-k3d-real.test.mjs | 33 ++++++++++++------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/tests/integration/sandbox-driver-openshell-k3d-real.test.mjs b/tests/integration/sandbox-driver-openshell-k3d-real.test.mjs index 8fbfaa1046..76c7b0800b 100644 --- a/tests/integration/sandbox-driver-openshell-k3d-real.test.mjs +++ b/tests/integration/sandbox-driver-openshell-k3d-real.test.mjs @@ -1168,17 +1168,28 @@ function createIntegrationSandboxDriverFactory( } }, async provisionHarness(context) { - // The real Driver receives unmodified Compute requirements for both Harness kinds. - const endpoint = await endpointForNamespace(context, { - sandboxServiceAccountName: agentSandboxServiceAccount(context), - exact: context.requirements.workloadIdentity !== undefined, - }); - return await delegate( - context.requirements, - context.namespace.name, - endpoint, - context, - ).provisionHarness(context); + try { + // The real Driver receives unmodified Compute requirements for both Harness kinds. + const endpoint = await endpointForNamespace(context, { + sandboxServiceAccountName: agentSandboxServiceAccount(context), + exact: context.requirements.workloadIdentity !== undefined, + }); + return await delegate( + context.requirements, + context.namespace.name, + endpoint, + context, + ).provisionHarness(context); + } catch (error) { + // Preserve main's first-failure evidence for startup and second-Agent diagnostics. + if (!provisioningFailures.has(context.revision.id)) { + process.stderr.write( + `OpenShell first provisioning failure for ${context.revision.id}: ${error.message}\n`, + ); + provisioningFailures.set(context.revision.id, error); + } + throw error; + } }, async harnessEndpoint(context) { const endpoint = await endpointForNamespace(context, { From 2a469a0091f2c332d707c053477e29c1dba6dac2 Mon Sep 17 00:00:00 2001 From: roboclaw-bot <309084314+roboclaw-bot@users.noreply.github.com> Date: Sun, 18 Oct 2026 01:50:32 +0000 Subject: [PATCH 07/10] test(openshell): remove retired bridge module import Co-authored-by: sallyom <11166065+sallyom@users.noreply.github.com> Signed-off-by: roboclaw-bot <309084314+roboclaw-bot@users.noreply.github.com> --- tests/integration/sandbox-driver-openshell-k3d-real.test.mjs | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/sandbox-driver-openshell-k3d-real.test.mjs b/tests/integration/sandbox-driver-openshell-k3d-real.test.mjs index 76c7b0800b..f927bca5be 100644 --- a/tests/integration/sandbox-driver-openshell-k3d-real.test.mjs +++ b/tests/integration/sandbox-driver-openshell-k3d-real.test.mjs @@ -13,7 +13,6 @@ import dns from "node:dns"; import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { request as httpRequest } from "node:http"; import { request as httpsRequest } from "node:https"; -import { createRequire } from "node:module"; import { connect, isIP, isIPv4 } from "node:net"; import { tmpdir } from "node:os"; import { basename, join } from "node:path"; From cec583ef402d7b5139b09366ea9567a85623a23b Mon Sep 17 00:00:00 2001 From: roboclaw-bot <309084314+roboclaw-bot@users.noreply.github.com> Date: Sun, 18 Oct 2026 12:39:34 +0000 Subject: [PATCH 08/10] test(openshell): apply review fixes to native fixture Check runtime logs against the selected model credential, keep ServicePrincipal credentials disabled in the shared fixture, and document current provider delivery behavior. Co-authored-by: sallyom <11166065+sallyom@users.noreply.github.com> Signed-off-by: roboclaw-bot <309084314+roboclaw-bot@users.noreply.github.com> --- docs/testing/openshell-coverage.md | 3 +- docs/testing/openshell.md | 4 +- tests/helpers/openshell-kubernetes-real.mjs | 9 +---- tests/integration/ci-openshell.test.mjs | 40 +++++++++++++++---- ...sandbox-driver-openshell-k3d-real.test.mjs | 9 ++--- 5 files changed, 41 insertions(+), 24 deletions(-) diff --git a/docs/testing/openshell-coverage.md b/docs/testing/openshell-coverage.md index 4d88d45b60..f5ee2f2d88 100644 --- a/docs/testing/openshell-coverage.md +++ b/docs/testing/openshell-coverage.md @@ -27,8 +27,7 @@ Missing prerequisites fail rather than skip. ## Native authority -The native `--harness openclaw` case uses production provider delivery and no -credential-copy bridge. Its separate +The native `--harness openclaw` case exercises production provider delivery. Its separate `tests/integration/sandbox-driver-openshell-native-k3d-real.test.mjs` entry runs in the registered OpenShell lane with `OPENAI_API_KEY`; the Codex entry cannot substitute for it. It provisions independent Agents A and B using the same diff --git a/docs/testing/openshell.md b/docs/testing/openshell.md index 908dd85c4f..85ad921102 100644 --- a/docs/testing/openshell.md +++ b/docs/testing/openshell.md @@ -142,8 +142,8 @@ the native test and builds the sibling `../openclaw` checkout. Set ./scripts/k3d demo --harness openclaw ``` -Both commands use unmodified Compute requirements and the production Driver, -without bootstrap Jobs or copied identity tokens. `test` requires an authorized +Both commands use unmodified Compute requirements and production Driver delivery. +`test` requires an authorized `OPENAI_API_KEY`; `demo` prompts for one if it is not set. `demo` leaves the proven environment running and opens the Control UI and OCE console on loopback. It leaves the dedicated worker slot free for the first browser session. diff --git a/tests/helpers/openshell-kubernetes-real.mjs b/tests/helpers/openshell-kubernetes-real.mjs index 8d40fa2959..ddc7390c07 100644 --- a/tests/helpers/openshell-kubernetes-real.mjs +++ b/tests/helpers/openshell-kubernetes-real.mjs @@ -233,14 +233,7 @@ export function createOpenShellInstallationConfiguration({ "limits.cpu": "16", "limits.memory": "8Gi", }; - configuration.drivers.compute.configuration.servicePrincipalCredentials = - process.env.OCC_TEST_OPENSHELL_HARNESS === "openclaw" - ? { - mode: "projectedServiceAccountToken", - audience: "openclaw-enterprise", - expirationSeconds: 3600, - } - : { mode: "disabled" }; + configuration.drivers.compute.configuration.servicePrincipalCredentials = { mode: "disabled" }; // The integration replaces this placeholder transport with each namespace's port-forward. configuration.backend = [ { diff --git a/tests/integration/ci-openshell.test.mjs b/tests/integration/ci-openshell.test.mjs index 7616224ea4..6a2d18ea96 100644 --- a/tests/integration/ci-openshell.test.mjs +++ b/tests/integration/ci-openshell.test.mjs @@ -155,7 +155,7 @@ test("OpenShell host probes retain the exposed hostname while connecting to loop assert.equal(observedHost, `${serviceHostname}:${address.port}`); }); -test("OpenShell privilege checks retain native status verification without a bridge bypass", () => { +test("OpenShell privilege checks require the Harness status verifier", () => { const { assertApprovedOpenShellPrivileges: check } = createOpenShellKubernetesFixture({}); const pod = { spec: { @@ -170,14 +170,11 @@ test("OpenShell privilege checks retain native status verification without a bri ], }, }; - // Native Compute now requests a temporary held-failure exposure. It requires the - // same verifier boundary as Codex; the retired bridge must not bypass that check. + // Native held-failure status and Codex transport both require a verifier, + // while the raw token must stay outside the Harness Pod spec. assert.doesNotThrow(() => check(pod)); pod.spec.containers[0].env = []; - assert.throws( - () => check(pod, { compatibilityBridge: true }), - /transport\/status token verifier/, - ); + assert.throws(() => check(pod), /transport\/status token verifier/); pod.spec.containers[0].env = [{ name: "APP_SERVER_TOKEN", value: "synthetic-token" }]; assert.throws(() => check(pod), /raw app-server token must stay outside/); }); @@ -537,6 +534,35 @@ test("OpenShell probe waits for a command that outlives its turn up to the fixtu } }); +test("OpenShell fixture disables ServicePrincipal credentials for both Harness selections", () => { + const previous = process.env.OCC_TEST_OPENSHELL_HARNESS; + try { + for (const harness of ["codex", "openclaw"]) { + process.env.OCC_TEST_OPENSHELL_HARNESS = harness; + const configuration = createOpenShellInstallationConfiguration({ + authentication: {}, + platformNamespace: "openclaw-system", + gatewayImage: "gateway-fixture", + codexImage: "codex-fixture", + cluster: { name: "fixture" }, + }); + // The real Driver rejects projected identity. Every fixture caller must + // receive the supported mode without patching Compute requirements later. + assert.deepEqual( + configuration.drivers.compute.configuration.servicePrincipalCredentials, + { mode: "disabled" }, + harness, + ); + } + } finally { + if (previous === undefined) { + delete process.env.OCC_TEST_OPENSHELL_HARNESS; + } else { + process.env.OCC_TEST_OPENSHELL_HARNESS = previous; + } + } +}); + test("OpenShell fixture networking admits the supervisor callback without granting tenant Pods", () => { const configuration = createOpenShellInstallationConfiguration({ authentication: {}, diff --git a/tests/integration/sandbox-driver-openshell-k3d-real.test.mjs b/tests/integration/sandbox-driver-openshell-k3d-real.test.mjs index f927bca5be..3011b225ef 100644 --- a/tests/integration/sandbox-driver-openshell-k3d-real.test.mjs +++ b/tests/integration/sandbox-driver-openshell-k3d-real.test.mjs @@ -1417,7 +1417,6 @@ async function prepareProductionInstallation( if (harnessId === "openclaw") { // scripts/k3d builds this runtime from an OpenClaw source with native worker support. configuration.runtime = { nativeWorkerSupport: "custom-image" }; - configuration.drivers.compute.configuration.servicePrincipalCredentials = { mode: "disabled" }; configuration.drivers.compute.configuration.runtime.nativeOpenClawSessionCapacity = 2; // The node and two retained session workers share one container; the single-Harness // 2 GiB budget OOM-kills the second turn. Keep Compute and Sandbox budgets aligned. @@ -1983,7 +1982,7 @@ async function prepareProductionInstallation( assertNoProjectedAgentIdentity(harnessPod); assertApprovedOpenShellPrivileges(harnessPod); } else { - // Native delivery uses the production provider path without copied identity tokens or Jobs. + // Native provider delivery keeps projected identity out of the Harness. assertNoProjectedAgentIdentity(harnessPod); assertApprovedOpenShellPrivileges(harnessPod); assert.deepEqual( @@ -1991,7 +1990,7 @@ async function prepareProductionInstallation( metadata.name.startsWith("openshell-cred-"), ), [], - "native delivery must not create bootstrap Jobs", + "native provider delivery must not create credential Jobs", ); } process.stderr.write( @@ -2013,7 +2012,7 @@ async function prepareProductionInstallation( const logs = await request("GET", `${runtimePath}/logs?source=gateway&tailLines=10`); assert.equal(logs.status, 200, "the demo API must read bounded Gateway logs"); assert.ok(Array.isArray(logs.data.records)); - assert.equal(JSON.stringify(logs).includes(process.env.OPENAI_API_KEY), false); + assert.equal(JSON.stringify(logs).includes(modelCredential), false); assert.ok( runtime.data.sources.some(({ id }) => id === "sandbox"), "the wrapper must preserve OpenShell policy-log support", @@ -2021,7 +2020,7 @@ async function prepareProductionInstallation( const sandboxLogs = await request("GET", `${runtimePath}/logs?source=sandbox&tailLines=10`); assert.equal(sandboxLogs.status, 200, "the demo API must read OpenShell policy logs"); assert.ok(Array.isArray(sandboxLogs.data.records)); - assert.equal(JSON.stringify(sandboxLogs).includes(process.env.OPENAI_API_KEY), false); + assert.equal(JSON.stringify(sandboxLogs).includes(modelCredential), false); const agentServiceName = openShellAgentName(agent.data.id); const gatewayPods = (await resources("pods", gatewayPlacement)).filter( From 5ed73046b9808e4759e47c33d736d97eb7cc0166 Mon Sep 17 00:00:00 2001 From: sallyom Date: Mon, 19 Oct 2026 23:48:20 -0700 Subject: [PATCH 09/10] fix(compute): preserve native Gateway state across restarts Retain the Gateway client credential store alongside its database so Pod replacement preserves migration receipts. Reuse only the exact generated native worker profile on container restart, retaining rejection of conflicting device settings. Signed-off-by: sallyom --- .../src/drivers/compute/kubernetes/index.ts | 2 ++ .../compute/kubernetes/runtime-entrypoints.ts | 12 ++++--- docs/flows/agent-native-admin.md | 14 ++++++-- .../storage-and-credentials.md | 3 ++ tests/conformance/kubernetes-compute.test.mjs | 35 ++++++++++++++++--- .../runtime-image-startup.test.mjs | 28 +++++++++++++-- 6 files changed, 80 insertions(+), 14 deletions(-) diff --git a/apps/controller/src/drivers/compute/kubernetes/index.ts b/apps/controller/src/drivers/compute/kubernetes/index.ts index 5466967072..bbc0cf77da 100644 --- a/apps/controller/src/drivers/compute/kubernetes/index.ts +++ b/apps/controller/src/drivers/compute/kubernetes/index.ts @@ -1278,6 +1278,8 @@ const WORKSPACE_NODE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; const WORKSPACE_NODE_ENROLLED_REVISION_KEY = "enrolledRevisionId"; const GATEWAY_PRIVATE_STATE_CATEGORIES = Object.freeze([ ["state", "/home/node/.openclaw/state"], + // Client migration receipts refer to the retained Gateway database. + ["client", "/home/node/.openclaw/client"], ["agent", "/home/node/.openclaw/agents/main/agent"], ["media", "/home/node/.openclaw/media"], // Session records retain linked worktree paths and their shared Git metadata. diff --git a/apps/controller/src/drivers/compute/kubernetes/runtime-entrypoints.ts b/apps/controller/src/drivers/compute/kubernetes/runtime-entrypoints.ts index 979dea64a0..a7b887910e 100644 --- a/apps/controller/src/drivers/compute/kubernetes/runtime-entrypoints.ts +++ b/apps/controller/src/drivers/compute/kubernetes/runtime-entrypoints.ts @@ -2339,7 +2339,12 @@ function configureNativeWorkerProfile() { const config = readOpenClawConfig(); const cloudWorkers = isPlainObject(config.cloudWorkers) ? config.cloudWorkers : {}; const profiles = isPlainObject(cloudWorkers.profiles) ? cloudWorkers.profiles : {}; - if (profiles[profileId] !== undefined) { + const profile = { + provider: "device", + settings: { device: deviceId, inference: "worker" }, + }; + // A container restart retains the writable configuration generated by this wrapper. + if (profiles[profileId] !== undefined && !pluginDeepEqual(profiles[profileId], profile)) { throw new Error("Dedicated OpenClaw worker profile conflicts with admitted configuration."); } writeOpenClawConfig({ @@ -2349,10 +2354,7 @@ function configureNativeWorkerProfile() { requiredProfile: profileId, profiles: { ...profiles, - [profileId]: { - provider: "device", - settings: { device: deviceId, inference: "worker" }, - }, + [profileId]: profile, }, }, }); diff --git a/docs/flows/agent-native-admin.md b/docs/flows/agent-native-admin.md index b02a6b0020..3fa4261bda 100644 --- a/docs/flows/agent-native-admin.md +++ b/docs/flows/agent-native-admin.md @@ -1,7 +1,7 @@ --- created: "2026-09-19" -updated: 2026-10-15 -last_updated_session: authoring-run/6d9b833d-e4fc-4f58-98a8-f0fac29f709d +updated: 2026-10-19 +last_updated_session: authoring-run/45df354e-6d46-4430-83a5-3885d8a9fb76 --- # Agent Native Admin UI Flow @@ -189,6 +189,14 @@ volume. The gateway mounts that same home subdirectory at `/home/node`, so nativ edits remain Pod-local and the next Pod starts from the admitted configuration. The init container cannot write through the gateway's later mount path. +For a dedicated native Harness, `configureNativeWorkerProfile` in +`apps/controller/src/drivers/compute/kubernetes/runtime-entrypoints.ts` accepts +the exact driver-generated profile on container restart; a different device or +profile setting still fails closed. `KubernetesComputeDriver.gatewayPrivateStateVolumeMounts` +retains `client/` alongside the Gateway database, preserving the client credential +store and its migration receipt across Pod replacement. The native admin +Configuration itself remains Pod-local. + ## Debugging and Verification - `AGENT_NATIVE_ADMIN_INVALID` at startup points to native admin enablement, the Agent domain, or a missing shared cookie domain. A bad shared cookie domain or public origin reports `AUTH_BASE_URL_INVALID`, short auth secret material `AUTH_SECRET_INVALID`, and a missing gateway API key path `GATEWAY_API_KEY_UNAVAILABLE`. @@ -218,6 +226,8 @@ The init container cannot write through the gateway's later mount path. ## Changelog +- 2026-10-19 23:14: Retain Gateway client state and reuse the exact generated native worker profile on restart. (authoring-run/45df354e-6d46-4430-83a5-3885d8a9fb76 - c584934287f78a4f3cf7c84cfb02febd9f3437ab) + - 2026-10-15 21:30: Present the authorized launch and its access boundaries in the Agent overview. (authoring-run/6d9b833d-e4fc-4f58-98a8-f0fac29f709d - 7adac57cc03545bdd3cdc383c88219396a677fd9) - 2026-10-15 20:27: Redirect copied chat navigation to the canonical console. (authoring-run/98164c0a-da1e-45c5-9d9a-8f6649a3a9c3 - b899534fae0070fab31489d54a8763bb81e4842e) - 2026-10-14 11:53: Extracted shared hostname derivation so the first-Agent script starts without installed workspace dependencies. (authoring-run/fba0a85c-ba37-4764-871e-9fdf80ee1083 - 5511d28d1) diff --git a/docs/reference/drivers/kubernetes-compute/storage-and-credentials.md b/docs/reference/drivers/kubernetes-compute/storage-and-credentials.md index fa31882f12..dadd8a4253 100644 --- a/docs/reference/drivers/kubernetes-compute/storage-and-credentials.md +++ b/docs/reference/drivers/kubernetes-compute/storage-and-credentials.md @@ -67,6 +67,7 @@ databases and their WAL/SHM siblings: | Private subpath | Gateway mount | | --------------------------- | ------------------------------------------- | | `state` | `/home/node/.openclaw/state` | +| `client` | `/home/node/.openclaw/client` | | `agent` | `/home/node/.openclaw/agents/main/agent` | | `media` | `/home/node/.openclaw/media` | | `worktrees` | `/home/node/.openclaw/worktrees` | @@ -74,6 +75,8 @@ databases and their WAL/SHM siblings: | `sessions` (dedicated mode) | `/home/node/.openclaw/agents/main/sessions` | Linked worktrees and Git metadata in `worktree-sources` persist across Gateway replacement. +The client credential store also persists: its migration receipt refers to the +retained Gateway database, so replacing only the client store can prevent startup. Embedded Gateways also mount its `workspace` subpath at `/home/node/.openclaw/workspace`, the default workspace under the configured diff --git a/tests/conformance/kubernetes-compute.test.mjs b/tests/conformance/kubernetes-compute.test.mjs index 4f316cfe0c..8c6f470bff 100644 --- a/tests/conformance/kubernetes-compute.test.mjs +++ b/tests/conformance/kubernetes-compute.test.mjs @@ -8825,9 +8825,9 @@ test("dedicated OpenClaw renders an enrolled Harness without exposing model cred let started = false; // Execute the generated Gateway startup program: every session must use the // enrolled worker without requiring a user-selected Cloud Worker destination. - runInNewContext(GATEWAY_RUNTIME_ENTRYPOINT, { + // Keep JSON in the VM realm, as the wrapper's native deep comparison requires. + const sandbox = { Buffer, - JSON, URL, console, process: { @@ -8888,8 +8888,8 @@ test("dedicated OpenClaw renders an enrolled Harness without exposing model cred } return nodeRequire(specifier); }, - }); - await Promise.resolve(); + }; + await runInNewContext(GATEWAY_RUNTIME_ENTRYPOINT, sandbox); assert.equal(started, true); const effectiveConfiguration = JSON.parse(files.get("/home/node/.openclaw/openclaw.json")); assert.equal(effectiveConfiguration.cloudWorkers.requiredProfile, "dedicated-native"); @@ -8897,6 +8897,27 @@ test("dedicated OpenClaw renders an enrolled Harness without exposing model cred provider: "device", settings: { device: deviceId, inference: "worker" }, }); + // Restart the wrapper against its generated writable configuration. The + // admitted revision still owns the profile; only the exact generated value is reusable. + started = false; + const restart = { + ...sandbox, + process: { ...sandbox.process, env: { ...sandbox.process.env } }, + }; + await runInNewContext(GATEWAY_RUNTIME_ENTRYPOINT, restart); + assert.equal(started, true); + + effectiveConfiguration.cloudWorkers.profiles["dedicated-native"].settings.device = "b".repeat(64); + files.set("/home/node/.openclaw/openclaw.json", JSON.stringify(effectiveConfiguration)); + started = false; + await assert.rejects( + runInNewContext(GATEWAY_RUNTIME_ENTRYPOINT, { + ...sandbox, + process: { ...sandbox.process, env: { ...sandbox.process.env } }, + }), + /worker profile conflicts with admitted configuration/u, + ); + assert.equal(started, false); }); test("account-token authentication grants only the exact Codex revision outbound HTTPS", () => { @@ -13547,6 +13568,12 @@ test("Gateway and Harness storage are separate and preserve ephemeral Codex cred subPath: "state", readOnly: false, }, + { + name: privateVolume.name, + mountPath: "/home/node/.openclaw/client", + subPath: "client", + readOnly: false, + }, { name: privateVolume.name, mountPath: "/home/node/.openclaw/agents/main/agent", diff --git a/tests/integration/runtime-image-startup.test.mjs b/tests/integration/runtime-image-startup.test.mjs index 1a08db69cb..939f088476 100644 --- a/tests/integration/runtime-image-startup.test.mjs +++ b/tests/integration/runtime-image-startup.test.mjs @@ -917,15 +917,31 @@ function run(args, env, readinessUrl) { (gatewayBase.agents ??= {}).defaults ??= {}; gatewayBase.agents.defaults.workspace = "/tmp/runtime-image-provider-workspace"; fs.writeFileSync("/tmp/gateway/base.json", JSON.stringify(gatewayBase)); - const gatewayRun = await run(["-e", ...gatewayArgs], { + const gatewayEnvironment = { OPENCLAW_CONFIG_PATH: "/tmp/gateway/base.json", OPENCLAW_STATE_DIR: "/home/node/.openclaw", OPENCLAW_GATEWAY_PORT: "18789", OPENCLAW_GATEWAY_PASSWORD: "openclaw-runtime-image-schema-password", OPENCLAW_WORKSPACE_NODE_ID: workspaceNodeId, OPENCLAW_NATIVE_WORKER_PROFILE: "dedicated-native", - }, "http://127.0.0.1:18789/readyz"); + }; + const gatewayRun = await run(["-e", ...gatewayArgs], gatewayEnvironment, "http://127.0.0.1:18789/readyz"); const gatewayConfig = "/home/node/.openclaw/openclaw.json"; + // A container restart reuses the writable admin Configuration, including the + // profile generated by the first wrapper. It must still start the native Gateway. + const gatewayRestart = await run(["-e", ...gatewayArgs], { + ...gatewayEnvironment, OPENCLAW_CONFIG_PATH: gatewayConfig, + }, "http://127.0.0.1:18789/readyz"); + const generatedConfiguration = fs.readFileSync(gatewayConfig, "utf8"); + const conflictingConfig = JSON.parse(generatedConfiguration); + conflictingConfig.cloudWorkers.profiles["dedicated-native"].settings.device = + (workspaceNodeId[0] === "a" ? "b" : "a") + workspaceNodeId.slice(1); + fs.writeFileSync(gatewayConfig, JSON.stringify(conflictingConfig)); + // Restart tolerance must not admit a profile bound to a different worker. + const conflictingRestart = await run(["-e", ...gatewayArgs], { + ...gatewayEnvironment, OPENCLAW_CONFIG_PATH: gatewayConfig, + }); + fs.writeFileSync(gatewayConfig, generatedConfiguration); const probe = JSON.stringify({ auth: { probes: { results: [{ provider: "openai", model, source: "env", status: "ok" }] } } }); fs.writeFileSync("/tmp/probe.cjs", [ 'const cp = require("node:child_process");', @@ -956,7 +972,7 @@ function run(args, env, readinessUrl) { }); const harnessConfig = "/home/node/.openclaw-node/openclaw.json"; process.stdout.write(JSON.stringify({ - gateway: { ...gatewayRun, config: JSON.parse(fs.readFileSync(gatewayConfig, "utf8")), validation: validate(gatewayConfig) }, + gateway: { ...gatewayRun, restart: gatewayRestart, conflictingRestart, config: JSON.parse(fs.readFileSync(gatewayConfig, "utf8")), validation: validate(gatewayConfig) }, harness: { ...harnessRun, config: JSON.parse(fs.readFileSync(harnessConfig, "utf8")), validation: validate(harnessConfig) }, })); })().catch((error) => { console.error(error); process.exitCode = 1; }); @@ -1016,6 +1032,12 @@ function run(args, env, readinessUrl) { assert.deepEqual(harness.validation.issues, []); assert.equal(gateway.validation.valid, true); assert.equal(gateway.ready, true, gateway.output); + assert.equal(gateway.restart.ready, true, gateway.restart.output); + assert.equal(gateway.conflictingRestart.code, 1); + assert.match( + gateway.conflictingRestart.output, + /worker profile conflicts with admitted configuration/u, + ); assert.equal(harness.validation.valid, true); // Schema acceptance alone does not qualify the default image's complete native flow. // Runtime admission retains its separate native-worker support gate. From 73c39aed60cab99aa642b282ed99b74e48ce4b2a Mon Sep 17 00:00:00 2001 From: roboclaw-bot <309084314+roboclaw-bot@users.noreply.github.com> Date: Tue, 20 Oct 2026 14:55:48 +0000 Subject: [PATCH 10/10] test(openshell): split Gateway persistence repair into its own PR Remove the independent native Gateway persistence and restart change from this coverage PR. Preserve it with its tests and documentation on a separate main-based branch. Co-authored-by: sallyom <11166065+sallyom@users.noreply.github.com> Signed-off-by: roboclaw-bot <309084314+roboclaw-bot@users.noreply.github.com> --- .../src/drivers/compute/kubernetes/index.ts | 2 -- .../compute/kubernetes/runtime-entrypoints.ts | 12 +++---- docs/flows/agent-native-admin.md | 14 ++------ .../storage-and-credentials.md | 3 -- tests/conformance/kubernetes-compute.test.mjs | 35 +++---------------- .../runtime-image-startup.test.mjs | 28 ++------------- 6 files changed, 14 insertions(+), 80 deletions(-) diff --git a/apps/controller/src/drivers/compute/kubernetes/index.ts b/apps/controller/src/drivers/compute/kubernetes/index.ts index bbc0cf77da..5466967072 100644 --- a/apps/controller/src/drivers/compute/kubernetes/index.ts +++ b/apps/controller/src/drivers/compute/kubernetes/index.ts @@ -1278,8 +1278,6 @@ const WORKSPACE_NODE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u; const WORKSPACE_NODE_ENROLLED_REVISION_KEY = "enrolledRevisionId"; const GATEWAY_PRIVATE_STATE_CATEGORIES = Object.freeze([ ["state", "/home/node/.openclaw/state"], - // Client migration receipts refer to the retained Gateway database. - ["client", "/home/node/.openclaw/client"], ["agent", "/home/node/.openclaw/agents/main/agent"], ["media", "/home/node/.openclaw/media"], // Session records retain linked worktree paths and their shared Git metadata. diff --git a/apps/controller/src/drivers/compute/kubernetes/runtime-entrypoints.ts b/apps/controller/src/drivers/compute/kubernetes/runtime-entrypoints.ts index e17e7d169e..9cedb38f09 100644 --- a/apps/controller/src/drivers/compute/kubernetes/runtime-entrypoints.ts +++ b/apps/controller/src/drivers/compute/kubernetes/runtime-entrypoints.ts @@ -2339,12 +2339,7 @@ function configureNativeWorkerProfile() { const config = readOpenClawConfig(); const cloudWorkers = isPlainObject(config.cloudWorkers) ? config.cloudWorkers : {}; const profiles = isPlainObject(cloudWorkers.profiles) ? cloudWorkers.profiles : {}; - const profile = { - provider: "device", - settings: { device: deviceId, inference: "worker" }, - }; - // A container restart retains the writable configuration generated by this wrapper. - if (profiles[profileId] !== undefined && !pluginDeepEqual(profiles[profileId], profile)) { + if (profiles[profileId] !== undefined) { throw new Error("Dedicated OpenClaw worker profile conflicts with admitted configuration."); } writeOpenClawConfig({ @@ -2354,7 +2349,10 @@ function configureNativeWorkerProfile() { requiredProfile: profileId, profiles: { ...profiles, - [profileId]: profile, + [profileId]: { + provider: "device", + settings: { device: deviceId, inference: "worker" }, + }, }, }, }); diff --git a/docs/flows/agent-native-admin.md b/docs/flows/agent-native-admin.md index 925bb663ea..8056daf7b2 100644 --- a/docs/flows/agent-native-admin.md +++ b/docs/flows/agent-native-admin.md @@ -1,7 +1,7 @@ --- created: "2026-09-19" -updated: 2026-10-19 -last_updated_session: authoring-run/45df354e-6d46-4430-83a5-3885d8a9fb76 +updated: 2026-10-15 +last_updated_session: authoring-run/6d9b833d-e4fc-4f58-98a8-f0fac29f709d --- # Agent Native Admin UI Flow @@ -191,14 +191,6 @@ volume. The gateway mounts that same home subdirectory at `/home/node`, so nativ edits remain Pod-local and the next Pod starts from the admitted configuration. The init container cannot write through the gateway's later mount path. -For a dedicated native Harness, `configureNativeWorkerProfile` in -`apps/controller/src/drivers/compute/kubernetes/runtime-entrypoints.ts` accepts -the exact driver-generated profile on container restart; a different device or -profile setting still fails closed. `KubernetesComputeDriver.gatewayPrivateStateVolumeMounts` -retains `client/` alongside the Gateway database, preserving the client credential -store and its migration receipt across Pod replacement. The native admin -Configuration itself remains Pod-local. - ## Debugging and Verification - `AGENT_NATIVE_ADMIN_INVALID` at startup points to native admin enablement, the Agent domain, or a missing shared cookie domain. A bad shared cookie domain or public origin reports `AUTH_BASE_URL_INVALID`, short auth secret material `AUTH_SECRET_INVALID`, and a missing gateway API key path `GATEWAY_API_KEY_UNAVAILABLE`. @@ -228,8 +220,6 @@ Configuration itself remains Pod-local. ## Changelog -- 2026-10-19 23:14: Retain Gateway client state and reuse the exact generated native worker profile on restart. (authoring-run/45df354e-6d46-4430-83a5-3885d8a9fb76 - c584934287f78a4f3cf7c84cfb02febd9f3437ab) - - 2026-10-15 21:30: Present the authorized launch and its access boundaries in the Agent overview. (authoring-run/6d9b833d-e4fc-4f58-98a8-f0fac29f709d - 7adac57cc03545bdd3cdc383c88219396a677fd9) - 2026-10-15 20:27: Redirect copied chat navigation to the canonical console. (authoring-run/98164c0a-da1e-45c5-9d9a-8f6649a3a9c3 - b899534fae0070fab31489d54a8763bb81e4842e) - 2026-10-14 11:53: Extracted shared hostname derivation so the first-Agent script starts without installed workspace dependencies. (authoring-run/fba0a85c-ba37-4764-871e-9fdf80ee1083 - 5511d28d1) diff --git a/docs/reference/drivers/kubernetes-compute/storage-and-credentials.md b/docs/reference/drivers/kubernetes-compute/storage-and-credentials.md index 14e9540963..5051499692 100644 --- a/docs/reference/drivers/kubernetes-compute/storage-and-credentials.md +++ b/docs/reference/drivers/kubernetes-compute/storage-and-credentials.md @@ -67,7 +67,6 @@ databases and their WAL/SHM siblings: | Private subpath | Gateway mount | | --------------------------- | ------------------------------------------- | | `state` | `/home/node/.openclaw/state` | -| `client` | `/home/node/.openclaw/client` | | `agent` | `/home/node/.openclaw/agents/main/agent` | | `media` | `/home/node/.openclaw/media` | | `worktrees` | `/home/node/.openclaw/worktrees` | @@ -75,8 +74,6 @@ databases and their WAL/SHM siblings: | `sessions` (dedicated mode) | `/home/node/.openclaw/agents/main/sessions` | Linked worktrees and Git metadata in `worktree-sources` persist across Gateway replacement. -The client credential store also persists: its migration receipt refers to the -retained Gateway database, so replacing only the client store can prevent startup. Embedded Gateways mount the claim's `workspace` subpath at `/home/node/.openclaw/workspace`, the default workspace under the configured diff --git a/tests/conformance/kubernetes-compute.test.mjs b/tests/conformance/kubernetes-compute.test.mjs index ce1819ff13..0617d8a4eb 100644 --- a/tests/conformance/kubernetes-compute.test.mjs +++ b/tests/conformance/kubernetes-compute.test.mjs @@ -8833,9 +8833,9 @@ test("dedicated OpenClaw renders an enrolled Harness without exposing model cred let started = false; // Execute the generated Gateway startup program: every session must use the // enrolled worker without requiring a user-selected Cloud Worker destination. - // Keep JSON in the VM realm, as the wrapper's native deep comparison requires. - const sandbox = { + runInNewContext(GATEWAY_RUNTIME_ENTRYPOINT, { Buffer, + JSON, URL, console, process: { @@ -8896,8 +8896,8 @@ test("dedicated OpenClaw renders an enrolled Harness without exposing model cred } return nodeRequire(specifier); }, - }; - await runInNewContext(GATEWAY_RUNTIME_ENTRYPOINT, sandbox); + }); + await Promise.resolve(); assert.equal(started, true); const effectiveConfiguration = JSON.parse(files.get("/home/node/.openclaw/openclaw.json")); assert.equal(effectiveConfiguration.cloudWorkers.requiredProfile, "dedicated-native"); @@ -8905,27 +8905,6 @@ test("dedicated OpenClaw renders an enrolled Harness without exposing model cred provider: "device", settings: { device: deviceId, inference: "worker" }, }); - // Restart the wrapper against its generated writable configuration. The - // admitted revision still owns the profile; only the exact generated value is reusable. - started = false; - const restart = { - ...sandbox, - process: { ...sandbox.process, env: { ...sandbox.process.env } }, - }; - await runInNewContext(GATEWAY_RUNTIME_ENTRYPOINT, restart); - assert.equal(started, true); - - effectiveConfiguration.cloudWorkers.profiles["dedicated-native"].settings.device = "b".repeat(64); - files.set("/home/node/.openclaw/openclaw.json", JSON.stringify(effectiveConfiguration)); - started = false; - await assert.rejects( - runInNewContext(GATEWAY_RUNTIME_ENTRYPOINT, { - ...sandbox, - process: { ...sandbox.process, env: { ...sandbox.process.env } }, - }), - /worker profile conflicts with admitted configuration/u, - ); - assert.equal(started, false); }); test("account-token authentication grants only the exact Codex revision outbound HTTPS", () => { @@ -13576,12 +13555,6 @@ test("Gateway and Harness storage are separate and preserve ephemeral Codex cred subPath: "state", readOnly: false, }, - { - name: privateVolume.name, - mountPath: "/home/node/.openclaw/client", - subPath: "client", - readOnly: false, - }, { name: privateVolume.name, mountPath: "/home/node/.openclaw/agents/main/agent", diff --git a/tests/integration/runtime-image-startup.test.mjs b/tests/integration/runtime-image-startup.test.mjs index 2ef3659609..8f2437218c 100644 --- a/tests/integration/runtime-image-startup.test.mjs +++ b/tests/integration/runtime-image-startup.test.mjs @@ -899,31 +899,15 @@ function run(args, env, readinessUrl) { (gatewayBase.agents ??= {}).defaults ??= {}; gatewayBase.agents.defaults.workspace = "/tmp/runtime-image-provider-workspace"; fs.writeFileSync("/tmp/gateway/base.json", JSON.stringify(gatewayBase)); - const gatewayEnvironment = { + const gatewayRun = await run(["-e", ...gatewayArgs], { OPENCLAW_CONFIG_PATH: "/tmp/gateway/base.json", OPENCLAW_STATE_DIR: "/home/node/.openclaw", OPENCLAW_GATEWAY_PORT: "18789", OPENCLAW_GATEWAY_PASSWORD: "openclaw-runtime-image-schema-password", OPENCLAW_WORKSPACE_NODE_ID: workspaceNodeId, OPENCLAW_NATIVE_WORKER_PROFILE: "dedicated-native", - }; - const gatewayRun = await run(["-e", ...gatewayArgs], gatewayEnvironment, "http://127.0.0.1:18789/readyz"); - const gatewayConfig = "/home/node/.openclaw/openclaw.json"; - // A container restart reuses the writable admin Configuration, including the - // profile generated by the first wrapper. It must still start the native Gateway. - const gatewayRestart = await run(["-e", ...gatewayArgs], { - ...gatewayEnvironment, OPENCLAW_CONFIG_PATH: gatewayConfig, }, "http://127.0.0.1:18789/readyz"); - const generatedConfiguration = fs.readFileSync(gatewayConfig, "utf8"); - const conflictingConfig = JSON.parse(generatedConfiguration); - conflictingConfig.cloudWorkers.profiles["dedicated-native"].settings.device = - (workspaceNodeId[0] === "a" ? "b" : "a") + workspaceNodeId.slice(1); - fs.writeFileSync(gatewayConfig, JSON.stringify(conflictingConfig)); - // Restart tolerance must not admit a profile bound to a different worker. - const conflictingRestart = await run(["-e", ...gatewayArgs], { - ...gatewayEnvironment, OPENCLAW_CONFIG_PATH: gatewayConfig, - }); - fs.writeFileSync(gatewayConfig, generatedConfiguration); + const gatewayConfig = "/home/node/.openclaw/openclaw.json"; const probe = JSON.stringify({ auth: { probes: { results: [{ provider: "openai", model, source: "env", status: "ok" }] } } }); fs.writeFileSync("/tmp/probe.cjs", [ 'const cp = require("node:child_process");', @@ -954,7 +938,7 @@ function run(args, env, readinessUrl) { }); const harnessConfig = "/home/node/.openclaw-node/openclaw.json"; process.stdout.write(JSON.stringify({ - gateway: { ...gatewayRun, restart: gatewayRestart, conflictingRestart, config: JSON.parse(fs.readFileSync(gatewayConfig, "utf8")), validation: validate(gatewayConfig) }, + gateway: { ...gatewayRun, config: JSON.parse(fs.readFileSync(gatewayConfig, "utf8")), validation: validate(gatewayConfig) }, harness: { ...harnessRun, config: JSON.parse(fs.readFileSync(harnessConfig, "utf8")), validation: validate(harnessConfig) }, })); })().catch((error) => { console.error(error); process.exitCode = 1; }); @@ -1014,12 +998,6 @@ function run(args, env, readinessUrl) { assert.deepEqual(harness.validation.issues, []); assert.equal(gateway.validation.valid, true); assert.equal(gateway.ready, true, gateway.output); - assert.equal(gateway.restart.ready, true, gateway.restart.output); - assert.equal(gateway.conflictingRestart.code, 1); - assert.match( - gateway.conflictingRestart.output, - /worker profile conflicts with admitted configuration/u, - ); assert.equal(harness.validation.valid, true); // Schema acceptance alone does not qualify the default image's complete native flow. // Runtime admission retains its separate native-worker support gate.